alerting

type Alert struct {
	Labels      map[string]string
	Annotations map[string]string
	StartsAt    time.Time
	// EndsAt is when the receiver should consider the alert resolved if it is
	// not refreshed: now+3×interval while firing, now once resolved.
	EndsAt time.Time
	// Why is the instance's why-fired payload (see Why): structured evidence
	// that rides with the alert but is not part of its identity. The
	// Alertmanager notifier posts it as the WhyAnnotation annotation. Nil when
	// the rule's rows carried no WhyColumn.
	Why json.RawMessage
}

type AlertmanagerNotifier struct {

	// GeneratorURL, when set, is attached to every alert.
	GeneratorURL string
	// contains filtered or unexported fields
}

type Changes struct {
	Upsert []Instance
	Delete []InstanceRef
	Events []Event
}

type Duration time.Duration

type Evaluator struct {
	// contains filtered or unexported fields
}

type EvaluatorDeps struct {
	Querier  Querier
	Notifier Notifier
	Store    StateStore
	Limits   Limits
	Logger   *slog.Logger
	// Now is the clock; time.Now when nil.
	Now func() time.Time
}

type Event struct {
	RuleKey
	Fingerprint string
	Labels      map[string]string
	Transition  Transition
	Value       float64
	At          time.Time
	// Why is the payload the instance carried at this transition.
	Why json.RawMessage
}

type EventQuery struct {
	Owner string
	// Env narrows to one env; empty means all of the owner's envs.
	Env string
	// Since is the oldest event wanted.
	Since time.Time
	// Limit defaults to 200 and is capped at 1000.
	Limit int
}

type FileRuleSource struct {
	// Path of the rules document.
	Path string
	// Owner and Env key every rule. Owner may be any non-empty opaque
	// string; it only has to match what the Querier expects.
	Owner, Env string
	// Identity is stamped on every rule (see Rule.Identity).
	Identity map[string]string
	// Recheck is how often to stat the file. Default 5s.
	Recheck time.Duration
	Logger  *slog.Logger
	// contains filtered or unexported fields
}

type Instance struct {
	RuleKey
	Fingerprint string
	// Labels are the final labels, identity included.
	Labels      map[string]string
	Annotations map[string]string
	State       State
	// ActiveAt is when the series was first seen in its current run.
	ActiveAt time.Time
	// FiringAt is when it became firing; zero while pending.
	FiringAt time.Time
	// ClearedAt is when a firing instance first stopped returning, while flap
	// damping holds it open; zero when it is returning.
	ClearedAt  time.Time
	ResolvedAt time.Time
	// ResolveSent records that the resolved notification was delivered.
	ResolveSent bool
	LastEvalAt  time.Time
	LastValue   float64
	// Why is the latest why-fired payload the rule's row carried (WhyColumn),
	// refreshed on every evaluation. It is not a label, so it never changes
	// the instance's fingerprint.
	Why json.RawMessage
}

type InstanceRef struct {
	RuleKey
	Fingerprint string
}

type Limits struct {
	// MinInterval is the floor on a rule's interval.
	MinInterval time.Duration
	// MaxExecutionTime is the per-query timeout, applied server-side and to the
	// call's context.
	MaxExecutionTime time.Duration
	// MaxInstancesPerRule: a result with more rows than this is an error, and the
	// rule's existing instances are left untouched for that evaluation.
	MaxInstancesPerRule int
	// MaxRulesPerEnv: rules past this count in ONE env are not evaluated (see
	// Select for which). The unit is the env, not the owner: an owner's envs
	// are independent deployments, and a cap over their sum let one env's
	// rules push another env's alerts out of evaluation.
	MaxRulesPerEnv int
}

type Locker interface {
	Run(ctx context.Context, name string, fn func(ctx context.Context) error) error
}

type MemoryStore struct {
	// contains filtered or unexported fields
}

type Notifier interface {
	Notify(ctx context.Context, alerts []Alert) error
}

type Overflow struct {
	Env string
	// Configured is how many rules the env declares; Cap the limit it is over.
	Configured, Cap int
	// Dropped are the rules not evaluated, in the order they were dropped.
	Dropped []Rule
}

type Querier interface {
	Query(ctx context.Context, owner, query string, opts QueryOptions) ([]Row, error)
}

type QueryOptions struct {
	MaxExecutionTime time.Duration
	// MaxResultRows is one more than the instance cap, so exceeding the cap
	// surfaces as an error instead of a silently truncated result.
	MaxResultRows int
}

type Row map[string]any

type Rule struct {
	RuleKey

	// Expr is a single ClickHouse SELECT; see ValidateExpr.
	Expr string
	// For is how long a series must keep returning before it fires. Zero fires
	// on the first evaluation that returns it.
	For time.Duration
	// Interval is the evaluation period, raised to the scheduler's MinInterval.
	Interval time.Duration
	// KeepFiringFor holds a firing alert open this long after its series stops
	// returning (Prometheus's keep_firing_for). The hold is never shorter than
	// one evaluation, which is the flap damping every rule gets.
	KeepFiringFor time.Duration

	// Source is where the rule was declared.
	Source Source

	// Labels and Annotations are the rule author's. Labels whose name starts
	// with "reliant_" are dropped, and `alertname` is always the rule's name.
	Labels      map[string]string
	Annotations map[string]string

	// Identity is set by the RuleSource, never by the rule author, and is
	// applied after every other label: it is how a consumer routes an alert to
	// the owner it belongs to.
	Identity map[string]string
}

type RuleKey struct {
	Owner string
	Env   string
	Name  string
}

type RuleSet []RuleSpec

type RuleSource interface {
	// Owners lists every owner that has rules.
	Owners(ctx context.Context) ([]string, error)
	// Rules lists an owner's rules across all of its envs.
	Rules(ctx context.Context, owner string) ([]Rule, error)
}

type RuleSpec struct {
	// Alert is the rule's name, unique within an env: becomes `alertname`.
	Alert string `json:"alert"`
	// Expr is a single ClickHouse SELECT; every row it returns is a firing
	// instance. See ValidateExpr and the package doc.
	Expr string `json:"expr"`
	// For is how long a series must keep returning before it fires.
	For Duration `json:"for,omitempty"`
	// Interval is the evaluation period; zero means the evaluator's floor.
	Interval Duration `json:"interval,omitempty"`
	// KeepFiringFor holds a firing alert open this long after it stops
	// returning, so a flapping series does not page on every recovery.
	KeepFiringFor Duration `json:"keep_firing_for,omitempty"`
	// Labels carry routing data (`severity` is a label). Names starting with
	// "reliant_" are reserved and dropped.
	Labels      map[string]string `json:"labels,omitempty"`
	Annotations map[string]string `json:"annotations,omitempty"`
	// Source is user or default; empty means user.
	Source Source `json:"source,omitempty"`
}

type RulesDocument struct {
	Alerts RuleSet `json:"alerts"`
}

type Scheduler struct {
	// contains filtered or unexported fields
}

type SchedulerDeps struct {
	Rules RuleSource
	Store StateStore
	Eval  *Evaluator
	// Locker shards evaluation across replicas. Nil runs every shard in this
	// process, which is right for a single replica.
	Locker Locker
	// Shards is how many lease-owned partitions owners hash into. Default 16.
	Shards int
	// LeasePrefix names the leases: "<prefix>/shard/<n>". Default "alerting".
	LeasePrefix string
	// Tick is how often a shard looks for due rules. Default 10s.
	Tick time.Duration
	// Concurrency bounds owners evaluated at once within a shard. Default 4.
	Concurrency int
	Logger      *slog.Logger
	// Meter records the alerting.rules.* instruments. otel's global meter
	// provider when nil.
	Meter metric.Meter
	// Notifier receives the AlertRulesOverCap alert for an env that is over its
	// rule cap. Without one that alert is only logged and counted, so every
	// real deployment sets it (the Evaluator's own Notifier is the one to pass).
	Notifier Notifier
}

type Source string

type State string

type StateStore interface {
	// LoadOwner returns every stored instance of an owner, resolved included.
	LoadOwner(ctx context.Context, owner string) ([]Instance, error)
	// Owners lists owners that have stored instances.
	Owners(ctx context.Context) ([]string, error)
	// Apply persists one rule's changes atomically.
	Apply(ctx context.Context, ch Changes) error
	// Events reads the transition history, newest first.
	Events(ctx context.Context, q EventQuery) ([]Event, error)
}

type StoreError struct {
	Rule RuleKey
	Err  error
}

type Transition string

type Why struct {
	// Detector is one of the Detector* names.
	Detector string `json:"detector"`
	// Signal is what was measured: error_rate, p95, throughput, issue_events,
	// release_health.
	Signal string `json:"signal"`
	// DedupeKey names what the alert is about, so a receiver that gets several
	// alerts for one cause (a bad deploy raises a regression, a spike and an
	// error-rate alert) can start one piece of work, not one per alert. Built
	// by DedupeKey.
	DedupeKey string `json:"dedupe_key,omitempty"`

	// Observed is the measured value in Unit; Baseline is what it is judged
	// against; Spread is the baseline's dispersion, when the method has one.
	// Nil means the detector does not have the figure, never zero.
	Observed *float64 `json:"observed,omitempty"`
	Baseline *float64 `json:"baseline,omitempty"`
	Spread   *float64 `json:"spread,omitempty"`
	// Unit names Observed, Baseline and Spread: "fraction", "ratio", "ms",
	// "events/hour".
	Unit string `json:"unit,omitempty"`
	// Method says how Baseline was found, in a line a person can check.
	Method string `json:"method,omitempty"`
	// Thresholds are the detector's own limits that Observed crossed, by name
	// ("factor": 5, "min_events": 20).
	Thresholds map[string]float64 `json:"thresholds,omitempty"`

	// Window is the span Observed covers ("15m", "1h-2h"); BaselineWindow the
	// span Baseline covers. Since is when the condition began.
	Window         string    `json:"window,omitempty"`
	BaselineWindow string    `json:"baseline_window,omitempty"`
	Since          time.Time `json:"since,omitzero"`
	// HistoryDays is how much history the baseline had to work with.
	HistoryDays int `json:"history_days,omitempty"`

	Context WhyContext `json:"context"`

	// Next are commands that lead from the alert to the evidence, runnable as
	// written ("forge obs check prod").
	Next []string `json:"next,omitempty"`
}

type WhyActivity struct {
	At     time.Time `json:"at"`
	Actor  string    `json:"actor,omitempty"`
	Verb   string    `json:"verb"`
	Target string    `json:"target,omitempty"`
}

type WhyContext struct {
	// Findings are the individual findings a detector judged: one per
	// service, issue or threshold that moved.
	Findings []WhyFinding `json:"findings,omitempty"`
	// Releases are the releases promoted in the window, with their commit range.
	Releases []WhyRelease `json:"releases_in_window,omitempty"`
	// Activity is the other changes in the window (a config change, a deploy).
	Activity []WhyActivity `json:"activity_in_window,omitempty"`
	// TopIssues are the issues most active in the window.
	TopIssues []WhyIssue `json:"top_issues,omitempty"`
	// ShiftedAttributes name attributes whose distribution shifted
	// ("build=3f9c: 92% vs 4% baseline").
	ShiftedAttributes []string `json:"shifted_attributes,omitempty"`
}

type WhyFinding struct {
	// Code is the detector's own name for it (a release-health finding code).
	Code    string   `json:"code"`
	Service string   `json:"service,omitempty"`
	Ref     string   `json:"ref,omitempty"`
	Value   float64  `json:"value"`
	Limit   float64  `json:"limit"`
	Before  *float64 `json:"before,omitempty"`
	After   *float64 `json:"after,omitempty"`
	// Policy says whether the finding counted ("fail") or only warned.
	Policy  string `json:"policy,omitempty"`
	Message string `json:"message,omitempty"`
}

type WhyIssue struct {
	ID      string `json:"id"`
	Service string `json:"service,omitempty"`
	Title   string `json:"title,omitempty"`
	// Events is the count over the alert's window.
	Events      int64     `json:"events"`
	FirstSeenAt time.Time `json:"first_seen_at,omitzero"`
	LastSeenAt  time.Time `json:"last_seen_at,omitzero"`
}

type WhyRelease struct {
	Version     string    `json:"version"`
	Previous    string    `json:"previous,omitempty"`
	PromotionID string    `json:"promotion_id,omitempty"`
	PromotedAt  time.Time `json:"promoted_at"`
	// Commit and PreviousCommit bound the change: PreviousCommit..Commit.
	Commit         string `json:"commit,omitempty"`
	PreviousCommit string `json:"previous_commit,omitempty"`
}