observability

type Config struct {
	ServiceName  string
	LogLevel     string
	LogFormat    string
	OTelEnabled  bool
	OTelEndpoint string
	// SampleRatio is the share of new root traces kept, in [0, 1].
	SampleRatio float64
	// SkipProbeSpans drops spans for /healthz, /readyz and /static/*.
	SkipProbeSpans bool
}

type Metrics struct {
	// Scheduler
	SchedulerLoopDuration prometheus.Histogram
	ScheduleDelay         prometheus.Histogram
	SchedulerWokenTicks   prometheus.Counter
	SchedulerDecisions    *prometheus.CounterVec
	TasksUndispatchable   *prometheus.CounterVec
	SchedulerStepDowns    *prometheus.CounterVec // #311 leader churn observability
	SchedulerReacquire    prometheus.Histogram   // #311 step-down → re-acquire latency
	AlertsDispatched      *prometheus.CounterVec // #424 on-failure alerts, by outcome

	// Task lifecycle
	TaskStateTransitions    *prometheus.CounterVec
	TaskDuration            *prometheus.HistogramVec
	TaskRetries             *prometheus.CounterVec
	TaskPodCreationDuration prometheus.Histogram
	TaskColdStart           *prometheus.HistogramVec

	// XCom
	XComSize     *prometheus.HistogramVec
	XComPush     *prometheus.CounterVec
	XComPull     *prometheus.CounterVec
	XComRejected *prometheus.CounterVec

	// API
	HTTPRequests        *prometheus.CounterVec
	HTTPRequestDuration *prometheus.HistogramVec
	AuthFailures        *prometheus.CounterVec

	// Agent credentials: task tokens authenticated without an attempt_epoch
	// claim, accepted under the epoch-0 rule (ADR 0051 amendment).
	AgentLegacyAttemptTokens prometheus.Counter
	// AgentTokenRejections counts agent RPCs refused for their bearer token, by
	// reason (#1125).
	AgentTokenRejections *prometheus.CounterVec
	// Reconciler: durable SUCCESS records settled over a reaper's provisional
	// infra mark, by the mark overridden (ADR 0052 amendment).
	ReconcileInfraOverrides *prometheus.CounterVec

	// Executor (Kubernetes)
	PodsCreated        *prometheus.CounterVec
	PodPendingDuration prometheus.Histogram
	KubernetesAPICalls *prometheus.CounterVec

	// Dispatch pool (#127, ADR 0031): depth gauge, capacity-saturation counter,
	// per-dispatch latency, inner-dispatcher error counter.
	DispatchQueueDepth  prometheus.Gauge
	DispatchAtCapacity  prometheus.Counter
	DispatchLatency     prometheus.Histogram
	DispatchInnerErrors prometheus.Counter
	// UnitMisfits counts tasks let through under executor.unit.enforce=warn
	// although they do not fit their size, by stage (register, dispatch;
	// ADR 0066 §3).
	UnitMisfits *prometheus.CounterVec
	// UnpinnedImages counts images not pinned by digest that
	// deploy.require_digest warned about or refused, by stage (register,
	// set_current) and outcome (warned, refused) (#1703).
	UnpinnedImages *prometheus.CounterVec
	// CIOIDCExchanges counts CI OIDC token exchanges by issuer entry
	// ("unknown" when the token named none) and outcome (ADR 0092).
	CIOIDCExchanges *prometheus.CounterVec

	// Redis observability — port of the #311 step-down pattern for Redis (Pro
	// only; Lite uses Postgres + in-process tailer per ADR 0026, so these
	// gauges stay at 0). Operators alert on the per-reason rate, not on log
	// content: a sudden rate spike on RedisCommandFailures{reason="timeout"}
	// or RedisDialFailures{reason="tls_handshake"} catches an outage before
	// the surrounding components (XCom, log tailer) start surfacing tail
	// errors to users.
	RedisCommandFailures *prometheus.CounterVec
	RedisDialFailures    *prometheus.CounterVec
	RedisDialDuration    prometheus.Histogram
	RedisPoolActive      prometheus.Gauge
	RedisPoolIdle        prometheus.Gauge
	RedisPoolTotalConns  prometheus.Gauge
	RedisPoolTimeouts    prometheus.Counter

	// Retention janitor (D5): rows deleted per table, rows a dry run found
	// eligible per table, and the duration of each cycle.
	RetentionRowsDeleted   *prometheus.CounterVec
	RetentionRowsEligible  *prometheus.GaugeVec
	RetentionCycleDuration prometheus.Histogram

	// UsageEventsLag is the age of the oldest usage event (ADR 0096) the
	// service API cannot serve yet because an older transaction is still
	// open, set by the leader; 0 while recording is off.
	UsageEventsLag prometheus.Gauge
	// TaskAttempts and TaskSlotSeconds are the per-tenant usage counters,
	// advanced by the leader from the usage events when
	// observability.usage_metrics is on. Their cardinality is the number of
	// tenants, so they stay empty by default.
	TaskAttempts    *prometheus.CounterVec
	TaskSlotSeconds *prometheus.CounterVec

	// TLSReloadFailures counts certificate or key files that changed on disk
	// but failed to load, by listener (api, agent_grpc). The listener keeps
	// serving the last good pair, so a non-zero rate means a rotation is stuck
	// and the served certificate will expire unless the files are fixed (#637).
	TLSReloadFailures *prometheus.CounterVec

	// LogTailReplayBytes is what the on_demand live-tail gates hold for replay
	// together, and LogTailReplayEvictions the rings logs.tail.replay_budget_bytes
	// dropped to stay under it (#1621).
	LogTailReplayBytes     prometheus.Gauge
	LogTailReplayEvictions prometheus.Counter

	// ResilienceBackstopDisabled is 1 for each resilience backstop a boot
	// setting removed and 0 for each one in place, set once at boot (#941).
	ResilienceBackstopDisabled *prometheus.GaugeVec
	// contains filtered or unexported fields
}

type Telemetry struct {
	Logger   *slog.Logger
	Metrics  *Metrics
	Registry *prometheus.Registry
	Tracer   trace.Tracer
}