Documentation
¶
Overview ¶
Package alerting implements threshold rules over internal/telemetry's metrics and logs, and crashloop detection as a built-in rule kind sharing the same evaluate/notify path rather than a separate mechanism.
Index ¶
- Constants
- Variables
- func BackupOverdue(history []store.BackupHistory, schedule string, grace time.Duration, ...) (overdue bool, anchor time.Time, anchorIsSuccess bool, err error)
- func BudgetRemaining(bad, total, target float64) float64
- func BurnRate(bad, total, target float64) float64
- func CertExpiryStatus(notAfter, now time.Time, warningWindow time.Duration) string
- func CertRenewalStates(infos []CertInfo, obs []CertExpiryObservation, threshold time.Duration, ...) map[string]string
- func ClampReleaseNotes(s string, limit int) string
- func DeleteCertificateForDomain(ctx context.Context, source CertSource, deleter CertDeleter, domain string) (found bool, err error)
- func DomainTrafficDropParams(r Rule) (window time.Duration, percent float64, minBaseline int)
- func EvaluateCertExpiryPolicy(ctx context.Context, certs CertSource, snapshots CertExpiryObservationStore, ...) (Rule, CertEvalResult, error)
- func EvaluateLogRule(ctx context.Context, src LogRuleSource, r Rule, policy LogRulePolicy, ...) (Rule, LogRuleResult, error)
- func FormatDigest(items []InboxItem) string
- func IsDatabaseKind(k Kind) bool
- func IsMaskedNotifyTarget(s string) bool
- func IsPlatformKind(k Kind) bool
- func IsPluginKind(k NotifyKind) bool
- func IsRequestSignal(metric string) bool
- func IsTrafficKind(k Kind) bool
- func MaskNotifyTarget(kind NotifyKind, target string) string
- func MaybeAutoRollback(ctx context.Context, st AutoRollbackStore, nudger deploy.ReconcileNudger, ...)
- func MaybeAutoRollbackOnSLOBurn(ctx context.Context, st SLOAutoRollbackStore, history SLOBurnHistoryRecorder, ...)
- func NewDeployTargetID() (string, error)
- func NewHistoryID() (string, error)
- func NewMaintenanceWindowID() (string, error)
- func NewNotificationChannelID() (string, error)
- func NewNotificationDeliveryID() (string, error)
- func NewRuleID() (string, error)
- func NewSilenceID() (string, error)
- func NotificationChannelSecretsKey(id string) string
- func PluginName(k NotifyKind) string
- func ReleaseNotesMaxBytes() int
- func RequestPathMetrics() []string
- type ACMEFailureSource
- type AlertContext
- type AppDomainSource
- type ApprovalRequestInfo
- type AutoRollbackStore
- type AutoRollbackTracker
- type BackupCopyPending
- type BackupCopySource
- type BackupSource
- type BudgetSource
- type BurnTier
- type BurnWindowStat
- type CertDeleter
- type CertEvalResult
- type CertExpiryObservation
- type CertExpiryObservationStore
- type CertExpiryPolicy
- type CertFact
- type CertInfo
- type CertLinks
- type CertPayload
- type CertSource
- type CertSuppressor
- type ChangeSource
- type ChannelSecrets
- type Comparator
- type ControlPlaneBackupSource
- type ControlPlaneDRSource
- type CostBudgetSource
- type DB
- func (db *DB) ArchiveInbox(ctx context.Context, principal string, ids []string, archived bool) error
- func (db *DB) CertExpiryEmail(ctx context.Context) (bool, error)
- func (db *DB) ClearInbox(ctx context.Context, principal, action string, sel InboxSelector, dryRun bool) (int, error)
- func (db *DB) CountHistoryBefore(ctx context.Context, cutoff time.Time) (int64, error)
- func (db *DB) CountPlaintextNotificationChannels(ctx context.Context) (int, error)
- func (db *DB) CreateSilence(ctx context.Context, s Silence) error
- func (db *DB) DeleteChannelQuietHours(ctx context.Context, channelID string) error
- func (db *DB) DeleteDeployTarget(ctx context.Context, id string) error
- func (db *DB) DeleteInbox(ctx context.Context, principal string, ids []string, restore bool, ...) error
- func (db *DB) DeleteMaintenanceWindow(ctx context.Context, id string) error
- func (db *DB) DeleteNotificationChannel(ctx context.Context, id string) error
- func (db *DB) DeleteRule(ctx context.Context, id string) error
- func (db *DB) DigestDue(ctx context.Context, now time.Time) ([]InboxPrefs, error)
- func (db *DB) ExpireSilence(ctx context.Context, id string, now time.Time) (*Silence, error)
- func (db *DB) GetChannelQuietHours(ctx context.Context, channelID string) (*QuietHours, error)
- func (db *DB) GetDeployTarget(ctx context.Context, id string) (*DeployTarget, error)
- func (db *DB) GetInboxPrefs(ctx context.Context, principal string) (InboxPrefs, error)
- func (db *DB) GetInboxState(ctx context.Context, principal string) (InboxState, error)
- func (db *DB) GetMaintenanceWindow(ctx context.Context, id string) (*MaintenanceWindow, error)
- func (db *DB) GetNotificationChannel(ctx context.Context, id string) (*NotificationChannel, error)
- func (db *DB) GetRule(ctx context.Context, id string) (*Rule, error)
- func (db *DB) GetSilence(ctx context.Context, id string) (*Silence, error)
- func (db *DB) HistoryStats(ctx context.Context) (rows int64, oldest time.Time, bytes int64, err error)
- func (db *DB) InboxUnreadCount(ctx context.Context, principal string) (int, error)
- func (db *DB) ListActiveSilences(ctx context.Context, now time.Time) ([]Silence, error)
- func (db *DB) ListAllCertExpiryObservations(ctx context.Context) ([]CertExpiryObservation, error)
- func (db *DB) ListCertExpiryObservations(ctx context.Context, ruleID string) ([]CertExpiryObservation, error)
- func (db *DB) ListDefaultRules(ctx context.Context) ([]Rule, error)
- func (db *DB) ListDeployTargetsForResource(ctx context.Context, resourceID string) ([]DeployTarget, error)
- func (db *DB) ListEnabledRules(ctx context.Context) ([]Rule, error)
- func (db *DB) ListHistory(ctx context.Context, f HistoryFilter) ([]HistoryEntry, error)
- func (db *DB) ListInbox(ctx context.Context, principal string, f InboxFilter) ([]InboxItem, error)
- func (db *DB) ListMaintenanceWindows(ctx context.Context) ([]MaintenanceWindow, error)
- func (db *DB) ListNotificationChannels(ctx context.Context) ([]NotificationChannel, error)
- func (db *DB) ListNotificationDeliveries(ctx context.Context, channelID string, limit int, before *time.Time) ([]NotificationDelivery, error)
- func (db *DB) ListRules(ctx context.Context) ([]Rule, error)
- func (db *DB) ListRulesForResource(ctx context.Context, resourceID string) ([]Rule, error)
- func (db *DB) ListSilences(ctx context.Context, now time.Time, includeExpired bool, limit int) ([]Silence, error)
- func (db *DB) MarkAllInboxRead(ctx context.Context, principal string, now time.Time) error
- func (db *DB) MarkDigestSent(ctx context.Context, principal string, now time.Time) error
- func (db *DB) MarkInboxRead(ctx context.Context, principal string, ids []string) error
- func (db *DB) PruneHistory(ctx context.Context, cutoff time.Time) (int64, error)
- func (db *DB) RecordHistory(ctx context.Context, e HistoryEntry) error
- func (db *DB) RecordNotificationDelivery(ctx context.Context, d NotificationDelivery) error
- func (db *DB) ResetDefaultRules(ctx context.Context) ([]Rule, error)
- func (db *DB) ResolveAllowed(ctx context.Context, ruleID, resourceID string, kind Kind) (bool, error)
- func (db *DB) RevealChannelTarget(ctx context.Context, channelID, target string) (string, error)
- func (db *DB) RevealNotificationChannel(ctx context.Context, c NotificationChannel) (NotificationChannel, error)
- func (db *DB) RouteDefaultRules(ctx context.Context, adminEmail string) (int, error)
- func (db *DB) SaveDeployTarget(ctx context.Context, t DeployTarget) error
- func (db *DB) SaveInboxPrefs(ctx context.Context, p InboxPrefs, now time.Time) error
- func (db *DB) SaveMaintenanceWindow(ctx context.Context, w MaintenanceWindow) error
- func (db *DB) SaveNotificationChannel(ctx context.Context, c NotificationChannel) error
- func (db *DB) SaveRule(ctx context.Context, r Rule) error
- func (db *DB) SealNotificationChannelSecrets(ctx context.Context) (int, error)
- func (db *DB) SeedDefaults(ctx context.Context, cfg DefaultsConfig) error
- func (db *DB) SelectInbox(ctx context.Context, principal string, sel InboxSelector) ([]string, error)
- func (db *DB) SetCertExpiryEmail(ctx context.Context, address string, on bool) error
- func (db *DB) SetChannelQuietHours(ctx context.Context, q QuietHours) error
- func (db *DB) SetChannelSecrets(cs ChannelSecrets)
- func (db *DB) SetDefaultRuleEnabled(ctx context.Context, id string, enabled bool) (*Rule, error)
- func (db *DB) UpdateState(ctx context.Context, id string, pendingSince, firingSince *time.Time, ...) error
- func (db *DB) UpsertCertExpiryObservation(ctx context.Context, o CertExpiryObservation) error
- type DatabaseBackupSource
- type DatabaseHealthSource
- type DatabaseLister
- type DatabaseLivenessSource
- type DatabasePoolerSource
- type DefaultRuleSpec
- type DefaultsConfig
- type DeliveryRecorder
- type DeployDispatcher
- func (d *DeployDispatcher) Dispatch(ctx context.Context, resourceID string, ev DeployOutcome)
- func (d *DeployDispatcher) Outcomes() DeployFailureSource
- func (d *DeployDispatcher) SendApprovalRequest(ctx context.Context, c NotificationChannel, info ApprovalRequestInfo) error
- func (d *DeployDispatcher) SendChannelTest(ctx context.Context, c NotificationChannel) error
- func (d *DeployDispatcher) SendNotice(ctx context.Context, c NotificationChannel, text string) error
- func (d *DeployDispatcher) SendTest(ctx context.Context, kind NotifyKind, notifyURL string) error
- func (d *DeployDispatcher) WithLogSources(runtime LogsSource, build DeployLogSource) *DeployDispatcher
- func (d *DeployDispatcher) WithOutcomeTracker(t *DeployOutcomeTracker) *DeployDispatcher
- type DeployFailure
- type DeployFailureSource
- type DeployLogSource
- type DeployOutcome
- type DeployOutcomeTracker
- type DeployTarget
- type DigestRunner
- type DigestSender
- type DomainCheckSource
- type DomainListSource
- type DomainTrafficSource
- type EmailDeliverySource
- type Engine
- func (e *Engine) EvaluateCertRenewalStalled(ctx context.Context, r Rule, now time.Time) (Rule, []string, error)
- func (e *Engine) EvaluateDomainNotResolving(ctx context.Context, r Rule, now time.Time) (Rule, []string, error)
- func (e *Engine) EvaluateDomainTrafficDrop(ctx context.Context, r Rule, now time.Time) (Rule, []string, error)
- func (e *Engine) EvaluateParkedDomainCertExpiry(ctx context.Context, r Rule, now time.Time) (Rule, []string, error)
- func (e *Engine) Run(ctx context.Context, interval time.Duration) error
- func (e *Engine) SetAutoRollback(st AutoRollbackStore, nudger deploy.ReconcileNudger)
- func (e *Engine) SetBackupCopy(src BackupCopySource, defaultLag time.Duration)
- func (e *Engine) SetBudgets(src BudgetSource)
- func (e *Engine) SetCertExpiryPolicy(p CertExpiryPolicy)
- func (e *Engine) SetChanges(src ChangeSource, link func(app string) string)
- func (e *Engine) SetControlPlaneBackups(src ControlPlaneBackupSource, maxAge time.Duration)
- func (e *Engine) SetDatabaseHealth(src DatabaseHealthSource)
- func (e *Engine) SetDeployFailures(src DeployFailureSource)
- func (e *Engine) SetDomainTraffic(src DomainTrafficSource)
- func (e *Engine) SetEmailDelivery(src EmailDeliverySource)
- func (e *Engine) SetExportHealth(src ExportHealthSource)
- func (e *Engine) SetLogArchive(src LogArchiveSource)
- func (e *Engine) SetMonitorDowns(src MonitorDownSource)
- func (e *Engine) SetNodeCertThresholds(warning, critical time.Duration)
- func (e *Engine) SetNoiseControl(n *NoiseControl)
- func (e *Engine) SetRegistryRefresh(src RegistryRefreshSource)
- func (e *Engine) SetRestoreDrills(src RestoreDrillSource)
- func (e *Engine) SetSLOBurnAutoRollback(st SLOAutoRollbackStore, nudger deploy.ReconcileNudger, ...)
- func (e *Engine) SetTrafficSources(domains DomainListSource, acme ACMEFailureSource)
- func (e *Engine) SetVersionSkew(settings VersionSkewSettingsSource, fetchers upgrade.Fetchers)
- func (e *Engine) Tick(ctx context.Context) error
- type Event
- type EventSource
- type ExportHealth
- type ExportHealthSource
- type HistoryEntry
- type HistoryFilter
- type InboxFilter
- type InboxItem
- type InboxPrefs
- type InboxSelector
- type InboxState
- type Kind
- type LogArchiveHealth
- type LogArchiveSource
- type LogRuleConfig
- type LogRulePolicy
- type LogRuleResult
- type LogRuleSource
- type LogsSource
- type MailRenderFunc
- type MaintenanceWindow
- type MetricClass
- type MetricsSource
- type MonitorDown
- type MonitorDownSource
- type MonitorDownTracker
- type MuteHit
- type NodeAlertState
- type NodeAlertStatus
- type NodeCertState
- type NodeServiceSource
- type NodeSource
- type NoiseConfig
- type NoiseControl
- func (n *NoiseControl) ApplyStreak(prev, next Rule, now time.Time) Rule
- func (n *NoiseControl) FlushAll(ctx context.Context, now time.Time, send SendFunc)
- func (n *NoiseControl) Recheck(ctx context.Context, r Rule, now time.Time, send SendFunc)
- func (n *NoiseControl) RecordSkipped(ctx context.Context, r Rule, resolved bool, now time.Time)
- func (n *NoiseControl) Route(ctx context.Context, ev Event, now time.Time, send SendFunc)
- func (n *NoiseControl) SetFiredHook(fn func(HistoryEntry))
- func (n *NoiseControl) Sweep(ctx context.Context, now time.Time, send SendFunc)
- type NoiseStore
- type NotificationChannel
- type NotificationDelivery
- type Notifier
- type NotifyKind
- type ParkedDomainSource
- type PlacementSource
- type PluginNotice
- type PluginSender
- type PushSender
- type QuietHours
- type RegistryRefreshHealth
- type RegistryRefreshSource
- type RequestCounts
- type RestartRecorder
- type RestartTracker
- func (t *RestartTracker) CountContainersSince(names []string, since time.Time) int
- func (t *RestartTracker) CountSince(resourceID string, since time.Time) int
- func (t *RestartTracker) Observe(resourceID, containerName string, at time.Time)
- func (t *RestartTracker) Prune(olderThan time.Time)
- func (t *RestartTracker) Run(ctx context.Context, source EventSource, services ServiceLister, ...) error
- func (t *RestartTracker) WithRecorder(recorder RestartRecorder, logger *slog.Logger) *RestartTracker
- type RestoreDrillProblem
- type RestoreDrillSource
- type Rule
- func EvaluateBackupCopyBehind(ctx context.Context, source BackupCopySource, defaultLag time.Duration, r Rule, ...) (Rule, string, error)
- func EvaluateBackupMissing(ctx context.Context, source BackupSource, r Rule, ...) (Rule, string, error)
- func EvaluateBudgetThreshold(ctx context.Context, src BudgetSource, r Rule, now time.Time) (Rule, []string, []string, error)
- func EvaluateCertExpiring(ctx context.Context, certs CertSource, r Rule, now time.Time, ...) (Rule, []string, error)
- func EvaluateCertExpiry(ctx context.Context, certs CertSource, snapshots CertExpiryObservationStore, ...) (Rule, []string, error)
- func EvaluateControlPlaneBackupStale(source ControlPlaneBackupSource, r Rule, defaultMaxAge time.Duration, ...) (Rule, string, error)
- func EvaluateCrashloop(tracker *RestartTracker, r Rule, now time.Time) Rule
- func EvaluateDatabaseRule(ctx context.Context, health DatabaseHealthSource, backups DatabaseBackupSource, ...) (Rule, string, error)
- func EvaluateDatabaseRuleAll(ctx context.Context, health DatabaseHealthSource, backups DatabaseBackupSource, ...) (Rule, string, error)
- func EvaluateDeployFailed(_ context.Context, src DeployFailureSource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateDomainHealth(ctx context.Context, apps AppDomainSource, checker DomainCheckSource, r Rule, ...) (Rule, []string, error)
- func EvaluateEmailDeliveryFailing(ctx context.Context, src EmailDeliverySource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateExportBehind(ctx context.Context, source ExportHealthSource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateLogArchiveStale(ctx context.Context, source LogArchiveSource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateMonitorDown(_ context.Context, src MonitorDownSource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateNodeCertExpiring(ctx context.Context, nodes NodeSource, r Rule, warning, critical time.Duration, ...) (Rule, []string, error)
- func EvaluateNodeDiskSpace(ctx context.Context, nodes NodeSource, metrics MetricsSource, r Rule, ...) (Rule, []string, error)
- func EvaluateNodeOffline(ctx context.Context, nodes NodeSource, r Rule, now time.Time) (Rule, []string, error)
- func EvaluateNodeResourceUsage(ctx context.Context, nodes NodeSource, services NodeServiceSource, ...) (Rule, []string, error)
- func EvaluatePatchStatus(ctx context.Context, nodes NodeSource, metrics MetricsSource, r Rule, ...) (Rule, []string, error)
- func EvaluateRegistryRefreshFailing(ctx context.Context, source RegistryRefreshSource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateRestoreDrillFailed(ctx context.Context, source RestoreDrillSource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateSLOBurn(ctx context.Context, src MetricsSource, r Rule, p SLOPolicy, now time.Time) (Rule, string, error)
- func EvaluateScheduledTaskFailure(ctx context.Context, tasks ScheduledTaskSource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateScheduledTaskHealth(ctx context.Context, tasks ScheduledTaskSource, r Rule, now time.Time) (Rule, string, error)
- func EvaluateThreshold(ctx context.Context, source MetricsSource, r Rule, now time.Time) (Rule, error)
- func EvaluateVersionSkew(ctx context.Context, settings VersionSkewSettingsSource, ...) (Rule, string, error)
- type RuleStore
- type SLOAutoRollbackStore
- type SLOBurnHistoryRecorder
- type SLOConfig
- type SLOPolicy
- type SLOStatus
- type ScheduledTaskSource
- type SendFunc
- type ServiceLister
- type Silence
- type SilenceMatcher
- type SpendCapSource
- type TemplatedSender
- type TidyConfig
- type TidyResult
- type TidyRunner
- type TierStatus
- type VersionSkewSettingsSource
- type WindowState
Constants ¶
const ( DefaultCertLastResortWindow = 72 * time.Hour DefaultCertReminderInterval = 24 * time.Hour // DefaultCertRenewalRatio mirrors the fraction of lifetime left at which the ingress renews. DefaultCertRenewalRatio = 1.0 / 3.0 )
Defaults for the cert expiry policy, each overridable through the environment.
const ( CertActionRenew = "renew" CertActionDNS = "dns" )
Fix actions a cert alert can offer.
const ( CertFailureRenewalFailing = "renewal_failing" CertFailureDNS = "dns" CertFailureOther = "other" )
Failure kinds carried on the structured cert payload.
const ( CertRenewalOK = "ok" CertRenewalStalled = "stalled" )
Certificate renewal states reported by GET /api/v1/certificates.
const ( SlackApprovalActionApprove = "deploy_approval_approve" SlackApprovalActionReject = "deploy_approval_reject" )
Shared action_id constants, so the Slack buttons and their handler in internal/api cannot drift apart.
const ( DiscordApprovalCustomIDApprove = "deploy_approval_approve" DiscordApprovalCustomIDReject = "deploy_approval_reject" )
DiscordApprovalCustomIDApprove/Reject prefix a button's custom_id; chat_interactions.go splits on ":" to recover the approval ID.
const ( DefaultDatabaseConnectionsPercent = 80.0 DefaultDatabaseDiskPercent = 85.0 )
Defaults when a rule leaves Threshold unset.
const ( // EnvReleaseNotesMaxBytes caps release notes attached to a deploy notification. EnvReleaseNotesMaxBytes = "RELEASE_NOTES_MAX_BYTES" // EnvReleaseNotesPreviewBytes caps the excerpt sent to short-text channels. EnvReleaseNotesPreviewBytes = "RELEASE_NOTES_PREVIEW_BYTES" )
const ( DefaultDomainTrafficDropWindow = time.Hour DefaultDomainTrafficDropPercent = 50.0 DefaultDomainTrafficDropMinBaseline = 20 DefaultParkedCertExpiryDays = 14 )
Defaults for the domain lifecycle kinds; a rule's own fields override them.
const ( EmailDeliveryFailingWindow = 30 * time.Minute DefaultEmailDeliveryFailureFloor = 3 )
Defaults for KindEmailDeliveryFailing: the look-back window and the failure count that must be exceeded.
const ( EventFired = "fired" EventResolved = "resolved" EventFlapping = "flapping" EventFlapEnded = "flap_ended" EventGroupFlush = "group" )
History event names.
const ( OutcomeSent = "sent" OutcomeSilenced = "silenced" OutcomeGrouped = "grouped" OutcomeInhibited = "inhibited" OutcomeFailed = "failed" OutcomeRateLimited = "ratelimited" OutcomeFlapping = "flapping" OutcomeSkipped = "skipped" OutcomeQuiet = "quiet_hours" )
History outcome names: what happened to the notification.
const ( InboxDefaultMinSeverity = SeverityWarning InboxDefaultDigestHour = 8 )
Inbox defaults: warning and critical only, so a new principal sees no info chatter.
const ( InboxScopeRead = "read" InboxScopeResolved = "resolved" InboxScopeArchived = "archived" InboxActionArchive = "archive" InboxActionDelete = "delete" )
Bulk scopes and actions accepted by ClearInbox.
const ( EnvInboxAutoArchiveAfter = "APP_INBOX_AUTO_ARCHIVE_RESOLVED_AFTER" EnvInboxRetention = "APP_INBOX_ARCHIVE_RETENTION" EnvInboxTidyInterval = "APP_INBOX_TIDY_INTERVAL" )
Env vars for the inbox sweep. A zero duration disables that step; an invalid one falls back to the default.
const ( DefaultLogRuleEvalEvery = time.Minute DefaultLogRuleMaxWindow = time.Hour DefaultLogRuleMaxRows = 5000 DefaultLogRuleSamples = 5 )
Env vars and defaults bounding log rule evaluation.
const ( ScopeAll = "all" ScopeApp = "app" ScopeNode = "node" )
Maintenance window scopes.
const ( DefaultNodeCertWarning = 21 * 24 * time.Hour DefaultNodeCertCritical = 7 * 24 * time.Hour )
Defaults for the node certificate thresholds. Healthy agents renew with a third of a 90 day lifetime left, so a certificate inside 21 days means renewal has been failing for over a week.
const ( DefaultFlapThreshold = 5 DefaultFlapWindow = 30 * time.Minute DefaultRateLimit = 30 DefaultRateWindow = 10 * time.Minute DefaultHistoryRetention = 30 * 24 * time.Hour )
Defaults for NoiseConfig, overridable per environment in cmd/levelrail.
const ( MetricRequestErrorRate5xxPct = "request_error_rate_5xx_pct" MetricRequestP95Ms = "request_p95_ms" MetricRequestP99Ms = "request_p99_ms" // EnvRequestSignalWindow overrides the trailing window of derived request metrics. EnvRequestSignalWindow = "APP_ALERT_REQUEST_WINDOW" )
Derived request metrics a KindThreshold rule can watch. They are computed from the ingress request counters over a trailing window, not stored.
const ( SeverityInfo = "info" SeverityWarning = "warning" SeverityCritical = "critical" )
Rule severities, used by silence matchers and shown in history.
const ( SilencePending = "pending" SilenceActive = "active" SilenceExpired = "expired" )
Silence status values.
const ( SLOAvailability = "availability" SLOLatency = "latency" )
SLO objective kinds.
const ( DefaultSLOWindow = 30 * 24 * time.Hour DefaultSLOMinRequests = 10 DefaultSLOEvalEvery = time.Minute )
Defaults follow the Google SRE workbook multiwindow multi-burn-rate table.
const ( DefaultCertExpiringDays = 14 DefaultDomainNotResolvingWindow = 30 * time.Minute )
Defaults for the traffic kinds; a rule's own fields override them.
const DefaultAdminEmailChannelID = "chn_default_admin_email"
DefaultAdminEmailChannelID is the notification channel the seeded rules are routed to when email is configured.
const DefaultBackupMissingGracePeriod = 6 * time.Hour
DefaultBackupMissingGracePeriod is the slack past a backup's expected interval before it counts as overdue.
const DefaultCertExpiryWarningWindow = 14 * 24 * time.Hour
DefaultCertExpiryWarningWindow is the default lead time before NotAfter for a certificate to report expiring_soon.
const DefaultCertRenewalStalledThreshold = 6 * time.Hour
DefaultCertRenewalStalledThreshold is how long NotAfter may stay unchanged in the warning window before renewal is reported stalled.
const DefaultControlPlaneBackupMaxAge = 3 * 24 * time.Hour
DefaultControlPlaneBackupMaxAge matches the doctor check's warning age.
const DefaultDatabaseBackupStaleHours = 26.0
DefaultDatabaseBackupStaleHours is the age of the newest successful backup past which database_backup_stale fires when Threshold is unset.
const DefaultDeployFailedWindow = time.Hour
DefaultDeployFailedWindow is how long a failed deploy keeps a deploy_failed rule firing when RestartWindow is unset.
const DefaultDomainHealthCheckInterval = 5 * time.Minute
DefaultDomainHealthCheckInterval is how often a domain_health rule runs a real DNS check by default. Checking more often than Engine's 30s tick only adds outbound DNS traffic.
const DefaultExportBehindAfter = 15 * time.Minute
DefaultExportBehindAfter is how long a failing destination may go without a successful delivery before the rule fires.
const DefaultNodeCPUThresholdPercent = 80.0
DefaultNodeCPUThresholdPercent is the summed node CPU percent that fires a node_resource_usage rule when unset.
const DefaultNodeDiskSpaceThresholdPercent = 90.0
DefaultNodeDiskSpaceThresholdPercent is the percent-used level a node_disk_space rule fires at when none is configured. Percent-used, not a free-byte floor, because fixed floors do not scale across disk sizes.
const DefaultNodeMemoryThresholdBytes = 4 * 1024 * 1024 * 1024
DefaultNodeMemoryThresholdBytes is the summed node memory that fires a node_resource_usage rule when unset. No honest host memory total exists to compare against, so this absolute floor should be tuned per node.
const DefaultPatchStatusThreshold = 1
DefaultPatchStatusThreshold is the pending security patch count that fires a patch_status rule; 1 means any pending patch.
const DefaultPoolerSaturatedWaitSeconds = 5.0
DefaultPoolerSaturatedWaitSeconds is how long the longest queued client waits before pooler_saturated fires when Threshold is unset.
const DefaultRuleResource = "platform"
DefaultRuleResource is the ResourceID of every seeded rule; they are platform-wide, not tied to an app.
const EventSLOBurnWouldRollback = "slo_burn_would_rollback"
EventSLOBurnWouldRollback is the history event for a dry-run rollback that was evaluated but not applied.
const MaskedChannelSecret = "(hidden)"
MaskedChannelSecret stands in for a sealed credential in every column and API response.
const MaxMaintenanceDuration = 30 * 24 * time.Hour
MaxMaintenanceDuration bounds a single window so a typo cannot silence alerts for months.
const NotifyPluginPrefix = "plugin:"
NotifyPluginPrefix starts the channel kind of a notification_channel plugin, for example "plugin:example-notify".
const RegistryRefreshFixPath = "/settings/registry-credentials"
RegistryRefreshFixPath is the settings page whose Refresh now button fixes the alert.
Variables ¶
var ( // but no master key is configured to open it. ErrChannelSecretsUnavailable = errors.New("alerting: notification channel secrets need a master key") // ErrChannelSecretNotRevealed means a send was attempted with the // masked placeholder instead of the decrypted credential. ErrChannelSecretNotRevealed = errors.New("alerting: notification target is still masked") )
var ( ErrPluginChannelUnsupported = errors.New("alerting: plugin channels are not supported for this notice yet") )
Plugin channel failures. A plugin kind never falls back to the generic webhook shape.
var DefaultCertExpiryDays = []int{14, 7, 3, 1}
DefaultCertExpiryDays are the escalating lead times, in days, for the seeded cert_expiry rules.
ErrDatabaseHealthUnavailable means the probe cannot read the database right now (stopped, remote node, unsupported engine); the rule goes quiet.
var ErrDeployTargetNotFound = errors.New("alerting: deploy target not found")
ErrDeployTargetNotFound is returned by GetDeployTarget when no target has that ID.
var ErrInboxInvalid = errors.New("invalid inbox input")
ErrInboxInvalid marks a preferences or request value the caller can fix.
var ErrInboxNotDeletable = fmt.Errorf("%w: only stored alert items can be deleted", ErrInboxInvalid)
ErrInboxNotDeletable marks ids that are derived live (approvals, failed deploys, failed upgrades) and can only be archived.
var ErrInteractiveApprovalUnsupported = fmt.Errorf("alerting: send approval request: channel kind does not support interactive approval")
ErrInteractiveApprovalUnsupported is returned for channel kinds other than Slack or Discord. Reaching it means the row predates save-time validation.
var ErrMaintenanceWindowNotFound = errors.New("alerting: maintenance window not found")
ErrMaintenanceWindowNotFound is returned when no window has that ID.
var ErrNotificationChannelNotFound = errors.New("alerting: notification channel not found")
ErrNotificationChannelNotFound is returned by GetNotificationChannel and DeleteNotificationChannel when id doesn't match any row.
var ErrRuleNotFound = errors.New("alerting: rule not found")
ErrRuleNotFound is returned by GetRule when no rule has that ID.
var ErrSilenceNotFound = errors.New("alerting: silence not found")
ErrSilenceNotFound is returned when no silence has that ID.
Functions ¶
func BackupOverdue ¶ added in v0.2.0
func BackupOverdue(history []store.BackupHistory, schedule string, grace time.Duration, now time.Time) (overdue bool, anchor time.Time, anchorIsSuccess bool, err error)
BackupOverdue reports whether the newest history (newest first, non-empty) trails its cron schedule by more than one expected interval plus grace. A target that never succeeded anchors on its oldest attempt.
func BudgetRemaining ¶ added in v0.2.0
BudgetRemaining is the fraction of the error budget left (1 untouched, 0 spent, negative overspent). No traffic leaves it untouched.
func BurnRate ¶ added in v0.2.0
BurnRate is how many times faster than sustainable the error budget is being spent: the observed bad ratio divided by the allowed one. No traffic burns nothing.
func CertExpiryStatus ¶
CertExpiryStatus buckets notAfter into expired, expiring_soon or healthy. A notAfter exactly warningWindow away is still healthy: the window is when warnings start. boundary.
func CertRenewalStates ¶ added in v0.2.0
func CertRenewalStates(infos []CertInfo, obs []CertExpiryObservation, threshold time.Duration, now time.Time) map[string]string
CertRenewalStates maps each domain to CertRenewalOK or CertRenewalStalled. Expired is always stalled; expiring_soon is stalled once NotAfter is unchanged for threshold, as in EvaluateCertExpiry. the same test EvaluateCertExpiry applies.
func ClampReleaseNotes ¶
ClampReleaseNotes trims s to at most limit bytes (marker included) on a rune boundary.
func DeleteCertificateForDomain ¶ added in v0.2.0
func DeleteCertificateForDomain(ctx context.Context, source CertSource, deleter CertDeleter, domain string) (found bool, err error)
DeleteCertificateForDomain removes matching entries; no match is a no-op since reconcile re-issues anyway.
func DomainTrafficDropParams ¶
DomainTrafficDropParams returns the rule's effective window, percent and minimum baseline.
func EvaluateCertExpiryPolicy ¶
func EvaluateCertExpiryPolicy(ctx context.Context, certs CertSource, snapshots CertExpiryObservationStore, r Rule, warningWindow, stalledThreshold time.Duration, now time.Time, logger *slog.Logger, policy CertExpiryPolicy) (Rule, CertEvalResult, error)
EvaluateCertExpiryPolicy is EvaluateCertExpiry with the quieting policy applied: one window per certificate, no alert for certificates the platform renews itself.
func EvaluateLogRule ¶ added in v0.2.0
func EvaluateLogRule(ctx context.Context, src LogRuleSource, r Rule, policy LogRulePolicy, now time.Time) (Rule, LogRuleResult, error)
EvaluateLogRule runs one log_pattern or log_level_rate rule: it reads at most policy.MaxRows of the newest lines in the window, matches them and advances the rule's state.
func FormatDigest ¶
FormatDigest renders items as a short plain-text summary, or "" when there is nothing to say.
func IsDatabaseKind ¶
IsDatabaseKind reports whether k is one of the database rule kinds.
func IsMaskedNotifyTarget ¶ added in v0.3.0
IsMaskedNotifyTarget reports whether s is MaskNotifyTarget's output for a sealed credential rather than a usable destination.
func IsPlatformKind ¶ added in v0.2.0
IsPlatformKind reports whether a rule kind watches the whole control plane rather than one app.
func IsPluginKind ¶
func IsPluginKind(k NotifyKind) bool
IsPluginKind reports whether k names a plugin channel.
func IsRequestSignal ¶
IsRequestSignal reports whether metric is a derived request metric.
func IsTrafficKind ¶ added in v0.2.0
IsTrafficKind reports whether k is one of the traffic kinds above.
func MaskNotifyTarget ¶ added in v0.3.0
func MaskNotifyTarget(kind NotifyKind, target string) string
MaskNotifyTarget hides the credential in target, keeping only where it points. An email recipient is an address, not a credential.
func MaybeAutoRollback ¶ added in v0.2.0
func MaybeAutoRollback(ctx context.Context, st AutoRollbackStore, nudger deploy.ReconcileNudger, tracker *AutoRollbackTracker, resourceID string, logger *slog.Logger)
MaybeAutoRollback rolls resourceID back to its previous good image when opted in, and logs rather than returns failures.
func MaybeAutoRollbackOnSLOBurn ¶ added in v0.2.0
func MaybeAutoRollbackOnSLOBurn(ctx context.Context, st SLOAutoRollbackStore, history SLOBurnHistoryRecorder, nudger deploy.ReconcileNudger, tracker *AutoRollbackTracker, r Rule, notice string, logger *slog.Logger)
MaybeAutoRollbackOnSLOBurn applies the app's SLO burn auto-rollback mode (auto, dry_run or pause_for_human) when a KindSLOBurn rule fires. Failures are logged, never returned, so one rule cannot block evaluation of the rest.
func NewDeployTargetID ¶
NewDeployTargetID generates a random, URL-safe deploy target identifier.
func NewHistoryID ¶ added in v0.2.0
NewHistoryID generates a random history identifier.
func NewMaintenanceWindowID ¶ added in v0.2.0
NewMaintenanceWindowID generates a random window identifier.
func NewNotificationChannelID ¶
NewNotificationChannelID mirrors NewDeployTargetID's exact shape.
func NewNotificationDeliveryID ¶
NewNotificationDeliveryID mirrors NewNotificationChannelID's exact shape.
func NewRuleID ¶
NewRuleID returns a random URL-safe rule ID, so callers never choose one and cannot collide or predict it.
func NewSilenceID ¶ added in v0.2.0
NewSilenceID generates a random silence identifier.
func NotificationChannelSecretsKey ¶ added in v0.3.0
NotificationChannelSecretsKey is the internal/secrets serviceName a channel's credentials are stored under.
func PluginName ¶
func PluginName(k NotifyKind) string
PluginName returns the plugin a plugin kind points at.
func ReleaseNotesMaxBytes ¶
func ReleaseNotesMaxBytes() int
ReleaseNotesMaxBytes reads the notes size cap from the environment.
func RequestPathMetrics ¶
func RequestPathMetrics() []string
RequestPathMetrics lists every metric classed as request-path, sorted.
Types ¶
type ACMEFailureSource ¶ added in v0.2.0
ACMEFailureSource reports whether the CA's last attempt for a domain failed.
func NewSuppressedACMESource ¶ added in v0.2.0
func NewSuppressedACMESource(inner ACMEFailureSource, suppress CertSuppressor) ACMEFailureSource
NewSuppressedACMESource reports no failure for hostnames suppress names, so a stale attempt made before a proxy took over TLS never raises an alert.
type AlertContext ¶ added in v0.2.0
type AlertContext struct {
RuleID string
Kind string
Metric string
App string
NodeID string
NodeName string
Labels map[string]string
Severity string
}
AlertContext is what a silence matcher or maintenance window is evaluated against: the rule plus where its app is placed.
type AppDomainSource ¶
type AppDomainSource interface {
GetDesiredService(ctx context.Context, name string) (*store.DesiredService, error)
}
AppDomainSource is the narrow store surface EvaluateDomainHealth needs: which domains a domain_health rule's own app currently has configured. *store.DB satisfies this structurally.
type ApprovalRequestInfo ¶ added in v0.2.0
type ApprovalRequestInfo struct {
ApprovalID string
ServiceName string
// Action is one of store.DeployApprovalAction* ("deploy" or
// "promote"), duplicated here as a plain string so this package
// doesn't need to import internal/store just for two constants.
Action string
Image string
RequestedBy string
EnvironmentID string
}
ApprovalRequestInfo identifies the gated change and the approval ID each button carries back on click.
type AutoRollbackStore ¶ added in v0.2.0
type AutoRollbackStore interface {
deploy.ImageDeployStore
GetDesiredService(ctx context.Context, name string) (*store.DesiredService, error)
ListDeployAttempts(ctx context.Context, serviceName string) ([]store.DeployAttempt, error)
}
AutoRollbackStore is the narrow store surface MaybeAutoRollback needs, which *store.DB satisfies.
type AutoRollbackTracker ¶ added in v0.2.0
type AutoRollbackTracker struct {
// contains filtered or unexported fields
}
AutoRollbackTracker records the image each service was last auto-rolled back to, so a new bad deploy can trigger another rollback while the rule still fires.
func NewAutoRollbackTracker ¶ added in v0.2.0
func NewAutoRollbackTracker() *AutoRollbackTracker
NewAutoRollbackTracker builds an empty AutoRollbackTracker.
type BackupCopyPending ¶
type BackupCopyPending struct {
Resource string
BackupID string
FinishedAt time.Time
LastError string
}
BackupCopyPending is one backup that has not reached the second destination yet.
type BackupCopySource ¶
type BackupCopySource interface {
PendingBackupCopies(ctx context.Context) ([]BackupCopyPending, error)
}
BackupCopySource lists the backups still waiting for their second copy.
type BackupSource ¶ added in v0.2.0
type BackupSource interface {
GetDesiredDatabase(ctx context.Context, name string) (*store.DesiredDatabase, error)
ListBackupHistory(ctx context.Context, databaseName string, limit int, before *time.Time) ([]store.BackupHistory, error)
GetServiceVolumeBackupSchedule(ctx context.Context, serviceName, volumeName string) (store.ServiceVolumeBackupConfig, error)
ListServiceVolumeBackupHistory(ctx context.Context, serviceName, volumeName string, limit int, before *time.Time) ([]store.BackupHistory, error)
}
BackupSource is the narrow store surface a backup-missing evaluation needs; *store.DB satisfies it.
type BudgetSource ¶ added in v0.2.0
type BudgetSource interface {
BudgetStatuses(ctx context.Context) ([]usagecost.BudgetStatus, error)
}
BudgetSource reports each budget's month to date position. The api Router satisfies it.
type BurnTier ¶ added in v0.2.0
type BurnTier struct {
Name string `json:"name"`
Factor float64 `json:"factor"`
Long time.Duration `json:"long"`
Short time.Duration `json:"short"`
Page bool `json:"page"`
}
BurnTier fires when the burn rate is at least Factor over both Long and Short. The short window makes the alert reset quickly once the burn stops.
type BurnWindowStat ¶ added in v0.2.0
type BurnWindowStat struct {
Window time.Duration `json:"window"`
Requests float64 `json:"requests"`
Bad float64 `json:"bad"`
Burn float64 `json:"burn_rate"`
}
BurnWindowStat is the observed burn over one window.
type CertDeleter ¶ added in v0.2.0
CertDeleter is the narrow storage surface DeleteCertificateForDomain needs beyond CertSource's two read methods: *store.DB satisfies this structurally, same as CertSource.
type CertEvalResult ¶
type CertEvalResult struct {
Notices []string
Facts []CertFact
// Superseded is true when this rule saw unhealthy certificates but a tighter rule owns them all.
Superseded bool
}
CertEvalResult is what one cert_expiry evaluation reports besides the rule state.
type CertExpiryObservation ¶
type CertExpiryObservation struct {
RuleID string
Domain string
Status string
NotAfter time.Time
EpisodeNotAfter time.Time
EpisodeStartedAt time.Time
ObservedAt time.Time
}
CertExpiryObservation is persisted per rule and domain so the next tick can tell an unchanged episode from a new one.
type CertExpiryObservationStore ¶
type CertExpiryObservationStore interface {
UpsertCertExpiryObservation(ctx context.Context, o CertExpiryObservation) error
ListCertExpiryObservations(ctx context.Context, ruleID string) ([]CertExpiryObservation, error)
}
CertExpiryObservationStore is the narrow store surface EvaluateCertExpiry needs to remember what it saw last tick, per domain. *DB satisfies this structurally.
type CertExpiryPolicy ¶
type CertExpiryPolicy struct {
Enabled bool
// Windows lists every enabled cert_expiry rule window so only the tightest matching one reports a certificate.
Windows []time.Duration
LastResort time.Duration
RenewRatio float64
// ReminderInterval is the minimum gap between reminders for a still-firing rule.
ReminderInterval time.Duration
ACME ACMEFailureSource
// BaseURL is the instance's public dashboard address used for email deep links.
BaseURL string
}
CertExpiryPolicy decides which certificates a cert_expiry rule reports. The zero value keeps the legacy behavior of reporting every non-healthy certificate.
func CertPolicyFromEnv ¶
func CertPolicyFromEnv(lookup func(string) string) CertExpiryPolicy
CertPolicyFromEnv reads the policy tunables; invalid values keep the defaults.
type CertFact ¶
type CertFact struct {
Domain string
Issuer string
DaysLeft int
Reason string
Action string
// NotAfter and FailureKind feed CertPayload.
NotAfter time.Time
FailureKind string
}
CertFact is the structured form of one cert alert notice, used by the email and the Inbox card.
func (CertFact) Notice ¶
Notice renders the fact as the one-line notice shown in history, webhooks and the plain-text email.
func (CertFact) Payload ¶
func (f CertFact) Payload() CertPayload
Payload returns the structured form of the fact; an unset failure kind reads as other.
type CertInfo ¶
type CertInfo struct {
Domain string
SANs []string
Issuer string
NotBefore time.Time
NotAfter time.Time
// Status is "healthy", "expiring_soon", or "expired"; see
// CertExpiryStatus.
Status string
// Internal marks a self-signed or local-CA certificate the platform issues by design.
Internal bool
}
CertInfo is one certificate's expiry fields, shared by the API and cert_expiry rules so they never disagree.
type CertLinks ¶
CertLinks are the dashboard deep links a cert alert offers; every field is empty when no public base URL is configured.
type CertPayload ¶
type CertPayload struct {
Domain string `json:"domain"`
Issuer string `json:"issuer,omitempty"`
NotAfter time.Time `json:"not_after"`
DaysLeft int `json:"days_left"`
FailureKind string `json:"failure_kind"`
}
CertPayload is the structured cert data stored beside an alert history entry so clients need not parse its text.
type CertSource ¶
type CertSource interface {
ListCertStorageKeys(ctx context.Context, prefix string, recursive bool) ([]string, error)
GetCertStorageValue(ctx context.Context, key string) (*store.CertStorageValue, error)
}
CertSource is redeclared here to avoid an import cycle, since internal/api imports this package.
func NewSuppressedCertSource ¶ added in v0.2.0
func NewSuppressedCertSource(inner CertSource, suppress CertSuppressor) CertSource
NewSuppressedCertSource hides stored certificates whose hostname suppress reports, so cert expiry and renewal rules never fire on a certificate visitors do not receive. A nil suppress returns inner unchanged.
type CertSuppressor ¶ added in v0.2.0
CertSuppressor returns, for one evaluation, whether a stored certificate's hostname must be ignored because a proxy terminates its TLS.
type ChangeSource ¶ added in v0.2.0
type ChangeSource interface {
Collect(ctx context.Context, app string, until time.Time) changes.Result
}
ChangeSource reports what changed on an app before an alert fired. *changes.Aggregator satisfies it.
type ChannelSecrets ¶ added in v0.3.0
type ChannelSecrets interface {
SetValue(ctx context.Context, serviceName, envKey, plaintext string) error
Resolve(ctx context.Context, serviceName, envKey string) (string, error)
DeleteAll(ctx context.Context, serviceName string) error
}
ChannelSecrets is the envelope-encryption surface channel credentials are sealed with; *secrets.Manager satisfies it.
type Comparator ¶
type Comparator string
Comparator is how a threshold Rule compares the latest sample value against its Threshold.
const ( GreaterThan Comparator = ">" LessThan Comparator = "<" GreaterOrEqual Comparator = ">=" LessOrEqual Comparator = "<=" )
The four comparators a threshold Rule can use.
type ControlPlaneBackupSource ¶ added in v0.2.0
ControlPlaneBackupSource reports when the newest control plane snapshot was taken.
type ControlPlaneDRSource ¶ added in v0.2.0
ControlPlaneDRSource is an optional extension of ControlPlaneBackupSource: DRProblem describes a failed or overdue off-box backup or restore drill, or returns "" when healthy.
type CostBudgetSource ¶
type CostBudgetSource interface {
CostBudgetBreaches(ctx context.Context) ([]costbudget.Breach, error)
}
CostBudgetSource is optional on a BudgetSource: named cost budgets add the highest threshold each one fired this period to the same rule.
type DB ¶
DB wraps alerting.db, kept apart from levelrail.db and telemetry.db because rule evaluation writes on every tick.
func Open ¶
Open opens (creating if needed) the SQLite database at path, applies pragmas, and runs every pending migration, matching internal/store and internal/telemetry's own Open functions exactly.
func (*DB) ArchiveInbox ¶
func (db *DB) ArchiveInbox(ctx context.Context, principal string, ids []string, archived bool) error
ArchiveInbox archives the given item ids for principal, or restores them when archived is false.
func (*DB) CertExpiryEmail ¶
CertExpiryEmail reports whether the default cert expiry rules currently route to the admin email channel.
func (*DB) ClearInbox ¶
func (db *DB) ClearInbox(ctx context.Context, principal, action string, sel InboxSelector, dryRun bool) (int, error)
ClearInbox archives or deletes every item sel matches and returns how many it touched. With dryRun it only counts.
func (*DB) CountHistoryBefore ¶
CountHistoryBefore counts what PruneHistory would delete for the same cutoff.
func (*DB) CountPlaintextNotificationChannels ¶ added in v0.3.0
CountPlaintextNotificationChannels returns how many channels still hold a credential in plaintext.
func (*DB) CreateSilence ¶ added in v0.2.0
CreateSilence inserts a new silence. Never updates in place: expiring early is ExpireSilence, so history stays intact.
func (*DB) DeleteChannelQuietHours ¶
DeleteChannelQuietHours removes the channel's quiet hours; absent is not an error.
func (*DB) DeleteDeployTarget ¶
DeleteDeployTarget removes a deploy target by ID. Deleting a target that doesn't exist is not an error, the same idempotent-delete convention DeleteRule already follows.
func (*DB) DeleteInbox ¶
func (db *DB) DeleteInbox(ctx context.Context, principal string, ids []string, restore bool, now time.Time) error
DeleteInbox hides stored alert items from the principal's inbox, or brings them back when restore is true. Alert history itself is kept; ids of derived items are rejected so they can only be archived.
func (*DB) DeleteMaintenanceWindow ¶ added in v0.2.0
DeleteMaintenanceWindow removes a window; deleting a missing one is a no-op.
func (*DB) DeleteNotificationChannel ¶
DeleteNotificationChannel removes a channel row. Attached deploy_notify_targets rows have channel_id cleared automatically by migrations/0004's ON DELETE SET NULL, never blocking this delete.
func (*DB) DeleteRule ¶
DeleteRule removes a rule by ID. Deleting a rule that doesn't exist is not an error, matching internal/store.DeleteDesiredService's own idempotent-delete convention.
func (*DB) DigestDue ¶
DigestDue lists principals whose digest is on, whose hour (UTC) has arrived and who got none in the last 20 hours.
func (*DB) ExpireSilence ¶ added in v0.2.0
ExpireSilence ends a silence at now. Expiring an already-expired silence is a no-op; the row is kept for history.
func (*DB) GetChannelQuietHours ¶
GetChannelQuietHours returns the channel's quiet hours, or nil when none are set.
func (*DB) GetDeployTarget ¶
GetDeployTarget returns the deploy target with this ID, or ErrDeployTargetNotFound.
func (*DB) GetInboxPrefs ¶
GetInboxPrefs returns saved preferences or the defaults.
func (*DB) GetInboxState ¶
GetInboxState returns the principal's per-item read and archive rows.
func (*DB) GetMaintenanceWindow ¶ added in v0.2.0
GetMaintenanceWindow returns one window or ErrMaintenanceWindowNotFound.
func (*DB) GetNotificationChannel ¶
GetNotificationChannel returns the channel with this ID, or ErrNotificationChannelNotFound.
func (*DB) GetSilence ¶ added in v0.2.0
GetSilence returns one silence or ErrSilenceNotFound.
func (*DB) HistoryStats ¶
func (db *DB) HistoryStats(ctx context.Context) (rows int64, oldest time.Time, bytes int64, err error)
HistoryStats returns the alert_history row count, its oldest entry (zero when empty) and its size in bytes when SQLite reports it.
func (*DB) InboxUnreadCount ¶
InboxUnreadCount counts visible unread items, capped at the newest 200.
func (*DB) ListActiveSilences ¶ added in v0.2.0
ListActiveSilences returns only the silences applying at now.
func (*DB) ListAllCertExpiryObservations ¶ added in v0.2.0
func (db *DB) ListAllCertExpiryObservations(ctx context.Context) ([]CertExpiryObservation, error)
ListAllCertExpiryObservations returns every observation row across all rules, so a reader with no rule in hand can see episodes any kind=cert_expiry rule has recorded.
func (*DB) ListCertExpiryObservations ¶
func (db *DB) ListCertExpiryObservations(ctx context.Context, ruleID string) ([]CertExpiryObservation, error)
ListCertExpiryObservations returns every observation row for ruleID, one per domain, unordered.
func (*DB) ListDefaultRules ¶
ListDefaultRules returns the seeded rules that still exist.
func (*DB) ListDeployTargetsForResource ¶
func (db *DB) ListDeployTargetsForResource(ctx context.Context, resourceID string) ([]DeployTarget, error)
ListDeployTargetsForResource returns every deploy target for resourceID, enabled or not.
func (*DB) ListEnabledRules ¶
ListEnabledRules returns only enabled rules; disabled ones are skipped before any metrics query.
func (*DB) ListHistory ¶ added in v0.2.0
func (db *DB) ListHistory(ctx context.Context, f HistoryFilter) ([]HistoryEntry, error)
ListHistory returns entries newest first.
func (*DB) ListInbox ¶
ListInbox returns the principal's items newest first, honoring its routing rules.
func (*DB) ListMaintenanceWindows ¶ added in v0.2.0
func (db *DB) ListMaintenanceWindows(ctx context.Context) ([]MaintenanceWindow, error)
ListMaintenanceWindows returns every window ordered by name.
func (*DB) ListNotificationChannels ¶
func (db *DB) ListNotificationChannels(ctx context.Context) ([]NotificationChannel, error)
ListNotificationChannels returns every channel, oldest first (creation order, the same order the settings page listing them wants).
func (*DB) ListNotificationDeliveries ¶
func (db *DB) ListNotificationDeliveries(ctx context.Context, channelID string, limit int, before *time.Time) ([]NotificationDelivery, error)
ListNotificationDeliveries returns up to limit records for channelID, newest first, paginated by an optional before cutoff.
func (*DB) ListRulesForResource ¶
ListRulesForResource returns every rule scoped to resourceID, enabled or not, ordered by name.
func (*DB) ListSilences ¶ added in v0.2.0
func (db *DB) ListSilences(ctx context.Context, now time.Time, includeExpired bool, limit int) ([]Silence, error)
ListSilences returns silences newest first, capped at limit. When includeExpired is false only pending and active ones at now are returned.
func (*DB) MarkAllInboxRead ¶
MarkAllInboxRead moves the principal's read cursor to now and drops the per-item rows it covers.
func (*DB) MarkDigestSent ¶
MarkDigestSent records that a digest went out at now.
func (*DB) MarkInboxRead ¶
MarkInboxRead marks the given history ids read for principal.
func (*DB) PruneHistory ¶ added in v0.2.0
PruneHistory deletes entries older than cutoff and returns how many went.
func (*DB) RecordHistory ¶ added in v0.2.0
func (db *DB) RecordHistory(ctx context.Context, e HistoryEntry) error
RecordHistory appends one entry. Insert-only.
func (*DB) RecordNotificationDelivery ¶
func (db *DB) RecordNotificationDelivery(ctx context.Context, d NotificationDelivery) error
RecordNotificationDelivery persists one delivery attempt. Insert-only: a delivery record is never updated once written.
func (*DB) ResetDefaultRules ¶
ResetDefaultRules rewrites every default rule to its shipped definition, re-creating deleted ones, and returns them.
func (*DB) ResolveAllowed ¶
func (db *DB) ResolveAllowed(ctx context.Context, ruleID, resourceID string, kind Kind) (bool, error)
ResolveAllowed reports whether a resolved row may follow the rule's last fired or resolved row. A second resolved is never allowed, and a cert expiry resolve also needs a delivered firing before it.
func (*DB) RevealChannelTarget ¶ added in v0.3.0
RevealChannelTarget decrypts target when it is a sealed channel's masked destination, as joined into a deploy target or alert rule.
func (*DB) RevealNotificationChannel ¶ added in v0.3.0
func (db *DB) RevealNotificationChannel(ctx context.Context, c NotificationChannel) (NotificationChannel, error)
RevealNotificationChannel returns c with its sealed credentials decrypted. Call it only on the send or verify path, and never persist or return the result to a client.
func (*DB) RouteDefaultRules ¶
RouteDefaultRules attaches every seeded rule that has no destination yet to an admin email channel, once per rule. Cert expiry rules are skipped: they default to the Inbox and email is opt-in per rule. An empty adminEmail (no email provider or no admin) leaves them in-app only, which the alert history and inbox always record. A rule is routed only once, so an operator who later detaches or changes its channel is never overridden.
func (*DB) SaveDeployTarget ¶
func (db *DB) SaveDeployTarget(ctx context.Context, t DeployTarget) error
SaveDeployTarget creates or fully replaces a deploy target's configuration, the same upsert shape SaveRule already uses.
func (*DB) SaveInboxPrefs ¶
SaveInboxPrefs validates and upserts preferences. LastDigestAt is not touched.
func (*DB) SaveMaintenanceWindow ¶ added in v0.2.0
func (db *DB) SaveMaintenanceWindow(ctx context.Context, w MaintenanceWindow) error
SaveMaintenanceWindow creates or fully replaces a window.
func (*DB) SaveNotificationChannel ¶
func (db *DB) SaveNotificationChannel(ctx context.Context, c NotificationChannel) error
SaveNotificationChannel creates or replaces a channel's configuration; secrets are sealed and a masked value keeps the stored one.
func (*DB) SaveRule ¶
SaveRule creates or replaces a rule's configuration. It never touches evaluation state, which only UpdateState writes.
func (*DB) SealNotificationChannelSecrets ¶ added in v0.3.0
SealNotificationChannelSecrets moves every plaintext channel credential under envelope encryption, verifying each by decrypting it before the plaintext column is cleared. Idempotent and safe to interrupt.
func (*DB) SeedDefaults ¶
func (db *DB) SeedDefaults(ctx context.Context, cfg DefaultsConfig) error
SeedDefaults creates each missing default rule once. A seed key is recorded after its rule exists, so operator edits, disables and deletes are never undone, and a crash between the two steps heals on the next call.
func (*DB) SelectInbox ¶
func (db *DB) SelectInbox(ctx context.Context, principal string, sel InboxSelector) ([]string, error)
SelectInbox returns the ids of the principal's visible items that match sel.
func (*DB) SetCertExpiryEmail ¶
SetCertExpiryEmail opts the default cert expiry rules in to (or out of) email at address. Other rules are untouched.
func (*DB) SetChannelQuietHours ¶
func (db *DB) SetChannelQuietHours(ctx context.Context, q QuietHours) error
SetChannelQuietHours creates or replaces the channel's quiet hours.
func (*DB) SetChannelSecrets ¶ added in v0.3.0
func (db *DB) SetChannelSecrets(cs ChannelSecrets)
SetChannelSecrets seals channel credentials from now on. nil keeps the plaintext fallback for a control plane with no master key.
func (*DB) SetDefaultRuleEnabled ¶
SetDefaultRuleEnabled flips only the enabled flag of a seeded rule, leaving every other operator edit alone.
func (*DB) UpdateState ¶
func (db *DB) UpdateState(ctx context.Context, id string, pendingSince, firingSince *time.Time, firing bool, evaluatedAt time.Time, value *float64) error
UpdateState persists a rule's new evaluation state after one evaluator pass. The only method that writes Pending/Firing/ LastEvaluated/LastValue; SaveRule above never touches them.
func (*DB) UpsertCertExpiryObservation ¶
func (db *DB) UpsertCertExpiryObservation(ctx context.Context, o CertExpiryObservation) error
UpsertCertExpiryObservation creates or replaces the (rule_id, domain) row.
type DatabaseBackupSource ¶
type DatabaseBackupSource interface {
ListBackupHistory(ctx context.Context, databaseName string, limit int, before *time.Time) ([]store.BackupHistory, error)
}
DatabaseBackupSource is the slice of the store the backup-failing rule reads.
type DatabaseHealthSource ¶
type DatabaseHealthSource interface {
DatabaseConnections(ctx context.Context, name string) (used, limit int, err error)
DatabaseDiskUsedPercent(ctx context.Context, name string) (float64, error)
}
DatabaseHealthSource reads live connection and disk figures from a managed database.
type DatabaseLister ¶
type DatabaseLister interface {
ListDesiredDatabases(ctx context.Context) ([]store.DesiredDatabase, error)
}
DatabaseLister lists every managed database; *store.DB satisfies it.
type DatabaseLivenessSource ¶
type DatabaseLivenessSource interface {
DatabaseUp(ctx context.Context, name string) (bool, error)
}
DatabaseLivenessSource is an optional extension of DatabaseHealthSource that lets database_down rules probe reachability.
type DatabasePoolerSource ¶
type DatabasePoolerSource interface {
PoolerSaturation(ctx context.Context, name string) (waiting int, maxWaitSeconds float64, err error)
}
DatabasePoolerSource reads the connection pooler's client queue; ErrDatabaseHealthUnavailable means no pooler is running.
type DefaultRuleSpec ¶
DefaultRuleSpec is one rule the seeder creates once; Key is stable and doubles as the rule ID.
func DefaultRuleSpecs ¶
func DefaultRuleSpecs(cfg DefaultsConfig) []DefaultRuleSpec
DefaultRuleSpecs lists the rules seeded on first boot. Only kinds the engine already evaluates are used.
type DefaultsConfig ¶
type DefaultsConfig struct {
CertExpiryDays []int
// MonitorsEnabled adds the monitor_down default; the monitor engine is off unless APP_MONITORS_ENABLED is true.
MonitorsEnabled bool
}
DefaultsConfig tunes which default rules are seeded.
func DefaultsConfigFromEnv ¶
func DefaultsConfigFromEnv(lookup func(string) string) DefaultsConfig
DefaultsConfigFromEnv reads APP_DEFAULT_ALERT_CERT_DAYS (comma-separated days, e.g. "14,7,3,1"); bad or empty input keeps the defaults.
type DeliveryRecorder ¶
type DeliveryRecorder interface {
RecordNotificationDelivery(ctx context.Context, d NotificationDelivery) error
}
DeliveryRecorder is the narrow surface Engine and DeployDispatcher need to persist a delivery attempt. *DB satisfies it structurally.
type DeployDispatcher ¶
type DeployDispatcher struct {
// contains filtered or unexported fields
}
DeployDispatcher sends deploy-outcome notifications to the enabled targets of one app.
func NewDeployDispatcher ¶
func NewDeployDispatcher(db *DB, client *http.Client, sender email.Sender, pushSender PushSender, logger *slog.Logger) *DeployDispatcher
NewDeployDispatcher builds a DeployDispatcher; nil senders fail sends with a 'not configured' error.
func (*DeployDispatcher) Dispatch ¶
func (d *DeployDispatcher) Dispatch(ctx context.Context, resourceID string, ev DeployOutcome)
Dispatch sends ev to every enabled DeployTarget for its app. Call it only once the attempt is terminal. Failures are logged, never returned: the deploy has already finished and must not look failed.
func (*DeployDispatcher) Outcomes ¶
func (d *DeployDispatcher) Outcomes() DeployFailureSource
Outcomes returns the tracker fed by this dispatcher, or nil when none is attached.
func (*DeployDispatcher) SendApprovalRequest ¶ added in v0.2.0
func (d *DeployDispatcher) SendApprovalRequest(ctx context.Context, c NotificationChannel, info ApprovalRequestInfo) error
SendApprovalRequest posts a deploy-approval request with Approve/Deny buttons to c. Delivery history is recorded under trigger deploy-approval-requested.
func (*DeployDispatcher) SendChannelTest ¶ added in v0.3.0
func (d *DeployDispatcher) SendChannelTest(ctx context.Context, c NotificationChannel) error
SendChannelTest fires a test notification through a saved channel, decrypting its credential only for this send.
func (*DeployDispatcher) SendNotice ¶ added in v0.2.0
func (d *DeployDispatcher) SendNotice(ctx context.Context, c NotificationChannel, text string) error
SendNotice sends a plain-text notice through channel c, using the same per-kind payload logic and SSRF-guarded client as SendTest.
func (*DeployDispatcher) SendTest ¶
func (d *DeployDispatcher) SendTest(ctx context.Context, kind NotifyKind, notifyURL string) error
SendTest reuses the dispatcher's HTTP client and email sender, so a passing test predicts a real send.
func (*DeployDispatcher) WithLogSources ¶ added in v0.2.0
func (d *DeployDispatcher) WithLogSources(runtime LogsSource, build DeployLogSource) *DeployDispatcher
WithLogSources lets the dispatcher attach the tail of the failing build or container output to a failed deploy notification. Either may be nil.
func (*DeployDispatcher) WithOutcomeTracker ¶
func (d *DeployDispatcher) WithOutcomeTracker(t *DeployOutcomeTracker) *DeployDispatcher
WithOutcomeTracker feeds every terminal deploy outcome to t, which the deploy_failed rule reads.
type DeployFailure ¶
DeployFailure is the latest failed deploy of one app that no later success has cleared.
type DeployFailureSource ¶
type DeployFailureSource interface {
RecentDeployFailures(since time.Time) []DeployFailure
}
DeployFailureSource lists unresolved deploy failures newer than since.
type DeployLogSource ¶ added in v0.2.0
type DeployLogSource interface {
QueryDeployLog(ctx context.Context, attemptID string) ([]telemetry.DeployLogEntry, error)
}
DeployLogSource reads a deploy attempt's build output; *telemetry.DB satisfies it.
type DeployOutcome ¶
type DeployOutcome struct {
// AppName is the service name; the caller computes the resource ID used for lookup.
AppName string
// Image is the tag this attempt deployed or tried to deploy
// (store.DeployAttempt.Image).
Image string
// Succeeded is store.DeployAttempt.Status == succeeded. Never called
// for the non-terminal "running" status: see Dispatch's own doc
// comment.
Succeeded bool
// Error is store.DeployAttempt.Error: only meaningful, and only ever
// non-empty, when !Succeeded.
Error string
// AttemptID is the deploy attempt, used to read a failed build's output.
AttemptID string
// LogLines is the tail of the failing build or container output. The
// dispatcher fills it for a failed deploy when a log source is set.
LogLines []string
// ReleaseNotes is optional markdown for a successful production or
// promotion deploy. Dispatch clamps it to the configured byte cap.
ReleaseNotes string
}
DeployOutcome is one finished deploy attempt's terminal result, as sent by Dispatch.
type DeployOutcomeTracker ¶
type DeployOutcomeTracker struct {
// contains filtered or unexported fields
}
DeployOutcomeTracker remembers each app's unresolved deploy failure, fed from DeployDispatcher.Dispatch (in memory, cleared by a success or a restart).
func NewDeployOutcomeTracker ¶
func NewDeployOutcomeTracker() *DeployOutcomeTracker
NewDeployOutcomeTracker returns an empty tracker.
func (*DeployOutcomeTracker) RecentDeployFailures ¶
func (t *DeployOutcomeTracker) RecentDeployFailures(since time.Time) []DeployFailure
RecentDeployFailures implements DeployFailureSource, ordered by app name.
func (*DeployOutcomeTracker) Record ¶
func (t *DeployOutcomeTracker) Record(ev DeployOutcome, at time.Time)
Record stores a failure or clears the app's earlier one on success.
type DeployTarget ¶
type DeployTarget struct {
ID string
ResourceID string
// ChannelID attaches an already-connected NotificationChannel; empty
// for legacy rows, which use NotifyURL/NotifyKind below directly.
ChannelID string
NotifyURL string
NotifyKind NotifyKind
Enabled bool
}
DeployTarget is a deploy-outcome notify destination scoped to one app.
type DigestRunner ¶
DigestRunner sends each opted-in principal one summary of the last day's inbox.
type DigestSender ¶
type DigestSender interface {
SendNotice(ctx context.Context, c NotificationChannel, text string) error
}
DigestSender is the slice of DeployDispatcher the digest needs.
type DomainCheckSource ¶
type DomainCheckSource interface {
CheckDomainStatus(ctx context.Context, domain string) (status string, err error)
}
DomainCheckSource is the narrow DNS check surface EvaluateDomainHealth needs, shared with the dashboard's Check now button.
type DomainListSource ¶ added in v0.2.0
type DomainListSource interface {
ListServiceDomains(ctx context.Context) ([]store.ServiceDomain, error)
}
DomainListSource lists every configured domain. *store.DB satisfies it.
type DomainTrafficSource ¶
type DomainTrafficSource interface {
DomainRequests(ctx context.Context, app, domain string, since, until time.Time) (int, bool)
}
DomainTrafficSource counts requests for one domain of an app in a time range. Implementations read the visitor rollups and live ring, never raw logs.
type EmailDeliverySource ¶
type EmailDeliverySource interface {
EmailDeliveryStats(ctx context.Context, since time.Time) (store.EmailDeliveryStats, error)
}
EmailDeliverySource reads the outbound email delivery log. *store.DB satisfies it.
type Engine ¶
type Engine struct {
// contains filtered or unexported fields
}
Engine evaluates enabled rules each tick and notifies only on firing or resolved transitions, never on repeats.
func NewEngine ¶
func NewEngine(rules RuleStore, metrics MetricsSource, logs LogsSource, tracker *RestartTracker, certs CertSource, scheduledTasks ScheduledTaskSource, certExpiryWarningWindow, certRenewalStalledThreshold time.Duration, nodes NodeSource, patchStatusThreshold, nodeDiskSpaceThreshold float64, nodeServices NodeServiceSource, nodeCPUThreshold, nodeMemoryThreshold float64, domainApps AppDomainSource, domainChecker DomainCheckSource, domainHealthCheckInterval time.Duration, backups BackupSource, backupMissingGracePeriod time.Duration, newNotifier func(Rule) Notifier, logger *slog.Logger) *Engine
NewEngine builds an Engine; a nil optional dependency disables only the rule kinds that need it. Zero thresholds fall back to their Default constants.
func (*Engine) EvaluateCertRenewalStalled ¶ added in v0.2.0
func (e *Engine) EvaluateCertRenewalStalled(ctx context.Context, r Rule, now time.Time) (Rule, []string, error)
EvaluateCertRenewalStalled runs one KindCertRenewalStalled rule. It keeps its own expiry episodes (keyed by the rule ID) through EvaluateCertExpiry.
func (*Engine) EvaluateDomainNotResolving ¶ added in v0.2.0
func (e *Engine) EvaluateDomainNotResolving(ctx context.Context, r Rule, now time.Time) (Rule, []string, error)
EvaluateDomainNotResolving runs one KindDomainNotResolving rule.
func (*Engine) EvaluateDomainTrafficDrop ¶
func (e *Engine) EvaluateDomainTrafficDrop(ctx context.Context, r Rule, now time.Time) (Rule, []string, error)
EvaluateDomainTrafficDrop runs one KindDomainTrafficDrop rule. Missing data never fires.
func (*Engine) EvaluateParkedDomainCertExpiry ¶
func (e *Engine) EvaluateParkedDomainCertExpiry(ctx context.Context, r Rule, now time.Time) (Rule, []string, error)
EvaluateParkedDomainCertExpiry runs one KindParkedDomainCertExpiry rule. A parked domain with no certificate is not an expiry, so it never fires.
func (*Engine) Run ¶
Run calls Tick on interval until ctx is done, matching the shape of every other periodic loop in this codebase (telemetry.Collector.Run, cmd/levelrail's retention sweeps).
func (*Engine) SetAutoRollback ¶ added in v0.2.0
func (e *Engine) SetAutoRollback(st AutoRollbackStore, nudger deploy.ReconcileNudger)
SetAutoRollback enables crashloop auto-rollback; a setter so optional wiring leaves NewEngine's signature alone.
func (*Engine) SetBackupCopy ¶
func (e *Engine) SetBackupCopy(src BackupCopySource, defaultLag time.Duration)
SetBackupCopy enables kind=backup_copy_behind rules; defaultLag applies when a rule sets no ForDuration.
func (*Engine) SetBudgets ¶ added in v0.2.0
func (e *Engine) SetBudgets(src BudgetSource)
SetBudgets enables kind=budget_threshold rules.
func (*Engine) SetCertExpiryPolicy ¶
func (e *Engine) SetCertExpiryPolicy(p CertExpiryPolicy)
SetCertExpiryPolicy turns on the quieter cert_expiry behavior: one window per certificate, platform-managed certificates skipped, and a rate-limited reminder.
func (*Engine) SetChanges ¶ added in v0.2.0
func (e *Engine) SetChanges(src ChangeSource, link func(app string) string)
SetChanges attaches a "recent changes" section to firing app alerts. link, when non-nil, maps an app name to its dashboard URL.
func (*Engine) SetControlPlaneBackups ¶ added in v0.2.0
func (e *Engine) SetControlPlaneBackups(src ControlPlaneBackupSource, maxAge time.Duration)
SetControlPlaneBackups enables kind=control_plane_backup_stale rules. It is left unset when scheduled control plane backups are disabled, so those rules stay quiet. maxAge <= 0 falls back to DefaultControlPlaneBackupMaxAge.
func (*Engine) SetDatabaseHealth ¶
func (e *Engine) SetDatabaseHealth(src DatabaseHealthSource)
SetDatabaseHealth wires the live probe the database_connections and database_disk rules read.
func (*Engine) SetDeployFailures ¶
func (e *Engine) SetDeployFailures(src DeployFailureSource)
SetDeployFailures wires the source the deploy_failed rule reads.
func (*Engine) SetDomainTraffic ¶
func (e *Engine) SetDomainTraffic(src DomainTrafficSource)
SetDomainTraffic enables domain_traffic_drop; nil leaves the kind skipped.
func (*Engine) SetEmailDelivery ¶
func (e *Engine) SetEmailDelivery(src EmailDeliverySource)
SetEmailDelivery enables kind=email_delivery_failing rules.
func (*Engine) SetExportHealth ¶
func (e *Engine) SetExportHealth(src ExportHealthSource)
SetExportHealth enables kind=export_behind rules.
func (*Engine) SetLogArchive ¶ added in v0.2.0
func (e *Engine) SetLogArchive(src LogArchiveSource)
SetLogArchive enables kind=log_archive_stale rules.
func (*Engine) SetMonitorDowns ¶
func (e *Engine) SetMonitorDowns(src MonitorDownSource)
SetMonitorDowns wires the source the monitor_down rule reads; unset keeps the rule inert.
func (*Engine) SetNodeCertThresholds ¶ added in v0.2.0
SetNodeCertThresholds sets the default warning window and critical threshold for kind=node_cert_expiring rules. Zero keeps the defaults.
func (*Engine) SetNoiseControl ¶ added in v0.2.0
func (e *Engine) SetNoiseControl(n *NoiseControl)
SetNoiseControl enables silences, grouping, flapping control and alert history. Unset, every transition notifies directly.
func (*Engine) SetRegistryRefresh ¶
func (e *Engine) SetRegistryRefresh(src RegistryRefreshSource)
SetRegistryRefresh enables kind=registry_refresh_failing rules.
func (*Engine) SetRestoreDrills ¶ added in v0.2.0
func (e *Engine) SetRestoreDrills(src RestoreDrillSource)
SetRestoreDrills enables kind=restore_drill_failed rules.
func (*Engine) SetSLOBurnAutoRollback ¶ added in v0.2.0
func (e *Engine) SetSLOBurnAutoRollback(st SLOAutoRollbackStore, nudger deploy.ReconcileNudger, history SLOBurnHistoryRecorder)
SetSLOBurnAutoRollback enables SLO-burn auto-rollback; history must be a separate *DB from st.
func (*Engine) SetTrafficSources ¶ added in v0.2.0
func (e *Engine) SetTrafficSources(domains DomainListSource, acme ACMEFailureSource)
SetTrafficSources enables domain_not_resolving and the ACME half of cert_renewal_stalled. Either may be nil.
func (*Engine) SetVersionSkew ¶ added in v0.2.0
func (e *Engine) SetVersionSkew(settings VersionSkewSettingsSource, fetchers upgrade.Fetchers)
SetVersionSkew enables kind=version_skew rules using the shared GitHub update lookup.
type Event ¶
type Event struct {
Rule Rule
// Resolved events are sent too, so operators do not learn to mute a channel that only reports problems starting.
Resolved bool
// LogLines is populated only for a firing (not resolved) crashloop
// event: the last up-to-200 lines of the failing container's logs.
// Nil for threshold rules and for resolved events.
LogLines []string
// CertNotices is set only on a firing cert_expiry event.
CertNotices []string
// CertFacts and CertBaseURL drive the actionable cert expiry email; the base URL is empty when no public address is configured.
CertFacts []CertFact
CertBaseURL string
// PatchNotices is set only on a firing patch_status event.
PatchNotices []string
// DiskSpaceNotices is set only on a firing node_disk_space event.
DiskSpaceNotices []string
// NodeOfflineNotices is populated only for a firing node_offline
// event: one line per offline node, from EvaluateNodeOffline.
NodeOfflineNotices []string
// NodeCertNotices is populated only for a firing node_cert_expiring
// event: one line per node whose agent certificate needs attention.
NodeCertNotices []string
// ResourceUsageNotices is set only on a firing node_resource_usage event.
ResourceUsageNotices []string
// TaskFailureNotice is set only on a firing scheduled_task_failure event.
TaskFailureNotice string
// DomainHealthNotices is set only on a firing domain_health event.
DomainHealthNotices []string
// BackupMissingNotice is set only on a firing backup_missing event.
BackupMissingNotice string
// VersionSkewNotice is set only on a firing version_skew event.
VersionSkewNotice string
// BudgetNotices is populated only for a firing budget_threshold event:
// one line per budget that crossed a threshold.
BudgetNotices []string
// SLONotice is set only for a firing slo_burn event: which burn-rate tier
// tripped and how much error budget is left.
SLONotice string
// LogNotice is set for a firing log_pattern or log_level_rate event: the
// count or rate that tripped. LogLines then holds the matching samples.
LogNotice string
// Changes is what changed on the app shortly before it fired; set for
// firing app-scoped events only. ChangesLink points at its dashboard page.
Changes *changes.Result
ChangesLink string
// Headline replaces the default first line for flapping and grouped
// messages; GroupNotices lists the alerts folded into a grouped one.
Headline string
GroupNotices []string
GroupCount int
}
Event is what a firing (or resolved) rule hands to a Notifier: enough context to write a useful message without the notifier needing to go query anything itself.
type EventSource ¶
EventSource is the narrow Docker surface RestartTracker needs. *docker.Client satisfies this structurally (it already implements Runtime, which embeds Events).
type ExportHealth ¶
type ExportHealth struct {
Name string
Enabled bool
CreatedAt time.Time
LastSuccessAt time.Time
LastError string
CircuitOpen bool
}
ExportHealth is one export destination's delivery state, as the engine sees it.
type ExportHealthSource ¶
type ExportHealthSource interface {
ExportHealth(ctx context.Context) ([]ExportHealth, error)
}
ExportHealthSource lists every export destination's delivery state.
type HistoryEntry ¶ added in v0.2.0
type HistoryEntry struct {
ID string
At time.Time
RuleID string
RuleName string
RuleKind string
ResourceID string
App string
Node string
Severity string
Event string
Outcome string
Detail string
SilenceID string
ChannelID string
Error string
// Cert is set on cert_expiry entries; older rows have none.
Cert *CertPayload
}
HistoryEntry is one alert state change and what became of its notification.
type HistoryFilter ¶ added in v0.2.0
type HistoryFilter struct {
App string
RuleID string
Outcome string
Event string
Since time.Time
Before *time.Time
Limit int
}
HistoryFilter narrows ListHistory. Zero fields match everything.
type InboxFilter ¶
InboxFilter narrows ListInbox.
type InboxItem ¶
type InboxItem struct {
HistoryEntry
Read bool
Resolved bool
}
InboxItem is one history entry as seen by a principal.
type InboxPrefs ¶
type InboxPrefs struct {
Principal string
MinSeverity string
MutedApps []string
DigestEnabled bool
DigestHour int
DigestChannelID string
LastDigestAt time.Time
}
InboxPrefs are one principal's routing rules and digest settings.
func DefaultInboxPrefs ¶
func DefaultInboxPrefs(principal string) InboxPrefs
DefaultInboxPrefs is what a principal with no saved row gets.
func (InboxPrefs) Validate ¶
func (p InboxPrefs) Validate() error
Validate rejects unknown severities, hours outside 0-23 and an enabled digest with no channel.
func (InboxPrefs) Visible ¶
func (p InboxPrefs) Visible(e HistoryEntry) bool
Visible reports whether routing rules let e into the principal's inbox.
type InboxSelector ¶
InboxSelector picks a principal's items for a bulk action.
type InboxState ¶
InboxState is the explicit read and archived ids a principal holds, for items the server does not list itself.
type Kind ¶
type Kind string
Kind distinguishes what a Rule evaluates.
const ( KindDatabaseConnections Kind = "database_connections_high" KindDatabaseDisk Kind = "database_disk_high" KindDatabaseBackupFailing Kind = "database_backup_failing" KindDatabaseDown Kind = "database_down" KindDatabaseBackupStale Kind = "database_backup_stale" KindPoolerSaturated Kind = "pooler_saturated" )
Database rule kinds. They target BackupDatabaseName and reuse Threshold (percent) and ForDuration, so they need no schema change.
const ( // KindDomainTrafficDrop fires when a domain's requests in the last window fall below // Threshold percent of the same window a day earlier. ResourceID "service:<app>" watches // that app's domains, "domain:<name>" one domain. ForDuration is the window, // RestartCountThreshold the minimum baseline request count. KindDomainTrafficDrop Kind = "domain_traffic_drop" // KindParkedDomainCertExpiry fires for parked domains whose certificate has fewer than // Threshold days left. Platform-wide. KindParkedDomainCertExpiry Kind = "parked_domain_cert_expiry" )
Domain lifecycle rule kinds. Both are explicit opt-in: nothing creates them automatically.
const ( // KindLogPattern fires when lines matching a text, regex or field filter // reach Count within Window. KindLogPattern Kind = "log_pattern" // KindLogLevelRate fires when lines at or above Level arrive faster than // PerMinute over Window. KindLogLevelRate Kind = "log_level_rate" )
Log rule kinds. Both are app-scoped and evaluated from the node-local log store on a bounded cadence, never over an unbounded range.
const ( KindThreshold Kind = "threshold" KindCrashloop Kind = "crashloop" KindCertExpiry Kind = "cert_expiry" KindPatchStatus Kind = "patch_status" KindScheduledTaskFailure Kind = "scheduled_task_failure" // KindScheduledTaskMissed and KindScheduledTaskRunningLong watch one // job's schedule health: skipped occurrences and a run in flight past // its warning threshold (scheduled_task_health.go). KindScheduledTaskMissed Kind = "scheduled_task_missed" KindScheduledTaskRunningLong Kind = "scheduled_task_running_long" KindNodeDiskSpace Kind = "node_disk_space" // KindNodeHostGuardrail is registered for rules that watch per-node disk, inode, memory and swap guardrails. KindNodeHostGuardrail Kind = "node_host_guardrail" KindNodeResourceUsage Kind = "node_resource_usage" KindDomainHealth Kind = "domain_health" KindBackupMissing Kind = "backup_missing" // KindControlPlaneBackupStale is platform-wide: it watches the newest control plane snapshot's age. KindControlPlaneBackupStale Kind = "control_plane_backup_stale" KindNodeOffline Kind = "node_offline" // KindNodeCertExpiring is platform-wide: it fires while any node's agent // certificate is close to expiry or expired (node_cert_expiring.go). KindNodeCertExpiring Kind = "node_cert_expiring" // KindLogArchiveStale is platform-wide: it fires when a log archive policy fails or stops succeeding. KindLogArchiveStale Kind = "log_archive_stale" // KindExportBehind is platform-wide: it fires when an export destination (audit or log sink) keeps failing. KindExportBehind Kind = "export_behind" // KindBackupCopyBehind is platform-wide: it fires while a backup has not reached the second destination within its lag limit. KindBackupCopyBehind Kind = "backup_copy_behind" // KindRegistryRefreshFailing is platform-wide: it fires while a cloud registry credential cannot refresh. KindRegistryRefreshFailing Kind = "registry_refresh_failing" // KindRestoreDrillFailed is platform-wide: it fires while a resource's latest restore drill is failed. KindRestoreDrillFailed Kind = "restore_drill_failed" // KindVersionSkew is platform-wide: it fires while the running build // is behind the configured update channel's latest release // (update_settings.channel, migrations/0258_update_settings.sql). KindVersionSkew Kind = "version_skew" // KindBudgetThreshold is platform-wide: it fires while any project or // team budget is at its warn level, over its limit, or forecast to // overrun (budget_threshold.go). KindBudgetThreshold Kind = "budget_threshold" // KindEmailDeliveryFailing is platform-wide: it fires while most recent // outbound emails are failing (email_delivery_failing.go). KindEmailDeliveryFailing Kind = "email_delivery_failing" )
Kinds other than threshold and crashloop ignore the threshold fields and watch platform-wide or backup-schedule resources. For most of them ResourceID is only a display label.
const ( // KindCertExpiring fires while a certificate expires within Threshold // days (default DefaultCertExpiringDays). ResourceID "domain:<name>" // narrows it to one hostname; empty watches every certificate. KindCertExpiring Kind = "cert_expiring" // KindCertRenewalStalled fires when a renewal has not moved NotAfter for // the stalled threshold, or an ACME failure persists that long. KindCertRenewalStalled Kind = "cert_renewal_stalled" // KindDomainNotResolving fires when a domain has not resolved for // ForDuration (default DefaultDomainNotResolvingWindow). ResourceID // "service:<app>" narrows it to one app's domains. KindDomainNotResolving Kind = "domain_not_resolving" )
Domain and certificate rule kinds used by the traffic area. The spec's dns_drift, zone_not_delegated and proxy_route_error kinds are documented in docs/observability.md and not evaluated yet.
const KindBackupFailed Kind = "backup_failed"
KindBackupFailed is platform-wide: it fires while the newest backup attempt of any scheduled database failed.
const KindDeployFailed Kind = "deploy_failed"
KindDeployFailed is platform-wide: it fires while any app has an unresolved failed deploy (fed by DeployDispatcher).
const KindMonitorDown Kind = "monitor_down"
KindMonitorDown is platform-wide: it fires while any uptime monitor holds a confirmed down state (fed by the monitor notifier).
const KindSLOBurn Kind = "slo_burn"
KindSLOBurn fires on error-budget burn rate for a request-based SLO over the app's ingress request metrics, using the multiwindow multi-burn-rate method.
type LogArchiveHealth ¶ added in v0.2.0
type LogArchiveHealth struct {
// Scope is the app name, or empty for the global policy.
Scope string
Enabled bool
Interval time.Duration
CreatedAt time.Time
LastSuccessAt time.Time
LastError string
}
LogArchiveHealth is one log archive policy's freshness, as the engine sees it.
type LogArchiveSource ¶ added in v0.2.0
type LogArchiveSource interface {
LogArchiveHealth(ctx context.Context) ([]LogArchiveHealth, error)
}
LogArchiveSource lists every log archive policy's health.
type LogRuleConfig ¶ added in v0.2.0
type LogRuleConfig struct {
// Query is a full-text phrase matched by the store's FTS index.
Query string `json:"query,omitempty"`
// Regex further narrows the lines by their message (RE2 syntax).
Regex string `json:"regex,omitempty"`
// Fields are key=value, key!=value, key~text or key>number filters.
Fields []string `json:"fields,omitempty"`
// Level keeps lines at or above it. log_level_rate defaults to error.
Level string `json:"level,omitempty"`
// Count is the match count that fires a log_pattern rule.
Count int `json:"count,omitempty"`
// PerMinute is the lines per minute that fires a log_level_rate rule.
PerMinute float64 `json:"per_minute,omitempty"`
// WindowSeconds is the look-back; zero means five minutes.
WindowSeconds int `json:"window_seconds,omitempty"`
// CooldownSeconds suppresses a new firing for this long after the last one.
CooldownSeconds int `json:"cooldown_seconds,omitempty"`
}
LogRuleConfig is what a log_pattern or log_level_rate rule watches.
func (LogRuleConfig) Validate ¶ added in v0.2.0
func (c LogRuleConfig) Validate(kind Kind, p LogRulePolicy) error
Validate rejects a config that cannot be evaluated within policy.
func (LogRuleConfig) Window ¶ added in v0.2.0
func (c LogRuleConfig) Window() time.Duration
Window returns the look-back with its default applied.
type LogRulePolicy ¶ added in v0.2.0
type LogRulePolicy struct {
EvalEvery time.Duration
MaxWindow time.Duration
MaxRows int
Samples int
}
LogRulePolicy bounds one evaluation: how often a rule runs, how far back it may look and how many rows it may read.
func LogRulePolicyFromEnv ¶ added in v0.2.0
func LogRulePolicyFromEnv() LogRulePolicy
LogRulePolicyFromEnv applies APP_LOG_RULE_* overrides to the defaults. A malformed or non-positive value keeps the default.
type LogRuleResult ¶ added in v0.2.0
type LogRuleResult struct {
Matched int
Value float64
Truncated bool
Samples []string
Notice string
Satisfied bool
}
LogRuleResult is one evaluation's outcome.
func MatchLogRule ¶ added in v0.2.0
func MatchLogRule(kind Kind, cfg LogRuleConfig, entries []telemetry.LogEntry, truncated bool, now time.Time, policy LogRulePolicy) (LogRuleResult, error)
MatchLogRule applies cfg to entries (oldest first, already limited to the window) and returns the result. It is pure so rules test with seeded lines.
type LogRuleSource ¶ added in v0.2.0
type LogRuleSource interface {
QueryLogs(ctx context.Context, resourceID string, from, to time.Time, query string) ([]telemetry.LogEntry, error)
}
LogRuleSource reads bounded log windows; *telemetry.Federator satisfies it.
type LogsSource ¶
type LogsSource interface {
QueryLogs(ctx context.Context, resourceID string, from, to time.Time, query string) ([]telemetry.LogEntry, error)
}
LogsSource is the narrow surface Engine needs to attach log lines to a firing crashloop event. *telemetry.Federator satisfies this structurally, the same as MetricsSource in evaluate.go.
type MailRenderFunc ¶
type MailRenderFunc func(ctx context.Context, key, to string, values map[string]string) (email.Message, bool)
MailRenderFunc renders one mailtemplates event for a recipient; ok is false when no template applies.
type MaintenanceWindow ¶ added in v0.2.0
type MaintenanceWindow struct {
ID string
Name string
Cron string
Duration time.Duration
Timezone string
Scope string
Targets []string
Enabled bool
CreatedBy string
CreatedAt time.Time
UpdatedAt time.Time
}
MaintenanceWindow is a recurring silence: it starts at each cron match (evaluated in Timezone) and lasts Duration.
func (MaintenanceWindow) Covers ¶ added in v0.2.0
func (w MaintenanceWindow) Covers(c AlertContext) bool
Covers reports whether the window's scope includes the alert c.
func (MaintenanceWindow) StateAt ¶ added in v0.2.0
func (w MaintenanceWindow) StateAt(now time.Time) (WindowState, error)
StateAt evaluates the window in its timezone, and durations use the wall clock so a DST-spanning window keeps its nominal length.
func (MaintenanceWindow) Validate ¶ added in v0.2.0
func (w MaintenanceWindow) Validate() error
Validate checks every field of the window.
type MetricClass ¶
type MetricClass string
MetricClass says whether a threshold metric is a request-path signal or a real resource or runtime one.
const ( // MetricClassRequestPath signals are meaningless while the ingress answers 503 itself. MetricClassRequestPath MetricClass = "request_path" // MetricClassResource signals stay real during maintenance: the container keeps running. MetricClassResource MetricClass = "resource" )
func ClassifyMetric ¶
func ClassifyMetric(metric string) MetricClass
ClassifyMetric returns the class of metric; unknown metrics are resource signals.
type MetricsSource ¶
type MetricsSource interface {
QueryMetrics(ctx context.Context, resourceID, metric string, from, to time.Time) ([]telemetry.Sample, error)
}
MetricsSource is the narrow surface a threshold evaluation needs. *telemetry.Federator satisfies this structurally; a hand-written fake in tests avoids needing a real store for pure evaluation-logic tests.
type MonitorDown ¶
MonitorDown is one monitor the state machine confirmed down and has not yet confirmed recovered.
type MonitorDownSource ¶
type MonitorDownSource interface {
MonitorsDown() []MonitorDown
}
MonitorDownSource lists monitors currently confirmed down.
type MonitorDownTracker ¶
type MonitorDownTracker struct {
// contains filtered or unexported fields
}
MonitorDownTracker holds confirmed-down monitors in memory, so a restart starts clear and the monitor engine re-confirms real outages on its own.
func NewMonitorDownTracker ¶
func NewMonitorDownTracker() *MonitorDownTracker
NewMonitorDownTracker returns an empty tracker.
func (*MonitorDownTracker) Forget ¶
func (t *MonitorDownTracker) Forget(id string)
Forget drops a deleted monitor so it cannot keep the rule firing.
func (*MonitorDownTracker) MonitorsDown ¶
func (t *MonitorDownTracker) MonitorsDown() []MonitorDown
MonitorsDown implements MonitorDownSource, ordered by name.
type MuteHit ¶ added in v0.2.0
MuteHit names the silence or maintenance window muting an alert.
func FindMute ¶ added in v0.2.0
func FindMute(silences []Silence, windows []MaintenanceWindow, c AlertContext, now time.Time) (MuteHit, bool)
FindMute reports whether any active silence or enabled maintenance window in effect at now mutes the alert c.
type NodeAlertState ¶
type NodeAlertState string
NodeAlertState is one node-scoped alert kind's live status for one specific node, computed on demand by CheckNodeAlertStatus rather than read from a rule's stored, all-nodes-aggregated LastValue.
const ( NodeAlertOK NodeAlertState = "ok" NodeAlertFiring NodeAlertState = "firing" NodeAlertUnknown NodeAlertState = "unknown" )
NodeAlertUnknown is kept distinct from OK so a node with no recent sample is never reported healthy.
type NodeAlertStatus ¶
type NodeAlertStatus struct {
PatchStatus NodeAlertState
NodeDiskSpace NodeAlertState
NodeResourceUsage NodeAlertState
}
NodeAlertStatus is one node's live status across the three node-scoped, platform-wide alert kinds.
func CheckNodeAlertStatus ¶
func CheckNodeAlertStatus(ctx context.Context, node store.Node, services NodeServiceSource, metrics MetricsSource, patchThreshold, diskThresholdPercent, cpuThresholdPercent, memoryThresholdBytes float64, now time.Time, logger *slog.Logger) NodeAlertStatus
CheckNodeAlertStatus calls each evaluator's per-node check directly, so it cannot drift from what a real rule decides.
type NodeCertState ¶ added in v0.2.0
type NodeCertState string
NodeCertState summarizes a node's agent certificate.
const ( NodeCertOK NodeCertState = "ok" NodeCertExpiring NodeCertState = "expiring" NodeCertCritical NodeCertState = "critical" NodeCertExpired NodeCertState = "expired" NodeCertRevoked NodeCertState = "revoked" NodeCertUnknown NodeCertState = "unknown" )
Node certificate states, from healthy to unusable.
type NodeServiceSource ¶
type NodeServiceSource interface {
ListDesiredServicesByNode(ctx context.Context, nodeID string) ([]store.DesiredService, error)
}
NodeServiceSource lists the services placed on a node; *store.DB satisfies it structurally.
type NodeSource ¶
NodeSource is the narrow node-listing surface EvaluatePatchStatus needs. *store.DB satisfies this structurally, the same as CertSource above.
type NoiseConfig ¶ added in v0.2.0
type NoiseConfig struct {
ConsecutiveFailures int
FlapThreshold int
FlapWindow time.Duration
GroupWindow time.Duration
RateLimit int
RateWindow time.Duration
HistoryRetention time.Duration
// HistoryRetentionFn, when set, supplies the live retention (0 keeps everything) instead of HistoryRetention.
HistoryRetentionFn func(ctx context.Context) time.Duration
}
NoiseConfig holds the control plane defaults for alert noise control. A zero value disables the corresponding feature; per-rule fields on Rule override ConsecutiveFailures and the flap settings.
type NoiseControl ¶ added in v0.2.0
type NoiseControl struct {
// contains filtered or unexported fields
}
NoiseControl sits between evaluation and notification and applies debounce, flapping, silences, and rate limits. Its state is in memory, so a restart forgets pending groups and streaks.
func NewNoiseControl ¶ added in v0.2.0
func NewNoiseControl(cfg NoiseConfig, st NoiseStore, placement PlacementSource, logger *slog.Logger) *NoiseControl
NewNoiseControl builds a NoiseControl. placement may be nil, which disables node-based matching and inhibition.
func (*NoiseControl) ApplyStreak ¶ added in v0.2.0
func (n *NoiseControl) ApplyStreak(prev, next Rule, now time.Time) Rule
ApplyStreak enforces the consecutive-failure threshold on one evaluated rule before its state is persisted.
func (*NoiseControl) FlushAll ¶ added in v0.2.0
FlushAll sends every buffered group immediately, for shutdown.
func (*NoiseControl) Recheck ¶ added in v0.2.0
Recheck releases a held firing notification once its silence, window or node outage is over while the rule is still firing.
func (*NoiseControl) RecordSkipped ¶ added in v0.2.0
RecordSkipped notes an event dropped because its rule or channel is disabled.
func (*NoiseControl) Route ¶ added in v0.2.0
Route decides what happens to one firing or resolved event.
func (*NoiseControl) SetFiredHook ¶
func (n *NoiseControl) SetFiredHook(fn func(HistoryEntry))
SetFiredHook registers fn to be called with each recorded firing.
type NoiseStore ¶ added in v0.2.0
type NoiseStore interface {
ListActiveSilences(ctx context.Context, now time.Time) ([]Silence, error)
ListMaintenanceWindows(ctx context.Context) ([]MaintenanceWindow, error)
RecordHistory(ctx context.Context, e HistoryEntry) error
PruneHistory(ctx context.Context, cutoff time.Time) (int64, error)
}
NoiseStore is what NoiseControl reads and writes. *DB satisfies it.
type NotificationChannel ¶
type NotificationChannel struct {
ID string
Name string
Kind NotifyKind
NotifyURL string
Enabled bool
// Off by default: it needs InteractiveSecret and a real Slack app or Discord application, not just a webhook URL.
InteractiveApprovals bool
// Verifies inbound button clicks came from this channel's app: a Slack signing secret or a Discord Ed25519 public key.
InteractiveSecret string
// NotifyDeviceLogin opts this channel into a link-only notice when a CLI
// device login awaits approval. The notice never carries the code.
NotifyDeviceLogin bool
// NotifyDeviceLoginExpired opts this channel into a link-only notice when
// a pending CLI login lapses unapproved.
NotifyDeviceLoginExpired bool
CreatedAt string
UpdatedAt string
}
NotificationChannel is a global, connect-once notify destination (migrations/0003_notification_channels.sql's own doc comment), reused across apps' DeployTarget rows by ID instead of a fresh URL per app.
type NotificationDelivery ¶
type NotificationDelivery struct {
ID string
ChannelID string
Trigger string
Success bool
Error string
CreatedAt string
}
NotificationDelivery is one recorded send attempt through a NotificationChannel, real or test.
type Notifier ¶
Notifier sends one Event somewhere. Every notify* function below satisfies this via notifyFunc, so the dispatch table in Dispatch stays a plain map, no interface boilerplate per channel.
func NewNotifier ¶
NewNotifier picks the Notifier for r.NotifyKind, falling back to generic so a mistyped kind still notifies someone.
func NewNotifierWithPlugins ¶
func NewNotifierWithPlugins(client *http.Client, sender email.Sender, pushSender PushSender, plugins PluginSender, r Rule) Notifier
NewNotifierWithPlugins is NewNotifier plus plugin channel support. plugins may be nil.
type NotifyKind ¶
type NotifyKind string
NotifyKind selects the notification payload shape (notify.go).
const ( NotifyGeneric NotifyKind = "generic" NotifySlack NotifyKind = "slack" NotifyDiscord NotifyKind = "discord" NotifyTelegram NotifyKind = "telegram" NotifyEmail NotifyKind = "email" NotifyPushover NotifyKind = "pushover" NotifyPagerDuty NotifyKind = "pagerduty" NotifyTeams NotifyKind = "teams" NotifyResend NotifyKind = "resend" NotifyNtfy NotifyKind = "ntfy" NotifyGotify NotifyKind = "gotify" NotifyMattermost NotifyKind = "mattermost" NotifyLark NotifyKind = "lark" NotifyRocketChat NotifyKind = "rocketchat" NotifyOpsgenie NotifyKind = "opsgenie" NotifyWebex NotifyKind = "webex" NotifyGoogleChat NotifyKind = "googlechat" NotifyWebpush NotifyKind = "webpush" )
NotifyKind payload shapes. Kinds whose NotifyURL is not a webhook URL (email, PagerDuty, Resend, Opsgenie, Webpush) are documented in notify.go.
type ParkedDomainSource ¶
type ParkedDomainSource interface {
ListParkedDomains(ctx context.Context) ([]store.ParkedDomain, error)
}
ParkedDomainSource lists parked domains. *store.DB satisfies it.
type PlacementSource ¶ added in v0.2.0
type PlacementSource interface {
GetDesiredService(ctx context.Context, name string) (*store.DesiredService, error)
ListNodes(ctx context.Context) ([]store.Node, error)
}
PlacementSource resolves which node an app runs on and whether it is offline. *store.DB satisfies it.
type PluginNotice ¶
type PluginNotice struct {
ID string
Title string
Body string
Status string
Severity string
ResourceID string
At time.Time
// LogLines are log excerpts the plugin runtime releases only to plugins
// approved for read:sensitive.
LogLines []string
}
PluginNotice is the event handed to a plugin: only what it needs to deliver one message.
type PluginSender ¶
type PluginSender interface {
SendPlugin(ctx context.Context, plugin, secret string, n PluginNotice) error
}
PluginSender delivers a notice through an installed plugin. secret is the channel's stored credential, passed for this call only.
type PushSender ¶ added in v0.2.0
PushSender sends one browser push to every registered subscription; *webpush.Sender satisfies it.
type QuietHours ¶
QuietHours holds back non-critical firing notifications on one channel between Start and End (HH:MM, evaluated in Timezone). A window that ends before it starts spans midnight.
func (QuietHours) ActiveAt ¶
func (q QuietHours) ActiveAt(t time.Time) bool
ActiveAt reports whether t falls inside the quiet window.
func (QuietHours) Validate ¶
func (q QuietHours) Validate() error
Validate rejects unparsable times, an empty window and unknown zones.
type RegistryRefreshHealth ¶
type RegistryRefreshHealth struct {
Name string
Preset string
LastError string
ExpiresAt *time.Time
}
RegistryRefreshHealth is one cloud registry credential's refresh state.
type RegistryRefreshSource ¶
type RegistryRefreshSource interface {
RegistryRefreshHealth(ctx context.Context) ([]RegistryRefreshHealth, error)
}
RegistryRefreshSource lists preset registry credentials and their refresh state.
type RequestCounts ¶ added in v0.2.0
RequestCounts is the good/bad split of an app's requests over a span.
func CountRequests ¶ added in v0.2.0
func CountRequests(ctx context.Context, src MetricsSource, app string, from, to time.Time, cfg SLOConfig) (RequestCounts, error)
CountRequests sums an app's request counters over [from, to], treating reset artefacts (negative or non-finite) as zero.
type RestartRecorder ¶ added in v0.2.0
type RestartRecorder interface {
RecordContainerRestart(ctx context.Context, serviceName string, at time.Time) error
}
RestartRecorder is the narrow store surface RestartTracker needs, defined here so internal/telemetry never imports alerting.
type RestartTracker ¶
type RestartTracker struct {
// contains filtered or unexported fields
}
RestartTracker counts each container start after a container's first as a restart, since a crashloop is repeated restarts.
func NewRestartTracker ¶
func NewRestartTracker() *RestartTracker
NewRestartTracker returns a tracker that keeps only in-memory counts until WithRecorder is called.
func (*RestartTracker) CountContainersSince ¶
func (t *RestartTracker) CountContainersSince(names []string, since time.Time) int
CountContainersSince is CountSince for specific containers, so a release's watch is not charged for restarts of another release of the same service.
func (*RestartTracker) CountSince ¶
func (t *RestartTracker) CountSince(resourceID string, since time.Time) int
CountSince returns how many restarts resourceID has had strictly after since. Safe for concurrent use.
func (*RestartTracker) Observe ¶
func (t *RestartTracker) Observe(resourceID, containerName string, at time.Time)
Observe records one start event for containerName, owned by resourceID, and is safe for concurrent use.
func (*RestartTracker) Prune ¶
func (t *RestartTracker) Prune(olderThan time.Time)
Prune drops restart records older than olderThan, so this map doesn't grow without bound over a long-running process. Call periodically (Run does this once per resync), not per-evaluation.
func (*RestartTracker) Run ¶
func (t *RestartTracker) Run(ctx context.Context, source EventSource, services ServiceLister, resync time.Duration, logger *slog.Logger) error
Run consumes source's events until ctx is done, re-resolving services on every resync so new services work without a restart.
func (*RestartTracker) WithRecorder ¶ added in v0.2.0
func (t *RestartTracker) WithRecorder(recorder RestartRecorder, logger *slog.Logger) *RestartTracker
WithRecorder persists each counted restart, and a failed write is logged without stopping Observe.
type RestoreDrillProblem ¶ added in v0.2.0
RestoreDrillProblem is one resource whose latest restore drill failed.
type RestoreDrillSource ¶ added in v0.2.0
type RestoreDrillSource interface {
RestoreDrillProblems(ctx context.Context) ([]RestoreDrillProblem, error)
}
RestoreDrillSource lists resources whose most recent restore drill failed.
type Rule ¶
type Rule struct {
ID string
Name string
Kind Kind
ResourceID string
// Threshold-kind fields.
Metric string
Comparator Comparator
Threshold float64
ForDuration time.Duration
// RestartCountThreshold also means consecutive failed runs for KindScheduledTaskFailure, where RestartWindow is unused.
RestartCountThreshold int
RestartWindow time.Duration
// ScheduledTaskID is the KindScheduledTaskFailure-only field: which
// of ResourceID's app's scheduled tasks this rule watches. Empty for
// every other kind.
ScheduledTaskID string
// Backup* fields pick the schedule a KindBackupMissing rule watches. ForDuration is the grace period past the expected interval.
BackupResourceKind string
BackupDatabaseName string
BackupServiceName string
BackupVolumeName string
// ChannelID attaches an already-connected NotificationChannel; empty
// for legacy rules, which use NotifyURL/NotifyKind below directly.
// SLO is the KindSLOBurn-only request-based SLO; nil for every other kind.
SLO *SLOConfig
// LogRule is the KindLogPattern/KindLogLevelRate-only match config; nil for every other kind.
LogRule *LogRuleConfig
ChannelID string
NotifyURL string
NotifyKind NotifyKind
// Enabled reflects both this rule's own flag and, once resolved by a
// read (GetRule/ListRules*), its attached channel's: a disabled
// channel silences the rule too.
Enabled bool
// Noise control: Severity and Labels feed silence matchers;
// ConsecutiveFailures, FlapThreshold and FlapWindow override the
// control plane defaults when non-zero.
Severity string
Labels map[string]string
ConsecutiveFailures int
FlapThreshold int
FlapWindow time.Duration
// Evaluation state, read-only from a caller's perspective: only the
// evaluator (evaluate.go, crashloop.go) writes these, via
// UpdateState below.
PendingSince *time.Time
Firing bool
FiringSince *time.Time
LastEvaluatedAt *time.Time
LastValue *float64
}
Rule is one alert rule, either a threshold check or a crashloop check, plus its current evaluation state.
func EvaluateBackupCopyBehind ¶
func EvaluateBackupCopyBehind(ctx context.Context, source BackupCopySource, defaultLag time.Duration, r Rule, now time.Time) (Rule, string, error)
EvaluateBackupCopyBehind runs one KindBackupCopyBehind rule. A backup is behind when its copy is still missing past the rule's ForDuration, else defaultLag.
func EvaluateBackupMissing ¶ added in v0.2.0
func EvaluateBackupMissing(ctx context.Context, source BackupSource, r Rule, defaultGracePeriod time.Duration, now time.Time) (Rule, string, error)
EvaluateBackupMissing runs one KindBackupMissing rule and returns its updated state and, when firing, a notice line. Missing is measured against the same cron schedule internal/backup.Scheduler reads.
func EvaluateBudgetThreshold ¶ added in v0.2.0
func EvaluateBudgetThreshold(ctx context.Context, src BudgetSource, r Rule, now time.Time) (Rule, []string, []string, error)
EvaluateBudgetThreshold fires while any budget is over its warn level, its limit, or a forecast overrun. It returns one notice per breach and the breach keys used to decide when to notify again.
func EvaluateCertExpiring ¶ added in v0.2.0
func EvaluateCertExpiring(ctx context.Context, certs CertSource, r Rule, now time.Time, logger *slog.Logger) (Rule, []string, error)
EvaluateCertExpiring runs one KindCertExpiring rule.
func EvaluateCertExpiry ¶
func EvaluateCertExpiry(ctx context.Context, certs CertSource, snapshots CertExpiryObservationStore, r Rule, warningWindow, stalledThreshold time.Duration, now time.Time, logger *slog.Logger) (Rule, []string, error)
EvaluateCertExpiry returns one notice per non-healthy certificate, with no debounce since expiry moves only with real time. A renewal stuck on the same NotAfter past stalledThreshold is reported as stalled, not a plain warning.
func EvaluateControlPlaneBackupStale ¶ added in v0.2.0
func EvaluateControlPlaneBackupStale(source ControlPlaneBackupSource, r Rule, defaultMaxAge time.Duration, now time.Time) (Rule, string, error)
EvaluateControlPlaneBackupStale runs one KindControlPlaneBackupStale rule. It fires when the newest snapshot is older than the rule's ForDuration (else defaultMaxAge). No snapshot yet stays quiet, like the doctor check.
func EvaluateCrashloop ¶
func EvaluateCrashloop(tracker *RestartTracker, r Rule, now time.Time) Rule
EvaluateCrashloop evaluates one crashloop rule with no extra debounce, since RestartWindow already requires sustained restarts.
func EvaluateDatabaseRule ¶
func EvaluateDatabaseRule(ctx context.Context, health DatabaseHealthSource, backups DatabaseBackupSource, r Rule, now time.Time) (Rule, string, error)
EvaluateDatabaseRule runs one database rule and returns its new state and, when firing, a notice line.
func EvaluateDatabaseRuleAll ¶
func EvaluateDatabaseRuleAll(ctx context.Context, health DatabaseHealthSource, backups DatabaseBackupSource, r Rule, now time.Time) (Rule, string, error)
EvaluateDatabaseRuleAll runs a database rule against its named database, or against every managed database when the rule names none (the seeded defaults).
func EvaluateDeployFailed ¶
func EvaluateDeployFailed(_ context.Context, src DeployFailureSource, r Rule, now time.Time) (Rule, string, error)
EvaluateDeployFailed fires while any app's latest deploy failed within the window (RestartWindow, default one hour) and has not since succeeded.
func EvaluateDomainHealth ¶
func EvaluateDomainHealth(ctx context.Context, apps AppDomainSource, checker DomainCheckSource, r Rule, now time.Time, logger *slog.Logger) (Rule, []string, error)
EvaluateDomainHealth runs one domain_health rule against every domain of the app in its ResourceID. An app with no domains, or one since deleted, is treated as no data: it neither fires nor resolves.
func EvaluateEmailDeliveryFailing ¶
func EvaluateEmailDeliveryFailing(ctx context.Context, src EmailDeliverySource, r Rule, now time.Time) (Rule, string, error)
EvaluateEmailDeliveryFailing fires while recent sends mostly fail: at least Threshold failures (default 3) in the window, outnumbering successes.
func EvaluateExportBehind ¶
func EvaluateExportBehind(ctx context.Context, source ExportHealthSource, r Rule, now time.Time) (Rule, string, error)
EvaluateExportBehind runs one KindExportBehind rule. A destination is behind when it is failing (or its breaker is open) and has not delivered within the rule's ForDuration (else DefaultExportBehindAfter).
func EvaluateLogArchiveStale ¶ added in v0.2.0
func EvaluateLogArchiveStale(ctx context.Context, source LogArchiveSource, r Rule, now time.Time) (Rule, string, error)
EvaluateLogArchiveStale runs one KindLogArchiveStale rule. A policy is unhealthy when its last run failed or it has not succeeded within the rule's ForDuration (else three intervals, at least two hours).
func EvaluateMonitorDown ¶
func EvaluateMonitorDown(_ context.Context, src MonitorDownSource, r Rule, now time.Time) (Rule, string, error)
EvaluateMonitorDown fires while any monitor is confirmed down. Confirmation already happened in the monitor state machine, so the rule's own ForDuration is the only extra delay.
func EvaluateNodeCertExpiring ¶ added in v0.2.0
func EvaluateNodeCertExpiring(ctx context.Context, nodes NodeSource, r Rule, warning, critical time.Duration, now time.Time) (Rule, []string, error)
EvaluateNodeCertExpiring fires while any node's agent certificate is inside its warning window or expired; revoked nodes are skipped.
func EvaluateNodeDiskSpace ¶
func EvaluateNodeDiskSpace(ctx context.Context, nodes NodeSource, metrics MetricsSource, r Rule, thresholdPercent float64, now time.Time, logger *slog.Logger) (Rule, []string, error)
EvaluateNodeDiskSpace fires as soon as any node's used-disk percent meets thresholdPercent, with no debounce. Nodes without a recent sample or with a non-positive total are skipped; a failing node query is logged and skipped.
func EvaluateNodeOffline ¶ added in v0.2.0
func EvaluateNodeOffline(ctx context.Context, nodes NodeSource, r Rule, now time.Time) (Rule, []string, error)
EvaluateNodeOffline fires while any node is offline and resolves once all are back. No debounce: the heartbeat sweep already applies its own grace period.
func EvaluateNodeResourceUsage ¶
func EvaluateNodeResourceUsage(ctx context.Context, nodes NodeSource, services NodeServiceSource, metrics MetricsSource, r Rule, cpuThresholdPercent, memoryThresholdBytes float64, now time.Time, logger *slog.Logger) (Rule, []string, error)
EvaluateNodeResourceUsage checks each node's summed CPU and memory against its threshold, returning one notice per breaching node. Memory is compared in absolute bytes, since no host total exists; LastValue holds only the max CPU percent.
func EvaluatePatchStatus ¶
func EvaluatePatchStatus(ctx context.Context, nodes NodeSource, metrics MetricsSource, r Rule, threshold float64, now time.Time, logger *slog.Logger) (Rule, []string, error)
EvaluatePatchStatus evaluates one patch_status rule across nodes; firing is immediate, with no debounce, since the collector samples hourly. Nodes with no recent sample are skipped, neither confirming nor denying the condition.
func EvaluateRegistryRefreshFailing ¶
func EvaluateRegistryRefreshFailing(ctx context.Context, source RegistryRefreshSource, r Rule, now time.Time) (Rule, string, error)
EvaluateRegistryRefreshFailing runs one KindRegistryRefreshFailing rule. A credential is failing while its last refresh errored.
func EvaluateRestoreDrillFailed ¶ added in v0.2.0
func EvaluateRestoreDrillFailed(ctx context.Context, source RestoreDrillSource, r Rule, now time.Time) (Rule, string, error)
EvaluateRestoreDrillFailed runs one KindRestoreDrillFailed rule. It fires while any resource's latest restore drill is failed and resolves once a later drill passes.
func EvaluateSLOBurn ¶ added in v0.2.0
func EvaluateSLOBurn(ctx context.Context, src MetricsSource, r Rule, p SLOPolicy, now time.Time) (Rule, string, error)
EvaluateSLOBurn runs one KindSLOBurn rule. It returns the rule's next state and, while firing, the notice for the notification. A rule with no SLO config never fires.
func EvaluateScheduledTaskFailure ¶
func EvaluateScheduledTaskFailure(ctx context.Context, tasks ScheduledTaskSource, r Rule, now time.Time) (Rule, string, error)
EvaluateScheduledTaskFailure reads the task's persisted ConsecutiveFailures counter and fires at RestartCountThreshold. No ForDuration debounce: the consecutive-failure count is already the debounce.
func EvaluateScheduledTaskHealth ¶ added in v0.2.0
func EvaluateScheduledTaskHealth(ctx context.Context, tasks ScheduledTaskSource, r Rule, now time.Time) (Rule, string, error)
EvaluateScheduledTaskHealth evaluates a KindScheduledTaskMissed or KindScheduledTaskRunningLong rule from the job's persisted counters, so firing does not depend on how often the engine ticks.
func EvaluateThreshold ¶
func EvaluateThreshold(ctx context.Context, source MetricsSource, r Rule, now time.Time) (Rule, error)
EvaluateThreshold returns the rule's updated state and never writes to the database itself. The condition must hold for ForDuration before firing, but one good sample resolves it at once, on purpose: a false resolve is less harmful than a false page.
func EvaluateVersionSkew ¶ added in v0.2.0
func EvaluateVersionSkew(ctx context.Context, settings VersionSkewSettingsSource, fetchers upgrade.Fetchers, cache *upgrade.Cache, r Rule, currentVersion string, now time.Time) (Rule, string, error)
EvaluateVersionSkew fires a KindVersionSkew rule while currentVersion is behind the configured channel's latest release.
type RuleStore ¶
type RuleStore interface {
ListEnabledRules(ctx context.Context) ([]Rule, error)
UpdateState(ctx context.Context, id string, pendingSince, firingSince *time.Time, firing bool, evaluatedAt time.Time, value *float64) error
RecordNotificationDelivery(ctx context.Context, d NotificationDelivery) error
UpsertCertExpiryObservation(ctx context.Context, o CertExpiryObservation) error
ListCertExpiryObservations(ctx context.Context, ruleID string) ([]CertExpiryObservation, error)
}
RuleStore is the narrow store surface Engine needs; *DB satisfies it structurally.
type SLOAutoRollbackStore ¶ added in v0.2.0
type SLOAutoRollbackStore interface {
AutoRollbackStore
SaveDeployApproval(ctx context.Context, a store.DeployApproval) error
}
SLOAutoRollbackStore extends AutoRollbackStore (crashloop.go) with the save-approval surface store.AutoRollbackSLOBurnPauseForHuman mode needs. *store.DB satisfies this structurally.
type SLOBurnHistoryRecorder ¶ added in v0.2.0
type SLOBurnHistoryRecorder interface {
RecordHistory(ctx context.Context, e HistoryEntry) error
}
SLOBurnHistoryRecorder is the narrow DB surface needed to log a dry-run would-have-rolled-back event.
type SLOConfig ¶ added in v0.2.0
type SLOConfig struct {
Objective string `json:"objective"`
Target float64 `json:"target"`
LatencyMs float64 `json:"latency_ms,omitempty"`
}
SLOConfig is the request-based SLO a KindSLOBurn rule watches. Target is a percentage of good requests (99.9). For SLOLatency a request is good when it finishes within LatencyMs, rounded down to the nearest histogram bound.
func (SLOConfig) ErrorBudget ¶ added in v0.2.0
ErrorBudget is the allowed fraction of bad requests, 0.001 for a 99.9 target.
type SLOPolicy ¶ added in v0.2.0
type SLOPolicy struct {
Tiers []BurnTier
Window time.Duration
MinRequests float64
EvalEvery time.Duration
}
SLOPolicy is the engine-wide burn-rate configuration.
func DefaultSLOPolicy ¶ added in v0.2.0
func DefaultSLOPolicy() SLOPolicy
DefaultSLOPolicy returns the SRE workbook defaults: 14.4x over 1h and 5m and 6x over 6h and 30m page, 3x over 1d and 2h and 1x over 3d and 6h open a ticket.
func SLOPolicyFromEnv ¶ added in v0.2.0
func SLOPolicyFromEnv() SLOPolicy
SLOPolicyFromEnv applies APP_SLO_* overrides to the defaults. A malformed or non-positive value keeps the default.
type SLOStatus ¶ added in v0.2.0
type SLOStatus struct {
Config SLOConfig `json:"config"`
HasTraffic bool `json:"has_traffic"`
BudgetRemaining float64 `json:"budget_remaining"`
BudgetRequests float64 `json:"budget_window_requests"`
Windows []BurnWindowStat `json:"windows"`
Tiers []TierStatus `json:"tiers"`
Firing bool `json:"firing"`
Page bool `json:"page"`
MaxBurn float64 `json:"max_burn"`
}
SLOStatus is the full evaluation of an SLO at one instant.
func ComputeSLOStatus ¶ added in v0.2.0
func ComputeSLOStatus(ctx context.Context, src MetricsSource, app string, cfg SLOConfig, p SLOPolicy, now time.Time) (SLOStatus, error)
ComputeSLOStatus queries every distinct window once and evaluates all tiers. Partial windows (an app newer than the window) are judged on the traffic that exists, and a tier whose long window saw fewer than MinRequests cannot fire.
type ScheduledTaskSource ¶
type ScheduledTaskSource interface {
GetScheduledTask(ctx context.Context, id string) (store.ScheduledTask, error)
}
ScheduledTaskSource is the narrow store surface a scheduled-task-failure evaluation needs. *store.DB satisfies this structurally.
type ServiceLister ¶
type ServiceLister interface {
ListDesiredServices(ctx context.Context) ([]store.DesiredService, error)
}
ServiceLister is the narrow store surface RestartTracker needs to resolve a container name to the service that owns it. *store.DB satisfies this structurally.
type Silence ¶ added in v0.2.0
type Silence struct {
ID string
Matchers SilenceMatcher
StartsAt time.Time
EndsAt time.Time
CreatedBy string
Reason string
CreatedAt time.Time
ExpiredAt *time.Time
}
Silence suppresses notifications for matching alerts between StartsAt and EndsAt.
func (Silence) EffectiveEnd ¶ added in v0.2.0
EffectiveEnd is when the silence stopped or will stop applying.
type SilenceMatcher ¶ added in v0.2.0
type SilenceMatcher struct {
RuleIDs []string `json:"rule_ids,omitempty"`
Kinds []string `json:"kinds,omitempty"`
Apps []string `json:"apps,omitempty"`
Nodes []string `json:"nodes,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Severities []string `json:"severities,omitempty"`
// Metrics narrows rules that carry a metric (threshold rules); rules without one are not constrained by it.
Metrics []string `json:"metrics,omitempty"`
}
SilenceMatcher selects alerts. Every non-empty field must match (AND); within a list field any entry may match (OR).
func (SilenceMatcher) IsEmpty ¶ added in v0.2.0
func (m SilenceMatcher) IsEmpty() bool
IsEmpty reports whether the matcher has no criteria at all.
func (SilenceMatcher) Matches ¶ added in v0.2.0
func (m SilenceMatcher) Matches(c AlertContext) bool
Matches reports whether c satisfies every criterion in m.
func (SilenceMatcher) Validate ¶ added in v0.2.0
func (m SilenceMatcher) Validate() error
Validate rejects a matcher that would match everything or names an unknown severity.
type SpendCapSource ¶
SpendCapSource is optional on a BudgetSource: per-app spend caps add their crossed thresholds to the same rule.
type TemplatedSender ¶
type TemplatedSender struct {
email.Sender
Render MailRenderFunc
}
TemplatedSender is an email.Sender that can also render the branded event templates.
func (TemplatedSender) RenderMail ¶
func (t TemplatedSender) RenderMail(ctx context.Context, key, to string, values map[string]string) (email.Message, bool)
RenderMail implements mailRenderer.
func (TemplatedSender) SendMessage ¶
SendMessage keeps the HTML part and headers when the wrapped sender supports them.
type TidyConfig ¶
type TidyConfig struct {
ArchiveResolvedAfter time.Duration
Retention time.Duration
Interval time.Duration
}
TidyConfig sets when resolved items archive themselves and when archived ones are pruned.
func TidyConfigFromEnv ¶
func TidyConfigFromEnv(getenv func(string) string) TidyConfig
TidyConfigFromEnv reads the sweep settings, falling back to the defaults.
type TidyResult ¶
TidyResult counts what one sweep did.
type TidyRunner ¶
TidyRunner archives resolved alert items and prunes old archived ones for every principal that uses the inbox.
func (*TidyRunner) Run ¶
func (r *TidyRunner) Run(ctx context.Context) error
Run sweeps every Cfg.Interval until ctx is done.
func (*TidyRunner) RunOnce ¶
func (r *TidyRunner) RunOnce(ctx context.Context) (TidyResult, error)
RunOnce performs one sweep.
type TierStatus ¶ added in v0.2.0
type TierStatus struct {
BurnTier
LongBurn float64 `json:"long_burn"`
ShortBurn float64 `json:"short_burn"`
Effective float64 `json:"effective_burn"`
Firing bool `json:"firing"`
}
TierStatus is one tier's evaluation. Effective is the lower of the long and short burn rates, the value compared against Factor.
type VersionSkewSettingsSource ¶ added in v0.2.0
type VersionSkewSettingsSource interface {
GetUpdateSettings(ctx context.Context) (store.UpdateSettings, error)
}
VersionSkewSettingsSource is the narrow store surface EvaluateVersionSkew needs.
Source Files
¶
- backup_copy_behind.go
- backup_missing.go
- budget_threshold.go
- cert_expiry.go
- cert_expiry_observation.go
- cert_links.go
- cert_policy.go
- cert_renewal.go
- cert_suppress.go
- channel_secrets.go
- chat_approval.go
- cp_backup_stale.go
- crashloop.go
- database_fanout.go
- database_health.go
- db.go
- default_routing.go
- defaults.go
- deploy_failed.go
- deploy_notify.go
- deploy_release_notes.go
- disk_space.go
- domain_health.go
- domain_lifecycle_kinds.go
- email_delivery_failing.go
- engine.go
- evaluate.go
- export_behind.go
- history.go
- history_stats.go
- inbox.go
- inbox_digest.go
- inbox_manage.go
- inbox_tidy.go
- log_archive_stale.go
- log_regex_cache.go
- log_rule_runtime.go
- log_rules.go
- mail_render.go
- maintenance.go
- metric_class.go
- migrate.go
- monitor_down.go
- node_alert_status.go
- node_cert_expiring.go
- node_offline.go
- node_resource_usage.go
- noise.go
- noise_state.go
- notification_channel.go
- notification_delivery.go
- notify.go
- patch_status.go
- plugin_channel.go
- quiet_hours.go
- registry_refresh.go
- request_signals.go
- restore_drill.go
- rules.go
- scheduled_task_failure.go
- scheduled_task_health.go
- silence.go
- slo.go
- slo_burn.go
- slo_burn_rollback.go
- traffic_kinds.go
- version_skew.go