Documentation
¶
Overview ¶
Package prommetrics provides Prometheus metric definitions and recording helpers for SolidPing.
Index ¶
- Constants
- Variables
- func DecCheckRunnerAbandonedActive()
- func IncCheckRunnerAbandonedActive()
- func NewDBStatsCollector(db *sql.DB, backend string) prometheus.Collector
- func NewMemInfoCollector() prometheus.Collector
- func NewSubsystemCollector(sizes SubsystemSizes) prometheus.Collector
- func RecordCheckRunnerAbandoned(checkType string)
- func RecordCheckStage(stage string, durationSeconds float64)
- func RecordClaimJobsOutcome(outcome string)
- func RecordDBBusyRetry(backend string)
- func RecordDBQuery(operation, backend, callsite string, durationSeconds float64, ok bool)
- func RecordExecution(checkType, status, region, org string, durationMs float64)
- func RecordHTTPRequest(method, route, status string, durationSeconds float64)
- func RecordIncidentCreated(org, checkType string)
- func RecordJobDuration(jobType, outcome string, seconds float64)
- func RecordJobLeaseLost(jobType string)
- func RecordJobProcessed(jobType, outcome string)
- func RecordJobReaped(outcome string, n int)
- func RecordJobSchedulingDelay(jobType string, seconds float64)
- func RecordLaneClaims(lane string, n int)
- func RecordResultsReaped(n int)
- func RecordSchedulingDelay(region string, delaySeconds float64)
- func RecordWorkerJobClaimed(workerUID, region string)
- func Register(reg prometheus.Registerer)
- func RegisterDB(db *sql.DB, backend string)
- func RegisterSubsystems(reg prometheus.Registerer, sizes SubsystemSizes)
- func SetCheckRunnerParked(workerUID, region string, count float64)
- func SetCheckStatus(checkSlug, checkType, region, org string, up bool)
- func SetCheckStatusStreak(checkSlug, checkType, org string, streak float64)
- func SetChecksConfigured(checkType, org, enabled string, count float64)
- func SetIncidentsActive(org string, count float64)
- func SetJobsQueueDepth(status string, count float64)
- func SetResultsRowCount(periodType string, count float64)
- func SetWorkerFreeRunners(workerUID, region string, count float64)
- func SetWorkersActive(region string, count float64)
- type SubsystemSizes
Constants ¶
const ( // LaneLabelFast is the lane label for fast-lane (lane 0) claims. LaneLabelFast = "fast" // LaneLabelSlow is the lane label for slow-lane (lane 1) claims. LaneLabelSlow = "slow" )
Lane label values for CheckLaneClaims (spec 2026-07-01-03).
Variables ¶
var ( // CheckExecutions counts total check executions. CheckExecutions = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_check_executions_total", Help: "Total number of check executions", }, []string{labelCheckType, labelStatus, labelRegion, labelOrganization}, ) // CheckDuration observes check execution duration in seconds. CheckDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "solidping_check_duration_seconds", Help: "Check execution duration in seconds", Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, }, []string{labelCheckType, labelStatus, labelRegion, labelOrganization}, ) // SchedulingDelay observes delay between scheduled and actual execution time. SchedulingDelay = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "solidping_check_scheduling_delay_seconds", Help: "Delay between scheduled and actual execution time", Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60}, }, []string{labelRegion}, ) // CheckUp indicates whether a check is currently UP (1) or DOWN (0). CheckUp = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_check_up", Help: "1 if check is currently UP, 0 otherwise", }, []string{labelCheckSlug, labelCheckType, labelRegion, labelOrganization}, ) // CheckStatusStreak tracks consecutive results with current status. CheckStatusStreak = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_check_status_streak", Help: "Consecutive results with current status", }, []string{labelCheckSlug, labelCheckType, labelOrganization}, ) // ChecksConfigured tracks the number of configured checks. ChecksConfigured = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_checks_configured", Help: "Number of configured checks", }, []string{labelCheckType, labelOrganization, "enabled"}, ) // WorkersActive tracks the number of active workers. WorkersActive = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_workers_active", Help: "Number of active workers", }, []string{labelRegion}, ) // WorkerFreeRunners tracks available runner slots per worker. WorkerFreeRunners = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_worker_free_runners", Help: "Available runner slots per worker", }, []string{labelWorkerUID, labelRegion}, ) // CheckRunnerParked tracks runner slots per worker currently occupied by // a claimed job sleeping until its scheduled_at — claimed but not yet // due (spec 2026-07-05-08 D5). Visibility into how much of the pool the // bounded claim-ahead window (D3) is occupying; alongside // WorkerFreeRunners this distinguishes "idle" from "parked" instead of // both looking like "not free". CheckRunnerParked = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_check_runner_parked", Help: "Runner slots currently occupied by a claimed job sleeping until its scheduled time", }, []string{labelWorkerUID, labelRegion}, ) // WorkerJobsClaimed counts total jobs claimed by each worker. WorkerJobsClaimed = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_worker_jobs_claimed_total", Help: "Total jobs claimed by worker", }, []string{labelWorkerUID, labelRegion}, ) // IncidentsActive tracks currently open incidents. IncidentsActive = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_incidents_active", Help: "Currently open incidents", }, []string{labelOrganization}, ) // IncidentsTotal counts total incidents created. IncidentsTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_incidents_total", Help: "Total incidents created", }, []string{labelOrganization, labelCheckType}, ) // ChecksRateLimited counts check executions skipped because the // org's MaxChecksPerMinute entitlement was already drained for the // current bucket. Skipped jobs are simply rescheduled for next // period — no result row is written. ChecksRateLimited = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_checks_rate_limited_total", Help: "Total check executions skipped due to MaxChecksPerMinute entitlement", }, []string{labelOrganization}, ) // HTTPRateLimited counts requests intercepted by the per-IP HTTP rate or // concurrency limiters. The reason label has four values: // "rate" — rejected with 429: token bucket empty and slow lane full / waited out. // "rate_delayed" — succeeded after waiting in the rate-limit slow lane. // "concurrency" — rejected with 429: no slot free and waiting room full / waited out. // "concurrency_queued" — succeeded after waiting for a concurrency slot. HTTPRateLimited = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_http_rate_limited_total", Help: "Total requests rejected or queued by the per-IP HTTP rate or concurrency limiter", }, []string{"reason"}, ) // HTTPRequestDuration observes HTTP handler latency by route pattern. HTTPRequestDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "solidping_http_request_duration_seconds", Help: "HTTP handler duration in seconds, keyed by route pattern (low cardinality)", Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, }, []string{labelMethod, labelRoute, labelStatus}, ) // HTTPRequestsTotal counts HTTP requests by route pattern and status. HTTPRequestsTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_http_requests_total", Help: "Total HTTP requests by route pattern and status", }, []string{labelMethod, labelRoute, labelStatus}, ) // DBQueryDuration observes SQL query latency by operation, backend and // callsite. Recorded from the bun sloghook on every query (SELECT, INSERT, // UPDATE, DELETE, BEGIN/COMMIT). Status is "ok" or "error". callsite is a // low-cardinality label threaded through ctx by the calling package (see // internal/db/sloghook.WithCallsite) — bounded by construction to the // handful of packages that annotate their context, plus "unlabelled" for // everything else, so cardinality stays proportional to the number of // annotated call paths rather than to traffic or argument values. DBQueryDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "solidping_db_query_duration_seconds", Help: "SQL query duration in seconds, by operation, backend and callsite", Buckets: []float64{0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5}, }, []string{labelOperation, labelBackend, labelStatus, labelCallsite}, ) // ResultsRowCount gauges the total row count in the results table by // period_type (raw/hour/day/month), across all organizations. Refreshed on // the aggregation job's cadence (internal/jobs/jobtypes/job_aggregation.go), // never per-request — a table-wide COUNT(*) is exactly what this table // cannot afford on every page load. Makes ingest growth visible before it // crosses a shared_buffers cache cliff (spec 2026-08-17-04). ResultsRowCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_results_rows", Help: "Total rows in the results table, by period_type", }, []string{"period_type"}, ) // DBBusyRetries counts SQLite SQLITE_BUSY and PostgreSQL // serialization-failure errors observed by the sloghook. A non-zero // rate indicates write contention; on SQLite it usually means the // 30s busy_timeout is being hit. DBBusyRetries = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_db_busy_retries_total", Help: "Database busy / serialization-failure errors (SQLite SQLITE_BUSY, PG 40001)", }, []string{labelBackend}, ) // CheckStageDuration breaks the per-check lifecycle into named stages // (claim, execute, save_result, process_incident, release_lease, // fetch). Lets us see where wall-clock time actually goes when // throughput plateaus. CheckStageDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "solidping_check_stage_duration_seconds", Help: "Per-stage wall-clock duration inside the check execution lifecycle", Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 30}, }, []string{labelStage}, ) // ClaimJobsResult counts the outcome of each ClaimJobs call from // fetcherLoop. Distinguishes "jobs returned" from "no due jobs" and // "due jobs were locked by another worker / optimistic conflict". ClaimJobsResult = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_claim_jobs_result_total", Help: "Outcome of ClaimJobs calls (jobs, empty, lock_conflict, error)", }, []string{labelOutcome}, ) // CheckLaneClaims counts claimed check jobs by lane (fast | slow), the // per-lane companion to ClaimJobsResult (spec 2026-07-01-03 D6). A slow // lane pinned at zero while slow work is due means the reservation budget // is saturated (busySlow == pool − fast_lane_reserved) — the intended, // contained failure mode where slow checks degrade to best-effort while // fast checks stay on time. CheckLaneClaims = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_check_lane_claims_total", Help: "Check jobs claimed by the pool fetcher, by lane (fast, slow)", }, []string{labelLane}, ) // JobsProcessed counts background-jobs processed by the job worker, // labeled by job_type and terminal outcome (success | retried | failed). // job_type is bounded by the jobdef enum; never use job_uid, org, or error // text as labels (cardinality). JobsProcessed = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_jobs_processed_total", Help: "Total background jobs processed, by type and outcome", }, []string{labelJobType, labelOutcome}, ) // JobDuration observes background-job execution wall-clock duration in // seconds, labeled by job_type and outcome. Buckets span fast webhooks to // slow discovery fan-outs. JobDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "solidping_job_duration_seconds", Help: "Background job execution duration in seconds, by type and outcome", Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120}, }, []string{labelJobType, labelOutcome}, ) // JobSchedulingDelay observes the delay between a job's scheduled_at and the // time it actually started running (clamped >= 0) — the queue-lateness // signal, mirroring solidping_check_scheduling_delay_seconds. JobSchedulingDelay = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "solidping_job_scheduling_delay_seconds", Help: "Delay between a job's scheduled time and when it started running", Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60, 300}, }, []string{labelJobType}, ) // EmailDeliveryLatency observes the round-trip latency of a send-mode SMTP // probe email (spec 2026-08-19-04): time between the sending SMTP check's // X-SolidPing-Sent-At header and the receiving email check's JMAP // receivedAt. Only recorded when both headers are present and the // resulting latency passes the sanity clamp (non-negative, not absurdly // large) — see emailcheck/handler.go. EmailDeliveryLatency = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "solidping_email_delivery_latency_seconds", Help: "Round-trip latency of a send-mode SMTP probe email from send to JMAP receipt", Buckets: []float64{0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1800}, }, []string{labelOrganization}, ) // JobsQueueDepth is the point-in-time backlog of background jobs by status // (pending | running). Set by a periodic sampler that zero-fills statuses // with no rows so a drained status drops to 0 rather than going stale. JobsQueueDepth = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_jobs_queue_depth", Help: "Background-job queue depth by status (pending, running)", }, []string{labelStatus}, ) // JobsReaped counts jobs the stuck-job reaper recovered, by outcome // ("retried" = rescheduled via the retry chain, "failed" = retry cap // reached). A spike signals worker instability (orphaned jobs from // restarts/crashes/deploys), not normal operation. JobsReaped = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_jobs_reaped_total", Help: "Total background jobs recovered by the stuck-job reaper, by outcome (retried, failed)", }, []string{labelOutcome}, ) // JobsLeaseLost counts terminal job writes the worker discarded because the // reaper had already moved the row out of 'running' (the worker lost its // job to the reaper). Non-zero means the stuck-timeout is firing on jobs // that were still alive — consider raising SP_JOBS_STUCK_TIMEOUT. JobsLeaseLost = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_jobs_lease_lost_total", Help: "Worker terminal writes discarded because the reaper already transitioned the job, by job type", }, []string{labelJobType}, ) // ResultsReaped counts raw results the abandoned-result reaper finalized // from a stale created marker into a terminal error (spec 2026-08-18-03). // A spike signals a worker that crashed or restarted mid-cycle across many // checks, not normal operation — these results are deliberately excluded // from availability, so a spike here should never itself be read as a // customer-facing availability dip. ResultsReaped = prometheus.NewCounter( prometheus.CounterOpts{ Name: "solidping_results_reaped_total", Help: "Total raw results finalized by the abandoned-result reaper (stale created rows)", }, ) // RealtimeConnections tracks currently open realtime hint WebSocket // connections. Global gauge — no per-org label so cardinality stays bounded. RealtimeConnections = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "solidping_realtime_connections", Help: "Currently open realtime hint stream connections", }, ) // RealtimeHintsPublished counts org hint events published to the notifier // bus (after coalescing). RealtimeHintsPublished = prometheus.NewCounter( prometheus.CounterOpts{ Name: "solidping_realtime_hints_published_total", Help: "Total org hint events published to the notifier bus", }, ) // RealtimeHintsCoalesced counts hint publications absorbed by the // leading-edge coalescer (merged into a pending per-org dirty set instead // of producing an immediate bus publish). RealtimeHintsCoalesced = prometheus.NewCounter( prometheus.CounterOpts{ Name: "solidping_realtime_hints_coalesced_total", Help: "Total hint publications merged by the coalescer instead of published immediately", }, ) // RealtimeHintsDelivered counts hint deliveries to local stream // subscribers (one increment per subscriber that received a hint). RealtimeHintsDelivered = prometheus.NewCounter( prometheus.CounterOpts{ Name: "solidping_realtime_hints_delivered_total", Help: "Total hint events delivered to local realtime stream subscribers", }, ) // RealtimeSubscriptions tracks currently active per-connection scope // subscriptions (sum across every open connection). Global gauge — no // per-org label so cardinality stays bounded. RealtimeSubscriptions = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "solidping_realtime_subscriptions", Help: "Currently active realtime scope subscriptions across all connections", }, ) // RealtimeMessagesReceived counts client->server WebSocket messages // processed by the realtime handler, labeled by message type (auth, // subscribe, unsubscribe, and unknown/malformed). RealtimeMessagesReceived = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_realtime_messages_received_total", Help: "Total client->server realtime WebSocket messages processed, by type", }, []string{labelMessageType}, ) // CheckRunnerAbandoned counts checker executions the watchdog gave up on // because the checker did not honor its context deadline within // execTimeout + abandonGrace (spec 2026-07-05-05 D1/D3). A lost runner // goroutine is otherwise silent and cumulative; this is the loud signal. CheckRunnerAbandoned = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_check_runner_abandoned_total", Help: "Total checker executions abandoned by the watchdog because the checker ignored its context", }, []string{labelCheckType}, ) // CheckRunnerAbandonedActive gauges checker goroutines currently // abandoned-but-still-running: incremented when the watchdog gives up on // an execution, decremented by the child goroutine's own deferred // cleanup if it ever returns (normally or via a late panic). Non-zero // for longer than a few executions means leaked goroutines are // accumulating (the exact failure mode from the 2026-07-04/05 // incident) — this is the direct "leaked goroutines right now" signal // that was invisible then. No labels: a single fleet-wide count per D3. CheckRunnerAbandonedActive = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "solidping_check_runner_abandoned_active", Help: "Checker goroutines currently abandoned by the watchdog but still running (leaked)", }, ) // TLSEdgeConnections counts connections classified by the TLS edge's // fallback splitter, per listener ("http"/"https") and outcome // ("local", "forwarded", "refused", "dial_failed"). A chained deployment // is otherwise silent: without this, "the downstream instance stopped // getting traffic" and "the next hop is unreachable" look identical. TLSEdgeConnections = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_tlsedge_connections_total", Help: "Connections classified by the TLS edge fallback splitter", }, []string{labelListener, labelOutcome}, ) // SupportCapture counts inbound human messages the support inbox tried to // capture, per channel and outcome ("captured", "deduplicated", "throttled", // "failed"). Capture is best-effort for the request — a failure must never // break the channel it came from — so without a counter a silent capture // outage is indistinguishable from nobody writing in (spec 2026-08-22-02). SupportCapture = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_support_capture_total", Help: "Inbound support messages by channel and capture outcome", }, []string{labelChannel, labelOutcome}, ) // SupportMirror counts the notification emails mirroring captured messages // to the support mailbox, by outcome ("sent", "folded", "throttled", // "failed", "disabled"). SupportMirror = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_support_mirror_total", Help: "Support-inbox mirror notifications by outcome", }, []string{labelOutcome}, ) // OperatorNotice counts instance-level operator notices by event // ("support.message", "user.registered", "watchdog.digest", "test"), // contact type and outcome ("sent", "failed", "skipped"). Mirrors // solidping_support_mirror_total: a notice that never reaches anybody is // otherwise indistinguishable from an instance where nothing happened. OperatorNotice = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_operator_notice_total", Help: "Operator notices by event, contact type and delivery outcome", }, []string{labelEvent, labelContactType, labelOutcome}, ) // deliver direct messages because they were installed before the DM scope // existed and have not been re-authorized. // // It is the observable half of "degrade cleanly". Slack does not grant new // scopes to an existing install, so such a workspace simply never delivers // message.im — which from the inbox looks exactly like nobody writing in. A // gauge rather than a counter because this is a STATE (how many workspaces // still owe a reinstall), not an event stream. SupportDMUnavailable = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_support_dm_unavailable", Help: "Connected integrations whose DM capture needs a reinstall to work", }, []string{labelChannel}, ) // OrgProviderLinksDangling reports how many LIVE organization_providers rows // point at an organization that no longer resolves. // // Same "degrade cleanly and observably" treatment as SupportDMUnavailable, // and for a failure mode that is even quieter: such a link silently wins the // partial unique lookup on (provider_type, provider_id), so the workspace or // guild behind it cannot sign in or reinstall, and the only symptom is a // server-side error nobody reads. The healers clear a row the moment someone // tries again — this gauge is what makes the rows visible BEFORE that. // // A gauge, not a counter: it is a state (how many rows are dangling right // now), sampled once at boot. OrgProviderLinksDangling = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "solidping_org_provider_links_dangling", Help: "Live organization_providers rows whose organization no longer resolves", }, ) // WatchdogAnomalies reports how many platform anomalies the hourly // watchdog found, per detector and severity (spec 2026-08-24-10). // // This is the OUT-OF-BAND half of the watchdog. The in-band digest is // delivered by the very process it is monitoring, which is better than // nothing but must never be the only signal: if the API is what broke, the // message reporting it does not go out. A scraped gauge lets an external // Prometheus alert on the same facts without depending on us at all. WatchdogAnomalies = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "solidping_watchdog_anomalies", Help: "Platform anomalies found by the last watchdog run, by detector and severity", }, []string{labelDetector, labelSeverity}, ) // WatchdogStrandedJobs is the total overdue-job count across every region // the watchdog reported as dark or backlogged. 419 stranded jobs is the // single number that told the story on 2026-08-24, so it gets its own // series rather than living inside a label. WatchdogStrandedJobs = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "solidping_watchdog_stranded_jobs", Help: "Overdue check_jobs across every region the watchdog flagged", }, ) // WatchdogStaleIncidents is the count of active incidents whose check has // stopped producing results — the "frozen incident" symptom. WatchdogStaleIncidents = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "solidping_watchdog_stale_incidents", Help: "Active incidents whose check has produced no result for longer than its staleness threshold", }, ) // WatchdogDetectorFailures counts detector errors. A detector that cannot // run is a blind spot, and a blind spot in the thing that watches for // blind spots has to be visible from outside. WatchdogDetectorFailures = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_watchdog_detector_failures_total", Help: "Watchdog detector runs that ended in an error", }, []string{labelDetector}, ) // WatchdogLastRun is the unix timestamp of the last completed watchdog // run. Its absence or staleness is itself alertable — a watchdog that // stopped running is exactly the failure nobody notices. WatchdogLastRun = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "solidping_watchdog_last_run_timestamp_seconds", Help: "Unix timestamp of the last completed platform watchdog run", }, ) // HeartbeatPushBeats counts beats seen by the embedded TCP/UDP heartbeat // listeners. transport is "tcp" or "udp"; outcome is one of: // "accepted" — verified and recorded as a heartbeat result. // "malformed" — did not parse as an SP1/SP2 line. // "rejected" — parsed but refused (unknown target, bad token, bad // MAC, replayed counter, stale timestamp, require_hmac). // Deliberately ONE bucket: the wire must not distinguish // these, and neither should a dashboard that someone // might screenshot into a ticket. // "rate_limited" — dropped by the per-source-IP budget before parsing. // "error" — an internal fault (a database failure), the only // outcome worth alerting on. HeartbeatPushBeats = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_heartbeat_push_beats_total", Help: "Total beats seen by the embedded TCP/UDP heartbeat listeners, by transport and outcome", }, []string{"transport", "outcome"}, ) // HeartbeatPushConnections counts TCP connections accepted by the embedded // heartbeat listener, and those refused because the connection cap was // already reached ("refused"). HeartbeatPushConnections = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "solidping_heartbeat_push_connections_total", Help: "Total TCP connections handled by the embedded heartbeat listener, by outcome", }, []string{"outcome"}, ) )
Functions ¶
func DecCheckRunnerAbandonedActive ¶
func DecCheckRunnerAbandonedActive()
DecCheckRunnerAbandonedActive marks a previously-abandoned checker goroutine as no longer outstanding (it finally returned, normally or via a recovered panic). Must only be called after a matching IncCheckRunnerAbandonedActive.
func IncCheckRunnerAbandonedActive ¶
func IncCheckRunnerAbandonedActive()
IncCheckRunnerAbandonedActive marks one more checker goroutine as abandoned-but-still-running. Paired with DecCheckRunnerAbandonedActive when (if) the child goroutine ever returns.
func NewDBStatsCollector ¶
func NewDBStatsCollector(db *sql.DB, backend string) prometheus.Collector
NewDBStatsCollector returns a Prometheus collector that emits the fields of sql.DBStats as gauges labeled with the given backend ("sqlite" or "postgres").
func NewMemInfoCollector ¶
func NewMemInfoCollector() prometheus.Collector
NewMemInfoCollector returns a collector reading the live process. The snapshot source is injectable so a test can drive it from fixtures.
func NewSubsystemCollector ¶
func NewSubsystemCollector(sizes SubsystemSizes) prometheus.Collector
NewSubsystemCollector returns a Prometheus collector exposing the suspect subsystem sizes plus runtime.NumGoroutine.
func RecordCheckRunnerAbandoned ¶
func RecordCheckRunnerAbandoned(checkType string)
RecordCheckRunnerAbandoned increments the abandoned-execution counter for the given check type. Called once per watchdog abandonment (spec 2026-07-05-05 D3).
func RecordCheckStage ¶
RecordCheckStage observes wall-clock duration for one stage of the check-execution lifecycle. Stages: "fetch", "claim", "execute", "save_result", "process_incident", "release_lease".
func RecordClaimJobsOutcome ¶
func RecordClaimJobsOutcome(outcome string)
RecordClaimJobsOutcome increments the claim-jobs outcome counter. outcome ∈ {"jobs", "empty", "lock_conflict", "error"}.
func RecordDBBusyRetry ¶
func RecordDBBusyRetry(backend string)
RecordDBBusyRetry increments the busy-retry counter for the given backend. Called when the bun hook spots a SQLITE_BUSY (SQLite) or a serialization-failure (Postgres).
func RecordDBQuery ¶
RecordDBQuery records a SQL query observation. backend is "sqlite" or "postgres"; operation is the bun-reported verb ("SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK"); callsite is the bounded label from internal/db/sloghook.WithCallsite (or "unlabelled"); ok=false means the query returned a non-ErrNoRows error.
func RecordExecution ¶
RecordExecution records a check execution's counter increment and duration observation. durationMs is provided in milliseconds and converted to seconds for the histogram.
func RecordHTTPRequest ¶
RecordHTTPRequest records the duration and outcome of an HTTP request, keyed by the route pattern (not the raw path) to keep cardinality bounded. Pass an empty route to skip recording — useful when the request hit a 404 catch-all that doesn't correspond to a registered route.
func RecordIncidentCreated ¶
func RecordIncidentCreated(org, checkType string)
RecordIncidentCreated increments the total incidents counter.
func RecordJobDuration ¶
RecordJobDuration observes a background job's execution duration in seconds, labeled by job type and terminal outcome.
func RecordJobLeaseLost ¶
func RecordJobLeaseLost(jobType string)
RecordJobLeaseLost increments the lease-lost counter for the given job type. Called when a worker's terminal write is discarded because the reaper already transitioned the job out of 'running'.
func RecordJobProcessed ¶
func RecordJobProcessed(jobType, outcome string)
RecordJobProcessed increments the processed-jobs counter for the given job type and terminal outcome ("success" | "retried" | "failed").
func RecordJobReaped ¶
RecordJobReaped increments the reaped-jobs counter by n for the given outcome ("retried" | "failed"). n is the number of jobs reaped in one sweep with that outcome; a zero n is a no-op.
func RecordJobSchedulingDelay ¶
RecordJobSchedulingDelay observes the delay (in seconds, clamped >= 0) between a job's scheduled time and when it actually started running.
func RecordLaneClaims ¶
RecordLaneClaims adds n claimed jobs to the per-lane claim counter. lane ∈ {LaneLabelFast, LaneLabelSlow}. A zero n is a no-op so callers can pass raw per-batch counts without pre-filtering.
func RecordResultsReaped ¶
func RecordResultsReaped(n int)
RecordResultsReaped increments the abandoned-result reaper counter by n. n is the number of raw results finalized in one sweep; a zero n is a no-op.
func RecordSchedulingDelay ¶
RecordSchedulingDelay records the delay between scheduled and actual execution time.
func RecordWorkerJobClaimed ¶
func RecordWorkerJobClaimed(workerUID, region string)
RecordWorkerJobClaimed increments the jobs claimed counter for a worker.
func Register ¶
func Register(reg prometheus.Registerer)
Register registers all SolidPing metrics with the given registerer, plus the Go runtime and process collectors (heap/RSS/goroutine/GC time series) that memory leak detection depends on. Called for every node role so the API server and worker expose the same /metrics surface.
func RegisterDB ¶
RegisterDB registers a DB-pool stats collector for the given backend with the default Prometheus registerer. Safe to call multiple times for the same backend label: a duplicate registration is silently ignored so embedded-PG bootstraps in tests don't panic.
func RegisterSubsystems ¶
func RegisterSubsystems(reg prometheus.Registerer, sizes SubsystemSizes)
RegisterSubsystems registers a subsystem-sizes collector with reg. Safe to call once per process; a duplicate registration is ignored so tests that re-bootstrap don't panic.
func SetCheckRunnerParked ¶
SetCheckRunnerParked sets the number of runner slots for a worker currently occupied by a claimed job sleeping until its scheduled time (spec 2026-07-05-08 D5).
func SetCheckStatus ¶
SetCheckStatus sets the up/down gauge for a specific check.
func SetCheckStatusStreak ¶
SetCheckStatusStreak sets the consecutive status streak for a check.
func SetChecksConfigured ¶
SetChecksConfigured sets the number of configured checks for a given type/org/enabled combo.
func SetIncidentsActive ¶
SetIncidentsActive sets the number of currently open incidents for an organization.
func SetJobsQueueDepth ¶
SetJobsQueueDepth sets the queue-depth gauge for the given status ("pending" | "running"). Callers must zero-fill statuses with no rows so a drained status reports 0 rather than going stale.
func SetResultsRowCount ¶
SetResultsRowCount sets the results-row-count gauge for the given period_type. Called by the aggregation-job-cadence sampler, never per-request.
func SetWorkerFreeRunners ¶
SetWorkerFreeRunners sets the available runner slots for a worker.
func SetWorkersActive ¶
SetWorkersActive sets the number of active workers in a region.
Types ¶
type SubsystemSizes ¶
type SubsystemSizes struct {
// DEKCacheEntries returns the per-org DEK cache size (grows O(orgs), never
// evicted).
DEKCacheEntries func() int
// RateLimitEntries returns the per-IP rate-limiter map size (grows O(unique
// IPs); pruned by a cleanup loop).
RateLimitEntries func() int
// EventListeners returns the total registered notifier listener channels
// (channel-per-Listen; leaks if not deregistered).
EventListeners func() int
}
SubsystemSizes provides scrape-time counts for the long-lived in-memory structures whose growth go_memstats cannot attribute. Closures are injected at registration so prommetrics stays free of import cycles (middleware/credentials/notifier all import nothing from here).
Any function may be nil; a nil function is reported as 0. NumGoroutine is always sourced from the runtime directly and cross-checks the Go collector's go_goroutines.