prommetrics

package
v0.0.0-...-81c9f84 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 9, 2026 License: AGPL-3.0 Imports: 7 Imported by: 0

Documentation

Overview

Package prommetrics provides Prometheus metric definitions and recording helpers for SolidPing.

Index

Constants

View Source
const (
	// LaneLabelFast is the lane label for fast-lane (lane 0) claims.
	LaneLabelFast = "fast"
	// LaneLabelSlow is the lane label for slow-lane (lane 1) claims.
	LaneLabelSlow = "slow"
)

Lane label values for CheckLaneClaims (spec 2026-07-01-03).

Variables

View Source
var (
	// CheckExecutions counts total check executions.
	CheckExecutions = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_check_executions_total",
			Help: "Total number of check executions",
		},
		[]string{labelCheckType, labelStatus, labelRegion, labelOrganization},
	)

	// CheckDuration observes check execution duration in seconds.
	CheckDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "solidping_check_duration_seconds",
			Help:    "Check execution duration in seconds",
			Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30},
		},
		[]string{labelCheckType, labelStatus, labelRegion, labelOrganization},
	)

	// SchedulingDelay observes delay between scheduled and actual execution time.
	SchedulingDelay = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "solidping_check_scheduling_delay_seconds",
			Help:    "Delay between scheduled and actual execution time",
			Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60},
		},
		[]string{labelRegion},
	)

	// CheckUp indicates whether a check is currently UP (1) or DOWN (0).
	CheckUp = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_check_up",
			Help: "1 if check is currently UP, 0 otherwise",
		},
		[]string{labelCheckSlug, labelCheckType, labelRegion, labelOrganization},
	)

	// CheckStatusStreak tracks consecutive results with current status.
	CheckStatusStreak = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_check_status_streak",
			Help: "Consecutive results with current status",
		},
		[]string{labelCheckSlug, labelCheckType, labelOrganization},
	)

	// ChecksConfigured tracks the number of configured checks.
	ChecksConfigured = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_checks_configured",
			Help: "Number of configured checks",
		},
		[]string{labelCheckType, labelOrganization, "enabled"},
	)

	// WorkersActive tracks the number of active workers.
	WorkersActive = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_workers_active",
			Help: "Number of active workers",
		},
		[]string{labelRegion},
	)

	// WorkerFreeRunners tracks available runner slots per worker.
	WorkerFreeRunners = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_worker_free_runners",
			Help: "Available runner slots per worker",
		},
		[]string{labelWorkerUID, labelRegion},
	)

	// CheckRunnerParked tracks runner slots per worker currently occupied by
	// a claimed job sleeping until its scheduled_at — claimed but not yet
	// due (spec 2026-07-05-08 D5). Visibility into how much of the pool the
	// bounded claim-ahead window (D3) is occupying; alongside
	// WorkerFreeRunners this distinguishes "idle" from "parked" instead of
	// both looking like "not free".
	CheckRunnerParked = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_check_runner_parked",
			Help: "Runner slots currently occupied by a claimed job sleeping until its scheduled time",
		},
		[]string{labelWorkerUID, labelRegion},
	)

	// WorkerJobsClaimed counts total jobs claimed by each worker.
	WorkerJobsClaimed = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_worker_jobs_claimed_total",
			Help: "Total jobs claimed by worker",
		},
		[]string{labelWorkerUID, labelRegion},
	)

	// IncidentsActive tracks currently open incidents.
	IncidentsActive = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_incidents_active",
			Help: "Currently open incidents",
		},
		[]string{labelOrganization},
	)

	// IncidentsTotal counts total incidents created.
	IncidentsTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_incidents_total",
			Help: "Total incidents created",
		},
		[]string{labelOrganization, labelCheckType},
	)

	// ChecksRateLimited counts check executions skipped because the
	// org's MaxChecksPerMinute entitlement was already drained for the
	// current bucket. Skipped jobs are simply rescheduled for next
	// period — no result row is written.
	ChecksRateLimited = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_checks_rate_limited_total",
			Help: "Total check executions skipped due to MaxChecksPerMinute entitlement",
		},
		[]string{labelOrganization},
	)

	// HTTPRateLimited counts requests intercepted by the per-IP HTTP rate or
	// concurrency limiters. The reason label has four values:
	//   "rate"               — rejected with 429: token bucket empty and slow lane full / waited out.
	//   "rate_delayed"       — succeeded after waiting in the rate-limit slow lane.
	//   "concurrency"        — rejected with 429: no slot free and waiting room full / waited out.
	//   "concurrency_queued" — succeeded after waiting for a concurrency slot.
	HTTPRateLimited = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_http_rate_limited_total",
			Help: "Total requests rejected or queued by the per-IP HTTP rate or concurrency limiter",
		},
		[]string{"reason"},
	)

	// HTTPRequestDuration observes HTTP handler latency by route pattern.
	HTTPRequestDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "solidping_http_request_duration_seconds",
			Help:    "HTTP handler duration in seconds, keyed by route pattern (low cardinality)",
			Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
		},
		[]string{labelMethod, labelRoute, labelStatus},
	)

	// HTTPRequestsTotal counts HTTP requests by route pattern and status.
	HTTPRequestsTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_http_requests_total",
			Help: "Total HTTP requests by route pattern and status",
		},
		[]string{labelMethod, labelRoute, labelStatus},
	)

	// DBQueryDuration observes SQL query latency by operation, backend and
	// callsite. Recorded from the bun sloghook on every query (SELECT, INSERT,
	// UPDATE, DELETE, BEGIN/COMMIT). Status is "ok" or "error". callsite is a
	// low-cardinality label threaded through ctx by the calling package (see
	// internal/db/sloghook.WithCallsite) — bounded by construction to the
	// handful of packages that annotate their context, plus "unlabelled" for
	// everything else, so cardinality stays proportional to the number of
	// annotated call paths rather than to traffic or argument values.
	DBQueryDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "solidping_db_query_duration_seconds",
			Help:    "SQL query duration in seconds, by operation, backend and callsite",
			Buckets: []float64{0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5},
		},
		[]string{labelOperation, labelBackend, labelStatus, labelCallsite},
	)

	// ResultsRowCount gauges the total row count in the results table by
	// period_type (raw/hour/day/month), across all organizations. Refreshed on
	// the aggregation job's cadence (internal/jobs/jobtypes/job_aggregation.go),
	// never per-request — a table-wide COUNT(*) is exactly what this table
	// cannot afford on every page load. Makes ingest growth visible before it
	// crosses a shared_buffers cache cliff (spec 2026-08-17-04).
	ResultsRowCount = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{

			Name: "solidping_results_rows",
			Help: "Total rows in the results table, by period_type",
		},
		[]string{"period_type"},
	)

	// DBBusyRetries counts SQLite SQLITE_BUSY and PostgreSQL
	// serialization-failure errors observed by the sloghook. A non-zero
	// rate indicates write contention; on SQLite it usually means the
	// 30s busy_timeout is being hit.
	DBBusyRetries = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_db_busy_retries_total",
			Help: "Database busy / serialization-failure errors (SQLite SQLITE_BUSY, PG 40001)",
		},
		[]string{labelBackend},
	)

	// CheckStageDuration breaks the per-check lifecycle into named stages
	// (claim, execute, save_result, process_incident, release_lease,
	// fetch). Lets us see where wall-clock time actually goes when
	// throughput plateaus.
	CheckStageDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "solidping_check_stage_duration_seconds",
			Help:    "Per-stage wall-clock duration inside the check execution lifecycle",
			Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 30},
		},
		[]string{labelStage},
	)

	// ClaimJobsResult counts the outcome of each ClaimJobs call from
	// fetcherLoop. Distinguishes "jobs returned" from "no due jobs" and
	// "due jobs were locked by another worker / optimistic conflict".
	ClaimJobsResult = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_claim_jobs_result_total",
			Help: "Outcome of ClaimJobs calls (jobs, empty, lock_conflict, error)",
		},
		[]string{labelOutcome},
	)

	// CheckLaneClaims counts claimed check jobs by lane (fast | slow), the
	// per-lane companion to ClaimJobsResult (spec 2026-07-01-03 D6). A slow
	// lane pinned at zero while slow work is due means the reservation budget
	// is saturated (busySlow == pool − fast_lane_reserved) — the intended,
	// contained failure mode where slow checks degrade to best-effort while
	// fast checks stay on time.
	CheckLaneClaims = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_check_lane_claims_total",
			Help: "Check jobs claimed by the pool fetcher, by lane (fast, slow)",
		},
		[]string{labelLane},
	)

	// JobsProcessed counts background-jobs processed by the job worker,
	// labeled by job_type and terminal outcome (success | retried | failed).
	// job_type is bounded by the jobdef enum; never use job_uid, org, or error
	// text as labels (cardinality).
	JobsProcessed = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_jobs_processed_total",
			Help: "Total background jobs processed, by type and outcome",
		},
		[]string{labelJobType, labelOutcome},
	)

	// JobDuration observes background-job execution wall-clock duration in
	// seconds, labeled by job_type and outcome. Buckets span fast webhooks to
	// slow discovery fan-outs.
	JobDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "solidping_job_duration_seconds",
			Help:    "Background job execution duration in seconds, by type and outcome",
			Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120},
		},
		[]string{labelJobType, labelOutcome},
	)

	// JobSchedulingDelay observes the delay between a job's scheduled_at and the
	// time it actually started running (clamped >= 0) — the queue-lateness
	// signal, mirroring solidping_check_scheduling_delay_seconds.
	JobSchedulingDelay = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "solidping_job_scheduling_delay_seconds",
			Help:    "Delay between a job's scheduled time and when it started running",
			Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60, 300},
		},
		[]string{labelJobType},
	)

	// EmailDeliveryLatency observes the round-trip latency of a send-mode SMTP
	// probe email (spec 2026-08-19-04): time between the sending SMTP check's
	// X-SolidPing-Sent-At header and the receiving email check's JMAP
	// receivedAt. Only recorded when both headers are present and the
	// resulting latency passes the sanity clamp (non-negative, not absurdly
	// large) — see emailcheck/handler.go.
	EmailDeliveryLatency = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "solidping_email_delivery_latency_seconds",
			Help:    "Round-trip latency of a send-mode SMTP probe email from send to JMAP receipt",
			Buckets: []float64{0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600, 1800},
		},
		[]string{labelOrganization},
	)

	// JobsQueueDepth is the point-in-time backlog of background jobs by status
	// (pending | running). Set by a periodic sampler that zero-fills statuses
	// with no rows so a drained status drops to 0 rather than going stale.
	JobsQueueDepth = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_jobs_queue_depth",
			Help: "Background-job queue depth by status (pending, running)",
		},
		[]string{labelStatus},
	)

	// JobsReaped counts jobs the stuck-job reaper recovered, by outcome
	// ("retried" = rescheduled via the retry chain, "failed" = retry cap
	// reached). A spike signals worker instability (orphaned jobs from
	// restarts/crashes/deploys), not normal operation.
	JobsReaped = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_jobs_reaped_total",
			Help: "Total background jobs recovered by the stuck-job reaper, by outcome (retried, failed)",
		},
		[]string{labelOutcome},
	)

	// JobsLeaseLost counts terminal job writes the worker discarded because the
	// reaper had already moved the row out of 'running' (the worker lost its
	// job to the reaper). Non-zero means the stuck-timeout is firing on jobs
	// that were still alive — consider raising SP_JOBS_STUCK_TIMEOUT.
	JobsLeaseLost = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_jobs_lease_lost_total",
			Help: "Worker terminal writes discarded because the reaper already transitioned the job, by job type",
		},
		[]string{labelJobType},
	)

	// ResultsReaped counts raw results the abandoned-result reaper finalized
	// from a stale created marker into a terminal error (spec 2026-08-18-03).
	// A spike signals a worker that crashed or restarted mid-cycle across many
	// checks, not normal operation — these results are deliberately excluded
	// from availability, so a spike here should never itself be read as a
	// customer-facing availability dip.
	ResultsReaped = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "solidping_results_reaped_total",
			Help: "Total raw results finalized by the abandoned-result reaper (stale created rows)",
		},
	)

	// RealtimeConnections tracks currently open realtime hint WebSocket
	// connections. Global gauge — no per-org label so cardinality stays bounded.
	RealtimeConnections = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "solidping_realtime_connections",
			Help: "Currently open realtime hint stream connections",
		},
	)

	// RealtimeHintsPublished counts org hint events published to the notifier
	// bus (after coalescing).
	RealtimeHintsPublished = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "solidping_realtime_hints_published_total",
			Help: "Total org hint events published to the notifier bus",
		},
	)

	// RealtimeHintsCoalesced counts hint publications absorbed by the
	// leading-edge coalescer (merged into a pending per-org dirty set instead
	// of producing an immediate bus publish).
	RealtimeHintsCoalesced = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "solidping_realtime_hints_coalesced_total",
			Help: "Total hint publications merged by the coalescer instead of published immediately",
		},
	)

	// RealtimeHintsDelivered counts hint deliveries to local stream
	// subscribers (one increment per subscriber that received a hint).
	RealtimeHintsDelivered = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "solidping_realtime_hints_delivered_total",
			Help: "Total hint events delivered to local realtime stream subscribers",
		},
	)

	// RealtimeSubscriptions tracks currently active per-connection scope
	// subscriptions (sum across every open connection). Global gauge — no
	// per-org label so cardinality stays bounded.
	RealtimeSubscriptions = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "solidping_realtime_subscriptions",
			Help: "Currently active realtime scope subscriptions across all connections",
		},
	)

	// RealtimeMessagesReceived counts client->server WebSocket messages
	// processed by the realtime handler, labeled by message type (auth,
	// subscribe, unsubscribe, and unknown/malformed).
	RealtimeMessagesReceived = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_realtime_messages_received_total",
			Help: "Total client->server realtime WebSocket messages processed, by type",
		},
		[]string{labelMessageType},
	)

	// CheckRunnerAbandoned counts checker executions the watchdog gave up on
	// because the checker did not honor its context deadline within
	// execTimeout + abandonGrace (spec 2026-07-05-05 D1/D3). A lost runner
	// goroutine is otherwise silent and cumulative; this is the loud signal.
	CheckRunnerAbandoned = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_check_runner_abandoned_total",
			Help: "Total checker executions abandoned by the watchdog because the checker ignored its context",
		},
		[]string{labelCheckType},
	)

	// CheckRunnerAbandonedActive gauges checker goroutines currently
	// abandoned-but-still-running: incremented when the watchdog gives up on
	// an execution, decremented by the child goroutine's own deferred
	// cleanup if it ever returns (normally or via a late panic). Non-zero
	// for longer than a few executions means leaked goroutines are
	// accumulating (the exact failure mode from the 2026-07-04/05
	// incident) — this is the direct "leaked goroutines right now" signal
	// that was invisible then. No labels: a single fleet-wide count per D3.
	CheckRunnerAbandonedActive = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "solidping_check_runner_abandoned_active",
			Help: "Checker goroutines currently abandoned by the watchdog but still running (leaked)",
		},
	)

	// TLSEdgeConnections counts connections classified by the TLS edge's
	// fallback splitter, per listener ("http"/"https") and outcome
	// ("local", "forwarded", "refused", "dial_failed"). A chained deployment
	// is otherwise silent: without this, "the downstream instance stopped
	// getting traffic" and "the next hop is unreachable" look identical.
	TLSEdgeConnections = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_tlsedge_connections_total",
			Help: "Connections classified by the TLS edge fallback splitter",
		},
		[]string{labelListener, labelOutcome},
	)

	// SupportCapture counts inbound human messages the support inbox tried to
	// capture, per channel and outcome ("captured", "deduplicated", "throttled",
	// "failed"). Capture is best-effort for the request — a failure must never
	// break the channel it came from — so without a counter a silent capture
	// outage is indistinguishable from nobody writing in (spec 2026-08-22-02).
	SupportCapture = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_support_capture_total",
			Help: "Inbound support messages by channel and capture outcome",
		},
		[]string{labelChannel, labelOutcome},
	)

	// SupportMirror counts the notification emails mirroring captured messages
	// to the support mailbox, by outcome ("sent", "folded", "throttled",
	// "failed", "disabled").
	SupportMirror = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_support_mirror_total",
			Help: "Support-inbox mirror notifications by outcome",
		},
		[]string{labelOutcome},
	)

	// OperatorNotice counts instance-level operator notices by event
	// ("support.message", "user.registered", "watchdog.digest", "test"),
	// contact type and outcome ("sent", "failed", "skipped"). Mirrors
	// solidping_support_mirror_total: a notice that never reaches anybody is
	// otherwise indistinguishable from an instance where nothing happened.
	OperatorNotice = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_operator_notice_total",
			Help: "Operator notices by event, contact type and delivery outcome",
		},
		[]string{labelEvent, labelContactType, labelOutcome},
	)

	// SupportDMUnavailable reports how many connected integrations cannot
	// deliver direct messages because they were installed before the DM scope
	// existed and have not been re-authorized.
	//
	// It is the observable half of "degrade cleanly". Slack does not grant new
	// scopes to an existing install, so such a workspace simply never delivers
	// message.im — which from the inbox looks exactly like nobody writing in. A
	// gauge rather than a counter because this is a STATE (how many workspaces
	// still owe a reinstall), not an event stream.
	SupportDMUnavailable = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_support_dm_unavailable",
			Help: "Connected integrations whose DM capture needs a reinstall to work",
		},
		[]string{labelChannel},
	)

	// OrgProviderLinksDangling reports how many LIVE organization_providers rows
	// point at an organization that no longer resolves.
	//
	// Same "degrade cleanly and observably" treatment as SupportDMUnavailable,
	// and for a failure mode that is even quieter: such a link silently wins the
	// partial unique lookup on (provider_type, provider_id), so the workspace or
	// guild behind it cannot sign in or reinstall, and the only symptom is a
	// server-side error nobody reads. The healers clear a row the moment someone
	// tries again — this gauge is what makes the rows visible BEFORE that.
	//
	// A gauge, not a counter: it is a state (how many rows are dangling right
	// now), sampled once at boot.
	OrgProviderLinksDangling = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "solidping_org_provider_links_dangling",
			Help: "Live organization_providers rows whose organization no longer resolves",
		},
	)

	// WatchdogAnomalies reports how many platform anomalies the hourly
	// watchdog found, per detector and severity (spec 2026-08-24-10).
	//
	// This is the OUT-OF-BAND half of the watchdog. The in-band digest is
	// delivered by the very process it is monitoring, which is better than
	// nothing but must never be the only signal: if the API is what broke, the
	// message reporting it does not go out. A scraped gauge lets an external
	// Prometheus alert on the same facts without depending on us at all.
	WatchdogAnomalies = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "solidping_watchdog_anomalies",
			Help: "Platform anomalies found by the last watchdog run, by detector and severity",
		},
		[]string{labelDetector, labelSeverity},
	)

	// WatchdogStrandedJobs is the total overdue-job count across every region
	// the watchdog reported as dark or backlogged. 419 stranded jobs is the
	// single number that told the story on 2026-08-24, so it gets its own
	// series rather than living inside a label.
	WatchdogStrandedJobs = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "solidping_watchdog_stranded_jobs",
			Help: "Overdue check_jobs across every region the watchdog flagged",
		},
	)

	// WatchdogStaleIncidents is the count of active incidents whose check has
	// stopped producing results — the "frozen incident" symptom.
	WatchdogStaleIncidents = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "solidping_watchdog_stale_incidents",
			Help: "Active incidents whose check has produced no result for longer than its staleness threshold",
		},
	)

	// WatchdogDetectorFailures counts detector errors. A detector that cannot
	// run is a blind spot, and a blind spot in the thing that watches for
	// blind spots has to be visible from outside.
	WatchdogDetectorFailures = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_watchdog_detector_failures_total",
			Help: "Watchdog detector runs that ended in an error",
		},
		[]string{labelDetector},
	)

	// WatchdogLastRun is the unix timestamp of the last completed watchdog
	// run. Its absence or staleness is itself alertable — a watchdog that
	// stopped running is exactly the failure nobody notices.
	WatchdogLastRun = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "solidping_watchdog_last_run_timestamp_seconds",
			Help: "Unix timestamp of the last completed platform watchdog run",
		},
	)

	// HeartbeatPushBeats counts beats seen by the embedded TCP/UDP heartbeat
	// listeners. transport is "tcp" or "udp"; outcome is one of:
	//   "accepted"     — verified and recorded as a heartbeat result.
	//   "malformed"    — did not parse as an SP1/SP2 line.
	//   "rejected"     — parsed but refused (unknown target, bad token, bad
	//                    MAC, replayed counter, stale timestamp, require_hmac).
	//                    Deliberately ONE bucket: the wire must not distinguish
	//                    these, and neither should a dashboard that someone
	//                    might screenshot into a ticket.
	//   "rate_limited" — dropped by the per-source-IP budget before parsing.
	//   "error"        — an internal fault (a database failure), the only
	//                    outcome worth alerting on.
	HeartbeatPushBeats = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_heartbeat_push_beats_total",
			Help: "Total beats seen by the embedded TCP/UDP heartbeat listeners, by transport and outcome",
		},
		[]string{"transport", "outcome"},
	)

	// HeartbeatPushConnections counts TCP connections accepted by the embedded
	// heartbeat listener, and those refused because the connection cap was
	// already reached ("refused").
	HeartbeatPushConnections = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "solidping_heartbeat_push_connections_total",
			Help: "Total TCP connections handled by the embedded heartbeat listener, by outcome",
		},
		[]string{"outcome"},
	)
)

Functions

func DecCheckRunnerAbandonedActive

func DecCheckRunnerAbandonedActive()

DecCheckRunnerAbandonedActive marks a previously-abandoned checker goroutine as no longer outstanding (it finally returned, normally or via a recovered panic). Must only be called after a matching IncCheckRunnerAbandonedActive.

func IncCheckRunnerAbandonedActive

func IncCheckRunnerAbandonedActive()

IncCheckRunnerAbandonedActive marks one more checker goroutine as abandoned-but-still-running. Paired with DecCheckRunnerAbandonedActive when (if) the child goroutine ever returns.

func NewDBStatsCollector

func NewDBStatsCollector(db *sql.DB, backend string) prometheus.Collector

NewDBStatsCollector returns a Prometheus collector that emits the fields of sql.DBStats as gauges labeled with the given backend ("sqlite" or "postgres").

func NewMemInfoCollector

func NewMemInfoCollector() prometheus.Collector

NewMemInfoCollector returns a collector reading the live process. The snapshot source is injectable so a test can drive it from fixtures.

func NewSubsystemCollector

func NewSubsystemCollector(sizes SubsystemSizes) prometheus.Collector

NewSubsystemCollector returns a Prometheus collector exposing the suspect subsystem sizes plus runtime.NumGoroutine.

func RecordCheckRunnerAbandoned

func RecordCheckRunnerAbandoned(checkType string)

RecordCheckRunnerAbandoned increments the abandoned-execution counter for the given check type. Called once per watchdog abandonment (spec 2026-07-05-05 D3).

func RecordCheckStage

func RecordCheckStage(stage string, durationSeconds float64)

RecordCheckStage observes wall-clock duration for one stage of the check-execution lifecycle. Stages: "fetch", "claim", "execute", "save_result", "process_incident", "release_lease".

func RecordClaimJobsOutcome

func RecordClaimJobsOutcome(outcome string)

RecordClaimJobsOutcome increments the claim-jobs outcome counter. outcome ∈ {"jobs", "empty", "lock_conflict", "error"}.

func RecordDBBusyRetry

func RecordDBBusyRetry(backend string)

RecordDBBusyRetry increments the busy-retry counter for the given backend. Called when the bun hook spots a SQLITE_BUSY (SQLite) or a serialization-failure (Postgres).

func RecordDBQuery

func RecordDBQuery(operation, backend, callsite string, durationSeconds float64, ok bool)

RecordDBQuery records a SQL query observation. backend is "sqlite" or "postgres"; operation is the bun-reported verb ("SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK"); callsite is the bounded label from internal/db/sloghook.WithCallsite (or "unlabelled"); ok=false means the query returned a non-ErrNoRows error.

func RecordExecution

func RecordExecution(checkType, status, region, org string, durationMs float64)

RecordExecution records a check execution's counter increment and duration observation. durationMs is provided in milliseconds and converted to seconds for the histogram.

func RecordHTTPRequest

func RecordHTTPRequest(method, route, status string, durationSeconds float64)

RecordHTTPRequest records the duration and outcome of an HTTP request, keyed by the route pattern (not the raw path) to keep cardinality bounded. Pass an empty route to skip recording — useful when the request hit a 404 catch-all that doesn't correspond to a registered route.

func RecordIncidentCreated

func RecordIncidentCreated(org, checkType string)

RecordIncidentCreated increments the total incidents counter.

func RecordJobDuration

func RecordJobDuration(jobType, outcome string, seconds float64)

RecordJobDuration observes a background job's execution duration in seconds, labeled by job type and terminal outcome.

func RecordJobLeaseLost

func RecordJobLeaseLost(jobType string)

RecordJobLeaseLost increments the lease-lost counter for the given job type. Called when a worker's terminal write is discarded because the reaper already transitioned the job out of 'running'.

func RecordJobProcessed

func RecordJobProcessed(jobType, outcome string)

RecordJobProcessed increments the processed-jobs counter for the given job type and terminal outcome ("success" | "retried" | "failed").

func RecordJobReaped

func RecordJobReaped(outcome string, n int)

RecordJobReaped increments the reaped-jobs counter by n for the given outcome ("retried" | "failed"). n is the number of jobs reaped in one sweep with that outcome; a zero n is a no-op.

func RecordJobSchedulingDelay

func RecordJobSchedulingDelay(jobType string, seconds float64)

RecordJobSchedulingDelay observes the delay (in seconds, clamped >= 0) between a job's scheduled time and when it actually started running.

func RecordLaneClaims

func RecordLaneClaims(lane string, n int)

RecordLaneClaims adds n claimed jobs to the per-lane claim counter. lane ∈ {LaneLabelFast, LaneLabelSlow}. A zero n is a no-op so callers can pass raw per-batch counts without pre-filtering.

func RecordResultsReaped

func RecordResultsReaped(n int)

RecordResultsReaped increments the abandoned-result reaper counter by n. n is the number of raw results finalized in one sweep; a zero n is a no-op.

func RecordSchedulingDelay

func RecordSchedulingDelay(region string, delaySeconds float64)

RecordSchedulingDelay records the delay between scheduled and actual execution time.

func RecordWorkerJobClaimed

func RecordWorkerJobClaimed(workerUID, region string)

RecordWorkerJobClaimed increments the jobs claimed counter for a worker.

func Register

func Register(reg prometheus.Registerer)

Register registers all SolidPing metrics with the given registerer, plus the Go runtime and process collectors (heap/RSS/goroutine/GC time series) that memory leak detection depends on. Called for every node role so the API server and worker expose the same /metrics surface.

func RegisterDB

func RegisterDB(db *sql.DB, backend string)

RegisterDB registers a DB-pool stats collector for the given backend with the default Prometheus registerer. Safe to call multiple times for the same backend label: a duplicate registration is silently ignored so embedded-PG bootstraps in tests don't panic.

func RegisterSubsystems

func RegisterSubsystems(reg prometheus.Registerer, sizes SubsystemSizes)

RegisterSubsystems registers a subsystem-sizes collector with reg. Safe to call once per process; a duplicate registration is ignored so tests that re-bootstrap don't panic.

func SetCheckRunnerParked

func SetCheckRunnerParked(workerUID, region string, count float64)

SetCheckRunnerParked sets the number of runner slots for a worker currently occupied by a claimed job sleeping until its scheduled time (spec 2026-07-05-08 D5).

func SetCheckStatus

func SetCheckStatus(checkSlug, checkType, region, org string, up bool)

SetCheckStatus sets the up/down gauge for a specific check.

func SetCheckStatusStreak

func SetCheckStatusStreak(checkSlug, checkType, org string, streak float64)

SetCheckStatusStreak sets the consecutive status streak for a check.

func SetChecksConfigured

func SetChecksConfigured(checkType, org, enabled string, count float64)

SetChecksConfigured sets the number of configured checks for a given type/org/enabled combo.

func SetIncidentsActive

func SetIncidentsActive(org string, count float64)

SetIncidentsActive sets the number of currently open incidents for an organization.

func SetJobsQueueDepth

func SetJobsQueueDepth(status string, count float64)

SetJobsQueueDepth sets the queue-depth gauge for the given status ("pending" | "running"). Callers must zero-fill statuses with no rows so a drained status reports 0 rather than going stale.

func SetResultsRowCount

func SetResultsRowCount(periodType string, count float64)

SetResultsRowCount sets the results-row-count gauge for the given period_type. Called by the aggregation-job-cadence sampler, never per-request.

func SetWorkerFreeRunners

func SetWorkerFreeRunners(workerUID, region string, count float64)

SetWorkerFreeRunners sets the available runner slots for a worker.

func SetWorkersActive

func SetWorkersActive(region string, count float64)

SetWorkersActive sets the number of active workers in a region.

Types

type SubsystemSizes

type SubsystemSizes struct {
	// DEKCacheEntries returns the per-org DEK cache size (grows O(orgs), never
	// evicted).
	DEKCacheEntries func() int
	// RateLimitEntries returns the per-IP rate-limiter map size (grows O(unique
	// IPs); pruned by a cleanup loop).
	RateLimitEntries func() int
	// EventListeners returns the total registered notifier listener channels
	// (channel-per-Listen; leaks if not deregistered).
	EventListeners func() int
}

SubsystemSizes provides scrape-time counts for the long-lived in-memory structures whose growth go_memstats cannot attribute. Closures are injected at registration so prommetrics stays free of import cycles (middleware/credentials/notifier all import nothing from here).

Any function may be nil; a nil function is reported as 0. NumGoroutine is always sourced from the runtime directly and cross-checks the Go collector's go_goroutines.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL