Documentation
¶
Index ¶
- Constants
- Variables
- func AdminAuth(adminKey string, next http.Handler) http.Handler
- func FilterHeaders(h http.Header) map[string]string
- func Fingerprint(route, msg, topFrame string) uint64
- func NewTraceID() string
- func NormalizeMessage(msg string) string
- func ParseUserAgent(ua string) (browser, os string)
- func ShouldPersistTrace(s Sample) bool
- func TopFrame(stack []Frame) string
- func TraceIDFromCtx(ctx context.Context) string
- func WithSpanTracker(ctx context.Context) context.Context
- func WithTraceID(ctx context.Context, id string) context.Context
- type Alert
- type Alerter
- type AnomalyDetector
- func (d *AnomalyDetector) GetCount(tenantID string) int64
- func (d *AnomalyDetector) IncrCounter(tenantID string)
- func (d *AnomalyDetector) Observe(tenantID string, ms float64) (bool, float64)
- func (d *AnomalyDetector) RecentAnomalies(tenantID string, n int) []AnomalyEvent
- func (d *AnomalyDetector) RecordAnomaly(tenantID string, latencyUS, z float64)
- type AnomalyEvent
- type Attribution
- type AttributionThresholds
- type Check
- type CheckResult
- type CooldownAlerter
- type DBClientStats
- type DBServerProbe
- type DBServerStats
- type DynamicCheck
- type ErrGroup
- type ErrorCapture
- type ErrorGroup
- type ErrorStore
- type Frame
- type FullSnapshot
- type GeoLookup
- type HistoryPoint
- type Metrics
- func (m *Metrics) Gatherer() prometheus.Gatherer
- func (m *Metrics) Handler() http.Handler
- func (m *Metrics) IncGoroutinePanic()
- func (m *Metrics) IncRequestPanic()
- func (m *Metrics) ObserveMigration(tenantID, status string, durationSeconds float64)
- func (m *Metrics) ObserveRequest(tenantID, method, path, status string, durationSeconds float64)
- func (m *Metrics) Register(c prometheus.Collector) error
- func (m *Metrics) SetActiveTenants(n int)
- type MinuteBucket
- type NewErrorNotifier
- type NoopAlerter
- type ObsServer
- func (s *ObsServer) DebugRouter(adminKey string) http.Handler
- func (s *ObsServer) Resources() *ResourceCollector
- func (s *ObsServer) Router(adminKey string) *chi.Mux
- func (s *ObsServer) ServeResources(w http.ResponseWriter, r *http.Request)
- func (s *ObsServer) ServeResourcesSnapshot(w http.ResponseWriter, r *http.Request)
- func (s *ObsServer) ServeTenantData(w http.ResponseWriter, r *http.Request)
- func (s *ObsServer) SetGeo(g *GeoLookup)
- func (s *ObsServer) SetResources(rc *ResourceCollector)
- func (s *ObsServer) SetTracesHandler(h http.Handler)
- type ObsStore
- func (s *ObsStore) Close() error
- func (s *ObsStore) ErrorGroups(tenantID string, hours int) ([]ErrorGroup, error)
- func (s *ObsStore) Flush(tenantID string, snap Snapshot) error
- func (s *ObsStore) History(tenantID string, hours int) ([]Snapshot, error)
- func (s *ObsStore) Path() string
- func (s *ObsStore) Prune() error
- func (s *ObsStore) SaveSlowTrace(tenantID string, tv TraceView) error
- func (s *ObsStore) SlowTraces(tenantID string, hours int) ([]TraceView, error)
- func (s *ObsStore) UpsertErrorGroup(tenantID string, g ErrorGroup, userID string) (isNew bool, err error)
- type PSILine
- type PercentileSnapshot
- type PoolStat
- type PressureStats
- type ProcessStats
- type QueryTracer
- type RecentRequest
- type RequestStats
- type ResourceCollector
- func (c *ResourceCollector) Config() ResourceConfig
- func (c *ResourceCollector) Count() int
- func (c *ResourceCollector) FootprintBytes() int64
- func (c *ResourceCollector) Interval() time.Duration
- func (c *ResourceCollector) Latest() *ResourceSnapshot
- func (c *ResourceCollector) Live() bool
- func (c *ResourceCollector) Observe(durUS, queryUS int64, status int)
- func (c *ResourceCollector) PromCollector() prometheus.Collector
- func (c *ResourceCollector) Run(ctx context.Context)
- func (c *ResourceCollector) Series(n int) []ResourceSnapshot
- func (c *ResourceCollector) SetDB(stat func() PoolStat, probe DBServerProbe)
- func (c *ResourceCollector) Started() bool
- func (c *ResourceCollector) Touch()
- type ResourceConfig
- type ResourceSnapshot
- type Rings
- func (rs *Rings) Count() int
- func (rs *Rings) Recent(tenantID string) []RecentRequest
- func (rs *Rings) RecentTraces(tenantID string, n int) []TraceView
- func (rs *Rings) Record(tenantID string, s Sample)
- func (rs *Rings) RouteID(route string) uint16
- func (rs *Rings) RouteName(id uint16) string
- func (rs *Rings) Snapshot(tenantID string) []Sample
- func (rs *Rings) TenantIDs() []string
- type RuntimeStats
- type SLOConfig
- type SLOEngine
- type SLOSnapshot
- type Sample
- type Signal
- type SlackAlerter
- type Snapshot
- type Span
- type SpanTracker
- func (t *SpanTracker) ElapsedUS() int64
- func (t *SpanTracker) Failed() bool
- func (t *SpanTracker) Finish() []Span
- func (t *SpanTracker) Mark(name string)
- func (t *SpanTracker) MarkFailed(name string)
- func (t *SpanTracker) NoteFailedQuery(sql string, err error)
- func (t *SpanTracker) RecordError(msg string)
- func (t *SpanTracker) SetCapture(c *ErrorCapture)
- func (t *SpanTracker) Spans() []Span
- func (t *SpanTracker) TotalUS() int32
- type SyntheticMonitor
- type TenantHistogram
- type TenantRing
- type TraceView
- type Verdict
- type WindowSummary
Constants ¶
const ( LevelWarning = "warning" LevelCritical = "critical" )
Alert levels.
const ( KindSLO = "" // historical default KindNewError = "new_error" // first occurrence of a fingerprint KindStorm = "storm" // many new groups at once, summarized )
Alert kinds: an SLO breach (burn rate / p95) or a NEW ERROR GROUP — a defect seen for the first time (OBSERVABILIDAD-ERRORES-S1).
const ( DefaultBackgroundInterval = 10 * time.Second DefaultLiveInterval = 1 * time.Second DefaultLiveWindow = 60 * time.Second )
Default cadences. Background is what a box pays 24/7; live is what the /admin correlation view asks for while an operator (or a k6) is looking, and it decays back to background LiveWindow after the last poll.
const PersistErrors = true
PersistErrors, when true, persists every error trace (status >= 400) regardless of latency, so a fast 401/403/422/500 is still captured for debugging.
const ResourceRingSize = 900
ResourceRingSize bounds the correlation series: 900 ticks = 15 minutes in live mode (1 s) or 2.5 hours in background mode (10 s). A fixed array — never a slice that grows — so the collector's memory is known before the first request (the A-54 RSS proxy is stated in bytes for this reason).
const SlowTraceThresholdUS = 50_000
SlowTraceThresholdUS is the per-request duration above which a trace is persisted to slow_traces (Phase 1: a fixed 50ms, not a per-tenant p95).
const TraceIDKey = contextKey("trace_id")
TraceIDKey is the context key under which the per-request trace id is stored.
Variables ¶
var Attributions = []Attribution{ AttrCPUThrottled, AttrMemoryPressure, AttrGCPressure, AttrCPUSaturated, AttrPoolExhausted, AttrDBBound, AttrLockContention, AttrHealthy, }
Attributions lists the eight values in the priority order the rules use.
var ResourcesVersion = "dev"
ResourcesVersion is set by the engine so the exported snapshot names the build it came from (an attachment to a report must say which binary).
Functions ¶
func AdminAuth ¶
AdminAuth wraps next so it is reachable only with a matching X-Admin-Key header. This is the single gate shared by the /debug/* endpoints and /metrics.
func FilterHeaders ¶
FilterHeaders flattens an http.Header into a name→value map, redacting the values of sensitive (credential) headers to "[Filtered]". Only the first value of each header is kept. Called only when a trace is persisted (off the hot path), so its allocation never taxes the 200-OK common case.
func Fingerprint ¶ added in v0.1.16
Fingerprint hashes (route, normalized message, top frame) into the group key. topFrame may be "" (no stack known); route is the matched template.
func NewTraceID ¶
func NewTraceID() string
NewTraceID returns a 16-hex-character trace id (8 random bytes, no hyphens), using crypto/rand and the stdlib only — no google/uuid. crypto/rand.Read in modern Go is backed by a fast userspace CSPRNG, so this stays cheap.
func NormalizeMessage ¶ added in v0.1.16
NormalizeMessage returns the message with per-occurrence data replaced by placeholders: uuids → <uuid>, long hex ids → <hex>, quoted literals → <q>, numbers → <n>; whitespace collapsed; SQLSTATE codes preserved.
func ParseUserAgent ¶
ParseUserAgent extracts a coarse browser and OS from a User-Agent string using stdlib-only substring matching — no external dependency, ~1µs. Unknown values fall back to "Unknown".
Note: Android UAs also contain "Linux" (Android is Linux-based) and iOS UAs contain "like Mac OS X", so Android/iOS are matched BEFORE Linux/macOS.
func ShouldPersistTrace ¶
ShouldPersistTrace reports whether a request's trace should be written to slow_traces. A trace is persisted when it is either slow (DurUS above the threshold) or — when PersistErrors — an error response (Status >= 400).
(A third heuristic, "any span with dur_us == 0", was considered for detecting a prematurely-cut pipeline but rejected: sub-microsecond stages such as the RBAC map lookup legitimately round to 0µs, so it would persist almost every request. "How far the pipeline got" is instead conveyed by which spans are present — error paths now mark their own stage.)
func TopFrame ¶ added in v0.1.16
TopFrame returns the first application frame of a stack ("" when none).
func TraceIDFromCtx ¶
TraceIDFromCtx returns the trace id from ctx, or "" if none is set.
func WithSpanTracker ¶
WithSpanTracker returns a context carrying a fresh SpanTracker.
Types ¶
type Alert ¶
type Alert struct {
TenantID string
Level string // LevelWarning | LevelCritical
Message string
BurnRate float64
P95ms float64
// New-error fields (Kind == KindNewError / KindStorm).
Kind string
Route string
TraceID string
Count int
}
Alert is a single notification.
type Alerter ¶
Alerter delivers SLO alerts somewhere (Slack, a no-op sink, …).
func NewSlackAlerterFromEnv ¶
func NewSlackAlerterFromEnv() Alerter
NewSlackAlerterFromEnv returns a SlackAlerter when SLACK_WEBHOOK_URL is set, or a NoopAlerter otherwise — so the server always starts cleanly without alerting config.
type AnomalyDetector ¶
type AnomalyDetector struct {
// contains filtered or unexported fields
}
AnomalyDetector flags requests whose latency is statistically anomalous using an exponentially-weighted moving average of mean and variance.
func NewAnomalyDetector ¶
func NewAnomalyDetector() *AnomalyDetector
func (*AnomalyDetector) GetCount ¶
func (d *AnomalyDetector) GetCount(tenantID string) int64
GetCount returns the total anomaly count for tenantID since startup.
func (*AnomalyDetector) IncrCounter ¶
func (d *AnomalyDetector) IncrCounter(tenantID string)
IncrCounter increments the anomaly count for tenantID by one.
func (*AnomalyDetector) Observe ¶
func (d *AnomalyDetector) Observe(tenantID string, ms float64) (bool, float64)
Observe updates the EWMA model for tenantID with a new latency sample (µs). Returns (true, z-score) when the sample is anomalous, (false, 0) otherwise.
func (*AnomalyDetector) RecentAnomalies ¶
func (d *AnomalyDetector) RecentAnomalies(tenantID string, n int) []AnomalyEvent
RecentAnomalies returns up to n of tenantID's most recent anomalies, newest first. Returns a non-nil empty slice when none are recorded, so it marshals to [] not null.
func (*AnomalyDetector) RecordAnomaly ¶
func (d *AnomalyDetector) RecordAnomaly(tenantID string, latencyUS, z float64)
RecordAnomaly increments the tenant's anomaly counter AND appends the event to its recent-anomaly ring. Call it (instead of IncrCounter) when Observe reports an anomaly: latencyUS is the offending request's latency and z the detected z-score. Off the common request path — runs only on a detection (z > zThreshold).
type AnomalyEvent ¶
type AnomalyEvent struct {
TS int64 `json:"ts"` // unix microseconds (matches trace TS)
LatencyUS float64 `json:"latency_us"` // request latency in microseconds
ZScore float64 `json:"z_score"`
}
AnomalyEvent is one detected latency anomaly, retained so the admin observability panel can show WHEN it happened, the request latency, and the z-score — data the detector already computes on detection but previously only logged + counted.
type Attribution ¶ added in v0.1.14
type Attribution string
Attribution is the verdict vocabulary, exactly as the spec defines it.
const ( AttrCPUSaturated Attribution = "cpu_saturated" AttrGCPressure Attribution = "gc_pressure" AttrCPUThrottled Attribution = "cpu_throttled" AttrPoolExhausted Attribution = "pool_exhausted" AttrDBBound Attribution = "db_bound" AttrMemoryPressure Attribution = "memory_pressure" AttrLockContention Attribution = "lock_contention" AttrHealthy Attribution = "healthy" )
type AttributionThresholds ¶ added in v0.1.14
type AttributionThresholds struct {
HighP99Ms float64 // absolute "slow" floor (default 50 ms)
BaselineRise float64 // p99 ≥ BaselineRise × healthy baseline is "slow" too (default 3×), if ≥ MinRiseMs
MinRiseMs float64 // (default 10 ms)
MinRPS float64 // below this the window is idle → healthy (default 1)
ThrottledFraction float64 // throttled_usec / interval ≥ this → cpu_throttled (default 0.02: 20 ms of every second stopped by the quota is a p99 the operator feels)
MemUseFraction float64 // memory.current / memory.max ≥ this → memory_pressure (default 0.90)
MemPSISome10 float64 // memory some avg10 ≥ this % → memory_pressure (default 10)
GCCPUFraction float64 // gc cpu / busy cpu ≥ this → gc_pressure (default 0.25)
GCPauseP99Ms float64 // gc pause p99 ≥ this with ≥ GCCyclesPerS → gc_pressure (default 5)
GCCyclesPerS float64 // (default 2)
SchedLatP99Ms float64 // sched latency p99 ≥ max(this, 5 % of the request p99) → cpu_saturated (default 2 ms: a 1-vCPU box shared with anything shows ~1 ms wakeup latency that explains nothing)
CPUPSISome10 float64 // cpu some avg10 ≥ this % corroborates (default 10)
CPUBusyFraction float64 // busy cpu / (interval × GOMAXPROCS) ≥ this corroborates when PSI is unavailable (default 0.85)
PoolWaitFraction float64 // empty_acquire_wait / interval ≥ this → pool_exhausted even without latency_high (default 0.10)
MutexWaitFraction float64 // mutex wait / interval ≥ this → lock_contention (default 0.10)
}
AttributionThresholds are the rule constants. Every one is documented in ADR-030 with why; the engine maps APPXIMO_SELFMON_P99_MS onto HighP99Ms (the one an operator most plausibly tunes: "what is slow for MY app").
type Check ¶
type Check struct {
Name string
URL string
Method string
Headers map[string]string
Expected int // expected HTTP status code; 0 = any 2xx
}
Check describes one synthetic health probe.
type CheckResult ¶
type CheckResult struct {
Name string `json:"name"`
Status string `json:"status"`
LatencyMs int64 `json:"latency_ms"`
LastCheck time.Time `json:"last_check"`
LastError string `json:"last_error,omitempty"`
Uptime float64 `json:"uptime_pct"`
// contains filtered or unexported fields
}
CheckResult is the last known state of a synthetic check.
type CooldownAlerter ¶
type CooldownAlerter struct {
// contains filtered or unexported fields
}
CooldownAlerter wraps another Alerter and drops repeat alerts for the same (tenant, level) pair that arrive within the cooldown window.
func NewCooldownAlerter ¶
func NewCooldownAlerter(inner Alerter, cooldown time.Duration) *CooldownAlerter
NewCooldownAlerter wraps inner so that each (tenant, level) fires at most once per cooldown.
type DBClientStats ¶ added in v0.1.14
type DBClientStats struct {
MaxConns int32 `json:"max_conns"`
TotalConns int32 `json:"total_conns"`
AcquiredConns int32 `json:"acquired_conns"`
IdleConns int32 `json:"idle_conns"`
ConstructingConns int32 `json:"constructing_conns"`
AcquireCount int64 `json:"acquire_count"`
AcquireDurationMs float64 `json:"acquire_duration_ms"` // cumulative, per pgxpool
EmptyAcquireCount int64 `json:"empty_acquire_count"`
EmptyAcquireWaitMs float64 `json:"empty_acquire_wait_ms"` // cumulative
CanceledAcquireCount int64 `json:"canceled_acquire_count"`
NewConnsCount int64 `json:"new_conns_count"`
MaxLifetimeDestroy int64 `json:"max_lifetime_destroy_count"`
MaxIdleDestroy int64 `json:"max_idle_destroy_count"`
NewConnsDelta int64 `json:"new_conns_delta"`
AcquireDelta int64 `json:"acquire_delta"`
EmptyAcquireDelta int64 `json:"empty_acquire_delta"`
EmptyAcquireWaitDelta float64 `json:"empty_acquire_wait_delta_ms"`
AcquireWaitDeltaMs float64 `json:"acquire_duration_delta_ms"`
CanceledAcquireDelta int64 `json:"canceled_acquire_delta"`
QueryLatencyP50Ms float64 `json:"query_latency_p50_ms"` // client-side query stage, this tick (the "query" span)
QueryLatencyP99Ms float64 `json:"query_latency_p99_ms"`
QueryCount int64 `json:"query_count"`
Saturated bool `json:"saturated"` // acquired == max && idle == 0
// Warming is the COLD-START of the pool, not a wall: the pool has not
// reached MaxConns yet and it opened at least one connection during this
// tick, so the goroutines that "found no free connection" were waiting for
// a connection to be CONSTRUCTED (TCP + TLS + auth + the session's SET),
// which every process pays exactly once per connection. The pool_exhausted
// rule ignores a warming tick (CAPACIDAD-USL-S1): the first second of every
// load run used to read pool_exhausted and a false positive in the first
// tick of every run poisons every later reading of the same series.
Warming bool `json:"warming"`
}
DBClientStats is layer 4a — pgxpool.Stat(), always observable, remote DB included. EmptyAcquireCount is THE pool signal: a goroutine asked for a connection and the pool had none idle.
type DBServerProbe ¶ added in v0.1.14
type DBServerProbe func(ctx context.Context, out *DBServerStats) error
DBServerProbe runs the server-side statement (pg_stat_database + pg_database_size + pg_stat_activity counts) and fills out. It MUST bound its own connection acquire (the engine's implementation uses a 250 ms timeout on pool.Acquire and returns an error the collector reports as "skipped: pool busy") — the probe never competes with requests for a connection.
type DBServerStats ¶ added in v0.1.14
type DBServerStats struct {
Observable bool `json:"observable"`
Reason string `json:"reason,omitempty"` // why not observable / why skipped this tick
ProbedAt int64 `json:"probed_at,omitempty"`
DBSizeBytes int64 `json:"db_size_bytes"`
CacheHitRatio float64 `json:"cache_hit_ratio"` // blks_hit / (blks_hit + blks_read), cumulative
BlksHit int64 `json:"blks_hit"`
BlksRead int64 `json:"blks_read"`
XactCommit int64 `json:"xact_commit"`
XactRollback int64 `json:"xact_rollback"`
Deadlocks int64 `json:"deadlocks"`
TempBytes int64 `json:"temp_bytes"`
ActiveConns int64 `json:"active_conns"`
IdleInTx int64 `json:"idle_in_transaction"`
Waiting int64 `json:"waiting"` // active backends with a wait_event
TotalBackends int64 `json:"total_backends"`
StatementsExt bool `json:"pg_stat_statements"` // extension present
}
DBServerStats is layer 4b — pg_stat_* views, ONLY when the database is local. Observable=false with a Reason is a correct answer, not a gap.
type DynamicCheck ¶
DynamicCheck resolves its probe lazily on every tick. Resolve returns the Check to run, or nil plus a human-readable reason — reported as status "pending" instead of a failing probe. This exists for canaries whose target only makes sense once external state exists (e.g. an API canary on a fresh install, before any tenant is registered: probing would just spam 4xx for a route nobody created).
type ErrGroup ¶
type ErrGroup struct {
Count atomic.Int64
FirstSeen int64
LastSeen atomic.Int64
Message string
Stack []Frame // symbolized only on first occurrence
// contains filtered or unexported fields
}
ErrGroup aggregates repeated occurrences of the same logical error.
type ErrorCapture ¶
type ErrorCapture struct {
TraceID string `json:"trace_id"`
TenantID string `json:"tenant_id"`
Route string `json:"route"`
Method string `json:"method"`
ErrMsg string `json:"error_msg"`
Stack []Frame `json:"stack"`
// SQL is the exact statement the driver rejected, when the error came
// from the database (the QueryTracer noted it on the request's tracker).
SQL string `json:"sql,omitempty"`
UserID string `json:"user_id"`
Role string `json:"role"`
Timestamp int64 `json:"ts"`
}
ErrorCapture is a snapshot of a server error (HTTP 500) linked to the request's trace, so the stack trace can be shown next to the timeline. For client errors (4xx) the Stack is empty — those are not server bugs and never pay for runtime.Callers. UserID/Role/Route/Method are filled by the caller (the handler) because observability must not import auth (auth imports observability).
func CaptureError ¶
func CaptureError(ctx context.Context, err error) ErrorCapture
CaptureError snapshots the current goroutine's stack for a server error (500) and links it to the request's trace_id and tenant (from ctx). It symbolizes the stack at most once per call site; repeat occurrences reuse the cached frames with ZERO allocations.
The hit path captures only 3 PCs into a stack-local array used solely for the fingerprint (so nothing escapes to the heap); the expensive full-stack capture + symbolization lives in captureAndSymbolize, called only on a cache miss, so its unavoidable PC-slice escape never taxes the common (repeat) path.
Returned BY VALUE so a caller that discards it allocates nothing. NEVER call on the happy path — even a hit pays for runtime.Callers.
func CaptureFromPCs ¶ added in v0.1.16
func CaptureFromPCs(ctx context.Context, err error, pcs []uintptr) ErrorCapture
CaptureFromPCs builds a capture from program counters collected EARLIER, at the site that mattered — a Ctx.Error(5xx, …) call inside a handler, or the Ctx database method the handler called — rather than in the middleware that eventually writes the response (whose own stack would only name the middleware). Symbolized once per (message, site) like CaptureError.
type ErrorGroup ¶ added in v0.1.16
type ErrorGroup struct {
Fingerprint uint64 `json:"fingerprint"`
Route string `json:"route"`
Status int `json:"status"`
Message string `json:"message"`
TopFrame string `json:"top_frame,omitempty"`
FirstSeen int64 `json:"first_seen"` // unix µs
LastSeen int64 `json:"last_seen"`
Count int64 `json:"count"`
Users []string `json:"users"`
SampleTraceID string `json:"sample_trace_id"`
}
ErrorGroup is one row of error_groups — a defect, not an occurrence.
type ErrorStore ¶
type ErrorStore struct {
// contains filtered or unexported fields
}
ErrorStore deduplicates errors per tenant by fingerprinting type + message prefix + call site.
func NewErrorStore ¶
func NewErrorStore() *ErrorStore
func (*ErrorStore) Record ¶
func (es *ErrorStore) Record(tenantID string, err error)
Record fingerprints err, captures a cheap PC snapshot on every call, and symbolizes the stack exactly once per unique fingerprint via stackOnce.
type Frame ¶
type Frame struct {
Function string `json:"function"`
File string `json:"file"`
Line int `json:"line"`
}
Frame is a single symbolized stack frame.
type FullSnapshot ¶
type FullSnapshot struct {
Cached *PercentileSnapshot `json:"cached,omitempty"`
Uncached *PercentileSnapshot `json:"uncached,omitempty"`
}
FullSnapshot splits latency into cache-hit vs cache-miss buckets.
type GeoLookup ¶
type GeoLookup struct {
// contains filtered or unexported fields
}
GeoLookup resolves an IP to an ISO country code using an in-memory GeoLite2 database. Lookups are ~1µs and never touch the network. A GeoLookup whose database failed to load returns "" for everything — graceful degradation, so a missing/corrupt mmdb never blocks startup or panics.
func DefaultGeoLookup ¶
func DefaultGeoLookup() *GeoLookup
DefaultGeoLookup builds a lookup from the embedded GeoLite2 database.
func NewGeoLookup ¶
NewGeoLookup builds a lookup from a GeoLite2-Country mmdb buffer. An empty or invalid buffer yields a usable no-op GeoLookup (Country → "") rather than an error.
type HistoryPoint ¶
type HistoryPoint struct {
TS int64 `json:"ts"`
P50US int64 `json:"p50_us"`
P95US int64 `json:"p95_us"`
BurnRate float64 `json:"burn_rate"`
ErrorRatio float64 `json:"error_ratio"`
SLOStatus string `json:"slo_status"`
}
HistoryPoint is the trimmed per-snapshot projection returned under "history". BurnRate/ErrorRatio come straight from the persisted snapshot so the panel can plot the SLO burn-rate over time (with the multi-window thresholds overlaid) without a second query.
type Metrics ¶
type Metrics struct {
// contains filtered or unexported fields
}
Metrics holds the four Prometheus collectors described in DEPLOYMENT_INFRA.md, backed by a dedicated registry rather than the global default. The dedicated registry keeps construction side-effect-free (safe to build more than once, e.g. in tests) and scopes /metrics output to exactly these series plus the standard Go/process collectors.
func (*Metrics) Gatherer ¶
func (m *Metrics) Gatherer() prometheus.Gatherer
Gatherer exposes the dedicated registry as a prometheus.Gatherer. It enables end-to-end metric assertions (prometheus/testutil.GatherAndCompare) against exactly this Metrics' series, without reaching for the global default registry. Like Handler(), it carries no auth of its own.
func (*Metrics) Handler ¶
Handler returns the Prometheus exposition handler scoped to this registry. It carries no auth of its own — mount it behind the admin-key middleware.
func (*Metrics) IncGoroutinePanic ¶
func (m *Metrics) IncGoroutinePanic()
IncGoroutinePanic records one panic recovered inside a Ctx.SafeGo goroutine.
func (*Metrics) IncRequestPanic ¶
func (m *Metrics) IncRequestPanic()
IncRequestPanic records one panic recovered by the request-chain Recoverer.
func (*Metrics) ObserveMigration ¶
ObserveMigration records the duration and outcome of a tenant migration.
func (*Metrics) ObserveRequest ¶
ObserveRequest records one served request: increments the counter and observes the duration histogram. path should be the chi route pattern (e.g. "/api/{entity}") rather than the raw URL path, to keep label cardinality bounded.
func (*Metrics) Register ¶ added in v0.1.14
func (m *Metrics) Register(c prometheus.Collector) error
Register adds a collector to the dedicated registry — how the resource collector's gauges reach /metrics without a second registry or endpoint.
func (*Metrics) SetActiveTenants ¶
SetActiveTenants updates the gauge of tenants currently loaded in cache.
type MinuteBucket ¶
MinuteBucket is one minute of aggregated traffic. Three uint32 fields, stored in a fixed array — no per-minute allocations.
type NewErrorNotifier ¶ added in v0.1.16
type NewErrorNotifier struct {
// contains filtered or unexported fields
}
NewErrorNotifier turns "this fingerprint was never seen for this tenant" into ONE alert — at the first occurrence, not when the SLO budget burns (a systematic, reproducible 500 used to generate nothing until then). The brake: at most maxPerMinute new-group alerts per tenant per minute; past that, the notifier sends ONE storm summary per stormCooldown naming how many groups it suppressed. A thousand new groups in a deploy gone wrong is one message, not a thousand.
func NewNewErrorNotifier ¶ added in v0.1.16
func NewNewErrorNotifier(inner Alerter, maxPerMinute int, stormCooldown time.Duration) *NewErrorNotifier
NewNewErrorNotifier wraps inner with the per-tenant brake.
type NoopAlerter ¶
type NoopAlerter struct{}
NoopAlerter discards alerts (used when no SLACK_WEBHOOK_URL is configured).
type ObsServer ¶
type ObsServer struct {
// contains filtered or unexported fields
}
ObsServer exposes internal observability data over HTTP.
func NewObsServer ¶
func NewObsServer( hist *TenantHistogram, errors *ErrorStore, anomaly *AnomalyDetector, synthmon *SyntheticMonitor, rings *Rings, slo *SLOEngine, store *ObsStore, ) *ObsServer
func (*ObsServer) DebugRouter ¶
DebugRouter returns an admin-gated sub-router serving /tenant/{id} and /synthetic. Mount it at "/debug" on any router (the main :8080 server or the control plane).
func (*ObsServer) Resources ¶ added in v0.1.14
func (s *ObsServer) Resources() *ResourceCollector
Resources returns the wired collector (nil when self-monitoring is off).
func (*ObsServer) Router ¶
Router returns a chi.Mux exposing the /debug/... paths, admin-protected. Mount at "/" on an existing router (used by the control plane on :9090).
func (*ObsServer) ServeResources ¶ added in v0.1.14
func (s *ObsServer) ServeResources(w http.ResponseWriter, r *http.Request)
ServeResources answers the live board + the correlation series.
func (*ObsServer) ServeResourcesSnapshot ¶ added in v0.1.14
func (s *ObsServer) ServeResourcesSnapshot(w http.ResponseWriter, r *http.Request)
ServeResourcesSnapshot is the exportable document of a run (spec §5, §7): engine + host identity, the window verdict, every tick. Served as an attachment so the browser saves it.
func (*ObsServer) ServeTenantData ¶
func (s *ObsServer) ServeTenantData(w http.ResponseWriter, r *http.Request)
ServeTenantData serves the per-tenant observability JSON (the same payload as the admin-gated GET /debug/tenant/{id}), reading the tenant id from the chi "id" URL param. It performs NO authorization itself — the caller (e.g. the admin API in pkg/platformadmin) is responsible for authorizing the request first (platform super-admin → any tenant; tenant admin → its own). This exists so the observability logic is REUSED with a different authorization gate, never duplicated. The data is already tenant-scoped (filtered by tenant_id), so no cross-tenant data can leak through it (R5 of the persistence audit).
func (*ObsServer) SetGeo ¶
SetGeo installs the GeoLite2 lookup used to enrich recent_traces with country.
func (*ObsServer) SetResources ¶ added in v0.1.14
func (s *ObsServer) SetResources(rc *ResourceCollector)
SetResources wires the collector into the obs server (nil = not enabled).
func (*ObsServer) SetTracesHandler ¶
SetTracesHandler installs the handler served at /debug/traces. It is injected from package main (where the HTML is go:embed'd and the admin key lives) so the route applies ITS OWN auth — query param (?key=) OR X-Admin-Key header — which is intentionally looser than the header-only gate on the JSON debug APIs, so the page can be opened directly in a browser.
type ObsStore ¶
type ObsStore struct {
// contains filtered or unexported fields
}
ObsStore persists observability snapshots to a local SQLite file (modernc, no CGO).
func OpenStore ¶
OpenStore opens (creating if needed) the SQLite store at path and ensures the schema exists. An empty path selects defaultObsDBPath (a persistent location). The parent directory is created if missing; if it cannot be created or written, the store falls back to an ephemeral temp file so observability still works — a bad path is logged as a WARNING, never a boot failure. A resolved path on an ephemeral filesystem (/tmp or a tmpfs) is also logged, so the operator knows the history will not survive a restart.
func (*ObsStore) ErrorGroups ¶ added in v0.1.16
func (s *ObsStore) ErrorGroups(tenantID string, hours int) ([]ErrorGroup, error)
ErrorGroups lists a tenant's groups seen in the last `hours`, most recent first (capped at 100).
func (*ObsStore) History ¶
History returns the tenant's snapshots from the last `hours`, newest first.
func (*ObsStore) Path ¶
Path returns the resolved on-disk location the store opened. It may differ from the requested path when an unwritable directory forced the ephemeral fallback.
func (*ObsStore) Prune ¶
Prune enforces retention: it deletes snapshots and slow traces older than the retention window, AND caps slow_traces at maxSlowTraceRows (oldest dropped). It must be called periodically, not only at startup — otherwise a long-running server never reclaims space.
func (*ObsStore) SaveSlowTrace ¶
SaveSlowTrace persists a single slow request's trace (id, route, total, span breakdown). tv.TS is the request start in unix microseconds, consistent with the ring's Sample.Start. Designed to be called asynchronously from the request path so it never blocks the response.
func (*ObsStore) SlowTraces ¶
SlowTraces returns the tenant's persisted slow traces from the last `hours`, newest first (capped at 200), with spans decoded back into TraceViews.
func (*ObsStore) UpsertErrorGroup ¶ added in v0.1.16
func (s *ObsStore) UpsertErrorGroup(tenantID string, g ErrorGroup, userID string) (isNew bool, err error)
UpsertErrorGroup records one occurrence of a fingerprint for a tenant and reports whether the group is NEW (first occurrence ever for this tenant) — the signal the first-occurrence alert fires on. One transaction, so two concurrent occurrences of a brand-new group cannot both read "new".
type PSILine ¶ added in v0.1.14
type PSILine struct {
SomeAvg10 float64 `json:"some_avg10"`
SomeAvg60 float64 `json:"some_avg60"`
SomeAvg300 float64 `json:"some_avg300"`
SomeTotal int64 `json:"some_total_usec"`
FullAvg10 float64 `json:"full_avg10"`
FullAvg60 float64 `json:"full_avg60"`
FullAvg300 float64 `json:"full_avg300"`
FullTotal int64 `json:"full_total_usec"`
}
PSILine is one PSI resource (cpu | memory | io): the "some" and "full" stall percentages over 10 / 60 / 300 s windows plus the cumulative totals.
type PercentileSnapshot ¶
type PercentileSnapshot struct {
P50Us float64 `json:"p50_us"`
P95Us float64 `json:"p95_us"`
P99Us float64 `json:"p99_us"`
P999Us float64 `json:"p999_us"`
Count int64 `json:"count"`
Mean float64 `json:"mean_us"`
}
PercentileSnapshot holds a point-in-time latency summary for one bucket. TenantID is intentionally absent — the response envelope provides it.
type PoolStat ¶ added in v0.1.14
type PoolStat struct {
MaxConns int32
TotalConns int32
AcquiredConns int32
IdleConns int32
ConstructingConns int32
AcquireCount int64
AcquireDuration time.Duration
EmptyAcquireCount int64
EmptyAcquireWaitTime time.Duration
CanceledAcquireCount int64
NewConnsCount int64
MaxLifetimeDestroyCount int64
MaxIdleDestroyCount int64
}
PoolStat is the subset of pgxpool.Stat the collector reads (pgx v5 names).
type PressureStats ¶ added in v0.1.14
type PressureStats struct {
Source string `json:"source"` // "cgroup" | "host" | "unavailable"
CPU PSILine `json:"cpu"`
Memory PSILine `json:"memory"`
IO PSILine `json:"io"`
// The spec's §7 shorthand, duplicated for the correlation chart.
CPUSomeAvg10 float64 `json:"cpu_some_avg10"`
MemSomeAvg10 float64 `json:"mem_some_avg10"`
IOSomeAvg10 float64 `json:"io_some_avg10"`
}
PressureStats is layer 3 — PSI. Source says WHOSE pressure it is: the process's own cgroup (preferred — inside a container the host view may be the whole host or nothing) or the host's /proc/pressure/*.
type ProcessStats ¶ added in v0.1.14
type ProcessStats struct {
Source string `json:"source"` // "cgroup" | "proc" | "unavailable"
CgroupPath string `json:"cgroup_path"` // relative to /sys/fs/cgroup
MemCurrentBytes int64 `json:"mem_current_bytes"`
MemMaxBytes int64 `json:"mem_max_bytes"` // -1 = "max" (unlimited)
MemPeakBytes int64 `json:"mem_peak_bytes"` // -1 = unavailable (memory.peak is Linux 5.19+); /proc fallback uses VmHWM
MemSwapBytes int64 `json:"mem_swap_bytes"`
CPUUsageUsec int64 `json:"cpu_usage_usec"`
CPUUserUsec int64 `json:"cpu_user_usec"`
CPUSystemUsec int64 `json:"cpu_system_usec"`
CPUNrPeriods int64 `json:"cpu_nr_periods"`
CPUNrThrottled int64 `json:"cpu_nr_throttled"`
CPUThrottledUsec int64 `json:"cpu_throttled_usec"`
CPUUsageDeltaUsec int64 `json:"cpu_usage_delta_usec"`
CPUThrottledDelta int64 `json:"cpu_throttled_delta_usec"`
CPUNrThrottledDlt int64 `json:"cpu_nr_throttled_delta"`
CPUQuotaUsec int64 `json:"cpu_quota_usec"` // cpu.max quota; -1 = "max" (no quota)
CPUPeriodUsec int64 `json:"cpu_period_usec"` // cpu.max period
PidsCurrent int64 `json:"pids_current"`
PidsMax int64 `json:"pids_max"` // -1 = max
Threads int64 `json:"threads"` // /proc/self/status Threads (runtime/metrics has no thread count)
RSSBytes int64 `json:"rss_bytes"` // /proc/self/status VmRSS — always read (cheap), the classic number
// > this process's thread count): a login session scope, a container
// running a supervisor + the app. Then memory.current and cpu.stat are the
// cgroup's, not the process's — the cards say so and lean on RSS. A
// systemd service unit (the production layout) is never shared.
CgroupShared bool `json:"cgroup_shared"`
}
ProcessStats is layer 2 — the process seen through its cgroup v2 (or /proc/self when there is none). Cumulative counters carry their tick delta.
type QueryTracer ¶ added in v0.1.16
type QueryTracer struct{}
QueryTracer is the pgx.QueryTracer the engine's pool runs with (OBSERVABILIDAD-ERRORES-S1). It does ONE thing and does it for every route, generated or custom, because it hangs off the driver and not off a handler: when a statement FAILS, it notes the exact SQL and the driver's message on the request's SpanTracker, so a 5xx capture can say WHICH query broke — the piece the field report ("EL 500 MUDO") had to guess. Bound parameter VALUES are never recorded (they can be personal data; the SQL template names the column, which is what locates the bug).
Cost on the happy path: one context lookup per query start (returns ctx unchanged — no allocation) and one nil check + one error check at the end.
func (QueryTracer) TraceQueryEnd ¶ added in v0.1.16
func (QueryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData)
TraceQueryEnd notes a failed statement on the request's tracker.
func (QueryTracer) TraceQueryStart ¶ added in v0.1.16
func (QueryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context
TraceQueryStart remembers the statement about to run on the request's tracker (a string assignment — pgx hands the SQL in Start and only the verdict in End) and returns ctx unchanged: no allocation on the way in.
type RecentRequest ¶
type RecentRequest struct {
StartUS int64 `json:"start_us"`
DurUS int32 `json:"dur_us"`
QueryUS int32 `json:"query_us"`
Route string `json:"route"`
Status uint16 `json:"status"`
}
RecentRequest is the JSON-friendly projection of a Sample with its route id resolved back to a human-readable pattern.
type RequestStats ¶ added in v0.1.14
type RequestStats struct {
Count int64 `json:"count"`
RPS float64 `json:"rps"`
LatencyP50Ms float64 `json:"latency_p50_ms"`
LatencyP95Ms float64 `json:"latency_p95_ms"`
LatencyP99Ms float64 `json:"latency_p99_ms"`
LatencyMaxMs float64 `json:"latency_max_ms"`
Errors5xx int64 `json:"errors_5xx"`
Status429 int64 `json:"status_429"` // shed by the tenant limiter — load the box refused, on purpose
Status503 int64 `json:"status_503"` // shed by the breaker / memory guard
}
RequestStats is the request path's own view of the tick: throughput and the latency histogram of the requests that FINISHED in the window.
type ResourceCollector ¶ added in v0.1.14
type ResourceCollector struct {
// contains filtered or unexported fields
}
ResourceCollector owns the four layers, the windowed request histograms, the ring and the one goroutine that reads everything.
func NewResourceCollector ¶ added in v0.1.14
func NewResourceCollector(cfg ResourceConfig) *ResourceCollector
NewResourceCollector builds a collector. It does NOT start reading: call Run in a goroutine (the engine does it from startBackground). Nothing is read at construction either, so an app that disables self-monitoring pays only the struct.
func (*ResourceCollector) Config ¶ added in v0.1.14
func (c *ResourceCollector) Config() ResourceConfig
Config returns the effective configuration (defaults applied).
func (*ResourceCollector) Count ¶ added in v0.1.14
func (c *ResourceCollector) Count() int
Count reports how many ticks the ring holds (≤ ResourceRingSize).
func (*ResourceCollector) FootprintBytes ¶ added in v0.1.14
func (c *ResourceCollector) FootprintBytes() int64
FootprintBytes is the collector's fixed memory: the ring array plus the four windowed histograms — the A-54 RSS proxy, declared in bytes (the ring slots' evidence slices are allocated per tick on the heap and counted separately by the test that reports this).
func (*ResourceCollector) Interval ¶ added in v0.1.14
func (c *ResourceCollector) Interval() time.Duration
Interval returns the cadence in force for the next tick.
func (*ResourceCollector) Latest ¶ added in v0.1.14
func (c *ResourceCollector) Latest() *ResourceSnapshot
Latest returns a copy of the most recent snapshot with its verdict described, or nil before the first tick. The copy (and the sentence) are the READER's cost, never the tick's.
func (*ResourceCollector) Live ¶ added in v0.1.14
func (c *ResourceCollector) Live() bool
Live reports whether the collector is currently on the live cadence.
func (*ResourceCollector) Observe ¶ added in v0.1.14
func (c *ResourceCollector) Observe(durUS, queryUS int64, status int)
Observe is THE request-path entry point. It is called from the logging tap once per finished request with the total duration, the "query" stage duration (0 when the request ran no query) and the status. Cost: two atomic adds and one or two HDR RecordValue under a mutex. No allocation.
func (*ResourceCollector) PromCollector ¶ added in v0.1.14
func (c *ResourceCollector) PromCollector() prometheus.Collector
PromCollector returns the Prometheus projection of this collector.
func (*ResourceCollector) Run ¶ added in v0.1.14
func (c *ResourceCollector) Run(ctx context.Context)
Run is the collector goroutine: ONE timer, one tick at a time, until ctx ends. It is the only place the four layers are read.
func (*ResourceCollector) Series ¶ added in v0.1.14
func (c *ResourceCollector) Series(n int) []ResourceSnapshot
Series copies out the last n snapshots, oldest first (n ≤ ResourceRingSize).
func (*ResourceCollector) SetDB ¶ added in v0.1.14
func (c *ResourceCollector) SetDB(stat func() PoolStat, probe DBServerProbe)
SetDB wires layer 4. stat is pgxpool's Stat (a func so this package does not import pgx); probe runs the server-side statement when the database is declared local (nil ⇒ never). Call before Run.
func (*ResourceCollector) Started ¶ added in v0.1.14
func (c *ResourceCollector) Started() bool
Started reports whether Run has been entered (the admin surface answers 503 with a reason until then, never an empty snapshot dressed as data).
func (*ResourceCollector) Touch ¶ added in v0.1.14
func (c *ResourceCollector) Touch()
Touch switches the collector to live cadence (1 s by default) for LiveWindow after the call. The /admin correlation view calls it on every poll, so the 1 s series exists exactly while someone is looking at it.
type ResourceConfig ¶ added in v0.1.14
type ResourceConfig struct {
BackgroundInterval time.Duration
LiveInterval time.Duration
LiveWindow time.Duration
// DBServerLocal declares that Postgres runs on THIS host (loopback / unix
// socket DSN): only then does the collector read pg_stat_* — a remote
// database's internals are "not observable from the app", by design.
DBServerLocal bool
// DBServerEvery bounds the server-side probe cadence (default 10 s): it
// borrows ONE pool connection with a 250 ms acquire timeout and gives up
// (reported as skipped) rather than compete with requests for the pool.
DBServerEvery time.Duration
// Thresholds for the attribution rules; zero values take the documented
// defaults (see attribution.go).
Thresholds AttributionThresholds
}
ResourceConfig configures a ResourceCollector. Zero values take the defaults above; the engine maps APPXIMO_SELFMON_INTERVAL / _LIVE_INTERVAL onto it.
type ResourceSnapshot ¶ added in v0.1.14
type ResourceSnapshot struct {
TS int64 `json:"ts"` // unix milliseconds
IntervalMs int64 `json:"interval_ms"`
Mode string `json:"mode"` // "live" | "background"
Runtime RuntimeStats `json:"runtime"`
Process ProcessStats `json:"process_cgroup"`
Pressure PressureStats `json:"pressure"`
DBClient DBClientStats `json:"db_client"`
DBServer DBServerStats `json:"db_server_local_only"`
Request RequestStats `json:"request"`
// Attribution is the verdict of the §4 table for THIS tick; Verdict
// carries its reason and the signals that fired.
Attribution Attribution `json:"attribution"`
Verdict Verdict `json:"verdict"`
}
ResourceSnapshot is one tick: the four layers + the request view + the verdict (the §7 data model of the spec, with the deltas the rules use).
func (*ResourceSnapshot) Describe ¶ added in v0.1.14
func (s *ResourceSnapshot) Describe()
Describe fills the read-side half of the verdict — the evidence (Signals, Also) and the sentence — from the snapshot's numbers, with the default thresholds. Idempotent; call it on a COPY of a ring slot (Latest and Series do, with the collector's own thresholds), never on the slot itself.
type Rings ¶
type Rings struct {
// contains filtered or unexported fields
}
Rings holds one TenantRing per tenant plus a route-pattern interner that maps variable-length route strings to compact uint16 ids stored in each Sample.
func (*Rings) Count ¶
Count returns the number of distinct tenants that have recorded at least one request.
func (*Rings) Recent ¶
func (rs *Rings) Recent(tenantID string) []RecentRequest
Recent returns the tenant's recent requests as RecentRequest values, ordered most-recent first, with route ids resolved to strings.
func (*Rings) RecentTraces ¶
RecentTraces returns up to n of the tenant's most recent requests projected as TraceViews (trace id resolved to hex, route resolved to string, spans copied out of the fixed array).
func (*Rings) Record ¶
Record appends a sample to the given tenant's ring, creating the ring on first use.
func (*Rings) RouteID ¶
RouteID returns a stable uint16 id for route, interning it on first sight. Reads are lock-free-ish (RLock); only the first occurrence of a route takes the write lock. Ids saturate at the uint16 max to avoid overflow wraparound.
type RuntimeStats ¶ added in v0.1.14
type RuntimeStats struct {
SchedLatencyP99S float64 `json:"sched_latency_p99_s"` // /sched/latencies:seconds, p99 of the tick's delta — the CPU-saturation signal
GCPauseTotalP99S float64 `json:"gc_pause_total_p99_s"` // /sched/pauses/total/gc:seconds, p99 of the tick's delta
Goroutines uint64 `json:"goroutines"` // /sched/goroutines:goroutines
GOMAXPROCS uint64 `json:"gomaxprocs"` // /sched/gomaxprocs:threads
MutexWaitTotalS float64 `json:"mutex_wait_total_s"` // /sync/mutex/wait/total:seconds (cumulative)
MutexWaitDeltaS float64 `json:"mutex_wait_delta_s"` // this tick
HeapObjectsBytes uint64 `json:"heap_objects_bytes"` // /memory/classes/heap/objects:bytes (live heap)
MemoryTotalBytes uint64 `json:"memory_total_bytes"` // /memory/classes/total:bytes (everything the runtime mapped)
HeapGoalBytes uint64 `json:"heap_goal_bytes"` // /gc/heap/goal:bytes
GOGCPercent uint64 `json:"gogc_percent"` // /gc/gogc:percent
GOMEMLIMITBytes uint64 `json:"gomemlimit_bytes"` // /gc/gomemlimit:bytes (math.MaxInt64 = unset)
GCCyclesTotal uint64 `json:"gc_cycles_total"` // /gc/cycles/total:gc-cycles (cumulative)
GCCyclesDelta uint64 `json:"gc_cycles_delta"` // this tick
CPUUserS float64 `json:"cpu_user_s"` // /cpu/classes/user:cpu-seconds (cumulative)
CPUGCS float64 `json:"cpu_gc_s"` // /cpu/classes/gc/total:cpu-seconds (cumulative)
CPUTotalS float64 `json:"cpu_total_s"` // /cpu/classes/total:cpu-seconds (cumulative)
CPUIdleS float64 `json:"cpu_idle_s"` // /cpu/classes/idle:cpu-seconds (cumulative)
CPUScavengeS float64 `json:"cpu_scavenge_s"` // /cpu/classes/scavenge/total:cpu-seconds (cumulative)
CPUTotalDeltaS float64 `json:"cpu_total_delta_s"` // this tick
CPUGCDeltaS float64 `json:"cpu_gc_delta_s"` // this tick
GCCPUFraction float64 `json:"gc_cpu_fraction"` // cpu_gc_delta / (cpu_total_delta − cpu_idle_delta), this tick
CPUBusyFraction float64 `json:"cpu_busy_fraction"` // (cpu_total_delta − cpu_idle_delta) / (interval × GOMAXPROCS)
SchedLatencyP50S float64 `json:"sched_latency_p50_s"` // for the chart
GCPauseTotalMaxS float64 `json:"gc_pause_total_max_s"` // the tick's longest STW
SchedOtherPauseS float64 `json:"sched_pause_other_p99_s"`
}
RuntimeStats is layer 1 — the Go runtime, per tick. Cumulative fields are the runtime's counters; *_delta / p99 fields are computed over THIS tick.
type SLOConfig ¶
type SLOConfig struct {
TargetUptime float64 // e.g. 0.999 (99.9%)
LatencySLOms float64 // e.g. 100.0 ms — requests slower than this count as errors
AlertCooldown time.Duration // e.g. 15min between repeat alerts of the same level
}
SLOConfig holds the per-tenant objectives and alert cadence.
func DefaultSLOConfig ¶
func DefaultSLOConfig() SLOConfig
DefaultSLOConfig is the objective applied to every tenant unless overridden.
type SLOEngine ¶
type SLOEngine struct {
// contains filtered or unexported fields
}
SLOEngine rolls the per-tenant request ring up into minute buckets every 60s, evaluates multi-window burn rates, and fires alerts through alerter. It is fully independent of the AnomalyDetector.
func NewSLOEngine ¶
func NewSLOEngine(rings *Rings, hist *TenantHistogram, alerter Alerter) *SLOEngine
NewSLOEngine builds an engine with the default SLO config. The provided alerter is wrapped in a CooldownAlerter so each (tenant, level) fires at most once per cooldown.
func NewSLOEngineWithConfig ¶
func NewSLOEngineWithConfig(rings *Rings, hist *TenantHistogram, alerter Alerter, cfg SLOConfig) *SLOEngine
NewSLOEngineWithConfig is NewSLOEngine with an explicit config (used in tests to shrink the cooldown window).
func (*SLOEngine) Run ¶
Run evaluates all tenants immediately, then every 60s until ctx is cancelled.
func (*SLOEngine) Snapshot ¶
func (e *SLOEngine) Snapshot(tid string) SLOSnapshot
Snapshot returns the current SLO state for a tenant (used by /debug and tests).
type SLOSnapshot ¶
type SLOSnapshot struct {
ErrorRatio5m float64 `json:"error_ratio_5m"`
BurnRate5m float64 `json:"burn_rate_5m"`
ErrorRatio1h float64 `json:"error_ratio_1h"`
BurnRate1h float64 `json:"burn_rate_1h"`
Status string `json:"status"`
}
SLOSnapshot is the JSON projection exposed under /debug/tenant/{id} → "slo".
type Sample ¶
type Sample struct {
Start int64
DurUS int32
QueryUS int32
Route uint16
Status uint16
TraceID [8]byte
Spans [maxSpans]Span
NSpans uint8
// ErrMsg is the error message for an errored request ("" otherwise).
// ErrorCapture carries the symbolized stack for a 500 (nil otherwise) — a
// pointer, so non-error samples cost only 8 bytes and Record stays alloc-free.
ErrMsg string
ErrorCapture *ErrorCapture
// Client context (raw): browser/OS are parsed and country geo-resolved lazily
// in the projection, so the hot path only stores two strings.
IP string
UserAgent string
}
Sample is a fixed-size record of a single request. All fields are value types so a Sample never escapes to the heap and TenantRing.Record stays allocation-free.
Start request start time, unix microseconds DurUS total request duration, microseconds QueryUS time spent in DB queries, microseconds (0 when unmeasured) Route interned route-pattern id (resolve via Rings.RouteName) Status HTTP status code TraceID first 8 bytes of the request trace id (16 hex chars decoded) Spans per-stage durations captured by the request's SpanTracker NSpans number of valid entries in Spans
Spans/TraceID are value types too, so Record stays allocation-free.
type Signal ¶ added in v0.1.14
type Signal struct {
Name string `json:"name"`
Value float64 `json:"value"`
Threshold float64 `json:"threshold"`
Unit string `json:"unit"`
Fired bool `json:"fired"`
}
Signal is one measured value the rule read, with the threshold it was compared against — the evidence line under the verdict.
type SlackAlerter ¶
type SlackAlerter struct {
// contains filtered or unexported fields
}
SlackAlerter posts alerts to a Slack incoming-webhook URL. The HTTP client is the shared SSRF-safe egress client (blocks loopback/private/ link-local and refuses redirects), so a misconfigured webhook can't be used to reach internal services.
func NewSlackAlerter ¶
func NewSlackAlerter(webhookURL string) *SlackAlerter
NewSlackAlerter builds a SlackAlerter posting to webhookURL with a 5s timeout.
type Snapshot ¶
type Snapshot struct {
TenantID string `json:"tenant_id"`
TS int64 `json:"ts"` // unix seconds
P50US int64 `json:"p50_us"`
P95US int64 `json:"p95_us"`
ErrorRatio float64 `json:"error_ratio"`
BurnRate float64 `json:"burn_rate"`
SLOStatus string `json:"slo_status"`
}
Snapshot is one persisted point-in-time observation for a tenant.
type Span ¶
type Span struct {
Name string `json:"name"`
DurUS int32 `json:"dur_us"`
// Err marks the stage during which the request's error was recorded — the
// bar in the waterfall where the failure happened (OBSERVABILIDAD-ERRORES-S1).
// omitempty keeps the JSON of a healthy span byte-identical to before.
Err bool `json:"err,omitempty"`
}
Span is one timed stage within a request. Name is a constant literal (e.g. "jwt"), so storing it never allocates.
type SpanTracker ¶
type SpanTracker struct {
// HasError/ErrMsg are set on ANY error response (4xx/5xx) via RecordError.
// Capture is set only for 500s (carries the symbolized stack) via SetCapture.
HasError bool
ErrMsg string
Capture *ErrorCapture
// UserID/Role are set by the JWT middleware once the token is validated,
// so the request line and the trace can say WHO even though the claims
// live in a derived context the logger never sees.
UserID, Role string
// LastSQL/LastSQLErr are the most recent FAILED statement seen by the
// database tracer (QueryTracer) on this request — the exact query the
// driver rejected, kept as data (never logged here) for the 5xx capture.
// A handled failure (a 404 after a miss, a 409 after a unique violation)
// leaves them set but harmless: only the 5xx writers read them.
LastSQL string
LastSQLErr string
// contains filtered or unexported fields
}
SpanTracker records up to maxSpans stage durations for a single request. It is a fixed array (not a slice) so Mark never allocates and the tracker never escapes beyond the one heap allocation made by NewSpanTracker. It is NOT safe for concurrent Mark calls — but a request is handled by one goroutine through the middleware chain, so the spans are marked sequentially.
func NewSpanTracker ¶
func NewSpanTracker() *SpanTracker
NewSpanTracker starts a tracker whose clock begins now.
func SpanTrackerFromCtx ¶
func SpanTrackerFromCtx(ctx context.Context) *SpanTracker
SpanTrackerFromCtx returns the SpanTracker in ctx, or nil if none is set. Callers must nil-check before calling Mark.
func (*SpanTracker) ElapsedUS ¶ added in v0.1.11
func (t *SpanTracker) ElapsedUS() int64
ElapsedUS is the engine time spent on the request so far: the marked stages plus the tail since the last mark. TotalUS counts only the marks.
func (*SpanTracker) Failed ¶ added in v0.1.16
func (t *SpanTracker) Failed() bool
Failed reports whether some stage has already been marked as the failure.
func (*SpanTracker) Finish ¶
func (t *SpanTracker) Finish() []Span
Finish records a final "done" span (the tail since the last mark) and returns the recorded spans. The returned slice is backed by the tracker's array — no allocation — and must be copied if it needs to outlive the tracker.
func (*SpanTracker) Mark ¶
func (t *SpanTracker) Mark(name string)
Mark records the elapsed time since the previous Mark (or since creation) as a span named name. Once maxSpans are recorded, further marks are ignored (the clock still advances), so a runaway caller can never overflow or panic.
func (*SpanTracker) MarkFailed ¶ added in v0.1.16
func (t *SpanTracker) MarkFailed(name string)
MarkFailed closes the stage that just FAILED — called at the source (the driver tracer on a rejected statement, the route on a handler's error), so the waterfall marks the bar the failure happened in, not the next one.
func (*SpanTracker) NoteFailedQuery ¶ added in v0.1.16
func (t *SpanTracker) NoteFailedQuery(sql string, err error)
NoteFailedQuery remembers the statement the driver just rejected. Called by the QueryTracer on EVERY failed query; two string assignments, no allocation.
func (*SpanTracker) RecordError ¶
func (t *SpanTracker) RecordError(msg string)
RecordError marks the request as errored and stores the error message. Cheap (no stack): used for client errors (401/403/422) and as the message for 500s.
func (*SpanTracker) SetCapture ¶
func (t *SpanTracker) SetCapture(c *ErrorCapture)
SetCapture attaches a symbolized error capture (the stack) to the request — used only for server errors (500). Implies HasError.
func (*SpanTracker) Spans ¶ added in v0.1.11
func (t *SpanTracker) Spans() []Span
TotalUS returns the sum of all recorded span durations. Spans returns the stages marked so far (a copy) — what a handler can publish to the client BEFORE the response is written (Server-Timing).
func (*SpanTracker) TotalUS ¶
func (t *SpanTracker) TotalUS() int32
type SyntheticMonitor ¶
type SyntheticMonitor struct {
// contains filtered or unexported fields
}
SyntheticMonitor periodically probes a list of HTTP endpoints.
func NewSyntheticMonitor ¶
func NewSyntheticMonitor(checks []Check) *SyntheticMonitor
func (*SyntheticMonitor) AddDynamic ¶
func (sm *SyntheticMonitor) AddDynamic(d DynamicCheck)
AddDynamic registers a lazily-resolved check. Must be called before Start.
func (*SyntheticMonitor) Results ¶
func (sm *SyntheticMonitor) Results() map[string]*CheckResult
Results returns all current check results as a map keyed by check name.
type TenantHistogram ¶
type TenantHistogram struct {
// contains filtered or unexported fields
}
TenantHistogram records request latency per tenant using HDR histograms, split by whether the response was served from cache.
func NewTenantHistogram ¶
func NewTenantHistogram() *TenantHistogram
func (*TenantHistogram) FullSnapshot ¶
func (th *TenantHistogram) FullSnapshot(tenantID string) *FullSnapshot
FullSnapshot returns the cached/uncached latency split for a tenant, or nil buckets when no data has been recorded for that category.
func (*TenantHistogram) Record ¶
func (th *TenantHistogram) Record(tenantID string, durationUs int64, fromCache bool)
Record adds a latency sample (in microseconds) for a given tenant. fromCache distinguishes cache hits (sub-ms) from full DB-backed responses.
func (*TenantHistogram) Snapshot ¶
func (th *TenantHistogram) Snapshot(tenantID string) *PercentileSnapshot
Snapshot returns the uncached latency snapshot. Kept for test backward-compatibility; prefer FullSnapshot in production paths.
type TenantRing ¶
type TenantRing struct {
// contains filtered or unexported fields
}
TenantRing is a fixed circular buffer of the last ringSize samples for one tenant.
A mutex — not lock-free atomics — guards the buffer on purpose: once head laps the array, two writers separated by ringSize records target the SAME physical slot, so an atomic head only serializes index allocation, not the 24-byte Sample copy. Those concurrent same-slot writes are a genuine data race (caught by -race). The lock keeps Record allocation-free (a single array assignment, no slices).
func (*TenantRing) Record ¶
func (r *TenantRing) Record(s Sample)
Record stores s in the next slot, overwriting the oldest sample once the buffer is full. Allocation-free.
func (*TenantRing) Snapshot ¶
func (r *TenantRing) Snapshot() []Sample
Snapshot returns a copy of the retained samples ordered most-recent first. Returns a non-nil empty slice (not nil) when nothing has been recorded, so it marshals to a JSON [] rather than null.
type TraceView ¶
type TraceView struct {
TraceID string `json:"trace_id"`
TS int64 `json:"ts"` // request start, unix microseconds
Route string `json:"route"`
TotalUS int32 `json:"total_us"`
Status uint16 `json:"status"`
Spans []Span `json:"spans"`
ErrMsg string `json:"error_msg,omitempty"` // set for error responses
Stack []Frame `json:"stack,omitempty"` // set only for 500s
// Request context for the error panel — populated for recent_traces from the
// in-memory ErrorCapture (slow_traces persists only error_msg + stack).
Method string `json:"method,omitempty"`
UserID string `json:"user_id,omitempty"`
Role string `json:"role,omitempty"`
// Client context — IP, parsed browser/OS, and geo country.
IP string `json:"ip,omitempty"`
UserAgent string `json:"user_agent,omitempty"`
Browser string `json:"browser,omitempty"`
OS string `json:"os,omitempty"`
Country string `json:"country,omitempty"`
// Reproducible-request context (persisted traces only): filtered headers +
// full URL. Method (above) completes the curl reconstruction.
Headers map[string]string `json:"headers,omitempty"`
FullURL string `json:"full_url,omitempty"`
// OBSERVABILIDAD-ERRORES-S1: the statement the driver rejected (5xx from
// the database), the redacted request body (opt-in, APPXIMO_TRACE_BODY),
// and the error group this trace belongs to.
SQL string `json:"sql,omitempty"`
Body string `json:"body,omitempty"`
Fingerprint uint64 `json:"fingerprint,omitempty"`
}
TraceView is the JSON projection of a Sample's trace: id, route, total, and the per-stage span breakdown. Served under /debug/tenant/{id} → "recent_traces".
type Verdict ¶ added in v0.1.14
type Verdict struct {
Attribution Attribution `json:"attribution"`
// Owner says whose problem it is: "appximo" (code / memory), "host"
// (the plan's quota, the box's RAM), "database" (pool config / the DB /
// the network to it), "none".
Owner string `json:"owner"`
// Reason is one sentence an operator can act on.
Reason string `json:"reason"`
// Signals are every rule input that was evaluated (fired or not), so the
// operator sees the numbers behind the sentence.
Signals []Signal `json:"signals"`
// Also lists lower-priority attributions whose rule ALSO fired this tick.
Also []Attribution `json:"also,omitempty"`
// LatencyHigh is the gate most code-side rules share: p99 over the
// absolute floor, or a multiple of the healthy baseline.
LatencyHigh bool `json:"latency_high"`
BaselineP99 float64 `json:"baseline_p99_ms"`
}
Verdict is the human-readable side of an Attribution.
type WindowSummary ¶ added in v0.1.14
type WindowSummary struct {
From int64 `json:"from"`
To int64 `json:"to"`
Ticks int `json:"ticks"`
TrafficTicks int `json:"traffic_ticks"`
Dominant Attribution `json:"dominant"`
Owner string `json:"owner"`
Reason string `json:"reason"`
Distribution map[Attribution]int `json:"distribution"`
PeakRPS float64 `json:"peak_rps"`
PeakP99Ms float64 `json:"peak_p99_ms"`
PeakTick *ResourceSnapshot `json:"peak_tick,omitempty"`
Requests int64 `json:"requests"`
Shed int64 `json:"shed_429_503"`
Errors5xx int64 `json:"errors_5xx"`
}
WindowSummary aggregates a series into the load-test verdict: the dominant non-healthy attribution over the window (if non-healthy ticks are at least MinShare of the traffic-bearing ticks), the distribution, and the peaks — what the operator attaches to a report after a k6.
func Summarize ¶ added in v0.1.14
func Summarize(series []ResourceSnapshot) WindowSummary
Summarize computes the window verdict over a series (oldest first).
Source Files
¶
- alerter.go
- anomaly.go
- attribution.go
- errors.go
- fingerprint.go
- geo.go
- headers.go
- histogram.go
- limits.go
- metrics.go
- resources.go
- resources_db.go
- resources_http.go
- resources_linux.go
- resources_prom.go
- resources_runtime.go
- ring.go
- server.go
- slo.go
- span.go
- store.go
- store_ephemeral_linux.go
- synthetic.go
- trace.go
- tracer.go
- useragent.go