Documentation
¶
Overview ¶
Package metrics provides Prometheus metrics for ephemerd.
Containerd's metrics (containerd_*) are automatically registered via the builtins import and appear alongside ephemerd's metrics on the same endpoint.
Index ¶
Constants ¶
const ( RuntimeWindowsHyperV = "windows-hyperv" RuntimeLinuxVM = "linux-vm" RuntimeLinuxNative = "linux-native" )
Runtime labels for per-container metrics. The label distinguishes which runtime instance produced the sample — see docs/arch/container-metrics.md.
const DefaultContainerdNamespace = "ephemerd"
DefaultContainerdNamespace is the namespace ephemerd's runtime uses for every job container — kept in sync with pkg/runtime. The Linux sampler applies this to its Sample context so the containerd Task.Metrics RPC resolves the right task without the caller having to thread a namespaced ctx through the gRPC stream handler.
Variables ¶
var ( // ContainerCPUSeconds is the cumulative CPU time consumed by a container. // Use rate() in promQL to derive utilization. ContainerCPUSeconds = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ephemerd_container_cpu_usage_seconds_total", Help: "Cumulative CPU time consumed by the container, in seconds.", }, containerLabels) // ContainerMemoryBytes is the current memory in use by a container. ContainerMemoryBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "ephemerd_container_memory_bytes", Help: "Current memory usage of the container, in bytes.", }, containerLabels) // ContainerMemoryAnonBytes is the current anonymous memory in use. On // Windows this is always 0 — HCS doesn't split anon from file-backed. ContainerMemoryAnonBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "ephemerd_container_memory_anon_bytes", Help: "Current anonymous (non-file-backed) memory usage of the container, in bytes. 0 on Windows.", }, containerLabels) // ContainerMemoryLimitBytes is the configured memory cap. 0 = unlimited. ContainerMemoryLimitBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "ephemerd_container_memory_limit_bytes", Help: "Configured memory limit for the container, in bytes. 0 means unlimited.", }, containerLabels) // ContainerCPULimit is the configured vCPU count. 0 = unlimited. ContainerCPULimit = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "ephemerd_container_cpu_limit", Help: "Configured vCPU count for the container. 0 means unlimited.", }, containerLabels) // ContainerNetworkRxBytes is the cumulative network bytes received by // the container's network namespace. Counts the runner container only // — sibling containers spawned via dind get their own netns and are // not included. See docs/arch/container-metrics.md for the rationale. ContainerNetworkRxBytes = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ephemerd_container_network_rx_bytes_total", Help: "Cumulative bytes received by the container's network namespace (loopback excluded).", }, containerLabels) // ContainerNetworkTxBytes is the cumulative network bytes transmitted. ContainerNetworkTxBytes = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ephemerd_container_network_tx_bytes_total", Help: "Cumulative bytes transmitted by the container's network namespace (loopback excluded).", }, containerLabels) )
var ( // JobsTotal counts completed jobs by provider, repo, and status. JobsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ephemerd_jobs_total", Help: "Total number of jobs processed.", }, []string{"provider", "repo", "status"}) // JobsActive tracks currently running jobs. JobsActive = promauto.NewGauge(prometheus.GaugeOpts{ Name: "ephemerd_jobs_active", Help: "Number of currently running jobs.", }) // JobsQueuedTotal counts jobs received from webhook or poll. JobsQueuedTotal = promauto.NewCounter(prometheus.CounterOpts{ Name: "ephemerd_jobs_queued_total", Help: "Total number of jobs received (queued events).", }) // JobDuration tracks the full lifecycle duration of a job. JobDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "ephemerd_job_duration_seconds", Help: "Time from container creation to destruction.", Buckets: []float64{10, 30, 60, 120, 300, 600, 1800, 3600, 7200}, }, []string{"provider", "repo"}) // JobStartup tracks time from queued event to runner registered with GitHub. JobStartup = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "ephemerd_job_startup_seconds", Help: "Time from queued event to runner environment ready.", Buckets: []float64{1, 2, 5, 10, 20, 30, 60, 120}, }, []string{"repo"}) // JobQueueWait tracks time spent waiting for a concurrency slot. JobQueueWait = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "ephemerd_job_queue_wait_seconds", Help: "Time spent waiting for a concurrency semaphore slot.", Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60}, }) // GitHubAPIRequests counts GitHub API calls by endpoint and status. GitHubAPIRequests = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ephemerd_github_api_requests_total", Help: "Total GitHub API requests.", }, []string{"endpoint", "status_code"}) // GitHubAPIRateRemaining tracks the remaining GitHub API rate limit. // Updated on every GitHub API response from X-RateLimit-Remaining; // pair with GitHubAPIRateUpdatedSeconds so operators can distinguish // "0 and current" from "0 and stale". GitHubAPIRateRemaining = promauto.NewGauge(prometheus.GaugeOpts{ Name: "ephemerd_github_api_rate_remaining", Help: "Remaining GitHub API rate limit quota (last observed).", }) // GitHubAPIRateLimit reports the ceiling for the current window // (typically 5000/hr for app installations, 60 for unauthenticated). GitHubAPIRateLimit = promauto.NewGauge(prometheus.GaugeOpts{ Name: "ephemerd_github_api_rate_limit", Help: "GitHub API rate limit ceiling for the current window (last observed).", }) // GitHubAPIRateResetSeconds is the unix timestamp at which the rate // window resets, taken from X-RateLimit-Reset. GitHubAPIRateResetSeconds = promauto.NewGauge(prometheus.GaugeOpts{ Name: "ephemerd_github_api_rate_reset_seconds", Help: "Unix timestamp of the next GitHub rate-limit window reset.", }) // GitHubAPIRateUpdatedSeconds is the unix timestamp of the most // recent GitHub API response that carried rate headers. Operators // use `time() - ephemerd_github_api_rate_updated_seconds` to age // out a stale-looking `_rate_remaining` reading — if the update // timestamp is older than the reset timestamp, the gauge is stale. GitHubAPIRateUpdatedSeconds = promauto.NewGauge(prometheus.GaugeOpts{ Name: "ephemerd_github_api_rate_updated_seconds", Help: "Unix timestamp of the last GitHub API response that carried rate headers.", }) // GitHubPollTotal counts polling cycles. GitHubPollTotal = promauto.NewCounter(prometheus.CounterOpts{ Name: "ephemerd_github_poll_total", Help: "Total number of GitHub API poll cycles executed.", }) // GitHubWebhookEventsTotal counts received webhook events by type. GitHubWebhookEventsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ephemerd_github_webhook_events_total", Help: "Total webhook events received.", }, []string{"event_type"}) // JITRegistrationErrors counts JIT runner registration failures. JITRegistrationErrors = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "ephemerd_github_jit_registration_errors_total", Help: "Total JIT runner registration failures.", }, []string{"repo", "reason"}) // UptimeSeconds tracks daemon uptime. UptimeSeconds = promauto.NewGauge(prometheus.GaugeOpts{ Name: "ephemerd_uptime_seconds", Help: "Time since daemon started in seconds.", }) // ConcurrentCapacity reports the configured max concurrent jobs. ConcurrentCapacity = promauto.NewGauge(prometheus.GaugeOpts{ Name: "ephemerd_concurrent_capacity", Help: "Maximum number of concurrent jobs (max_concurrent setting).", }) // Draining reports whether the daemon is in drain mode. Draining = promauto.NewGauge(prometheus.GaugeOpts{ Name: "ephemerd_draining", Help: "Whether the daemon is draining (1) or accepting jobs (0).", }) )
Functions ¶
func DeleteContainerSeries ¶
func DeleteContainerSeries(id, repo, runtimeLabel string)
DeleteContainerSeries removes every metric series for (id, repo, runtime). Call when a container is destroyed to bound cardinality.
func RecordContainerStats ¶
func RecordContainerStats(id, repo, runtimeLabel string, s ContainerStats)
RecordContainerStats applies a single sample to the metric series for (id, repo, runtime). Cumulative fields (CPU, network bytes) are translated into Prometheus counter Add() calls by diffing against the previous sample; gauges (memory, limits) are set directly.
func Serve ¶
func Serve(ctx context.Context, cfg ServerConfig) func()
Serve starts the metrics HTTP server and blocks until ctx is cancelled. Returns a cleanup function that shuts down the server gracefully.
Types ¶
type ContainerStats ¶
type ContainerStats struct {
CPUUsageNanos uint64
MemoryBytes uint64
MemoryAnonBytes uint64
CPULimit uint64 // cores, 0 = unlimited
MemoryLimitBytes uint64 // 0 = unlimited
NetworkRxBytes uint64 // cumulative bytes received (loopback excluded)
NetworkTxBytes uint64 // cumulative bytes transmitted (loopback excluded)
}
ContainerStats is the platform-neutral sample shape produced by all samplers and consumed by RecordContainerStats.
type LinuxSampler ¶
type LinuxSampler struct {
// contains filtered or unexported fields
}
LinuxSampler reads cgroupv2 stats for a running container by way of containerd's Task.Metrics RPC, plus optional netns-scoped network counters via netlink. The configured CPU + memory limits are constants for the container's lifetime and are baked in at construction.
func NewLinuxSampler ¶
func NewLinuxSampler(reader taskMetricsReader, cpuLimit, memLimitBytes uint64) *LinuxSampler
NewLinuxSampler builds a Sampler over a containerd Task. cpuLimit is in cores (0 = unlimited); memLimitBytes is in bytes (0 = unlimited). When netnsPath is non-empty, the sampler also reads per-namespace network counters via netlink on every Sample. Empty netnsPath disables network sampling (network bytes report as 0).
func (*LinuxSampler) Sample ¶
func (s *LinuxSampler) Sample(ctx context.Context) (ContainerStats, error)
Sample reads the latest cgroupv2 metrics. cgroup v1 is intentionally not supported — every kernel ephemerd ships runs cgroupv2.
func (*LinuxSampler) WithLogger ¶
func (s *LinuxSampler) WithLogger(log *slog.Logger) *LinuxSampler
WithLogger attaches a logger for non-fatal sampler diagnostics (currently: netns lookup failures, reported once per container).
func (*LinuxSampler) WithNamespace ¶
func (s *LinuxSampler) WithNamespace(ns string) *LinuxSampler
WithNamespace overrides the containerd namespace applied to Sample's context. Only callers that put containers in a non-default namespace need this.
func (*LinuxSampler) WithNetwork ¶
func (s *LinuxSampler) WithNetwork(netnsPath string) *LinuxSampler
WithNetwork enables per-netns network sampling against the given namespace path (e.g. /var/run/netns/cni-xyz, /proc/<pid>/ns/net). Returns the sampler for chaining.
type Sampler ¶
type Sampler interface {
Sample(ctx context.Context) (ContainerStats, error)
}
Sampler is the per-container resource sampler interface. Implementations live in sampler_linux.go and sampler_windows.go (with a no-op stub in sampler_other.go for cross-compile).
type SamplerRegistry ¶
type SamplerRegistry struct {
// contains filtered or unexported fields
}
SamplerRegistry tracks active host-local Samplers and ticks them on a shared interval. Use this for native containers whose stats can be read directly in-process. The Linux-VM path bypasses this registry entirely — the in-VM ephemerd runs its own ticker and pushes batches to the host over the Dispatch stream, which calls RecordContainerStats directly.
func NewSamplerRegistry ¶
func NewSamplerRegistry(interval time.Duration, log *slog.Logger) *SamplerRegistry
NewSamplerRegistry creates a registry that ticks every interval. The returned registry must be Start()ed before samples flow, and Stop()ped at shutdown to drain the ticker goroutine.
func (*SamplerRegistry) Register ¶
func (r *SamplerRegistry) Register(id, repo, runtimeLabel string, s Sampler)
Register adds a sampler for (id, repo, runtime). Replaces any prior sampler at the same key.
func (*SamplerRegistry) Start ¶
func (r *SamplerRegistry) Start(ctx context.Context)
Start launches the ticker goroutine. Safe to call once.
func (*SamplerRegistry) Stop ¶
func (r *SamplerRegistry) Stop()
Stop signals the ticker goroutine and waits for it to drain.
func (*SamplerRegistry) Unregister ¶
func (r *SamplerRegistry) Unregister(id, repo, runtimeLabel string)
Unregister removes the sampler and deletes the associated metric series.