metrics

package
v0.0.0-...-d11b0ee Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 12, 2026 License: MIT Imports: 21 Imported by: 0

Documentation

Overview

Package metrics provides Prometheus metrics for ephemerd.

Containerd's metrics (containerd_*) are automatically registered via the builtins import and appear alongside ephemerd's metrics on the same endpoint.

Index

Constants

View Source
const (
	RuntimeWindowsHyperV = "windows-hyperv"
	RuntimeLinuxVM       = "linux-vm"
	RuntimeLinuxNative   = "linux-native"
)

Runtime labels for per-container metrics. The label distinguishes which runtime instance produced the sample — see docs/arch/container-metrics.md.

View Source
const DefaultContainerdNamespace = "ephemerd"

DefaultContainerdNamespace is the namespace ephemerd's runtime uses for every job container — kept in sync with pkg/runtime. The Linux sampler applies this to its Sample context so the containerd Task.Metrics RPC resolves the right task without the caller having to thread a namespaced ctx through the gRPC stream handler.

Variables

View Source
var (
	// ContainerCPUSeconds is the cumulative CPU time consumed by a container.
	// Use rate() in promQL to derive utilization.
	ContainerCPUSeconds = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_container_cpu_usage_seconds_total",
		Help: "Cumulative CPU time consumed by the container, in seconds.",
	}, containerLabels)

	// ContainerMemoryBytes is the current memory in use by a container.
	ContainerMemoryBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "ephemerd_container_memory_bytes",
		Help: "Current memory usage of the container, in bytes.",
	}, containerLabels)

	// ContainerMemoryAnonBytes is the current anonymous memory in use. On
	// Windows this is always 0 — HCS doesn't split anon from file-backed.
	ContainerMemoryAnonBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "ephemerd_container_memory_anon_bytes",
		Help: "Current anonymous (non-file-backed) memory usage of the container, in bytes. 0 on Windows.",
	}, containerLabels)

	// ContainerMemoryLimitBytes is the configured memory cap. 0 = unlimited.
	ContainerMemoryLimitBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "ephemerd_container_memory_limit_bytes",
		Help: "Configured memory limit for the container, in bytes. 0 means unlimited.",
	}, containerLabels)

	// ContainerCPULimit is the configured vCPU count. 0 = unlimited.
	ContainerCPULimit = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "ephemerd_container_cpu_limit",
		Help: "Configured vCPU count for the container. 0 means unlimited.",
	}, containerLabels)

	// ContainerNetworkRxBytes is the cumulative network bytes received by
	// the container's network namespace. Counts the runner container only
	// — sibling containers spawned via dind get their own netns and are
	// not included. See docs/arch/container-metrics.md for the rationale.
	ContainerNetworkRxBytes = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_container_network_rx_bytes_total",
		Help: "Cumulative bytes received by the container's network namespace (loopback excluded).",
	}, containerLabels)

	// ContainerNetworkTxBytes is the cumulative network bytes transmitted.
	ContainerNetworkTxBytes = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_container_network_tx_bytes_total",
		Help: "Cumulative bytes transmitted by the container's network namespace (loopback excluded).",
	}, containerLabels)
)
View Source
var (
	// JobsTotal counts completed jobs by provider, repo, and status.
	JobsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_jobs_total",
		Help: "Total number of jobs processed.",
	}, []string{"provider", "repo", "status"})

	// JobsActive tracks currently running jobs.
	JobsActive = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_jobs_active",
		Help: "Number of currently running jobs.",
	})

	// JobsQueuedTotal counts jobs received from webhook or poll.
	JobsQueuedTotal = promauto.NewCounter(prometheus.CounterOpts{
		Name: "ephemerd_jobs_queued_total",
		Help: "Total number of jobs received (queued events).",
	})

	// JobDuration tracks the full lifecycle duration of a job.
	JobDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "ephemerd_job_duration_seconds",
		Help:    "Time from container creation to destruction.",
		Buckets: []float64{10, 30, 60, 120, 300, 600, 1800, 3600, 7200},
	}, []string{"provider", "repo"})

	// JobStartup tracks time from queued event to runner registered with GitHub.
	JobStartup = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "ephemerd_job_startup_seconds",
		Help:    "Time from queued event to runner environment ready.",
		Buckets: []float64{1, 2, 5, 10, 20, 30, 60, 120},
	}, []string{"repo"})

	// JobQueueWait tracks time spent waiting for a concurrency slot.
	JobQueueWait = promauto.NewHistogram(prometheus.HistogramOpts{
		Name:    "ephemerd_job_queue_wait_seconds",
		Help:    "Time spent waiting for a concurrency semaphore slot.",
		Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60},
	})

	// GitHubAPIRequests counts GitHub API calls by endpoint and status.
	GitHubAPIRequests = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_github_api_requests_total",
		Help: "Total GitHub API requests.",
	}, []string{"endpoint", "status_code"})

	// GitHubAPIRateRemaining tracks the remaining GitHub API rate limit.
	// Updated on every GitHub API response from X-RateLimit-Remaining;
	// pair with GitHubAPIRateUpdatedSeconds so operators can distinguish
	// "0 and current" from "0 and stale".
	GitHubAPIRateRemaining = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_github_api_rate_remaining",
		Help: "Remaining GitHub API rate limit quota (last observed).",
	})

	// GitHubAPIRateLimit reports the ceiling for the current window
	// (typically 5000/hr for app installations, 60 for unauthenticated).
	GitHubAPIRateLimit = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_github_api_rate_limit",
		Help: "GitHub API rate limit ceiling for the current window (last observed).",
	})

	// GitHubAPIRateResetSeconds is the unix timestamp at which the rate
	// window resets, taken from X-RateLimit-Reset.
	GitHubAPIRateResetSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_github_api_rate_reset_seconds",
		Help: "Unix timestamp of the next GitHub rate-limit window reset.",
	})

	// GitHubAPIRateUpdatedSeconds is the unix timestamp of the most
	// recent GitHub API response that carried rate headers. Operators
	// use `time() - ephemerd_github_api_rate_updated_seconds` to age
	// out a stale-looking `_rate_remaining` reading — if the update
	// timestamp is older than the reset timestamp, the gauge is stale.
	GitHubAPIRateUpdatedSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_github_api_rate_updated_seconds",
		Help: "Unix timestamp of the last GitHub API response that carried rate headers.",
	})

	// GitHubPollTotal counts polling cycles.
	GitHubPollTotal = promauto.NewCounter(prometheus.CounterOpts{
		Name: "ephemerd_github_poll_total",
		Help: "Total number of GitHub API poll cycles executed.",
	})

	// GitHubWebhookEventsTotal counts received webhook events by type.
	GitHubWebhookEventsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_github_webhook_events_total",
		Help: "Total webhook events received.",
	}, []string{"event_type"})

	// JITRegistrationErrors counts JIT runner registration failures.
	JITRegistrationErrors = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_github_jit_registration_errors_total",
		Help: "Total JIT runner registration failures.",
	}, []string{"repo", "reason"})

	// UptimeSeconds tracks daemon uptime.
	UptimeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_uptime_seconds",
		Help: "Time since daemon started in seconds.",
	})

	// ConcurrentCapacity reports the configured max concurrent jobs.
	ConcurrentCapacity = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_concurrent_capacity",
		Help: "Maximum number of concurrent jobs (max_concurrent setting).",
	})

	// Draining reports whether the daemon is in drain mode.
	Draining = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_draining",
		Help: "Whether the daemon is draining (1) or accepting jobs (0).",
	})
)

Functions

func DeleteContainerSeries

func DeleteContainerSeries(id, repo, runtimeLabel string)

DeleteContainerSeries removes every metric series for (id, repo, runtime). Call when a container is destroyed to bound cardinality.

func RecordContainerStats

func RecordContainerStats(id, repo, runtimeLabel string, s ContainerStats)

RecordContainerStats applies a single sample to the metric series for (id, repo, runtime). Cumulative fields (CPU, network bytes) are translated into Prometheus counter Add() calls by diffing against the previous sample; gauges (memory, limits) are set directly.

func Serve

func Serve(ctx context.Context, cfg ServerConfig) func()

Serve starts the metrics HTTP server and blocks until ctx is cancelled. Returns a cleanup function that shuts down the server gracefully.

Types

type ContainerStats

type ContainerStats struct {
	CPUUsageNanos    uint64
	MemoryBytes      uint64
	MemoryAnonBytes  uint64
	CPULimit         uint64 // cores, 0 = unlimited
	MemoryLimitBytes uint64 // 0 = unlimited
	NetworkRxBytes   uint64 // cumulative bytes received (loopback excluded)
	NetworkTxBytes   uint64 // cumulative bytes transmitted (loopback excluded)
}

ContainerStats is the platform-neutral sample shape produced by all samplers and consumed by RecordContainerStats.

type LinuxSampler

type LinuxSampler struct {
	// contains filtered or unexported fields
}

LinuxSampler reads cgroupv2 stats for a running container by way of containerd's Task.Metrics RPC, plus optional netns-scoped network counters via netlink. The configured CPU + memory limits are constants for the container's lifetime and are baked in at construction.

func NewLinuxSampler

func NewLinuxSampler(reader taskMetricsReader, cpuLimit, memLimitBytes uint64) *LinuxSampler

NewLinuxSampler builds a Sampler over a containerd Task. cpuLimit is in cores (0 = unlimited); memLimitBytes is in bytes (0 = unlimited). When netnsPath is non-empty, the sampler also reads per-namespace network counters via netlink on every Sample. Empty netnsPath disables network sampling (network bytes report as 0).

func (*LinuxSampler) Sample

func (s *LinuxSampler) Sample(ctx context.Context) (ContainerStats, error)

Sample reads the latest cgroupv2 metrics. cgroup v1 is intentionally not supported — every kernel ephemerd ships runs cgroupv2.

func (*LinuxSampler) WithLogger

func (s *LinuxSampler) WithLogger(log *slog.Logger) *LinuxSampler

WithLogger attaches a logger for non-fatal sampler diagnostics (currently: netns lookup failures, reported once per container).

func (*LinuxSampler) WithNamespace

func (s *LinuxSampler) WithNamespace(ns string) *LinuxSampler

WithNamespace overrides the containerd namespace applied to Sample's context. Only callers that put containers in a non-default namespace need this.

func (*LinuxSampler) WithNetwork

func (s *LinuxSampler) WithNetwork(netnsPath string) *LinuxSampler

WithNetwork enables per-netns network sampling against the given namespace path (e.g. /var/run/netns/cni-xyz, /proc/<pid>/ns/net). Returns the sampler for chaining.

type Sampler

type Sampler interface {
	Sample(ctx context.Context) (ContainerStats, error)
}

Sampler is the per-container resource sampler interface. Implementations live in sampler_linux.go and sampler_windows.go (with a no-op stub in sampler_other.go for cross-compile).

type SamplerRegistry

type SamplerRegistry struct {
	// contains filtered or unexported fields
}

SamplerRegistry tracks active host-local Samplers and ticks them on a shared interval. Use this for native containers whose stats can be read directly in-process. The Linux-VM path bypasses this registry entirely — the in-VM ephemerd runs its own ticker and pushes batches to the host over the Dispatch stream, which calls RecordContainerStats directly.

func NewSamplerRegistry

func NewSamplerRegistry(interval time.Duration, log *slog.Logger) *SamplerRegistry

NewSamplerRegistry creates a registry that ticks every interval. The returned registry must be Start()ed before samples flow, and Stop()ped at shutdown to drain the ticker goroutine.

func (*SamplerRegistry) Register

func (r *SamplerRegistry) Register(id, repo, runtimeLabel string, s Sampler)

Register adds a sampler for (id, repo, runtime). Replaces any prior sampler at the same key.

func (*SamplerRegistry) Start

func (r *SamplerRegistry) Start(ctx context.Context)

Start launches the ticker goroutine. Safe to call once.

func (*SamplerRegistry) Stop

func (r *SamplerRegistry) Stop()

Stop signals the ticker goroutine and waits for it to drain.

func (*SamplerRegistry) Unregister

func (r *SamplerRegistry) Unregister(id, repo, runtimeLabel string)

Unregister removes the sampler and deletes the associated metric series.

type ServerConfig

type ServerConfig struct {
	Port    int
	Path    string
	TLSCert string
	TLSKey  string
	Log     *slog.Logger
}

ServerConfig configures the metrics HTTP server.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL