Help us shape the future of Go! Take the 2026 Go Developer Survey

metrics

package
v0.1.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 15, 2026 License: MIT Imports: 21 Imported by: 0

Documentation

Overview

Package metrics provides Prometheus metrics for ephemerd.

Containerd's metrics (containerd_*) are automatically registered via the builtins import and appear alongside ephemerd's metrics on the same endpoint.

Index

Constants

View Source
const (
	RuntimeWindowsHyperV = "windows-hyperv"
	RuntimeLinuxVM       = "linux-vm"
	RuntimeLinuxNative   = "linux-native"
)

Runtime labels for per-container metrics. The label distinguishes which runtime instance produced the sample — see docs/arch/container-metrics.md.

View Source
const DefaultContainerdNamespace = "ephemerd"

DefaultContainerdNamespace is the namespace ephemerd's runtime uses for every job container — kept in sync with pkg/runtime. The Linux sampler applies this to its Sample context so the containerd Task.Metrics RPC resolves the right task without the caller having to thread a namespaced ctx through the gRPC stream handler.

Variables

View Source
var (
	// ContainerCPUSeconds is the cumulative CPU time consumed by a container.
	// Use rate() in promQL to derive utilization.
	ContainerCPUSeconds = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_container_cpu_usage_seconds_total",
		Help: "Cumulative CPU time consumed by the container, in seconds.",
	}, containerLabels)

	// ContainerMemoryBytes is the current memory in use by a container.
	ContainerMemoryBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "ephemerd_container_memory_bytes",
		Help: "Current memory usage of the container, in bytes.",
	}, containerLabels)

	// ContainerMemoryAnonBytes is the current anonymous memory in use. On
	// Windows this is always 0 — HCS doesn't split anon from file-backed.
	ContainerMemoryAnonBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "ephemerd_container_memory_anon_bytes",
		Help: "Current anonymous (non-file-backed) memory usage of the container, in bytes. 0 on Windows.",
	}, containerLabels)

	// ContainerMemoryLimitBytes is the configured memory cap. 0 = unlimited.
	ContainerMemoryLimitBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "ephemerd_container_memory_limit_bytes",
		Help: "Configured memory limit for the container, in bytes. 0 means unlimited.",
	}, containerLabels)

	// ContainerCPULimit is the configured vCPU count. 0 = unlimited.
	ContainerCPULimit = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Name: "ephemerd_container_cpu_limit",
		Help: "Configured vCPU count for the container. 0 means unlimited.",
	}, containerLabels)

	// ContainerNetworkRxBytes is the cumulative network bytes received by
	// the container's network namespace. Counts the runner container only
	// — sibling containers spawned via dind get their own netns and are
	// not included. See docs/arch/container-metrics.md for the rationale.
	ContainerNetworkRxBytes = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_container_network_rx_bytes_total",
		Help: "Cumulative bytes received by the container's network namespace (loopback excluded).",
	}, containerLabels)

	// ContainerNetworkTxBytes is the cumulative network bytes transmitted.
	ContainerNetworkTxBytes = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_container_network_tx_bytes_total",
		Help: "Cumulative bytes transmitted by the container's network namespace (loopback excluded).",
	}, containerLabels)
)
View Source
var (
	// JobsTotal counts completed jobs by provider, repo, and status.
	JobsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_jobs_total",
		Help: "Total number of jobs processed.",
	}, []string{"provider", "repo", "status"})

	// JobsActive tracks currently running jobs.
	JobsActive = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_jobs_active",
		Help: "Number of currently running jobs.",
	})

	// JobsQueuedTotal counts jobs received from webhook or poll.
	JobsQueuedTotal = promauto.NewCounter(prometheus.CounterOpts{
		Name: "ephemerd_jobs_queued_total",
		Help: "Total number of jobs received (queued events).",
	})

	// JobDuration tracks the full lifecycle duration of a job.
	JobDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "ephemerd_job_duration_seconds",
		Help:    "Time from container creation to destruction.",
		Buckets: []float64{10, 30, 60, 120, 300, 600, 1800, 3600, 7200},
	}, []string{"provider", "repo"})

	// JobStartup tracks time from queued event to runner registered with GitHub.
	JobStartup = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "ephemerd_job_startup_seconds",
		Help:    "Time from queued event to runner environment ready.",
		Buckets: []float64{1, 2, 5, 10, 20, 30, 60, 120},
	}, []string{"repo"})

	// JobQueueWait tracks time spent waiting for a concurrency slot.
	JobQueueWait = promauto.NewHistogram(prometheus.HistogramOpts{
		Name:    "ephemerd_job_queue_wait_seconds",
		Help:    "Time spent waiting for a concurrency semaphore slot.",
		Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60},
	})

	// GitHubAPIRequests counts GitHub API calls by endpoint and status.
	GitHubAPIRequests = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_github_api_requests_total",
		Help: "Total GitHub API requests.",
	}, []string{"endpoint", "status_code"})

	// GitHubAPIRateRemaining tracks the remaining GitHub API rate limit.
	// Updated on every GitHub API response from X-RateLimit-Remaining;
	// pair with GitHubAPIRateUpdatedSeconds so operators can distinguish
	// "0 and current" from "0 and stale".
	GitHubAPIRateRemaining = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_github_api_rate_remaining",
		Help: "Remaining GitHub API rate limit quota (last observed).",
	})

	// GitHubAPIRateLimit reports the ceiling for the current window
	// (typically 5000/hr for app installations, 60 for unauthenticated).
	GitHubAPIRateLimit = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_github_api_rate_limit",
		Help: "GitHub API rate limit ceiling for the current window (last observed).",
	})

	// GitHubAPIRateResetSeconds is the unix timestamp at which the rate
	// window resets, taken from X-RateLimit-Reset.
	GitHubAPIRateResetSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_github_api_rate_reset_seconds",
		Help: "Unix timestamp of the next GitHub rate-limit window reset.",
	})

	// GitHubAPIRateUpdatedSeconds is the unix timestamp of the most
	// recent GitHub API response that carried rate headers. Operators
	// use `time() - ephemerd_github_api_rate_updated_seconds` to age
	// out a stale-looking `_rate_remaining` reading — if the update
	// timestamp is older than the reset timestamp, the gauge is stale.
	GitHubAPIRateUpdatedSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_github_api_rate_updated_seconds",
		Help: "Unix timestamp of the last GitHub API response that carried rate headers.",
	})

	// GitHubPollTotal counts polling cycles.
	GitHubPollTotal = promauto.NewCounter(prometheus.CounterOpts{
		Name: "ephemerd_github_poll_total",
		Help: "Total number of GitHub API poll cycles executed.",
	})

	// GitHubWebhookEventsTotal counts received webhook events by type.
	GitHubWebhookEventsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_github_webhook_events_total",
		Help: "Total webhook events received.",
	}, []string{"event_type"})

	// JITRegistrationErrors counts JIT runner registration failures.
	JITRegistrationErrors = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "ephemerd_github_jit_registration_errors_total",
		Help: "Total JIT runner registration failures.",
	}, []string{"repo", "reason"})

	// UptimeSeconds tracks daemon uptime.
	UptimeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_uptime_seconds",
		Help: "Time since daemon started in seconds.",
	})

	// ConcurrentCapacity reports the configured max concurrent jobs.
	ConcurrentCapacity = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_concurrent_capacity",
		Help: "Maximum number of concurrent jobs (max_concurrent setting).",
	})

	// Draining reports whether the daemon is in drain mode.
	Draining = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "ephemerd_draining",
		Help: "Whether the daemon is draining (1) or accepting jobs (0).",
	})
)

Functions

func DeleteContainerSeries

func DeleteContainerSeries(id, repo, runtimeLabel string)

DeleteContainerSeries removes every metric series for (id, repo, runtime). Call when a container is destroyed to bound cardinality.

func RecordContainerStats

func RecordContainerStats(id, repo, runtimeLabel string, s ContainerStats)

RecordContainerStats applies a single sample to the metric series for (id, repo, runtime). Cumulative fields (CPU, network bytes) are translated into Prometheus counter Add() calls by diffing against the previous sample; gauges (memory, limits) are set directly.

func Serve

func Serve(ctx context.Context, cfg ServerConfig) func()

Serve starts the metrics HTTP server and blocks until ctx is cancelled. Returns a cleanup function that shuts down the server gracefully.

Types

type ContainerStats

type ContainerStats struct {
	CPUUsageNanos    uint64
	MemoryBytes      uint64
	MemoryAnonBytes  uint64
	CPULimit         uint64 // cores, 0 = unlimited
	MemoryLimitBytes uint64 // 0 = unlimited
	NetworkRxBytes   uint64 // cumulative bytes received (loopback excluded)
	NetworkTxBytes   uint64 // cumulative bytes transmitted (loopback excluded)
}

ContainerStats is the platform-neutral sample shape produced by all samplers and consumed by RecordContainerStats.

type LinuxSampler

type LinuxSampler struct {
	// contains filtered or unexported fields
}

LinuxSampler reads cgroupv2 stats for a running container by way of containerd's Task.Metrics RPC, plus optional netns-scoped network counters via netlink. The configured CPU + memory limits are constants for the container's lifetime and are baked in at construction.

func NewLinuxSampler

func NewLinuxSampler(reader taskMetricsReader, cpuLimit, memLimitBytes uint64) *LinuxSampler

NewLinuxSampler builds a Sampler over a containerd Task. cpuLimit is in cores (0 = unlimited); memLimitBytes is in bytes (0 = unlimited). When netnsPath is non-empty, the sampler also reads per-namespace network counters via netlink on every Sample. Empty netnsPath disables network sampling (network bytes report as 0).

func (*LinuxSampler) Sample

func (s *LinuxSampler) Sample(ctx context.Context) (ContainerStats, error)

Sample reads the latest cgroupv2 metrics. cgroup v1 is intentionally not supported — every kernel ephemerd ships runs cgroupv2.

func (*LinuxSampler) WithLogger

func (s *LinuxSampler) WithLogger(log *slog.Logger) *LinuxSampler

WithLogger attaches a logger for non-fatal sampler diagnostics (currently: netns lookup failures, reported once per container).

func (*LinuxSampler) WithNamespace

func (s *LinuxSampler) WithNamespace(ns string) *LinuxSampler

WithNamespace overrides the containerd namespace applied to Sample's context. Only callers that put containers in a non-default namespace need this.

func (*LinuxSampler) WithNetwork

func (s *LinuxSampler) WithNetwork(netnsPath string) *LinuxSampler

WithNetwork enables per-netns network sampling against the given namespace path (e.g. /var/run/netns/cni-xyz, /proc/<pid>/ns/net). Returns the sampler for chaining.

type Sampler

type Sampler interface {
	Sample(ctx context.Context) (ContainerStats, error)
}

Sampler is the per-container resource sampler interface. Implementations live in sampler_linux.go and sampler_windows.go (with a no-op stub in sampler_other.go for cross-compile).

type SamplerRegistry

type SamplerRegistry struct {
	// contains filtered or unexported fields
}

SamplerRegistry tracks active host-local Samplers and ticks them on a shared interval. Use this for native containers whose stats can be read directly in-process. The Linux-VM path bypasses this registry entirely — the in-VM ephemerd runs its own ticker and pushes batches to the host over the Dispatch stream, which calls RecordContainerStats directly.

func NewSamplerRegistry

func NewSamplerRegistry(interval time.Duration, log *slog.Logger) *SamplerRegistry

NewSamplerRegistry creates a registry that ticks every interval. The returned registry must be Start()ed before samples flow, and Stop()ped at shutdown to drain the ticker goroutine.

func (*SamplerRegistry) Register

func (r *SamplerRegistry) Register(id, repo, runtimeLabel string, s Sampler)

Register adds a sampler for (id, repo, runtime). Replaces any prior sampler at the same key.

func (*SamplerRegistry) Start

func (r *SamplerRegistry) Start(ctx context.Context)

Start launches the ticker goroutine. Safe to call once.

func (*SamplerRegistry) Stop

func (r *SamplerRegistry) Stop()

Stop signals the ticker goroutine and waits for it to drain.

func (*SamplerRegistry) Unregister

func (r *SamplerRegistry) Unregister(id, repo, runtimeLabel string)

Unregister removes the sampler and deletes the associated metric series.

type ServerConfig

type ServerConfig struct {
	Port    int
	Path    string
	TLSCert string
	TLSKey  string
	Log     *slog.Logger
}

ServerConfig configures the metrics HTTP server.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL