telemetry

package
v0.0.0-...-a8bd47a Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 5, 2026 License: Apache-2.0 Imports: 37 Imported by: 35

Documentation

Index

Constants

View Source
const (
	GCPTraceContextHeader = "X-Cloud-Trace-Context"
	AWSTraceContextHeader = "X-Amzn-Trace-Id"
)
View Source
const (
	ApiOrchestratorCountMeterName GaugeIntType = "api.orchestrator.status"
	OrchestratorStatusGaugeName   GaugeIntType = "orchestrator.status"

	// Orchestrator node resources allocated to running sandboxes (sum across running sandboxes)
	OrchestratorCpuAllocatedGaugeName    GaugeIntType = "orchestrator.sandbox.cpu.allocated"
	OrchestratorMemoryAllocatedGaugeName GaugeIntType = "orchestrator.sandbox.memory.allocated"
	OrchestratorDiskAllocatedGaugeName   GaugeIntType = "orchestrator.sandbox.disk.allocated"

	// Sandbox metrics
	SandboxRamUsedGaugeName   GaugeIntType = "e2b.sandbox.ram.used"
	SandboxRamTotalGaugeName  GaugeIntType = "e2b.sandbox.ram.total"
	SandboxRamCacheGaugeName  GaugeIntType = "e2b.sandbox.ram.cache"
	SandboxCpuTotalGaugeName  GaugeIntType = "e2b.sandbox.cpu.total"
	SandboxDiskUsedGaugeName  GaugeIntType = "e2b.sandbox.disk.used"
	SandboxDiskTotalGaugeName GaugeIntType = "e2b.sandbox.disk.total"

	// Team metrics
	TeamSandboxRunningGaugeName GaugeIntType = "e2b.team.sandbox.running"

	SandboxCountGaugeName GaugeIntType = "api.env.instance.running"

	// Build resource metrics
	BuildRootfsSizeHistogramName HistogramType = "template.build.rootfs.size"
)
View Source
const DefaultPprofPort = 6060

DefaultPprofPort is the default port that we should use to mount pprof endpoint.

Variables

View Source
var (
	// Pre-allocated result attributes for use with PrecomputeAttrs.
	Success = attribute.String(resultAttr, resultTypeSuccess)
	Failure = attribute.String(resultAttr, resultTypeFailure)
)
View Source
var SubMillisecondMsBuckets = []float64{
	0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000,
}

SubMillisecondMsBuckets resolve sub-ms operations (mmap / cache hits) that the default OTEL buckets (first boundary 5ms) collapse into one, while still covering remote reads to ~10s.

Functions

func GetCounter

func GetCounter(meter metric.Meter, name CounterType) (metric.Int64Counter, error)

func GetFloatHistogram

func GetFloatHistogram(meter metric.Meter, name HistogramType) (metric.Float64Histogram, error)

func GetGaugeFloat

func GetGaugeFloat(meter metric.Meter, name GaugeFloatType) (metric.Float64ObservableGauge, error)

func GetGaugeInt

func GetGaugeInt(meter metric.Meter, name GaugeIntType) (metric.Int64ObservableGauge, error)

func GetHistogram

func GetHistogram(meter metric.Meter, name HistogramType) (metric.Int64Histogram, error)

GetHistogram returns an Int64 histogram with the registered description and unit. It sets no bucket boundaries: histogramAggregation discards them.

func GetResource

func GetResource(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, additional ...attribute.KeyValue) (*resource.Resource, error)

func NewEventWriter

func NewEventWriter(ctx context.Context, name string) io.Writer

func NewMeterExporter

func NewMeterExporter(ctx context.Context, extraOption ...otlpmetricgrpc.Option) (sdkmetric.Exporter, error)

func NewMeterProvider

func NewMeterProvider(metricsExporter sdkmetric.Exporter, metricExportPeriod time.Duration, res *resource.Resource, extraOption ...sdkmetric.Option) (*sdkmetric.MeterProvider, error)

func NewPprofMux

func NewPprofMux() *http.ServeMux

NewPprofMux returns a ServeMux with pprof handlers registered on a dedicated mux rather than http.DefaultServeMux, preventing accidental exposure.

Mirrors the five handlers that `_ "net/http/pprof"` registers in its init(). The Index handler already serves all runtime/pprof profiles (heap, goroutine, allocs, block, mutex, threadcreate, etc.) by name from the URL path.

func NewPprofServer

func NewPprofServer() *http.Server

func NewSpanExporter

func NewSpanExporter(ctx context.Context, extraOption ...otlptracegrpc.Option) (sdktrace.SpanExporter, error)

func NewTextPropagator

func NewTextPropagator() propagation.TextMapPropagator

func NewTracerProvider

func NewTracerProvider(spanExporter sdktrace.SpanExporter, res *resource.Resource) trace.TracerProvider

func OTELCollectorGRPCEndpoint

func OTELCollectorGRPCEndpoint() string

func Observe0

func Observe0(ctx context.Context, tracer trace.Tracer, name string, fn func(context.Context) error, opts ...trace.SpanStartOption) error

Observe0 records one synchronous operation returning only an error.

func Observe1

func Observe1[T any](ctx context.Context, tracer trace.Tracer, name string, fn func(context.Context) (T, error), opts ...trace.SpanStartOption) (T, error)

Observe1 records one synchronous operation returning one value and an error.

func ParseEdgeTraceID

func ParseEdgeTraceID(gcpHeader, awsHeader string) (string, bool)

ParseEdgeTraceID extracts a trace ID from cloud-provider trace headers. These headers are untrusted and are only used for cross-service correlation.

func PprofPort

func PprofPort() int

PprofPort returns the port the pprof server should bind to: the value of PPROF_PORT when set to a valid port, otherwise DefaultPprofPort.

func PrecomputeAttrs

func PrecomputeAttrs(kv ...attribute.KeyValue) metric.MeasurementOption

PrecomputeAttrs builds a reusable MeasurementOption from the given attribute key-values. The option must include all attributes (including "result"). Use with Stopwatch.Record to avoid per-call attribute allocation.

func ReportCriticalError

func ReportCriticalError(ctx context.Context, message string, err error, attrs ...attribute.KeyValue)

func ReportError

func ReportError(ctx context.Context, message string, err error, attrs ...attribute.KeyValue)

func ReportErrorByCode

func ReportErrorByCode(ctx context.Context, code int, message string, err error, attrs ...attribute.KeyValue)

func ReportEvent

func ReportEvent(ctx context.Context, name string, attrs ...attribute.KeyValue)

func SetAttributes

func SetAttributes(ctx context.Context, attrs ...attribute.KeyValue)

func WithBuildID

func WithBuildID(buildID string) attribute.KeyValue

func WithClusterID

func WithClusterID(clusterID uuid.UUID) attribute.KeyValue

func WithEdgeTraceID

func WithEdgeTraceID(traceID string) attribute.KeyValue

func WithEnvdVersion

func WithEnvdVersion(envdVersion string) attribute.KeyValue

func WithFirecrackerVersion

func WithFirecrackerVersion(firecrackerVersion string) attribute.KeyValue

func WithKernelVersion

func WithKernelVersion(kernelVersion string) attribute.KeyValue

func WithMaskedAPIKey

func WithMaskedAPIKey(maskedAPIKey string) attribute.KeyValue

func WithNodeID

func WithNodeID(nodeID string) attribute.KeyValue

func WithSandboxID

func WithSandboxID(sandboxID string) attribute.KeyValue

func WithServiceInstanceID

func WithServiceInstanceID(serviceInstanceID string) attribute.KeyValue

func WithTeamID

func WithTeamID(teamID string) attribute.KeyValue

func WithTemplateID

func WithTemplateID(templateID string) attribute.KeyValue

func WithUserID

func WithUserID(userID string) attribute.KeyValue

Types

type Client

type Client struct {
	MetricExporter sdkmetric.Exporter
	MeterProvider  metric.MeterProvider

	SpanExporter    sdktrace.SpanExporter
	TracerProvider  trace.TracerProvider
	TracePropagator propagation.TextMapPropagator
	LogsProvider    LogProvider
	// contains filtered or unexported fields
}

func New

func New(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, additional ...attribute.KeyValue) (*Client, error)

New creates a telemetry client that exports traces, metrics, and logs via gRPC. Telemetry is enabled when the OTEL_COLLECTOR_GRPC_ENDPOINT environment variable is set (e.g. "localhost:4317"). When unset, a noop client is returned with zero overhead.

func NewAnonymous

func NewAnonymous(ctx context.Context, serviceName string) (*Client, error)

NewAnonymous creates a telemetry client for tools and CLI commands that don't have build-time injected metadata (commitSHA, version, nodeID). serviceName is the primary identifier used for filtering traces and metrics in observability tools (e.g. Grafana). The remaining resource attributes are filled with sensible defaults (hostname, "unknown" commit, "dev" version).

func NewNoopClient

func NewNoopClient() *Client

func (*Client) Shutdown

func (t *Client) Shutdown(ctx context.Context) error

func (*Client) StartRuntimeInstrumentation

func (t *Client) StartRuntimeInstrumentation() error

StartRuntimeInstrumentation registers OTEL Go runtime metric callbacks.

Collected metrics (semantic-convention names):

  • go.memory.used
  • go.memory.limit
  • go.memory.allocated
  • go.memory.allocations
  • go.memory.gc.goal
  • go.goroutine.count
  • go.processor.limit
  • go.config.gogc

The callbacks are invoked by the MeterProvider and stop automatically when it shuts down — no separate goroutine is spawned.

type CounterType

type CounterType string
const (
	ApiOrchestratorCreatedSandboxes      CounterType = "api.orchestrator.created_sandboxes"
	ApiOrchestratorResumeOriginNodeRemap CounterType = "api.orchestrator.resume_origin_node_remapped"
	// ApiOrchestratorPauseRefusalRestore counts what became of a pause the
	// node refused retryably: outcome restored | restore_failed |
	// route_restore_failed, caller request | eviction.
	ApiOrchestratorPauseRefusalRestore CounterType = "api.orchestrator.pause_refusal_restore"
	// ApiEvictorFsOnlyAutoPause counts timeout auto-pauses whose policy asked
	// for a filesystem-only snapshot. Unlabeled since the fs-only version
	// gate was dropped: every counted eviction takes the fs-only path.
	ApiEvictorFsOnlyAutoPause CounterType = "api.evictor.fs_only_auto_pause"
	// ApiEvictorAutoPauseDegraded counts timeout auto-pauses the evictor
	// downgraded to a filesystem-only snapshot after the node refused the
	// memory one, by cause: admission_refused (no retry budget, degraded at
	// the first refusal) or overstay_budget (the retry budget since the first
	// refusal ran out); a subset of ApiEvictorFsOnlyAutoPause.
	ApiEvictorAutoPauseDegraded CounterType = "api.evictor.auto_pause_degraded"
	SandboxCreateMeterName      CounterType = "api.env.instance.started"

	TeamSandboxCreated CounterType = "e2b.team.sandbox.created"

	EnvdInitCalls CounterType = "orchestrator.sandbox.envd.init.calls"

	// 2 MiB chunks the pre-pause envd heap collapse attempted, split by the
	// result attribute (collapsed|skipped): attempts = total, successful =
	// collapsed.
	EnvdCollapseChunks CounterType = "orchestrator.sandbox.envd.collapse.chunks"
	// Incremented by the balance_dirty_pages thread count at every 200 ms poll
	// for the lifetime of the process. rate() shows dirty-page throttle
	// intensity in real-time; 0 when no stalls are occurring.
	OrchestratorHostBalanceDirtyPagesThreads CounterType = "orchestrator.host.balance_dirty_pages.threads"

	OrchestratorSandboxKilledCounterName CounterType = "orchestrator.sandbox.killed"
	// OrchestratorSandboxPauseAdmissionCounterName counts every snapshot-
	// admission decision. refused is the told-the-caller-to-retry rate;
	// ready_after_wait counts the grace paying off — their ratio tunes the
	// grace flag.
	OrchestratorSandboxPauseAdmissionCounterName CounterType = "orchestrator.sandbox.pause_admission"
	// OrchestratorSandboxCheckpointCounterName counts Checkpoint RPCs by flow —
	// the denominator that makes the in_place-labeled duration histograms
	// cuttable as a ramp (what fraction went in-place, at what success rate).
	OrchestratorSandboxCheckpointCounterName CounterType = "orchestrator.sandbox.checkpoint"
	// OrchestratorFPRResumeCounterName counts free-page-reporting resume
	// outcomes after a CoW window paused reporting. Guest memory is hugetlb
	// with no swap and no memcg reclaim, so FPR discards are the ONLY
	// mechanism returning freed guest pages to the host: outcome="abandoned"
	// is lasting per-sandbox host-memory retention and the alertable signal
	// for a leaked pause.
	OrchestratorFPRResumeCounterName CounterType = "orchestrator.sandbox.fpr_resume"

	// OrchestratorSnapshotUploadFailedCounterName counts pause-snapshot uploads
	// that never landed durably (budget exhausted or a non-retryable error).
	// A non-zero rate means lost snapshots.
	OrchestratorSnapshotUploadFailedCounterName CounterType = "orchestrator.snapshot.upload.failed"

	// SandboxPauseFsQuiescedCounterName counts filesystem-only pauses by whether
	// the captured rootfs was frozen (quiesced=true, crash-consistent) vs a plain
	// sync fallback (quiesced=false). quiesced/total is the fraction of newly
	// minted fs-only snapshots that are safe to cold-boot / rewrite — the eligible
	// population for the offline envd upgrade built on top of this flag.
	SandboxPauseFsQuiescedCounterName CounterType = "orchestrator.sandbox.pause.fs_quiesced"

	// SandboxResumeWPModeCounterName counts sandbox resumes by the write-protect
	// tracking mode the resume chose (mode=sync|async, the use-sync-wp flag
	// decision). This is the denominator for every sync-WP burn-in signal:
	// wp_resolve rates and dirty-divergence readings only come from mode=sync
	// sandboxes, so without this counter the fleet's sync coverage — and
	// whether an error rate is "all sync sandboxes" or "one loud one" — is
	// invisible.
	SandboxResumeWPModeCounterName CounterType = "orchestrator.sandbox.resume.wp_mode"

	// OrchestratorEnvdUpgradeAttempts counts resume-time envd live-upgrade
	// attempts, by result (success|delivery_failed|not_ready|panic) and
	// from_version/to_version. success/total is the rollout success rate;
	// attempts/resumes is the fire rate.
	OrchestratorEnvdUpgradeAttempts CounterType = "orchestrator.envd.upgrade.attempts"
	// OrchestratorEnvdUpgradeGated counts resumes the envd-upgrade-target flag
	// targeted but a gate declined, by reason — a silent no-op worth watching during
	// a ramp. The reasons are the resolver's and the gates' own vocabulary; read them
	// off `sum by (reason)` rather than from a list here, which would go stale.
	OrchestratorEnvdUpgradeGated CounterType = "orchestrator.envd.upgrade.gated"
	// OrchestratorEnvdUpgradeHandover counts live-upgrade handover items by item
	// (proc|retained|watcher) and result (ok|failed), reported back by the new
	// envd on /init. failed/(ok+failed) is the handover error rate — the
	// fleet-visible signal (otherwise only logged in-guest) that the new envd
	// dropped or degraded something it re-adopted across the swap.
	OrchestratorEnvdUpgradeHandover CounterType = "orchestrator.envd.upgrade.handover"

	// EnvdDefaultsApplied counts MEMORY RESUMES by where the default user the orchestrator
	// sent came from, so the derivation has a positive engagement signal rather than an
	// inferred one: a derivation that produced nothing and one that never ran read
	// identically on a dashboard.
	//
	// Resumes, not starts: the cold-boot path does its own reconstruction and is not
	// counted here. Split by sandbox_type, because a build's own internal resumes go
	// through the same path and are not customer sandbox starts.
	EnvdDefaultsApplied CounterType = "orchestrator.envd.defaults.applied"
	// EnvdDefaultsMismatch counts /init responses where the defaults envd reports as
	// EFFECTIVE differ from what the orchestrator sent, by field. Zero is the only
	// acceptable steady state: a non-zero value means a sandbox is resolving requests
	// against an identity nobody asked for. Note the header is written inside a guest,
	// so a single hostile or broken sandbox can hold this above zero; investigate via
	// the per-sandbox error log rather than treating the aggregate as authoritative.
	EnvdDefaultsMismatch CounterType = "orchestrator.envd.defaults.mismatch"
	// EnvdDefaultsWorkdirWithheld counts MEMORY RESUMES where the template recorded a
	// default workdir the orchestrator could not prove the build sent, so it was not
	// re-sent — the same event EnvdDefaultsApplied counts, so the two divide. An
	// UPPER BOUND on the population that keeps the wrong working directory after a live
	// upgrade: a workdir equal to the resolved user's home is withheld to no effect, and
	// whether it coincides cannot be decided outside the guest.
	EnvdDefaultsWorkdirWithheld CounterType = "orchestrator.envd.defaults.workdir_withheld"
	// EnvdDefaultsBuiltinFallback counts /init responses where envd reports it was never
	// told which user to run as, so it is serving the value it was compiled with. This is
	// the REALIZED loss, and the only signal that can observe one: a mismatch cannot,
	// because envd stores exactly what /init sent it and then reports that field back.
	//
	// Split by whether a user was sent, and by sandbox type. sent=true is a defect at any
	// type, because a delivered user must never read back as never-told. sent=false is only
	// about the population at risk at sandbox_type="sandbox": the build tree starts its own
	// sandboxes with no default user configured, on the host envd, so it lands in
	// sent=false from the first day of the rollout and would otherwise dominate the bucket.
	EnvdDefaultsBuiltinFallback CounterType = "orchestrator.envd.defaults.builtin_fallback"

	// OrchestratorEnvdOfflineUpgradeAttempts counts every OFFLINE envd upgrade
	// OUTCOME on the cold boot of a filesystem-only snapshot — not only the ones that
	// reach the rootfs binary swap — by result and from_version/to_version:
	//
	//	success | swap_failed | unrecoverable | envd_too_large | envd_missing | stat_unparseable
	//	    the swap ran, or the rootfs itself declined it (see rootfs.SwapEnvdBinary)
	//	not_quiesced
	//	    an upgrade was wanted, but the snapshot's rootfs was not frozen at pause,
	//	    so it must not be rewritten
	//	same_version
	//	    already on the target
	//	not_staged | downgrade | invalid_target | getversion_failed
	//	    the resolver refused the configured target
	//
	// Every no-op except flag-off is counted, so the eligible population adds up.
	// Flag-off is deliberately absent: it is the whole filesystem-only population
	// minus the rest, already available as sandbox.create.duration{fs_only="true"}.
	// to_version is empty on the resolver's refusals — it has no target to name.
	//
	// On result=success a refire label says whether the rootfs ALREADY held the target
	// bytes. from_version cannot answer that on its own: it is a claim read off the
	// snapshot record, and the swap keys on that record without ever advancing it, so
	// an already-upgraded snapshot resolves the same upgrade on every cold boot and
	// rewrites identical bytes while reporting the same from_version as a genuine
	// upgrade. refire="false" is the count that actually moved sandboxes. The label is
	// absent on every other result, where nothing was compared.
	OrchestratorEnvdOfflineUpgradeAttempts CounterType = "orchestrator.envd.offline_upgrade.attempts"

	// OrchestratorFsRecoveryRuns counts every pre-boot filesystem-recovery
	// decision on a cold boot, by result and trigger. Recovery is journal replay
	// only (`e2fsck -p -E journal_only`), so the results are:
	//
	//	skipped_quiesced
	//	    the rootfs was frozen at pause; nothing to replay
	//	replayed
	//	    the journal was replayed (or regenerated, or there was nothing to do);
	//	    the fs is mountable and the boot proceeded — the expected outcome for
	//	    the admitted population, NOT a corruption signal.
	//	failed_operational
	//	    the run did not complete a clean replay AND e2fsck may have opened the
	//	    device (an e2fsck non-replay exit, or a timeout that could have killed it
	//	    mid-write). Fail closed — the start failed — but always retryable on
	//	    another node, never a permanent snapshot verdict (its exit codes cannot
	//	    tell an unmountable filesystem apart from a transient fault).
	//	failed_open
	//	    recovery could not run AND e2fsck provably never opened the device (the
	//	    jail could not launch it, or the host cannot exec e2fsck), so the disk is
	//	    what a flag-off cold boot would mount and the guest kernel replays the
	//	    journal itself. The boot proceeded. A host-image signal — a non-trivial
	//	    rate means the recovery tooling is broken fleet-wide (roll back), even
	//	    though sandboxes still boot.
	//
	// A bounded "reason" attribute sub-labels each result so a ramp can act on it
	// from a dashboard instead of grepping create-failure logs:
	//
	//	replayed:         nothing_to_do (no replay needed) | journal_replayed
	//	                  (journal applied) — the efficacy split
	//	failed_operational: timeout (Go deadline fired mid-run) | killed (the unit was
	//	                  signalled mid-run — exit -1 or 128+N; OOM, RuntimeMaxSec,
	//	                  external stop; NOT a tooling failure) | e2fsck_4 | e2fsck_8 |
	//	                  e2fsck_other (e2fsck's
	//	                  own non-replay exits — an unreplayable snapshot, expected and
	//	                  small) | no_sentinel (ran but lost its result, or the pre-launch
	//	                  device guard)
	//	failed_open:      launcher_failure (the unit failed to START — e2fsck never ran) |
	//	                  exec_failure (126/127: the host could not exec e2fsck) —
	//	                  both host-image regressions, roll back
	//	skipped_quiesced: quiesced
	//
	// It never carries a raw exit code or any tenant-influenced bytes.
	//
	// trigger separates the two admitted populations: "rescue" (the request
	// demanded a filesystem boot of a memory snapshot) vs "legacy_fs_only"
	// (a filesystem-only snapshot whose pause fell back to sync).
	OrchestratorFsRecoveryRuns CounterType = "orchestrator.sandbox.fs_recovery.runs"

	// OrchestratorFsRecoveryToolingUnsupported fires once per orchestrator process
	// when the host e2fsck does not accept `-E journal_only` (probed against a
	// nonexistent device, so it reads no filesystem). Expected to be flat zero: node
	// images ship an e2fsprogs that has supported the option for years. A non-zero
	// fleet sum means some node's tooling silently no-ops recovery (the guest kernel
	// still replays at mount, so boots are unaffected) — a signal to fix that image,
	// not a per-sandbox failure.
	OrchestratorFsRecoveryToolingUnsupported CounterType = "orchestrator.sandbox.fs_recovery.tooling_unsupported"

	// TemplateBuildCmdlineArgs counts template builds by the guest kernel command line
	// parameters they actually booted with, after parsing and validation. This is the
	// engagement signal for the per-team cmdline-variant flag: a non-zero rate on a
	// non-empty label is proof builds ran the path, which is what the flag
	// reading "on" in the feature-flag service does not tell you. Zero here while the
	// flag is targeted is the alarm that the opt-in is silently inert. Because it
	// records what was APPLIED, a rejected fragment shows up as the empty label rather
	// than as the parameters that were asked for.
	TemplateBuildCmdlineArgs CounterType = "orchestrator.template.build.cmdline_args"

	// PauseResumePrefetchHarvestAttempts counts pause-resume prefetch harvest
	// attempts, by result (success|resume_failed|collect_failed|skipped). The
	// throwaway is absent from Prometheus otherwise (registration-skip), so this
	// is the harvest-activity / failure-rate signal.
	PauseResumePrefetchHarvestAttempts CounterType = "orchestrator.sandbox.pause_resume_prefetch.harvest.attempts"

	ApiRedisStoragePublisherPublished CounterType = "api.redis_storage.publisher.published"
	ApiRedisStoragePublisherDropped   CounterType = "api.redis_storage.publisher.dropped"

	// ApiRedisStorageExpirationIndexHealed counts sandboxes the healer re-added
	// to the global expiration index. Healthy steady state is zero; a sustained
	// non-zero rate means expiration index writes are being lost and sandboxes
	// would otherwise become invisible to the evictor (immortal).
	ApiRedisStorageExpirationIndexHealed CounterType = "api.redis_storage.expiration_index.healed"
	// ApiRedisStorageExpirationIndexSwept counts members removed from the
	// global expiration index by the evictor scan
	// (reason=orphan|dead_execution|invalid).
	ApiRedisStorageExpirationIndexSwept CounterType = "api.redis_storage.expiration_index.swept"
	// ApiRedisStorageExpirationIndexRescored counts live members whose index
	// score drifted from the stored EndTime and were re-scored by the evictor
	// scan. Sustained non-zero rate means score updates are being lost.
	ApiRedisStorageExpirationIndexRescored CounterType = "api.redis_storage.expiration_index.rescored"
)
const (
	// Build result counters
	BuildResultCounterName      CounterType = "template.build.result"
	BuildCacheResultCounterName CounterType = "template.build.cache.result"

	// TCP Firewall counters
	TCPFirewallConnectionsTotal CounterType = "orchestrator.tcpfirewall.connections.total"
	TCPFirewallErrorsTotal      CounterType = "orchestrator.tcpfirewall.errors.total"
	TCPFirewallDecisionsTotal   CounterType = "orchestrator.tcpfirewall.decisions.total"

	// Ingress proxy counters
	IngressProxyConnectionsBlockedTotal CounterType = "orchestrator.proxy.connections.blocked.total"

	// cmux counters
	CmuxErrorsTotal CounterType = "orchestrator.cmux.errors.total"

	// Firecracker net counters — global totals, no sandbox_id (low cardinality).
	// All carry a direction=tx/rx attribute. Per-sandbox distributions are histograms below.
	SandboxFCNetFails         CounterType = "orchestrator.sandbox.fc.net.fails"
	SandboxFCNetNoAvailBuffer CounterType = "orchestrator.sandbox.fc.net.no_avail_buffer"
	SandboxFCNetTapIOFails    CounterType = "orchestrator.sandbox.fc.net.tap_io_fails"

	// Firecracker block counters — global totals, no sandbox_id (low cardinality).
	// Carry a direction=read/write attribute where applicable.
	SandboxFCBlockFails         CounterType = "orchestrator.sandbox.fc.block.fails"
	SandboxFCBlockNoAvailBuffer CounterType = "orchestrator.sandbox.fc.block.no_avail_buffer"
)

type EventWriter

type EventWriter struct {
	// contains filtered or unexported fields
}

func (*EventWriter) Write

func (w *EventWriter) Write(p []byte) (n int, err error)

type FloatTimerFactory

type FloatTimerFactory struct {
	// contains filtered or unexported fields
}

FloatTimerFactory records duration as fractional milliseconds so sub-ms operations aren't truncated to 0. The duration histogram and event counter share <metricName> (rate()-friendly); only the bytes counter splits out to <metricName>.size so Grafana's unit detection doesn't conflate ms with By.

func NewFloatTimerFactory

func NewFloatTimerFactory(
	meter metric.Meter,
	metricName, durationDescription, bytesDescription string,
) (FloatTimerFactory, error)

func (*FloatTimerFactory) Record

func (f *FloatTimerFactory) Record(ctx context.Context, dur time.Duration, total int64, attrs metric.MeasurementOption)

type GaugeFloatType

type GaugeFloatType string
const (
	SandboxCpuUsedGaugeName GaugeFloatType = "e2b.sandbox.cpu.used"
)

type GaugeIntType

type GaugeIntType string

type HistogramType

type HistogramType string
const (
	// Build timing histograms
	BuildDurationHistogramName      HistogramType = "template.build.duration"
	BuildPhaseDurationHistogramName HistogramType = "template.build.phase.duration"
	BuildStepDurationHistogramName  HistogramType = "template.build.step.duration"

	// Sandbox timing histograms
	OrchestratorSandboxCreateDurationName HistogramType = "orchestrator.sandbox.create.duration"
	WaitForEnvdDurationHistogramName      HistogramType = "orchestrator.sandbox.envd.init.duration"
	GuestSyncDurationHistogramName        HistogramType = "orchestrator.sandbox.guest_sync.duration"
	PauseDurationHistogramName            HistogramType = "orchestrator.sandbox.pause.duration"
	SnapshotProcessMemoryDurationName     HistogramType = "orchestrator.sandbox.snapshot.process_memory.duration"
	SnapshotProcessRootfsDurationName     HistogramType = "orchestrator.sandbox.snapshot.process_rootfs.duration"
	SnapshotRootfsSealDurationName        HistogramType = "orchestrator.sandbox.snapshot.rootfs_seal.duration"
	SnapshotGuestFreezeDurationName       HistogramType = "orchestrator.sandbox.snapshot.guest_freeze.duration"
	SnapshotMemorySealDurationName        HistogramType = "orchestrator.sandbox.snapshot.memory_seal.duration"

	// OrchestratorSandboxExecutionDurationName is one sample per Firecracker
	// run, so a sandbox that is paused and resumed records one per run.
	OrchestratorSandboxExecutionDurationName HistogramType = "orchestrator.sandbox.execution.duration"

	// OrchestratorEnvdUpgradeDurationName is the wall-time of a resume-time envd
	// live-upgrade (delivery + trigger + WaitForEnvd) = overhead added to the
	// resume. Labeled by result.
	OrchestratorEnvdUpgradeDurationName HistogramType = "orchestrator.envd.upgrade.duration"

	// OrchestratorEnvdOfflineUpgradeDurationName is the wall-time of the offline
	// rootfs envd swap (jailed debugfs), recorded once per swap attempt. Catches
	// pathological rewrites; the swap runs in the cold-boot PreBootFn, so it adds
	// directly to resume latency.
	OrchestratorEnvdOfflineUpgradeDurationName HistogramType = "orchestrator.envd.offline_upgrade.duration"

	// OrchestratorFsRecoveryDurationName is the wall-time of the jailed pre-boot
	// journal-replay run on a cold boot, recorded for every outcome the run reaches
	// (replayed/failed_operational) — only skipped_quiesced,
	// which never runs e2fsck, has no duration sample. Replay is journal-bounded, so
	// this should stay sub-second even on large filesystems. Labeled like
	// OrchestratorFsRecoveryRuns.
	OrchestratorFsRecoveryDurationName HistogramType = "orchestrator.sandbox.fs_recovery.duration"

	// Pre-pause envd heap collapse round-trip duration (the pause-path cost of
	// POST /collapse: network plus envd's madvise work), recorded once per pause
	// when the collapse-envd-heap flag is on.
	EnvdCollapseDurationHistogramName HistogramType = "orchestrator.sandbox.envd.collapse.duration"

	// Background memfile dedup latency: provisional header creation at pause to
	// the durable-header swap. Recorded once per swap; no attributes.
	OrchestratorSandboxMemfileDedupDurationName HistogramType = "orchestrator.sandbox.memfile_dedup.duration"

	// How long snapshot admission actually waited whenever it waited, labeled
	// by outcome (ready_after_wait/refused only).
	OrchestratorSandboxPauseAdmissionWaitDurationName HistogramType = "orchestrator.sandbox.pause_admission.wait.duration"

	// Pause-resume prefetch harvest cost, recorded once per harvest attempt.
	// duration is the SLOT-HOLD cost alone — the throwaway resume, its trace
	// collection and its reap — which is what the harvest-timeout flag caps.
	// persist_wait is the separate, resource-free wait on the in-flight snapshot
	// upload that the mapping has to be written after; keeping it out of duration
	// is what lets "harvests at the timeout" stay a meaningful signal.
	// pages is the harvested trace size (distinct 2 MiB blocks), recorded only on
	// success, so its bottom bucket surfaces the empty-trace (idle-at-pause) rate.
	PauseResumePrefetchHarvestDurationName     HistogramType = "orchestrator.sandbox.pause_resume_prefetch.harvest.duration"
	PauseResumePrefetchHarvestPagesName        HistogramType = "orchestrator.sandbox.pause_resume_prefetch.harvest.pages"
	PauseResumePrefetchSealWaitDurationName    HistogramType = "orchestrator.sandbox.pause_resume_prefetch.seal_wait.duration"
	PauseResumePrefetchPersistWaitDurationName HistogramType = "orchestrator.sandbox.pause_resume_prefetch.persist_wait.duration"

	// Sandbox startup working-set histograms: demand-fault pages/bytes a guest
	// needed to reach a successful envd init, recorded once per start. Sampled
	// per start (not per fault), so histogram_quantile yields per-sandbox
	// percentiles.
	EnvdFreezeDurationHistogramName     HistogramType = "orchestrator.sandbox.envd.freeze.duration"
	EnvdFreezeSweepHistogramName        HistogramType = "orchestrator.sandbox.envd.freeze.sweep"
	EnvdFreezeWaitHistogramName         HistogramType = "orchestrator.sandbox.envd.freeze.wait"
	EnvdFreezeVisitedHistogramName      HistogramType = "orchestrator.sandbox.envd.freeze.visited"
	EnvdFreezeAuditHistogramName        HistogramType = "orchestrator.sandbox.envd.freeze.audit"
	EnvdFreezeCgroupsHistogramName      HistogramType = "orchestrator.sandbox.envd.freeze.cgroups"
	EnvdUnfreezeDurationHistogramName   HistogramType = "orchestrator.sandbox.envd.unfreeze.duration"
	UffdStartupPagesHistogramName       HistogramType = "orchestrator.sandbox.uffd.startup.pages"
	UffdStartupSourcePagesHistogramName HistogramType = "orchestrator.sandbox.uffd.startup.source_pages"
	UffdStartupBytesHistogramName       HistogramType = "orchestrator.sandbox.uffd.startup.bytes"

	// TCP Firewall histograms
	TCPFirewallConnectionDurationHistogramName    HistogramType = "orchestrator.tcpfirewall.connection.duration"
	TCPFirewallConnectionsPerSandboxHistogramName HistogramType = "orchestrator.tcpfirewall.connections.per_sandbox"

	// Ingress proxy histograms
	IngressProxyConnectionDurationHistogramName    HistogramType = "orchestrator.proxy.connection.duration"
	IngressProxyConnectionsPerSandboxHistogramName HistogramType = "orchestrator.proxy.connections.per_sandbox"
)
const (
	ApiRedisStoragePublisherPublishDuration HistogramType = "api.redis_storage.publisher.publish.duration"

	// Firecracker net histograms — per-sandbox distribution per metrics flush, no sandbox_id.
	// Firecracker serializes SharedIncMetric as per-flush deltas (default flush interval: 60 s).
	// Symmetric TX/RX metrics carry a direction=tx/rx attribute; TX-only metrics always use direction=tx.
	SandboxFCNetBytes                HistogramType = "orchestrator.sandbox.fc.net.bytes"
	SandboxFCNetPackets              HistogramType = "orchestrator.sandbox.fc.net.packets"
	SandboxFCNetCount                HistogramType = "orchestrator.sandbox.fc.net.count"
	SandboxFCNetRateLimiterThrottled HistogramType = "orchestrator.sandbox.fc.net.rate_limiter_throttled"
	// TX-only: no RX equivalent in Firecracker metrics.
	SandboxFCNetRateLimiterEventCount HistogramType = "orchestrator.sandbox.fc.net.rate_limiter_event_count"
	SandboxFCNetRemainingReqs         HistogramType = "orchestrator.sandbox.fc.net.remaining_reqs"

	// Firecracker block histograms — per-sandbox distribution per metrics flush, no sandbox_id.
	// Symmetric read/write metrics carry a direction=read/write attribute.
	SandboxFCBlockBytes                 HistogramType = "orchestrator.sandbox.fc.block.bytes"
	SandboxFCBlockCount                 HistogramType = "orchestrator.sandbox.fc.block.count"
	SandboxFCBlockQueueEventCount       HistogramType = "orchestrator.sandbox.fc.block.queue_event_count"
	SandboxFCBlockRateLimiterThrottled  HistogramType = "orchestrator.sandbox.fc.block.rate_limiter_throttled"
	SandboxFCBlockRateLimiterEventCount HistogramType = "orchestrator.sandbox.fc.block.rate_limiter_event_count"
	SandboxFCBlockIOEngineThrottled     HistogramType = "orchestrator.sandbox.fc.block.io_engine_throttled"
	SandboxFCBlockRemainingReqs         HistogramType = "orchestrator.sandbox.fc.block.remaining_reqs"

	SnapshotDiffBytes  HistogramType = "orchestrator.sandbox.snapshot.diff.bytes"
	SnapshotDiffRatio  HistogramType = "orchestrator.sandbox.snapshot.diff.ratio"
	SnapshotTotalBytes HistogramType = "orchestrator.sandbox.snapshot.total.bytes"

	UploadUncompressedBytes HistogramType = "orchestrator.sandbox.upload.uncompressed.bytes"
	UploadCompressedBytes   HistogramType = "orchestrator.sandbox.upload.compressed.bytes"
	UploadCompressionRatio  HistogramType = "orchestrator.sandbox.upload.compression.ratio"
)

type LogProvider

type LogProvider interface {
	log.LoggerProvider
	Shutdown(ctx context.Context) error
}

LogProvider extends log.LoggerProvider with a Shutdown method so the batch processor and exporter are properly flushed on exit.

func NewLogProvider

func NewLogProvider(ctx context.Context, res *resource.Resource, extraOpts ...otlploggrpc.Option) (LogProvider, error)

func NewNoopLogProvider

func NewNoopLogProvider() LogProvider

type ObservableCounterType

type ObservableCounterType string
const (
	ApiOrchestratorSbxCreateSuccess ObservableCounterType = "api.orchestrator.sandbox.create.success"
	ApiOrchestratorSbxCreateFailure ObservableCounterType = "api.orchestrator.sandbox.create.failure"
)

type ObservableUpDownCounterType

type ObservableUpDownCounterType string
const (
	OrchestratorSandboxCountMeterName ObservableUpDownCounterType = "orchestrator.env.sandbox.running"

	ClientProxyServerConnectionsMeterCounterName ObservableUpDownCounterType = "client_proxy.proxy.server.connections.open"
	ClientProxyPoolConnectionsMeterCounterName   ObservableUpDownCounterType = "client_proxy.proxy.pool.connections.open"
	ClientProxyPoolSizeMeterCounterName          ObservableUpDownCounterType = "client_proxy.proxy.pool.size"

	OrchestratorProxyServerConnectionsMeterCounterName ObservableUpDownCounterType = "orchestrator.proxy.server.connections.open"
	OrchestratorProxyPoolConnectionsMeterCounterName   ObservableUpDownCounterType = "orchestrator.proxy.pool.connections.open"
	OrchestratorProxyPoolSizeMeterCounterName          ObservableUpDownCounterType = "orchestrator.proxy.pool.size"

	BuildCounterMeterName       ObservableUpDownCounterType = "api.env.build.running"
	EvictionsRunningCounterName ObservableUpDownCounterType = "api.evictor.evictions.running"

	TCPFirewallActiveConnections ObservableUpDownCounterType = "orchestrator.tcpfirewall.connections.active"

	ApiRedisStoragePublisherQueueDepth ObservableUpDownCounterType = "api.redis_storage.publisher.queue.depth"
)

type Stopwatch

type Stopwatch struct {
	// contains filtered or unexported fields
}

func (Stopwatch) Failure

func (t Stopwatch) Failure(ctx context.Context, total int64, kv ...attribute.KeyValue)

func (Stopwatch) RecordRaw

func (t Stopwatch) RecordRaw(ctx context.Context, total int64, allAttrs metric.MeasurementOption)

RecordRaw records an operation using a precomputed attribute option, it does not include any previous attributes passed at Begin(). Zero-allocation alternative to Success/Failure for hot paths.

func (Stopwatch) Success

func (t Stopwatch) Success(ctx context.Context, total int64, kv ...attribute.KeyValue)

type TimerFactory

type TimerFactory struct {
	// contains filtered or unexported fields
}

func NewTimerFactory

func NewTimerFactory(
	blocksMeter metric.Meter,
	metricName, durationDescription, bytesDescription, counterDescription string,
) (TimerFactory, error)

func (*TimerFactory) Begin

func (f *TimerFactory) Begin(kv ...attribute.KeyValue) *Stopwatch

type ZapFieldToOTELAttributeEncoder

type ZapFieldToOTELAttributeEncoder struct {
	attribute.KeyValue
}

func (*ZapFieldToOTELAttributeEncoder) AddArray

func (*ZapFieldToOTELAttributeEncoder) AddBinary

func (z *ZapFieldToOTELAttributeEncoder) AddBinary(key string, value []byte)

func (*ZapFieldToOTELAttributeEncoder) AddBool

func (z *ZapFieldToOTELAttributeEncoder) AddBool(key string, value bool)

func (*ZapFieldToOTELAttributeEncoder) AddByteString

func (z *ZapFieldToOTELAttributeEncoder) AddByteString(key string, value []byte)

func (*ZapFieldToOTELAttributeEncoder) AddComplex64

func (z *ZapFieldToOTELAttributeEncoder) AddComplex64(key string, value complex64)

func (*ZapFieldToOTELAttributeEncoder) AddComplex128

func (z *ZapFieldToOTELAttributeEncoder) AddComplex128(key string, value complex128)

func (*ZapFieldToOTELAttributeEncoder) AddDuration

func (z *ZapFieldToOTELAttributeEncoder) AddDuration(key string, value time.Duration)

func (*ZapFieldToOTELAttributeEncoder) AddFloat32

func (z *ZapFieldToOTELAttributeEncoder) AddFloat32(key string, value float32)

func (*ZapFieldToOTELAttributeEncoder) AddFloat64

func (z *ZapFieldToOTELAttributeEncoder) AddFloat64(key string, value float64)

func (*ZapFieldToOTELAttributeEncoder) AddInt

func (z *ZapFieldToOTELAttributeEncoder) AddInt(key string, value int)

func (*ZapFieldToOTELAttributeEncoder) AddInt8

func (z *ZapFieldToOTELAttributeEncoder) AddInt8(key string, value int8)

func (*ZapFieldToOTELAttributeEncoder) AddInt16

func (z *ZapFieldToOTELAttributeEncoder) AddInt16(key string, value int16)

func (*ZapFieldToOTELAttributeEncoder) AddInt32

func (z *ZapFieldToOTELAttributeEncoder) AddInt32(key string, value int32)

func (*ZapFieldToOTELAttributeEncoder) AddInt64

func (z *ZapFieldToOTELAttributeEncoder) AddInt64(key string, value int64)

func (*ZapFieldToOTELAttributeEncoder) AddObject

func (*ZapFieldToOTELAttributeEncoder) AddReflected

func (z *ZapFieldToOTELAttributeEncoder) AddReflected(key string, value any) error

func (*ZapFieldToOTELAttributeEncoder) AddString

func (z *ZapFieldToOTELAttributeEncoder) AddString(key, value string)

func (*ZapFieldToOTELAttributeEncoder) AddTime

func (z *ZapFieldToOTELAttributeEncoder) AddTime(key string, value time.Time)

func (*ZapFieldToOTELAttributeEncoder) AddUint

func (z *ZapFieldToOTELAttributeEncoder) AddUint(key string, value uint)

func (*ZapFieldToOTELAttributeEncoder) AddUint8

func (z *ZapFieldToOTELAttributeEncoder) AddUint8(key string, value uint8)

func (*ZapFieldToOTELAttributeEncoder) AddUint16

func (z *ZapFieldToOTELAttributeEncoder) AddUint16(key string, value uint16)

func (*ZapFieldToOTELAttributeEncoder) AddUint32

func (z *ZapFieldToOTELAttributeEncoder) AddUint32(key string, value uint32)

func (*ZapFieldToOTELAttributeEncoder) AddUint64

func (z *ZapFieldToOTELAttributeEncoder) AddUint64(key string, value uint64)

func (*ZapFieldToOTELAttributeEncoder) AddUintptr

func (z *ZapFieldToOTELAttributeEncoder) AddUintptr(key string, value uintptr)

func (*ZapFieldToOTELAttributeEncoder) OpenNamespace

func (z *ZapFieldToOTELAttributeEncoder) OpenNamespace(_ string)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL