Documentation
¶
Index ¶
- Constants
- Variables
- func GetCounter(meter metric.Meter, name CounterType) (metric.Int64Counter, error)
- func GetFloatHistogram(meter metric.Meter, name HistogramType) (metric.Float64Histogram, error)
- func GetGaugeFloat(meter metric.Meter, name GaugeFloatType) (metric.Float64ObservableGauge, error)
- func GetGaugeInt(meter metric.Meter, name GaugeIntType) (metric.Int64ObservableGauge, error)
- func GetHistogram(meter metric.Meter, name HistogramType) (metric.Int64Histogram, error)
- func GetObservableCounter(meter metric.Meter, name ObservableCounterType, callback metric.Int64Callback) (metric.Int64ObservableCounter, error)
- func GetObservableUpDownCounter(meter metric.Meter, name ObservableUpDownCounterType, ...) (metric.Int64ObservableUpDownCounter, error)
- func GetResource(ctx context.Context, ...) (*resource.Resource, error)
- func NewEventWriter(ctx context.Context, name string) io.Writer
- func NewMeterExporter(ctx context.Context, extraOption ...otlpmetricgrpc.Option) (sdkmetric.Exporter, error)
- func NewMeterProvider(metricsExporter sdkmetric.Exporter, metricExportPeriod time.Duration, ...) (*sdkmetric.MeterProvider, error)
- func NewPprofMux() *http.ServeMux
- func NewPprofServer() *http.Server
- func NewSpanExporter(ctx context.Context, extraOption ...otlptracegrpc.Option) (sdktrace.SpanExporter, error)
- func NewTextPropagator() propagation.TextMapPropagator
- func NewTracerProvider(spanExporter sdktrace.SpanExporter, res *resource.Resource) trace.TracerProvider
- func OTELCollectorGRPCEndpoint() string
- func Observe0(ctx context.Context, tracer trace.Tracer, name string, ...) error
- func Observe1[T any](ctx context.Context, tracer trace.Tracer, name string, ...) (T, error)
- func ParseEdgeTraceID(gcpHeader, awsHeader string) (string, bool)
- func PprofPort() int
- func PrecomputeAttrs(kv ...attribute.KeyValue) metric.MeasurementOption
- func ReportCriticalError(ctx context.Context, message string, err error, attrs ...attribute.KeyValue)
- func ReportError(ctx context.Context, message string, err error, attrs ...attribute.KeyValue)
- func ReportErrorByCode(ctx context.Context, code int, message string, err error, ...)
- func ReportEvent(ctx context.Context, name string, attrs ...attribute.KeyValue)
- func SetAttributes(ctx context.Context, attrs ...attribute.KeyValue)
- func WithBuildID(buildID string) attribute.KeyValue
- func WithClusterID(clusterID uuid.UUID) attribute.KeyValue
- func WithEdgeTraceID(traceID string) attribute.KeyValue
- func WithEnvdVersion(envdVersion string) attribute.KeyValue
- func WithFirecrackerVersion(firecrackerVersion string) attribute.KeyValue
- func WithKernelVersion(kernelVersion string) attribute.KeyValue
- func WithMaskedAPIKey(maskedAPIKey string) attribute.KeyValue
- func WithNodeID(nodeID string) attribute.KeyValue
- func WithSandboxID(sandboxID string) attribute.KeyValue
- func WithServiceInstanceID(serviceInstanceID string) attribute.KeyValue
- func WithTeamID(teamID string) attribute.KeyValue
- func WithTemplateID(templateID string) attribute.KeyValue
- func WithUserID(userID string) attribute.KeyValue
- type Client
- type CounterType
- type EventWriter
- type FloatTimerFactory
- type GaugeFloatType
- type GaugeIntType
- type HistogramType
- type LogProvider
- type ObservableCounterType
- type ObservableUpDownCounterType
- type Stopwatch
- type TimerFactory
- type ZapFieldToOTELAttributeEncoder
- func (z *ZapFieldToOTELAttributeEncoder) AddArray(_ string, _ zapcore.ArrayMarshaler) error
- func (z *ZapFieldToOTELAttributeEncoder) AddBinary(key string, value []byte)
- func (z *ZapFieldToOTELAttributeEncoder) AddBool(key string, value bool)
- func (z *ZapFieldToOTELAttributeEncoder) AddByteString(key string, value []byte)
- func (z *ZapFieldToOTELAttributeEncoder) AddComplex64(key string, value complex64)
- func (z *ZapFieldToOTELAttributeEncoder) AddComplex128(key string, value complex128)
- func (z *ZapFieldToOTELAttributeEncoder) AddDuration(key string, value time.Duration)
- func (z *ZapFieldToOTELAttributeEncoder) AddFloat32(key string, value float32)
- func (z *ZapFieldToOTELAttributeEncoder) AddFloat64(key string, value float64)
- func (z *ZapFieldToOTELAttributeEncoder) AddInt(key string, value int)
- func (z *ZapFieldToOTELAttributeEncoder) AddInt8(key string, value int8)
- func (z *ZapFieldToOTELAttributeEncoder) AddInt16(key string, value int16)
- func (z *ZapFieldToOTELAttributeEncoder) AddInt32(key string, value int32)
- func (z *ZapFieldToOTELAttributeEncoder) AddInt64(key string, value int64)
- func (z *ZapFieldToOTELAttributeEncoder) AddObject(_ string, _ zapcore.ObjectMarshaler) error
- func (z *ZapFieldToOTELAttributeEncoder) AddReflected(key string, value any) error
- func (z *ZapFieldToOTELAttributeEncoder) AddString(key, value string)
- func (z *ZapFieldToOTELAttributeEncoder) AddTime(key string, value time.Time)
- func (z *ZapFieldToOTELAttributeEncoder) AddUint(key string, value uint)
- func (z *ZapFieldToOTELAttributeEncoder) AddUint8(key string, value uint8)
- func (z *ZapFieldToOTELAttributeEncoder) AddUint16(key string, value uint16)
- func (z *ZapFieldToOTELAttributeEncoder) AddUint32(key string, value uint32)
- func (z *ZapFieldToOTELAttributeEncoder) AddUint64(key string, value uint64)
- func (z *ZapFieldToOTELAttributeEncoder) AddUintptr(key string, value uintptr)
- func (z *ZapFieldToOTELAttributeEncoder) OpenNamespace(_ string)
Constants ¶
const ( GCPTraceContextHeader = "X-Cloud-Trace-Context" AWSTraceContextHeader = "X-Amzn-Trace-Id" )
const ( ApiOrchestratorCountMeterName GaugeIntType = "api.orchestrator.status" OrchestratorStatusGaugeName GaugeIntType = "orchestrator.status" // Orchestrator node resources allocated to running sandboxes (sum across running sandboxes) OrchestratorCpuAllocatedGaugeName GaugeIntType = "orchestrator.sandbox.cpu.allocated" OrchestratorMemoryAllocatedGaugeName GaugeIntType = "orchestrator.sandbox.memory.allocated" OrchestratorDiskAllocatedGaugeName GaugeIntType = "orchestrator.sandbox.disk.allocated" // Sandbox metrics SandboxRamUsedGaugeName GaugeIntType = "e2b.sandbox.ram.used" SandboxRamTotalGaugeName GaugeIntType = "e2b.sandbox.ram.total" SandboxRamCacheGaugeName GaugeIntType = "e2b.sandbox.ram.cache" SandboxCpuTotalGaugeName GaugeIntType = "e2b.sandbox.cpu.total" SandboxDiskUsedGaugeName GaugeIntType = "e2b.sandbox.disk.used" SandboxDiskTotalGaugeName GaugeIntType = "e2b.sandbox.disk.total" // Team metrics TeamSandboxRunningGaugeName GaugeIntType = "e2b.team.sandbox.running" SandboxCountGaugeName GaugeIntType = "api.env.instance.running" // Build resource metrics BuildRootfsSizeHistogramName HistogramType = "template.build.rootfs.size" )
const DefaultPprofPort = 6060
DefaultPprofPort is the default port that we should use to mount pprof endpoint.
Variables ¶
var ( // Pre-allocated result attributes for use with PrecomputeAttrs. Success = attribute.String(resultAttr, resultTypeSuccess) Failure = attribute.String(resultAttr, resultTypeFailure) )
var SubMillisecondMsBuckets = []float64{
0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000,
}
SubMillisecondMsBuckets resolve sub-ms operations (mmap / cache hits) that the default OTEL buckets (first boundary 5ms) collapse into one, while still covering remote reads to ~10s.
Functions ¶
func GetCounter ¶
func GetCounter(meter metric.Meter, name CounterType) (metric.Int64Counter, error)
func GetFloatHistogram ¶
func GetFloatHistogram(meter metric.Meter, name HistogramType) (metric.Float64Histogram, error)
func GetGaugeFloat ¶
func GetGaugeFloat(meter metric.Meter, name GaugeFloatType) (metric.Float64ObservableGauge, error)
func GetGaugeInt ¶
func GetGaugeInt(meter metric.Meter, name GaugeIntType) (metric.Int64ObservableGauge, error)
func GetHistogram ¶
func GetHistogram(meter metric.Meter, name HistogramType) (metric.Int64Histogram, error)
GetHistogram returns an Int64 histogram with the registered description and unit. It sets no bucket boundaries: histogramAggregation discards them.
func GetObservableCounter ¶
func GetObservableCounter(meter metric.Meter, name ObservableCounterType, callback metric.Int64Callback) (metric.Int64ObservableCounter, error)
func GetObservableUpDownCounter ¶
func GetObservableUpDownCounter(meter metric.Meter, name ObservableUpDownCounterType, callback metric.Int64Callback) (metric.Int64ObservableUpDownCounter, error)
func GetResource ¶
func NewMeterExporter ¶
func NewMeterProvider ¶
func NewPprofMux ¶
NewPprofMux returns a ServeMux with pprof handlers registered on a dedicated mux rather than http.DefaultServeMux, preventing accidental exposure.
Mirrors the five handlers that `_ "net/http/pprof"` registers in its init(). The Index handler already serves all runtime/pprof profiles (heap, goroutine, allocs, block, mutex, threadcreate, etc.) by name from the URL path.
func NewPprofServer ¶
func NewSpanExporter ¶
func NewSpanExporter(ctx context.Context, extraOption ...otlptracegrpc.Option) (sdktrace.SpanExporter, error)
func NewTextPropagator ¶
func NewTextPropagator() propagation.TextMapPropagator
func NewTracerProvider ¶
func NewTracerProvider(spanExporter sdktrace.SpanExporter, res *resource.Resource) trace.TracerProvider
func OTELCollectorGRPCEndpoint ¶
func OTELCollectorGRPCEndpoint() string
func Observe0 ¶
func Observe0(ctx context.Context, tracer trace.Tracer, name string, fn func(context.Context) error, opts ...trace.SpanStartOption) error
Observe0 records one synchronous operation returning only an error.
func Observe1 ¶
func Observe1[T any](ctx context.Context, tracer trace.Tracer, name string, fn func(context.Context) (T, error), opts ...trace.SpanStartOption) (T, error)
Observe1 records one synchronous operation returning one value and an error.
func ParseEdgeTraceID ¶
ParseEdgeTraceID extracts a trace ID from cloud-provider trace headers. These headers are untrusted and are only used for cross-service correlation.
func PprofPort ¶
func PprofPort() int
PprofPort returns the port the pprof server should bind to: the value of PPROF_PORT when set to a valid port, otherwise DefaultPprofPort.
func PrecomputeAttrs ¶
func PrecomputeAttrs(kv ...attribute.KeyValue) metric.MeasurementOption
PrecomputeAttrs builds a reusable MeasurementOption from the given attribute key-values. The option must include all attributes (including "result"). Use with Stopwatch.Record to avoid per-call attribute allocation.
func ReportCriticalError ¶
func ReportError ¶
func ReportErrorByCode ¶
func WithBuildID ¶
func WithEdgeTraceID ¶
func WithEnvdVersion ¶
func WithFirecrackerVersion ¶
func WithKernelVersion ¶
func WithMaskedAPIKey ¶
func WithNodeID ¶
func WithSandboxID ¶
func WithServiceInstanceID ¶
func WithTeamID ¶
func WithTemplateID ¶
func WithUserID ¶
Types ¶
type Client ¶
type Client struct {
MetricExporter sdkmetric.Exporter
MeterProvider metric.MeterProvider
SpanExporter sdktrace.SpanExporter
TracerProvider trace.TracerProvider
TracePropagator propagation.TextMapPropagator
LogsProvider LogProvider
// contains filtered or unexported fields
}
func New ¶
func New(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, additional ...attribute.KeyValue) (*Client, error)
New creates a telemetry client that exports traces, metrics, and logs via gRPC. Telemetry is enabled when the OTEL_COLLECTOR_GRPC_ENDPOINT environment variable is set (e.g. "localhost:4317"). When unset, a noop client is returned with zero overhead.
func NewAnonymous ¶
NewAnonymous creates a telemetry client for tools and CLI commands that don't have build-time injected metadata (commitSHA, version, nodeID). serviceName is the primary identifier used for filtering traces and metrics in observability tools (e.g. Grafana). The remaining resource attributes are filled with sensible defaults (hostname, "unknown" commit, "dev" version).
func NewNoopClient ¶
func NewNoopClient() *Client
func (*Client) StartRuntimeInstrumentation ¶
StartRuntimeInstrumentation registers OTEL Go runtime metric callbacks.
Collected metrics (semantic-convention names):
- go.memory.used
- go.memory.limit
- go.memory.allocated
- go.memory.allocations
- go.memory.gc.goal
- go.goroutine.count
- go.processor.limit
- go.config.gogc
The callbacks are invoked by the MeterProvider and stop automatically when it shuts down — no separate goroutine is spawned.
type CounterType ¶
type CounterType string
const ( ApiOrchestratorCreatedSandboxes CounterType = "api.orchestrator.created_sandboxes" ApiOrchestratorResumeOriginNodeRemap CounterType = "api.orchestrator.resume_origin_node_remapped" // ApiOrchestratorPauseRefusalRestore counts what became of a pause the // node refused retryably: outcome restored | restore_failed | // route_restore_failed, caller request | eviction. ApiOrchestratorPauseRefusalRestore CounterType = "api.orchestrator.pause_refusal_restore" // ApiEvictorFsOnlyAutoPause counts timeout auto-pauses whose policy asked // for a filesystem-only snapshot. Unlabeled since the fs-only version // gate was dropped: every counted eviction takes the fs-only path. ApiEvictorFsOnlyAutoPause CounterType = "api.evictor.fs_only_auto_pause" // ApiEvictorAutoPauseDegraded counts timeout auto-pauses the evictor // downgraded to a filesystem-only snapshot after the node refused the // memory one, by cause: admission_refused (no retry budget, degraded at // the first refusal) or overstay_budget (the retry budget since the first // refusal ran out); a subset of ApiEvictorFsOnlyAutoPause. ApiEvictorAutoPauseDegraded CounterType = "api.evictor.auto_pause_degraded" SandboxCreateMeterName CounterType = "api.env.instance.started" TeamSandboxCreated CounterType = "e2b.team.sandbox.created" EnvdInitCalls CounterType = "orchestrator.sandbox.envd.init.calls" // 2 MiB chunks the pre-pause envd heap collapse attempted, split by the // result attribute (collapsed|skipped): attempts = total, successful = // collapsed. EnvdCollapseChunks CounterType = "orchestrator.sandbox.envd.collapse.chunks" // Incremented by the balance_dirty_pages thread count at every 200 ms poll // for the lifetime of the process. rate() shows dirty-page throttle // intensity in real-time; 0 when no stalls are occurring. OrchestratorHostBalanceDirtyPagesThreads CounterType = "orchestrator.host.balance_dirty_pages.threads" OrchestratorSandboxKilledCounterName CounterType = "orchestrator.sandbox.killed" // OrchestratorSandboxPauseAdmissionCounterName counts every snapshot- // admission decision. refused is the told-the-caller-to-retry rate; // ready_after_wait counts the grace paying off — their ratio tunes the // grace flag. OrchestratorSandboxPauseAdmissionCounterName CounterType = "orchestrator.sandbox.pause_admission" // OrchestratorSandboxCheckpointCounterName counts Checkpoint RPCs by flow — // the denominator that makes the in_place-labeled duration histograms // cuttable as a ramp (what fraction went in-place, at what success rate). OrchestratorSandboxCheckpointCounterName CounterType = "orchestrator.sandbox.checkpoint" // OrchestratorFPRResumeCounterName counts free-page-reporting resume // outcomes after a CoW window paused reporting. Guest memory is hugetlb // with no swap and no memcg reclaim, so FPR discards are the ONLY // mechanism returning freed guest pages to the host: outcome="abandoned" // is lasting per-sandbox host-memory retention and the alertable signal // for a leaked pause. OrchestratorFPRResumeCounterName CounterType = "orchestrator.sandbox.fpr_resume" // OrchestratorSnapshotUploadFailedCounterName counts pause-snapshot uploads // that never landed durably (budget exhausted or a non-retryable error). // A non-zero rate means lost snapshots. OrchestratorSnapshotUploadFailedCounterName CounterType = "orchestrator.snapshot.upload.failed" // SandboxPauseFsQuiescedCounterName counts filesystem-only pauses by whether // the captured rootfs was frozen (quiesced=true, crash-consistent) vs a plain // sync fallback (quiesced=false). quiesced/total is the fraction of newly // minted fs-only snapshots that are safe to cold-boot / rewrite — the eligible // population for the offline envd upgrade built on top of this flag. SandboxPauseFsQuiescedCounterName CounterType = "orchestrator.sandbox.pause.fs_quiesced" // SandboxResumeWPModeCounterName counts sandbox resumes by the write-protect // tracking mode the resume chose (mode=sync|async, the use-sync-wp flag // decision). This is the denominator for every sync-WP burn-in signal: // wp_resolve rates and dirty-divergence readings only come from mode=sync // sandboxes, so without this counter the fleet's sync coverage — and // whether an error rate is "all sync sandboxes" or "one loud one" — is // invisible. SandboxResumeWPModeCounterName CounterType = "orchestrator.sandbox.resume.wp_mode" // OrchestratorEnvdUpgradeAttempts counts resume-time envd live-upgrade // attempts, by result (success|delivery_failed|not_ready|panic) and // from_version/to_version. success/total is the rollout success rate; // attempts/resumes is the fire rate. OrchestratorEnvdUpgradeAttempts CounterType = "orchestrator.envd.upgrade.attempts" // OrchestratorEnvdUpgradeGated counts resumes the envd-upgrade-target flag // targeted but a gate declined, by reason — a silent no-op worth watching during // a ramp. The reasons are the resolver's and the gates' own vocabulary; read them // off `sum by (reason)` rather than from a list here, which would go stale. OrchestratorEnvdUpgradeGated CounterType = "orchestrator.envd.upgrade.gated" // OrchestratorEnvdUpgradeHandover counts live-upgrade handover items by item // (proc|retained|watcher) and result (ok|failed), reported back by the new // envd on /init. failed/(ok+failed) is the handover error rate — the // fleet-visible signal (otherwise only logged in-guest) that the new envd // dropped or degraded something it re-adopted across the swap. OrchestratorEnvdUpgradeHandover CounterType = "orchestrator.envd.upgrade.handover" // EnvdDefaultsApplied counts MEMORY RESUMES by where the default user the orchestrator // sent came from, so the derivation has a positive engagement signal rather than an // inferred one: a derivation that produced nothing and one that never ran read // identically on a dashboard. // // Resumes, not starts: the cold-boot path does its own reconstruction and is not // counted here. Split by sandbox_type, because a build's own internal resumes go // through the same path and are not customer sandbox starts. EnvdDefaultsApplied CounterType = "orchestrator.envd.defaults.applied" // EnvdDefaultsMismatch counts /init responses where the defaults envd reports as // EFFECTIVE differ from what the orchestrator sent, by field. Zero is the only // acceptable steady state: a non-zero value means a sandbox is resolving requests // against an identity nobody asked for. Note the header is written inside a guest, // so a single hostile or broken sandbox can hold this above zero; investigate via // the per-sandbox error log rather than treating the aggregate as authoritative. EnvdDefaultsMismatch CounterType = "orchestrator.envd.defaults.mismatch" // EnvdDefaultsWorkdirWithheld counts MEMORY RESUMES where the template recorded a // default workdir the orchestrator could not prove the build sent, so it was not // re-sent — the same event EnvdDefaultsApplied counts, so the two divide. An // UPPER BOUND on the population that keeps the wrong working directory after a live // upgrade: a workdir equal to the resolved user's home is withheld to no effect, and // whether it coincides cannot be decided outside the guest. EnvdDefaultsWorkdirWithheld CounterType = "orchestrator.envd.defaults.workdir_withheld" // EnvdDefaultsBuiltinFallback counts /init responses where envd reports it was never // told which user to run as, so it is serving the value it was compiled with. This is // the REALIZED loss, and the only signal that can observe one: a mismatch cannot, // because envd stores exactly what /init sent it and then reports that field back. // // Split by whether a user was sent, and by sandbox type. sent=true is a defect at any // type, because a delivered user must never read back as never-told. sent=false is only // about the population at risk at sandbox_type="sandbox": the build tree starts its own // sandboxes with no default user configured, on the host envd, so it lands in // sent=false from the first day of the rollout and would otherwise dominate the bucket. EnvdDefaultsBuiltinFallback CounterType = "orchestrator.envd.defaults.builtin_fallback" // OrchestratorEnvdOfflineUpgradeAttempts counts every OFFLINE envd upgrade // OUTCOME on the cold boot of a filesystem-only snapshot — not only the ones that // reach the rootfs binary swap — by result and from_version/to_version: // // success | swap_failed | unrecoverable | envd_too_large | envd_missing | stat_unparseable // the swap ran, or the rootfs itself declined it (see rootfs.SwapEnvdBinary) // not_quiesced // an upgrade was wanted, but the snapshot's rootfs was not frozen at pause, // so it must not be rewritten // same_version // already on the target // not_staged | downgrade | invalid_target | getversion_failed // the resolver refused the configured target // // Every no-op except flag-off is counted, so the eligible population adds up. // Flag-off is deliberately absent: it is the whole filesystem-only population // minus the rest, already available as sandbox.create.duration{fs_only="true"}. // to_version is empty on the resolver's refusals — it has no target to name. // // On result=success a refire label says whether the rootfs ALREADY held the target // bytes. from_version cannot answer that on its own: it is a claim read off the // snapshot record, and the swap keys on that record without ever advancing it, so // an already-upgraded snapshot resolves the same upgrade on every cold boot and // rewrites identical bytes while reporting the same from_version as a genuine // upgrade. refire="false" is the count that actually moved sandboxes. The label is // absent on every other result, where nothing was compared. OrchestratorEnvdOfflineUpgradeAttempts CounterType = "orchestrator.envd.offline_upgrade.attempts" // OrchestratorFsRecoveryRuns counts every pre-boot filesystem-recovery // decision on a cold boot, by result and trigger. Recovery is journal replay // only (`e2fsck -p -E journal_only`), so the results are: // // skipped_quiesced // the rootfs was frozen at pause; nothing to replay // replayed // the journal was replayed (or regenerated, or there was nothing to do); // the fs is mountable and the boot proceeded — the expected outcome for // the admitted population, NOT a corruption signal. // failed_operational // the run did not complete a clean replay AND e2fsck may have opened the // device (an e2fsck non-replay exit, or a timeout that could have killed it // mid-write). Fail closed — the start failed — but always retryable on // another node, never a permanent snapshot verdict (its exit codes cannot // tell an unmountable filesystem apart from a transient fault). // failed_open // recovery could not run AND e2fsck provably never opened the device (the // jail could not launch it, or the host cannot exec e2fsck), so the disk is // what a flag-off cold boot would mount and the guest kernel replays the // journal itself. The boot proceeded. A host-image signal — a non-trivial // rate means the recovery tooling is broken fleet-wide (roll back), even // though sandboxes still boot. // // A bounded "reason" attribute sub-labels each result so a ramp can act on it // from a dashboard instead of grepping create-failure logs: // // replayed: nothing_to_do (no replay needed) | journal_replayed // (journal applied) — the efficacy split // failed_operational: timeout (Go deadline fired mid-run) | killed (the unit was // signalled mid-run — exit -1 or 128+N; OOM, RuntimeMaxSec, // external stop; NOT a tooling failure) | e2fsck_4 | e2fsck_8 | // e2fsck_other (e2fsck's // own non-replay exits — an unreplayable snapshot, expected and // small) | no_sentinel (ran but lost its result, or the pre-launch // device guard) // failed_open: launcher_failure (the unit failed to START — e2fsck never ran) | // exec_failure (126/127: the host could not exec e2fsck) — // both host-image regressions, roll back // skipped_quiesced: quiesced // // It never carries a raw exit code or any tenant-influenced bytes. // // trigger separates the two admitted populations: "rescue" (the request // demanded a filesystem boot of a memory snapshot) vs "legacy_fs_only" // (a filesystem-only snapshot whose pause fell back to sync). OrchestratorFsRecoveryRuns CounterType = "orchestrator.sandbox.fs_recovery.runs" // OrchestratorFsRecoveryToolingUnsupported fires once per orchestrator process // when the host e2fsck does not accept `-E journal_only` (probed against a // nonexistent device, so it reads no filesystem). Expected to be flat zero: node // images ship an e2fsprogs that has supported the option for years. A non-zero // fleet sum means some node's tooling silently no-ops recovery (the guest kernel // still replays at mount, so boots are unaffected) — a signal to fix that image, // not a per-sandbox failure. OrchestratorFsRecoveryToolingUnsupported CounterType = "orchestrator.sandbox.fs_recovery.tooling_unsupported" // TemplateBuildCmdlineArgs counts template builds by the guest kernel command line // parameters they actually booted with, after parsing and validation. This is the // engagement signal for the per-team cmdline-variant flag: a non-zero rate on a // non-empty label is proof builds ran the path, which is what the flag // reading "on" in the feature-flag service does not tell you. Zero here while the // flag is targeted is the alarm that the opt-in is silently inert. Because it // records what was APPLIED, a rejected fragment shows up as the empty label rather // than as the parameters that were asked for. TemplateBuildCmdlineArgs CounterType = "orchestrator.template.build.cmdline_args" // PauseResumePrefetchHarvestAttempts counts pause-resume prefetch harvest // attempts, by result (success|resume_failed|collect_failed|skipped). The // throwaway is absent from Prometheus otherwise (registration-skip), so this // is the harvest-activity / failure-rate signal. PauseResumePrefetchHarvestAttempts CounterType = "orchestrator.sandbox.pause_resume_prefetch.harvest.attempts" ApiRedisStoragePublisherPublished CounterType = "api.redis_storage.publisher.published" ApiRedisStoragePublisherDropped CounterType = "api.redis_storage.publisher.dropped" // ApiRedisStorageExpirationIndexHealed counts sandboxes the healer re-added // to the global expiration index. Healthy steady state is zero; a sustained // non-zero rate means expiration index writes are being lost and sandboxes // would otherwise become invisible to the evictor (immortal). ApiRedisStorageExpirationIndexHealed CounterType = "api.redis_storage.expiration_index.healed" // ApiRedisStorageExpirationIndexSwept counts members removed from the // global expiration index by the evictor scan // (reason=orphan|dead_execution|invalid). ApiRedisStorageExpirationIndexSwept CounterType = "api.redis_storage.expiration_index.swept" // ApiRedisStorageExpirationIndexRescored counts live members whose index // score drifted from the stored EndTime and were re-scored by the evictor // scan. Sustained non-zero rate means score updates are being lost. ApiRedisStorageExpirationIndexRescored CounterType = "api.redis_storage.expiration_index.rescored" )
const ( // Build result counters BuildResultCounterName CounterType = "template.build.result" BuildCacheResultCounterName CounterType = "template.build.cache.result" // TCP Firewall counters TCPFirewallConnectionsTotal CounterType = "orchestrator.tcpfirewall.connections.total" TCPFirewallErrorsTotal CounterType = "orchestrator.tcpfirewall.errors.total" TCPFirewallDecisionsTotal CounterType = "orchestrator.tcpfirewall.decisions.total" // Ingress proxy counters IngressProxyConnectionsBlockedTotal CounterType = "orchestrator.proxy.connections.blocked.total" // cmux counters CmuxErrorsTotal CounterType = "orchestrator.cmux.errors.total" // Firecracker net counters — global totals, no sandbox_id (low cardinality). // All carry a direction=tx/rx attribute. Per-sandbox distributions are histograms below. SandboxFCNetFails CounterType = "orchestrator.sandbox.fc.net.fails" SandboxFCNetNoAvailBuffer CounterType = "orchestrator.sandbox.fc.net.no_avail_buffer" SandboxFCNetTapIOFails CounterType = "orchestrator.sandbox.fc.net.tap_io_fails" // Firecracker block counters — global totals, no sandbox_id (low cardinality). // Carry a direction=read/write attribute where applicable. SandboxFCBlockFails CounterType = "orchestrator.sandbox.fc.block.fails" SandboxFCBlockNoAvailBuffer CounterType = "orchestrator.sandbox.fc.block.no_avail_buffer" )
type EventWriter ¶
type EventWriter struct {
// contains filtered or unexported fields
}
type FloatTimerFactory ¶
type FloatTimerFactory struct {
// contains filtered or unexported fields
}
FloatTimerFactory records duration as fractional milliseconds so sub-ms operations aren't truncated to 0. The duration histogram and event counter share <metricName> (rate()-friendly); only the bytes counter splits out to <metricName>.size so Grafana's unit detection doesn't conflate ms with By.
func NewFloatTimerFactory ¶
func NewFloatTimerFactory( meter metric.Meter, metricName, durationDescription, bytesDescription string, ) (FloatTimerFactory, error)
func (*FloatTimerFactory) Record ¶
func (f *FloatTimerFactory) Record(ctx context.Context, dur time.Duration, total int64, attrs metric.MeasurementOption)
type GaugeFloatType ¶
type GaugeFloatType string
const (
SandboxCpuUsedGaugeName GaugeFloatType = "e2b.sandbox.cpu.used"
)
type GaugeIntType ¶
type GaugeIntType string
type HistogramType ¶
type HistogramType string
const ( // Build timing histograms BuildDurationHistogramName HistogramType = "template.build.duration" BuildPhaseDurationHistogramName HistogramType = "template.build.phase.duration" BuildStepDurationHistogramName HistogramType = "template.build.step.duration" // Sandbox timing histograms OrchestratorSandboxCreateDurationName HistogramType = "orchestrator.sandbox.create.duration" WaitForEnvdDurationHistogramName HistogramType = "orchestrator.sandbox.envd.init.duration" GuestSyncDurationHistogramName HistogramType = "orchestrator.sandbox.guest_sync.duration" PauseDurationHistogramName HistogramType = "orchestrator.sandbox.pause.duration" SnapshotProcessMemoryDurationName HistogramType = "orchestrator.sandbox.snapshot.process_memory.duration" SnapshotProcessRootfsDurationName HistogramType = "orchestrator.sandbox.snapshot.process_rootfs.duration" SnapshotRootfsSealDurationName HistogramType = "orchestrator.sandbox.snapshot.rootfs_seal.duration" SnapshotGuestFreezeDurationName HistogramType = "orchestrator.sandbox.snapshot.guest_freeze.duration" SnapshotMemorySealDurationName HistogramType = "orchestrator.sandbox.snapshot.memory_seal.duration" // OrchestratorSandboxExecutionDurationName is one sample per Firecracker // run, so a sandbox that is paused and resumed records one per run. OrchestratorSandboxExecutionDurationName HistogramType = "orchestrator.sandbox.execution.duration" // OrchestratorEnvdUpgradeDurationName is the wall-time of a resume-time envd // live-upgrade (delivery + trigger + WaitForEnvd) = overhead added to the // resume. Labeled by result. OrchestratorEnvdUpgradeDurationName HistogramType = "orchestrator.envd.upgrade.duration" // OrchestratorEnvdOfflineUpgradeDurationName is the wall-time of the offline // rootfs envd swap (jailed debugfs), recorded once per swap attempt. Catches // pathological rewrites; the swap runs in the cold-boot PreBootFn, so it adds // directly to resume latency. OrchestratorEnvdOfflineUpgradeDurationName HistogramType = "orchestrator.envd.offline_upgrade.duration" // OrchestratorFsRecoveryDurationName is the wall-time of the jailed pre-boot // journal-replay run on a cold boot, recorded for every outcome the run reaches // (replayed/failed_operational) — only skipped_quiesced, // which never runs e2fsck, has no duration sample. Replay is journal-bounded, so // this should stay sub-second even on large filesystems. Labeled like // OrchestratorFsRecoveryRuns. OrchestratorFsRecoveryDurationName HistogramType = "orchestrator.sandbox.fs_recovery.duration" // Pre-pause envd heap collapse round-trip duration (the pause-path cost of // POST /collapse: network plus envd's madvise work), recorded once per pause // when the collapse-envd-heap flag is on. EnvdCollapseDurationHistogramName HistogramType = "orchestrator.sandbox.envd.collapse.duration" // Background memfile dedup latency: provisional header creation at pause to // the durable-header swap. Recorded once per swap; no attributes. OrchestratorSandboxMemfileDedupDurationName HistogramType = "orchestrator.sandbox.memfile_dedup.duration" // How long snapshot admission actually waited whenever it waited, labeled // by outcome (ready_after_wait/refused only). OrchestratorSandboxPauseAdmissionWaitDurationName HistogramType = "orchestrator.sandbox.pause_admission.wait.duration" // Pause-resume prefetch harvest cost, recorded once per harvest attempt. // duration is the SLOT-HOLD cost alone — the throwaway resume, its trace // collection and its reap — which is what the harvest-timeout flag caps. // persist_wait is the separate, resource-free wait on the in-flight snapshot // upload that the mapping has to be written after; keeping it out of duration // is what lets "harvests at the timeout" stay a meaningful signal. // pages is the harvested trace size (distinct 2 MiB blocks), recorded only on // success, so its bottom bucket surfaces the empty-trace (idle-at-pause) rate. PauseResumePrefetchHarvestDurationName HistogramType = "orchestrator.sandbox.pause_resume_prefetch.harvest.duration" PauseResumePrefetchHarvestPagesName HistogramType = "orchestrator.sandbox.pause_resume_prefetch.harvest.pages" PauseResumePrefetchSealWaitDurationName HistogramType = "orchestrator.sandbox.pause_resume_prefetch.seal_wait.duration" PauseResumePrefetchPersistWaitDurationName HistogramType = "orchestrator.sandbox.pause_resume_prefetch.persist_wait.duration" // Sandbox startup working-set histograms: demand-fault pages/bytes a guest // needed to reach a successful envd init, recorded once per start. Sampled // per start (not per fault), so histogram_quantile yields per-sandbox // percentiles. EnvdFreezeDurationHistogramName HistogramType = "orchestrator.sandbox.envd.freeze.duration" EnvdFreezeSweepHistogramName HistogramType = "orchestrator.sandbox.envd.freeze.sweep" EnvdFreezeWaitHistogramName HistogramType = "orchestrator.sandbox.envd.freeze.wait" EnvdFreezeVisitedHistogramName HistogramType = "orchestrator.sandbox.envd.freeze.visited" EnvdFreezeAuditHistogramName HistogramType = "orchestrator.sandbox.envd.freeze.audit" EnvdFreezeCgroupsHistogramName HistogramType = "orchestrator.sandbox.envd.freeze.cgroups" EnvdUnfreezeDurationHistogramName HistogramType = "orchestrator.sandbox.envd.unfreeze.duration" UffdStartupPagesHistogramName HistogramType = "orchestrator.sandbox.uffd.startup.pages" UffdStartupSourcePagesHistogramName HistogramType = "orchestrator.sandbox.uffd.startup.source_pages" UffdStartupBytesHistogramName HistogramType = "orchestrator.sandbox.uffd.startup.bytes" // TCP Firewall histograms TCPFirewallConnectionDurationHistogramName HistogramType = "orchestrator.tcpfirewall.connection.duration" TCPFirewallConnectionsPerSandboxHistogramName HistogramType = "orchestrator.tcpfirewall.connections.per_sandbox" // Ingress proxy histograms IngressProxyConnectionDurationHistogramName HistogramType = "orchestrator.proxy.connection.duration" IngressProxyConnectionsPerSandboxHistogramName HistogramType = "orchestrator.proxy.connections.per_sandbox" )
const ( ApiRedisStoragePublisherPublishDuration HistogramType = "api.redis_storage.publisher.publish.duration" // Firecracker net histograms — per-sandbox distribution per metrics flush, no sandbox_id. // Firecracker serializes SharedIncMetric as per-flush deltas (default flush interval: 60 s). // Symmetric TX/RX metrics carry a direction=tx/rx attribute; TX-only metrics always use direction=tx. SandboxFCNetBytes HistogramType = "orchestrator.sandbox.fc.net.bytes" SandboxFCNetPackets HistogramType = "orchestrator.sandbox.fc.net.packets" SandboxFCNetCount HistogramType = "orchestrator.sandbox.fc.net.count" SandboxFCNetRateLimiterThrottled HistogramType = "orchestrator.sandbox.fc.net.rate_limiter_throttled" // TX-only: no RX equivalent in Firecracker metrics. SandboxFCNetRateLimiterEventCount HistogramType = "orchestrator.sandbox.fc.net.rate_limiter_event_count" SandboxFCNetRemainingReqs HistogramType = "orchestrator.sandbox.fc.net.remaining_reqs" // Firecracker block histograms — per-sandbox distribution per metrics flush, no sandbox_id. // Symmetric read/write metrics carry a direction=read/write attribute. SandboxFCBlockBytes HistogramType = "orchestrator.sandbox.fc.block.bytes" SandboxFCBlockCount HistogramType = "orchestrator.sandbox.fc.block.count" SandboxFCBlockQueueEventCount HistogramType = "orchestrator.sandbox.fc.block.queue_event_count" SandboxFCBlockRateLimiterThrottled HistogramType = "orchestrator.sandbox.fc.block.rate_limiter_throttled" SandboxFCBlockRateLimiterEventCount HistogramType = "orchestrator.sandbox.fc.block.rate_limiter_event_count" SandboxFCBlockIOEngineThrottled HistogramType = "orchestrator.sandbox.fc.block.io_engine_throttled" SandboxFCBlockRemainingReqs HistogramType = "orchestrator.sandbox.fc.block.remaining_reqs" SnapshotDiffBytes HistogramType = "orchestrator.sandbox.snapshot.diff.bytes" SnapshotDiffRatio HistogramType = "orchestrator.sandbox.snapshot.diff.ratio" SnapshotTotalBytes HistogramType = "orchestrator.sandbox.snapshot.total.bytes" UploadUncompressedBytes HistogramType = "orchestrator.sandbox.upload.uncompressed.bytes" UploadCompressedBytes HistogramType = "orchestrator.sandbox.upload.compressed.bytes" UploadCompressionRatio HistogramType = "orchestrator.sandbox.upload.compression.ratio" )
type LogProvider ¶
type LogProvider interface {
log.LoggerProvider
Shutdown(ctx context.Context) error
}
LogProvider extends log.LoggerProvider with a Shutdown method so the batch processor and exporter are properly flushed on exit.
func NewLogProvider ¶
func NewLogProvider(ctx context.Context, res *resource.Resource, extraOpts ...otlploggrpc.Option) (LogProvider, error)
func NewNoopLogProvider ¶
func NewNoopLogProvider() LogProvider
type ObservableCounterType ¶
type ObservableCounterType string
const ( ApiOrchestratorSbxCreateSuccess ObservableCounterType = "api.orchestrator.sandbox.create.success" ApiOrchestratorSbxCreateFailure ObservableCounterType = "api.orchestrator.sandbox.create.failure" )
type ObservableUpDownCounterType ¶
type ObservableUpDownCounterType string
const ( OrchestratorSandboxCountMeterName ObservableUpDownCounterType = "orchestrator.env.sandbox.running" ClientProxyServerConnectionsMeterCounterName ObservableUpDownCounterType = "client_proxy.proxy.server.connections.open" ClientProxyPoolConnectionsMeterCounterName ObservableUpDownCounterType = "client_proxy.proxy.pool.connections.open" ClientProxyPoolSizeMeterCounterName ObservableUpDownCounterType = "client_proxy.proxy.pool.size" OrchestratorProxyServerConnectionsMeterCounterName ObservableUpDownCounterType = "orchestrator.proxy.server.connections.open" OrchestratorProxyPoolConnectionsMeterCounterName ObservableUpDownCounterType = "orchestrator.proxy.pool.connections.open" OrchestratorProxyPoolSizeMeterCounterName ObservableUpDownCounterType = "orchestrator.proxy.pool.size" BuildCounterMeterName ObservableUpDownCounterType = "api.env.build.running" EvictionsRunningCounterName ObservableUpDownCounterType = "api.evictor.evictions.running" TCPFirewallActiveConnections ObservableUpDownCounterType = "orchestrator.tcpfirewall.connections.active" ApiRedisStoragePublisherQueueDepth ObservableUpDownCounterType = "api.redis_storage.publisher.queue.depth" )
type Stopwatch ¶
type Stopwatch struct {
// contains filtered or unexported fields
}
type TimerFactory ¶
type TimerFactory struct {
// contains filtered or unexported fields
}
func NewTimerFactory ¶
func NewTimerFactory( blocksMeter metric.Meter, metricName, durationDescription, bytesDescription, counterDescription string, ) (TimerFactory, error)
type ZapFieldToOTELAttributeEncoder ¶
func (*ZapFieldToOTELAttributeEncoder) AddArray ¶
func (z *ZapFieldToOTELAttributeEncoder) AddArray(_ string, _ zapcore.ArrayMarshaler) error
func (*ZapFieldToOTELAttributeEncoder) AddBinary ¶
func (z *ZapFieldToOTELAttributeEncoder) AddBinary(key string, value []byte)
func (*ZapFieldToOTELAttributeEncoder) AddBool ¶
func (z *ZapFieldToOTELAttributeEncoder) AddBool(key string, value bool)
func (*ZapFieldToOTELAttributeEncoder) AddByteString ¶
func (z *ZapFieldToOTELAttributeEncoder) AddByteString(key string, value []byte)
func (*ZapFieldToOTELAttributeEncoder) AddComplex64 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddComplex64(key string, value complex64)
func (*ZapFieldToOTELAttributeEncoder) AddComplex128 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddComplex128(key string, value complex128)
func (*ZapFieldToOTELAttributeEncoder) AddDuration ¶
func (z *ZapFieldToOTELAttributeEncoder) AddDuration(key string, value time.Duration)
func (*ZapFieldToOTELAttributeEncoder) AddFloat32 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddFloat32(key string, value float32)
func (*ZapFieldToOTELAttributeEncoder) AddFloat64 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddFloat64(key string, value float64)
func (*ZapFieldToOTELAttributeEncoder) AddInt ¶
func (z *ZapFieldToOTELAttributeEncoder) AddInt(key string, value int)
func (*ZapFieldToOTELAttributeEncoder) AddInt8 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddInt8(key string, value int8)
func (*ZapFieldToOTELAttributeEncoder) AddInt16 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddInt16(key string, value int16)
func (*ZapFieldToOTELAttributeEncoder) AddInt32 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddInt32(key string, value int32)
func (*ZapFieldToOTELAttributeEncoder) AddInt64 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddInt64(key string, value int64)
func (*ZapFieldToOTELAttributeEncoder) AddObject ¶
func (z *ZapFieldToOTELAttributeEncoder) AddObject(_ string, _ zapcore.ObjectMarshaler) error
func (*ZapFieldToOTELAttributeEncoder) AddReflected ¶
func (z *ZapFieldToOTELAttributeEncoder) AddReflected(key string, value any) error
func (*ZapFieldToOTELAttributeEncoder) AddString ¶
func (z *ZapFieldToOTELAttributeEncoder) AddString(key, value string)
func (*ZapFieldToOTELAttributeEncoder) AddTime ¶
func (z *ZapFieldToOTELAttributeEncoder) AddTime(key string, value time.Time)
func (*ZapFieldToOTELAttributeEncoder) AddUint ¶
func (z *ZapFieldToOTELAttributeEncoder) AddUint(key string, value uint)
func (*ZapFieldToOTELAttributeEncoder) AddUint8 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddUint8(key string, value uint8)
func (*ZapFieldToOTELAttributeEncoder) AddUint16 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddUint16(key string, value uint16)
func (*ZapFieldToOTELAttributeEncoder) AddUint32 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddUint32(key string, value uint32)
func (*ZapFieldToOTELAttributeEncoder) AddUint64 ¶
func (z *ZapFieldToOTELAttributeEncoder) AddUint64(key string, value uint64)
func (*ZapFieldToOTELAttributeEncoder) AddUintptr ¶
func (z *ZapFieldToOTELAttributeEncoder) AddUintptr(key string, value uintptr)
func (*ZapFieldToOTELAttributeEncoder) OpenNamespace ¶
func (z *ZapFieldToOTELAttributeEncoder) OpenNamespace(_ string)