featureflags

package
v0.0.0-...-8e108db Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 26, 2026 License: Apache-2.0 Imports: 15 Imported by: 27

Documentation

Index

Constants

View Source
const (
	SandboxKind                        ldcontext.Kind = "sandbox"
	SandboxTemplateAttribute           string         = "template-id"
	SandboxKernelVersionAttribute      string         = "kernel-version"
	SandboxFirecrackerVersionAttribute string         = "firecracker-version"
	// SandboxTypeAttribute distinguishes "sandbox" from "build" runs.
	SandboxTypeAttribute string = "sandbox-type"

	TeamKind    ldcontext.Kind = "team"
	UserKind    ldcontext.Kind = "user"
	ClusterKind ldcontext.Kind = "cluster"

	TierKind             ldcontext.Kind = "tier"
	ServiceKind          ldcontext.Kind = "service"
	TemplateKind         ldcontext.Kind = "template"
	VolumeKind           ldcontext.Kind = "volume"
	CompressFileTypeKind ldcontext.Kind = "compress-file-type"
	CompressUseCaseKind  ldcontext.Kind = "compress-use-case"

	OrchestratorKind            ldcontext.Kind = "orchestrator"
	OrchestratorCommitAttribute string         = "commit"
)

kinds

View Source
const (
	DefaultFirecackerV1_10Version = "v1.10.1_30cbb07"
	DefaultFirecackerV1_12Version = "v1.12.1_210cbac"
	DefaultFirecackerV1_14Version = "v1.14.1_bd85e43"
	DefaultFirecrackerVersion     = DefaultFirecackerV1_12Version
)

The Firecracker version the last tag + the short SHA (so we can build our dev previews) TODO: The short tag here has only 7 characters — the one from our build pipeline will likely have exactly 8 so this will break.

View Source
const (
	DefaultKernelVersion = "vmlinux-6.1.158"
)

Variables

View Source
var (
	MetricsWriteFlag                    = NewBoolFlag("sandbox-metrics-write", true)
	MetricsReadFlag                     = NewBoolFlag("sandbox-metrics-read", true)
	SnapshotFeatureFlag                 = NewBoolFlag("use-nfs-for-snapshots", env.IsDevelopment())
	TemplateFeatureFlag                 = NewBoolFlag("use-nfs-for-templates", env.IsDevelopment())
	EnableWriteThroughCacheFlag         = NewBoolFlag("write-to-cache-on-writes", false)
	UseNFSCacheForBuildingTemplatesFlag = NewBoolFlag("use-nfs-for-building-templates", env.IsDevelopment())
	BestOfKCanFitFlag                   = NewBoolFlag("best-of-k-can-fit", true)
	BestOfKTooManyStartingFlag          = NewBoolFlag("best-of-k-too-many-starting", false)
	EdgeProvidedSandboxMetricsFlag      = NewBoolFlag("edge-provided-sandbox-metrics", false)
	CreateStorageCacheSpansFlag         = NewBoolFlag("create-storage-cache-spans", env.IsDevelopment())
	SandboxAutoResumeFlag               = NewBoolFlag("sandbox-auto-resume", env.IsDevelopment())
	OrchAcceptsCombinedHostFlag         = NewBoolFlag("orch-accepts-combined-host", false)

	// UseMemFdFlag asks Firecracker to back guest memory with a memfd and
	// pass the fd over the UFFD socket; the orchestrator then mmaps it
	// directly instead of using process_vm_readv on pause.
	UseMemFdFlag = NewBoolFlag("use-memfd", false)

	// MemfdBackgroundCopyFlag streams the memfd into the snapshot cache on
	// a goroutine so Pause returns as soon as the diff metadata is written.
	// Only takes effect when UseMemFdFlag is also on.
	MemfdBackgroundCopyFlag = NewBoolFlag("memfd-background-copy", false)

	// MemfileDiffDedupFlag enables 4 KiB-page dedup of the memfile diff
	// against the base memfile. bestEffort skips uncached blocks;
	// directIO opens the dedup output with O_DIRECT.
	MemfileDiffDedupFlag = NewJSONFlag("memfile-diff-dedup", ldvalue.FromJSONMarshal(map[string]any{
		"enabled":    false,
		"bestEffort": false,
		"directIO":   false,
	}))

	// PeerToPeerChunkTransferFlag enables peer-to-peer chunk routing.
	PeerToPeerChunkTransferFlag = NewBoolFlag("peer-to-peer-chunk-transfer", false)
	// PeerToPeerAsyncCheckpointFlag makes Checkpoint upload fire-and-forget instead
	// of synchronous. Only safe to enable after PeerToPeerChunkTransferFlag is ON.
	PeerToPeerAsyncCheckpointFlag = NewBoolFlag("peer-to-peer-async-checkpoint", false)

	PersistentVolumesFlag            = NewBoolFlag("can-use-persistent-volumes", env.IsDevelopment())
	ExecutionMetricsOnWebhooksFlag   = NewBoolFlag("execution-metrics-on-webhooks", false) // TODO: Remove NLT 20250315
	SandboxLabelBasedSchedulingFlag  = NewBoolFlag("sandbox-label-based-scheduling", false)
	OptimisticResourceAccountingFlag = NewBoolFlag("sandbox-placement-optimistic-resource-accounting", false)
	FreePageReportingFlag            = NewBoolFlag("free-page-reporting", false)
	FreezeUserCgroupFlag             = NewBoolFlag("freeze-user-cgroup", env.IsDevelopment())

	NetworkTransformRulesFlag = NewBoolFlag("network-transform-rules", env.IsDevelopment())

	// V4HeaderForUncompressedFlag forces the V4 header layout on uncompressed
	// uploads. Independent of compress-config: it changes the header format,
	// not whether data is compressed.
	V4HeaderForUncompressedFlag = NewBoolFlag("v4-header-for-uncompressed", false)
)
View Source
var (
	MaxSandboxesPerNode           = NewIntFlag("max-sandboxes-per-node", 200)
	GcloudConcurrentUploadLimit   = NewIntFlag("gcloud-concurrent-upload-limit", 8)
	GcloudMaxTasks                = NewIntFlag("gcloud-max-tasks", 16)
	ClickhouseBatcherMaxBatchSize = NewIntFlag("clickhouse-batcher-max-batch-size", 100)
	ClickhouseBatcherMaxDelay     = NewIntFlag("clickhouse-batcher-max-delay", 1000) // 1s in milliseconds
	ClickhouseBatcherQueueSize    = NewIntFlag("clickhouse-batcher-queue-size", 1000)
	BestOfKSampleSize             = NewIntFlag("best-of-k-sample-size", 3)                   // Default K=3
	BestOfKMaxOvercommit          = NewIntFlag("best-of-k-max-overcommit", 400)              // Default R=4 (stored as percentage, max over-commit ratio)
	BestOfKAlpha                  = NewIntFlag("best-of-k-alpha", 50)                        // Default Alpha=0.5 (stored as percentage for int flag, current usage weight)
	EnvdInitTimeoutMilliseconds   = NewIntFlag("envd-init-request-timeout-milliseconds", 50) // Timeout for envd init request in milliseconds
	HostStatsSamplingInterval     = NewIntFlag("host-stats-sampling-interval", 5000)         // Host stats sampling interval in milliseconds (default 5s)
	MaxCacheWriterConcurrencyFlag = NewIntFlag("max-cache-writer-concurrency", 10)

	// BuildCacheMaxUsagePercentage the maximum percentage of the cache disk storage
	// that can be used before the cache starts evicting items.
	BuildCacheMaxUsagePercentage = NewIntFlag("build-cache-max-usage-percentage", 85)
	BuildProvisionVersion        = NewIntFlag("build-provision-version", 0)

	// NBDConnectionsPerDevice the number of NBD socket connections per device
	NBDConnectionsPerDevice = NewIntFlag("nbd-connections-per-device", 1)

	// MemoryPrefetchMaxFetchWorkers is the maximum number of parallel fetch workers per sandbox for memory prefetching.
	// Fetching is I/O bound so we can have more parallelism.
	MemoryPrefetchMaxFetchWorkers = NewIntFlag("memory-prefetch-max-fetch-workers", 16)

	// MemoryPrefetchMaxCopyWorkers is the maximum number of parallel copy workers per sandbox for memory prefetching.
	// Copy uses uffd syscalls, so we limit parallelism to avoid overwhelming the system.
	MemoryPrefetchMaxCopyWorkers = NewIntFlag("memory-prefetch-max-copy-workers", 8)

	// TCPFirewallMaxConnectionsPerSandbox is the maximum number of concurrent TCP firewall
	// connections allowed per sandbox. Negative means no limit.
	TCPFirewallMaxConnectionsPerSandbox = NewIntFlag("tcpfirewall-max-connections-per-sandbox", -1)

	// SandboxMaxIncomingConnections is the maximum number of concurrent HTTP proxy
	// connections allowed per sandbox. Negative means no limit.
	SandboxMaxIncomingConnections = NewIntFlag("sandbox-max-incoming-connections", -1)

	// BuildBaseRootfsSizeLimitMB is the maximum size of the base rootfs filesystem created from the OCI image, in MB.
	BuildBaseRootfsSizeLimitMB = NewIntFlag("build-base-rootfs-size-limit-mb", 25000)

	// MinAutoResumeTimeoutSeconds is the minimum auto-resume timeout in seconds.
	// This prevents thrashing from very short timeouts.
	MinAutoResumeTimeoutSeconds = NewIntFlag("minimum-autoresume-timeout", 300)

	// BuildReservedDiskSpaceMB is the amount of disk space in MB reserved for root on the guest filesystem.
	// Reserved blocks are only usable by root (uid 0), protecting the guest OS from disk-full conditions.
	BuildReservedDiskSpaceMB = NewIntFlag("build-reserved-disk-space-mb", 0)

	// MaxStartingInstancesPerNode limits concurrent sandbox start/resume operations on a single orchestrator node.
	// Must be > 0.
	MaxStartingInstancesPerNode = NewIntFlag("max-starting-instances-per-node", 3)

	// MaxConcurrentEvictions caps the number of sandbox evictions that can run
	// in parallel per API instance. Excess items remain expired in the store
	// and are picked up by the next eviction tick. Must be > 0; non-positive
	// values are ignored at refresh time.
	MaxConcurrentEvictions = NewIntFlag("max-concurrent-evictions", 256)

	// MaxConcurrentSnapshotUpserts limits concurrent UpsertSnapshot calls (pause + snapshot template paths).
	// 0 or negative disables throttling (unlimited concurrency).
	MaxConcurrentSnapshotUpserts = NewIntFlag("max-concurrent-snapshot-upserts", 0)
	// MaxConcurrentSandboxListQueries limits concurrent GetSnapshotsWithCursor calls in the sandbox list path.
	// 0 or negative disables throttling (unlimited concurrency).
	MaxConcurrentSandboxListQueries = NewIntFlag("max-concurrent-sandbox-list-queries", 0)
	// MaxConcurrentSnapshotBuildQueries limits concurrent GetSnapshotBuilds calls (e.g. sandbox delete).
	// 0 or negative disables throttling (unlimited concurrency).
	MaxConcurrentSnapshotBuildQueries = NewIntFlag("max-concurrent-snapshot-build-queries", 0)

	MinChunkerReadSizeKB = NewIntFlag("min-chunker-read-size-kb", 16)
)
View Source
var (
	BuildFirecrackerVersion     = NewStringFlag("build-firecracker-version", env.GetEnv("DEFAULT_FIRECRACKER_VERSION", DefaultFirecrackerVersion))
	BuildKernelVersion          = NewStringFlag("build-kernel-version", env.GetEnv("DEFAULT_KERNEL_VERSION", DefaultKernelVersion))
	BuildIoEngine               = NewStringFlag("build-io-engine", "Sync")
	DefaultPersistentVolumeType = NewStringFlag("default-persistent-volume-type", "")
	BuildNodeInfo               = NewJSONFlag("preferred-build-node", ldvalue.Null())
	FirecrackerVersions         = NewJSONFlag("firecracker-versions", ldvalue.FromJSONMarshal(FirecrackerVersionMap))
)

BuildIoEngine Sync is used by default as there seems to be a bad interaction between Async and a lot of io operations.

View Source
var BlockDriveThrottleConfig = NewJSONFlag("block-drive-throttle-config", ldvalue.FromJSONMarshal(map[string]any{
	"ops":       map[string]any{"bucketSize": -1, "oneTimeBurst": 0, "refillTimeMs": 1000},
	"bandwidth": map[string]any{"bucketSize": -1, "oneTimeBurst": 0, "refillTimeMs": 1000},
}))

BlockDriveThrottleConfig controls per-sandbox block device (disk) throttling via Firecracker's VMM-level token bucket rate limiters on the rootfs drive. Structure mirrors the Firecracker RateLimiter API: two independent token buckets. Set bucketSize to -1 to disable a bucket.

Ops bucket (IOPS): effective rate = ops.bucketSize * 1000 / ops.refillTimeMs ops/s. Bandwidth bucket (bytes): effective rate = bandwidth.bucketSize * 1000 / bandwidth.refillTimeMs bytes/s.

View Source
var CleanNFSCache = NewJSONFlag("clean-nfs-cache", ldvalue.Null())
View Source
var CompressConfigFlag = NewJSONFlag("compress-config", ldvalue.FromJSONMarshal(map[string]any{
	"compressBuilds":     false,
	"compressionType":    "",
	"compressionLevel":   0,
	"frameSizeKB":        0,
	"minPartSizeMB":      0,
	"frameEncodeWorkers": 0,
	"encoderConcurrency": 0,
}))

CompressConfigFlag controls compression during template builds. When compressBuilds is true, builds upload exclusively compressed data (no uncompressed fallback). When false, exclusively uncompressed with V3 headers (unless V4HeaderForUncompressedFlag is set).

View Source
var FreePageHintingConfig = NewJSONFlag("free-page-hinting-config", ldvalue.Null())

FreePageHintingConfig controls virtio-balloon free-page-hinting. "enabled" configures FreePageHinting=true on the balloon at install time (kernel-side eligibility is targeted separately via the LD context — the race fixed in https://lore.kernel.org/lkml/20240429125100.7393-1-david@redhat.com/ is on the hinting flow, gated by the per-use-case timeouts below). "pause"/"build" are pre-pause drain timeouts in ms keyed by SnapshotUseCase; missing/zero/negative disables the drain for that use case. Example: {"enabled": true, "pause": 500, "build": 0}

View Source
var RateLimitConfigFlag = NewJSONFlag("rate-limit-config", ldvalue.Null())

RateLimitConfigFlag provides per-team rate limit overrides. JSON format:

{
  "/sandboxes/": {"rate": 50, "burst": 100},
  "/sandboxes/:sandboxID/pause": {"rate": 10, "burst": 20}
}

When non-null, values override the code defaults. Target specific teams in LaunchDarkly.

View Source
var ReclaimConfigFlag = NewJSONFlag("guest-pause-reclaim", ldvalue.Null())

ReclaimConfigFlag holds per-step caps in milliseconds for the pre-pause reclaim chain. Missing/zero/negative values disable the step. Example: {"sync":500,"drop_caches":200,"compact_memory":1000,"fstrim":500}

View Source
var TCPFirewallEgressThrottleConfig = NewJSONFlag("tcpfirewall-egress-throttle-config", ldvalue.FromJSONMarshal(map[string]any{
	"ops":       map[string]any{"bucketSize": -1, "oneTimeBurst": 0, "refillTimeMs": 1000},
	"bandwidth": map[string]any{"bucketSize": -1, "oneTimeBurst": 0, "refillTimeMs": 1000},
}))

TCPFirewallEgressThrottleConfig controls per-sandbox egress throttling via Firecracker's VMM-level token bucket rate limiters on the network interface. Structure mirrors the Firecracker RateLimiter API: two independent token buckets. Set bucketSize to -1 to disable a bucket.

Ops bucket (packets): effective rate = ops.bucketSize * 1000 / ops.refillTimeMs ops/s. Bandwidth bucket (bytes): effective rate = bandwidth.bucketSize * 1000 / bandwidth.refillTimeMs bytes/s.

View Source
var TrackedTemplatesForMetrics = NewJSONFlag("tracked-templates-for-metrics", ldvalue.FromJSONMarshal(defaultTrackedTemplates))

TrackedTemplatesForMetrics is a JSON flag that defines which template aliases should be tracked in sandbox start time metrics. Templates not in this list will be grouped under "other" to reduce metric cardinality. JSON format: {"base": true, "code-interpreter-v1": true, ...}

Functions

func AddToContext

func AddToContext(ctx context.Context, contexts ...ldcontext.Context) context.Context

func ClusterContext

func ClusterContext(clusterID string) ldcontext.Context

func CompressFileTypeContext

func CompressFileTypeContext(fileType string) ldcontext.Context

func CompressUseCaseContext

func CompressUseCaseContext(useCase string) ldcontext.Context

func GetFreePageHintingTimeout

func GetFreePageHintingTimeout(ctx context.Context, ff *Client, useCase string, contexts ...ldcontext.Context) time.Duration

GetFreePageHintingTimeout returns the pre-pause FPH drain timeout for the given SnapshotUseCase. Zero means disabled.

func GetTrackedTemplatesSet

func GetTrackedTemplatesSet(ctx context.Context, ff *Client) map[string]struct{}

GetTrackedTemplatesSet fetches the TrackedTemplatesForMetrics flag and returns it as a set for efficient lookup. Only keys with a truthy value are included; keys set to false are ignored.

func IsFreePageHintingEnabled

func IsFreePageHintingEnabled(ctx context.Context, ff *Client, contexts ...ldcontext.Context) bool

IsFreePageHintingEnabled reports whether FPH should be configured on the balloon at install time.

func OverrideBoolFlag

func OverrideBoolFlag(flag BoolFlag, value bool)

OverrideBoolFlag forces a bool flag to a specific value in the offline store. Only takes effect when LAUNCH_DARKLY_API_KEY is not set (i.e. dev/CLI tools).

func OverrideJSONFlag

func OverrideJSONFlag(flag JSONFlag, value ldvalue.Value)

OverrideJSONFlag forces a JSON flag to a specific value in the offline store. Only takes effect when LAUNCH_DARKLY_API_KEY is not set (i.e. dev/CLI tools).

func ResolveFirecrackerVersion

func ResolveFirecrackerVersion(ctx context.Context, ff *Client, buildVersion string) string

ResolveFirecrackerVersion resolves the firecracker version using the FirecrackerVersions feature flag. The buildVersion format is "v1.12.1_210cbac" — we extract "v1.12" as the lookup key.

func SandboxContext

func SandboxContext(sandboxID string) ldcontext.Context

func ServiceContext

func ServiceContext(serviceName string) ldcontext.Context

func TeamContext

func TeamContext(teamID string) ldcontext.Context

func TeamContextWithName

func TeamContextWithName(teamID, teamName string) ldcontext.Context

func TemplateContext

func TemplateContext(templateID string) ldcontext.Context

func TierContext

func TierContext(tierID, tierName string) ldcontext.Context

func UserContext

func UserContext(userID string) ldcontext.Context

func VersionContext

func VersionContext(orchestratorID, commit string) ldcontext.Context

func VolumeContext

func VolumeContext(volumeName string) ldcontext.Context

Types

type BlockDriveThrottleConfigValue

type BlockDriveThrottleConfigValue struct {
	Ops       TokenBucketConfig
	Bandwidth TokenBucketConfig
}

BlockDriveThrottleConfigValue holds the parsed values of BlockDriveThrottleConfig.

func GetBlockDriveThrottleConfig

func GetBlockDriveThrottleConfig(ctx context.Context, ff *Client) BlockDriveThrottleConfigValue

GetBlockDriveThrottleConfig fetches and parses the BlockDriveThrottleConfig flag.

type BoolFlag

type BoolFlag struct {
	// contains filtered or unexported fields
}

func NewBoolFlag

func NewBoolFlag(name string, fallback bool) BoolFlag

func (BoolFlag) Fallback

func (f BoolFlag) Fallback() bool

func (BoolFlag) Key

func (f BoolFlag) Key() string

func (BoolFlag) String

func (f BoolFlag) String() string

type Client

type Client struct {
	// contains filtered or unexported fields
}

func NewClient

func NewClient() (*Client, error)

func NewClientWithDatasource

func NewClientWithDatasource(source *ldtestdata.TestDataSource) (*Client, error)

func NewClientWithLogLevel

func NewClientWithLogLevel(logLevel ldlog.LogLevel) (*Client, error)

NewClientWithLogLevel creates a client with a specific log level. Use ldlog.Error to suppress INFO/WARN logs in CLI tools.

func (*Client) BoolFlag

func (c *Client) BoolFlag(ctx context.Context, flag BoolFlag, contexts ...ldcontext.Context) bool

func (*Client) Close

func (c *Client) Close(ctx context.Context) error

func (*Client) IntFlag

func (c *Client) IntFlag(ctx context.Context, flag IntFlag, contexts ...ldcontext.Context) int

func (*Client) JSONFlag

func (c *Client) JSONFlag(ctx context.Context, flag JSONFlag, contexts ...ldcontext.Context) ldvalue.Value

func (*Client) SetDeploymentName

func (c *Client) SetDeploymentName(deploymentName string)

func (*Client) SetServiceName

func (c *Client) SetServiceName(serviceName string)

func (*Client) StringFlag

func (c *Client) StringFlag(ctx context.Context, flag StringFlag, contexts ...ldcontext.Context) string

func (*Client) WatchJSONFlag

func (c *Client) WatchJSONFlag(ctx context.Context, flag JSONFlag, contexts ...ldcontext.Context) (<-chan interfaces.FlagValueChangeEvent, func())

type IntFlag

type IntFlag struct {
	// contains filtered or unexported fields
}

func NewIntFlag

func NewIntFlag(name string, fallback int) IntFlag

func (IntFlag) Fallback

func (f IntFlag) Fallback() int

func (IntFlag) Key

func (f IntFlag) Key() string

func (IntFlag) String

func (f IntFlag) String() string

type JSONFlag

type JSONFlag struct {
	// contains filtered or unexported fields
}

func NewJSONFlag

func NewJSONFlag(name string, fallback ldvalue.Value) JSONFlag

func (JSONFlag) Fallback

func (f JSONFlag) Fallback() ldvalue.Value

func (JSONFlag) Key

func (f JSONFlag) Key() string

func (JSONFlag) String

func (f JSONFlag) String() string

type ReclaimConfig

type ReclaimConfig struct {
	Sync          time.Duration
	DropCaches    time.Duration
	CompactMemory time.Duration
	Fstrim        time.Duration
}

func GetReclaimConfig

func GetReclaimConfig(ctx context.Context, ff *Client, contexts ...ldcontext.Context) ReclaimConfig

type StringFlag

type StringFlag struct {
	// contains filtered or unexported fields
}

func NewStringFlag

func NewStringFlag(name string, fallback string) StringFlag

func (StringFlag) Fallback

func (f StringFlag) Fallback() string

func (StringFlag) Key

func (f StringFlag) Key() string

func (StringFlag) String

func (f StringFlag) String() string

type TCPFirewallEgressThrottleConfigValue

type TCPFirewallEgressThrottleConfigValue struct {
	Ops       TokenBucketConfig
	Bandwidth TokenBucketConfig
}

TCPFirewallEgressThrottleConfigValue holds the parsed values of TCPFirewallEgressThrottleConfig.

func GetTCPFirewallEgressThrottleConfig

func GetTCPFirewallEgressThrottleConfig(ctx context.Context, ff *Client) TCPFirewallEgressThrottleConfigValue

GetTCPFirewallEgressThrottleConfig fetches and parses the TCPFirewallEgressThrottleConfig flag.

type TokenBucketConfig

type TokenBucketConfig struct {
	BucketSize   int64
	OneTimeBurst int64
	RefillTimeMs int64
}

TokenBucketConfig holds parameters for a single Firecracker token bucket. BucketSize < 0 disables the bucket.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL