Documentation
¶
Overview ¶
Package orchestrator defines the interface for sandbox providers and implements the boot flow that provisions or restores environments.
Index ¶
- Constants
- Variables
- func IsNotFound(err error) bool
- func IsTerminalState(s VMState) bool
- func NewEventID() string
- func NormalizeArch(arch string) string
- func Schedule(spec Spec, hosts []*Host, policy PlacementPolicy) (*Host, PlacementDecision, error)
- func SchedulePreferring(spec Spec, hosts []*Host, policy PlacementPolicy, hints PlacementHints) (*Host, PlacementDecision, error)
- type AgentSpec
- type ArtifactEndpoint
- type ArtifactHolder
- type ArtifactMove
- type ArtifactMoved
- type ArtifactMover
- type AttachSpec
- type Attacher
- type BootOptions
- type BootResult
- type CapacityProber
- type Checkpoint
- type DeadLetterKind
- type DeadLetterRecord
- type Endpoint
- type EndpointReporter
- type Environment
- type EnvironmentEvent
- type EventRecord
- type ExecOptions
- type ExecResult
- type ExposeSpec
- type FleetConfig
- type FleetManager
- func (fm *FleetManager) Attach(ctx context.Context, vmID string, spec AttachSpec) (io.ReadWriteCloser, error)
- func (fm *FleetManager) AuditEvent(ctx context.Context, entityType, entityID, eventType string, ...)
- func (fm *FleetManager) CompleteTask(taskID string) error
- func (fm *FleetManager) CordonHost(ctx context.Context, hostID string) error
- func (fm *FleetManager) CreateSnapshot(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotRecord, error)
- func (fm *FleetManager) DeleteSnapshot(ctx context.Context, vmID, snapshotID string) error
- func (fm *FleetManager) DeleteSnapshotByID(ctx context.Context, snapshotID string) error
- func (fm *FleetManager) DestroyVM(ctx context.Context, vmID string) error
- func (fm *FleetManager) Drain(ctx context.Context, vmID string) error
- func (fm *FleetManager) Exec(ctx context.Context, vmID string, cmd []string, opts ExecOptions) (ExecResult, error)
- func (fm *FleetManager) ForkEnvironment(ctx context.Context, srcVMID string, opts ForkOptions) (string, error)
- func (fm *FleetManager) GetHost(hostID string) (Host, bool)
- func (fm *FleetManager) GetSnapshot(ctx context.Context, vmID, snapshotID string) (SnapshotRecord, error)
- func (fm *FleetManager) GetSnapshotByID(ctx context.Context, snapshotID string) (SnapshotRecord, error)
- func (fm *FleetManager) GetVM(vmID string) (VMInfo, bool)
- func (fm *FleetManager) GetVMByTask(taskID string) (VMInfo, bool)
- func (fm *FleetManager) HostsHoldingArtifact(ctx context.Context, tenantID, digest string) (map[string]ArtifactHolder, error)
- func (fm *FleetManager) ListFleet() []VMInfo
- func (fm *FleetManager) ListFleetFiltered(filter VMFilter) ([]VMInfo, string, error)
- func (fm *FleetManager) ListHosts() []Host
- func (fm *FleetManager) ListHostsFiltered(filter HostFilter) ([]Host, string, error)
- func (fm *FleetManager) ListSnapshots(ctx context.Context, vmID string) ([]SnapshotRecord, error)
- func (fm *FleetManager) ListSnapshotsFiltered(ctx context.Context, filter SnapshotFilter) ([]SnapshotRecord, string, error)
- func (fm *FleetManager) ProvisionAndAssign(ctx context.Context, taskID string, spec Spec, manifest []byte, ...) (*VMInfo, error)
- func (fm *FleetManager) RegisterHost(ctx context.Context, h Host, p Provider) error
- func (fm *FleetManager) RemoveHost(ctx context.Context, hostID string) error
- func (fm *FleetManager) ResolveLayer(ctx context.Context, tenantID, layerKey, arch string) (SnapshotRecord, bool, error)
- func (fm *FleetManager) RestoreSnapshot(ctx context.Context, vmID, snapshotID string) error
- func (fm *FleetManager) RestoreSnapshotByID(ctx context.Context, snapshotID string) error
- func (fm *FleetManager) RotateToken(ctx context.Context, vmID string) error
- func (fm *FleetManager) Start(ctx context.Context)
- func (fm *FleetManager) Stop()
- func (fm *FleetManager) SubscribeEnvironmentEvents(vmID string) (<-chan EnvironmentEvent, func())
- func (fm *FleetManager) UncordonHost(ctx context.Context, hostID string) error
- type ForkOptions
- type GPUDevice
- type HTTPStatusError
- type HealthState
- type HealthStatus
- type HealthcheckHTTP
- type HealthcheckSpec
- type Host
- type HostBackend
- type HostCapacity
- type HostFilter
- type HostRecord
- type HostState
- type MIGInstance
- type MemoryStateStore
- func (s *MemoryStateStore) AppendEvent(_ context.Context, event EventRecord) error
- func (s *MemoryStateStore) DeleteHost(_ context.Context, hostID string) error
- func (s *MemoryStateStore) DeleteSnapshot(_ context.Context, snapshotID string) error
- func (s *MemoryStateStore) DeleteVM(_ context.Context, vmID string) error
- func (s *MemoryStateStore) FindSnapshotByLayerKey(_ context.Context, tenantID, layerKey, arch string) (SnapshotRecord, bool, error)
- func (s *MemoryStateStore) GetHost(_ context.Context, hostID string) (HostRecord, error)
- func (s *MemoryStateStore) GetSnapshot(_ context.Context, snapshotID string) (SnapshotRecord, error)
- func (s *MemoryStateStore) ListDeadLetters(_ context.Context) ([]DeadLetterRecord, error)
- func (s *MemoryStateStore) ListHosts(_ context.Context) ([]HostRecord, error)
- func (s *MemoryStateStore) ListSnapshots(_ context.Context) ([]SnapshotRecord, error)
- func (s *MemoryStateStore) ListSnapshotsByDigest(_ context.Context, tenantID, digest string) ([]SnapshotRecord, error)
- func (s *MemoryStateStore) ListSnapshotsByVM(_ context.Context, vmID string) ([]SnapshotRecord, error)
- func (s *MemoryStateStore) ListTasks(_ context.Context) ([]TaskRecord, error)
- func (s *MemoryStateStore) ListVMs(_ context.Context) ([]VMRecord, error)
- func (s *MemoryStateStore) SnapshotQuotaUsage(_ context.Context, tenantID string) (SnapshotQuotaUsage, error)
- func (s *MemoryStateStore) UpsertDeadLetter(_ context.Context, entry DeadLetterRecord) error
- func (s *MemoryStateStore) UpsertHost(_ context.Context, host HostRecord) error
- func (s *MemoryStateStore) UpsertSnapshot(_ context.Context, snapshot SnapshotRecord) error
- func (s *MemoryStateStore) UpsertTask(_ context.Context, task TaskRecord) error
- func (s *MemoryStateStore) UpsertVM(_ context.Context, vm VMRecord) error
- type PlacementDecision
- type PlacementHints
- type PlacementPolicy
- type PostgresStateStore
- func (s *PostgresStateStore) AppendEvent(ctx context.Context, event EventRecord) error
- func (s *PostgresStateStore) ApplyMigrations(ctx context.Context) error
- func (s *PostgresStateStore) DeleteHost(ctx context.Context, hostID string) error
- func (s *PostgresStateStore) DeleteSnapshot(ctx context.Context, snapshotID string) error
- func (s *PostgresStateStore) DeleteVM(ctx context.Context, vmID string) error
- func (s *PostgresStateStore) FindSnapshotByLayerKey(ctx context.Context, tenantID, layerKey, arch string) (SnapshotRecord, bool, error)
- func (s *PostgresStateStore) GetHost(ctx context.Context, hostID string) (HostRecord, error)
- func (s *PostgresStateStore) GetSnapshot(ctx context.Context, snapshotID string) (SnapshotRecord, error)
- func (s *PostgresStateStore) ListDeadLetters(ctx context.Context) ([]DeadLetterRecord, error)
- func (s *PostgresStateStore) ListHosts(ctx context.Context) ([]HostRecord, error)
- func (s *PostgresStateStore) ListSnapshots(ctx context.Context) ([]SnapshotRecord, error)
- func (s *PostgresStateStore) ListSnapshotsByDigest(ctx context.Context, tenantID, digest string) ([]SnapshotRecord, error)
- func (s *PostgresStateStore) ListSnapshotsByVM(ctx context.Context, vmID string) ([]SnapshotRecord, error)
- func (s *PostgresStateStore) ListTasks(ctx context.Context) ([]TaskRecord, error)
- func (s *PostgresStateStore) ListVMs(ctx context.Context) ([]VMRecord, error)
- func (s *PostgresStateStore) SnapshotQuotaUsage(ctx context.Context, tenantID string) (SnapshotQuotaUsage, error)
- func (s *PostgresStateStore) UpsertDeadLetter(ctx context.Context, entry DeadLetterRecord) error
- func (s *PostgresStateStore) UpsertHost(ctx context.Context, h HostRecord) error
- func (s *PostgresStateStore) UpsertSnapshot(ctx context.Context, snapshot SnapshotRecord) error
- func (s *PostgresStateStore) UpsertTask(ctx context.Context, task TaskRecord) error
- func (s *PostgresStateStore) UpsertVM(ctx context.Context, vm VMRecord) error
- type Provider
- type ReconcileMetrics
- type ReconcileSummary
- type SnapshotCapable
- type SnapshotDeleter
- type SnapshotDigester
- type SnapshotExportRecord
- type SnapshotExportStatus
- type SnapshotFilter
- type SnapshotForkable
- type SnapshotMode
- type SnapshotOptions
- type SnapshotQuotaUsage
- type SnapshotRecord
- type SnapshotState
- type Spec
- type StateStore
- type TaskRecord
- type TaskRunStatus
- type TokenSetter
- type VMFilter
- type VMInfo
- type VMRecord
- type VMState
Constants ¶
const ( GuestHealthcheckPath = "/fuse/healthcheck.json" GuestHealthStatePath = "/fuse/health.json" )
Guest paths for the healthcheck. Both live under /fuse, which the guest mounts on tmpfs, alongside the manifest and secrets (see agent_profile.go).
GuestHealthcheckPath is written by the orchestrator before boot and is how the probe reaches the guest at all: the firecracker host agent ignores AgentSpec.Command and sends structured paths on a frozen wire, so a new flag would never be passed. fused looks for the config at this fixed path instead of being told about it.
GuestHealthStatePath is written by fused after every attempt and is what reconcileHealth reads back.
const ( DefaultPageLimit = 50 MaxPageLimit = 200 )
DefaultPageLimit is the page size used when a caller passes limit <= 0. MaxPageLimit is the ceiling a larger request is clamped to (never rejected — a caller that asks for too much just gets less than it asked for plus a cursor to keep paging).
const ( DefaultExecTimeout = 60 * time.Second MaxExecTimeout = 600 * time.Second )
Exec timeout bounds. Guest commands are bounded so a hung command cannot pin an SSH session on the host agent forever. MaxExecTimeout matches the host agent's own per-exec ceiling; asking for more would let the caller's deadline outlive the guest command it is waiting on.
const ( // ArchAMD64 is x86_64. It is also what an empty arch means everywhere: // hosts registered before arch existed are all x86_64. ArchAMD64 = "amd64" // ArchARM64 is aarch64 / Apple Silicon. ArchARM64 = "arm64" )
Host architectures, in GOARCH vocabulary. NormalizeArch maps the uname -m spellings agents and operators actually type onto these.
const DefaultFusedDrainCommand = "systemctl stop fused 2>/dev/null || pkill -TERM fused"
DefaultFusedDrainCommand is the fused profile's graceful-stop invocation, run inside the guest via Environment.ExecStream on Drain.
The fused agent traps SIGTERM and tears down its service DAG before exiting, so stopping it with a signal IS a full graceful drain. On the firecracker profile fused runs as a systemd unit, so `systemctl stop` delivers SIGTERM, waits for the clean exit, and (because the stop is clean) the unit's Restart=on-failure does NOT bring it back. The `pkill -TERM` fallback covers non-systemd guests, which also quiesce on SIGTERM.
No trailing `|| true`: a genuine failure (neither a unit nor a process was stopped) propagates a non-zero exit so Drain records the error and keeps the VM Draining. A graceful-stop command that silently no-op'd (e.g. a missing helper hidden behind `|| true`) would defeat that.
const DefaultMaxStartupScriptTimeout = 55 * time.Second
DefaultMaxStartupScriptTimeout is the largest bound a caller may request via BootOptions.StartupScriptTimeout when FleetConfig.MaxStartupScriptTimeout is unset.
The ceiling exists because the bound is caller-supplied but the cost is paid by the server: the script runs synchronously inside the create request, so a caller-chosen timeout is really a caller-chosen hold on a connection. 55s keeps the whole create under the default 60s HTTP write timeout while still giving authors most of a minute, nearly double the default.
Raising it requires raising --write-timeout to match; an operator who does both gets a longer setup budget, and one who raises only this gets a truncated response instead of a classified error, which main.go refuses at startup rather than discovering per request.
const DefaultStartupScriptTimeout = 30 * time.Second
DefaultStartupScriptTimeout bounds a boot-time startup script when BootOptions.StartupScriptTimeout is unset. Chosen to leave headroom under the default 60s HTTP write timeout so the API can answer with a classified error rather than having the response cut off mid-flight.
const FuseSecretsPath = fuseSecretsPath
FuseSecretsPath is exported so fleet.reuploadSecrets can re-upload to the profile-declared path without hardcoding the literal.
const ReservedGuestDir = "/fuse"
fused guest paths. These follow the fused `/fuse/` convention (see PRD-08): the guest fc-agent mounts /fuse on tmpfs so secrets never reach persistent storage. They are profile details, not core defaults. ReservedGuestDir is the directory every path below lives under. It is exported so the API can refuse a caller file that would land in it: /fuse is where the guest agent keeps the manifest, the resolved secrets, and the TLS credentials, and a caller writing there would either clobber them or race them. The Fusefile compiler applies the same rule client-side.
Variables ¶
var ( // ErrSeedUnplaceable is returned when no host in the fleet can take an // environment seeded from an artifact: not the host holding it, not any // other host holding a copy, and not any host the artifact could be moved // to. The wrapping context names each rung that was tried, because the // operator asked for an environment and needs to know which of the three // reasons stopped it. ErrSeedUnplaceable = fmt.Errorf("%w: no host can take an environment seeded from this artifact", ErrNoCapacity) // ErrArtifactImmovable is returned when an artifact cannot be copied off // the host that holds it: no digest was recorded for it (an agent build // that predates artifact hashing, so there is nothing to verify a transfer // against), or no ArtifactMover is configured, or no host holding it is // still registered with a usable agent URL. // // This is deliberately not a hard failure on its own. Placement treats an // immovable artifact the way it always did: the workload is pinned to the // artifact rather than the artifact following the workload. ErrArtifactImmovable = fmt.Errorf("%w: artifact cannot be copied to another host", ErrNoCapacity) // ErrArtifactTransferFailed is returned when a peer pull was attempted and // did not complete. It is NOT wrapped into ErrNoCapacity: the fleet had // room, the copy failed, and reporting that as a capacity shortfall would // send an operator to look at the wrong thing. ErrArtifactTransferFailed = errors.New("artifact transfer failed") )
Artifact failures at the API boundary.
The two "cannot place this" cases wrap ErrNoCapacity on purpose. They ARE capacity outcomes from the caller's point of view (the fleet cannot host this environment right now), the handler layer already maps ErrNoCapacity to 503, and 503 is the honest answer: retrying later may well succeed. Wrapping is also what keeps the mapping in one place rather than requiring every new boundary condition to be threaded through the HTTP layer.
var ( // ErrTaskAlreadyAssigned is returned by ProvisionAndAssign when a // VM is already tracked for the given task id. The existing VM's // id is included in the wrapping fmt.Errorf context. ErrTaskAlreadyAssigned = errors.New("task already assigned") // ErrVMNotFound is returned by DestroyVM, CreateSnapshot, // RestoreSnapshot, and ListSnapshots when the vm id does not // match a tracked VM. ErrVMNotFound = errors.New("vm not found") // ErrTaskNotFound is returned by CompleteTask when the task id // does not match any VM. ErrTaskNotFound = errors.New("task not found") // ErrSnapshotNotFound is returned when a snapshot ID does not // correspond to a persisted snapshot record for the requested VM. ErrSnapshotNotFound = errors.New("snapshot not found") // ErrSnapshotQuotaExceeded is returned when creating a snapshot // would exceed the configured retention quota. ErrSnapshotQuotaExceeded = errors.New("snapshot quota exceeded") // ErrSnapshotInvalidState is returned when an operation targets a // snapshot that is not currently in the required lifecycle state. ErrSnapshotInvalidState = errors.New("snapshot invalid state") // ErrSnapshotHasChildren is returned when deleting a snapshot that // still has descendant snapshots in the lineage graph. ErrSnapshotHasChildren = errors.New("snapshot has children") // ErrVMNotRunning is returned by operations that require a VM to be // in the Running state (e.g. Drain). The current state is included // in the wrapping fmt.Errorf context so callers and operators can // see why the transition was rejected. ErrVMNotRunning = errors.New("vm not running") // ErrExecUnsupported is returned by Exec when the environment has no // real guest to run commands in (e.g. the in-memory stub a provider // falls back to when its BaseURL is unset). Reporting it is what keeps // a misconfigured host from answering exec with a fabricated success. ErrExecUnsupported = errors.New("exec not supported by provider") // ErrAttachUnsupported is returned by Attach when the environment does // not implement Attacher. ErrAttachUnsupported = errors.New("attach not supported by provider") // ErrSnapshotUnsupported is returned by CreateSnapshot and // RestoreSnapshot when the environment does not implement // SnapshotCapable. Like ErrExecUnsupported this is a provider // capability gap, not a server fault. ErrSnapshotUnsupported = errors.New("snapshot not supported by provider") // ErrGPUUnsupported is returned by CreateSnapshot and ForkEnvironment // when the target vm holds a gpu passthrough device. A vfio device // cannot be checkpointed (d4), so this is a permanent property of the // environment rather than a transient failure. ErrGPUUnsupported = errors.New("operation not supported for gpu environments") // ErrHostHasVMs is returned by RemoveHost when VMs are still assigned // to the host. Callers should cordon/drain and wait for the VMs to // leave before retrying. ErrHostHasVMs = errors.New("host still has vms assigned") // ErrStartupScriptTimeout is returned during boot when the caller's // startup script did not finish within StartupScriptTimeout. The boot // path runs the script synchronously, so a script that never returns // (a foreground server, `sleep infinity`) would otherwise hang the // create until the HTTP write timeout killed the response and the // caller saw an opaque 500. Long-lived processes belong in `services`, // or must be backgrounded by the script itself. ErrStartupScriptTimeout = errors.New("startup script did not complete in time") // ErrStartupScriptTimeoutTooLarge is returned by ProvisionAndAssign when // the caller asks for a startup script bound above the fleet's ceiling. // The request is refused rather than clamped: a caller that silently got // 55s when it asked for 10m would read the resulting // ErrStartupScriptTimeout as "my script is slow" instead of "my bound was // never honored". ErrStartupScriptTimeoutTooLarge = errors.New("startup script timeout exceeds the configured maximum") )
Sentinel errors exposed by the FleetManager public API. Callers (notably the REST handler layer) use errors.Is to map these to HTTP status codes without string-matching. These are kept intentionally coarse — they cover the cases that differ at the API boundary, not every internal failure mode.
var DefaultFusedManifest = []byte(`{"version":"1","machine":{"workspace":"/workspace"},"services":{}}`)
DefaultFusedManifest is the fused profile's default compiled manifest, used by the API when a caller omits an inline manifest. It encodes the fused manifest schema and is profile data, not a generic core default.
var ErrHostNotFound = fmt.Errorf("host not found")
ErrHostNotFound is returned when a host operation targets an unregistered host ID.
var ErrHostPinUnschedulable = errors.New("pinned host cannot take this environment")
ErrHostPinUnschedulable is returned by Schedule when spec.HostID names a registered host that no longer clears one of the other gates (cordoned, wrong region or backend, mismatched labels, or full). The wrapped message names the gate that rejected it. The REST handler maps this to 503; an unknown host id is rejected earlier, at the API boundary, as a 404.
var ErrInvalidCursor = errors.New("invalid pagination cursor")
ErrInvalidCursor is returned when a caller-supplied pagination cursor cannot be decoded. It is a client error (maps to 400 at the API boundary), not a server fault.
var ErrNoCapacity = errors.New("no host has sufficient capacity")
ErrNoCapacity is returned by Schedule when no registered host can fit the requested spec. The REST handler maps this to 503.
var ErrNoHosts = errors.New("no hosts registered")
ErrNoHosts is returned by Schedule when the host registry is empty.
var ErrNoLabelMatch = errors.New("no host matches the requested placement labels")
ErrNoLabelMatch is returned by Schedule when spec.Labels matched no schedulable host at all, which is a selector mistake rather than a capacity shortfall. The REST handler maps this to 503.
Functions ¶
func IsNotFound ¶ added in v0.9.1
IsNotFound reports whether err is, or wraps, a provider HTTP 404. Both the firecracker and qemu providers surface agent responses as *HTTPStatusError, so cleanup paths use this to tell "the resource is already gone" apart from a real provider failure. A vanished resource means the work is already done, not that the caller should retry forever.
func IsTerminalState ¶
IsTerminalState reports whether the given VM state is a terminal state from the SSE subscriber's perspective. After a terminal event the broadcaster will not publish further events for the same VM, so the handler closes the stream.
func NewEventID ¶
func NewEventID() string
NewEventID returns a 128-bit hex-encoded random identifier suitable for use as an SSE `id:` field. Using crypto/rand avoids pulling in a UUID dependency and gives the same uniqueness guarantee. The stringified form is opaque to clients — they round-trip it via Last-Event-ID for resume.
Exported so the api package can mint ids for synthesised snapshot events without importing crypto/rand directly.
func NormalizeArch ¶ added in v0.16.0
NormalizeArch maps an architecture string onto GOARCH vocabulary. It accepts the uname -m spellings ("x86_64", "aarch64") alongside the GOARCH ones, lowercased. Empty stays empty (meaning "not set", which the scheduler reads as amd64); anything unrecognized is returned lowercased as-is so validation can name it in an error.
func Schedule ¶
func Schedule(spec Spec, hosts []*Host, policy PlacementPolicy) (*Host, PlacementDecision, error)
Schedule picks the best host for the given spec according to the placement policy. It is a pure function with no side effects — the caller is responsible for updating the host's Allocated counters and persisting the decision.
Filter pipeline:
- If spec.HostID is non-empty, consider only that host; a pin that clears none of the gates below fails with ErrHostPinUnschedulable naming the gate that rejected it.
- Exclude non-schedulable hosts (cordoned, draining).
- If spec.Region is non-empty, exclude hosts in a different region.
- Exclude hosts whose backend cannot serve a GPU request.
- If spec.Labels is non-empty, exclude hosts missing any requested pair.
- Exclude hosts that don't fit the spec (capacity - allocated < spec).
- Among survivors, pick by policy (binpack or spread).
Ties within a policy are broken by host ID for determinism.
func SchedulePreferring ¶ added in v0.19.0
func SchedulePreferring(spec Spec, hosts []*Host, policy PlacementPolicy, hints PlacementHints) (*Host, PlacementDecision, error)
SchedulePreferring is Schedule with placement preferences applied.
The preferences are consulted only AFTER the filter pipeline above has run, on the hosts that survived it, so nothing in PlacementHints can admit a host the gates rejected. Within that surviving set, artifact locality outranks the placement policy: seeding from a local artifact is a `cp --reflink=auto` on the host, while seeding from a remote one means copying hundreds of megabytes to tens of gigabytes across the fleet first. No binpack or spread score is worth that difference, and the policy still decides among equally local hosts.
Types ¶
type AgentSpec ¶
type AgentSpec struct {
Files map[string][]byte // arbitrary files to upload into the guest (path -> bytes)
DownloadURL string // fetch the agent binary (e.g. GitHub releases)
Command string // how to launch the daemon
AuthToken string // pass-through bearer token
Gateway string // pass-through gateway websocket URL
GatewayToken string // pass-through gateway token
DrainCommand string // command run inside the guest for graceful shutdown (” => skip)
Expose []ExposeSpec // guest ports to publish as reachable endpoints, if any
}
AgentSpec is the generic, provider-agnostic description of the guest agent to launch inside a sandbox. fused is expressed as one configuration of this spec via FusedAgentSpec (see agent_profile.go); nothing fuse-specific is hardcoded in the core boot path.
func FusedAgentSpec ¶
func FusedAgentSpec(manifest []byte, secretMap map[string]string, creds *secrets.VMCredentials, opts BootOptions) AgentSpec
FusedAgentSpec builds the fused profile: the full AgentSpec that reproduces the orchestrator's original boot behavior. Files carries the manifest, the secrets JSON (defaulting to "{}" when nil/empty), and — when creds is set — the TLS/auth credential files. Command is the fused launch line.
NOTE on Command: the Command field is the generic launch line for providers that run a free-form shell command. The firecracker host agent IGNORES Command and instead reads structured fields off the frozen /start-surfd wire (manifest/secrets/TLS paths), sourcing them from its own path constants that mirror the consts above.
type ArtifactEndpoint ¶ added in v0.19.0
ArtifactEndpoint identifies one side of a host-to-host artifact transfer. Token is the host agent's own bearer token, which the orchestrator already stores in order to talk to that host at all.
type ArtifactHolder ¶ added in v0.19.0
ArtifactHolder is one host that holds a copy of an artifact, and the snapshot id that host stores it under. The id differs per host: a pulled copy is a distinct row with a distinct id, because "which host has it" is a per-copy fact and a single record has only one host_id.
type ArtifactMove ¶ added in v0.19.0
type ArtifactMove struct {
Digest string
SnapshotID string
From ArtifactEndpoint
To ArtifactEndpoint
}
ArtifactMove describes one transfer: fetch Digest from From, and commit it on To under SnapshotID.
SnapshotID is the id the RECEIVING host stores it under, and it is chosen by the orchestrator rather than the agent. Committing under an id the control plane already knows is what makes a pulled artifact indistinguishable from a locally created one: a later create with that seed id resolves it out of the receiving agent's snapshot store with no special case anywhere.
type ArtifactMoved ¶ added in v0.19.0
ArtifactMoved is what the receiving host reports once the artifact is verified and committed.
type ArtifactMover ¶ added in v0.19.0
type ArtifactMover interface {
MoveArtifact(ctx context.Context, move ArtifactMove) (ArtifactMoved, error)
}
ArtifactMover performs one host-to-host artifact transfer.
It is an interface on FleetConfig rather than a direct call into internal/hostwire because hostwire imports this package (for HTTPStatusError), so depending on it here would be an import cycle. The concrete implementation lives at the composition root, alongside the host provider factory, and is the only thing in the system that holds both hosts' agent tokens at once.
Implementations MUST mint the grant with the SERVING host's token (hostwire.MintArtifactGrant) and present only that grant to the serving host. Handing the pulling host a bearer token for the serving host would trade one blob read for full control of that host.
A nil mover means artifacts cannot move. That is a degraded mode, not an error: placement falls back to pinning the workload to the host that already holds the artifact, which is exactly the behaviour that predates this file.
type AttachSpec ¶ added in v0.4.0
type AttachSpec struct {
// Cmd is the argv to run. Empty means the guest's login shell.
Cmd []string
// TTY requests a pty for the process. Rows and Cols seed its initial
// window size; further resizes arrive as frames on the stream itself.
TTY bool
Rows uint16
Cols uint16
}
AttachSpec describes an interactive attach to a sandbox.
type Attacher ¶ added in v0.4.0
type Attacher interface {
Attach(ctx context.Context, spec AttachSpec) (io.ReadWriteCloser, error)
}
Attacher is implemented by environments that can open a raw duplex byte stream to a process inside the sandbox. The stream carries the fuse-attach/1 frame protocol (see docs/attach.md); nothing between the caller and the guest interprets those frames, so the orchestrator relays them verbatim.
Environments with no real guest (e.g. in-memory stubs) omit this method; callers type-assert and report ErrAttachUnsupported when it is absent.
type BootOptions ¶
type BootOptions struct {
StartupScript string
GatewayURL string
GatewayToken string
Expose []ExposeSpec
// Healthcheck is the environment-level readiness probe, or nil when the
// caller declared none. It is shipped into the guest as a file (see
// FusedAgentSpec) and evaluated there; the orchestrator only reads the
// verdict back. See health.go.
Healthcheck *HealthcheckSpec
// Files are caller-supplied guest files (a Fusefile's `copy` block),
// keyed by absolute guest path. The agent profile merges them into
// AgentSpec.Files ahead of its own entries, so they are uploaded on the
// same pass as the manifest and credentials and therefore land before
// StartupScript runs, and so a caller can never displace a profile file.
// The API refuses a path under ReservedGuestDir before it reaches here.
Files map[string][]byte
// StartupScriptTimeout bounds StartupScript. Zero means
// DefaultStartupScriptTimeout. The script runs synchronously inside the
// create request, so this must stay comfortably under the orchestrator's
// HTTP write timeout (60s by default) for the caller to receive a real
// error instead of a truncated response.
StartupScriptTimeout time.Duration
}
type BootResult ¶
type BootResult struct {
Env Environment
BootTime time.Duration
FromCache bool // true if restored from checkpoint
AuthTokenEncrypted []byte // AES-GCM encrypted per-VM auth token for persistence
DrainCommand string // graceful-shutdown command for the configured agent (” => skip)
Endpoints []Endpoint // published endpoints, if the provider reported any
}
BootResult is returned after provisioning or restoring an environment.
func Boot ¶
func Boot(ctx context.Context, p Provider, spec Spec, manifest []byte, secretMap map[string]string, opts BootOptions, encryptionKey []byte) (*BootResult, error)
Boot provisions or restores an environment, uploads the agent's files, and starts the configured guest agent. If a matching checkpoint exists, it restores from it. Otherwise creates fresh. When encryptionKey is non-nil (32 bytes), per-VM TLS credentials and an auth token are generated and injected. The encrypted token is returned in BootResult for persistence.
The guest agent is described by an AgentSpec; today that is always the fused profile (FusedAgentSpec), which carries the fused manifest/secrets/TLS files and launch command. Boot itself is profile-agnostic.
type CapacityProber ¶ added in v0.4.0
type CapacityProber interface {
Capacity(ctx context.Context) (HostCapacity, error)
}
CapacityProber is implemented by providers that can report the real hardware capacity of the host they front (CPU count, total RAM, free disk, GPU inventory) instead of trusting operator-declared numbers. The registerHost API handler type-asserts the provider to this interface at registration time to source capacity for any field the operator left unset; providers that cannot probe (e.g. a stub with no real hardware behind it) simply return an error.
type Checkpoint ¶
type Checkpoint struct {
ID string
Comment string
SizeBytes int64
CreatedAt time.Time
// Digest is the hex sha256 the host agent computed over the artifact as it
// wrote it, or "" from an agent (or a backend) that does not hash. It
// verifies a later transfer of these exact bytes and nothing else: two
// builds of the same recipe produce different rootfs bytes, so it is never
// an identity two artifacts can share and never a cache key.
Digest string
}
Checkpoint is a snapshot of a sandbox.
type DeadLetterKind ¶
type DeadLetterKind string
DeadLetterKind identifies the kind of failure a dead-letter entry represents.
const ( // DeadLetterOrphanDestroy records repeated failures to destroy an // orphan VM observed by reconcile but not tracked in the fleet. DeadLetterOrphanDestroy DeadLetterKind = "orphan_destroy" // DeadLetterStuckTask records a task that exceeded its runtime ceiling // and was torn down by the reconcile loop. DeadLetterStuckTask DeadLetterKind = "stuck_task" // DeadLetterIdleTimeout records an environment torn down because it went // longer than its idle timeout with no exec and no attach session. DeadLetterIdleTimeout DeadLetterKind = "idle_timeout" )
type DeadLetterRecord ¶
type DeadLetterRecord struct {
ID int64
Kind DeadLetterKind
EntityID string
TaskID string
Reason string
RetryCount int
Payload json.RawMessage
FirstSeenAt time.Time
LastSeenAt time.Time
}
DeadLetterRecord is a failure the reconciler has given up on retrying. Entries are keyed uniquely by (Kind, EntityID); repeated failures update the RetryCount and LastSeenAt fields rather than inserting new rows.
type Endpoint ¶ added in v0.2.0
type Endpoint struct {
As string // caller-chosen label, e.g. "http"
URL string // reachable address, e.g. "http://203.0.113.5:41231"
Port int // the guest-side port this endpoint publishes
}
Endpoint is a published network endpoint for an environment (e.g. an ingress port exposed via the Fusefile's `expose` list).
type EndpointReporter ¶ added in v0.2.0
type EndpointReporter interface {
Endpoints() []Endpoint
}
EndpointReporter is implemented by environments that can report additional network endpoints published during StartAgent (e.g. via ingress/expose). Providers that don't support ingress simply omit it, in which case Boot reports no endpoints — consistent with how SnapshotCapable/SnapshotForkable are optional per-provider capabilities.
type Environment ¶
type Environment interface {
// Name returns the sandbox identifier.
Name() string
// URL returns the sandbox's reachable address for the guest agent.
URL() string
// Exec runs argv inside the sandbox and reports the result. A non-zero
// guest exit code is carried in ExecResult.ExitCode, not returned as an
// error; err is reserved for transport and provider failures, so callers
// must check both.
Exec(ctx context.Context, cmd []string, opts ExecOptions) (ExecResult, error)
// ExecStream runs a command with stdout/stderr wired to writers.
ExecStream(ctx context.Context, stdout, stderr io.Writer, name string, args ...string) error
// Upload writes data to a file path inside the sandbox.
Upload(ctx context.Context, data []byte, path string) error
// StartAgent launches the configured guest agent inside the sandbox.
StartAgent(ctx context.Context, spec AgentSpec) error
// Token returns the per-sandbox auth token that callers must include
// when reaching URL(). Empty string for providers whose URL
// self-authenticates.
Token() string
}
Environment is a running sandbox.
type EnvironmentEvent ¶
type EnvironmentEvent struct {
ID string `json:"id"`
Kind string `json:"event"`
VMID string `json:"vm_id"`
State VMState `json:"state"`
URL string `json:"url,omitempty"`
Error string `json:"error,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
EnvironmentEvent is a single state-change notification published by the FleetManager broadcaster and consumed by SSE subscribers.
The struct is deliberately a flat, JSON-serialisable shape so the REST handler can encode it directly into an SSE `data:` line. The `Kind` field always serialises as `event` (matching the SSE event dispatch contract on the client) — for v1 the only kind emitted is "state". Future event kinds (e.g. "log", "snapshot") would be added here without breaking existing subscribers.
Wire shape (one line of SSE data:):
{
"id": "<event-uuid>",
"event": "state",
"vm_id": "...",
"state": "running",
"url": "host:port",
"error": "...",
"updated_at": "..."
}
type EventRecord ¶
type EventRecord struct {
ID int64
EntityType string
EntityID string
EventType string
Payload json.RawMessage
CreatedAt time.Time
}
EventRecord stores an audit event for critical lifecycle transitions.
type ExecOptions ¶ added in v0.4.0
type ExecOptions struct {
// Timeout bounds how long the guest command may run. Zero means
// DefaultExecTimeout. Values above MaxExecTimeout are clamped.
Timeout time.Duration
}
ExecOptions tunes a single Exec call.
type ExecResult ¶ added in v0.4.0
ExecResult is the outcome of running a command inside a sandbox. Stdout and Stderr are kept separate and byte-exact; ExitCode is the guest command's own status, which is meaningful even when it is non-zero.
type ExposeSpec ¶ added in v0.2.0
ExposeSpec requests that a guest port be published as a reachable endpoint by the provider during StartAgent.
type FleetConfig ¶
type FleetConfig struct {
Provider Provider
StateStore StateStore
Prefix string // VM name prefix, e.g. "fuse-"
ReconcileInterval time.Duration // default 30s
// TaskStuckTimeout is the maximum age of a Running VM with no state
// transitions before it is considered stuck. This is a leak-detection
// ceiling, NOT a heartbeat — healthy long-running tasks must set
// Spec.MaxRuntime to override this. Default 2h.
TaskStuckTimeout time.Duration
// DefaultIdleTimeout applies to VMs whose Spec.IdleTimeout is zero. It is
// how long a Running VM may go with no exec and no attach session before
// it is destroyed. Unlike TaskStuckTimeout this is a liveness check, but
// only over control-plane activity. Zero (the default) disables idle
// expiry for VMs that do not ask for it.
DefaultIdleTimeout time.Duration
// OrphanDestroyMaxRetries is the number of consecutive reconcile cycles
// an orphan VM may fail to destroy before being dead-lettered and
// skipped on subsequent cycles. Default 5.
OrphanDestroyMaxRetries int
// StartupScriptTimeout bounds a caller's boot-time startup script when
// BootOptions does not set its own. Default
// DefaultStartupScriptTimeout. Keep it under the HTTP write timeout so
// a create that trips it still returns a classified error.
StartupScriptTimeout time.Duration
// MaxStartupScriptTimeout is the largest bound a caller may request via
// BootOptions.StartupScriptTimeout. Default
// DefaultMaxStartupScriptTimeout. A request above it is refused with
// ErrStartupScriptTimeoutTooLarge. Keep it under the HTTP write timeout
// for the same reason StartupScriptTimeout is kept under it.
MaxStartupScriptTimeout time.Duration
// DefaultSnapshotRetention applies to snapshots created without an
// explicit retention window. Zero leaves snapshots unbounded until
// explicitly deleted.
DefaultSnapshotRetention time.Duration
// SnapshotQuotaMaxCount limits ready/in-flight snapshots per tenant.
// Zero disables count-based enforcement.
SnapshotQuotaMaxCount int
// SnapshotQuotaMaxBytes limits the aggregate size of ready/in-flight
// snapshots per tenant. Zero disables byte-based enforcement.
SnapshotQuotaMaxBytes int64
// ArtifactMover copies a build artifact from one host agent to another.
// Nil (the default) means artifacts cannot move, in which case an
// environment seeded from one is pinned to the host that holds it — the
// behaviour that predates peer-to-peer distribution. See ArtifactMover.
ArtifactMover ArtifactMover
// ArtifactPullTimeout bounds a single host-to-host artifact transfer.
// Zero inherits the caller's context, which for a create is the HTTP
// request. Transfer time scales with artifact size and link speed, so any
// fixed bound is either too small for the largest legitimate artifact or
// too large to detect anything; set it only where the fleet's artifacts and
// links are known.
ArtifactPullTimeout time.Duration
// ArtifactIdleTTL is how long a layer artifact may go unreferenced and
// unused before the reconcile loop collects it. "Used" includes cache hits,
// not just environments holding it open. Zero (the default) disables idle
// collection: an orchestrator upgrade must not start deleting artifacts an
// existing fleet relies on.
ArtifactIdleTTL time.Duration
// ArtifactMaxPerTenant caps how many ready layer artifacts one tenant may
// keep, evicting least recently used on overflow. It is the cheap ceiling
// behind ArtifactIdleTTL, for a tenant that builds often enough to keep
// every layer inside the idle window. Zero (the default) disables it.
ArtifactMaxPerTenant int
// PlacementPolicy controls how the scheduler picks among hosts
// when multiple are registered. Default is spread.
PlacementPolicy PlacementPolicy
// TokenEncryptionKey is the 32-byte AES-256 key used to encrypt
// per-VM auth tokens before storing them in the state store.
// When nil, VM credential generation is skipped (insecure mode).
TokenEncryptionKey []byte
// HostProviderFactory builds a Provider for a given (url, token,
// backend) triple. The fleet uses it during recoverState to
// rehydrate the per-host providers that RegisterHost would have
// created during normal operation. Without it, an orchestrator
// restart loses the scheduler's host registry and falls back to
// single-provider mode for all subsequent placements.
HostProviderFactory func(url, token string, backend HostBackend) Provider
Metrics ReconcileMetrics
Logger *slog.Logger
}
FleetConfig configures the fleet manager.
type FleetManager ¶
type FleetManager struct {
// contains filtered or unexported fields
}
FleetManager tracks and manages a fleet of VMs.
func NewFleetManager ¶
func NewFleetManager(cfg FleetConfig) *FleetManager
NewFleetManager creates a fleet manager. Call Start to begin the reconciliation loop.
func (*FleetManager) Attach ¶ added in v0.4.0
func (fm *FleetManager) Attach(ctx context.Context, vmID string, spec AttachSpec) (io.ReadWriteCloser, error)
Attach opens a raw duplex byte stream to a process inside a running VM's guest. The stream carries fuse-attach/1 frames; the fleet does not interpret them, it only hands the stream back to the caller to relay.
The caller owns the returned stream and must Close it.
func (*FleetManager) AuditEvent ¶
func (fm *FleetManager) AuditEvent(ctx context.Context, entityType, entityID, eventType string, payload map[string]any)
AuditEvent records a security or operational event in the state store's event table. It is the public face of appendEvent, exposed so the REST middleware can log auth failures and IP rejections without reaching into FleetManager internals.
func (*FleetManager) CompleteTask ¶
func (fm *FleetManager) CompleteTask(taskID string) error
CompleteTask marks a task as done and triggers async VM destruction.
func (*FleetManager) CordonHost ¶
func (fm *FleetManager) CordonHost(ctx context.Context, hostID string) error
CordonHost marks a host as cordoned (no new VMs). Existing VMs are left running.
func (*FleetManager) CreateSnapshot ¶
func (fm *FleetManager) CreateSnapshot(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotRecord, error)
CreateSnapshot quiesces the given VM, invokes Environment.Checkpoint, persists a first-class SnapshotRecord, and marks it ready once the provider confirms the new checkpoint exists.
func (*FleetManager) DeleteSnapshot ¶
func (fm *FleetManager) DeleteSnapshot(ctx context.Context, vmID, snapshotID string) error
DeleteSnapshot removes a leaf snapshot resource after deleting the provider-side artifact.
func (*FleetManager) DeleteSnapshotByID ¶
func (fm *FleetManager) DeleteSnapshotByID(ctx context.Context, snapshotID string) error
DeleteSnapshotByID removes a leaf snapshot resource by global snapshot ID.
func (*FleetManager) DestroyVM ¶
func (fm *FleetManager) DestroyVM(ctx context.Context, vmID string) error
DestroyVM forcefully tears down a VM by ID.
func (*FleetManager) Drain ¶
func (fm *FleetManager) Drain(ctx context.Context, vmID string) error
Drain transitions a Running VM into the Draining state and runs the configured drain command inside the guest (via Environment.ExecStream) to quiesce in-guest workloads gracefully. It is the first phase of two-phase environment teardown:
- POST /v1/environments/{vmId}?action=drain → Drain → guest quiesce
- DELETE /v1/environments/{vmId} → DestroyVM → VM gone
Drain is intentionally narrow:
- Only Running VMs can be drained. Any other state (Provisioning, Draining, Destroying) returns ErrVMNotRunning so callers can map it to a 409 Conflict and inspect the current state.
- Drain does not destroy the VM on success or on failure. Even if the drain command fails, the VM stays in Draining so the caller can still issue DELETE through the existing path. Auto-destroy here would defeat the whole point of giving the harness a chance to recover from a partial drain.
- An empty drain command leaves the VM Draining for the caller to DELETE (back-compat): no graceful command is run.
- Drain takes its own context; if the caller's request context is cancelled we abort the command cleanly. The default drainTimeout is layered on top so a slow guest cannot hold a request thread indefinitely.
The state transition (Running → Draining) is performed and persisted before the drain command fires. That ordering matters: if the orchestrator crashes mid-drain, recovery will see a Draining VM and the operator can decide whether to retry Drain or proceed straight to DELETE. The alternative (command first, then state flip) would lose the intent of the drain on a crash.
func (*FleetManager) Exec ¶ added in v0.4.0
func (fm *FleetManager) Exec(ctx context.Context, vmID string, cmd []string, opts ExecOptions) (ExecResult, error)
Exec runs argv inside a running VM's guest and reports the result.
A non-zero guest exit code is not an error: it is carried in ExecResult.ExitCode with stdout and stderr intact, so a caller can tell "the command ran and failed" apart from "the command could not be run". The returned error is reserved for the latter — unknown VM, wrong state, or a transport failure reaching the host.
Only Running VMs can be exec'd into. A Draining VM is deliberately refused even though its guest is still reachable: drain means the workload is being quiesced, and letting new commands in behind that would undo it.
func (*FleetManager) ForkEnvironment ¶ added in v0.2.0
func (fm *FleetManager) ForkEnvironment(ctx context.Context, srcVMID string, opts ForkOptions) (string, error)
ForkEnvironment creates a brand-new vm seeded from a checkpoint of an existing running vm. it obtains a seed snapshot (creating one when ReuseSnapshotID is empty), asks a SnapshotForkable provider to build a new environment from that checkpoint, registers the new vm as running, and records lineage so the fork references its seed snapshot.
lineage is recorded by persisting a SnapshotRecord for the new vm whose ParentSnapshotID is the seed snapshot id (state ready). this reuses the same store path CreateSnapshot uses (upsertSnapshotRecord) and is directly assertable via ListSnapshots / GetSnapshotByID.
the fork is charged to whichever host it lands on, which is the source's host unless that host can no longer fit it and the seed artifact can be copied elsewhere (see placeFork). it is given its OWN guest credentials either way, since it boots a copy of the source's disk and would otherwise answer to the source's token.
providers that cannot fork do not implement SnapshotForkable, so this reports fork as unsupported for them. the firecracker provider implements it; the qemu provider deliberately does not (vfio gpu passthrough cannot be checkpointed).
func (*FleetManager) GetHost ¶
func (fm *FleetManager) GetHost(hostID string) (Host, bool)
GetHost returns a snapshot of a registered host.
func (*FleetManager) GetSnapshot ¶
func (fm *FleetManager) GetSnapshot(ctx context.Context, vmID, snapshotID string) (SnapshotRecord, error)
GetSnapshot returns one persisted snapshot resource scoped to the VM.
func (*FleetManager) GetSnapshotByID ¶
func (fm *FleetManager) GetSnapshotByID(ctx context.Context, snapshotID string) (SnapshotRecord, error)
GetSnapshotByID returns one persisted snapshot resource by its global ID.
func (*FleetManager) GetVM ¶
func (fm *FleetManager) GetVM(vmID string) (VMInfo, bool)
GetVM returns info for a specific VM.
func (*FleetManager) GetVMByTask ¶
func (fm *FleetManager) GetVMByTask(taskID string) (VMInfo, bool)
GetVMByTask returns the VM assigned to a given task.
func (*FleetManager) HostsHoldingArtifact ¶ added in v0.19.0
func (fm *FleetManager) HostsHoldingArtifact(ctx context.Context, tenantID, digest string) (map[string]ArtifactHolder, error)
HostsHoldingArtifact returns every host that already holds digest for tenantID, keyed by host id.
This is the index. It is a query over snapshot records rather than a maintained map, so it cannot drift from the records it describes and needs no recovery step at startup. Records with no host id are skipped: they describe an artifact whose location is unknown, which is not the same as an artifact nobody has.
func (*FleetManager) ListFleet ¶
func (fm *FleetManager) ListFleet() []VMInfo
ListFleet returns every tracked VM matching no filter, transparently paginating through the full result set. Intended for internal/test callers that want "everything" rather than one page; the REST API uses ListFleetFiltered directly so it can return a cursor to the caller.
func (*FleetManager) ListFleetFiltered ¶
func (fm *FleetManager) ListFleetFiltered(filter VMFilter) ([]VMInfo, string, error)
ListFleetFiltered returns one page of tracked VMs filtered by optional exact-match fields, sorted ascending by VM id. It returns the page, a next-page cursor (empty when there are no more results), and an error only for a malformed cursor.
The filter+copy pass runs under fm.mu.RLock (reading vm fields is only safe under lock); sorting and windowing the resulting VMInfo copies runs after the lock is released, since those are plain value data by then.
func (*FleetManager) ListHosts ¶
func (fm *FleetManager) ListHosts() []Host
ListHosts returns every registered host, transparently paginating through the full result set. Intended for internal/test callers that want "everything"; the REST API uses ListHostsFiltered directly so it can return a cursor to the caller.
func (*FleetManager) ListHostsFiltered ¶ added in v0.17.0
func (fm *FleetManager) ListHostsFiltered(filter HostFilter) ([]Host, string, error)
ListHostsFiltered returns one page of registered hosts sorted ascending by host id. It returns the page, a next-page cursor (empty when there are no more results), and an error only for a malformed cursor.
func (*FleetManager) ListSnapshots ¶
func (fm *FleetManager) ListSnapshots(ctx context.Context, vmID string) ([]SnapshotRecord, error)
ListSnapshots returns all snapshots known to the state store for the given VM ID, newest first. Returns an empty slice when the VM exists but has no snapshots. Scoped to one VM's snapshots, so it is not the full-table scan ListSnapshotsFiltered can be; it does not paginate.
func (*FleetManager) ListSnapshotsFiltered ¶
func (fm *FleetManager) ListSnapshotsFiltered(ctx context.Context, filter SnapshotFilter) ([]SnapshotRecord, string, error)
ListSnapshotsFiltered returns one page of snapshots across the fleet filtered by optional exact-match fields, newest first (see sortSnapshotRecords). Missing resources yield an empty list. It returns the page, a next-page cursor (empty when there are no more results), and an error only for a malformed cursor or a store failure.
When filter.VMID is set, the underlying lookup is scoped to that VM at the store layer instead of loading every snapshot in the fleet.
func (*FleetManager) ProvisionAndAssign ¶
func (fm *FleetManager) ProvisionAndAssign(ctx context.Context, taskID string, spec Spec, manifest []byte, secretMap map[string]string, opts BootOptions) (*VMInfo, error)
ProvisionAndAssign provisions a new VM, boots fused, and assigns the given task. Blocks until the VM is ready or an error occurs.
func (*FleetManager) RegisterHost ¶
RegisterHost adds a host to the scheduler's registry. If the host already exists, its capacity, URL, token, and region are updated (useful for heartbeat-like refreshes). The host starts in HostActive state.
The caller must supply a Provider for this host (typically a firecracker.Provider constructed with the host's URL/token). FleetManager holds only the Provider interface to avoid import cycles with concrete provider packages.
func (*FleetManager) RemoveHost ¶
func (fm *FleetManager) RemoveHost(ctx context.Context, hostID string) error
RemoveHost deletes a host from the registry. It must have no VMs assigned to it; callers should cordon/drain and wait for VMs to leave before removing.
func (*FleetManager) ResolveLayer ¶ added in v0.19.0
func (fm *FleetManager) ResolveLayer(ctx context.Context, tenantID, layerKey, arch string) (SnapshotRecord, bool, error)
ResolveLayer returns the newest ready artifact materializing layerKey for (tenantID, arch), or ok=false when the cache is cold for that key.
A miss is an ordinary outcome, not a failure: the first build of any recipe misses every key in its chain, and a caller walks its chain deepest-first probing one key at a time, so misses are the common case rather than the exceptional one.
tenantID is a security boundary. An artifact carries whatever the build baked into it, so it is only ever served back to the tenant that built it; the store enforces that and a caller cannot widen it.
This does not go through ListSnapshotsFiltered because that loads every snapshot in the fleet and filters in Go, and this runs once per chain key on every build.
func (*FleetManager) RestoreSnapshot ¶
func (fm *FleetManager) RestoreSnapshot(ctx context.Context, vmID, snapshotID string) error
RestoreSnapshot rolls a VM back to a prior snapshot via Environment.Restore after validating the metadata record and provider visibility of the checkpoint.
func (*FleetManager) RestoreSnapshotByID ¶
func (fm *FleetManager) RestoreSnapshotByID(ctx context.Context, snapshotID string) error
RestoreSnapshotByID restores a VM from a snapshot identified by its global ID.
func (*FleetManager) RotateToken ¶
func (fm *FleetManager) RotateToken(ctx context.Context, vmID string) error
RotateToken generates new TLS credentials and an auth token for a running VM, uploads them to the guest filesystem, and persists the new encrypted token.
KNOWN GAP: fused reads --auth-token-file exactly once at process start (fused/main.go) and has no credential poller, so a live fused keeps serving the OLD token until it is restarted. Rotation therefore updates the orchestrator's copy and the guest's files but does not yet take effect in the running guest; it needs a StartAgent call (which restarts fused) the way ForkEnvironment does.
Rotation is a server-side operation: the orchestrator updates its own encrypted copy and the guest agent's credential files (paths owned by the agent profile). New inbound connections use the rotated credentials; existing connections retain the old cert until they reconnect.
func (*FleetManager) Start ¶
func (fm *FleetManager) Start(ctx context.Context)
Start begins the background reconciliation loop.
func (*FleetManager) Stop ¶
func (fm *FleetManager) Stop()
Stop cancels the reconciliation loop and waits for it to finish.
func (*FleetManager) SubscribeEnvironmentEvents ¶
func (fm *FleetManager) SubscribeEnvironmentEvents(vmID string) (<-chan EnvironmentEvent, func())
SubscribeEnvironmentEvents registers a subscriber for state-change events on a single VM. The returned channel receives events until cancel is called (idempotent) or the broadcaster drops the connection. Channel capacity is bounded; slow subscribers drop events with a logged warning rather than blocking publishers.
This is a single-process broadcaster: events published on one orchestrator replica are not visible to subscribers on a different replica. The orchestrator runs as a single process today, so this is acceptable; cross-replica fanout would require a Redis or NATS backplane and is intentionally deferred.
func (*FleetManager) UncordonHost ¶
func (fm *FleetManager) UncordonHost(ctx context.Context, hostID string) error
UncordonHost returns a cordoned or draining host to active scheduling.
type ForkOptions ¶ added in v0.2.0
type ForkOptions struct {
// Comment is attached to the seed snapshot when ForkEnvironment
// creates one (empty ReuseSnapshotID). ignored when reusing an
// existing snapshot.
Comment string
// ReuseSnapshotID selects an existing ready snapshot of the source
// vm to seed the fork from. empty means snapshot the source first.
ReuseSnapshotID string
}
ForkOptions tunes a ForkEnvironment call. All fields are optional.
type GPUDevice ¶ added in v0.7.0
type GPUDevice struct {
UUID string `json:"uuid,omitempty"`
Model string `json:"model,omitempty"`
PCIBusID string `json:"pci_bus_id,omitempty"`
MemoryMB int `json:"memory_mb,omitempty"`
DriverVersion string `json:"driver_version,omitempty"`
CUDAVersion string `json:"cuda_version,omitempty"`
ComputeCap string `json:"compute_cap,omitempty"`
MIGCapable bool `json:"mig_capable,omitempty"`
MIGMode string `json:"mig_mode,omitempty"`
IOMMUGroup string `json:"iommu_group,omitempty"`
}
GPUDevice is the per-device detail the host agent probes for a single whole GPU. Field names mirror the agent's gpu_devices payload. All fields are best-effort: the agent omits any it cannot determine.
type HTTPStatusError ¶ added in v0.6.0
HTTPStatusError carries the HTTP status code (and trimmed body) from a non-2xx host-agent response so callers can branch on the code without importing a specific provider package. Providers wrap their raw HTTP errors in this type; RegisterHost uses it to tell "agent rejected the token" (401) apart from "agent unreachable" or any other failure when the register-time capacity probe (see CapacityProber) errors out.
func (*HTTPStatusError) Error ¶ added in v0.6.0
func (e *HTTPStatusError) Error() string
type HealthState ¶ added in v0.21.0
type HealthState string
HealthState is the verdict of the environment-level probe.
It is deliberately NOT a VMState. VMState is exactly provisioning, running, draining and destroying plus the synthetic wire terminals, and every consumer of IsSettledState in the CLI and the three SDKs reasons about that closed set. A failing probe reports; it does not move an environment somewhere new, and nothing tears a VM down for it.
const ( // HealthStarting means the probe has not passed yet and is still inside // its start period, so failures are not being counted. HealthStarting HealthState = "starting" // HealthPassing means the most recent attempt succeeded. HealthPassing HealthState = "passing" // HealthFailing means the probe has failed `retries` times in a row after // the start period ended. HealthFailing HealthState = "failing" )
type HealthStatus ¶ added in v0.21.0
type HealthStatus struct {
State HealthState `json:"state"`
// Since is when the probe entered State, so a consumer can tell a
// long-standing failure from one that just started.
Since time.Time `json:"since,omitempty"`
// Failures is the count of consecutive failed attempts. It is reset by
// any success, so a passing probe always reports zero.
Failures int `json:"failures,omitempty"`
// Message is the last attempt's failure detail (a non-2xx status, a
// connection error, a non-zero exit), empty while passing.
Message string `json:"message,omitempty"`
}
HealthStatus is one verdict as reported by the guest agent.
It is live state, not durable state: it is never persisted, and an orchestrator restart simply re-reads it from the guest on the next tick. Persisting a liveness signal would only let a stale one outlive the thing it described.
type HealthcheckHTTP ¶ added in v0.21.0
HealthcheckHTTP is an HTTP GET against a port inside the guest. The probe runs in-guest, so the port needs no expose entry.
type HealthcheckSpec ¶ added in v0.21.0
type HealthcheckSpec struct {
HTTP *HealthcheckHTTP `json:"http,omitempty"`
Exec []string `json:"exec,omitempty"`
IntervalSeconds int64 `json:"interval_seconds,omitempty"`
TimeoutSeconds int64 `json:"timeout_seconds,omitempty"`
Retries int `json:"retries,omitempty"`
StartPeriodSeconds int64 `json:"start_period_seconds,omitempty"`
}
HealthcheckSpec is the environment-level readiness probe, as it travels from the create request to the guest. Exactly one of HTTP and Exec is set; the API layer rejects a request that sets both or neither.
Every duration is in seconds, and a zero means the field was omitted and the guest agent's default applies. The orchestrator deliberately does not fill those defaults in: the probe is executed in the guest, so the guest owns what "unset" means, and having two places invent defaults is how they drift.
The json tags are the guest wire: this struct is marshalled straight into GuestHealthcheckPath, so fused's config shape and this one are the same shape by construction.
type Host ¶
type Host struct {
ID string
URL string // base URL of the host agent (e.g. https://agent-1.local)
Token string // bearer token for this host's agent
Region string
Backend HostBackend // "firecracker" or "qemu"; empty means firecracker (default)
// Labels are operator-declared key/value pairs supplied at registration
// and matched against a spec's placement label selectors. They are never
// probed from the host agent, the same trust model as Capacity.GPUKind.
Labels map[string]string
Capacity HostCapacity
Allocated HostCapacity
State HostState
LastSeen time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
Host is a registered compute host in the fleet. It represents a single Firecracker host agent that can provision VMs.
type HostBackend ¶ added in v0.3.0
type HostBackend string
HostBackend identifies the virtualization backend a host agent runs. It determines which capabilities the host can offer the scheduler (e.g. only qemu hosts may advertise GPUs).
const ( // BackendFirecracker is the default backend: microVMs with no GPU // passthrough support. BackendFirecracker HostBackend = "firecracker" // BackendQEMU is a full-VM backend that supports GPU passthrough. BackendQEMU HostBackend = "qemu" )
type HostCapacity ¶
type HostCapacity struct {
CPUs int `json:"cpus"`
RamMB int `json:"ram_mb"`
StorageGB int `json:"storage_gb"`
VMCount int `json:"vm_count"` // max concurrent VMs
// Arch is the host's CPU architecture in GOARCH vocabulary ("amd64",
// "arm64"). Probed from the host agent like CPUs/RamMB; an operator may
// also declare it at registration. Empty means amd64: every host that
// existed before this field was x86_64, so old records stay schedulable
// without migration.
Arch string `json:"arch,omitempty"`
// GPUs is the count of whole GPU devices available on the host.
// Zero means no GPUs. Only qemu-backed hosts may report GPUs > 0
// (enforced at registration, see internal/api registerHost).
GPUs int `json:"gpus,omitempty"`
// GPUKind identifies the GPU model (e.g. "a100"). Empty when GPUs is 0.
GPUKind string `json:"gpu_kind,omitempty"`
// MIGProfiles is fractional GPU capacity: MIG instance count by
// profile name (e.g. {"1g.10gb": 4}). A GPU in MIG mode stays on the
// nvidia driver and exports mdev devices, so it is never part of the
// whole-device GPUs pool — the two counters are independent (D5).
// Only qemu-backed hosts may advertise MIG profiles.
//
// When the host reports per-instance MIG inventory (MIGInstances), this
// map is a derived summary (count by profile) and the scheduler binds
// specific instance uuids rather than decrementing a counter. When a
// host only declares counts (the --mig-profile registration override,
// or a hand-managed mig-inventory.txt without parent detail), this map
// remains the scheduling unit so legacy callers do not break.
MIGProfiles map[string]int `json:"mig_profiles,omitempty"`
// MIGInstances is the per-instance MIG inventory probed from the host
// agent (one entry per carved MIG GPU instance). When non-empty, the
// scheduler switches from count-based to instance-based allocation:
// fits() checks for free instances of the requested profile on a
// kind-matching card, and allocate binds concrete uuids to VMs. This
// is strictly additive — a host that reports no instances falls back
// to the MIGProfiles count path (issue #41).
MIGInstances []MIGInstance `json:"mig_instances,omitempty"`
// GPUDevices is the per-device detail probed from the host agent
// (one entry per whole GPU). It rides the capacity wire alongside the
// scalar GPUs/GPUKind counters. In-memory + wire only in this PR; the
// durable per-device schema is issue #37.
GPUDevices []GPUDevice `json:"gpu_devices,omitempty"`
// GPUDeviceUUIDs is the set of device uuids currently bound to VMs. It
// is populated only on a host's Allocated struct (never on Capacity),
// and only for hosts that report per-device inventory. fits() derives
// the free-device set as Capacity.GPUDevices minus this set. It is
// in-memory only: allocation/deallocation maintain it, and recovery
// recomputes it from live VM bindings, so it is never persisted as its
// own column (the durable source of truth is the per-VM gpu_uuids).
GPUDeviceUUIDs []string `json:"gpu_device_uuids,omitempty"`
// MIGInstanceUUIDs is the set of MIG instance uuids currently bound to
// VMs. Populated only on Allocated (never on Capacity), and only for
// hosts that report per-instance MIG inventory. fits() derives the
// free-instance set as Capacity.MIGInstances minus this set. In-memory
// only, like GPUDeviceUUIDs: the durable source of truth is the
// per-VM mig_instance_uuids binding.
MIGInstanceUUIDs []string `json:"mig_instance_uuids,omitempty"`
}
HostCapacity is the resource envelope of a host. It is reported by the host agent at registration time and refreshed by heartbeat. The scheduler compares Spec against (Capacity - Allocated) to make admission decisions.
type HostFilter ¶ added in v0.17.0
HostFilter controls pagination for ListHostsFiltered. There is no exact-match filtering today (registration/listing never narrowed hosts by field) — this only carries Limit/Cursor, kept as its own type for symmetry with VMFilter/SnapshotFilter and so a future field doesn't need a signature change.
type HostRecord ¶
type HostRecord struct {
ID string
URL string
TokenEncrypted []byte
Region string
State HostState
TenantID string
Backend HostBackend
Labels map[string]string
Capacity HostCapacity
Allocated HostCapacity
LastSeen time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
HostRecord is the durable representation of a compute host in the scheduler's registry. It maps 1:1 to a row in orchestrator_hosts.
TokenEncrypted is the agent bearer token sealed with AES-GCM using the orchestrator's TOKEN_ENCRYPTION_KEY (same key the per-VM tokens use). It is decrypted only into the in-memory Host.Token used by the provider client; the plaintext never enters the database.
type HostState ¶
type HostState string
HostState captures the scheduling eligibility of a registered host.
const ( // HostActive means the host is healthy and accepting new VMs. HostActive HostState = "active" // HostCordoned means the host is not accepting new VMs. Existing // VMs are left running. Cordon is the operator's "maintenance // soon, stop sending work here" signal. HostCordoned HostState = "cordoned" // HostDraining means the host is cordoned AND its existing VMs // are being evicted by the reconcile loop. Once all VMs are gone // the host is eligible for removal. Drain support in reconcile // is stubbed in this PR; the state is defined so the data model // doesn't need a follow-up migration. HostDraining HostState = "draining" )
type MIGInstance ¶ added in v0.9.0
type MIGInstance struct {
UUID string `json:"uuid,omitempty"`
Profile string `json:"profile,omitempty"`
Kind string `json:"kind,omitempty"`
ParentGPUUUID string `json:"parent_gpu_uuid,omitempty"`
}
MIGInstance is one carved MIG GPU instance advertised by the host agent. Field names mirror the agent's mig_instances payload. The orchestrator binds a specific instance uuid to a VM (Spec.MIGInstanceUUIDs) so it knows which instance went to which VM — the count-map path (MIGProfiles) cannot.
type MemoryStateStore ¶
type MemoryStateStore struct {
// contains filtered or unexported fields
}
MemoryStateStore is a process-local store useful for tests/default behavior.
func NewMemoryStateStore ¶
func NewMemoryStateStore() *MemoryStateStore
NewMemoryStateStore returns an in-memory StateStore implementation.
func (*MemoryStateStore) AppendEvent ¶
func (s *MemoryStateStore) AppendEvent(_ context.Context, event EventRecord) error
func (*MemoryStateStore) DeleteHost ¶
func (s *MemoryStateStore) DeleteHost(_ context.Context, hostID string) error
func (*MemoryStateStore) DeleteSnapshot ¶
func (s *MemoryStateStore) DeleteSnapshot(_ context.Context, snapshotID string) error
func (*MemoryStateStore) DeleteVM ¶
func (s *MemoryStateStore) DeleteVM(_ context.Context, vmID string) error
func (*MemoryStateStore) FindSnapshotByLayerKey ¶ added in v0.19.0
func (s *MemoryStateStore) FindSnapshotByLayerKey(_ context.Context, tenantID, layerKey, arch string) (SnapshotRecord, bool, error)
FindSnapshotByLayerKey scans for the newest ready layer artifact matching the triple. The winner is picked with snapshotRecordLess, the same ordering sortSnapshotRecords uses (CreatedAt desc, SnapshotID desc as tiebreak), so this store and Postgres cannot disagree about which of two concurrently built artifacts wins.
The tenant check is a security boundary: a hit from another tenant would serve that tenant's baked-in build output, so it has to be a miss.
func (*MemoryStateStore) GetHost ¶
func (s *MemoryStateStore) GetHost(_ context.Context, hostID string) (HostRecord, error)
func (*MemoryStateStore) GetSnapshot ¶
func (s *MemoryStateStore) GetSnapshot(_ context.Context, snapshotID string) (SnapshotRecord, error)
func (*MemoryStateStore) ListDeadLetters ¶
func (s *MemoryStateStore) ListDeadLetters(_ context.Context) ([]DeadLetterRecord, error)
func (*MemoryStateStore) ListHosts ¶
func (s *MemoryStateStore) ListHosts(_ context.Context) ([]HostRecord, error)
func (*MemoryStateStore) ListSnapshots ¶
func (s *MemoryStateStore) ListSnapshots(_ context.Context) ([]SnapshotRecord, error)
func (*MemoryStateStore) ListSnapshotsByDigest ¶ added in v0.19.0
func (s *MemoryStateStore) ListSnapshotsByDigest(_ context.Context, tenantID, digest string) ([]SnapshotRecord, error)
ListSnapshotsByDigest scans for every ready snapshot of this tenant holding the given artifact bytes. Results are sorted by snapshot id so the caller picks the same source peer on every call, which keeps a retried pull hitting the host that already answered rather than fanning out across the fleet.
An empty digest matches nothing, deliberately: every snapshot taken by an agent build that predates artifact hashing has one, so treating it as a value would report the whole fleet as holding a single phantom artifact.
func (*MemoryStateStore) ListSnapshotsByVM ¶ added in v0.17.0
func (s *MemoryStateStore) ListSnapshotsByVM(_ context.Context, vmID string) ([]SnapshotRecord, error)
func (*MemoryStateStore) ListTasks ¶
func (s *MemoryStateStore) ListTasks(_ context.Context) ([]TaskRecord, error)
func (*MemoryStateStore) ListVMs ¶
func (s *MemoryStateStore) ListVMs(_ context.Context) ([]VMRecord, error)
func (*MemoryStateStore) SnapshotQuotaUsage ¶ added in v0.17.0
func (s *MemoryStateStore) SnapshotQuotaUsage(_ context.Context, tenantID string) (SnapshotQuotaUsage, error)
func (*MemoryStateStore) UpsertDeadLetter ¶
func (s *MemoryStateStore) UpsertDeadLetter(_ context.Context, entry DeadLetterRecord) error
func (*MemoryStateStore) UpsertHost ¶
func (s *MemoryStateStore) UpsertHost(_ context.Context, host HostRecord) error
func (*MemoryStateStore) UpsertSnapshot ¶
func (s *MemoryStateStore) UpsertSnapshot(_ context.Context, snapshot SnapshotRecord) error
func (*MemoryStateStore) UpsertTask ¶
func (s *MemoryStateStore) UpsertTask(_ context.Context, task TaskRecord) error
type PlacementDecision ¶
type PlacementDecision struct {
HostID string `json:"host_id"`
Policy PlacementPolicy `json:"policy"`
Candidates int `json:"candidates"` // eligible hosts considered
HeadroomCPUs int `json:"headroom_cpus"` // CPUs remaining after placement
HeadroomRam int `json:"headroom_ram"` // RAM remaining after placement
// ArtifactLocal reports that the winning host already held the spec's seed
// artifact, so the environment boots from a local copy with no transfer.
// When it is false and the spec seeds from an artifact, the caller has to
// move the artifact to the winner before creating the VM, so this is the
// field that says whether a transfer is about to happen.
ArtifactLocal bool `json:"artifact_local,omitempty"`
// ArtifactHolders is how many of the eligible candidates held the artifact.
// Zero alongside ArtifactLocal=false means no host that could take the
// workload had it; a positive count alongside ArtifactLocal=false cannot
// happen, and reading both is how a decision stays self-explaining.
ArtifactHolders int `json:"artifact_holders,omitempty"`
}
PlacementDecision records why the scheduler picked a particular host. It is attached to the VM record for debugging.
type PlacementHints ¶ added in v0.19.0
type PlacementHints struct {
// ArtifactHosts are the hosts that already hold the spec's seed artifact.
//
// This is a PREFERENCE and never a constraint. A host in this set that
// cannot fit the workload, is cordoned, is the wrong arch, has the wrong
// labels, or cannot serve the GPU request is rejected by the ordinary gates
// before this map is ever consulted; locality only ever breaks a tie among
// hosts that already qualify. Letting it do more would trade a correct
// placement for a fast one, which is how a cache hit turns a working create
// into a failing one.
ArtifactHosts map[string]bool
}
PlacementHints carries scheduling preferences that come from fleet state rather than from anything the caller asked for. They are kept out of Spec on purpose: Spec is persisted with the VM, and a durable record must not carry a point-in-time observation about where some bytes happened to live.
type PlacementPolicy ¶
type PlacementPolicy string
PlacementPolicy controls how the scheduler picks among eligible hosts that all have sufficient capacity.
const ( // PlacementBinpack fills hosts as densely as possible. It picks // the host with the MOST already-allocated resources that still // has room. This minimizes the number of active hosts. PlacementBinpack PlacementPolicy = "binpack" // PlacementSpread distributes VMs as evenly as possible. It picks // the host with the LEAST already-allocated resources. This // maximizes isolation between VMs. PlacementSpread PlacementPolicy = "spread" )
type PostgresStateStore ¶
type PostgresStateStore struct {
// contains filtered or unexported fields
}
PostgresStateStore persists orchestrator state in Postgres.
func NewPostgresStateStore ¶
func NewPostgresStateStore(db *sql.DB) *PostgresStateStore
NewPostgresStateStore creates a Postgres-backed state store.
func (*PostgresStateStore) AppendEvent ¶
func (s *PostgresStateStore) AppendEvent(ctx context.Context, event EventRecord) error
func (*PostgresStateStore) ApplyMigrations ¶
func (s *PostgresStateStore) ApplyMigrations(ctx context.Context) error
ApplyMigrations creates and upgrades orchestrator state tables.
func (*PostgresStateStore) DeleteHost ¶
func (s *PostgresStateStore) DeleteHost(ctx context.Context, hostID string) error
func (*PostgresStateStore) DeleteSnapshot ¶
func (s *PostgresStateStore) DeleteSnapshot(ctx context.Context, snapshotID string) error
func (*PostgresStateStore) DeleteVM ¶
func (s *PostgresStateStore) DeleteVM(ctx context.Context, vmID string) error
func (*PostgresStateStore) FindSnapshotByLayerKey ¶ added in v0.19.0
func (s *PostgresStateStore) FindSnapshotByLayerKey(ctx context.Context, tenantID, layerKey, arch string) (SnapshotRecord, bool, error)
FindSnapshotByLayerKey serves the layer cache lookup off orchestrator_snapshots_layer_key_idx. The `layer_key <> ”` predicate is not redundant with the equality above it: it is what makes the query match the index's partial predicate, so this stays a single index probe instead of degrading into a scan as build artifacts accumulate.
The layer key is not unique by design (two concurrent builds of the same recipe both insert), so duplicates are resolved here rather than rejected at write time: newest ready artifact wins, with snapshot_id as the tiebreak so this agrees exactly with MemoryStateStore and with sortSnapshotRecords.
tenant_id is a security boundary, not a filter for convenience. Another tenant's artifact carries whatever their build baked in, so a cross-tenant row has to read as a miss.
func (*PostgresStateStore) GetHost ¶
func (s *PostgresStateStore) GetHost(ctx context.Context, hostID string) (HostRecord, error)
func (*PostgresStateStore) GetSnapshot ¶
func (s *PostgresStateStore) GetSnapshot(ctx context.Context, snapshotID string) (SnapshotRecord, error)
func (*PostgresStateStore) ListDeadLetters ¶
func (s *PostgresStateStore) ListDeadLetters(ctx context.Context) ([]DeadLetterRecord, error)
func (*PostgresStateStore) ListHosts ¶
func (s *PostgresStateStore) ListHosts(ctx context.Context) ([]HostRecord, error)
func (*PostgresStateStore) ListSnapshots ¶
func (s *PostgresStateStore) ListSnapshots(ctx context.Context) ([]SnapshotRecord, error)
func (*PostgresStateStore) ListSnapshotsByDigest ¶ added in v0.19.0
func (s *PostgresStateStore) ListSnapshotsByDigest(ctx context.Context, tenantID, digest string) ([]SnapshotRecord, error)
ListSnapshotsByDigest serves the artifact index off orchestrator_snapshots_digest_idx. The `digest <> ”` predicate matches the index's partial predicate, so this stays an index probe rather than degrading into a scan as snapshots accumulate; it is also a correctness guard, since every snapshot written by an agent that does not hash carries an empty digest and must not read as "this host holds artifact ”".
Ordering by snapshot_id (not created_at) is deliberate: the caller uses this to choose a peer to pull from, and a stable order means a retry asks the same host again instead of spreading half-finished transfers over the fleet.
func (*PostgresStateStore) ListSnapshotsByVM ¶ added in v0.17.0
func (s *PostgresStateStore) ListSnapshotsByVM(ctx context.Context, vmID string) ([]SnapshotRecord, error)
func (*PostgresStateStore) ListTasks ¶
func (s *PostgresStateStore) ListTasks(ctx context.Context) ([]TaskRecord, error)
func (*PostgresStateStore) ListVMs ¶
func (s *PostgresStateStore) ListVMs(ctx context.Context) ([]VMRecord, error)
func (*PostgresStateStore) SnapshotQuotaUsage ¶ added in v0.17.0
func (s *PostgresStateStore) SnapshotQuotaUsage(ctx context.Context, tenantID string) (SnapshotQuotaUsage, error)
SnapshotQuotaUsage aggregates count and size for tenantID directly in SQL, scoped to the same states/mode enforceSnapshotQuota cares about, so per-tenant quota checks never require a full-table scan.
func (*PostgresStateStore) UpsertDeadLetter ¶
func (s *PostgresStateStore) UpsertDeadLetter(ctx context.Context, entry DeadLetterRecord) error
func (*PostgresStateStore) UpsertHost ¶
func (s *PostgresStateStore) UpsertHost(ctx context.Context, h HostRecord) error
func (*PostgresStateStore) UpsertSnapshot ¶
func (s *PostgresStateStore) UpsertSnapshot(ctx context.Context, snapshot SnapshotRecord) error
func (*PostgresStateStore) UpsertTask ¶
func (s *PostgresStateStore) UpsertTask(ctx context.Context, task TaskRecord) error
type Provider ¶
type Provider interface {
// Create provisions a new sandbox.
Create(ctx context.Context, spec Spec) (Environment, error)
// Get returns a handle to an existing sandbox by name.
Get(ctx context.Context, name string) (Environment, error)
// Destroy tears down a sandbox.
Destroy(ctx context.Context, name string) error
// List returns all sandboxes matching the given prefix.
List(ctx context.Context, prefix string) ([]Environment, error)
// Close releases provider resources.
Close() error
}
Provider manages sandboxed environments.
type ReconcileMetrics ¶
type ReconcileMetrics interface {
ReconcileCompleted(summary ReconcileSummary)
}
ReconcileMetrics is an optional callback invoked at the end of every reconcile cycle with a counter summary. Implementations typically feed a Prometheus/OTel exporter — the orchestrator package intentionally does not depend on either. A nil ReconcileMetrics disables metrics reporting.
type ReconcileSummary ¶
type ReconcileSummary struct {
TrackedVMs int
ProviderVMs int
OrphansDestroyed int
OrphansFailed int
OrphansDeadLettered int
StuckTasksSuspected int
StuckTasksFailed int
IdleVMsSuspected int
IdleVMsFailed int
VMsMissingProvider int
// HealthChecked counts VMs whose guest returned a usable health verdict
// this cycle; HealthFailing is how many of those were failing. Neither
// causes a teardown: the environment-level probe reports, it does not act.
HealthChecked int
HealthFailing int
Duration time.Duration
}
ReconcileSummary captures counts from a single reconcile cycle.
type SnapshotCapable ¶
type SnapshotCapable interface {
Checkpoint(ctx context.Context, comment string) (string, error)
Restore(ctx context.Context, checkpointID string) error
ListCheckpoints(ctx context.Context) ([]Checkpoint, error)
}
SnapshotCapable is implemented by environments that support checkpoint/restore. Providers that cannot snapshot (e.g. some container-based providers) simply omit these methods; callers must type-assert to SnapshotCapable before invoking them.
type SnapshotDeleter ¶
SnapshotDeleter is implemented by environments that can delete a previously created checkpoint by ID.
type SnapshotDigester ¶ added in v0.19.0
type SnapshotDigester interface {
CheckpointWithDigest(ctx context.Context, comment string) (Checkpoint, error)
}
SnapshotDigester is an optional companion to SnapshotCapable for backends that hash the artifact while they write it. It exists as a separate interface rather than a wider Checkpoint signature because the digest is only ever available at creation time: the host agent computes it inline over the rootfs it just copied, and nothing recomputes it afterwards, so ListCheckpoints cannot supply it and changing Checkpoint's signature would churn every backend and caller for a value most of them cannot produce.
An environment that does not implement this (the in-memory stub, qemu) simply yields no digest, which is not an error: the digest is an integrity check on a later transfer of those bytes, never an identity a lookup depends on.
type SnapshotExportRecord ¶
type SnapshotExportRecord struct {
Destination string
Status SnapshotExportStatus
RequestedAt time.Time
UpdatedAt time.Time
LastError string
}
SnapshotExportRecord captures metadata for an optional exported artifact.
type SnapshotExportStatus ¶
type SnapshotExportStatus string
SnapshotExportStatus tracks the state of an optional export record.
const ( SnapshotExportPending SnapshotExportStatus = "pending" SnapshotExportReady SnapshotExportStatus = "ready" SnapshotExportError SnapshotExportStatus = "error" )
type SnapshotFilter ¶
type SnapshotFilter struct {
VMID string
TaskID string
TenantID string
State SnapshotState
// Mode narrows to one creation mode, e.g. SnapshotModeBuild to list only
// build artifacts.
Mode SnapshotMode
// LayerKey narrows to artifacts materializing one setup step. Unlike Name
// below, this is a real indexed column rather than a key read out of the
// metadata blob: it is an equality lookup performed on every build, so it
// cannot afford to be a JSON probe over every row. Note that listing is
// not the hot path for it either; see StateStore.FindSnapshotByLayerKey.
LayerKey string
// Arch narrows to artifacts built on one architecture. It is a separate
// field from LayerKey rather than folded into it because the key is
// derived client-side before any host is scheduled, so arch is only known
// after the build lands somewhere. It constrains what may be served, not
// what the recipe is.
Arch string
// Name matches the "name" key inside the snapshot's metadata blob. It is
// how a build artifact is found by something other than its random id.
// Metadata is free-form JSON, so an absent key or a malformed blob simply
// does not match rather than erroring.
Name string
Limit int
Cursor string
}
SnapshotFilter narrows ListSnapshotsFiltered results on exact-match fields, plus pagination controls.
Limit caps the page size (<=0 defaults to DefaultPageLimit, >MaxPageLimit is clamped down to it). Cursor, when non-empty, resumes after the last item of a previous page (see ListSnapshotsFiltered's NextCursor return).
type SnapshotForkable ¶ added in v0.2.0
type SnapshotForkable interface {
CreateFromCheckpoint(ctx context.Context, spec Spec, srcVMID, checkpointID string) (Environment, error)
}
SnapshotForkable is implemented by providers that can create a brand-new environment seeded from an existing checkpoint of another vm. this is the capability a true fork needs; the firecracker provider does not implement it yet, so ForkEnvironment reports fork as unsupported at runtime until a host wire endpoint exists.
type SnapshotMode ¶
type SnapshotMode string
SnapshotMode captures how a snapshot was created.
const ( SnapshotModeManual SnapshotMode = "manual" SnapshotModeAuto SnapshotMode = "auto" // SnapshotModeBuild marks a build artifact: a deliberately created, // durable rootfs meant to seed future environments (see `fuse build`). // Unlike manual and auto snapshots it is not an ephemeral checkpoint of // one vm's state, so retention gc and the per-tenant snapshot quota both // skip it. That means build artifacts accumulate until deleted // explicitly; there is no ceiling on them today. SnapshotModeBuild SnapshotMode = "build" )
type SnapshotOptions ¶
type SnapshotOptions struct {
// Comment is a free-form note attached to the snapshot. Passed
// through to the underlying Environment.Checkpoint implementation
// and persisted in the snapshot record.
Comment string
// Mode records whether this snapshot was taken manually (via the
// REST API) or by an automated process. Defaults to
// SnapshotModeManual when empty.
Mode SnapshotMode
// RetentionUntil, if non-nil, records when the snapshot is
// eligible for garbage collection.
RetentionUntil *time.Time
// Metadata augments the persisted metadata blob. Reserved fields
// such as "comment" are set by the orchestrator.
Metadata map[string]string
// Exports records optional object-storage export intents/status.
Exports []SnapshotExportRecord
// TenantID is the scope a layer artifact is filed under, and it is only
// consulted when LayerKey is set.
//
// It exists because the two halves of the cache have to agree on a scope,
// and the default one cannot. An ordinary snapshot is filed under
// snapshotTenantID, which is the task id, and `fuse build` derives a task
// id that is unique per build on purpose. A layer filed under that is
// filed under a name no later build can ever ask for, so the cache could
// never hit.
//
// This is a security boundary. The API layer derives it from how the
// caller authenticated and never from a query param or a request body,
// because a caller that could name its own scope could read another
// tenant's cache. See callerTenantID.
//
// It is deliberately ignored for non-layer snapshots, so nothing about
// existing snapshot tenancy or quota accounting moves.
TenantID string
// LayerKey is set by `fuse build` when this snapshot is a cacheable setup
// layer, and is empty for every ordinary snapshot. It is what makes an
// artifact findable by recipe rather than by id: without it a build
// artifact can only be named, and a name is not something a later build
// can derive from the Fusefile it is holding.
LayerKey string
}
SnapshotOptions tunes a CreateSnapshot call. All fields are optional.
type SnapshotQuotaUsage ¶ added in v0.17.0
SnapshotQuotaUsage summarizes a tenant's in-flight/ready snapshot usage (count and aggregate size) for quota enforcement, without requiring the caller to load every snapshot record to compute it.
type SnapshotRecord ¶
type SnapshotRecord struct {
SnapshotID string
VMID string
TaskID string
HostID string
TenantID string
ParentSnapshotID string
Mode SnapshotMode
// LayerKey is the derived cache key of the setup step this artifact
// materializes (see internal/fusefile.LayerKeys). It is what separates a
// layer from the rest of SnapshotModeBuild: layers reuse that mode rather
// than adding a new one, so a non-empty LayerKey is the only thing that
// marks a build artifact as cache-addressable. Empty on every snapshot
// that is not a layer, and empty never matches a lookup.
LayerKey string
// Arch is the goarch of the host that actually built this artifact, not
// of whoever asked for it. It is part of every layer lookup even though it
// is deliberately absent from LayerKey: an ext4 rootfs is not portable
// across architectures, but the key is derived client-side before a host
// is scheduled, so arch can only be observed after the fact and applied as
// a filter.
Arch string
// Digest is the hex sha256 of the artifact rootfs, recorded so a transfer
// of these bytes can be verified against them. It is an integrity check
// only and is NOT a cross-build identity: two builds of the same recipe
// produce different rootfs bytes (timestamps, inode ordering, package
// caches), so it can never serve as a cache key and nothing dedups on it.
Digest string
State SnapshotState
SizeBytes int64
RetentionUntil *time.Time
Metadata json.RawMessage
Exports []SnapshotExportRecord
LastError string
CreatedAt time.Time
UpdatedAt time.Time
}
SnapshotRecord tracks checkpoint lineage and retention metadata.
type SnapshotState ¶
type SnapshotState string
SnapshotState captures the lifecycle of a persisted snapshot resource.
const ( SnapshotStateCreating SnapshotState = "creating" SnapshotStateReady SnapshotState = "ready" SnapshotStateRestoring SnapshotState = "restoring" SnapshotStateDeleting SnapshotState = "deleting" SnapshotStateError SnapshotState = "error" )
type Spec ¶
type Spec struct {
Name string // e.g. "fuse-{task-id}"
CPUs int
RamMB int
StorageGB int
Region string
// Arch restricts scheduling to hosts of this CPU architecture, in
// GOARCH vocabulary ("amd64", "arm64"). Empty means any host: images
// are resolved host-side, so a spec that doesn't care about arch keeps
// the pre-arch behavior. A host with no reported arch counts as amd64.
Arch string
// GPUs is the count of GPU devices requested: whole devices, or MIG
// instances when GPUProfile is set. Zero means no GPU.
GPUs int32
// GPUKind identifies the requested GPU model (e.g. "a100"). Empty when
// GPUs is 0.
GPUKind string
// GPUProfile requests fractional GPU allocation: a MIG profile in
// mig-parted vocabulary (e.g. "1g.10gb"), passed through as an mdev
// device by the qemu host agent (decision D5). Empty means whole-device
// allocation.
GPUProfile string
// GPUUUIDs are the concrete device uuids this VM is bound to. Populated
// by allocation on hosts that report per-device inventory
// (Capacity.GPUDevices non-empty) and persisted per-VM so the binding
// survives an orchestrator restart. Empty on legacy homogeneous hosts
// that only track a scalar GPU count.
GPUUUIDs []string `json:"gpu_uuids,omitempty"`
// MIGInstanceUUIDs are the concrete MIG instance uuids this VM is bound
// to. Populated by allocation on hosts that report per-instance MIG
// inventory (Capacity.MIGInstances non-empty) and persisted per-VM so
// the binding survives an orchestrator restart. Empty when the host
// only tracks MIG allocation by profile count (the MIGProfiles path).
// Mirrors GPUUUIDs for whole devices (issue #41).
MIGInstanceUUIDs []string `json:"mig_instance_uuids,omitempty"`
// HostID pins this sandbox to an exact host id. Unlike GPUUUIDs and
// MIGInstanceUUIDs, which allocation fills in, this is caller-settable:
// it comes from the Fusefile's placement.host. A pin is a hard gate, not
// an override, so the pinned host still has to be active, run the right
// backend, and fit the spec.
HostID string
// Labels are placement label selectors: every pair must match the target
// host's declared labels. Caller-settable, from placement.labels.
Labels map[string]string
// MaxRuntime overrides FleetConfig.TaskStuckTimeout for this task.
// Zero means "use the fleet default". This is a leak-detection ceiling,
// not a liveness check — set it higher than any plausible healthy runtime.
MaxRuntime time.Duration
// IdleTimeout destroys this VM once it has gone this long with no exec
// and no attach session. Zero means "use FleetConfig.DefaultIdleTimeout"
// (itself zero by default, i.e. no idle expiry). Unlike MaxRuntime this
// is a liveness check, but only over control-plane activity: in-guest CPU
// and network traffic are not observed.
IdleTimeout time.Duration
// Image names a base rootfs for the provider to boot from, resolved by
// the provider (e.g. the firecracker host agent looks it up in its own
// named-rootfs directory). Empty means the provider's default base.
Image string
// SeedSnapshotID boots this VM from a persisted snapshot artifact instead
// of Image. Unlike a fork there is no live source VM: the artifact is
// resolved out of the host agent's snapshot store. It is host-local, which
// is why SeedSnapshotID always travels with PinnedHostID. Empty means boot
// from Image. Mutually exclusive with Image.
SeedSnapshotID string
// PinnedHostID restricts scheduling to a single host. Set when the VM must
// land where its host-local seed artifact already is. Empty means schedule
// across the fleet as usual.
PinnedHostID string
}
Spec describes the resources needed for a sandbox.
type StateStore ¶
type StateStore interface {
UpsertVM(ctx context.Context, vm VMRecord) error
DeleteVM(ctx context.Context, vmID string) error
ListVMs(ctx context.Context) ([]VMRecord, error)
UpsertTask(ctx context.Context, task TaskRecord) error
ListTasks(ctx context.Context) ([]TaskRecord, error)
UpsertSnapshot(ctx context.Context, snapshot SnapshotRecord) error
GetSnapshot(ctx context.Context, snapshotID string) (SnapshotRecord, error)
ListSnapshots(ctx context.Context) ([]SnapshotRecord, error)
DeleteSnapshot(ctx context.Context, snapshotID string) error
// ListSnapshotsByVM returns snapshots for a single VM, scoped at the
// store layer (a WHERE clause for Postgres) rather than requiring the
// caller to load and filter every snapshot in the fleet.
ListSnapshotsByVM(ctx context.Context, vmID string) ([]SnapshotRecord, error)
// ListSnapshotsByDigest returns every ready snapshot for tenantID whose
// artifact hashes to digest. It is the artifact index: the orchestrator
// answers "which hosts already hold these bytes" by reading the host_id of
// each row it returns.
//
// The index is DERIVED, never a second table. A snapshot row is already the
// statement "host H holds artifact A", so a separate index could only ever
// disagree with it, and the disagreement would be invisible until a pull was
// aimed at a host that no longer had the bytes. Deriving also makes recovery
// free: there is nothing to rehydrate at startup because the query is the
// index, and a row whose digest was never recorded is simply not findable
// rather than wrongly findable.
//
// tenantID is a security boundary for the same reason it is one in
// FindSnapshotByLayerKey: an artifact carries whatever its build baked in.
ListSnapshotsByDigest(ctx context.Context, tenantID, digest string) ([]SnapshotRecord, error)
// FindSnapshotByLayerKey returns the newest ready layer artifact matching
// (tenantID, layerKey, arch), or ok=false when there is no hit.
//
// This is its own store method rather than a SnapshotFilter because it
// runs once per chain key on every build, and ListSnapshotsFiltered loads
// every snapshot in the fleet and filters in Go. Postgres serves this off
// a single indexed query.
//
// tenantID scoping is a security boundary, not an optimization: serving
// one tenant's artifact to another leaks whatever that build baked in, so
// a cross-tenant match must be a miss, never a hit.
FindSnapshotByLayerKey(ctx context.Context, tenantID, layerKey, arch string) (SnapshotRecord, bool, error)
// SnapshotQuotaUsage returns the count and aggregate size of snapshots
// for tenantID that count toward the per-tenant quota: states
// Creating/Ready/Restoring, excluding SnapshotModeBuild artifacts (see
// FleetManager.enforceSnapshotQuota). Computed at the store layer (a
// COUNT/SUM query for Postgres) so quota checks don't require a
// full-table scan on every CreateSnapshot call.
SnapshotQuotaUsage(ctx context.Context, tenantID string) (SnapshotQuotaUsage, error)
AppendEvent(ctx context.Context, event EventRecord) error
// UpsertDeadLetter inserts or updates a dead-letter entry keyed by
// (Kind, EntityID). On update, RetryCount and LastSeenAt are advanced.
UpsertDeadLetter(ctx context.Context, entry DeadLetterRecord) error
// ListDeadLetters returns all dead-letter entries. Implementations
// may order arbitrarily.
ListDeadLetters(ctx context.Context) ([]DeadLetterRecord, error)
// UpsertHost inserts or updates a host registration.
UpsertHost(ctx context.Context, host HostRecord) error
// DeleteHost removes a host from the registry. No-op if absent.
DeleteHost(ctx context.Context, hostID string) error
// ListHosts returns all registered hosts.
ListHosts(ctx context.Context) ([]HostRecord, error)
// GetHost returns a single host by ID, or an error if not found.
GetHost(ctx context.Context, hostID string) (HostRecord, error)
}
StateStore persists orchestrator control-plane state.
type TaskRecord ¶
type TaskRecord struct {
TaskID string
VMID string
RunStatus TaskRunStatus
RetryCount int
LastError string
AssignedAt time.Time
UpdatedAt time.Time
}
TaskRecord tracks durable task assignment/run metadata.
type TaskRunStatus ¶
type TaskRunStatus string
TaskRunStatus captures the lifecycle of a task assignment.
const ( TaskRunAssigned TaskRunStatus = "assigned" TaskRunRunning TaskRunStatus = "running" TaskRunCompleted TaskRunStatus = "completed" TaskRunFailed TaskRunStatus = "failed" )
type TokenSetter ¶
type TokenSetter interface {
SetToken(token string)
}
TokenSetter is implemented by environments whose per-sandbox auth token can be updated after construction (e.g. set during Boot once credentials are generated, or refreshed by token rotation). Boot and RotateToken type-assert to this interface so the Environment surface stays read-only for callers.
type VMFilter ¶
VMFilter narrows ListFleetFiltered results on exact-match fields, plus pagination controls.
Limit caps the page size (<=0 defaults to DefaultPageLimit, >MaxPageLimit is clamped down to it). Cursor, when non-empty, resumes after the last item of a previous page (see ListFleetFiltered's NextCursor return).
type VMInfo ¶
type VMInfo struct {
ID string
State VMState
TaskID string
HostID string
URL string
Spec Spec
CreatedAt time.Time
UpdatedAt time.Time
Error string
Endpoints []Endpoint
// Health is the environment-level probe's last verdict, or nil when the
// environment declared no probe or none has been read back yet. See
// health.go.
Health *HealthStatus
}
VMInfo is a read-only snapshot of a managed VM.
type VMRecord ¶
type VMRecord struct {
ID string
HostID string
NetworkHost string
State VMState
URL string
TaskID string
TenantID string
Spec Spec
LastError string
AuthTokenEncrypted []byte // AES-GCM encrypted per-VM auth token (nil for legacy VMs)
SecretsEncrypted []byte // AES-GCM encrypted JSON of the secret map (nil when no secrets supplied)
Endpoints []Endpoint // published endpoints (e.g. ingress), if any
CreatedAt time.Time
UpdatedAt time.Time
}
VMRecord is the durable representation of a fleet VM.
HostID is the loose reference to the placement host (orchestrator_hosts.host_id). NetworkHost is the externally-reachable host:port Fuse clients dial; it is derived from the provider-returned URL and stored verbatim so reconcile can rebuild routing without re-parsing.
SecretsEncrypted holds the per-VM secret map sealed with AES-GCM under the orchestrator's TOKEN_ENCRYPTION_KEY. It exists so that an orchestrator restart can re-upload the same secrets to the guest agent without the caller resubmitting them (the secrets path lives in the agent profile); without it, a crash mid-deploy would leave the VM running with stale or missing secrets and no way to recover them.
type VMState ¶
type VMState string
VMState represents the lifecycle state of a managed VM.
Stored states (recorded on a vm and persisted to the state store):
- VMStateProvisioning, VMStateRunning, VMStateDraining, VMStateDestroying.
Synthetic terminal states (emitted only over the wire as SSE events — never stored on a vm record because the underlying record is gone by the time we publish them): VMStateDestroyed, VMStateFailed. Defined alongside the wire format in events.go.
VMStateDestroyed is a synthetic terminal state emitted over the wire when a VM is removed from the fleet map (either via successful destroy or reap). It is NOT stored as a VMState on a vm record — once the vm is gone from the in-memory map there is nothing left to have a state — but subscribers need a terminal signal so they can close their stream cleanly.
const ( VMStateProvisioning VMState = "provisioning" VMStateRunning VMState = "running" // VMStateDraining indicates the VM has been asked to gracefully quiesce // in-guest workloads (agent shutdown) but the VM itself has not yet // been destroyed. This is the "drain" phase of two-phase teardown: // the harness gets a clean shutdown signal and a chance to flush // outputs before the subsequent DELETE actually tears down the VM. // A Draining VM is still tracked, still consumes host capacity, // and may still be inspected via the API. DELETE is the only // supported state transition out of Draining (no "undrain"). VMStateDraining VMState = "draining" VMStateDestroying VMState = "destroying" )