Documentation
¶
Overview ¶
Package orchestrator defines the interface for sandbox providers and implements the boot flow that provisions or restores environments.
Index ¶
- Constants
- Variables
- func IsTerminalState(s VMState) bool
- func NewEventID() string
- func Schedule(spec Spec, hosts []*Host, policy PlacementPolicy) (*Host, PlacementDecision, error)
- type AgentSpec
- type AttachSpec
- type Attacher
- type BootOptions
- type BootResult
- type CapacityProber
- type Checkpoint
- type DeadLetterKind
- type DeadLetterRecord
- type Endpoint
- type EndpointReporter
- type Environment
- type EnvironmentEvent
- type EventRecord
- type ExecOptions
- type ExecResult
- type ExposeSpec
- type FleetConfig
- type FleetManager
- func (fm *FleetManager) Attach(ctx context.Context, vmID string, spec AttachSpec) (io.ReadWriteCloser, error)
- func (fm *FleetManager) AuditEvent(ctx context.Context, entityType, entityID, eventType string, ...)
- func (fm *FleetManager) CompleteTask(taskID string) error
- func (fm *FleetManager) CordonHost(ctx context.Context, hostID string) error
- func (fm *FleetManager) CreateSnapshot(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotRecord, error)
- func (fm *FleetManager) DeleteSnapshot(ctx context.Context, vmID, snapshotID string) error
- func (fm *FleetManager) DeleteSnapshotByID(ctx context.Context, snapshotID string) error
- func (fm *FleetManager) DestroyVM(ctx context.Context, vmID string) error
- func (fm *FleetManager) Drain(ctx context.Context, vmID string) error
- func (fm *FleetManager) Exec(ctx context.Context, vmID string, cmd []string, opts ExecOptions) (ExecResult, error)
- func (fm *FleetManager) ForkEnvironment(ctx context.Context, srcVMID string, opts ForkOptions) (string, error)
- func (fm *FleetManager) GetHost(hostID string) (Host, bool)
- func (fm *FleetManager) GetSnapshot(ctx context.Context, vmID, snapshotID string) (SnapshotRecord, error)
- func (fm *FleetManager) GetSnapshotByID(ctx context.Context, snapshotID string) (SnapshotRecord, error)
- func (fm *FleetManager) GetVM(vmID string) (VMInfo, bool)
- func (fm *FleetManager) GetVMByTask(taskID string) (VMInfo, bool)
- func (fm *FleetManager) ListFleet() []VMInfo
- func (fm *FleetManager) ListFleetFiltered(filter VMFilter) []VMInfo
- func (fm *FleetManager) ListHosts() []Host
- func (fm *FleetManager) ListSnapshots(ctx context.Context, vmID string) ([]SnapshotRecord, error)
- func (fm *FleetManager) ListSnapshotsFiltered(ctx context.Context, filter SnapshotFilter) ([]SnapshotRecord, error)
- func (fm *FleetManager) ProvisionAndAssign(ctx context.Context, taskID string, spec Spec, manifest []byte, ...) (*VMInfo, error)
- func (fm *FleetManager) RegisterHost(ctx context.Context, h Host, p Provider) error
- func (fm *FleetManager) RemoveHost(ctx context.Context, hostID string) error
- func (fm *FleetManager) RestoreSnapshot(ctx context.Context, vmID, snapshotID string) error
- func (fm *FleetManager) RestoreSnapshotByID(ctx context.Context, snapshotID string) error
- func (fm *FleetManager) RotateToken(ctx context.Context, vmID string) error
- func (fm *FleetManager) Start(ctx context.Context)
- func (fm *FleetManager) Stop()
- func (fm *FleetManager) SubscribeEnvironmentEvents(vmID string) (<-chan EnvironmentEvent, func())
- func (fm *FleetManager) UncordonHost(ctx context.Context, hostID string) error
- type ForkOptions
- type Host
- type HostBackend
- type HostCapacity
- type HostRecord
- type HostState
- type MemoryStateStore
- func (s *MemoryStateStore) AppendEvent(_ context.Context, event EventRecord) error
- func (s *MemoryStateStore) DeleteHost(_ context.Context, hostID string) error
- func (s *MemoryStateStore) DeleteSnapshot(_ context.Context, snapshotID string) error
- func (s *MemoryStateStore) DeleteVM(_ context.Context, vmID string) error
- func (s *MemoryStateStore) GetHost(_ context.Context, hostID string) (HostRecord, error)
- func (s *MemoryStateStore) GetSnapshot(_ context.Context, snapshotID string) (SnapshotRecord, error)
- func (s *MemoryStateStore) ListDeadLetters(_ context.Context) ([]DeadLetterRecord, error)
- func (s *MemoryStateStore) ListHosts(_ context.Context) ([]HostRecord, error)
- func (s *MemoryStateStore) ListSnapshots(_ context.Context) ([]SnapshotRecord, error)
- func (s *MemoryStateStore) ListTasks(_ context.Context) ([]TaskRecord, error)
- func (s *MemoryStateStore) ListVMs(_ context.Context) ([]VMRecord, error)
- func (s *MemoryStateStore) UpsertDeadLetter(_ context.Context, entry DeadLetterRecord) error
- func (s *MemoryStateStore) UpsertHost(_ context.Context, host HostRecord) error
- func (s *MemoryStateStore) UpsertSnapshot(_ context.Context, snapshot SnapshotRecord) error
- func (s *MemoryStateStore) UpsertTask(_ context.Context, task TaskRecord) error
- func (s *MemoryStateStore) UpsertVM(_ context.Context, vm VMRecord) error
- type PlacementDecision
- type PlacementPolicy
- type PostgresStateStore
- func (s *PostgresStateStore) AppendEvent(ctx context.Context, event EventRecord) error
- func (s *PostgresStateStore) ApplyMigrations(ctx context.Context) error
- func (s *PostgresStateStore) DeleteHost(ctx context.Context, hostID string) error
- func (s *PostgresStateStore) DeleteSnapshot(ctx context.Context, snapshotID string) error
- func (s *PostgresStateStore) DeleteVM(ctx context.Context, vmID string) error
- func (s *PostgresStateStore) GetHost(ctx context.Context, hostID string) (HostRecord, error)
- func (s *PostgresStateStore) GetSnapshot(ctx context.Context, snapshotID string) (SnapshotRecord, error)
- func (s *PostgresStateStore) ListDeadLetters(ctx context.Context) ([]DeadLetterRecord, error)
- func (s *PostgresStateStore) ListHosts(ctx context.Context) ([]HostRecord, error)
- func (s *PostgresStateStore) ListSnapshots(ctx context.Context) ([]SnapshotRecord, error)
- func (s *PostgresStateStore) ListTasks(ctx context.Context) ([]TaskRecord, error)
- func (s *PostgresStateStore) ListVMs(ctx context.Context) ([]VMRecord, error)
- func (s *PostgresStateStore) UpsertDeadLetter(ctx context.Context, entry DeadLetterRecord) error
- func (s *PostgresStateStore) UpsertHost(ctx context.Context, h HostRecord) error
- func (s *PostgresStateStore) UpsertSnapshot(ctx context.Context, snapshot SnapshotRecord) error
- func (s *PostgresStateStore) UpsertTask(ctx context.Context, task TaskRecord) error
- func (s *PostgresStateStore) UpsertVM(ctx context.Context, vm VMRecord) error
- type Provider
- type ReconcileMetrics
- type ReconcileSummary
- type SnapshotCapable
- type SnapshotDeleter
- type SnapshotExportRecord
- type SnapshotExportStatus
- type SnapshotFilter
- type SnapshotForkable
- type SnapshotMode
- type SnapshotOptions
- type SnapshotRecord
- type SnapshotState
- type Spec
- type StateStore
- type TaskRecord
- type TaskRunStatus
- type TokenSetter
- type VMFilter
- type VMInfo
- type VMRecord
- type VMState
Constants ¶
const ( DefaultExecTimeout = 60 * time.Second MaxExecTimeout = 600 * time.Second )
Exec timeout bounds. Guest commands are bounded so a hung command cannot pin an SSH session on the host agent forever. MaxExecTimeout matches the host agent's own per-exec ceiling; asking for more would let the caller's deadline outlive the guest command it is waiting on.
const DefaultFusedDrainCommand = "systemctl stop fused 2>/dev/null || pkill -TERM fused"
DefaultFusedDrainCommand is the fused profile's graceful-stop invocation, run inside the guest via Environment.ExecStream on Drain.
The fused agent traps SIGTERM and tears down its service DAG before exiting, so stopping it with a signal IS a full graceful drain. On the firecracker profile fused runs as a systemd unit, so `systemctl stop` delivers SIGTERM, waits for the clean exit, and (because the stop is clean) the unit's Restart=on-failure does NOT bring it back. The `pkill -TERM` fallback covers non-systemd guests, which also quiesce on SIGTERM.
No trailing `|| true`: a genuine failure (neither a unit nor a process was stopped) propagates a non-zero exit so Drain records the error and keeps the VM Draining. A graceful-stop command that silently no-op'd (e.g. a missing helper hidden behind `|| true`) would defeat that.
const FuseSecretsPath = fuseSecretsPath
FuseSecretsPath is exported so fleet.reuploadSecrets can re-upload to the profile-declared path without hardcoding the literal.
Variables ¶
var ( // ErrTaskAlreadyAssigned is returned by ProvisionAndAssign when a // VM is already tracked for the given task id. The existing VM's // id is included in the wrapping fmt.Errorf context. ErrTaskAlreadyAssigned = errors.New("task already assigned") // ErrVMNotFound is returned by DestroyVM, CreateSnapshot, // RestoreSnapshot, and ListSnapshots when the vm id does not // match a tracked VM. ErrVMNotFound = errors.New("vm not found") // ErrTaskNotFound is returned by CompleteTask when the task id // does not match any VM. ErrTaskNotFound = errors.New("task not found") // ErrSnapshotNotFound is returned when a snapshot ID does not // correspond to a persisted snapshot record for the requested VM. ErrSnapshotNotFound = errors.New("snapshot not found") // ErrSnapshotQuotaExceeded is returned when creating a snapshot // would exceed the configured retention quota. ErrSnapshotQuotaExceeded = errors.New("snapshot quota exceeded") // ErrSnapshotInvalidState is returned when an operation targets a // snapshot that is not currently in the required lifecycle state. ErrSnapshotInvalidState = errors.New("snapshot invalid state") // ErrSnapshotHasChildren is returned when deleting a snapshot that // still has descendant snapshots in the lineage graph. ErrSnapshotHasChildren = errors.New("snapshot has children") // ErrVMNotRunning is returned by operations that require a VM to be // in the Running state (e.g. Drain). The current state is included // in the wrapping fmt.Errorf context so callers and operators can // see why the transition was rejected. ErrVMNotRunning = errors.New("vm not running") // ErrExecUnsupported is returned by Exec when the environment has no // real guest to run commands in (e.g. the in-memory stub a provider // falls back to when its BaseURL is unset). Reporting it is what keeps // a misconfigured host from answering exec with a fabricated success. ErrExecUnsupported = errors.New("exec not supported by provider") // ErrAttachUnsupported is returned by Attach when the environment does // not implement Attacher. ErrAttachUnsupported = errors.New("attach not supported by provider") )
Sentinel errors exposed by the FleetManager public API. Callers (notably the REST handler layer) use errors.Is to map these to HTTP status codes without string-matching. These are kept intentionally coarse — they cover the cases that differ at the API boundary, not every internal failure mode.
var DefaultFusedManifest = []byte(`{"version":"1","machine":{"workspace":"/workspace"},"services":{}}`)
DefaultFusedManifest is the fused profile's default compiled manifest, used by the API when a caller omits an inline manifest. It encodes the fused manifest schema and is profile data, not a generic core default.
var ErrHostNotFound = fmt.Errorf("host not found")
ErrHostNotFound is returned when a host operation targets an unregistered host ID.
var ErrNoCapacity = errors.New("no host has sufficient capacity")
ErrNoCapacity is returned by Schedule when no registered host can fit the requested spec. The REST handler maps this to 503.
var ErrNoHosts = errors.New("no hosts registered")
ErrNoHosts is returned by Schedule when the host registry is empty.
Functions ¶
func IsTerminalState ¶
IsTerminalState reports whether the given VM state is a terminal state from the SSE subscriber's perspective. After a terminal event the broadcaster will not publish further events for the same VM, so the handler closes the stream.
func NewEventID ¶
func NewEventID() string
NewEventID returns a 128-bit hex-encoded random identifier suitable for use as an SSE `id:` field. Using crypto/rand avoids pulling in a UUID dependency and gives the same uniqueness guarantee. The stringified form is opaque to clients — they round-trip it via Last-Event-ID for resume.
Exported so the api package can mint ids for synthesised snapshot events without importing crypto/rand directly.
func Schedule ¶
func Schedule(spec Spec, hosts []*Host, policy PlacementPolicy) (*Host, PlacementDecision, error)
Schedule picks the best host for the given spec according to the placement policy. It is a pure function with no side effects — the caller is responsible for updating the host's Allocated counters and persisting the decision.
Filter pipeline:
- Exclude non-schedulable hosts (cordoned, draining).
- If spec.Region is non-empty, exclude hosts in a different region.
- Exclude hosts that don't fit the spec (capacity - allocated < spec).
- Among survivors, pick by policy (binpack or spread).
Ties within a policy are broken by host ID for determinism.
Types ¶
type AgentSpec ¶
type AgentSpec struct {
Files map[string][]byte // arbitrary files to upload into the guest (path -> bytes)
DownloadURL string // fetch the agent binary (e.g. GitHub releases)
Command string // how to launch the daemon
AuthToken string // pass-through bearer token
Gateway string // pass-through gateway websocket URL
GatewayToken string // pass-through gateway token
DrainCommand string // command run inside the guest for graceful shutdown (” => skip)
Expose []ExposeSpec // guest ports to publish as reachable endpoints, if any
}
AgentSpec is the generic, provider-agnostic description of the guest agent to launch inside a sandbox. fused is expressed as one configuration of this spec via FusedAgentSpec (see agent_profile.go); nothing fuse-specific is hardcoded in the core boot path.
func FusedAgentSpec ¶
func FusedAgentSpec(manifest []byte, secretMap map[string]string, creds *secrets.VMCredentials, opts BootOptions) AgentSpec
FusedAgentSpec builds the fused profile: the full AgentSpec that reproduces the orchestrator's original boot behavior. Files carries the manifest, the secrets JSON (defaulting to "{}" when nil/empty), and — when creds is set — the TLS/auth credential files. Command is the fused launch line.
NOTE on Command: the Command field is the generic launch line for providers that run a free-form shell command. The firecracker host agent IGNORES Command and instead reads structured fields off the frozen /start-surfd wire (manifest/secrets/TLS paths), sourcing them from its own path constants that mirror the consts above.
type AttachSpec ¶ added in v0.4.0
type AttachSpec struct {
// Cmd is the argv to run. Empty means the guest's login shell.
Cmd []string
// TTY requests a pty for the process. Rows and Cols seed its initial
// window size; further resizes arrive as frames on the stream itself.
TTY bool
Rows uint16
Cols uint16
}
AttachSpec describes an interactive attach to a sandbox.
type Attacher ¶ added in v0.4.0
type Attacher interface {
Attach(ctx context.Context, spec AttachSpec) (io.ReadWriteCloser, error)
}
Attacher is implemented by environments that can open a raw duplex byte stream to a process inside the sandbox. The stream carries the fuse-attach/1 frame protocol (see docs/attach.md); nothing between the caller and the guest interprets those frames, so the orchestrator relays them verbatim.
Environments with no real guest (e.g. in-memory stubs) omit this method; callers type-assert and report ErrAttachUnsupported when it is absent.
type BootOptions ¶
type BootOptions struct {
StartupScript string
GatewayURL string
GatewayToken string
Expose []ExposeSpec
}
type BootResult ¶
type BootResult struct {
Env Environment
BootTime time.Duration
FromCache bool // true if restored from checkpoint
AuthTokenEncrypted []byte // AES-GCM encrypted per-VM auth token for persistence
DrainCommand string // graceful-shutdown command for the configured agent (” => skip)
Endpoints []Endpoint // published endpoints, if the provider reported any
}
BootResult is returned after provisioning or restoring an environment.
func Boot ¶
func Boot(ctx context.Context, p Provider, spec Spec, manifest []byte, secretMap map[string]string, opts BootOptions, encryptionKey []byte) (*BootResult, error)
Boot provisions or restores an environment, uploads the agent's files, and starts the configured guest agent. If a matching checkpoint exists, it restores from it. Otherwise creates fresh. When encryptionKey is non-nil (32 bytes), per-VM TLS credentials and an auth token are generated and injected. The encrypted token is returned in BootResult for persistence.
The guest agent is described by an AgentSpec; today that is always the fused profile (FusedAgentSpec), which carries the fused manifest/secrets/TLS files and launch command. Boot itself is profile-agnostic.
type CapacityProber ¶ added in v0.4.0
type CapacityProber interface {
Capacity(ctx context.Context) (HostCapacity, error)
}
CapacityProber is implemented by providers that can report the real hardware capacity of the host they front (CPU count, total RAM, free disk) instead of trusting operator-declared numbers. RegisterHost type- asserts to this interface at registration time to source capacity for any field the operator left unset; providers that cannot probe (e.g. a stub with no real hardware behind it) simply return an error.
type Checkpoint ¶
Checkpoint is a snapshot of a sandbox.
type DeadLetterKind ¶
type DeadLetterKind string
DeadLetterKind identifies the kind of failure a dead-letter entry represents.
const ( // DeadLetterOrphanDestroy records repeated failures to destroy an // orphan VM observed by reconcile but not tracked in the fleet. DeadLetterOrphanDestroy DeadLetterKind = "orphan_destroy" // DeadLetterStuckTask records a task that exceeded its runtime ceiling // and was torn down by the reconcile loop. DeadLetterStuckTask DeadLetterKind = "stuck_task" )
type DeadLetterRecord ¶
type DeadLetterRecord struct {
ID int64
Kind DeadLetterKind
EntityID string
TaskID string
Reason string
RetryCount int
Payload json.RawMessage
FirstSeenAt time.Time
LastSeenAt time.Time
}
DeadLetterRecord is a failure the reconciler has given up on retrying. Entries are keyed uniquely by (Kind, EntityID); repeated failures update the RetryCount and LastSeenAt fields rather than inserting new rows.
type Endpoint ¶ added in v0.2.0
type Endpoint struct {
As string // caller-chosen label, e.g. "http"
URL string // reachable address, e.g. "http://203.0.113.5:41231"
Port int // the guest-side port this endpoint publishes
}
Endpoint is a published network endpoint for an environment (e.g. an ingress port exposed via the Fusefile's `expose` list).
type EndpointReporter ¶ added in v0.2.0
type EndpointReporter interface {
Endpoints() []Endpoint
}
EndpointReporter is implemented by environments that can report additional network endpoints published during StartAgent (e.g. via ingress/expose). Providers that don't support ingress simply omit it, in which case Boot reports no endpoints — consistent with how SnapshotCapable/SnapshotForkable are optional per-provider capabilities.
type Environment ¶
type Environment interface {
// Name returns the sandbox identifier.
Name() string
// URL returns the sandbox's reachable address for the guest agent.
URL() string
// Exec runs argv inside the sandbox and reports the result. A non-zero
// guest exit code is carried in ExecResult.ExitCode, not returned as an
// error; err is reserved for transport and provider failures, so callers
// must check both.
Exec(ctx context.Context, cmd []string, opts ExecOptions) (ExecResult, error)
// ExecStream runs a command with stdout/stderr wired to writers.
ExecStream(ctx context.Context, stdout, stderr io.Writer, name string, args ...string) error
// Upload writes data to a file path inside the sandbox.
Upload(ctx context.Context, data []byte, path string) error
// StartAgent launches the configured guest agent inside the sandbox.
StartAgent(ctx context.Context, spec AgentSpec) error
// Token returns the per-sandbox auth token that callers must include
// when reaching URL(). Empty string for providers whose URL
// self-authenticates.
Token() string
}
Environment is a running sandbox.
type EnvironmentEvent ¶
type EnvironmentEvent struct {
ID string `json:"id"`
Kind string `json:"event"`
VMID string `json:"vm_id"`
State VMState `json:"state"`
URL string `json:"url,omitempty"`
Error string `json:"error,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
EnvironmentEvent is a single state-change notification published by the FleetManager broadcaster and consumed by SSE subscribers.
The struct is deliberately a flat, JSON-serialisable shape so the REST handler can encode it directly into an SSE `data:` line. The `Kind` field always serialises as `event` (matching the SSE event dispatch contract on the client) — for v1 the only kind emitted is "state". Future event kinds (e.g. "log", "snapshot") would be added here without breaking existing subscribers.
Wire shape (one line of SSE data:):
{
"id": "<event-uuid>",
"event": "state",
"vm_id": "...",
"state": "running",
"url": "host:port",
"error": "...",
"updated_at": "..."
}
type EventRecord ¶
type EventRecord struct {
ID int64
EntityType string
EntityID string
EventType string
Payload json.RawMessage
CreatedAt time.Time
}
EventRecord stores an audit event for critical lifecycle transitions.
type ExecOptions ¶ added in v0.4.0
type ExecOptions struct {
// Timeout bounds how long the guest command may run. Zero means
// DefaultExecTimeout. Values above MaxExecTimeout are clamped.
Timeout time.Duration
}
ExecOptions tunes a single Exec call.
type ExecResult ¶ added in v0.4.0
ExecResult is the outcome of running a command inside a sandbox. Stdout and Stderr are kept separate and byte-exact; ExitCode is the guest command's own status, which is meaningful even when it is non-zero.
type ExposeSpec ¶ added in v0.2.0
ExposeSpec requests that a guest port be published as a reachable endpoint by the provider during StartAgent.
type FleetConfig ¶
type FleetConfig struct {
Provider Provider
StateStore StateStore
Prefix string // VM name prefix, e.g. "fuse-"
ReconcileInterval time.Duration // default 30s
// TaskStuckTimeout is the maximum age of a Running VM with no state
// transitions before it is considered stuck. This is a leak-detection
// ceiling, NOT a heartbeat — healthy long-running tasks must set
// Spec.MaxRuntime to override this. Default 2h.
TaskStuckTimeout time.Duration
// OrphanDestroyMaxRetries is the number of consecutive reconcile cycles
// an orphan VM may fail to destroy before being dead-lettered and
// skipped on subsequent cycles. Default 5.
OrphanDestroyMaxRetries int
// DefaultSnapshotRetention applies to snapshots created without an
// explicit retention window. Zero leaves snapshots unbounded until
// explicitly deleted.
DefaultSnapshotRetention time.Duration
// SnapshotQuotaMaxCount limits ready/in-flight snapshots per tenant.
// Zero disables count-based enforcement.
SnapshotQuotaMaxCount int
// SnapshotQuotaMaxBytes limits the aggregate size of ready/in-flight
// snapshots per tenant. Zero disables byte-based enforcement.
SnapshotQuotaMaxBytes int64
// PlacementPolicy controls how the scheduler picks among hosts
// when multiple are registered. Default is spread.
PlacementPolicy PlacementPolicy
// TokenEncryptionKey is the 32-byte AES-256 key used to encrypt
// per-VM auth tokens before storing them in the state store.
// When nil, VM credential generation is skipped (insecure mode).
TokenEncryptionKey []byte
// HostProviderFactory builds a Provider for a given (url, token,
// backend) triple. The fleet uses it during recoverState to
// rehydrate the per-host providers that RegisterHost would have
// created during normal operation. Without it, an orchestrator
// restart loses the scheduler's host registry and falls back to
// single-provider mode for all subsequent placements.
HostProviderFactory func(url, token string, backend HostBackend) Provider
Metrics ReconcileMetrics
Logger *slog.Logger
}
FleetConfig configures the fleet manager.
type FleetManager ¶
type FleetManager struct {
// contains filtered or unexported fields
}
FleetManager tracks and manages a fleet of VMs.
func NewFleetManager ¶
func NewFleetManager(cfg FleetConfig) *FleetManager
NewFleetManager creates a fleet manager. Call Start to begin the reconciliation loop.
func (*FleetManager) Attach ¶ added in v0.4.0
func (fm *FleetManager) Attach(ctx context.Context, vmID string, spec AttachSpec) (io.ReadWriteCloser, error)
Attach opens a raw duplex byte stream to a process inside a running VM's guest. The stream carries fuse-attach/1 frames; the fleet does not interpret them, it only hands the stream back to the caller to relay.
The caller owns the returned stream and must Close it.
func (*FleetManager) AuditEvent ¶
func (fm *FleetManager) AuditEvent(ctx context.Context, entityType, entityID, eventType string, payload map[string]any)
AuditEvent records a security or operational event in the state store's event table. It is the public face of appendEvent, exposed so the REST middleware can log auth failures and IP rejections without reaching into FleetManager internals.
func (*FleetManager) CompleteTask ¶
func (fm *FleetManager) CompleteTask(taskID string) error
CompleteTask marks a task as done and triggers async VM destruction.
func (*FleetManager) CordonHost ¶
func (fm *FleetManager) CordonHost(ctx context.Context, hostID string) error
CordonHost marks a host as cordoned (no new VMs). Existing VMs are left running.
func (*FleetManager) CreateSnapshot ¶
func (fm *FleetManager) CreateSnapshot(ctx context.Context, vmID string, opts SnapshotOptions) (SnapshotRecord, error)
CreateSnapshot quiesces the given VM, invokes Environment.Checkpoint, persists a first-class SnapshotRecord, and marks it ready once the provider confirms the new checkpoint exists.
func (*FleetManager) DeleteSnapshot ¶
func (fm *FleetManager) DeleteSnapshot(ctx context.Context, vmID, snapshotID string) error
DeleteSnapshot removes a leaf snapshot resource after deleting the provider-side artifact.
func (*FleetManager) DeleteSnapshotByID ¶
func (fm *FleetManager) DeleteSnapshotByID(ctx context.Context, snapshotID string) error
DeleteSnapshotByID removes a leaf snapshot resource by global snapshot ID.
func (*FleetManager) DestroyVM ¶
func (fm *FleetManager) DestroyVM(ctx context.Context, vmID string) error
DestroyVM forcefully tears down a VM by ID.
func (*FleetManager) Drain ¶
func (fm *FleetManager) Drain(ctx context.Context, vmID string) error
Drain transitions a Running VM into the Draining state and runs the configured drain command inside the guest (via Environment.ExecStream) to quiesce in-guest workloads gracefully. It is the first phase of two-phase environment teardown:
- POST /v1/environments/{vmId}?action=drain → Drain → guest quiesce
- DELETE /v1/environments/{vmId} → DestroyVM → VM gone
Drain is intentionally narrow:
- Only Running VMs can be drained. Any other state (Provisioning, Draining, Destroying) returns ErrVMNotRunning so callers can map it to a 409 Conflict and inspect the current state.
- Drain does not destroy the VM on success or on failure. Even if the drain command fails, the VM stays in Draining so the caller can still issue DELETE through the existing path. Auto-destroy here would defeat the whole point of giving the harness a chance to recover from a partial drain.
- An empty drain command leaves the VM Draining for the caller to DELETE (back-compat): no graceful command is run.
- Drain takes its own context; if the caller's request context is cancelled we abort the command cleanly. The default drainTimeout is layered on top so a slow guest cannot hold a request thread indefinitely.
The state transition (Running → Draining) is performed and persisted before the drain command fires. That ordering matters: if the orchestrator crashes mid-drain, recovery will see a Draining VM and the operator can decide whether to retry Drain or proceed straight to DELETE. The alternative (command first, then state flip) would lose the intent of the drain on a crash.
func (*FleetManager) Exec ¶ added in v0.4.0
func (fm *FleetManager) Exec(ctx context.Context, vmID string, cmd []string, opts ExecOptions) (ExecResult, error)
Exec runs argv inside a running VM's guest and reports the result.
A non-zero guest exit code is not an error: it is carried in ExecResult.ExitCode with stdout and stderr intact, so a caller can tell "the command ran and failed" apart from "the command could not be run". The returned error is reserved for the latter — unknown VM, wrong state, or a transport failure reaching the host.
Only Running VMs can be exec'd into. A Draining VM is deliberately refused even though its guest is still reachable: drain means the workload is being quiesced, and letting new commands in behind that would undo it.
func (*FleetManager) ForkEnvironment ¶ added in v0.2.0
func (fm *FleetManager) ForkEnvironment(ctx context.Context, srcVMID string, opts ForkOptions) (string, error)
ForkEnvironment creates a brand-new vm seeded from a checkpoint of an existing running vm. it obtains a seed snapshot (creating one when ReuseSnapshotID is empty), asks a SnapshotForkable provider to build a new environment from that checkpoint, registers the new vm as running, and records lineage so the fork references its seed snapshot.
lineage is recorded by persisting a SnapshotRecord for the new vm whose ParentSnapshotID is the seed snapshot id (state ready). this reuses the same store path CreateSnapshot uses (upsertSnapshotRecord) and is directly assertable via ListSnapshots / GetSnapshotByID.
the fork is charged to the source's host (a fork is pinned there: the seed snapshot's rootfs is host-local) and is given its OWN guest credentials, since it boots a copy of the source's disk and would otherwise answer to the source's token.
providers that cannot fork do not implement SnapshotForkable, so this reports fork as unsupported for them. the firecracker provider implements it; the qemu provider deliberately does not (vfio gpu passthrough cannot be checkpointed).
func (*FleetManager) GetHost ¶
func (fm *FleetManager) GetHost(hostID string) (Host, bool)
GetHost returns a snapshot of a registered host.
func (*FleetManager) GetSnapshot ¶
func (fm *FleetManager) GetSnapshot(ctx context.Context, vmID, snapshotID string) (SnapshotRecord, error)
GetSnapshot returns one persisted snapshot resource scoped to the VM.
func (*FleetManager) GetSnapshotByID ¶
func (fm *FleetManager) GetSnapshotByID(ctx context.Context, snapshotID string) (SnapshotRecord, error)
GetSnapshotByID returns one persisted snapshot resource by its global ID.
func (*FleetManager) GetVM ¶
func (fm *FleetManager) GetVM(vmID string) (VMInfo, bool)
GetVM returns info for a specific VM.
func (*FleetManager) GetVMByTask ¶
func (fm *FleetManager) GetVMByTask(taskID string) (VMInfo, bool)
GetVMByTask returns the VM assigned to a given task.
func (*FleetManager) ListFleet ¶
func (fm *FleetManager) ListFleet() []VMInfo
ListFleet returns a snapshot of all tracked VMs.
func (*FleetManager) ListFleetFiltered ¶
func (fm *FleetManager) ListFleetFiltered(filter VMFilter) []VMInfo
ListFleetFiltered returns tracked VMs filtered by optional exact-match fields.
func (*FleetManager) ListHosts ¶
func (fm *FleetManager) ListHosts() []Host
ListHosts returns all registered hosts.
func (*FleetManager) ListSnapshots ¶
func (fm *FleetManager) ListSnapshots(ctx context.Context, vmID string) ([]SnapshotRecord, error)
ListSnapshots returns all snapshots known to the state store for the given VM ID, newest first. Returns an empty slice when the VM exists but has no snapshots.
func (*FleetManager) ListSnapshotsFiltered ¶
func (fm *FleetManager) ListSnapshotsFiltered(ctx context.Context, filter SnapshotFilter) ([]SnapshotRecord, error)
ListSnapshotsFiltered returns snapshots across the fleet filtered by optional exact-match fields. Missing resources yield an empty list.
func (*FleetManager) ProvisionAndAssign ¶
func (fm *FleetManager) ProvisionAndAssign(ctx context.Context, taskID string, spec Spec, manifest []byte, secretMap map[string]string, opts BootOptions) (*VMInfo, error)
ProvisionAndAssign provisions a new VM, boots fused, and assigns the given task. Blocks until the VM is ready or an error occurs.
func (*FleetManager) RegisterHost ¶
RegisterHost adds a host to the scheduler's registry. If the host already exists, its capacity, URL, token, and region are updated (useful for heartbeat-like refreshes). The host starts in HostActive state.
The caller must supply a Provider for this host (typically a firecracker.Provider constructed with the host's URL/token). FleetManager holds only the Provider interface to avoid import cycles with concrete provider packages.
func (*FleetManager) RemoveHost ¶
func (fm *FleetManager) RemoveHost(ctx context.Context, hostID string) error
RemoveHost deletes a host from the registry. It must have no VMs assigned to it; callers should cordon/drain and wait for VMs to leave before removing.
func (*FleetManager) RestoreSnapshot ¶
func (fm *FleetManager) RestoreSnapshot(ctx context.Context, vmID, snapshotID string) error
RestoreSnapshot rolls a VM back to a prior snapshot via Environment.Restore after validating the metadata record and provider visibility of the checkpoint.
func (*FleetManager) RestoreSnapshotByID ¶
func (fm *FleetManager) RestoreSnapshotByID(ctx context.Context, snapshotID string) error
RestoreSnapshotByID restores a VM from a snapshot identified by its global ID.
func (*FleetManager) RotateToken ¶
func (fm *FleetManager) RotateToken(ctx context.Context, vmID string) error
RotateToken generates new TLS credentials and an auth token for a running VM, uploads them to the guest filesystem, and persists the new encrypted token.
KNOWN GAP: fused reads --auth-token-file exactly once at process start (cmd/fused/main.go) and has no credential poller, so a live fused keeps serving the OLD token until it is restarted. Rotation therefore updates the orchestrator's copy and the guest's files but does not yet take effect in the running guest; it needs a StartAgent call (which restarts fused) the way ForkEnvironment does.
Rotation is a server-side operation: the orchestrator updates its own encrypted copy and the guest agent's credential files (paths owned by the agent profile). New inbound connections use the rotated credentials; existing connections retain the old cert until they reconnect.
func (*FleetManager) Start ¶
func (fm *FleetManager) Start(ctx context.Context)
Start begins the background reconciliation loop.
func (*FleetManager) Stop ¶
func (fm *FleetManager) Stop()
Stop cancels the reconciliation loop and waits for it to finish.
func (*FleetManager) SubscribeEnvironmentEvents ¶
func (fm *FleetManager) SubscribeEnvironmentEvents(vmID string) (<-chan EnvironmentEvent, func())
SubscribeEnvironmentEvents registers a subscriber for state-change events on a single VM. The returned channel receives events until cancel is called (idempotent) or the broadcaster drops the connection. Channel capacity is bounded; slow subscribers drop events with a logged warning rather than blocking publishers.
This is a single-process broadcaster: events published on one orchestrator replica are not visible to subscribers on a different replica. The orchestrator runs as a single process today, so this is acceptable; cross-replica fanout would require a Redis or NATS backplane and is intentionally deferred.
func (*FleetManager) UncordonHost ¶
func (fm *FleetManager) UncordonHost(ctx context.Context, hostID string) error
UncordonHost returns a cordoned or draining host to active scheduling.
type ForkOptions ¶ added in v0.2.0
type ForkOptions struct {
// Comment is attached to the seed snapshot when ForkEnvironment
// creates one (empty ReuseSnapshotID). ignored when reusing an
// existing snapshot.
Comment string
// ReuseSnapshotID selects an existing ready snapshot of the source
// vm to seed the fork from. empty means snapshot the source first.
ReuseSnapshotID string
}
ForkOptions tunes a ForkEnvironment call. All fields are optional.
type Host ¶
type Host struct {
ID string
URL string // base URL of the host agent (e.g. https://agent-1.local)
Token string // bearer token for this host's agent
Region string
Backend HostBackend // "firecracker" or "qemu"; empty means firecracker (default)
Capacity HostCapacity
Allocated HostCapacity
State HostState
LastSeen time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
Host is a registered compute host in the fleet. It represents a single Firecracker host agent that can provision VMs.
type HostBackend ¶ added in v0.3.0
type HostBackend string
HostBackend identifies the virtualization backend a host agent runs. It determines which capabilities the host can offer the scheduler (e.g. only qemu hosts may advertise GPUs).
const ( // BackendFirecracker is the default backend: microVMs with no GPU // passthrough support. BackendFirecracker HostBackend = "firecracker" // BackendQEMU is a full-VM backend that supports GPU passthrough. BackendQEMU HostBackend = "qemu" )
type HostCapacity ¶
type HostCapacity struct {
CPUs int `json:"cpus"`
RamMB int `json:"ram_mb"`
StorageGB int `json:"storage_gb"`
VMCount int `json:"vm_count"` // max concurrent VMs
// GPUs is the count of whole GPU devices available on the host.
// Zero means no GPUs. Only qemu-backed hosts may report GPUs > 0
// (enforced at registration, see internal/api registerHost).
GPUs int `json:"gpus,omitempty"`
// GPUKind identifies the GPU model (e.g. "a100"). Empty when GPUs is 0.
GPUKind string `json:"gpu_kind,omitempty"`
}
HostCapacity is the resource envelope of a host. It is reported by the host agent at registration time and refreshed by heartbeat. The scheduler compares Spec against (Capacity - Allocated) to make admission decisions.
type HostRecord ¶
type HostRecord struct {
ID string
URL string
TokenEncrypted []byte
Region string
State HostState
TenantID string
Backend HostBackend
Capacity HostCapacity
Allocated HostCapacity
LastSeen time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
HostRecord is the durable representation of a compute host in the scheduler's registry. It maps 1:1 to a row in orchestrator_hosts.
TokenEncrypted is the agent bearer token sealed with AES-GCM using the orchestrator's TOKEN_ENCRYPTION_KEY (same key the per-VM tokens use). It is decrypted only into the in-memory Host.Token used by the provider client; the plaintext never enters the database.
type HostState ¶
type HostState string
HostState captures the scheduling eligibility of a registered host.
const ( // HostActive means the host is healthy and accepting new VMs. HostActive HostState = "active" // HostCordoned means the host is not accepting new VMs. Existing // VMs are left running. Cordon is the operator's "maintenance // soon, stop sending work here" signal. HostCordoned HostState = "cordoned" // HostDraining means the host is cordoned AND its existing VMs // are being evicted by the reconcile loop. Once all VMs are gone // the host is eligible for removal. Drain support in reconcile // is stubbed in this PR; the state is defined so the data model // doesn't need a follow-up migration. HostDraining HostState = "draining" )
type MemoryStateStore ¶
type MemoryStateStore struct {
// contains filtered or unexported fields
}
MemoryStateStore is a process-local store useful for tests/default behavior.
func NewMemoryStateStore ¶
func NewMemoryStateStore() *MemoryStateStore
NewMemoryStateStore returns an in-memory StateStore implementation.
func (*MemoryStateStore) AppendEvent ¶
func (s *MemoryStateStore) AppendEvent(_ context.Context, event EventRecord) error
func (*MemoryStateStore) DeleteHost ¶
func (s *MemoryStateStore) DeleteHost(_ context.Context, hostID string) error
func (*MemoryStateStore) DeleteSnapshot ¶
func (s *MemoryStateStore) DeleteSnapshot(_ context.Context, snapshotID string) error
func (*MemoryStateStore) DeleteVM ¶
func (s *MemoryStateStore) DeleteVM(_ context.Context, vmID string) error
func (*MemoryStateStore) GetHost ¶
func (s *MemoryStateStore) GetHost(_ context.Context, hostID string) (HostRecord, error)
func (*MemoryStateStore) GetSnapshot ¶
func (s *MemoryStateStore) GetSnapshot(_ context.Context, snapshotID string) (SnapshotRecord, error)
func (*MemoryStateStore) ListDeadLetters ¶
func (s *MemoryStateStore) ListDeadLetters(_ context.Context) ([]DeadLetterRecord, error)
func (*MemoryStateStore) ListHosts ¶
func (s *MemoryStateStore) ListHosts(_ context.Context) ([]HostRecord, error)
func (*MemoryStateStore) ListSnapshots ¶
func (s *MemoryStateStore) ListSnapshots(_ context.Context) ([]SnapshotRecord, error)
func (*MemoryStateStore) ListTasks ¶
func (s *MemoryStateStore) ListTasks(_ context.Context) ([]TaskRecord, error)
func (*MemoryStateStore) ListVMs ¶
func (s *MemoryStateStore) ListVMs(_ context.Context) ([]VMRecord, error)
func (*MemoryStateStore) UpsertDeadLetter ¶
func (s *MemoryStateStore) UpsertDeadLetter(_ context.Context, entry DeadLetterRecord) error
func (*MemoryStateStore) UpsertHost ¶
func (s *MemoryStateStore) UpsertHost(_ context.Context, host HostRecord) error
func (*MemoryStateStore) UpsertSnapshot ¶
func (s *MemoryStateStore) UpsertSnapshot(_ context.Context, snapshot SnapshotRecord) error
func (*MemoryStateStore) UpsertTask ¶
func (s *MemoryStateStore) UpsertTask(_ context.Context, task TaskRecord) error
type PlacementDecision ¶
type PlacementDecision struct {
HostID string `json:"host_id"`
Policy PlacementPolicy `json:"policy"`
Candidates int `json:"candidates"` // eligible hosts considered
HeadroomCPUs int `json:"headroom_cpus"` // CPUs remaining after placement
HeadroomRam int `json:"headroom_ram"` // RAM remaining after placement
}
PlacementDecision records why the scheduler picked a particular host. It is attached to the VM record for debugging.
type PlacementPolicy ¶
type PlacementPolicy string
PlacementPolicy controls how the scheduler picks among eligible hosts that all have sufficient capacity.
const ( // PlacementBinpack fills hosts as densely as possible. It picks // the host with the MOST already-allocated resources that still // has room. This minimizes the number of active hosts. PlacementBinpack PlacementPolicy = "binpack" // PlacementSpread distributes VMs as evenly as possible. It picks // the host with the LEAST already-allocated resources. This // maximizes isolation between VMs. PlacementSpread PlacementPolicy = "spread" )
type PostgresStateStore ¶
type PostgresStateStore struct {
// contains filtered or unexported fields
}
PostgresStateStore persists orchestrator state in Postgres.
func NewPostgresStateStore ¶
func NewPostgresStateStore(db *sql.DB) *PostgresStateStore
NewPostgresStateStore creates a Postgres-backed state store.
func (*PostgresStateStore) AppendEvent ¶
func (s *PostgresStateStore) AppendEvent(ctx context.Context, event EventRecord) error
func (*PostgresStateStore) ApplyMigrations ¶
func (s *PostgresStateStore) ApplyMigrations(ctx context.Context) error
ApplyMigrations creates and upgrades orchestrator state tables.
func (*PostgresStateStore) DeleteHost ¶
func (s *PostgresStateStore) DeleteHost(ctx context.Context, hostID string) error
func (*PostgresStateStore) DeleteSnapshot ¶
func (s *PostgresStateStore) DeleteSnapshot(ctx context.Context, snapshotID string) error
func (*PostgresStateStore) DeleteVM ¶
func (s *PostgresStateStore) DeleteVM(ctx context.Context, vmID string) error
func (*PostgresStateStore) GetHost ¶
func (s *PostgresStateStore) GetHost(ctx context.Context, hostID string) (HostRecord, error)
func (*PostgresStateStore) GetSnapshot ¶
func (s *PostgresStateStore) GetSnapshot(ctx context.Context, snapshotID string) (SnapshotRecord, error)
func (*PostgresStateStore) ListDeadLetters ¶
func (s *PostgresStateStore) ListDeadLetters(ctx context.Context) ([]DeadLetterRecord, error)
func (*PostgresStateStore) ListHosts ¶
func (s *PostgresStateStore) ListHosts(ctx context.Context) ([]HostRecord, error)
func (*PostgresStateStore) ListSnapshots ¶
func (s *PostgresStateStore) ListSnapshots(ctx context.Context) ([]SnapshotRecord, error)
func (*PostgresStateStore) ListTasks ¶
func (s *PostgresStateStore) ListTasks(ctx context.Context) ([]TaskRecord, error)
func (*PostgresStateStore) ListVMs ¶
func (s *PostgresStateStore) ListVMs(ctx context.Context) ([]VMRecord, error)
func (*PostgresStateStore) UpsertDeadLetter ¶
func (s *PostgresStateStore) UpsertDeadLetter(ctx context.Context, entry DeadLetterRecord) error
func (*PostgresStateStore) UpsertHost ¶
func (s *PostgresStateStore) UpsertHost(ctx context.Context, h HostRecord) error
func (*PostgresStateStore) UpsertSnapshot ¶
func (s *PostgresStateStore) UpsertSnapshot(ctx context.Context, snapshot SnapshotRecord) error
func (*PostgresStateStore) UpsertTask ¶
func (s *PostgresStateStore) UpsertTask(ctx context.Context, task TaskRecord) error
type Provider ¶
type Provider interface {
// Create provisions a new sandbox.
Create(ctx context.Context, spec Spec) (Environment, error)
// Get returns a handle to an existing sandbox by name.
Get(ctx context.Context, name string) (Environment, error)
// Destroy tears down a sandbox.
Destroy(ctx context.Context, name string) error
// List returns all sandboxes matching the given prefix.
List(ctx context.Context, prefix string) ([]Environment, error)
// Close releases provider resources.
Close() error
}
Provider manages sandboxed environments.
type ReconcileMetrics ¶
type ReconcileMetrics interface {
ReconcileCompleted(summary ReconcileSummary)
}
ReconcileMetrics is an optional callback invoked at the end of every reconcile cycle with a counter summary. Implementations typically feed a Prometheus/OTel exporter — the orchestrator package intentionally does not depend on either. A nil ReconcileMetrics disables metrics reporting.
type ReconcileSummary ¶
type ReconcileSummary struct {
TrackedVMs int
ProviderVMs int
OrphansDestroyed int
OrphansFailed int
OrphansDeadLettered int
StuckTasksSuspected int
StuckTasksFailed int
VMsMissingProvider int
Duration time.Duration
}
ReconcileSummary captures counts from a single reconcile cycle.
type SnapshotCapable ¶
type SnapshotCapable interface {
Checkpoint(ctx context.Context, comment string) (string, error)
Restore(ctx context.Context, checkpointID string) error
ListCheckpoints(ctx context.Context) ([]Checkpoint, error)
}
SnapshotCapable is implemented by environments that support checkpoint/restore. Providers that cannot snapshot (e.g. some container-based providers) simply omit these methods; callers must type-assert to SnapshotCapable before invoking them.
type SnapshotDeleter ¶
SnapshotDeleter is implemented by environments that can delete a previously created checkpoint by ID.
type SnapshotExportRecord ¶
type SnapshotExportRecord struct {
Destination string
Status SnapshotExportStatus
RequestedAt time.Time
UpdatedAt time.Time
LastError string
}
SnapshotExportRecord captures metadata for an optional exported artifact.
type SnapshotExportStatus ¶
type SnapshotExportStatus string
SnapshotExportStatus tracks the state of an optional export record.
const ( SnapshotExportPending SnapshotExportStatus = "pending" SnapshotExportReady SnapshotExportStatus = "ready" SnapshotExportError SnapshotExportStatus = "error" )
type SnapshotFilter ¶
type SnapshotFilter struct {
VMID string
TaskID string
TenantID string
State SnapshotState
}
SnapshotFilter narrows ListSnapshotsFiltered results on exact-match fields.
type SnapshotForkable ¶ added in v0.2.0
type SnapshotForkable interface {
CreateFromCheckpoint(ctx context.Context, spec Spec, srcVMID, checkpointID string) (Environment, error)
}
SnapshotForkable is implemented by providers that can create a brand-new environment seeded from an existing checkpoint of another vm. this is the capability a true fork needs; the firecracker provider does not implement it yet, so ForkEnvironment reports fork as unsupported at runtime until a host wire endpoint exists.
type SnapshotMode ¶
type SnapshotMode string
SnapshotMode captures how a snapshot was created.
const ( SnapshotModeManual SnapshotMode = "manual" SnapshotModeAuto SnapshotMode = "auto" )
type SnapshotOptions ¶
type SnapshotOptions struct {
// Comment is a free-form note attached to the snapshot. Passed
// through to the underlying Environment.Checkpoint implementation
// and persisted in the snapshot record.
Comment string
// Mode records whether this snapshot was taken manually (via the
// REST API) or by an automated process. Defaults to
// SnapshotModeManual when empty.
Mode SnapshotMode
// RetentionUntil, if non-nil, records when the snapshot is
// eligible for garbage collection.
RetentionUntil *time.Time
// Metadata augments the persisted metadata blob. Reserved fields
// such as "comment" are set by the orchestrator.
Metadata map[string]string
// Exports records optional object-storage export intents/status.
Exports []SnapshotExportRecord
}
SnapshotOptions tunes a CreateSnapshot call. All fields are optional.
type SnapshotRecord ¶
type SnapshotRecord struct {
SnapshotID string
VMID string
TaskID string
HostID string
TenantID string
ParentSnapshotID string
Mode SnapshotMode
State SnapshotState
SizeBytes int64
RetentionUntil *time.Time
Metadata json.RawMessage
Exports []SnapshotExportRecord
LastError string
CreatedAt time.Time
UpdatedAt time.Time
}
SnapshotRecord tracks checkpoint lineage and retention metadata.
type SnapshotState ¶
type SnapshotState string
SnapshotState captures the lifecycle of a persisted snapshot resource.
const ( SnapshotStateCreating SnapshotState = "creating" SnapshotStateReady SnapshotState = "ready" SnapshotStateRestoring SnapshotState = "restoring" SnapshotStateDeleting SnapshotState = "deleting" SnapshotStateError SnapshotState = "error" )
type Spec ¶
type Spec struct {
Name string // e.g. "fuse-{task-id}"
CPUs int
RamMB int
StorageGB int
Region string
// GPUs is the count of whole GPU devices requested. Zero means no GPU.
GPUs int32
// GPUKind identifies the requested GPU model (e.g. "a100"). Empty when
// GPUs is 0.
GPUKind string
// MaxRuntime overrides FleetConfig.TaskStuckTimeout for this task.
// Zero means "use the fleet default". This is a leak-detection ceiling,
// not a liveness check — set it higher than any plausible healthy runtime.
MaxRuntime time.Duration
// Image names a base rootfs for the provider to boot from, resolved by
// the provider (e.g. the firecracker host agent looks it up in its own
// named-rootfs directory). Empty means the provider's default base.
Image string
}
Spec describes the resources needed for a sandbox.
type StateStore ¶
type StateStore interface {
UpsertVM(ctx context.Context, vm VMRecord) error
DeleteVM(ctx context.Context, vmID string) error
ListVMs(ctx context.Context) ([]VMRecord, error)
UpsertTask(ctx context.Context, task TaskRecord) error
ListTasks(ctx context.Context) ([]TaskRecord, error)
UpsertSnapshot(ctx context.Context, snapshot SnapshotRecord) error
GetSnapshot(ctx context.Context, snapshotID string) (SnapshotRecord, error)
ListSnapshots(ctx context.Context) ([]SnapshotRecord, error)
DeleteSnapshot(ctx context.Context, snapshotID string) error
AppendEvent(ctx context.Context, event EventRecord) error
// UpsertDeadLetter inserts or updates a dead-letter entry keyed by
// (Kind, EntityID). On update, RetryCount and LastSeenAt are advanced.
UpsertDeadLetter(ctx context.Context, entry DeadLetterRecord) error
// ListDeadLetters returns all dead-letter entries. Implementations
// may order arbitrarily.
ListDeadLetters(ctx context.Context) ([]DeadLetterRecord, error)
// UpsertHost inserts or updates a host registration.
UpsertHost(ctx context.Context, host HostRecord) error
// DeleteHost removes a host from the registry. No-op if absent.
DeleteHost(ctx context.Context, hostID string) error
// ListHosts returns all registered hosts.
ListHosts(ctx context.Context) ([]HostRecord, error)
// GetHost returns a single host by ID, or an error if not found.
GetHost(ctx context.Context, hostID string) (HostRecord, error)
}
StateStore persists orchestrator control-plane state.
type TaskRecord ¶
type TaskRecord struct {
TaskID string
VMID string
RunStatus TaskRunStatus
RetryCount int
LastError string
AssignedAt time.Time
UpdatedAt time.Time
}
TaskRecord tracks durable task assignment/run metadata.
type TaskRunStatus ¶
type TaskRunStatus string
TaskRunStatus captures the lifecycle of a task assignment.
const ( TaskRunAssigned TaskRunStatus = "assigned" TaskRunRunning TaskRunStatus = "running" TaskRunCompleted TaskRunStatus = "completed" TaskRunFailed TaskRunStatus = "failed" )
type TokenSetter ¶
type TokenSetter interface {
SetToken(token string)
}
TokenSetter is implemented by environments whose per-sandbox auth token can be updated after construction (e.g. set during Boot once credentials are generated, or refreshed by token rotation). Boot and RotateToken type-assert to this interface so the Environment surface stays read-only for callers.
type VMInfo ¶
type VMInfo struct {
ID string
State VMState
TaskID string
HostID string
URL string
Spec Spec
CreatedAt time.Time
UpdatedAt time.Time
Error string
Endpoints []Endpoint
}
VMInfo is a read-only snapshot of a managed VM.
type VMRecord ¶
type VMRecord struct {
ID string
HostID string
NetworkHost string
State VMState
URL string
TaskID string
TenantID string
Spec Spec
LastError string
AuthTokenEncrypted []byte // AES-GCM encrypted per-VM auth token (nil for legacy VMs)
SecretsEncrypted []byte // AES-GCM encrypted JSON of the secret map (nil when no secrets supplied)
Endpoints []Endpoint // published endpoints (e.g. ingress), if any
CreatedAt time.Time
UpdatedAt time.Time
}
VMRecord is the durable representation of a fleet VM.
HostID is the loose reference to the placement host (orchestrator_hosts.host_id). NetworkHost is the externally-reachable host:port Fuse clients dial; it is derived from the provider-returned URL and stored verbatim so reconcile can rebuild routing without re-parsing.
SecretsEncrypted holds the per-VM secret map sealed with AES-GCM under the orchestrator's TOKEN_ENCRYPTION_KEY. It exists so that an orchestrator restart can re-upload the same secrets to the guest agent without the caller resubmitting them (the secrets path lives in the agent profile); without it, a crash mid-deploy would leave the VM running with stale or missing secrets and no way to recover them.
type VMState ¶
type VMState string
VMState represents the lifecycle state of a managed VM.
Stored states (recorded on a vm and persisted to the state store):
- VMStateProvisioning, VMStateRunning, VMStateDraining, VMStateDestroying.
Synthetic terminal states (emitted only over the wire as SSE events — never stored on a vm record because the underlying record is gone by the time we publish them): VMStateDestroyed, VMStateFailed. Defined alongside the wire format in events.go.
VMStateDestroyed is a synthetic terminal state emitted over the wire when a VM is removed from the fleet map (either via successful destroy or reap). It is NOT stored as a VMState on a vm record — once the vm is gone from the in-memory map there is nothing left to have a state — but subscribers need a terminal signal so they can close their stream cleanly.
const ( VMStateProvisioning VMState = "provisioning" VMStateRunning VMState = "running" // VMStateDraining indicates the VM has been asked to gracefully quiesce // in-guest workloads (agent shutdown) but the VM itself has not yet // been destroyed. This is the "drain" phase of two-phase teardown: // the harness gets a clean shutdown signal and a chance to flush // outputs before the subsequent DELETE actually tears down the VM. // A Draining VM is still tracked, still consumes host capacity, // and may still be inspected via the API. DELETE is the only // supported state transition out of Draining (no "undrain"). VMStateDraining VMState = "draining" VMStateDestroying VMState = "destroying" )