storage

package
v0.0.0-...-3e0e934 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 28, 2026 License: Apache-2.0, BSD-2-Clause Imports: 47 Imported by: 0

Documentation

Index

Constants

View Source
const (
	CowKindVolume   = cowKindVolume
	CowKindSnapshot = cowKindSnapshot
)
View Source
const (
	// CatalogKindRuntimeSnapshot is produced by CommitSandbox (taking a
	// snapshot of a running sandbox).
	CatalogKindRuntimeSnapshot = "runtime_snapshot"
	// CatalogKindTemplate is produced by AppSnapshot (building a template
	// from an image / one-shot sandbox).
	CatalogKindTemplate = "template"
)

Catalog entry kinds. See SnapshotCatalogEntry.Kind.

View Source
const StorageBackendCow = "cubecow"

StorageBackendCow is the canonical value of `storage_backend` for the cubecow (reflink-only copy-on-write) backend. cubelet refuses to boot when `storage_backend` is set to anything else under this build.

Variables

View Source
var DefaultEmptyDirFormats = []string{"512Mi", "others"}

DefaultEmptyDirFormats is the fallback list of format roots used by CleanupOrphanStorageFiles when callers do not supply their own.

View Source
var ErrCowObjectAlreadyExists = errors.New("cubecow object already exists")
View Source
var ErrCowObjectMissing = errors.New("cubecow object missing")
View Source
var ErrSnapshotCatalogNotFound = errors.New("snapshot catalog not found")

ErrSnapshotCatalogNotFound is returned when no catalog can be located for the given snapshot id under any registered snapshot root.

Functions

func AddSnapshotCatalogRoot

func AddSnapshotCatalogRoot(root string)

AddSnapshotCatalogRoot adds a snapshot root if not already present.

func CleanupCowTemplateObjects

func CleanupCowTemplateObjects(ctx context.Context, refs []CowObjectRef) error

func CleanupTemplateLocalData

func CleanupTemplateLocalData(ctx context.Context, templateID, snapshotPath string) error

func DeactivateCowObject

func DeactivateCowObject(ctx context.Context, name, kind string) error

func DeleteCowObject

func DeleteCowObject(ctx context.Context, name, kind string) error

func DeleteSnapshotCatalog

func DeleteSnapshotCatalog(snapshotID string)

DeleteSnapshotCatalog removes the in-memory cache entry for snapshotID. The on-disk file is cleared by CleanupTemplateLocalData which removes the entire snapshot directory; calling this directly is only useful when deletion happens out-of-band.

func Engine

func Engine() *cubecow.Engine

func GetCowMetrics

func GetCowMetrics(ctx context.Context) (map[string]uint64, error)

func GetPCIDiskInfo

func GetPCIDiskInfo(ctx context.Context, id string) (*disk.CubePCIDiskInfo, error)

func InspectStorageVolumes

func InspectStorageVolumes() (map[string]*StorageInfo, error)

InspectStorageVolumes returns every sandbox storage record cubelet currently owns, with cubecow device paths re-resolved through the live engine. It is the canonical data source for `cubecli storage ls` and any other tool that needs an authoritative cross-sandbox view.

func IsCowBackend

func IsCowBackend() bool

func PersistSandboxRootfsAfterRollback

func PersistSandboxRootfsAfterRollback(ctx context.Context, sandboxID string, rootfs *CowSnapshotObject) error

func ReadDir

func ReadDir(name string) ([]os.DirEntry, error)

func RefreshStorageInfoPathsWithEngine

func RefreshStorageInfoPathsWithEngine(ctx context.Context, info *StorageInfo, engine *cubecow.Engine) error

func ResolveCowDevPath

func ResolveCowDevPath(ctx context.Context, name, kind string) (string, error)

func SetSnapshotCatalogRoots

func SetSnapshotCatalogRoots(roots ...string)

SetSnapshotCatalogRoots replaces the configured set of snapshot root directories. Each root must point at the same parent that CommitSandbox uses for snapshot output (e.g. DefaultSnapshotDir). Duplicates are removed while preserving order.

func TemplateBuildRootfsName

func TemplateBuildRootfsName(templateID string) string

TemplateBuildRootfsName returns the deterministic cubecow volume name used for a template's temporary writable working layer during AppSnapshot. Exposed so non-storage callers (e.g. AppSnapshot handler writing snapshot catalog) can record the name without redeclaring the format string.

func WriteSnapshotCatalog

func WriteSnapshotCatalog(entry *SnapshotCatalogEntry) error

WriteSnapshotCatalog persists entry under <SnapshotPath>/catalog.json. Existing files are overwritten.

Types

type BackendFileInfo

type BackendFileInfo struct {
	Name string

	SizeLimit int64

	FSQuota int64

	FilePath string

	VolumeName string

	Kind string

	Gen uint32

	Type string

	SourcePath string

	SizeLimitQ string

	Medium cubebox.StorageMedium

	BDF string
}

type CleanupOrphanReport

type CleanupOrphanReport struct {
	Format   string
	FilePath string
	Removed  bool
	Err      error
}

CleanupOrphanReport describes the outcome of a single orphan file cleanup attempt produced by CleanupOrphanStorageFiles.

func CleanupOrphanStorageFiles

func CleanupOrphanStorageFiles(formats []string, dryRun bool) ([]CleanupOrphanReport, error)

CleanupOrphanStorageFiles scans the configured emptydir format roots and drops any file that has no live sandbox owner. When dryRun is true cubelet only reports the files it would delete. formats may be nil, in which case DefaultEmptyDirFormats is used.

type Config

type Config struct {
	RootPath string `toml:"root_path"`
	DataPath string `toml:"data_path"`

	DiskSize string `toml:"disksize"`

	WarningPercent int64 `toml:"warningPercent"`

	PoolDefaultFormatSizeList []string `toml:"pool_default_format_size_list"`

	BaseDiskUUID string `toml:"base_disk_uuid"`

	PoolSize int `toml:"pool_size"`

	PoolWorkers int `toml:"pool_worker_num"`

	FAdviseSize int `toml:"fadvise_size"`

	PoolType poolType `toml:"pool_type"`

	PoolTriggerIntervalInMs int `toml:"pool_trigger_interval_in_ms"`

	PoolTriggerBurst int `toml:"pool_trigger_burst"`

	DisableDiskCheck bool `toml:"disable_disk_check"`

	FreeBlocksThreshold int32 `toml:"free_blocks_threshold"`

	FreeInodesThreshold int32            `toml:"free_inodes_threshold"`
	ReconcileInterval   tomlext.Duration `toml:"reconcile_interval"`

	StorageBackend string          `toml:"storage_backend"`
	Cow            CowInlineConfig `toml:"cow"`

	// CmdTimeout overrides the per-command timeout for utils.ExecV
	// invocations in shell.go (cp / truncate / e2fsck / resize2fs /
	// mkfs.ext4). Defaults to defaultCmdTimeout when zero. The slow
	// ext4-create path on multi-GiB images can need noticeably more
	// than the 3s default; this knob lets operators bump it without
	// recompiling.
	CmdTimeout tomlext.Duration `toml:"cmd_timeout"`
}

func (*Config) BuildCowInitJSON

func (c *Config) BuildCowInitJSON() ([]byte, error)

func (*Config) PrepareCowInlineConfig

func (c *Config) PrepareCowInlineConfig() error

PrepareCowInlineConfig stamps cubelet-owned defaults (currently the reflink backend kind and its `root_dir` derived from `data_path`) onto the config so BuildCowInitJSON has everything cubecow needs.

type CowBackendConfig

type CowBackendConfig struct {
	Kind    string `toml:"-"`
	Reflink CowReflinkBackendConfig
}

CowBackendConfig is filled in by cubelet at init time, never by the user, and shipped to cubecow as the `[backend]` block.

type CowInlineConfig

type CowInlineConfig struct {
	Log     CowLogConfig     `toml:"log"`
	Backend CowBackendConfig `toml:"-"`
}

CowInlineConfig mirrors the cubecow `AppConfig` schema. cubecow is reflink-only and cubelet always owns the cubecow init payload (there is no external cubecow.toml fallback), so the only thing users can tune through TOML is the `[log]` block. The reflink backend's `root_dir` is derived from `data_path` and stamped onto `Backend.Reflink` in PrepareCowInlineConfig.

type CowLogConfig

type CowLogConfig struct {
	Level    *string `toml:"level"`
	Format   *string `toml:"format"`
	File     *string `toml:"file"`
	Rotation *string `toml:"rotation"`
}

type CowObjectMissingError

type CowObjectMissingError struct {
	VolumeName string
	Kind       string
	Cause      error
}

func (*CowObjectMissingError) Error

func (e *CowObjectMissingError) Error() string

func (*CowObjectMissingError) Is

func (e *CowObjectMissingError) Is(target error) bool

func (*CowObjectMissingError) Unwrap

func (e *CowObjectMissingError) Unwrap() error

type CowObjectRef

type CowObjectRef struct {
	Name string
	Kind string
	Role string
}

func DefaultTemplateObjectRefs

func DefaultTemplateObjectRefs(templateID string) []CowObjectRef

type CowObjectStatus

type CowObjectStatus struct {
	Name         string
	Kind         string
	Role         string
	Exists       bool
	DevicePath   string
	SizeBytes    uint64
	ErrorMessage string
}

func InspectCowObjects

func InspectCowObjects(ctx context.Context, refs []CowObjectRef) ([]CowObjectStatus, error)

type CowReflinkBackendConfig

type CowReflinkBackendConfig struct {
	RootDir *string `toml:"-"`
}

CowReflinkBackendConfig is the `[backend.reflink]` payload. cubelet always derives `RootDir` from `data_path` so reflink files live on the same FICLONE-capable filesystem as the rest of cubelet's state.

type CowRollbackSnapshotRefs

type CowRollbackSnapshotRefs struct {
	Rootfs *CowSnapshotObject
	Memory *CowSnapshotObject
}

func ResolveSnapshotForRollback

func ResolveSnapshotForRollback(ctx context.Context, rootfsVol, memoryVol, memoryKind string) (*CowRollbackSnapshotRefs, error)

ResolveSnapshotForRollback resolves the cubecow objects that back a snapshot for the purposes of rollback. memoryKind is honored so that snapshot replicas whose memory blob was produced by reflink-clone (kind=snapshot, used for incremental memory snapshots) work alongside the legacy empty-volume layout (kind=volume).

type CowSnapshotObject

type CowSnapshotObject struct {
	Name      string
	MountName string
	Kind      string
	DevPath   string
	SizeBytes uint64
	Gen       uint32
}

func CommitTemplateMemoryFromBase

func CommitTemplateMemoryFromBase(ctx context.Context, source *CowSnapshotObject, templateID string, sizeBytes uint64) (*CowSnapshotObject, error)

CommitTemplateMemoryFromBase clones an existing memory object (typically the memory blob backing the sandbox's current base snapshot/template) into the canonical template memory name for templateID. The resulting object is a cubecow snapshot whose content starts as an exact reflink-shared copy of the source, making it a valid base for the hypervisor's incremental (pagemap_anon) memory snapshot, which only writes CoW anonymous pages and relies on the rest of memory being preserved from the base file.

func CommitTemplateRootfs

func CommitTemplateRootfs(ctx context.Context, source *CowSnapshotObject, templateID string) (*CowSnapshotObject, error)

func CreateTemplateMemoryVolume

func CreateTemplateMemoryVolume(ctx context.Context, templateID string, sizeBytes uint64) (*CowSnapshotObject, error)

func CreateTemplateRootfsFromBuild

func CreateTemplateRootfsFromBuild(ctx context.Context, templateID string) (*CowSnapshotObject, error)

func GetSandboxRootfsForSnapshot

func GetSandboxRootfsForSnapshot(ctx context.Context, sandboxID, preferredVolumeName string) (*CowSnapshotObject, error)

func RollbackDeriveNewGen

func RollbackDeriveNewGen(ctx context.Context, sandboxID, snapshotRootfsVol string, newGen uint32, desiredSizeBytes uint64) (*CowSnapshotObject, error)

type CowVolumeManager

type CowVolumeManager struct {
	// contains filtered or unexported fields
}

func (*CowVolumeManager) CommitTemplateMemory

func (m *CowVolumeManager) CommitTemplateMemory(ctx context.Context, sourceName, templateID string, sizeBytes uint64) (*cowVolume, error)

CommitTemplateMemory clones an existing memory object (sourceName) into the canonical template memory name for templateID via cubecow's reflink-backed CreateSnapshot. Unlike CreateMemoryVolume which produces an empty volume, this preserves the source memory bytes so the hypervisor can perform an incremental (pagemap_anon) snapshot that only overwrites CoW anonymous pages while keeping the rest of the base memory intact.

activate=true is passed through so callers immediately receive a usable device path. With the reflink backend, activation is effectively a no-op (snapshots are addressable via their filesystem path).

func (*CowVolumeManager) CommitTemplateRootfs

func (m *CowVolumeManager) CommitTemplateRootfs(ctx context.Context, sourceName, templateID string) (*cowVolume, error)

func (*CowVolumeManager) CreateDefaultMediumVolume

func (m *CowVolumeManager) CreateDefaultMediumVolume(ctx context.Context, sandboxID, volumeName string, sizeBytes uint64) (*cowVolume, error)

func (*CowVolumeManager) CreateMemoryVolume

func (m *CowVolumeManager) CreateMemoryVolume(ctx context.Context, templateID string, sizeBytes uint64) (*cowVolume, error)

func (*CowVolumeManager) CreateSandboxRootfsFromTemplate

func (m *CowVolumeManager) CreateSandboxRootfsFromTemplate(ctx context.Context, sandboxID, templateID string, gen uint32, desiredSizeBytes uint64) (*cowVolume, error)

func (*CowVolumeManager) CreateTemplateBuildRootfs

func (m *CowVolumeManager) CreateTemplateBuildRootfs(ctx context.Context, templateID string, sizeBytes uint64) (*cowVolume, error)

func (*CowVolumeManager) DeactivateByKind

func (m *CowVolumeManager) DeactivateByKind(ctx context.Context, name, kind string) error

func (*CowVolumeManager) DeleteByKind

func (m *CowVolumeManager) DeleteByKind(ctx context.Context, name, kind string) error

func (*CowVolumeManager) GetMetrics

func (m *CowVolumeManager) GetMetrics(ctx context.Context) (map[string]uint64, error)

func (*CowVolumeManager) GetSizeBytes

func (m *CowVolumeManager) GetSizeBytes(ctx context.Context, name string) (uint64, error)

func (*CowVolumeManager) GetVolumeInfo

func (m *CowVolumeManager) GetVolumeInfo(ctx context.Context, name string) (*cubecow.Volume, error)

func (*CowVolumeManager) ResolveDevPath

func (m *CowVolumeManager) ResolveDevPath(ctx context.Context, name, kind string) (string, error)

func (*CowVolumeManager) RollbackDeriveNewGen

func (m *CowVolumeManager) RollbackDeriveNewGen(ctx context.Context, sandboxID, snapshotRootfsVol string, gen uint32, desiredSizeBytes uint64) (*cowVolume, error)

type HostDirBackendInfo

type HostDirBackendInfo struct {
	VolumeName string `json:"volume_name"`

	ShareDir string `json:"share_dir"`

	BindPath string `json:"bind_path"`

	ReadOnly bool `json:"read_only"`
}

type HostStorageMeta

type HostStorageMeta struct {
	InstanceID string
	IP         string
}

type Pool

type Pool interface {
	Get(ctx context.Context, size int64) (*devInfo, error)
	GetSync(ctx context.Context, size int64) (*devInfo, error)
	Close()
	InitBaseFile(ctx context.Context) error
}

type SnapshotCatalogEntry

type SnapshotCatalogEntry struct {
	SnapshotID   string `json:"snapshot_id"`
	InstanceType string `json:"instance_type,omitempty"`
	SpecDir      string `json:"spec_dir,omitempty"`
	SnapshotPath string `json:"snapshot_path"`
	MetaDir      string `json:"meta_dir"`
	RootfsVol    string `json:"rootfs_vol"`
	RootfsKind   string `json:"rootfs_kind"`
	MemoryVol    string `json:"memory_vol"`
	MemoryKind   string `json:"memory_kind"`
	// BuildRootfsVol/Kind track the temporary writable working layer created
	// during template build (AppSnapshot path). They must be cleaned up at
	// template delete time. Empty for runtime snapshots (CommitSandbox), which
	// never produce a build artifact.
	BuildRootfsVol  string `json:"build_rootfs_vol,omitempty"`
	BuildRootfsKind string `json:"build_rootfs_kind,omitempty"`
	RootfsSizeBytes uint64 `json:"rootfs_size_bytes,omitempty"`
	// Kind distinguishes the producer/semantics of this catalog entry so
	// CleanupTemplate/GetLocalSnapshot consumers can branch where needed.
	// Empty == legacy entry (pre-v4) and should be treated as a runtime
	// snapshot for backward compatibility.
	Kind      string `json:"kind,omitempty"`
	CreatedAt string `json:"created_at,omitempty"`
}

SnapshotCatalogEntry captures everything cubelet needs in order to roll back to a snapshot, restore-from-snapshot for a fresh sandbox, or clean up the underlying cubecow artifacts, without requiring master to carry physical cubecow/path references in its tables.

Persisted as <snapshot_path>/catalog.json by CommitSandbox (Kind=KindRuntimeSnapshot) and AppSnapshot (Kind=KindTemplate). Dev paths are re-resolved from cubecow on demand because they can change across activations. Unknown JSON fields are tolerated and missing fields decode as zero values so old catalog files keep working after schema extensions.

Runtime artifact identity belongs here, not in the rootfs artifact cache. In particular, future "redo snapshot" checks should compare the node's active kernel artifact identity with the identity recorded in this catalog entry; a kernel mismatch requires rebuilding the snapshot/template replica, but does not by itself require rebuilding the rootfs ext4 artifact.

func GetLocalSnapshot

func GetLocalSnapshot(ctx context.Context, snapshotID string) (*SnapshotCatalogEntry, error)

GetLocalSnapshot looks up the catalog for snapshotID. The cache is consulted first; on miss the on-disk roots are scanned. Returns ErrSnapshotCatalogNotFound if no record exists.

func ListLocalSnapshots

func ListLocalSnapshots(ctx context.Context) ([]*SnapshotCatalogEntry, error)

ListLocalSnapshots returns every catalog entry discoverable under the configured roots. The in-memory cache is refreshed as a side effect.

type SnapshotCatalogEntryResolved

type SnapshotCatalogEntryResolved struct {
	SnapshotCatalogEntry
	RootfsDev string `json:"rootfs_dev,omitempty"`
	MemoryDev string `json:"memory_dev,omitempty"`
}

SnapshotCatalogEntryResolved enriches a catalog entry with freshly re-resolved cubecow device paths.

func ResolveLocalSnapshot

func ResolveLocalSnapshot(ctx context.Context, snapshotID string) (*SnapshotCatalogEntryResolved, error)

ResolveLocalSnapshot returns the catalog entry plus freshly re-resolved device paths (rootfs/memory) via cubecow. Useful for callers that immediately need to feed the dev paths into a cubecow operation.

type StateRecoverer

type StateRecoverer interface {
	RecoverStorageState(ctx context.Context) error
	RecoverSandboxStorage(ctx context.Context, sandboxID string) error
}

type StorageInfo

type StorageInfo struct {
	Namespace string

	SandboxID string
	Volumes   map[string]*BackendFileInfo

	CubePCIDiskInfo *disk.CubePCIDiskInfo

	CubePCISystemDiskInfo *disk.CubePCISystemDiskInfo

	InstanceType string    `json:"instanceType,omitempty"`
	TemplateID   string    `json:"templateId,omitempty"`
	CreateAt     time.Time `json:"createAt,omitempty"`
	UpdateAt     time.Time `json:"updateAt,omitempty"`

	HostDirBackendInfos map[string]*HostDirBackendInfo `json:"hostDirBackendInfos,omitempty"`

	RestoreMemoryVolURL string `json:"-"`
}

func (*StorageInfo) GetNICQueues

func (i *StorageInfo) GetNICQueues() int64

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL