Documentation
¶
Index ¶
- Constants
- Variables
- func DisplayRef(ref string) string
- func EmitProgress(ctx context.Context, item string, current, total int64, unit string)
- func EnsureLease(ctx context.Context) (context.Context, error)
- func HasLazyLease(ctx context.Context) bool
- func IsNotFound(err error) bool
- func MakeTemporary(l *leases.Lease) error
- func NewLayerApplier(store content.Provider) diff.Applier
- func NewNetworkProgressReader(ctx context.Context, item string, total int64, r io.ReadCloser, ...) (io.ReadCloser, error)
- func NewProgressReader(ctx context.Context, item string, total int64, r io.ReadCloser) io.ReadCloser
- func ReleaseOperationLeasesAfterRestart(ctx context.Context, lm leases.Manager) error
- func ReleaseTransferLeasesAfterRestart(ctx context.Context, lm leases.Manager) error
- func WithLazyLease(ctx context.Context, lm leases.Manager, opts ...leases.Opt) (context.Context, func(context.Context) error, error)
- func WithLease(ctx context.Context, ls leases.Manager, opts ...leases.Opt) (context.Context, func(context.Context) error, error)
- func WithNamespace(ctx context.Context, ns string) context.Context
- func WithoutLazyLease(ctx context.Context) context.Context
- type Accessor
- type ChainContentError
- type ChainWork
- type Diff
- type ExportChain
- type ExportLayer
- type ExportStats
- type ImmutableRef
- type ImportImageOpts
- type ImportTiming
- type ImportedImage
- type ImportedLayerBlobKey
- type ImportedLayerBlobRow
- type ImportedLayerDiffKey
- type ImportedLayerDiffRow
- type LeaseManager
- func (l *LeaseManager) AddResource(ctx context.Context, lease leases.Lease, resource leases.Resource) error
- func (l *LeaseManager) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error)
- func (l *LeaseManager) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error
- func (l *LeaseManager) DeleteResource(ctx context.Context, lease leases.Lease, resource leases.Resource) error
- func (l *LeaseManager) List(ctx context.Context, filters ...string) ([]leases.Lease, error)
- func (l *LeaseManager) ListResources(ctx context.Context, lease leases.Lease) ([]leases.Resource, error)
- func (l *LeaseManager) Namespace() string
- func (l *LeaseManager) WithNamespace(ns string) *LeaseManager
- type LeaseRef
- type LocalMounterOpt
- type MergeSnapshotter
- type MetadataStore
- type Mountable
- type MountableRef
- type Mounter
- type MutableRef
- type PackProvider
- type PersistentMetadataRows
- type ProgressTracker
- type ReadOnlyMountReleaser
- type Ref
- type RefMetadata
- type RefOption
- type SharedMounter
- type SnapshotContentHashRow
- type SnapshotContentRow
- type SnapshotManager
- type SnapshotManagerOpt
- type SnapshotRecordMetadata
- type Snapshotter
- type Value
Constants ¶
const ( // using is kept for its next reader. DefaultSharedMountLinger = 30 * time.Second // using that are kept; beyond it, the least recently used are released. // A build reads the outputs of thousands of execs and reads most of them // again later (a Go build's link reads every package's archive), so the // bound holds a large build's outputs. An idle mount costs a view and a // bind mount, and collection releases idle mounts first. DefaultMaxIdleSharedMounts = 8192 )
const BuiltinImageLeaseLabel = "dagger.io/builtin-image"
BuiltinImageLeaseLabel marks the persistent leases that hold the engine's builtin images' blobs; the startup sweeps leave them alone.
const BuiltinLayersLeaseID = "dagger/builtin-image/layers"
BuiltinLayersLeaseID is the persistent lease holding the blobs a chain import copied from the builtin image store: builtin blobs, so engine lifetime like the rest of the builtin images' content.
const ContentHashMetadataKey = "buildkit.contenthash.v0"
ContentHashMetadataKey stores the serialized per-path hash tree. Imported hashes must survive a restart: rescanning uses a different file hash format.
Persisted rows record neither this key nor an encoding version, and unopened rows are carried forward on shutdown. Changes to this key or the record format require a dagql cachePersistenceSchemaVersion bump: incompatible records make Checksum fail rather than trigger a filesystem rescan.
const ProgressEmitInterval = 100 * time.Millisecond
ProgressEmitInterval throttles streaming progress records; the final record for an item is always emitted so consumers converge on the true completed state.
Variables ¶
var (
ErrLocked = errors.New("locked")
)
var ErrNoBlobs = errors.Errorf("no blobs for snapshot")
var NoUpdateLastUsed noUpdateLastUsed
Functions ¶
func DisplayRef ¶ added in v0.21.9
DisplayRef shortens a fully qualified image ref for transfer span names ("pulling <ref>", "unpacking <ref>"): these surface as labeled progress rows in the TUI, where the repo digest and default registry are noise.
func EmitProgress ¶ added in v0.21.9
EmitProgress sends one streaming-progress log record for the named item, attributed to the current span via ctx. The unit names what current/total count, e.g. "bytes" or "objects"; see engine/telemetryattrs for the conventions.
func HasLazyLease ¶
func IsNotFound ¶
func MakeTemporary ¶
func NewLayerApplier ¶
NewLayerApplier returns the engine's applier for image layers. It extracts a layer into a snapshot's directory as containerd's file system applier does, entry for entry, except that regular files are written by layerApplyWorkers goroutines while the tar is read. Mounts it doesn't write into directly (several mounts, a read-only bind, a user namespace) go to containerd's applier.
func NewNetworkProgressReader ¶
func NewNetworkProgressReader(ctx context.Context, item string, total int64, r io.ReadCloser, direction enginetelemetry.NetworkDirection) (io.ReadCloser, error)
NewNetworkProgressReader reports both display progress and operation-level network bytes while the response body is consumed.
func NewProgressReader ¶ added in v0.21.9
func NewProgressReader(ctx context.Context, item string, total int64, r io.ReadCloser) io.ReadCloser
NewProgressReader wraps r to stream its read progress as one item via the telemetry convention, attributed to the span carried by ctx. A total <= 0 means the size is unknown (indeterminate). The final state is emitted when the reader sees EOF or is closed.
func ReleaseOperationLeasesAfterRestart ¶
ReleaseOperationLeasesAfterRestart deletes the operation leases an earlier process left behind. Those leases never expire, so they would otherwise retain their snapshots forever. It must run only at startup, before any operation can start.
func ReleaseTransferLeasesAfterRestart ¶
ReleaseTransferLeasesAfterRestart must run only at startup, after durable snapshot owners have been restored and before any transfers can start. Retained typed refs are not released at clean shutdown. Their Go ownership ends with that process, so only durable owners should survive its transfers.
func WithLazyLease ¶
func WithNamespace ¶
WithNamespace is namespaces.WithNamespace, unless ctx already carries ns where namespaces.Namespace finds it, which is what the engine's in-process containerd stores read. The outgoing gRPC and ttrpc headers that namespaces.WithNamespace also attaches matter only to a proxied snapshotter, whose client attaches them itself (see engine/server's newSnapshotter). Checking the outgoing headers here would copy the context's gRPC metadata on every call through the namespace wrappers.
Types ¶
type Accessor ¶
type Accessor interface {
MetadataStore
Get(ctx context.Context, id string, opts ...RefOption) (ImmutableRef, error)
GetBySnapshotID(ctx context.Context, snapshotID string, opts ...RefOption) (ImmutableRef, error)
Scratch(ctx context.Context) (ImmutableRef, error)
New(ctx context.Context, parent ImmutableRef, opts ...RefOption) (MutableRef, error)
GetMutable(ctx context.Context, id string, opts ...RefOption) (MutableRef, error) // Rebase?
GetMutableBySnapshotID(ctx context.Context, snapshotID string, opts ...RefOption) (MutableRef, error)
ImportImage(ctx context.Context, img *ImportedImage, opts ImportImageOpts) (ImmutableRef, error)
ImportChain(ctx context.Context, chain *ExportChain) (ImmutableRef, error)
ApplySnapshotDiff(ctx context.Context, lower, upper ImmutableRef, opts ...RefOption) (ImmutableRef, error)
Merge(ctx context.Context, parents []ImmutableRef, opts ...RefOption) (ImmutableRef, error)
}
type ChainContentError ¶
ChainContentError identifies an unusable supplied chain. Storage/lease and receiver failures are deliberately not annotated with this type.
func (*ChainContentError) Error ¶
func (e *ChainContentError) Error() string
func (*ChainContentError) Unwrap ¶
func (e *ChainContentError) Unwrap() error
type ChainWork ¶
type ChainWork struct {
Begin func(ctx context.Context, i int) context.Context
End func(i int, blobBytes int64, err error)
}
ChainWork marks the span of each chain's own work in an export, for a profile: the builds of the layers the chain is the first to have. Begin is called when chain i's first build starts and returns the context its builds run under; End is called when its last build ends, with the bytes of the blobs they computed and the first error. A chain with no layer of its own begins and ends once the export is planned. The export's planning and its pinning of the built blobs into each chain belong to the export, not to a chain.
type ExportChain ¶
type ExportChain struct {
Layers []ExportLayer
// Prefixes maps each snapshot the export walked, from the root to the
// exported one, to the number of Layers that rebuild it: the layers
// emitted up to and including it. A snapshot that added no layer maps to
// the prefix before it, and a scratch root to none.
Prefixes map[string]int
Provider content.InfoReaderProvider
// Await, when set, is how the blobs of ImportChain's layers arrive, with
// no Provider: each layer the engine hasn't imported yet waits for Await
// to return before taking its blob from the content store, and nothing
// is downloaded ahead. Await returns once the blob is in the content
// store. A layer that waits holds no import lock, and stops waiting, its
// Await's context canceled, once another import has imported it.
Await func(ctx context.Context, desc ocispecs.Descriptor) error
// Timing, when set, receives where ImportChain's time went.
Timing *ImportTiming
// Applied, when set, is told of each layer ImportChain applies itself,
// with the snapshot it committed, while it still holds the layer's
// import lock: before any other import can reuse the snapshot.
Applied func(ctx context.Context, desc ocispecs.Descriptor, snapshotID string)
// Stats are what exporting the chain took.
Stats ExportStats
// contains filtered or unexported fields
}
func ExportChains ¶
func ExportChains(ctx context.Context, refs []ImmutableRef, work *ChainWork, refCfg config.RefConfig) (_ []*ExportChain, rerr error)
ExportChains exports refs' chains as one export, as ExportChain exports each: the layers they lack blobs for build up to refCfg.Builds at once, and a layer the chains share builds once. work, when not nil, marks each chain's own work. It returns the chains in refs' order; on error it returns none, and every chain it opened is released.
type ExportLayer ¶
type ExportLayer struct {
Descriptor ocispecs.Descriptor
Description string
CreatedAt *time.Time
// Computed is set when the export computed the layer's blob: it diffed
// and compressed the layer's snapshot, or converted the snapshot's
// stored blob to the export's compression. It is clear when the export
// used a blob that already existed, such as a pulled image layer's or
// one an earlier export computed.
Computed bool
}
type ExportStats ¶
type ExportStats struct {
// Layers counts the layers emitted.
Layers int
// LayersReused counts the layers whose blob already existed, and
// LayersDiffed those whose blob was computed.
LayersReused int
LayersDiffed int
// RawBytes is the on-disk usage of the diffed layers' snapshots, and
// BlobBytes the size of the blobs computed from them.
RawBytes int64
BlobBytes int64
// DiffTime is computing the diffed layers' blobs: reading the mounts,
// archiving, compressing and writing the content. Layers that build at
// once each add their own, so it can exceed the export's wall time.
DiffTime time.Duration
// LockWait is the wait for the per-snapshot export lock, held by another
// export of the same layer.
LockWait time.Duration
// BuildWait is the wait for a build slot, shared with other exports, to
// build the diffed layers' blobs.
BuildWait time.Duration
// LayersMerged counts the diffed layers whose snapshot carries the
// merge-usage label: a merge, a diff application, or a copy committed with
// its usage (CommitWithUsage). The Merged sums are their share of
// RawBytes, BlobBytes and DiffTime.
LayersMerged int
MergedRawBytes int64
MergedBlobBytes int64
MergedDiffTime time.Duration
}
ExportStats sum what exporting chains took, over their layers.
type ImmutableRef ¶
type ImmutableRef interface {
Ref
ExportChain(ctx context.Context, cfg config.RefConfig) (*ExportChain, error)
// ExportFileChain exports the file at filePath in the ref's filesystem
// as a chain of its own content.
ExportFileChain(ctx context.Context, cfg config.RefConfig, filePath string) (*ExportChain, error)
}
type ImportImageOpts ¶
type ImportImageOpts struct {
ImageRef string
RecordType client.UsageRecordType
}
type ImportTiming ¶
type ImportTiming struct {
// LayerWait is the wait for the per-layer import lock, held by another
// import of the same layer onto the same parent, or of the same layers.
LayerWait time.Duration
// Fetch is getting blobs into the content store: reading them, writing
// and verifying them, and committing them. A reused layer adds nothing.
Fetch time.Duration
// Apply is applying the fetched blobs to their snapshots.
Apply time.Duration
// Ahead counts the blobs whose download started early, while the layer
// before them applied, and Joined the layers whose blob came from a
// download another import had in progress.
Ahead int
Joined int
// Packed counts the layers whose blob the import got with its pack's:
// read, joined, or held from an earlier read.
Packed int
}
ImportTiming sums where a chain import's time went, over its layers. Each layer is fetched whole before it is applied. With fetching ahead, the next layers' blobs download while a layer applies. On a snapshotter that applies a chain's layers at once (overlayfs), the layers' fetches and applies overlap one another, so the sums can exceed the import's time; a chain applied in order spends them one after another, on its own path (Fetch is its wait for the blob, not the download's duration), and they never exceed it.
type ImportedImage ¶
type ImportedImage struct {
Ref string
ManifestDesc ocispecs.Descriptor
ConfigDesc ocispecs.Descriptor
Layers []ocispecs.Descriptor
Nonlayers []ocispecs.Descriptor
}
func (*ImportedImage) Blobs ¶
func (img *ImportedImage) Blobs() []ocispecs.Descriptor
Blobs lists every blob of the image: manifest, config, layers and the non-layer descriptors.
type ImportedLayerBlobKey ¶
type ImportedLayerBlobRow ¶
type ImportedLayerDiffKey ¶
type ImportedLayerDiffRow ¶
type LeaseManager ¶
type LeaseManager struct {
// contains filtered or unexported fields
}
func NewLeaseManager ¶
func NewLeaseManager(lm leases.Manager, ns string) *LeaseManager
func (*LeaseManager) AddResource ¶
func (*LeaseManager) DeleteResource ¶
func (*LeaseManager) ListResources ¶
func (*LeaseManager) Namespace ¶
func (l *LeaseManager) Namespace() string
func (*LeaseManager) WithNamespace ¶
func (l *LeaseManager) WithNamespace(ns string) *LeaseManager
type LocalMounterOpt ¶
type LocalMounterOpt func(*localMounter)
func ForceRemount ¶
func ForceRemount() LocalMounterOpt
type MergeSnapshotter ¶
type MergeSnapshotter interface {
Snapshotter
Merge(ctx context.Context, key string, diffs []Diff, opts ...snapshots.Opt) error
}
func NewMergeSnapshotter ¶
func NewMergeSnapshotter(ctx context.Context, sn Snapshotter, lm leases.Manager) MergeSnapshotter
type MetadataStore ¶
type Mountable ¶
type Mountable interface {
Mount(ctx context.Context, readonly bool) (MountableRef, error)
}
type MountableRef ¶
type MountableRef = executor.MountableRef
type Mounter ¶
func LocalMounter ¶
func LocalMounter(mountable MountableRef, opts ...LocalMounterOpt) Mounter
func LocalMounterWithMounts ¶
func LocalMounterWithMounts(mounts []mount.Mount, opts ...LocalMounterOpt) Mounter
type MutableRef ¶
type PackProvider ¶
type PackProvider interface {
// PackOf returns the ID of the pack that holds blob, if one does.
PackOf(blob digest.Digest) (digest.Digest, bool)
// OpenPack opens the whole pack that holds blob.
OpenPack(ctx context.Context, blob digest.Digest) (io.ReadCloser, error)
}
PackProvider is a content provider some of whose blobs are stored in packs: one object holding several small blobs, read whole.
type PersistentMetadataRows ¶
type PersistentMetadataRows struct {
SnapshotContent []SnapshotContentRow
ImportedByBlob []ImportedLayerBlobRow
ImportedByDiff []ImportedLayerDiffRow
ContentHashes []SnapshotContentHashRow
}
type ProgressTracker ¶ added in v0.21.9
type ProgressTracker struct {
// contains filtered or unexported fields
}
ProgressTracker streams one item's absolute progress with throttled updates. A known total emits an initial zero state immediately so the item appears as soon as work begins; an unknown total (<= 0) stays silent until progress is actually made, so no-op transfers emit nothing.
func NewProgressTracker ¶ added in v0.21.9
func (*ProgressTracker) Finish ¶ added in v0.21.9
func (pt *ProgressTracker) Finish()
Finish emits the final state: the last recorded amount, against the known total if there is one.
func (*ProgressTracker) Update ¶ added in v0.21.9
func (pt *ProgressTracker) Update(current int64)
Update records the item's absolute progress, emitting on a throttle; Finish guarantees the final state.
type ReadOnlyMountReleaser ¶
type ReadOnlyMountReleaser interface {
// using, so that garbage collection can reclaim the snapshots their
// views held. It returns once they are released.
ReleaseIdleSharedMounts() error
// WaitForBackgroundReleases releases the shared read-only mounts nobody
// is using and waits for the releases running in the background, such
// as before the metadata DBs close. Mounts released later are released
// on their callers' paths, and are no longer shared.
WaitForBackgroundReleases()
}
ReadOnlyMountReleaser is a snapshot manager that shares read-only mounts and releases them in the background.
type Ref ¶
type Ref interface {
Mountable
ID() string
SnapshotID() string
Release(context.Context) error
Size(context.Context) (int64, error)
}
Ref is a reference to cacheable objects.
type RefMetadata ¶
type RefMetadata interface {
ID() string
SnapshotID() string
GetDescription() string
SetDescription(string) error
GetCreatedAt() time.Time
SetCreatedAt(time.Time) error
GetLayerType() string
SetLayerType(string) error
GetRecordType() client.UsageRecordType
SetRecordType(client.UsageRecordType) error
// generic getters/setters for external packages
GetString(string) string
Get(string) *Value
SetString(key, val, index string) error
GetExternal(string) ([]byte, error)
SetExternal(string, []byte) error
ClearValueAndIndex(string, string) error
}
type RefOption ¶
type RefOption interface{}
func WithCreationTime ¶
func WithDescription ¶
func WithImageRef ¶
WithImageRef appends the given imageRef to the cache ref's metadata
func WithRecordType ¶
func WithRecordType(t client.UsageRecordType) RefOption
type SharedMounter ¶
type SharedMounter interface {
// and the mount it came from. release ends this reader's use of it; it
// must be called once.
MountShared(ctx context.Context) (root string, m mount.Mount, release func(), err error)
// layer, if a shared mount of the snapshot has been made and that is still
// remembered. A regular file there is the file the snapshot shows, whatever
// the layers below hold, so a reader of it needs no mount. The directory
// may be gone by the time it is read, if the snapshot is; a reader that
// doesn't find its file there has learned nothing and uses MountShared.
SharedMountTopLayer() (dir string, ok bool)
}
SharedMounter is a ref whose read-only mounts are shared across the engine: every reader of its snapshot uses one view of it and one local mount, which are kept while any reader uses them and, once none does, for a linger period. Immutable refs implement it.
type SnapshotContentHashRow ¶
type SnapshotContentRow ¶
type SnapshotManager ¶
type SnapshotManager interface {
Accessor
PinSnapshot(context.Context, string) (ImmutableRef, error)
LeaseExistingSnapshot(context.Context, string) (ImmutableRef, error)
SnapshotSize(ctx context.Context, snapshotID string) (int64, error)
// SnapshotParent returns the snapshot's parent ID, or "" for a base
// snapshot. Owner leases retain the whole parent chain.
SnapshotParent(ctx context.Context, snapshotID string) (string, error)
SnapshotRecordMetadata(ctx context.Context, snapshotID string) (SnapshotRecordMetadata, bool, error)
AttachLease(ctx context.Context, leaseID, snapshotID string) error
RemoveLease(ctx context.Context, leaseID string) error
PinContent(ctx context.Context, leaseID string, descs []ocispecs.Descriptor) error
// LoadPersistentMetadata takes ownership of the content-hash data buffers.
// Callers must not modify them after loading.
LoadPersistentMetadata(rows PersistentMetadataRows) error
PersistentMetadataRows() PersistentMetadataRows
DeleteStaleDaggerOwnerLeases(ctx context.Context, keep map[string]struct{}) error
Close() error
}
func NewSnapshotManager ¶
func NewSnapshotManager(opt SnapshotManagerOpt) (SnapshotManager, error)
type SnapshotManagerOpt ¶
type SnapshotManagerOpt struct {
Snapshotter Snapshotter
ContentStore content.Store
LeaseManager leases.Manager
Applier diff.Applier
// ApplyFrom, when set, returns an applier like Applier that reads blobs
// from provider instead of the content store. A chain import uses it to
// apply a layer from its blob decompressed ahead; nil applies the stored
// blob with Applier.
ApplyFrom func(provider content.Provider) diff.Applier
Differ diff.Comparer
MountPoolRoot string
// BuiltinContent is the engine's builtin image store, consulted by
// chain imports for a layer blob before the chain's provider. Nil means
// no such store.
BuiltinContent content.InfoReaderProvider
// MetadataDB is the containerd metadata database behind LeaseManager.
// When set, AttachLease makes its lease writes in one transaction.
MetadataDB metadata.Transactor
// LocalMounter mounts a snapshot's mounts on a local directory for
// shared read-only mounts; nil means LocalMounterWithMounts. Tests use
// one that needs no privileges.
LocalMounter func([]mount.Mount) Mounter
// SharedMountLinger and MaxIdleSharedMounts bound the shared read-only
// mounts nobody is using that are kept; zero means
// DefaultSharedMountLinger and DefaultMaxIdleSharedMounts.
}
type SnapshotRecordMetadata ¶
type SnapshotRecordMetadata struct {
RecordType client.UsageRecordType
Description string
}
type Snapshotter ¶
type Snapshotter interface {
Name() string
Mounts(ctx context.Context, key string) (MountableRef, error)
Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error
View(ctx context.Context, key, parent string, opts ...snapshots.Opt) (MountableRef, error)
Stat(ctx context.Context, key string) (snapshots.Info, error)
Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error)
Usage(ctx context.Context, key string) (snapshots.Usage, error)
Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error
Remove(ctx context.Context, key string) error
Walk(ctx context.Context, fn snapshots.WalkFunc, filters ...string) error
Close() error
}
Source Files
¶
- apply_ahead.go
- background_release.go
- blobs.go
- blobs_linux.go
- chain_error.go
- content_fetch.go
- diffapply_linux.go
- export_file_linux.go
- import.go
- keyed_lock.go
- layerapply_linux.go
- lease.go
- localmounter.go
- localmounter_linux.go
- manager.go
- merge.go
- metadata.go
- namespace.go
- pack_fetch.go
- persistent_metadata.go
- prefetch.go
- progress.go
- pull.go
- refs.go
- remote.go
- remote_type.go
- scratch.go
- shared_mount.go
- snapshotter.go
Directories
¶
| Path | Synopsis |
|---|---|
|
Package testutil builds real local snapshot and content stores for transfer tests.
|
Package testutil builds real local snapshot and content stores for transfer tests. |