Documentation
¶
Overview ¶
Package blob stores opaque byte objects under opaque keys.
It is the seam for a value too large to hold in memory or to move through a transaction. The package internal/storage owns the other kind: a small record that changes, with transactions, batches, and compare-and-set. A multi-megabyte value would move through every one of those operations, so file bytes get their own contract.
The contract is deliberately narrow. A store knows nothing about accounts, files, purposes, or expiry. It reads and writes bytes at a key. The owner of the key holds every meaning the bytes carry.
Index ¶
- Constants
- Variables
- func RestoreObjectStore(ctx context.Context, target *ObjectStore, source, scratch, operation string, ...) (resultErr error)
- func ValidateKey(key string) error
- func VerifyPublished(ctx context.Context, source PublicationReader, key string, size int64, ...) (resultErr error)
- type ArchiveProjection
- func (p *ArchiveProjection) ApplyPublication(ctx context.Context, key string, expected, next PublicationState, ...) error
- func (p *ArchiveProjection) Close() error
- func (ArchiveProjection) Format(state fmt.State, _ rune)
- func (p *ArchiveProjection) ReadPublished(ctx context.Context, key string) (io.ReadCloser, error)
- func (p *ArchiveProjection) Snapshot(ctx context.Context, destination string) (Snapshot, error)
- func (p *ArchiveProjection) StatPublished(ctx context.Context, key string) (Info, error)
- type Filesystem
- func (f *Filesystem) Backend() string
- func (f *Filesystem) Delete(ctx context.Context, key string) error
- func (f *Filesystem) EnsurePublicationReady(ctx context.Context) error
- func (f *Filesystem) Get(ctx context.Context, key string) (io.ReadCloser, error)
- func (f *Filesystem) InspectPublication(ctx context.Context, key string) (result PublicationState, resultErr error)
- func (f *Filesystem) Publish(ctx context.Context, key string, r io.Reader) (Info, error)
- func (f *Filesystem) Put(ctx context.Context, key string, r io.Reader) (Info, error)
- func (f *Filesystem) ReadPublished(ctx context.Context, key string) (io.ReadCloser, error)
- func (f *Filesystem) Retire(ctx context.Context, key string) error
- func (f *Filesystem) Stat(ctx context.Context, key string) (Info, error)
- func (f *Filesystem) StatPublished(ctx context.Context, key string) (Info, error)
- type ImportActivationInspector
- type ImportActivator
- type ImportInspector
- type ImportPublicationReplayer
- type ImportPublicationStep
- type ImportReplayActivator
- type ImportReplayInspector
- type ImportReplayPosition
- type ImportUnreleasedInspector
- type Info
- type ObjectStore
- func (o *ObjectStore) Backend() string
- func (o *ObjectStore) Delete(ctx context.Context, key string) error
- func (o *ObjectStore) EnsurePublicationReady(ctx context.Context) error
- func (o *ObjectStore) Get(ctx context.Context, key string) (io.ReadCloser, error)
- func (o *ObjectStore) InspectPublication(ctx context.Context, key string) (result PublicationState, resultErr error)
- func (o *ObjectStore) Publish(ctx context.Context, key string, r io.Reader) (Info, error)
- func (o *ObjectStore) Put(ctx context.Context, key string, r io.Reader) (Info, error)
- func (o *ObjectStore) ReadPublished(ctx context.Context, key string) (io.ReadCloser, error)
- func (o *ObjectStore) Retire(ctx context.Context, key string) error
- func (o *ObjectStore) Stat(ctx context.Context, key string) (Info, error)
- func (o *ObjectStore) StatPublished(ctx context.Context, key string) (Info, error)
- type ObjectStoreOptions
- type PopulatedControl
- type PopulatedControls
- type PopulatedImportClaimer
- type PublicationReader
- type PublicationState
- type PublicationStore
- type RecoveryPublicationReader
- type RecoveryTargetBinder
- type RestoreResult
- type RestoreTarget
- type Snapshot
- type SnapshotView
- type Store
Constants ¶
const ImportPublicationMaxBytes int64 = 1 << 30
ImportPublicationMaxBytes bounds one independently retained recovery asset.
const MaxKeyLength = 256
MaxKeyLength bounds a key. The bound keeps a key inside the length every backing medium accepts, including a filesystem path component and an object store key.
Variables ¶
var ( // ErrInvalidKey reports a key the contract refuses. A store returns it // before it touches its backing medium, so a rejected key never leaves a // partial object behind. ErrInvalidKey = errors.New("blob: invalid key") // ErrNotFound reports that no object exists at the key. ErrNotFound = errors.New("blob: object not found") )
Errors the contract defines. A backend returns these rather than an error of its own, so a caller can branch on the reason without naming a backend.
var ( // ErrPublicationExists prevents changes to a retained identity. ErrPublicationExists = errors.New("blob: publication identity already exists") // ErrCorruptPublication reports an invalid retained envelope. ErrCorruptPublication = errors.New("blob: corrupt publication envelope") )
var ErrActivationConflict = errors.New("blob: activation conflicts with retained ownership")
ErrActivationConflict preserves an import or a different activation decision.
var ErrConditionalPublicationUnsupported = errors.New("blob: storage did not enforce conditional publication")
ErrConditionalPublicationUnsupported reports a failed conditional-write probe.
var ErrImportRestricted = errors.New("blob: imported bytes require deployment recovery")
ErrImportRestricted retains the deployment recovery barrier on imported bytes.
var ErrImportTargetPopulated = errors.New("blob: import target contains existing objects")
ErrImportTargetPopulated refuses an import into an object store that already holds objects.
var ErrLayoutMigrationRequired = errors.New("blob: existing objects require explicit layout migration")
ErrLayoutMigrationRequired refuses an existing bucket without the current layout receipt.
ErrLayoutUnavailable reports an unverified blob storage format.
var ErrPopulatedClaimUnsupported = errors.New("blob: populated claim requires object storage")
ErrPopulatedClaimUnsupported refuses a populated claim outside object storage.
var ErrPopulatedContentsChanged = errors.New("blob: retained objects differ from the captured snapshot")
ErrPopulatedContentsChanged refuses retained objects that differ from the captured snapshot.
ErrPublicationUnavailable refuses work before it can create provider charges.
Functions ¶
func RestoreObjectStore ¶ added in v1.3.0
func RestoreObjectStore(ctx context.Context, target *ObjectStore, source, scratch, operation string, expected Snapshot) (resultErr error)
RestoreObjectStore imports into an empty prefix or resumes the same operation. It preserves existing objects and leaves its recovery barrier after every outcome. The coordinator must fence all target writers before this operation.
func ValidateKey ¶
ValidateKey reports whether the key is one this package stores under.
A key is opaque: it carries no structure a store reads. The rules exist so that one key means the same object on every backend. A path separator is the case that matters most, because a filesystem would read it as a directory step and an object store would read it as a prefix, and the same key would then name two different places.
func VerifyPublished ¶ added in v1.3.0
func VerifyPublished(ctx context.Context, source PublicationReader, key string, size int64, digest string) (resultErr error)
VerifyPublished checks retained payload length and an optional content digest. It streams bytes and never changes publication or retirement state.
Types ¶
type ArchiveProjection ¶ added in v1.3.0
type ArchiveProjection struct {
// contains filtered or unexported fields
}
ArchiveProjection changes only a private copy of an original verified archive. It exposes no configured store, native recovery claim, or activation permission.
func OpenArchiveProjection ¶ added in v1.3.0
func OpenArchiveProjection(ctx context.Context, source, scratch string, expected Snapshot) (*ArchiveProjection, error)
OpenArchiveProjection verifies the original envelope and copies its complete retained state.
func (*ArchiveProjection) ApplyPublication ¶ added in v1.3.0
func (p *ArchiveProjection) ApplyPublication(ctx context.Context, key string, expected, next PublicationState, input io.Reader) error
ApplyPublication validates exact historical preimages before changing private bytes. It writes no import claim, native cursor, or activation receipt.
func (*ArchiveProjection) Close ¶ added in v1.3.0
func (p *ArchiveProjection) Close() error
Close removes only the owned private archive copy.
func (ArchiveProjection) Format ¶ added in v1.3.0
func (ArchiveProjection) Format(state fmt.State, _ rune)
Format omits private paths and retained bytes.
func (*ArchiveProjection) ReadPublished ¶ added in v1.3.0
func (p *ArchiveProjection) ReadPublished(ctx context.Context, key string) (io.ReadCloser, error)
ReadPublished reads the private projected state for domain reference checks.
func (*ArchiveProjection) Snapshot ¶ added in v1.3.0
Snapshot writes a complete portable image of private projected bytes and retirements.
func (*ArchiveProjection) StatPublished ¶ added in v1.3.0
StatPublished reads private projected metadata without publishing bytes.
type Filesystem ¶
type Filesystem struct {
// contains filtered or unexported fields
}
Filesystem stores objects as files under a configured root directory.
It suits one node. A deployment that runs more than one Starport process against the same files needs the object store backend instead, which FIL2 implements behind this same contract.
func NewFilesystem ¶
func NewFilesystem(root string) (*Filesystem, error)
NewFilesystem opens a filesystem store rooted at the directory. It creates the root and its two subdirectories when they are absent, and it refuses a root it cannot write.
func (*Filesystem) Delete ¶
func (f *Filesystem) Delete(ctx context.Context, key string) error
Delete implements Store.
func (*Filesystem) EnsurePublicationReady ¶ added in v1.3.0
func (f *Filesystem) EnsurePublicationReady(ctx context.Context) error
EnsurePublicationReady uses the filesystem's qualified native operations. Each later write still reports its own permission and storage failures.
func (*Filesystem) Get ¶
func (f *Filesystem) Get(ctx context.Context, key string) (io.ReadCloser, error)
Get implements Store.
func (*Filesystem) InspectPublication ¶ added in v1.3.0
func (f *Filesystem) InspectPublication(ctx context.Context, key string) (result PublicationState, resultErr error)
InspectPublication reads complete retained bytes and checks their digest.
func (*Filesystem) Publish ¶ added in v1.3.0
Publish stages durable bytes, then links them without replacing an identity. A retirement marker occupies that same identity and rejects a delayed link.
func (*Filesystem) ReadPublished ¶ added in v1.3.0
func (f *Filesystem) ReadPublished(ctx context.Context, key string) (io.ReadCloser, error)
ReadPublished consumes the private envelope before exposing payload bytes.
func (*Filesystem) Retire ¶ added in v1.3.0
func (f *Filesystem) Retire(ctx context.Context, key string) error
Retire replaces either live bytes or an absent identity with a durable marker.
func (*Filesystem) StatPublished ¶ added in v1.3.0
StatPublished uses one open handle so retirement cannot mix two versions.
type ImportActivationInspector ¶ added in v1.3.0
type ImportActivationInspector interface {
CheckActivatedImportAt(context.Context, string, Snapshot, ImportReplayPosition, string) error
}
ImportActivationInspector verifies an exact native release without repair or publication. Its result establishes historical component completion, not current admission permission.
type ImportActivator ¶ added in v1.3.0
ImportActivator releases an exact import after the recovery coordinator checks fencing, history, and cross-store readiness. Activation alone grants no admission. The operation must be the complete bundle-bound operation used during restore.
type ImportInspector ¶ added in v1.3.0
type ImportInspector interface {
SnapshotImport(context.Context, string, string, Snapshot) (Snapshot, error)
CheckImport(context.Context, string, Snapshot) error
}
ImportInspector captures imported bytes while their startup barrier remains closed. Callers must fence writers and discard partial output after any error.
type ImportPublicationReplayer ¶ added in v1.3.0
type ImportPublicationReplayer interface {
ReplayPublication(context.Context, string, Snapshot, ImportPublicationStep, io.Reader, string) (string, error)
}
ImportPublicationReplayer changes retained bytes only under an exact import barrier. Every writer must remain fenced. A failed call can leave durable pending work.
type ImportPublicationStep ¶ added in v1.3.0
type ImportPublicationStep struct {
Sequence int64 `json:"sequence"`
PreviousSHA256 string `json:"previous_sha256"`
EvidenceSHA256 string `json:"evidence_sha256"`
Key string `json:"key"`
Expected PublicationState `json:"expected"`
Next PublicationState `json:"next"`
}
ImportPublicationStep binds one retained publication or retirement to independent evidence.
type ImportReplayActivator ¶ added in v1.3.0
type ImportReplayActivator interface {
ActivateImportAt(context.Context, string, Snapshot, ImportReplayPosition, string) error
}
ImportReplayActivator releases only the exact inspected completed blob chain.
type ImportReplayInspector ¶ added in v1.3.0
type ImportReplayInspector interface {
CheckImportPosition(context.Context, string, Snapshot, ImportReplayPosition) error
SnapshotImportAt(context.Context, string, string, Snapshot, ImportReplayPosition) (Snapshot, error)
}
ImportReplayInspector binds captures to the exact completed blob replay cursor.
type ImportReplayPosition ¶ added in v1.3.0
type ImportReplayPosition struct {
Sequence int64 `json:"sequence"`
ReceiptSHA256 string `json:"receipt_sha256"`
}
ImportReplayPosition identifies the exact completed local blob replay chain.
type ImportUnreleasedInspector ¶ added in v1.3.0
type ImportUnreleasedInspector interface {
CheckUnreleasedImport(context.Context, string, Snapshot) error
}
ImportUnreleasedInspector refuses any activation attempt before application preparation. It allows an existing replay cursor while requiring the original closed import claim.
type Info ¶
type Info struct {
// Key is the key the object is stored under.
Key string
// Size is the stored length in bytes.
Size int64
}
Info describes one stored object.
type ObjectStore ¶
type ObjectStore struct {
// contains filtered or unexported fields
}
ObjectStore stores objects in an S3-compatible bucket. It serves every node of a deployment, which the filesystem backend cannot.
func NewObjectStore ¶
func NewObjectStore(ctx context.Context, options ObjectStoreOptions) (*ObjectStore, error)
NewObjectStore opens a store against the bucket the options name.
It builds the client and returns. It does not reach the bucket, because a network call at construction would make startup depend on a remote service that the first upload will reach anyway.
func (*ObjectStore) Delete ¶
func (o *ObjectStore) Delete(ctx context.Context, key string) error
Delete implements Store.
func (*ObjectStore) EnsurePublicationReady ¶ added in v1.3.0
func (o *ObjectStore) EnsurePublicationReady(ctx context.Context) error
EnsurePublicationReady verifies this client's conditional-write support. Construction does not contact the bucket. Warm checks read process memory.
func (*ObjectStore) Get ¶
func (o *ObjectStore) Get(ctx context.Context, key string) (io.ReadCloser, error)
Get implements Store.
func (*ObjectStore) InspectPublication ¶ added in v1.3.0
func (o *ObjectStore) InspectPublication(ctx context.Context, key string) (result PublicationState, resultErr error)
InspectPublication reads complete retained bytes without creating layout records.
func (*ObjectStore) Publish ¶ added in v1.3.0
Publish requires conditional creation for both single-part and multipart completion. The configured S3 service must enforce If-None-Match: *.
func (*ObjectStore) Put ¶
Put implements Store.
The uploader sends the whole object in one request, or in parts when the stream is large. Either way the object becomes reachable only after the last part lands. A lost acknowledgment or a failed size lookup can return an error after publication. The owner must verify the retained object.
func (*ObjectStore) ReadPublished ¶ added in v1.3.0
func (o *ObjectStore) ReadPublished(ctx context.Context, key string) (io.ReadCloser, error)
ReadPublished checks the same envelope as the filesystem backend.
func (*ObjectStore) Retire ¶ added in v1.3.0
func (o *ObjectStore) Retire(ctx context.Context, key string) error
Retire writes a real object, never a delete marker. S3 permits conditional creation over a delete marker, which would let an old upload reappear.
func (*ObjectStore) StatPublished ¶ added in v1.3.0
StatPublished checks publication metadata from the atomic object write.
type ObjectStoreOptions ¶
type ObjectStoreOptions struct {
Bucket string
Region string
Endpoint string
Prefix string
// AccessKeyID and SecretAccessKey state static credentials. Both empty
// selects the ambient AWS credential chain.
AccessKeyID string
SecretAccessKey string
}
ObjectStoreOptions addresses one S3-compatible bucket.
One client reaches AWS S3, Cloudflare R2, MinIO, and Backblaze B2, because Endpoint selects the implementation; an absent Endpoint selects AWS. Immutable publication additionally requires verified conditional creation on PutObject and CompleteMultipartUpload. API compatibility alone does not qualify that concurrency contract. Retained objects require lifecycle rules that preserve current retirement markers.
type PopulatedControl ¶ added in v1.3.0
type PopulatedControl struct {
Present bool `json:"present"`
Value []byte `json:"value,omitempty"`
ETag string `json:"etag,omitempty"`
}
PopulatedControl records the exact bytes and ETag of one control object. Present is false when the object does not exist.
type PopulatedControls ¶ added in v1.3.0
type PopulatedControls struct {
Import PopulatedControl `json:"import"`
ActivationCurrent PopulatedControl `json:"activation_current"`
ReplayCurrent PopulatedControl `json:"replay_current"`
Closure PopulatedControl `json:"closure"`
}
PopulatedControls is the control preimage that the caller keeps from preparation. A captured snapshot excludes these objects.
type PopulatedImportClaimer ¶ added in v1.3.0
type PopulatedImportClaimer interface {
ObservePopulatedControls(context.Context) (PopulatedControls, error)
ClaimPopulated(context.Context, string, string, string, Snapshot, PopulatedControls) error
}
PopulatedImportClaimer puts an import claim on retained bytes in place. The coordinator must fence all target writers before it observes the controls and must keep them fenced through activation. A failed claim can leave durable restricted work. A retry with the same inputs continues it.
type PublicationReader ¶ added in v1.3.0
type PublicationReader interface {
ReadPublished(context.Context, string) (io.ReadCloser, error)
StatPublished(context.Context, string) (Info, error)
}
PublicationReader reads retained bytes without exposing publication or retirement.
type PublicationState ¶ added in v1.3.0
type PublicationState struct {
Kind string `json:"kind"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
}
PublicationState distinguishes an unused identity from permanent retirement. Digest and size refer to payload bytes, excluding the retained envelope.
type PublicationStore ¶ added in v1.3.0
type PublicationStore interface {
// EnsurePublicationReady checks capability before new work needs durable bytes.
EnsurePublicationReady(context.Context) error
// Publish creates a complete object only under an unused identity.
// Publish never replaces existing live bytes or retirement markers.
Publish(context.Context, string, io.Reader) (Info, error)
// ReadPublished opens live bytes. Missing and retired identities return ErrNotFound.
ReadPublished(context.Context, string) (io.ReadCloser, error)
// StatPublished reports the payload length, excluding the private envelope.
StatPublished(context.Context, string) (Info, error)
// Retire durably prevents every later publication, including an earlier
// writer that resumes. Only success permits the owner to release quota.
Retire(context.Context, string) error
}
PublicationStore owns immutable bytes and permanent retirement at each key. Its namespace is separate from mutable Put/Get objects. An error can mean that a write succeeded but its acknowledgment was lost.
Retirement markers must survive restart and backup restoration. A caller must not remove them while an earlier writer can resume. Payload quota does not include these markers. Operators must budget for their retained storage.
type RecoveryPublicationReader ¶ added in v1.3.0
type RecoveryPublicationReader interface {
InspectPublication(context.Context, string) (PublicationState, error)
}
RecoveryPublicationReader exposes exact retained identity state without writes.
type RecoveryTargetBinder ¶ added in v1.3.0
RecoveryTargetBinder identifies the selected scope without releasing import barriers. The digest supplements native claims and does not prove remote server continuity.
type RestoreResult ¶ added in v1.3.0
type RestoreResult struct{ Published bool }
RestoreResult reports a published directory separately from final durability. Published also identifies a verified directory retained from an exact retry. The import barrier remains until the complete deployment recovery procedure.
func RestoreFilesystem ¶ added in v1.3.0
func RestoreFilesystem(ctx context.Context, destination, source, operation string, expected Snapshot) (result RestoreResult, resultErr error)
RestoreFilesystem verifies a complete image before publishing a new directory. Existing paths remain intact. Imported bytes retain a startup barrier.
func RestoreFilesystemOnce ¶ added in v1.3.0
func RestoreFilesystemOnce(ctx context.Context, destination, source, operation string, expected Snapshot) (RestoreResult, error)
RestoreFilesystemOnce verifies an exact prior import before accepting its retry.
type RestoreTarget ¶ added in v1.3.0
RestoreTarget retains restricted bytes for an explicit deployment restore. A successful call grants no permission to start the gateway.
func FilesystemRestoreTarget ¶ added in v1.3.0
func FilesystemRestoreTarget(destination string) (RestoreTarget, error)
FilesystemRestoreTarget selects a new directory or an exact prior import.
func ObjectRestoreTarget ¶ added in v1.3.0
func ObjectRestoreTarget(store *ObjectStore) (RestoreTarget, error)
ObjectRestoreTarget selects an empty namespace or an exact prior import.
type Snapshot ¶ added in v1.3.0
type Snapshot struct {
Format string `json:"format"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
Objects int64 `json:"objects"`
Retired int64 `json:"retired"`
}
Snapshot identifies committed bytes, retirement markers, and activation history. Objects counts archive entries, including historical activation receipts. It excludes incomplete uploads, native activation authority, and noncurrent versions.
func Backup ¶ added in v1.3.0
Backup snapshots the selected blob namespace. The caller must fence all writers. The destination must be a new file beneath an existing private directory. An error can leave a partial file, which has no valid returned receipt.
func BackupLegacyObjects ¶ added in v1.3.0
func BackupLegacyObjects(ctx context.Context, source *ObjectStore, destination string) (Snapshot, error)
BackupLegacyObjects reads the former logical-key layout without changing it. Migration restores the resulting image into a different, empty prefix or directory.
type SnapshotView ¶ added in v1.3.0
type SnapshotView interface {
PublicationReader
Get(context.Context, string) (io.ReadCloser, error)
Stat(context.Context, string) (Info, error)
Close() error
}
SnapshotView owns a verified private archive copy. It exposes no writes.
func OpenSnapshot ¶ added in v1.3.0
func OpenSnapshot(ctx context.Context, source, scratch string, expected Snapshot) (SnapshotView, error)
OpenSnapshot verifies and extracts a private copy for offline reference checks. The caller must close it to remove the temporary copy.
type Store ¶
type Store interface {
PublicationStore
// Put stores the bytes the reader yields at the key. It reads until the
// reader reports io.EOF.
//
// A backend stages bytes and publishes a complete object. A lost reply
// can leave the new object present despite an error. The caller must
// inspect retained identity and content before deciding how to recover.
// A rejected input stream does not publish a partial object.
//
// Put returns ErrInvalidKey before it writes anything when the key breaks
// the key rules.
Put(ctx context.Context, key string, r io.Reader) (Info, error)
// Get opens the stored object for reading. The caller closes the reader.
// Get returns ErrNotFound when no object exists at the key.
Get(ctx context.Context, key string) (io.ReadCloser, error)
// Stat reports the object without reading its bytes. Stat returns
// ErrNotFound when no object exists at the key.
Stat(ctx context.Context, key string) (Info, error)
// Delete removes the object. Delete of an absent key returns nil, so a
// repeated delete is safe.
Delete(ctx context.Context, key string) error
// Backend names the implementation, such as "filesystem". An operator
// reads it once at startup to confirm where the bytes land.
Backend() string
}
Store reads and writes opaque bytes at an opaque key.
Every method takes a context and honors its cancellation. A backend that reaches a network respects the deadline the caller sets.
Source Files
¶
- activation.go
- activation_filesystem.go
- activation_inspection.go
- activation_objectstore.go
- activation_probe.go
- address.go
- archive_projection.go
- blob.go
- filesystem.go
- filesystem_publication.go
- import_inspection.go
- import_replay.go
- import_replay_native.go
- key.go
- objectstore.go
- objectstore_layout.go
- objectstore_publication.go
- objectstore_readiness.go
- populated_claim.go
- publication.go
- readiness.go
- recovery_publication.go
- recovery_target.go
- restore_retry.go
- restore_target.go
- snapshot.go
- snapshot_image.go
- snapshot_restore.go
- snapshot_view.go
- verification.go