storage

package
v1.1.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 14, 2026 License: MIT Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source
const (
	DriverFilesystem = "filesystem"
	DriverS3         = "s3"
)
View Source
const MaxIndexedObjects = 50_000

MaxIndexedObjects is the Azure block-list protocol limit. Keeping the limit here lets storage inspection reject corrupt legacy rows before allocating a bitmap derived from an untrusted database value.

Variables

View Source
var (
	ErrIndexedObjectMissing       = errors.New("indexed storage object is missing")
	ErrIndexedObjectLimitExceeded = errors.New("indexed storage object count exceeds supported limit")
)
View Source
var (
	ErrObjectNotFound     = errors.New("object not found in storage")
	ErrComposeUnsupported = errors.New("object sequence cannot be composed by this storage backend")
)

Functions

func MergedObject added in v1.1.1

func MergedObject(folderName string) string

func PartsFolder added in v1.1.1

func PartsFolder(folderName string) string

Types

type Adapter

type Adapter interface {
	UploadStream(ctx context.Context, objectName string, stream io.Reader) error
	CopyObject(ctx context.Context, sourceObjectName, destinationObjectName string) error
	CreateDownloadStream(ctx context.Context, objectName string) (io.ReadCloser, error)
	// CreateRangedDownloadStream returns exactly count bytes beginning at offset.
	// A short underlying object must surface as io.ErrUnexpectedEOF rather than
	// a successful EOF so callers never serve a silently truncated range.
	CreateRangedDownloadStream(ctx context.Context, objectName string, offset, count int64) (io.ReadCloser, error)
	InspectObject(ctx context.Context, objectName string) (ObjectMetadata, error)
	InspectFolderSummary(ctx context.Context, folderName string) (FolderSummary, error)
	InspectIndexedFolder(ctx context.Context, folderName string, expectedObjects int) (int64, error)
	// InspectIndexedFolderSizes validates the same contiguous numeric object set
	// as InspectIndexedFolder and returns sizes ordered by object index.
	InspectIndexedFolderSizes(ctx context.Context, folderName string, expectedObjects int) ([]int64, error)
	WalkTopLevelFolders(ctx context.Context, visit func(folderName string) error) error
	ObjectExists(ctx context.Context, objectName string) (bool, error)
	DeleteFolder(ctx context.Context, folderName string) error
	CountFilesInFolder(ctx context.Context, folderName string) (int, error)
	Clear(ctx context.Context) error
}

func NewAdapter

func NewAdapter(ctx context.Context, cfg config.StorageConfig) (Adapter, error)

type ComposeAdapter added in v1.0.6

type ComposeAdapter interface {
	ComposeObjects(ctx context.Context, destinationObjectName string, sourceObjectNames []string) error
}

type DirectDownloadAdapter

type DirectDownloadAdapter interface {
	CreateDownloadURL(ctx context.Context, objectName string, ttl time.Duration) (string, error)
}

type FilesystemAdapter

type FilesystemAdapter struct {
	// contains filtered or unexported fields
}

func NewFilesystemAdapter

func NewFilesystemAdapter(root string) (*FilesystemAdapter, error)

func (*FilesystemAdapter) CleanupTemporaryUploads added in v1.1.1

func (a *FilesystemAdapter) CleanupTemporaryUploads(ctx context.Context, candidates []ObjectMetadata, cutoff time.Time) (int, error)

func (*FilesystemAdapter) Clear

func (a *FilesystemAdapter) Clear(_ context.Context) error

func (*FilesystemAdapter) CopyObject added in v1.0.6

func (a *FilesystemAdapter) CopyObject(ctx context.Context, sourceObjectName, destinationObjectName string) error

func (*FilesystemAdapter) CountFilesInFolder

func (a *FilesystemAdapter) CountFilesInFolder(ctx context.Context, folderName string) (int, error)

func (*FilesystemAdapter) CreateDownloadStream

func (a *FilesystemAdapter) CreateDownloadStream(_ context.Context, objectName string) (io.ReadCloser, error)

func (*FilesystemAdapter) CreateRangedDownloadStream added in v1.1.2

func (a *FilesystemAdapter) CreateRangedDownloadStream(ctx context.Context, objectName string, offset, count int64) (io.ReadCloser, error)

func (*FilesystemAdapter) DeleteFolder

func (a *FilesystemAdapter) DeleteFolder(_ context.Context, folderName string) error

func (*FilesystemAdapter) FilesystemUsage added in v1.1.1

func (a *FilesystemAdapter) FilesystemUsage(ctx context.Context) (FilesystemUsage, error)

func (*FilesystemAdapter) InspectFolderSummary added in v1.1.2

func (a *FilesystemAdapter) InspectFolderSummary(ctx context.Context, folderName string) (FolderSummary, error)

func (*FilesystemAdapter) InspectIndexedFolder added in v1.1.2

func (a *FilesystemAdapter) InspectIndexedFolder(ctx context.Context, folderName string, expectedObjects int) (int64, error)

func (*FilesystemAdapter) InspectIndexedFolderSizes added in v1.1.2

func (a *FilesystemAdapter) InspectIndexedFolderSizes(ctx context.Context, folderName string, expectedObjects int) ([]int64, error)

func (*FilesystemAdapter) InspectObject added in v1.1.1

func (a *FilesystemAdapter) InspectObject(ctx context.Context, objectName string) (ObjectMetadata, error)

func (*FilesystemAdapter) ObjectExists added in v1.1.1

func (a *FilesystemAdapter) ObjectExists(ctx context.Context, objectName string) (bool, error)

func (*FilesystemAdapter) UploadStream

func (a *FilesystemAdapter) UploadStream(ctx context.Context, objectName string, stream io.Reader) error

func (*FilesystemAdapter) WalkTemporaryUploads added in v1.1.2

func (a *FilesystemAdapter) WalkTemporaryUploads(ctx context.Context, visit func(ObjectMetadata) error) error

func (*FilesystemAdapter) WalkTopLevelFolders added in v1.1.2

func (a *FilesystemAdapter) WalkTopLevelFolders(ctx context.Context, visit func(folderName string) error) error

type FilesystemUsage added in v1.1.1

type FilesystemUsage struct {
	CapacityBytes int64
	UsedBytes     int64
}

type FilesystemUsageAdapter added in v1.1.1

type FilesystemUsageAdapter interface {
	FilesystemUsage(ctx context.Context) (FilesystemUsage, error)
}

type FolderSummary added in v1.1.2

type FolderSummary struct {
	FolderName       string
	Exists           bool
	ObjectCount      int64
	PhysicalBytes    int64
	NewestModifiedAt time.Time
}

type IndexedObjectLimitExceededError added in v1.1.2

type IndexedObjectLimitExceededError struct {
	Count int
	Limit int
}

func (IndexedObjectLimitExceededError) Error added in v1.1.2

func (IndexedObjectLimitExceededError) Unwrap added in v1.1.2

type IndexedObjectMissingError added in v1.1.1

type IndexedObjectMissingError struct {
	Index int
}

func (IndexedObjectMissingError) Error added in v1.1.1

func (IndexedObjectMissingError) Unwrap added in v1.1.1

func (e IndexedObjectMissingError) Unwrap() error

type ObjectMetadata added in v1.1.1

type ObjectMetadata struct {
	Name       string
	SizeBytes  int64
	ModifiedAt time.Time
}

type ObjectNotFoundError

type ObjectNotFoundError struct {
	ObjectName string
}

func (ObjectNotFoundError) Error

func (e ObjectNotFoundError) Error() string

func (ObjectNotFoundError) Unwrap

func (e ObjectNotFoundError) Unwrap() error

type S3Adapter

type S3Adapter struct {
	// contains filtered or unexported fields
}

func NewS3Adapter

func NewS3Adapter(ctx context.Context, cfg config.StorageConfig) (*S3Adapter, error)

func (*S3Adapter) Clear

func (a *S3Adapter) Clear(ctx context.Context) error

func (*S3Adapter) ComposeObjects added in v1.0.6

func (a *S3Adapter) ComposeObjects(ctx context.Context, destinationObjectName string, sourceObjectNames []string) error

func (*S3Adapter) CopyObject added in v1.0.6

func (a *S3Adapter) CopyObject(ctx context.Context, sourceObjectName, destinationObjectName string) error

func (*S3Adapter) CountFilesInFolder

func (a *S3Adapter) CountFilesInFolder(ctx context.Context, folderName string) (int, error)

func (*S3Adapter) CreateDownloadStream

func (a *S3Adapter) CreateDownloadStream(ctx context.Context, objectName string) (io.ReadCloser, error)

func (*S3Adapter) CreateDownloadURL

func (a *S3Adapter) CreateDownloadURL(ctx context.Context, objectName string, ttl time.Duration) (string, error)

func (*S3Adapter) CreateRangedDownloadStream added in v1.1.2

func (a *S3Adapter) CreateRangedDownloadStream(ctx context.Context, objectName string, offset, count int64) (io.ReadCloser, error)

func (*S3Adapter) DeleteFolder

func (a *S3Adapter) DeleteFolder(ctx context.Context, folderName string) error

func (*S3Adapter) InspectFolderSummary added in v1.1.2

func (a *S3Adapter) InspectFolderSummary(ctx context.Context, folderName string) (FolderSummary, error)

func (*S3Adapter) InspectIndexedFolder added in v1.1.2

func (a *S3Adapter) InspectIndexedFolder(ctx context.Context, folderName string, expectedObjects int) (int64, error)

func (*S3Adapter) InspectIndexedFolderSizes added in v1.1.2

func (a *S3Adapter) InspectIndexedFolderSizes(ctx context.Context, folderName string, expectedObjects int) ([]int64, error)

func (*S3Adapter) InspectObject added in v1.1.1

func (a *S3Adapter) InspectObject(ctx context.Context, objectName string) (ObjectMetadata, error)

func (*S3Adapter) ObjectExists added in v1.1.1

func (a *S3Adapter) ObjectExists(ctx context.Context, objectName string) (bool, error)

func (*S3Adapter) UploadStream

func (a *S3Adapter) UploadStream(ctx context.Context, objectName string, stream io.Reader) error

func (*S3Adapter) WalkTopLevelFolders added in v1.1.2

func (a *S3Adapter) WalkTopLevelFolders(ctx context.Context, visit func(folderName string) error) error

type TemporaryUploadCleaner added in v1.1.1

type TemporaryUploadCleaner interface {
	WalkTemporaryUploads(ctx context.Context, visit func(ObjectMetadata) error) error
	CleanupTemporaryUploads(ctx context.Context, candidates []ObjectMetadata, cutoff time.Time) (int, error)
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL