storage

package
v1.23.4 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 27, 2026 License: BSD-3-Clause Imports: 19 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ComputeChecksum

func ComputeChecksum(path string) (string, error)

ComputeChecksum calculates SHA256 checksum of a file.

func VerifyChecksum

func VerifyChecksum(path, expected string) (bool, error)

VerifyChecksum verifies a file against expected checksum.

Types

type AzureStorage

type AzureStorage struct {
	// contains filtered or unexported fields
}

AzureStorage implements Storage for Azure Blob Storage.

func NewAzureStorage

func NewAzureStorage(ctx context.Context, cfg *Config) (*AzureStorage, error)

NewAzureStorage creates a new Azure Blob storage backend.

func (*AzureStorage) Bucket

func (a *AzureStorage) Bucket() string

Bucket returns the container name.

func (*AzureStorage) Close

func (a *AzureStorage) Close() error

Close releases any resources.

func (*AzureStorage) Copy

func (a *AzureStorage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies an object within Azure.

func (*AzureStorage) Delete

func (a *AzureStorage) Delete(ctx context.Context, key string) error

Delete removes an object from Azure.

func (*AzureStorage) DeleteMany

func (a *AzureStorage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple objects from Azure.

func (*AzureStorage) Download

func (a *AzureStorage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data from Azure to a writer.

func (*AzureStorage) DownloadFile

func (a *AzureStorage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads from Azure to a local file.

func (*AzureStorage) Exists

func (a *AzureStorage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if an object exists in Azure.

func (*AzureStorage) GetInfo

func (a *AzureStorage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves object metadata from Azure.

func (*AzureStorage) GetSignedURL

func (a *AzureStorage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL generates a SAS URL for Azure.

func (*AzureStorage) List

func (a *AzureStorage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists objects in Azure.

func (*AzureStorage) Provider

func (a *AzureStorage) Provider() Provider

Provider returns the storage provider type.

func (*AzureStorage) Upload

func (a *AzureStorage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to Azure.

func (*AzureStorage) UploadFile

func (a *AzureStorage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file to Azure.

type Config

type Config struct {
	Provider Provider
	Bucket   string
	Region   string
	Endpoint string // Custom endpoint for S3-compatible stores (MinIO, R2, etc.)

	// AWS-specific
	AWSProfile       string
	AWSAccessKey     string
	AWSSecretKey     string
	AWSSessionToken  string
	AWSAssumeRoleARN string

	// GCS-specific
	GCSCredentialsFile string
	GCSProjectID       string

	// Azure-specific
	AzureAccountName   string
	AzureAccountKey    string
	AzureConnectionStr string

	// SFTP-specific
	SFTPHost       string
	SFTPPort       int
	SFTPUser       string
	SFTPPassword   string
	SFTPPrivateKey string

	// Local-specific
	LocalBasePath string

	// Common options
	PathStyle  bool // Use path-style URLs (for MinIO, etc.)
	DisableSSL bool
	MaxRetries int
	Timeout    time.Duration
}

Config holds configuration for storage backends.

func ParseURI

func ParseURI(uri string) (*Config, string, error)

ParseURI parses a storage URI and returns config. Supported formats:

  • s3://bucket/path
  • gs://bucket/path
  • azure://container/path
  • file:///local/path
  • sftp://user@host:port/path

type DownloadOptions

type DownloadOptions struct {
	// Range for partial downloads (e.g., "bytes=0-1023")
	Range string
	// VersionID for versioned objects
	VersionID string
	// ProgressFunc reports download progress
	ProgressFunc func(bytesDownloaded, totalBytes int64)
}

DownloadOptions configures download behavior.

type GCSStorage

type GCSStorage struct {
	// contains filtered or unexported fields
}

GCSStorage implements Storage for Google Cloud Storage.

func NewGCSStorage

func NewGCSStorage(ctx context.Context, cfg *Config) (*GCSStorage, error)

NewGCSStorage creates a new GCS storage backend.

func (*GCSStorage) Bucket

func (g *GCSStorage) Bucket() string

Bucket returns the bucket name.

func (*GCSStorage) Close

func (g *GCSStorage) Close() error

Close releases any resources.

func (*GCSStorage) Copy

func (g *GCSStorage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies an object within GCS.

func (*GCSStorage) Delete

func (g *GCSStorage) Delete(ctx context.Context, key string) error

Delete removes an object from GCS.

func (*GCSStorage) DeleteMany

func (g *GCSStorage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple objects from GCS.

func (*GCSStorage) Download

func (g *GCSStorage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data from GCS to a writer.

func (*GCSStorage) DownloadFile

func (g *GCSStorage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads from GCS to a local file.

func (*GCSStorage) Exists

func (g *GCSStorage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if an object exists in GCS.

func (*GCSStorage) GetInfo

func (g *GCSStorage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves object metadata from GCS.

func (*GCSStorage) GetSignedURL

func (g *GCSStorage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL generates a pre-signed URL for GCS.

func (*GCSStorage) List

func (g *GCSStorage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists objects in GCS.

func (*GCSStorage) Provider

func (g *GCSStorage) Provider() Provider

Provider returns the storage provider type.

func (*GCSStorage) Upload

func (g *GCSStorage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to GCS.

func (*GCSStorage) UploadFile

func (g *GCSStorage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file to GCS.

type ListOptions

type ListOptions struct {
	Prefix     string
	Delimiter  string
	MaxKeys    int
	StartAfter string
}

ListOptions for listing objects.

type ListResult

type ListResult struct {
	Objects        []ObjectInfo
	CommonPrefixes []string
	IsTruncated    bool
	NextMarker     string
}

ListResult contains listing results.

type LocalStorage

type LocalStorage struct {
	// contains filtered or unexported fields
}

LocalStorage implements Storage for local filesystem.

func NewLocalStorage

func NewLocalStorage(cfg *Config) (*LocalStorage, error)

NewLocalStorage creates a new local filesystem storage backend.

func (*LocalStorage) Bucket

func (l *LocalStorage) Bucket() string

Bucket returns the base path.

func (*LocalStorage) Close

func (l *LocalStorage) Close() error

Close releases any resources.

func (*LocalStorage) Copy

func (l *LocalStorage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies a file.

func (*LocalStorage) Delete

func (l *LocalStorage) Delete(ctx context.Context, key string) error

Delete removes a file.

func (*LocalStorage) DeleteMany

func (l *LocalStorage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple files.

func (*LocalStorage) Download

func (l *LocalStorage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data to a writer.

func (*LocalStorage) DownloadFile

func (l *LocalStorage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads to a local file (copy).

func (*LocalStorage) Exists

func (l *LocalStorage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if a file exists.

func (*LocalStorage) GetInfo

func (l *LocalStorage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves file metadata.

func (*LocalStorage) GetSignedURL

func (l *LocalStorage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL is not supported for local storage.

func (*LocalStorage) List

func (l *LocalStorage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists files with the given options.

func (*LocalStorage) Provider

func (l *LocalStorage) Provider() Provider

Provider returns the storage provider type.

func (*LocalStorage) Upload

func (l *LocalStorage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to local filesystem.

func (*LocalStorage) UploadFile

func (l *LocalStorage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file (copy).

type ObjectInfo

type ObjectInfo struct {
	Key          string
	Size         int64
	LastModified time.Time
	ETag         string
	ContentType  string
	Metadata     map[string]string
	VersionID    string
	StorageClass string
}

ObjectInfo contains metadata about a stored object.

type Provider

type Provider string

Provider represents a cloud storage provider type.

const (
	ProviderS3    Provider = "s3"
	ProviderGCS   Provider = "gcs"
	ProviderAzure Provider = "azure"
	ProviderLocal Provider = "local"
	ProviderSFTP  Provider = "sftp"
)

type S3Storage

type S3Storage struct {
	// contains filtered or unexported fields
}

S3Storage implements Storage for AWS S3 and S3-compatible stores.

func NewS3Storage

func NewS3Storage(ctx context.Context, cfg *Config) (*S3Storage, error)

NewS3Storage creates a new S3 storage backend.

func (*S3Storage) Bucket

func (s *S3Storage) Bucket() string

Bucket returns the bucket name.

func (*S3Storage) Close

func (s *S3Storage) Close() error

Close releases any resources.

func (*S3Storage) Copy

func (s *S3Storage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies an object within S3.

func (*S3Storage) Delete

func (s *S3Storage) Delete(ctx context.Context, key string) error

Delete removes an object from S3.

func (*S3Storage) DeleteMany

func (s *S3Storage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple objects from S3.

func (*S3Storage) Download

func (s *S3Storage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data from S3 to a writer.

func (*S3Storage) DownloadFile

func (s *S3Storage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads from S3 to a local file.

func (*S3Storage) Exists

func (s *S3Storage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if an object exists in S3.

func (*S3Storage) GetInfo

func (s *S3Storage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves object metadata from S3.

func (*S3Storage) GetSignedURL

func (s *S3Storage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL generates a pre-signed URL for S3.

func (*S3Storage) List

func (s *S3Storage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists objects in S3.

func (*S3Storage) Provider

func (s *S3Storage) Provider() Provider

Provider returns the storage provider type.

func (*S3Storage) Upload

func (s *S3Storage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to S3.

func (*S3Storage) UploadFile

func (s *S3Storage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file to S3.

type SFTPStorage

type SFTPStorage struct {
	// contains filtered or unexported fields
}

SFTPStorage implements Storage for SFTP servers.

func NewSFTPStorage

func NewSFTPStorage(cfg *Config) (*SFTPStorage, error)

NewSFTPStorage creates a new SFTP storage backend.

func (*SFTPStorage) Bucket

func (s *SFTPStorage) Bucket() string

Bucket returns the base path.

func (*SFTPStorage) Close

func (s *SFTPStorage) Close() error

Close releases any resources.

func (*SFTPStorage) Copy

func (s *SFTPStorage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies a file on SFTP.

func (*SFTPStorage) Delete

func (s *SFTPStorage) Delete(ctx context.Context, key string) error

Delete removes a file from SFTP.

func (*SFTPStorage) DeleteMany

func (s *SFTPStorage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple files from SFTP.

func (*SFTPStorage) Download

func (s *SFTPStorage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data from SFTP to a writer.

func (*SFTPStorage) DownloadFile

func (s *SFTPStorage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads from SFTP to a local file.

func (*SFTPStorage) Exists

func (s *SFTPStorage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if a file exists on SFTP.

func (*SFTPStorage) GetInfo

func (s *SFTPStorage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves file metadata from SFTP.

func (*SFTPStorage) GetSignedURL

func (s *SFTPStorage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL is not supported for SFTP.

func (*SFTPStorage) List

func (s *SFTPStorage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists files on SFTP.

func (*SFTPStorage) Provider

func (s *SFTPStorage) Provider() Provider

Provider returns the storage provider type.

func (*SFTPStorage) Upload

func (s *SFTPStorage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to SFTP.

func (*SFTPStorage) UploadFile

func (s *SFTPStorage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file to SFTP.

type Storage

type Storage interface {
	// Upload uploads data from a reader to the storage.
	Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

	// UploadFile uploads a local file to storage.
	UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

	// Download downloads data from storage to a writer.
	Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

	// DownloadFile downloads from storage to a local file.
	DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

	// Delete removes an object from storage.
	Delete(ctx context.Context, key string) error

	// DeleteMany removes multiple objects.
	DeleteMany(ctx context.Context, keys []string) error

	// Exists checks if an object exists.
	Exists(ctx context.Context, key string) (bool, error)

	// GetInfo retrieves object metadata.
	GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

	// List lists objects with the given options.
	List(ctx context.Context, opts *ListOptions) (*ListResult, error)

	// GetSignedURL generates a pre-signed URL for temporary access.
	GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

	// Copy copies an object within the storage.
	Copy(ctx context.Context, srcKey, dstKey string) error

	// Provider returns the storage provider type.
	Provider() Provider

	// Bucket returns the bucket/container name.
	Bucket() string

	// Close releases any resources.
	Close() error
}

Storage defines the interface for cloud storage operations.

func New

func New(ctx context.Context, cfg *Config) (Storage, error)

New creates a new Storage instance based on the config.

type UploadOptions

type UploadOptions struct {
	// ContentType for the uploaded object
	ContentType string
	// Metadata key-value pairs
	Metadata map[string]string
	// ServerSideEncryption type (e.g., "AES256", "aws:kms")
	ServerSideEncryption string
	// KMSKeyID for KMS encryption
	KMSKeyID string
	// StorageClass (e.g., "STANDARD", "GLACIER", "NEARLINE")
	StorageClass string
	// ACL (e.g., "private", "public-read")
	ACL string
	// PartSize for multipart uploads (default 64MB)
	PartSize int64
	// Concurrency for parallel part uploads
	Concurrency int
	// ProgressFunc reports upload progress
	ProgressFunc func(bytesUploaded, totalBytes int64)
}

UploadOptions configures upload behavior.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL