Help us shape the future of Go! Take the 2026 Go Developer Survey

storage

package
v1.102.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 3, 2026 License: BSD-3-Clause Imports: 18 Imported by: 0

Documentation

Overview

Default-build stub for the GCS storage backend. cloud.google.com/go/storage pulls google.golang.org/grpc + s2a-go + cloud.google.com/go/iam + the full Google Cloud SDK transitive chain. grpc is opt-in only across the Lux/Hanzo stack, so GCS follows the same discipline: enable with `-tags gcs`. Without the tag, NewGCSStorage returns errGCSDisabled and the dispatch in storage.go returns the operator a clear error pointing them at s3:// (hanzoai/s3, MinIO-protocol).

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ComputeChecksum 

func ComputeChecksum(path string) (string, error)

ComputeChecksum calculates SHA256 checksum of a file.

func VerifyChecksum 

func VerifyChecksum(path, expected string) (bool, error)

VerifyChecksum verifies a file against expected checksum.

Types

type AzureStorage 

type AzureStorage struct {
	// contains filtered or unexported fields
}

AzureStorage implements Storage for Azure Blob Storage.

func NewAzureStorage 

func NewAzureStorage(ctx context.Context, cfg *Config) (*AzureStorage, error)

NewAzureStorage creates a new Azure Blob storage backend.

func (*AzureStorage) Bucket 

func (a *AzureStorage) Bucket() string

Bucket returns the container name.

func (*AzureStorage) Close 

func (a *AzureStorage) Close() error

Close releases any resources.

func (*AzureStorage) Copy 

func (a *AzureStorage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies an object within Azure.

func (*AzureStorage) Delete 

func (a *AzureStorage) Delete(ctx context.Context, key string) error

Delete removes an object from Azure.

func (*AzureStorage) DeleteMany 

func (a *AzureStorage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple objects from Azure.

func (*AzureStorage) Download 

func (a *AzureStorage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data from Azure to a writer.

func (*AzureStorage) DownloadFile 

func (a *AzureStorage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads from Azure to a local file.

func (*AzureStorage) Exists 

func (a *AzureStorage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if an object exists in Azure.

func (*AzureStorage) GetInfo 

func (a *AzureStorage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves object metadata from Azure.

func (*AzureStorage) GetSignedURL 

func (a *AzureStorage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL generates a SAS URL for Azure.

func (*AzureStorage) List 

func (a *AzureStorage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists objects in Azure.

func (*AzureStorage) Provider 

func (a *AzureStorage) Provider() Provider

Provider returns the storage provider type.

func (*AzureStorage) Upload 

func (a *AzureStorage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to Azure.

func (*AzureStorage) UploadFile 

func (a *AzureStorage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file to Azure.

type Config 

type Config struct {
	Provider Provider
	Bucket   string
	Region   string
	Endpoint string // Custom endpoint for S3-compatible stores (MinIO, R2, etc.)

	// AWS-specific
	AWSProfile       string
	AWSAccessKey     string
	AWSSecretKey     string
	AWSSessionToken  string
	AWSAssumeRoleARN string

	// GCS-specific
	GCSCredentialsFile string
	GCSProjectID       string

	// Azure-specific
	AzureAccountName   string
	AzureAccountKey    string
	AzureConnectionStr string

	// SFTP-specific
	SFTPHost       string
	SFTPPort       int
	SFTPUser       string
	SFTPPassword   string
	SFTPPrivateKey string

	// Local-specific
	LocalBasePath string

	// Common options
	PathStyle  bool // Use path-style URLs (for MinIO, etc.)
	DisableSSL bool
	MaxRetries int
	Timeout    time.Duration
}

Config holds configuration for storage backends.

func ParseURI 

func ParseURI(uri string) (*Config, string, error)

ParseURI parses a storage URI and returns config. Supported formats:

  • s3://bucket/path
  • gs://bucket/path
  • azure://container/path
  • file:///local/path
  • sftp://user@host:port/path

type DownloadOptions 

type DownloadOptions struct {
	// Range for partial downloads (e.g., "bytes=0-1023")
	Range string
	// VersionID for versioned objects
	VersionID string
	// ProgressFunc reports download progress
	ProgressFunc func(bytesDownloaded, totalBytes int64)
}

DownloadOptions configures download behavior.

type GCSStorage 

type GCSStorage struct{}

GCSStorage is the disabled-build type. NewGCSStorage always returns errGCSDisabled so the methods are never invoked; they only need to exist so the Storage interface is satisfied at type-check time.

func NewGCSStorage 

func NewGCSStorage(_ context.Context, _ *Config) (*GCSStorage, error)

func (*GCSStorage) Bucket 

func (*GCSStorage) Bucket() string

func (*GCSStorage) Close 

func (*GCSStorage) Close() error

func (*GCSStorage) Copy 

func (*GCSStorage) Copy(_ context.Context, _, _ string) error

func (*GCSStorage) Delete 

func (*GCSStorage) Delete(_ context.Context, _ string) error

func (*GCSStorage) DeleteMany 

func (*GCSStorage) DeleteMany(_ context.Context, _ []string) error

func (*GCSStorage) Download 

func (*GCSStorage) Download(_ context.Context, _ string, _ io.Writer, _ *DownloadOptions) error

func (*GCSStorage) DownloadFile 

func (*GCSStorage) DownloadFile(_ context.Context, _ string, _ string, _ *DownloadOptions) error

func (*GCSStorage) Exists 

func (*GCSStorage) Exists(_ context.Context, _ string) (bool, error)

func (*GCSStorage) GetInfo 

func (*GCSStorage) GetInfo(_ context.Context, _ string) (*ObjectInfo, error)

func (*GCSStorage) GetSignedURL 

func (*GCSStorage) GetSignedURL(_ context.Context, _ string, _ time.Duration, _ bool) (string, error)

func (*GCSStorage) List 

func (*GCSStorage) List(_ context.Context, _ *ListOptions) (*ListResult, error)

func (*GCSStorage) Provider 

func (*GCSStorage) Provider() Provider

func (*GCSStorage) Upload 

func (*GCSStorage) Upload(_ context.Context, _ string, _ io.Reader, _ int64, _ *UploadOptions) error

func (*GCSStorage) UploadFile 

func (*GCSStorage) UploadFile(_ context.Context, _ string, _ string, _ *UploadOptions) error

type ListOptions 

type ListOptions struct {
	Prefix     string
	Delimiter  string
	MaxKeys    int
	StartAfter string
}

ListOptions for listing objects.

type ListResult 

type ListResult struct {
	Objects        []ObjectInfo
	CommonPrefixes []string
	IsTruncated    bool
	NextMarker     string
}

ListResult contains listing results.

type LocalStorage 

type LocalStorage struct {
	// contains filtered or unexported fields
}

LocalStorage implements Storage for local filesystem.

func NewLocalStorage 

func NewLocalStorage(cfg *Config) (*LocalStorage, error)

NewLocalStorage creates a new local filesystem storage backend.

func (*LocalStorage) Bucket 

func (l *LocalStorage) Bucket() string

Bucket returns the base path.

func (*LocalStorage) Close 

func (l *LocalStorage) Close() error

Close releases any resources.

func (*LocalStorage) Copy 

func (l *LocalStorage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies a file.

func (*LocalStorage) Delete 

func (l *LocalStorage) Delete(ctx context.Context, key string) error

Delete removes a file.

func (*LocalStorage) DeleteMany 

func (l *LocalStorage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple files.

func (*LocalStorage) Download 

func (l *LocalStorage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data to a writer.

func (*LocalStorage) DownloadFile 

func (l *LocalStorage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads to a local file (copy).

func (*LocalStorage) Exists 

func (l *LocalStorage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if a file exists.

func (*LocalStorage) GetInfo 

func (l *LocalStorage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves file metadata.

func (*LocalStorage) GetSignedURL 

func (l *LocalStorage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL is not supported for local storage.

func (*LocalStorage) List 

func (l *LocalStorage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists files with the given options.

func (*LocalStorage) Provider 

func (l *LocalStorage) Provider() Provider

Provider returns the storage provider type.

func (*LocalStorage) Upload 

func (l *LocalStorage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to local filesystem.

func (*LocalStorage) UploadFile 

func (l *LocalStorage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file (copy).

type ObjectInfo 

type ObjectInfo struct {
	Key          string
	Size         int64
	LastModified time.Time
	ETag         string
	ContentType  string
	Metadata     map[string]string
	VersionID    string
	StorageClass string
}

ObjectInfo contains metadata about a stored object.

type Provider 

type Provider string

Provider represents a cloud storage provider type. 

const (
	ProviderS3    Provider = "s3"
	ProviderGCS   Provider = "gcs"
	ProviderAzure Provider = "azure"
	ProviderLocal Provider = "local"
	ProviderSFTP  Provider = "sftp"
)

type S3Storage 

type S3Storage struct {
	// contains filtered or unexported fields
}

S3Storage implements Storage for AWS S3 and S3-compatible stores.

func NewS3Storage 

func NewS3Storage(ctx context.Context, cfg *Config) (*S3Storage, error)

NewS3Storage creates a new S3 storage backend.

func (*S3Storage) Bucket 

func (s *S3Storage) Bucket() string

Bucket returns the bucket name.

func (*S3Storage) Close 

func (s *S3Storage) Close() error

Close releases any resources.

func (*S3Storage) Copy 

func (s *S3Storage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies an object within S3.

func (*S3Storage) Delete 

func (s *S3Storage) Delete(ctx context.Context, key string) error

Delete removes an object from S3.

func (*S3Storage) DeleteMany 

func (s *S3Storage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple objects from S3.

func (*S3Storage) Download 

func (s *S3Storage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data from S3 to a writer.

func (*S3Storage) DownloadFile 

func (s *S3Storage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads from S3 to a local file.

func (*S3Storage) Exists 

func (s *S3Storage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if an object exists in S3.

func (*S3Storage) GetInfo 

func (s *S3Storage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves object metadata from S3.

func (*S3Storage) GetSignedURL 

func (s *S3Storage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL generates a pre-signed URL for S3.

func (*S3Storage) List 

func (s *S3Storage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists objects in S3.

func (*S3Storage) Provider 

func (s *S3Storage) Provider() Provider

Provider returns the storage provider type.

func (*S3Storage) Upload 

func (s *S3Storage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to S3.

func (*S3Storage) UploadFile 

func (s *S3Storage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file to S3.

type SFTPStorage 

type SFTPStorage struct {
	// contains filtered or unexported fields
}

SFTPStorage implements Storage for SFTP servers.

func NewSFTPStorage 

func NewSFTPStorage(cfg *Config) (*SFTPStorage, error)

NewSFTPStorage creates a new SFTP storage backend.

func (*SFTPStorage) Bucket 

func (s *SFTPStorage) Bucket() string

Bucket returns the base path.

func (*SFTPStorage) Close 

func (s *SFTPStorage) Close() error

Close releases any resources.

func (*SFTPStorage) Copy 

func (s *SFTPStorage) Copy(ctx context.Context, srcKey, dstKey string) error

Copy copies a file on SFTP.

func (*SFTPStorage) Delete 

func (s *SFTPStorage) Delete(ctx context.Context, key string) error

Delete removes a file from SFTP.

func (*SFTPStorage) DeleteMany 

func (s *SFTPStorage) DeleteMany(ctx context.Context, keys []string) error

DeleteMany removes multiple files from SFTP.

func (*SFTPStorage) Download 

func (s *SFTPStorage) Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

Download downloads data from SFTP to a writer.

func (*SFTPStorage) DownloadFile 

func (s *SFTPStorage) DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

DownloadFile downloads from SFTP to a local file.

func (*SFTPStorage) Exists 

func (s *SFTPStorage) Exists(ctx context.Context, key string) (bool, error)

Exists checks if a file exists on SFTP.

func (*SFTPStorage) GetInfo 

func (s *SFTPStorage) GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

GetInfo retrieves file metadata from SFTP.

func (*SFTPStorage) GetSignedURL 

func (s *SFTPStorage) GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

GetSignedURL is not supported for SFTP.

func (*SFTPStorage) List 

func (s *SFTPStorage) List(ctx context.Context, opts *ListOptions) (*ListResult, error)

List lists files on SFTP.

func (*SFTPStorage) Provider 

func (s *SFTPStorage) Provider() Provider

Provider returns the storage provider type.

func (*SFTPStorage) Upload 

func (s *SFTPStorage) Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

Upload uploads data from a reader to SFTP.

func (*SFTPStorage) UploadFile 

func (s *SFTPStorage) UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

UploadFile uploads a local file to SFTP.

type Storage 

type Storage interface {
	// Upload uploads data from a reader to the storage.
	Upload(ctx context.Context, key string, reader io.Reader, size int64, opts *UploadOptions) error

	// UploadFile uploads a local file to storage.
	UploadFile(ctx context.Context, key string, localPath string, opts *UploadOptions) error

	// Download downloads data from storage to a writer.
	Download(ctx context.Context, key string, writer io.Writer, opts *DownloadOptions) error

	// DownloadFile downloads from storage to a local file.
	DownloadFile(ctx context.Context, key string, localPath string, opts *DownloadOptions) error

	// Delete removes an object from storage.
	Delete(ctx context.Context, key string) error

	// DeleteMany removes multiple objects.
	DeleteMany(ctx context.Context, keys []string) error

	// Exists checks if an object exists.
	Exists(ctx context.Context, key string) (bool, error)

	// GetInfo retrieves object metadata.
	GetInfo(ctx context.Context, key string) (*ObjectInfo, error)

	// List lists objects with the given options.
	List(ctx context.Context, opts *ListOptions) (*ListResult, error)

	// GetSignedURL generates a pre-signed URL for temporary access.
	GetSignedURL(ctx context.Context, key string, expiry time.Duration, forUpload bool) (string, error)

	// Copy copies an object within the storage.
	Copy(ctx context.Context, srcKey, dstKey string) error

	// Provider returns the storage provider type.
	Provider() Provider

	// Bucket returns the bucket/container name.
	Bucket() string

	// Close releases any resources.
	Close() error
}

Storage defines the interface for cloud storage operations.

func New 

func New(ctx context.Context, cfg *Config) (Storage, error)

New creates a new Storage instance based on the config.

type UploadOptions 

type UploadOptions struct {
	// ContentType for the uploaded object
	ContentType string
	// Metadata key-value pairs
	Metadata map[string]string
	// ServerSideEncryption type (e.g., "AES256", "aws:kms")
	ServerSideEncryption string
	// KMSKeyID for KMS encryption
	KMSKeyID string
	// StorageClass (e.g., "STANDARD", "GLACIER", "NEARLINE")
	StorageClass string
	// ACL (e.g., "private", "public-read")
	ACL string
	// PartSize for multipart uploads (default 64MB)
	PartSize int64
	// Concurrency for parallel part uploads
	Concurrency int
	// ProgressFunc reports upload progress
	ProgressFunc func(bytesUploaded, totalBytes int64)
}

UploadOptions configures upload behavior.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.