storage

package
v2.6.42 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 29, 2026 License: MIT Imports: 63 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// PipeBufferSize - size of ring buffer between stream handlers
	PipeBufferSize = 128 * 1024
)

Variables

View Source 
var (
	// ErrNotFound is returned when file/object cannot be found
	ErrNotFound = errors.New("key not found")
)

Functions

func AdjustValueByRange added in v2.6.18 

func AdjustValueByRange(value, minValue, maxSize int64) int64

Types

type AzureBlob 

type AzureBlob struct {
	Container azblob.ContainerURL
	Pipeline  pipeline.Pipeline
	CPK       azblob.ClientProvidedKeyOptions
	Config    *config.AzureBlobConfig
}

AzureBlob - presents methods for manipulate data on Azure

func (*AzureBlob) Close 

func (a *AzureBlob) Close(ctx context.Context) error

func (*AzureBlob) Connect 

func (a *AzureBlob) Connect(ctx context.Context) error

Connect - connect to Azure

func (*AzureBlob) CopyObject 

func (a *AzureBlob) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*AzureBlob) DeleteFile 

func (a *AzureBlob) DeleteFile(ctx context.Context, key string) error

func (*AzureBlob) DeleteFileFromObjectDiskBackup 

func (a *AzureBlob) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*AzureBlob) DeleteKeysBatch added in v2.6.42 

func (a *AzureBlob) DeleteKeysBatch(ctx context.Context, keys []string) error

DeleteKeysBatch implements BatchDeleter interface for Azure Blob Uses concurrent deletion since Azure SDK doesn't expose batch delete API

func (*AzureBlob) DeleteKeysFromObjectDiskBackupBatch added in v2.6.42 

func (a *AzureBlob) DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackupBatch implements BatchDeleter interface for Azure Blob

func (*AzureBlob) GetFileReader 

func (a *AzureBlob) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*AzureBlob) GetFileReaderAbsolute added in v2.5.0 

func (a *AzureBlob) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*AzureBlob) GetFileReaderWithLocalPath 

func (a *AzureBlob) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*AzureBlob) Kind 

func (a *AzureBlob) Kind() string

func (*AzureBlob) PutFile 

func (a *AzureBlob) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*AzureBlob) PutFileAbsolute added in v2.5.0 

func (a *AzureBlob) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*AzureBlob) StatFile 

func (a *AzureBlob) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*AzureBlob) StatFileAbsolute added in v2.6.26 

func (a *AzureBlob) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*AzureBlob) Walk 

func (a *AzureBlob) Walk(ctx context.Context, azPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*AzureBlob) WalkAbsolute added in v2.4.27 

func (a *AzureBlob) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type Backup 

type Backup struct {
	metadata.BackupMetadata
	Broken     string
	UploadDate time.Time `json:"upload_date"`
}

func GetBackupsToDeleteRemote added in v2.4.28 

func GetBackupsToDeleteRemote(backups []Backup, keep int) []Backup

type BackupDestination 

type BackupDestination struct {
	RemoteStorage
	// contains filtered or unexported fields
}

func NewBackupDestination 

func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhouse.ClickHouse, backupName string) (*BackupDestination, error)

func (*BackupDestination) BackupList 

func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, parseMetadataOnly string) ([]Backup, error)

func (*BackupDestination) DownloadCompressedStream 

func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remotePath string, localPath string, maxSpeed uint64) (int64, error)

func (*BackupDestination) DownloadPath 

func (bd *BackupDestination) DownloadPath(ctx context.Context, remotePath string, localPath string, RetriesOnFailure int, RetriesDuration time.Duration, RetriesJitter int8, RetrierClassifier retrier.Classifier, maxSpeed uint64) (int64, error)

func (*BackupDestination) RemoveBackupRemote added in v2.4.28 

func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup, cfg *config.Config, retrierClassifier retrier.Classifier) error

func (*BackupDestination) UploadCompressedStream 

func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLocalPath string, files []string, remotePath string, maxSpeed uint64) error

func (*BackupDestination) UploadPath 

func (bd *BackupDestination) UploadPath(ctx context.Context, baseLocalPath string, files []string, remotePath string, RetriesOnFailure int, RetriesDuration time.Duration, RetriesJitter int8, RertierClassifier retrier.Classifier, maxSpeed uint64) (int64, error)

type BatchDeleteError added in v2.6.42 

type BatchDeleteError struct {
	Message  string
	Failures []KeyError
}

BatchDeleteError represents errors that occurred during batch deletion

func (*BatchDeleteError) Error added in v2.6.42 

func (e *BatchDeleteError) Error() string

type BatchDeleter added in v2.6.42 

type BatchDeleter interface {
	// DeleteKeysBatch deletes a batch of keys
	// Batching (collecting keys up to DeleteBatchSize) should be done by the caller
	// Returns nil if all keys were deleted successfully
	// Returns BatchDeleteError if some keys failed to delete
	DeleteKeysBatch(ctx context.Context, keys []string) error

	// DeleteKeysFromObjectDiskBackupBatch deletes a batch of keys from object disk backup path
	DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error
}

BatchDeleter is an optional interface that storage backends can implement to support batch deletion of keys for improved performance

type COS 

type COS struct {
	Config     *config.COSConfig
	BufferSize int
	// contains filtered or unexported fields
}

func (*COS) Close 

func (c *COS) Close(ctx context.Context) error

func (*COS) Connect 

func (c *COS) Connect(ctx context.Context) error

Connect - connect to cos

func (*COS) CopyObject 

func (c *COS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*COS) DeleteFile 

func (c *COS) DeleteFile(ctx context.Context, key string) error

func (*COS) DeleteFileFromObjectDiskBackup 

func (c *COS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*COS) DeleteKeysBatch added in v2.6.42 

func (c *COS) DeleteKeysBatch(ctx context.Context, keys []string) error

DeleteKeysBatch implements BatchDeleter interface for COS Uses concurrent deletion with configurable concurrency

func (*COS) DeleteKeysFromObjectDiskBackupBatch added in v2.6.42 

func (c *COS) DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackupBatch implements BatchDeleter interface for COS

func (*COS) GetFileReader 

func (c *COS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*COS) GetFileReaderAbsolute added in v2.5.0 

func (c *COS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*COS) GetFileReaderWithLocalPath 

func (c *COS) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*COS) Kind 

func (c *COS) Kind() string

func (*COS) PutFile 

func (c *COS) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*COS) PutFileAbsolute added in v2.5.0 

func (c *COS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*COS) StatFile 

func (c *COS) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*COS) StatFileAbsolute added in v2.6.26 

func (c *COS) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*COS) Walk 

func (c *COS) Walk(ctx context.Context, cosPath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*COS) WalkAbsolute added in v2.4.27 

func (c *COS) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

type FTP 

type FTP struct {
	Config *config.FTPConfig
	// contains filtered or unexported fields
}

func (*FTP) Close 

func (f *FTP) Close(ctx context.Context) error

func (*FTP) Connect 

func (f *FTP) Connect(ctx context.Context) error

func (*FTP) CopyObject 

func (f *FTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*FTP) DeleteFile 

func (f *FTP) DeleteFile(ctx context.Context, key string) error

func (*FTP) DeleteFileFromObjectDiskBackup 

func (f *FTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*FTP) DeleteKeys added in v2.6.42 

func (f *FTP) DeleteKeys(ctx context.Context, keys []string) error

DeleteKeys implements BatchDeleter interface for FTP Uses concurrent deletion with connection pool

func (*FTP) DeleteKeysFromObjectDiskBackup added in v2.6.42 

func (f *FTP) DeleteKeysFromObjectDiskBackup(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackup implements BatchDeleter interface for FTP

func (*FTP) GetFileReader 

func (f *FTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*FTP) GetFileReaderAbsolute added in v2.5.0 

func (f *FTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*FTP) GetFileReaderWithLocalPath 

func (f *FTP) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*FTP) Kind 

func (f *FTP) Kind() string

func (*FTP) MkdirAll 

func (f *FTP) MkdirAll(key string, client *ftp.ServerConn) error

func (*FTP) PutFile 

func (f *FTP) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*FTP) PutFileAbsolute added in v2.5.0 

func (f *FTP) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*FTP) StatFile 

func (f *FTP) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*FTP) StatFileAbsolute added in v2.6.26 

func (f *FTP) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*FTP) Walk 

func (f *FTP) Walk(ctx context.Context, ftpPath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*FTP) WalkAbsolute added in v2.4.27 

func (f *FTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

type FTPFileReader 

type FTPFileReader struct {
	*ftp.Response
	// contains filtered or unexported fields
}

func (*FTPFileReader) Close 

func (fr *FTPFileReader) Close() error

type GCS 

type GCS struct {
	Config *config.GCSConfig
	// contains filtered or unexported fields
}

GCS - presents methods for manipulate data on GCS

func (*GCS) Close 

func (gcs *GCS) Close(ctx context.Context) error

func (*GCS) Connect 

func (gcs *GCS) Connect(ctx context.Context) error

Connect - connect to GCS

func (*GCS) CopyObject 

func (gcs *GCS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*GCS) DeleteFile 

func (gcs *GCS) DeleteFile(ctx context.Context, key string) error

func (*GCS) DeleteFileFromObjectDiskBackup 

func (gcs *GCS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*GCS) DeleteKeysBatch added in v2.6.42 

func (gcs *GCS) DeleteKeysBatch(ctx context.Context, keys []string) error

DeleteKeysBatch implements BatchDeleter interface for GCS Uses concurrent deletion with connection pool since GCS doesn't have batch delete API

func (*GCS) DeleteKeysFromObjectDiskBackupBatch added in v2.6.42 

func (gcs *GCS) DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackupBatch implements BatchDeleter interface for GCS

func (*GCS) GetFileReader 

func (gcs *GCS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*GCS) GetFileReaderAbsolute added in v2.5.0 

func (gcs *GCS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*GCS) GetFileReaderWithLocalPath 

func (gcs *GCS) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*GCS) Kind 

func (gcs *GCS) Kind() string

func (*GCS) PutFile 

func (gcs *GCS) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*GCS) PutFileAbsolute added in v2.5.0 

func (gcs *GCS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*GCS) StatFile 

func (gcs *GCS) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*GCS) StatFileAbsolute added in v2.6.26 

func (gcs *GCS) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*GCS) Walk 

func (gcs *GCS) Walk(ctx context.Context, gcsPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*GCS) WalkAbsolute added in v2.4.27 

func (gcs *GCS) WalkAbsolute(ctx context.Context, rootPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type KeyError added in v2.6.42 

type KeyError struct {
	Key string
	Err error
}

KeyError represents an error for a specific key during batch deletion

func (KeyError) Error added in v2.6.42 

func (e KeyError) Error() string

type RecalculateV4Signature 

type RecalculateV4Signature struct {
	// contains filtered or unexported fields
}

RecalculateV4Signature allow GCS over S3, remove Accept-Encoding header from sign https://stackoverflow.com/a/74382598/1204665, https://github.com/aws/aws-sdk-go-v2/issues/1816

func (*RecalculateV4Signature) RoundTrip 

func (lt *RecalculateV4Signature) RoundTrip(req *http.Request) (*http.Response, error)

type RemoteFile 

type RemoteFile interface {
	Size() int64
	Name() string
	LastModified() time.Time
}

RemoteFile - interface describe file on remote storage

type RemoteStorage 

type RemoteStorage interface {
	Kind() string
	Connect(ctx context.Context) error
	Close(ctx context.Context) error
	StatFile(ctx context.Context, key string) (RemoteFile, error)
	StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)
	DeleteFile(ctx context.Context, key string) error
	DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
	Walk(ctx context.Context, prefix string, recursive bool, fn func(context.Context, RemoteFile) error) error
	WalkAbsolute(ctx context.Context, absolutePrefix string, recursive bool, fn func(context.Context, RemoteFile) error) error
	GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
	GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
	GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)
	PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error
	PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error
	CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
}

RemoteStorage -

type S3 

type S3 struct {
	Config      *config.S3Config
	Concurrency int
	BufferSize  int
	// contains filtered or unexported fields
}

S3 - presents methods for manipulate data on s3

func (*S3) Close 

func (s *S3) Close(ctx context.Context) error

func (*S3) Connect 

func (s *S3) Connect(ctx context.Context) error

Connect - connect to s3

func (*S3) CopyObject 

func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*S3) DeleteFile 

func (s *S3) DeleteFile(ctx context.Context, key string) error

func (*S3) DeleteFileFromObjectDiskBackup 

func (s *S3) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*S3) DeleteKeysBatch added in v2.6.42 

func (s *S3) DeleteKeysBatch(ctx context.Context, keys []string) error

DeleteKeysBatch implements BatchDeleter interface for S3 Uses DeleteObjects API to delete up to 1000 keys per request

func (*S3) DeleteKeysFromObjectDiskBackupBatch added in v2.6.42 

func (s *S3) DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackupBatch implements BatchDeleter interface for S3

func (*S3) GetFileReader 

func (s *S3) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetFileReaderAbsolute added in v2.5.0 

func (s *S3) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetFileReaderWithLocalPath 

func (s *S3) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*S3) Kind 

func (s *S3) Kind() string

func (*S3) PutFile 

func (s *S3) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*S3) PutFileAbsolute added in v2.5.0 

func (s *S3) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*S3) ResolveEndpoint added in v2.6.0 

func (s *S3) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (endpoint smithyendpoints.Endpoint, err error)

func (*S3) StatFile 

func (s *S3) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*S3) StatFileAbsolute added in v2.6.26 

func (s *S3) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*S3) Walk 

func (s *S3) Walk(ctx context.Context, s3Path string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*S3) WalkAbsolute added in v2.4.27 

func (s *S3) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type S3LogToZeroLogAdapter added in v2.5.22 

type S3LogToZeroLogAdapter struct {
	// contains filtered or unexported fields
}

func (S3LogToZeroLogAdapter) Logf added in v2.5.22 

func (adapter S3LogToZeroLogAdapter) Logf(severity awsV2Logging.Classification, msg string, args ...interface{})

type SFTP 

type SFTP struct {
	Config *config.SFTPConfig
	// contains filtered or unexported fields
}

SFTP Implement RemoteStorage

func (*SFTP) Close 

func (sftp *SFTP) Close(ctx context.Context) error

func (*SFTP) Connect 

func (sftp *SFTP) Connect(ctx context.Context) error

func (*SFTP) CopyObject 

func (sftp *SFTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

func (*SFTP) Debug 

func (sftp *SFTP) Debug(msg string, v ...interface{})

func (*SFTP) DeleteDirectory 

func (sftp *SFTP) DeleteDirectory(ctx context.Context, dirPath string) error

func (*SFTP) DeleteFile 

func (sftp *SFTP) DeleteFile(ctx context.Context, key string) error

func (*SFTP) DeleteFileFromObjectDiskBackup 

func (sftp *SFTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*SFTP) DeleteKeys added in v2.6.42 

func (sftp *SFTP) DeleteKeys(ctx context.Context, keys []string) error

DeleteKeys implements BatchDeleter interface for SFTP SFTP uses sequential deletion due to protocol limitations (single connection)

func (*SFTP) DeleteKeysFromObjectDiskBackup added in v2.6.42 

func (sftp *SFTP) DeleteKeysFromObjectDiskBackup(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackup implements BatchDeleter interface for SFTP

func (*SFTP) GetFileReader 

func (sftp *SFTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*SFTP) GetFileReaderAbsolute added in v2.5.0 

func (sftp *SFTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*SFTP) GetFileReaderWithLocalPath 

func (sftp *SFTP) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*SFTP) Kind 

func (sftp *SFTP) Kind() string

func (*SFTP) PutFile 

func (sftp *SFTP) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*SFTP) PutFileAbsolute added in v2.5.0 

func (sftp *SFTP) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*SFTP) StatFile 

func (sftp *SFTP) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*SFTP) StatFileAbsolute added in v2.6.26 

func (sftp *SFTP) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*SFTP) Walk 

func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*SFTP) WalkAbsolute added in v2.4.27 

func (sftp *SFTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

Source Files

View all Source files

Directories

Path Synopsis
Package azblob forked from github.com/Azure/azure-storage-blob-go/azblob/blob/feature/clientprovidedkey because UploadStreamToBlockBlob does not expose CPK
 Package azblob forked from github.com/Azure/azure-storage-blob-go/azblob/blob/feature/clientprovidedkey because UploadStreamToBlockBlob does not expose CPK

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.