storage

package
v2.8.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 16, 2026 License: MIT Imports: 83 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// ManifestFileName is the name of the file manifest stored alongside metadata.json.
	// It is a gzip-compressed bbolt database whose keys are the relative paths of every
	// file in the backup, allowing restore operations to skip the expensive S3
	// ListObjects Walk. bbolt keeps keys sorted, so per-part lookups are prefix seeks
	// instead of full scans, and the read side is mmap-backed instead of loading the
	// whole listing into the heap.
	ManifestFileName = "manifest.bolt.gz"
	// ManifestVersion is the current manifest format version.
	ManifestVersion = 1
)

Variables

View Source 
var (
	// ErrNotFound is returned when file/object cannot be found
	ErrNotFound = errors.New("key not found")
)

Functions

func AdjustValueByRange added in v2.6.18 

func AdjustValueByRange(value, minValue, maxSize int64) int64

func NewErrNotFound added in v2.7.0 

func NewErrNotFound(key string) error

NewErrNotFound wraps ErrNotFound with key; errors.Is(err, ErrNotFound) still matches.

Types

type AzureBlob 

type AzureBlob struct {
	Container *container.Client
	CPK       *blob.CPKInfo
	Config    *config.AzureBlobConfig
}

AzureBlob - presents methods for manipulate data on Azure

func (*AzureBlob) Close 

func (a *AzureBlob) Close(_ context.Context) error

func (*AzureBlob) Connect 

func (a *AzureBlob) Connect(ctx context.Context) error

Connect - connect to Azure

func (*AzureBlob) CopyObject 

func (a *AzureBlob) CopyObject(ctx context.Context, _ int64, srcBucket, srcKey, dstKey string) (int64, error)

CopyObject server-side copy from srcBucket/srcKey to a.Config.Container/dstKey, both keys are absolute inside the container

func (*AzureBlob) DeleteFile 

func (a *AzureBlob) DeleteFile(ctx context.Context, key string) error

func (*AzureBlob) DeleteFileFromObjectDiskBackup 

func (a *AzureBlob) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*AzureBlob) DeleteKeysBatch added in v2.6.42 

func (a *AzureBlob) DeleteKeysBatch(ctx context.Context, keys []string) error

DeleteKeysBatch implements BatchDeleter interface for Azure Blob Uses concurrent deletion since Azure SDK doesn't expose batch delete API

func (*AzureBlob) DeleteKeysFromObjectDiskBackupBatch added in v2.6.42 

func (a *AzureBlob) DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackupBatch implements BatchDeleter interface for Azure Blob

func (*AzureBlob) GetFileReader 

func (a *AzureBlob) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*AzureBlob) GetFileReaderAbsolute added in v2.5.0 

func (a *AzureBlob) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*AzureBlob) GetFileReaderWithLocalPath 

func (a *AzureBlob) GetFileReaderWithLocalPath(ctx context.Context, key, _ string, _ int64) (io.ReadCloser, error)

func (*AzureBlob) Kind 

func (a *AzureBlob) Kind() string

func (*AzureBlob) PutFile 

func (a *AzureBlob) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*AzureBlob) PutFileAbsolute added in v2.5.0 

func (a *AzureBlob) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*AzureBlob) StatFile 

func (a *AzureBlob) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*AzureBlob) StatFileAbsolute added in v2.6.26 

func (a *AzureBlob) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*AzureBlob) Walk 

func (a *AzureBlob) Walk(ctx context.Context, azPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*AzureBlob) WalkAbsolute added in v2.4.27 

func (a *AzureBlob) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type Backup 

type Backup struct {
	metadata.BackupMetadata
	Broken     string
	UploadDate time.Time `json:"upload_date"`
}

func GetBackupsToDeleteRemote added in v2.4.28 

func GetBackupsToDeleteRemote(backups []Backup, keep int) []Backup

func GetOldestLiveBackupToRebase added in v2.8.0 

func GetOldestLiveBackupToRebase(backups []Backup, keep int) *Backup

GetOldestLiveBackupToRebase - return the oldest kept backup whose `required_backup` points to a backup outside the `keep` window, rebase of such backup makes it full, so the whole out-of-window chain loses `required_backup` references from kept backups and becomes deletable by GetBackupsToDeleteRemote

type BackupDestination 

type BackupDestination struct {
	RemoteStorage
	// contains filtered or unexported fields
}

func NewBackupDestination 

func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhouse.ClickHouse, backupName string) (*BackupDestination, error)

func (*BackupDestination) BackupList 

func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, parseMetadataOnly string) ([]Backup, error)

func (*BackupDestination) DownloadCompressedStream 

func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remotePath string, localPath string, maxSpeed uint64) (int64, error)

func (*BackupDestination) DownloadLimiter added in v2.7.1 

func (bd *BackupDestination) DownloadLimiter(maxBytesPerSecond uint64) *bwlimit.Limiter

DownloadLimiter mirrors UploadLimiter for the download direction.

func (*BackupDestination) DownloadManifest added in v2.8.0 

func (bd *BackupDestination) DownloadManifest(ctx context.Context, backupName string) *ManifestReader

DownloadManifest attempts to download, decompress and open the manifest for a given backup. Returns nil if the manifest does not exist or is unreadable (graceful fallback to Walk).

func (*BackupDestination) DownloadPath 

func (bd *BackupDestination) DownloadPath(ctx context.Context, remotePath string, localPath string, RetriesOnFailure int, RetriesDuration time.Duration, RetriesJitter int8, RetrierClassifier retrier.Classifier, maxSpeed uint64) (int64, error)

func (*BackupDestination) DownloadPathWithManifest added in v2.8.0 

func (bd *BackupDestination) DownloadPathWithManifest(ctx context.Context, remotePath string, localPath string, fileNames []string, RetriesOnFailure int, RetriesDuration time.Duration, RetriesJitter int8, RetrierClassifier retrier.Classifier, maxSpeed uint64) (int64, error)

DownloadPathWithManifest downloads the given files from remotePath using a pre-loaded manifest listing instead of calling Walk. fileNames are relative to remotePath, as returned by ManifestReader.FilesUnderPrefix.

func (*BackupDestination) RemoveBackupRemote added in v2.4.28 

func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup, cfg *config.Config, retrierClassifier retrier.Classifier) error

func (*BackupDestination) UpdateMetadataCacheEntry added in v2.8.0 

func (bd *BackupDestination) UpdateMetadataCacheEntry(ctx context.Context, backup Backup) error

UpdateMetadataCacheEntry overwrites one entry in the on-disk metadata cache, need after metadata.json rewrite on remote storage (e.g. `rebase` command), otherwise BackupList fast path returns the stale cached entry

func (*BackupDestination) UploadCompressedStream 

func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLocalPath string, files []string, remotePath string, maxSpeed uint64) error

func (*BackupDestination) UploadLimiter added in v2.7.1 

func (bd *BackupDestination) UploadLimiter(maxBytesPerSecond uint64) *bwlimit.Limiter

UploadLimiter returns the shared upload rate limiter for this destination, creating (or recreating, on a rate change) it lazily. It returns nil when throttling is disabled (maxBytesPerSecond == 0). The limiter is shared across all concurrent upload workers so the configured rate is an aggregate cap.

func (*BackupDestination) UploadManifest added in v2.8.0 

func (bd *BackupDestination) UploadManifest(ctx context.Context, backupName string, w *ManifestWriter) error

UploadManifest finalizes the manifest and uploads it to remote storage alongside metadata.json.

func (*BackupDestination) UploadPath 

func (bd *BackupDestination) UploadPath(ctx context.Context, baseLocalPath string, files []string, remotePath string, RetriesOnFailure int, RetriesDuration time.Duration, RetriesJitter int8, RertierClassifier retrier.Classifier, maxSpeed uint64) (int64, error)

func (*BackupDestination) WalkCompressedStream added in v2.8.0 

func (bd *BackupDestination) WalkCompressedStream(ctx context.Context, remotePath string, handler func(ctx context.Context, fileName string, content io.Reader) error) error

WalkCompressedStream streams a remote archive and calls handler for every file inside, without writing anything to the local filesystem, need for `rebase` object disk blobs copy

type BatchDeleteError added in v2.6.42 

type BatchDeleteError struct {
	Message  string
	Failures []KeyError
}

BatchDeleteError represents errors that occurred during batch deletion

func (*BatchDeleteError) Error added in v2.6.42 

func (e *BatchDeleteError) Error() string

type BatchDeleter added in v2.6.42 

type BatchDeleter interface {
	// DeleteKeysBatch deletes a batch of keys
	// Batching (collecting keys up to DeleteBatchSize) should be done by the caller
	// Returns nil if all keys were deleted successfully
	// Returns BatchDeleteError if some keys failed to delete
	DeleteKeysBatch(ctx context.Context, keys []string) error

	// DeleteKeysFromObjectDiskBackupBatch deletes a batch of keys from object disk backup path
	DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error
}

BatchDeleter is an optional interface that storage backends can implement to support batch deletion of keys for improved performance

type COS 

type COS struct {
	Config     *config.COSConfig
	BufferSize int
	// contains filtered or unexported fields
}

func (*COS) Close 

func (c *COS) Close(_ context.Context) error

func (*COS) Connect 

func (c *COS) Connect(ctx context.Context) error

Connect - connect to cos

func (*COS) CopyObject 

func (c *COS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

CopyObject server-side copy inside COS, empty srcBucket means the same bucket, non-empty srcBucket - another COS bucket in the same region (e.g. ClickHouse object disk over COS), MultiCopy internally switches to multipart copy for objects above the single-copy limit, when the source is not really on COS (e.g. object disk over MinIO) MultiCopy fails and callers fall back to streaming

func (*COS) DeleteFile 

func (c *COS) DeleteFile(ctx context.Context, key string) error

func (*COS) DeleteFileFromObjectDiskBackup 

func (c *COS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*COS) DeleteKeysBatch added in v2.6.42 

func (c *COS) DeleteKeysBatch(ctx context.Context, keys []string) error

DeleteKeysBatch implements BatchDeleter interface for COS Uses concurrent deletion with configurable concurrency

func (*COS) DeleteKeysFromObjectDiskBackupBatch added in v2.6.42 

func (c *COS) DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackupBatch implements BatchDeleter interface for COS

func (*COS) GetFileReader 

func (c *COS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*COS) GetFileReaderAbsolute added in v2.5.0 

func (c *COS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*COS) GetFileReaderWithLocalPath 

func (c *COS) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*COS) Kind 

func (c *COS) Kind() string

func (*COS) PutFile 

func (c *COS) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*COS) PutFileAbsolute added in v2.5.0 

func (c *COS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*COS) StatFile 

func (c *COS) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*COS) StatFileAbsolute added in v2.6.26 

func (c *COS) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*COS) Walk 

func (c *COS) Walk(ctx context.Context, cosPath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*COS) WalkAbsolute added in v2.4.27 

func (c *COS) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

type FTP 

type FTP struct {
	Config *config.FTPConfig
	// contains filtered or unexported fields
}

func (*FTP) Close 

func (f *FTP) Close(ctx context.Context) error

func (*FTP) Connect 

func (f *FTP) Connect(ctx context.Context) error

func (*FTP) CopyObject 

func (f *FTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

CopyObject copy file inside the same FTP server (like `cp` command), srcBucket is ignored, tries ProFTPD mod_copy `SITE CPFR`/`SITE CPTO` server-side copy first, then FXP (PASV+PORT, server transfers to itself without client bandwidth), falls back to streaming RETR from one pooled connection into STOR on another

func (*FTP) DeleteFile 

func (f *FTP) DeleteFile(ctx context.Context, key string) error

func (*FTP) DeleteFileFromObjectDiskBackup 

func (f *FTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*FTP) DeleteKeys added in v2.6.42 

func (f *FTP) DeleteKeys(ctx context.Context, keys []string) error

DeleteKeys implements BatchDeleter interface for FTP Uses concurrent deletion with connection pool

func (*FTP) DeleteKeysFromObjectDiskBackup added in v2.6.42 

func (f *FTP) DeleteKeysFromObjectDiskBackup(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackup implements BatchDeleter interface for FTP

func (*FTP) GetFileReader 

func (f *FTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*FTP) GetFileReaderAbsolute added in v2.5.0 

func (f *FTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*FTP) GetFileReaderWithLocalPath 

func (f *FTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string, _ int64) (io.ReadCloser, error)

func (*FTP) Kind 

func (f *FTP) Kind() string

func (*FTP) MkdirAll 

func (f *FTP) MkdirAll(key string, client *ftp.ServerConn) error

func (*FTP) PutFile 

func (f *FTP) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*FTP) PutFileAbsolute added in v2.5.0 

func (f *FTP) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, _ int64) error

func (*FTP) StatFile 

func (f *FTP) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*FTP) StatFileAbsolute added in v2.6.26 

func (f *FTP) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*FTP) Walk 

func (f *FTP) Walk(ctx context.Context, ftpPath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*FTP) WalkAbsolute added in v2.4.27 

func (f *FTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

type FTPFileReader 

type FTPFileReader struct {
	*ftp.Response
	// contains filtered or unexported fields
}

func (*FTPFileReader) Close 

func (fr *FTPFileReader) Close() error

type GCS 

type GCS struct {
	Config *config.GCSConfig
	// contains filtered or unexported fields
}

GCS - presents methods for manipulate data on GCS

func (*GCS) Close 

func (gcs *GCS) Close(ctx context.Context) error

func (*GCS) Connect 

func (gcs *GCS) Connect(ctx context.Context) error

Connect - connect to GCS

func (*GCS) CopyObject 

func (gcs *GCS) CopyObject(ctx context.Context, _ int64, srcBucket, srcKey, dstKey string) (int64, error)

CopyObject server-side copy from srcBucket/srcKey to gcs.Config.Bucket/dstKey, both keys are absolute inside the bucket

func (*GCS) DeleteFile 

func (gcs *GCS) DeleteFile(ctx context.Context, key string) error

func (*GCS) DeleteFileFromObjectDiskBackup 

func (gcs *GCS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*GCS) DeleteKeysBatch added in v2.6.42 

func (gcs *GCS) DeleteKeysBatch(ctx context.Context, keys []string) error

DeleteKeysBatch implements BatchDeleter interface for GCS Uses concurrent deletion with connection pool since GCS doesn't have batch delete API

func (*GCS) DeleteKeysFromObjectDiskBackupBatch added in v2.6.42 

func (gcs *GCS) DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackupBatch implements BatchDeleter interface for GCS

func (*GCS) GetFileReader 

func (gcs *GCS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*GCS) GetFileReaderAbsolute added in v2.5.0 

func (gcs *GCS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*GCS) GetFileReaderWithLocalPath 

func (gcs *GCS) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*GCS) Kind 

func (gcs *GCS) Kind() string

func (*GCS) PutFile 

func (gcs *GCS) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*GCS) PutFileAbsolute added in v2.5.0 

func (gcs *GCS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*GCS) StatFile 

func (gcs *GCS) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*GCS) StatFileAbsolute added in v2.6.26 

func (gcs *GCS) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*GCS) Walk 

func (gcs *GCS) Walk(ctx context.Context, gcsPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*GCS) WalkAbsolute added in v2.4.27 

func (gcs *GCS) WalkAbsolute(ctx context.Context, rootPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type KeyError added in v2.6.42 

type KeyError struct {
	Key string
	Err error
}

KeyError represents an error for a specific key during batch deletion

func (KeyError) Error added in v2.6.42 

func (e KeyError) Error() string

type ManifestReader added in v2.8.0 

type ManifestReader struct {
	// contains filtered or unexported fields
}

ManifestReader provides prefix lookups over a downloaded manifest. It is backed by a read-only (mmap) bolt database in a local temporary file, so lookups don't require loading the whole listing into memory.

func (*ManifestReader) Close added in v2.8.0 

func (r *ManifestReader) Close()

Close closes the underlying bolt database and removes the temporary file. Safe to call on a nil reader.

func (*ManifestReader) FilesUnderPrefix added in v2.8.0 

func (r *ManifestReader) FilesUnderPrefix(prefix string) ([]string, error)

FilesUnderPrefix returns the names (relative to the prefix) of all manifest entries whose path starts with the given prefix, using a sorted-key seek instead of a full scan. The prefix should NOT include the backup name (e.g., "shadow/default/my_table/default/part1").

type ManifestWriter added in v2.8.0 

type ManifestWriter struct {
	TotalFiles int
	// contains filtered or unexported fields
}

ManifestWriter incrementally records uploaded file paths into a local bbolt database, batching writes to bound memory usage. It is safe for concurrent use. After a write error it becomes a no-op and the error is returned by Finalize, so a broken manifest never fails the backup itself.

func NewManifestWriter added in v2.8.0 

func NewManifestWriter(backupName string) (*ManifestWriter, error)

NewManifestWriter creates a manifest writer backed by a temporary local bolt file.

func (*ManifestWriter) AddFile added in v2.8.0 

func (w *ManifestWriter) AddFile(relativePath string)

AddFile records a file path in the manifest. On flush failure the writer is marked broken and subsequent calls become no-ops; the error surfaces in Finalize.

func (*ManifestWriter) Close added in v2.8.0 

func (w *ManifestWriter) Close()

Close releases the writer's resources and removes its temporary files. Safe to call multiple times, after Finalize, and on a nil writer.

func (*ManifestWriter) Finalize added in v2.8.0 

func (w *ManifestWriter) Finalize() (string, error)

Finalize flushes pending entries, writes the manifest metadata, closes the bolt database and gzip-compresses it. It returns the path of the compressed file.

type RecalculateV4Signature 

type RecalculateV4Signature struct {
	// contains filtered or unexported fields
}

RecalculateV4Signature allow GCS over S3, remove Accept-Encoding header from sign https://stackoverflow.com/a/74382598/1204665, https://github.com/aws/aws-sdk-go-v2/issues/1816

func (*RecalculateV4Signature) RoundTrip 

func (lt *RecalculateV4Signature) RoundTrip(req *http.Request) (*http.Response, error)

type RemoteFile 

type RemoteFile interface {
	Size() int64
	Name() string
	LastModified() time.Time
}

RemoteFile - interface describe file on remote storage

type RemoteStorage 

type RemoteStorage interface {
	Kind() string
	Connect(ctx context.Context) error
	Close(ctx context.Context) error
	StatFile(ctx context.Context, key string) (RemoteFile, error)
	StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)
	DeleteFile(ctx context.Context, key string) error
	DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error
	Walk(ctx context.Context, prefix string, recursive bool, fn func(context.Context, RemoteFile) error) error
	WalkAbsolute(ctx context.Context, absolutePrefix string, recursive bool, fn func(context.Context, RemoteFile) error) error
	GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)
	GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)
	GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)
	PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error
	PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error
	CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)
}

RemoteStorage -

type S3 

type S3 struct {
	Config      *config.S3Config
	Concurrency int
	// contains filtered or unexported fields
}

S3 - presents methods for manipulate data on s3

func (*S3) Close 

func (s *S3) Close(_ context.Context) error

func (*S3) Connect 

func (s *S3) Connect(ctx context.Context) error

Connect - connect to s3

func (*S3) CopyObject 

func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

CopyObject server-side copy from srcBucket/srcKey to s.Config.Bucket/dstKey, both keys are absolute inside the bucket

func (*S3) DeleteFile 

func (s *S3) DeleteFile(ctx context.Context, key string) error

func (*S3) DeleteFileFromObjectDiskBackup 

func (s *S3) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*S3) DeleteKeysBatch added in v2.6.42 

func (s *S3) DeleteKeysBatch(ctx context.Context, keys []string) error

DeleteKeysBatch implements BatchDeleter interface for S3 Uses DeleteObjects API to delete up to 1000 keys per request

func (*S3) DeleteKeysFromObjectDiskBackupBatch added in v2.6.42 

func (s *S3) DeleteKeysFromObjectDiskBackupBatch(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackupBatch implements BatchDeleter interface for S3

func (*S3) GetFileReader 

func (s *S3) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetFileReaderAbsolute added in v2.5.0 

func (s *S3) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error)

func (*S3) GetFileReaderWithLocalPath 

func (s *S3) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string, remoteSize int64) (io.ReadCloser, error)

func (*S3) Kind 

func (s *S3) Kind() string

func (*S3) PutFile 

func (s *S3) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*S3) PutFileAbsolute added in v2.5.0 

func (s *S3) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*S3) ResolveEndpoint added in v2.6.0 

func (s *S3) ResolveEndpoint(ctx context.Context, params s3.EndpointParameters) (endpoint smithyendpoints.Endpoint, err error)

func (*S3) StatFile 

func (s *S3) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*S3) StatFileAbsolute added in v2.6.26 

func (s *S3) StatFileAbsolute(ctx context.Context, key string) (RemoteFile, error)

func (*S3) Walk 

func (s *S3) Walk(ctx context.Context, s3Path string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

func (*S3) WalkAbsolute added in v2.4.27 

func (s *S3) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error

type S3LogToZeroLogAdapter added in v2.5.22 

type S3LogToZeroLogAdapter struct {
	// contains filtered or unexported fields
}

func (S3LogToZeroLogAdapter) Logf added in v2.5.22 

func (adapter S3LogToZeroLogAdapter) Logf(severity awsV2Logging.Classification, msg string, args ...interface{})

type SFTP 

type SFTP struct {
	Config *config.SFTPConfig
	// contains filtered or unexported fields
}

SFTP Implement RemoteStorage

func (*SFTP) Close 

func (sftp *SFTP) Close(_ context.Context) error

func (*SFTP) Connect 

func (sftp *SFTP) Connect(_ context.Context) error

func (*SFTP) CopyObject 

func (sftp *SFTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error)

CopyObject copy file inside the same SFTP server (like `echo "cp src dst" | sftp -b -`), srcBucket is ignored, uses the `copy-data` SFTP protocol extension (OpenSSH 9.0+) for server-side copy, falls back to `hardlink@openssh.com` (OpenSSH 5.7+, backup files are immutable so a hardlink is equivalent to a copy), streams through the client as the last resort

func (*SFTP) Debug 

func (sftp *SFTP) Debug(msg string, v ...interface{})

func (*SFTP) DeleteDirectory 

func (sftp *SFTP) DeleteDirectory(ctx context.Context, dirPath string) error

func (*SFTP) DeleteFile 

func (sftp *SFTP) DeleteFile(ctx context.Context, key string) error

func (*SFTP) DeleteFileFromObjectDiskBackup 

func (sftp *SFTP) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error

func (*SFTP) DeleteKeys added in v2.6.42 

func (sftp *SFTP) DeleteKeys(ctx context.Context, keys []string) error

DeleteKeys implements BatchDeleter interface for SFTP SFTP uses sequential deletion due to protocol limitations (single connection)

func (*SFTP) DeleteKeysFromObjectDiskBackup added in v2.6.42 

func (sftp *SFTP) DeleteKeysFromObjectDiskBackup(ctx context.Context, keys []string) error

DeleteKeysFromObjectDiskBackup implements BatchDeleter interface for SFTP

func (*SFTP) GetFileReader 

func (sftp *SFTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error)

func (*SFTP) GetFileReaderAbsolute added in v2.5.0 

func (sftp *SFTP) GetFileReaderAbsolute(_ context.Context, key string) (io.ReadCloser, error)

func (*SFTP) GetFileReaderWithLocalPath 

func (sftp *SFTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string, _ int64) (io.ReadCloser, error)

func (*SFTP) Kind 

func (sftp *SFTP) Kind() string

func (*SFTP) PutFile 

func (sftp *SFTP) PutFile(ctx context.Context, key string, r io.ReadCloser, localSize int64) error

func (*SFTP) PutFileAbsolute added in v2.5.0 

func (sftp *SFTP) PutFileAbsolute(_ context.Context, key string, r io.ReadCloser, _ int64) error

func (*SFTP) StatFile 

func (sftp *SFTP) StatFile(ctx context.Context, key string) (RemoteFile, error)

func (*SFTP) StatFileAbsolute added in v2.6.26 

func (sftp *SFTP) StatFileAbsolute(_ context.Context, key string) (RemoteFile, error)

func (*SFTP) Walk 

func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, process func(context.Context, RemoteFile) error) error

func (*SFTP) WalkAbsolute added in v2.4.27 

func (sftp *SFTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.