Documentation
¶
Overview ¶
Package sync provides file synchronization between omnistorage backends.
Inspired by rclone, this package provides three main operations:
- Sync: Make destination match source (including deletes)
- Bisync: Two-way sync with conflict resolution
- CopyDir: Copy files recursively (no deletes)
- Check: Verify files match between backends
Basic usage:
result, err := sync.Sync(ctx, srcBackend, dstBackend, "data/", "backup/", sync.Options{
DeleteExtra: true,
DryRun: false,
})
fmt.Printf("Copied: %d, Deleted: %d\n", result.Copied, result.Deleted)
With logging:
result, err := sync.Sync(ctx, srcBackend, dstBackend, "data/", "backup/", sync.Options{
Logger: slog.Default(),
})
Index ¶
- func CopyBetweenPaths(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) error
- func CopyFile(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) error
- func CopyFromPath(ctx context.Context, src omnistorage.Backend, srcPath string, writer io.Writer) (int64, error)
- func CopyToPath(ctx context.Context, dst omnistorage.Backend, reader io.Reader, dstPath string, ...) error
- func IsRetryError(err error) bool
- func IsTemporaryError(err error) bool
- func MoveFile(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) error
- func NeedsUpdate(src, dst FileInfo, opts Options) bool
- func QuickVerify(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) (bool, error)
- func Verify(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, ...) (bool, error)
- func VerifyAllIntegrity(ctx context.Context, backend omnistorage.Backend, basePath string) ([]string, error)
- func VerifyAndReport(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, ...) (string, error)
- func VerifyChecksum(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) (bool, error)
- func VerifyFile(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) (bool, error)
- func VerifyIntegrity(ctx context.Context, backend omnistorage.Backend, filePath string) error
- type BisyncOptions
- type BisyncResult
- type CheckResult
- type Conflict
- type ConflictStrategy
- type DiffEntry
- type DiffStatus
- type FileError
- type FileInfo
- type MetadataOptions
- type Options
- type Phase
- type Progress
- type Result
- func Copy(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, ...) (*Result, error)
- func CopyDir(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) (*Result, error)
- func CopyWithProgress(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, ...) (*Result, error)
- func Move(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, ...) (*Result, error)
- func MustCopy(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) *Result
- func Sync(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, ...) (*Result, error)
- func TreeCopy(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, ...) (*Result, error)
- type RetryConfig
- type RetryError
- type VerifyResult
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CopyBetweenPaths ¶
func CopyBetweenPaths(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) error
CopyBetweenPaths copies between two paths, potentially on different backends. If src and dst are the same backend and support server-side copy, it will be used.
func CopyFile ¶
CopyFile copies a single file from source to destination backend.
This is a convenience function for copying individual files across backends. For copying multiple files, use Copy or Sync.
func CopyFromPath ¶
func CopyFromPath(ctx context.Context, src omnistorage.Backend, srcPath string, writer io.Writer) (int64, error)
CopyFromPath copies from a backend path to a writer. Useful for copying from a backend to any io.Writer destination.
func CopyToPath ¶
func CopyToPath(ctx context.Context, dst omnistorage.Backend, reader io.Reader, dstPath string, contentType string) error
CopyToPath copies from a reader to a destination path. Useful for copying from any io.Reader source to a backend.
func IsRetryError ¶
IsRetryError returns true if err is a RetryError.
func IsTemporaryError ¶
IsTemporaryError returns true if err is likely temporary and worth retrying. This is a reasonable default for RetryConfig.RetryableErrors.
func MoveFile ¶
MoveFile moves a single file from source to destination. The source file is deleted after successful copy.
func NeedsUpdate ¶
NeedsUpdate returns true if dst should be updated to match src.
func QuickVerify ¶
func QuickVerify(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) (bool, error)
QuickVerify performs a fast verification using only file metadata.
Compares files by size only, which is very fast but may miss files with identical sizes but different content.
func Verify ¶
func Verify(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, opts Options) (bool, error)
Verify checks if source and destination are in sync.
Returns true if all files match, false otherwise. This is a simplified interface to Check for common use cases.
By default, files are compared by size. Use opts.Checksum for content-based verification (slower but more accurate).
func VerifyAllIntegrity ¶
func VerifyAllIntegrity(ctx context.Context, backend omnistorage.Backend, basePath string) ([]string, error)
VerifyAllIntegrity checks integrity of all files under a path.
func VerifyAndReport ¶
func VerifyAndReport(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, opts Options) (string, error)
VerifyAndReport verifies and returns a human-readable report.
func VerifyChecksum ¶
func VerifyChecksum(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) (bool, error)
VerifyChecksum verifies files using content checksum comparison.
This is more accurate than size/time comparison but slower as it reads all file contents.
func VerifyFile ¶
func VerifyFile(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) (bool, error)
VerifyFile checks if a single file matches between source and destination.
Returns true if the files are identical, false otherwise. Uses checksum comparison for accurate verification.
func VerifyIntegrity ¶
VerifyIntegrity checks that a file's content is readable and intact.
This reads the entire file to verify it can be read without errors. Useful for detecting corrupted files in storage.
Types ¶
type BisyncOptions ¶
type BisyncOptions struct {
// ConflictStrategy determines how to handle files changed on both sides.
// Default is ConflictNewerWins.
ConflictStrategy ConflictStrategy
// ConflictSuffix is appended to filenames when using ConflictKeepBoth.
// Default is ".conflict".
ConflictSuffix string
// DryRun reports what would be done without making changes.
DryRun bool
// Checksum uses file checksums for comparison instead of size/time.
Checksum bool
// DeleteMissing deletes files that don't exist on the other side.
// Use with caution - this can result in data loss if a file was
// intentionally deleted on one side but still exists on the other.
DeleteMissing bool
// Progress is called with progress updates during sync.
Progress func(Progress)
// MaxErrors is the maximum number of errors before aborting.
MaxErrors int
// Concurrency is the number of concurrent file transfers.
// Default is 4.
Concurrency int
// Filter specifies which files to include/exclude.
Filter *filter.Filter
// BandwidthLimit is the maximum bytes per second for transfers.
BandwidthLimit int64
// Retry configures retry behavior for failed operations.
Retry *RetryConfig
// PreserveMetadata controls which metadata to preserve.
PreserveMetadata *MetadataOptions
// Logger for structured logging. If nil, no logging is performed.
Logger *slog.Logger
}
BisyncOptions configures bidirectional sync behavior.
func DefaultBisyncOptions ¶
func DefaultBisyncOptions() BisyncOptions
DefaultBisyncOptions returns BisyncOptions with sensible defaults.
type BisyncResult ¶
type BisyncResult struct {
// CopiedToPath2 is files copied from path1 to path2.
CopiedToPath2 int
// CopiedToPath1 is files copied from path2 to path1.
CopiedToPath1 int
// UpdatedInPath2 is files updated in path2 from path1.
UpdatedInPath2 int
// UpdatedInPath1 is files updated in path1 from path2.
UpdatedInPath1 int
// DeletedFromPath1 is files deleted from path1.
DeletedFromPath1 int
// DeletedFromPath2 is files deleted from path2.
DeletedFromPath2 int
// Conflicts is the list of conflicting files.
Conflicts []Conflict
// Skipped is the number of files skipped (already in sync).
Skipped int
// Errors contains any errors that occurred.
Errors []FileError
// BytesTransferred is the total bytes transferred.
BytesTransferred int64
// Duration is how long the sync took.
Duration time.Duration
// DryRun indicates if this was a dry run.
DryRun bool
}
BisyncResult contains the results of a bidirectional sync.
func Bisync ¶
func Bisync(ctx context.Context, backend1, backend2 omnistorage.Backend, path1, path2 string, opts BisyncOptions) (*BisyncResult, error)
Bisync performs bidirectional synchronization between two backends.
Unlike unidirectional Sync, Bisync propagates changes in both directions:
- Files only in path1 are copied to path2
- Files only in path2 are copied to path1
- Files changed on both sides are handled according to ConflictStrategy
This is useful for syncing two directories that may both have changes, such as syncing between a local folder and cloud storage where edits can happen on either side.
func (*BisyncResult) Success ¶
func (r *BisyncResult) Success() bool
Success returns true if bisync completed without errors.
func (*BisyncResult) TotalCopied ¶
func (r *BisyncResult) TotalCopied() int
TotalCopied returns the total number of files copied in both directions.
func (*BisyncResult) TotalDeleted ¶
func (r *BisyncResult) TotalDeleted() int
TotalDeleted returns the total number of files deleted from both sides.
func (*BisyncResult) TotalUpdated ¶
func (r *BisyncResult) TotalUpdated() int
TotalUpdated returns the total number of files updated in both directions.
type CheckResult ¶
type CheckResult struct {
// Match lists files that match between source and destination.
Match []string
// Differ lists files that exist in both but have different content.
Differ []string
// SrcOnly lists files that exist only in source.
SrcOnly []string
// DstOnly lists files that exist only in destination.
DstOnly []string
// Errors contains any errors that occurred during checking.
Errors []FileError
}
CheckResult contains the results of a check operation.
func Check ¶
func Check(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, opts Options) (*CheckResult, error)
Check compares files between source and destination backends.
It returns a CheckResult containing:
- Match: files that are identical in both
- Differ: files that exist in both but have different content
- SrcOnly: files that exist only in source
- DstOnly: files that exist only in destination
By default, files are compared by size and modification time. Set opts.Checksum to true for content-based comparison (slower but more accurate).
func (*CheckResult) InSync ¶
func (r *CheckResult) InSync() bool
InSync returns true if source and destination are in sync.
type Conflict ¶
type Conflict struct {
// Path is the relative path of the conflicting file.
Path string
// Path1Info is the file info from path1.
Path1Info FileInfo
// Path2Info is the file info from path2.
Path2Info FileInfo
// Resolution describes how the conflict was resolved.
Resolution string
// Error is set if the conflict could not be resolved.
Error error
}
Conflict represents a file that was changed on both sides.
type ConflictStrategy ¶
type ConflictStrategy int
ConflictStrategy defines how to handle files changed on both sides.
const ( // ConflictNewerWins keeps the file with the newer modification time. ConflictNewerWins ConflictStrategy = iota // ConflictLargerWins keeps the file with the larger size. ConflictLargerWins // ConflictSourceWins always keeps the source (path1) version. ConflictSourceWins // ConflictDestWins always keeps the destination (path2) version. ConflictDestWins // ConflictKeepBoth keeps both files, renaming one with a conflict suffix. ConflictKeepBoth // ConflictSkip skips conflicting files without copying either. ConflictSkip // ConflictError reports an error for each conflict. ConflictError )
type DiffEntry ¶
type DiffEntry struct {
Path string
Status DiffStatus
}
DiffEntry represents a single difference between backends.
type DiffStatus ¶
type DiffStatus string
DiffStatus represents the type of difference.
const ( // DiffStatusNew indicates a file exists only in source. DiffStatusNew DiffStatus = "new" // DiffStatusModified indicates a file differs between source and destination. DiffStatusModified DiffStatus = "modified" // DiffStatusDeleted indicates a file exists only in destination. DiffStatusDeleted DiffStatus = "deleted" )
type FileInfo ¶
type FileInfo struct {
Path string
Size int64
ModTime time.Time
Hash string // MD5 or other hash if available
IsDir bool
}
FileInfo represents a file for sync comparison.
type MetadataOptions ¶
type MetadataOptions struct {
// ContentType preserves the MIME content type.
// Default is true (always preserved).
ContentType bool
// ModTime preserves the modification time.
// Requires backend support (Features().SetModTime).
ModTime bool
// CustomMetadata preserves backend-specific custom metadata.
// For S3, this includes user-defined metadata headers.
CustomMetadata bool
}
MetadataOptions configures which metadata to preserve during sync.
func DefaultMetadataOptions ¶
func DefaultMetadataOptions() *MetadataOptions
DefaultMetadataOptions returns metadata options with sensible defaults.
type Options ¶
type Options struct {
// DeleteExtra deletes files in destination that don't exist in source.
// When false, behaves like CopyDir (only adds/updates, never deletes).
DeleteExtra bool
// DryRun reports what would be done without making changes.
DryRun bool
// Checksum uses file checksums (when available) for comparison
// instead of just size and modification time.
// This is slower but more accurate.
Checksum bool
// IgnoreExisting skips files that already exist in destination.
// Useful for resuming interrupted syncs.
IgnoreExisting bool
// IgnoreSize ignores size when comparing files.
// Only compares modification time (or checksum if Checksum is true).
IgnoreSize bool
// IgnoreTime ignores modification time when comparing files.
// Only compares size (or checksum if Checksum is true).
IgnoreTime bool
// SizeOnly compares files by size only, ignoring modification time.
SizeOnly bool
// Progress is called with progress updates during sync.
// Can be nil if progress updates aren't needed.
Progress func(Progress)
// MaxErrors is the maximum number of errors before aborting.
// 0 means abort on first error.
MaxErrors int
// Concurrency is the number of concurrent file transfers.
// Default is 4.
Concurrency int
// Filter specifies which files to include/exclude from sync.
// If nil, all files are included.
Filter *filter.Filter
// DeleteExcluded deletes files from destination that match exclude filters.
// Only applies when DeleteExtra is true.
DeleteExcluded bool
// BandwidthLimit is the maximum bytes per second for transfers.
// 0 means unlimited. The limit is shared across all concurrent transfers.
// Example: 1048576 for 1MB/s, or use filter.MB constant.
BandwidthLimit int64
// Retry configures retry behavior for failed file operations.
// If nil or MaxRetries is 0, operations are not retried.
Retry *RetryConfig
// PreserveMetadata controls which metadata to preserve during sync.
// If nil, only content-type is preserved (default behavior).
PreserveMetadata *MetadataOptions
// Logger is used for structured logging during sync operations.
// If nil, a null logger is used (no logging).
Logger *slog.Logger
}
Options configures sync behavior.
func DefaultOptions ¶
func DefaultOptions() Options
DefaultOptions returns Options with sensible defaults.
type Phase ¶
type Phase string
Phase represents a phase of the sync operation.
const ( // PhaseScanning indicates the sync is scanning for files. PhaseScanning Phase = "scanning" // PhaseComparing indicates the sync is comparing files. PhaseComparing Phase = "comparing" // PhaseTransferring indicates the sync is transferring files. PhaseTransferring Phase = "transferring" // PhaseDeleting indicates the sync is deleting extra files. PhaseDeleting Phase = "deleting" // PhaseComplete indicates the sync is complete. PhaseComplete Phase = "complete" )
type Progress ¶
type Progress struct {
// Phase indicates the current sync phase.
Phase Phase
// CurrentFile is the file currently being processed.
CurrentFile string
// BytesTransferred is the number of bytes transferred so far.
BytesTransferred int64
// TotalBytes is the total bytes to transfer (if known).
TotalBytes int64
// FilesTransferred is the number of files transferred so far.
FilesTransferred int
// TotalFiles is the total number of files to transfer.
TotalFiles int
// FilesDeleted is the number of files deleted so far.
FilesDeleted int
// Errors is the number of errors encountered so far.
Errors int
}
Progress represents the current state of a sync operation.
type Result ¶
type Result struct {
// Copied is the number of files copied.
Copied int
// Updated is the number of files updated (overwritten).
Updated int
// Deleted is the number of files deleted from destination.
Deleted int
// Skipped is the number of files skipped (already in sync).
Skipped int
// Errors contains any errors that occurred.
Errors []FileError
// BytesTransferred is the total bytes transferred.
BytesTransferred int64
// Duration is how long the sync took.
Duration time.Duration
// DryRun indicates if this was a dry run.
DryRun bool
}
Result contains the results of a sync operation.
func Copy ¶
func Copy(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, opts Options) (*Result, error)
Copy copies files from source to destination.
If srcPath is a file, it copies that single file. If srcPath is a directory (or prefix), it copies all files recursively.
Copy never deletes files from the destination. Use Sync with DeleteExtra=true for mirror behavior.
Options that affect Copy:
- DryRun: report what would be copied without copying
- IgnoreExisting: skip files that already exist in destination
- Progress: callback for progress updates
func CopyDir ¶
func CopyDir(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string) (*Result, error)
CopyDir copies all files from source to destination recursively. Unlike Sync, it never deletes files from destination. This is equivalent to Sync with DeleteExtra=false.
func CopyWithProgress ¶
func CopyWithProgress(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, progress func(currentFile string, bytesCopied int64)) (*Result, error)
CopyWithProgress copies files with a simple progress callback.
The callback receives the current file path and bytes copied so far. This is a convenience wrapper around Copy for simple progress reporting.
func Move ¶
func Move(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, opts Options) (*Result, error)
Move moves files from source to destination.
This is like Sync but also deletes files from source after successful copy. Use with caution as source files are permanently deleted.
func MustCopy ¶
MustCopy is like Copy but panics on error. Useful for scripts and tests where errors are unexpected.
func Sync ¶
func Sync(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, opts Options) (*Result, error)
Sync synchronizes files from source to destination.
By default, it copies new and updated files but does not delete extra files in the destination. Set Options.DeleteExtra to true to remove files from destination that don't exist in source (making destination a mirror of source).
Both backends should support List operation. If the source backend implements ExtendedBackend with Stat, it will be used for more accurate file comparison.
func TreeCopy ¶
func TreeCopy(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, opts Options) (*Result, error)
TreeCopy copies an entire directory tree, preserving structure. Unlike Copy which flattens paths relative to srcPath, TreeCopy preserves the full directory structure under dstPath.
type RetryConfig ¶
type RetryConfig struct {
// MaxRetries is the maximum number of retry attempts.
// 0 means no retries (fail on first error).
MaxRetries int
// InitialDelay is the delay before the first retry.
// Default is 1 second.
InitialDelay time.Duration
// MaxDelay is the maximum delay between retries.
// Default is 30 seconds.
MaxDelay time.Duration
// Multiplier is the factor by which delay increases after each retry.
// Default is 2.0 (exponential backoff).
Multiplier float64
// Jitter adds randomness to delays to prevent thundering herd.
// 0.1 means +/- 10% random variation. Default is 0.1.
Jitter float64
// RetryableErrors is a function that determines if an error should be retried.
// If nil, all errors are retried.
RetryableErrors func(error) bool
}
RetryConfig configures retry behavior for failed operations.
func DefaultRetryConfig ¶
func DefaultRetryConfig() RetryConfig
DefaultRetryConfig returns retry config with sensible defaults.
type RetryError ¶
RetryError indicates an operation failed after all retry attempts.
func (*RetryError) Error ¶
func (e *RetryError) Error() string
func (*RetryError) Unwrap ¶
func (e *RetryError) Unwrap() error
type VerifyResult ¶
type VerifyResult struct {
// Verified is true if all files match.
Verified bool
// TotalFiles is the total number of files checked.
TotalFiles int
// MatchingFiles is the number of files that match.
MatchingFiles int
// MismatchedFiles lists files that don't match.
MismatchedFiles []string
// MissingInDst lists files missing from destination.
MissingInDst []string
// ExtraInDst lists files in destination but not in source.
ExtraInDst []string
// Errors contains any errors encountered during verification.
Errors []FileError
}
VerifyResult contains detailed verification results.
func VerifyWithDetails ¶
func VerifyWithDetails(ctx context.Context, src, dst omnistorage.Backend, srcPath, dstPath string, opts Options) (*VerifyResult, error)
VerifyWithDetails performs verification and returns detailed results.
This is useful when you need to know exactly what differs, not just whether everything matches.