sync

package
v1.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 6, 2026 License: Apache-2.0 Imports: 39 Imported by: 6

Documentation

Index

Constants

View Source
const JFS_UMASK = "JFS_UMASK"

Variables

This section is empty.

Functions

func CopyData added in v0.1.3

func CopyData(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error)

func InitForCopyData added in v0.1.3

func InitForCopyData()

func ListAll

func ListAll(store object.ObjectStorage, prefix, start, end string, followLink bool) (<-chan object.Object, error)

ListAll on all the keys that starts at marker from object storage.

func Sync

func Sync(src, dst object.ObjectStorage, config *Config) error

Sync syncs all the keys between to object storage

Types

type Checkpoint added in v0.1.7

type Checkpoint struct {
	sync.RWMutex
	PrefixState      map[string]*PrefixState          `json:"prefix_state"`
	MultipartUploads map[string]*multipartUploadState `json:"multipart_uploads,omitempty"`
	Config           *Config                          `json:"config"`
	Stats            CheckpointStats                  `json:"stats"`
	SrcDelayDel      []string                         `json:"src_delay_del,omitempty"`
	DstDelayDel      []string                         `json:"dst_delay_del,omitempty"`
	UpdatedAt        time.Time                        `json:"updated_at"`
}

Checkpoint represents the complete checkpoint state

type CheckpointManager added in v0.1.7

type CheckpointManager struct {
	// contains filtered or unexported fields
}

CheckpointManager manages checkpoint persistence

func NewCheckpointManager added in v0.1.7

func NewCheckpointManager(src, dst object.ObjectStorage, config *Config) *CheckpointManager

NewCheckpointManager creates a new checkpoint manager

func (*CheckpointManager) AddPendingKey added in v0.1.7

func (m *CheckpointManager) AddPendingKey(prefix string, obj object.Object)

func (*CheckpointManager) DeleteCheckpoint added in v0.1.7

func (m *CheckpointManager) DeleteCheckpoint() error

DeleteCheckpoint removes the checkpoint file from storage.

func (CheckpointManager) EnsureMultipartUploadState added in v0.1.7

func (s CheckpointManager) EnsureMultipartUploadState(key string, size int64, mtime time.Time, partSize int64, upload *object.MultipartUpload) *multipartUploadState

func (CheckpointManager) FindMultipartUpload added in v0.1.7

func (s CheckpointManager) FindMultipartUpload(key string, size int64, mtime time.Time) *object.MultipartUpload

func (CheckpointManager) FinishMultipartUpload added in v0.1.7

func (s CheckpointManager) FinishMultipartUpload(key string)

func (*CheckpointManager) GetLastListedKey added in v0.1.7

func (m *CheckpointManager) GetLastListedKey(prefix string) string

GetLastListedKey returns the last listed key for a prefix, or "" if not tracked.

func (CheckpointManager) GetMultipartCheckpoint added in v0.1.7

func (s CheckpointManager) GetMultipartCheckpoint(key string, size int64, mtime time.Time) *multipartUploadState

func (CheckpointManager) GetMultipartPart added in v0.1.7

func (s CheckpointManager) GetMultipartPart(state *multipartUploadState, num int, calChksum bool) (*object.Part, uint32, bool)

func (*CheckpointManager) GetOrCreatePrefixState added in v0.1.7

func (m *CheckpointManager) GetOrCreatePrefixState(prefix string) *PrefixState

GetOrCreatePrefixState gets or creates a prefix state

func (*CheckpointManager) ListPrefixes added in v0.1.7

func (m *CheckpointManager) ListPrefixes() []string

ListPrefixes returns a snapshot of all prefix keys currently tracked in checkpoint.

func (*CheckpointManager) Load added in v0.1.7

func (m *CheckpointManager) Load() (*Checkpoint, error)

Load loads checkpoint from object storage

func (*CheckpointManager) MarkCompleted added in v0.1.7

func (m *CheckpointManager) MarkCompleted(key string)

MarkCompleted removes a key from PendingKeys after successful completion

func (*CheckpointManager) MarkFailed added in v0.1.7

func (m *CheckpointManager) MarkFailed(key string)

MarkFailed moves a key from PendingKeys to FailedKeys

func (*CheckpointManager) MarkListDone added in v0.1.7

func (m *CheckpointManager) MarkListDone(prefix string)

func (CheckpointManager) MarkMultipartPart added in v0.1.7

func (s CheckpointManager) MarkMultipartPart(key string, state *multipartUploadState, part *object.Part, chksum uint32, calChksum bool)

func (CheckpointManager) PutMultipartCheckpoint added in v0.1.7

func (s CheckpointManager) PutMultipartCheckpoint(key string, state *multipartUploadState)

func (*CheckpointManager) RegisterChildPrefix added in v0.1.7

func (m *CheckpointManager) RegisterChildPrefix(childPrefix string, listDepth int)

RegisterChildPrefix registers a child prefix discovered during listing.

func (*CheckpointManager) Reset added in v0.1.7

func (m *CheckpointManager) Reset(config *Config)

Reset discards the current checkpoint and starts fresh with the given config.

func (*CheckpointManager) RestorePrefix added in v0.1.7

func (m *CheckpointManager) RestorePrefix(prefix string) (objs []object.Object, listDone bool, listDepth int, found bool)

RestorePrefix restores pending+failed keys for a prefix, merging failed into pending.

func (*CheckpointManager) Save added in v0.1.7

func (m *CheckpointManager) Save(ckpt *Checkpoint) error

Save saves checkpoint to object storage

func (*CheckpointManager) SaveOnSignal added in v0.1.7

func (m *CheckpointManager) SaveOnSignal()

func (*CheckpointManager) StartPeriodicSave added in v0.1.7

func (m *CheckpointManager) StartPeriodicSave(interval time.Duration)

func (*CheckpointManager) Stop added in v0.1.7

func (m *CheckpointManager) Stop()

func (*CheckpointManager) TrackKey added in v0.1.7

func (m *CheckpointManager) TrackKey(key, prefix string)

func (*CheckpointManager) UpdateLastListedKey added in v0.1.7

func (m *CheckpointManager) UpdateLastListedKey(prefix string, obj object.Object)

func (*CheckpointManager) ValidateConfig added in v0.1.7

func (m *CheckpointManager) ValidateConfig(current *Config) bool

ValidateConfig checks if checkpoint config matches current config

type CheckpointStats added in v0.1.7

type CheckpointStats struct {
	Copied       int64 `json:"copied"`
	CopiedBytes  int64 `json:"copied_bytes"`
	Checked      int64 `json:"checked"`
	CheckedBytes int64 `json:"checked_bytes"`
	Deleted      int64 `json:"deleted"`
	Skipped      int64 `json:"skipped"`
	SkippedBytes int64 `json:"skipped_bytes"`
	Failed       int64 `json:"failed"`
	Handled      int64 `json:"handled"`
}

CheckpointStats stores cumulative statistics

type Config

type Config struct {
	StorageClass      string
	Start             string
	End               string
	Threads           int
	Update            bool
	ForceUpdate       bool
	Perms             bool
	MaxFailure        int64
	Dry               bool
	DeleteSrc         bool
	DeleteSrcAfter    bool
	DeleteDst         bool
	MatchFullPath     bool
	Dirs              bool
	Exclude           []string
	Include           []string
	Existing          bool
	IgnoreExisting    bool
	Links             bool
	Inplace           bool
	Limit             int64
	Manager           string
	Workers           []string
	ManagerAddr       string
	ListThreads       int
	ListDepth         int
	BWLimit           int64
	TrafficControlURL string
	NoHTTPS           bool
	Verbose           bool
	Quiet             bool
	CheckAll          bool
	CheckNew          bool
	CheckChange       bool
	MaxSize           int64
	MinSize           int64
	MaxAge            time.Duration
	MinAge            time.Duration
	StartTime         time.Time
	EndTime           time.Time
	Env               map[string]string `json:"-"`

	FilesFrom string

	EnableCheckpoint     bool
	CheckpointInterval   time.Duration
	CheckpointForceReset bool

	Registerer prometheus.Registerer `json:"-"`
	// contains filtered or unexported fields
}

func NewConfigFromCli

func NewConfigFromCli(c *cli.Context) *Config

type PrefixState added in v0.1.7

type PrefixState struct {
	sync.RWMutex
	ListDone      bool                     `json:"list_done"`
	LastListedKey string                   `json:"last_listed_key"`
	ListDepth     int                      `json:"list_depth"`
	PendingKeys   map[string]object.Object `json:"-"`
	FailedKeys    map[string]object.Object `json:"-"`
	// contains filtered or unexported fields
}

PrefixState maintains the state for a specific prefix

func (*PrefixState) MarshalJSON added in v0.1.7

func (s *PrefixState) MarshalJSON() ([]byte, error)

func (*PrefixState) UnmarshalJSON added in v0.1.7

func (s *PrefixState) UnmarshalJSON(data []byte) error

type Stat

type Stat struct {
	Copied           int64                            // the number of copied files
	CopiedBytes      int64                            // total amount of copied data in bytes
	Checked          int64                            // the number of checked files
	CheckedBytes     int64                            // total amount of checked data in bytes
	Deleted          int64                            // the number of deleted files
	Skipped          int64                            // the number of files skipped
	SkippedBytes     int64                            // total amount of skipped data in bytes
	Failed           int64                            // the number of files that fail to copy
	DelayDelDir      []string                         // the directories that need to be deleted
	CompletedKeys    []string                         `json:"completed_keys,omitempty"` // checkpoint: keys completed by this worker
	FailedKeys       []string                         `json:"failed_keys,omitempty"`    // checkpoint: keys failed by this worker
	MultipartUploads map[string]*multipartUploadState `json:"multipart_uploads,omitempty"`
}

Stat has the counters to represent the progress.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL