sync

package
v1.4.0-beta2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 29, 2026 License: Apache-2.0 Imports: 39 Imported by: 6

Documentation

Index

Constants

View Source
const JFS_UMASK = "JFS_UMASK"

Variables

This section is empty.

Functions

func CopyData added in v0.1.3

func CopyData(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error)

func InitForCopyData added in v0.1.3

func InitForCopyData()

func ListAll

func ListAll(store object.ObjectStorage, prefix, start, end string, followLink bool) (<-chan object.Object, error)

ListAll on all the keys that starts at marker from object storage.

func Sync

func Sync(src, dst object.ObjectStorage, config *Config) error

Sync syncs all the keys between to object storage

Types

type Checkpoint

type Checkpoint struct {
	sync.RWMutex
	PrefixState      map[string]*PrefixState          `json:"prefix_state"`
	MultipartUploads map[string]*multipartUploadState `json:"multipart_uploads,omitempty"`
	Config           *Config                          `json:"config"`
	Stats            CheckpointStats                  `json:"stats"`
	SrcDelayDel      []string                         `json:"src_delay_del,omitempty"`
	DstDelayDel      []string                         `json:"dst_delay_del,omitempty"`
	UpdatedAt        time.Time                        `json:"updated_at"`
}

Checkpoint represents the complete checkpoint state

type CheckpointManager

type CheckpointManager struct {
	// contains filtered or unexported fields
}

CheckpointManager manages checkpoint persistence

func NewCheckpointManager

func NewCheckpointManager(src, dst object.ObjectStorage, config *Config) *CheckpointManager

NewCheckpointManager creates a new checkpoint manager

func (*CheckpointManager) AddPendingKey

func (m *CheckpointManager) AddPendingKey(prefix string, obj object.Object)

func (*CheckpointManager) DeleteCheckpoint

func (m *CheckpointManager) DeleteCheckpoint() error

DeleteCheckpoint removes the checkpoint file from storage.

func (CheckpointManager) EnsureMultipartUploadState

func (s CheckpointManager) EnsureMultipartUploadState(key string, size int64, mtime time.Time, partSize int64, upload *object.MultipartUpload) *multipartUploadState

func (CheckpointManager) FindMultipartUpload

func (s CheckpointManager) FindMultipartUpload(key string, size int64, mtime time.Time) *object.MultipartUpload

func (CheckpointManager) FinishMultipartUpload

func (s CheckpointManager) FinishMultipartUpload(key string)

func (*CheckpointManager) GetLastListedKey

func (m *CheckpointManager) GetLastListedKey(prefix string) string

GetLastListedKey returns the last listed key for a prefix, or "" if not tracked.

func (CheckpointManager) GetMultipartCheckpoint

func (s CheckpointManager) GetMultipartCheckpoint(key string, size int64, mtime time.Time) *multipartUploadState

func (CheckpointManager) GetMultipartPart

func (s CheckpointManager) GetMultipartPart(state *multipartUploadState, num int, calChksum bool) (*object.Part, uint32, bool)

func (*CheckpointManager) GetOrCreatePrefixState

func (m *CheckpointManager) GetOrCreatePrefixState(prefix string) *PrefixState

GetOrCreatePrefixState gets or creates a prefix state

func (*CheckpointManager) ListPrefixes

func (m *CheckpointManager) ListPrefixes() []string

ListPrefixes returns a snapshot of all prefix keys currently tracked in checkpoint.

func (*CheckpointManager) Load

func (m *CheckpointManager) Load() (*Checkpoint, error)

Load loads checkpoint from object storage

func (*CheckpointManager) MarkCompleted

func (m *CheckpointManager) MarkCompleted(key string)

MarkCompleted removes a key from PendingKeys after successful completion

func (*CheckpointManager) MarkFailed

func (m *CheckpointManager) MarkFailed(key string)

MarkFailed moves a key from PendingKeys to FailedKeys

func (*CheckpointManager) MarkListDone

func (m *CheckpointManager) MarkListDone(prefix string)

func (CheckpointManager) MarkMultipartPart

func (s CheckpointManager) MarkMultipartPart(key string, state *multipartUploadState, part *object.Part, chksum uint32, calChksum bool)

func (CheckpointManager) PutMultipartCheckpoint

func (s CheckpointManager) PutMultipartCheckpoint(key string, state *multipartUploadState)

func (*CheckpointManager) RegisterChildPrefix

func (m *CheckpointManager) RegisterChildPrefix(childPrefix string, listDepth int)

RegisterChildPrefix registers a child prefix discovered during listing.

func (*CheckpointManager) Reset

func (m *CheckpointManager) Reset(config *Config)

Reset discards the current checkpoint and starts fresh with the given config.

func (*CheckpointManager) RestorePrefix

func (m *CheckpointManager) RestorePrefix(prefix string) (objs []object.Object, listDone bool, listDepth int, found bool)

RestorePrefix restores pending+failed keys for a prefix, merging failed into pending.

func (*CheckpointManager) Save

func (m *CheckpointManager) Save(ckpt *Checkpoint) error

Save saves checkpoint to object storage

func (*CheckpointManager) SaveOnSignal

func (m *CheckpointManager) SaveOnSignal()

func (*CheckpointManager) StartPeriodicSave

func (m *CheckpointManager) StartPeriodicSave(interval time.Duration)

func (*CheckpointManager) Stop

func (m *CheckpointManager) Stop()

func (*CheckpointManager) TrackKey

func (m *CheckpointManager) TrackKey(key, prefix string)

func (*CheckpointManager) UpdateLastListedKey

func (m *CheckpointManager) UpdateLastListedKey(prefix string, obj object.Object)

func (*CheckpointManager) ValidateConfig

func (m *CheckpointManager) ValidateConfig(current *Config) bool

ValidateConfig checks if checkpoint config matches current config

type CheckpointStats

type CheckpointStats struct {
	Copied       int64 `json:"copied"`
	CopiedBytes  int64 `json:"copied_bytes"`
	Checked      int64 `json:"checked"`
	CheckedBytes int64 `json:"checked_bytes"`
	Deleted      int64 `json:"deleted"`
	Skipped      int64 `json:"skipped"`
	SkippedBytes int64 `json:"skipped_bytes"`
	Failed       int64 `json:"failed"`
	Handled      int64 `json:"handled"`
}

CheckpointStats stores cumulative statistics

type Config

type Config struct {
	StorageClass      string
	Start             string
	End               string
	Threads           int
	Update            bool
	ForceUpdate       bool
	Perms             bool
	MaxFailure        int64
	Dry               bool
	DeleteSrc         bool
	DeleteSrcAfter    bool
	DeleteDst         bool
	MatchFullPath     bool
	Dirs              bool
	Exclude           []string
	Include           []string
	Existing          bool
	IgnoreExisting    bool
	Links             bool
	Inplace           bool
	Limit             int64
	Manager           string
	Workers           []string
	ManagerAddr       string
	ListThreads       int
	ListDepth         int
	BWLimit           int64
	TrafficControlURL string
	NoHTTPS           bool
	Verbose           bool
	Quiet             bool
	CheckAll          bool
	CheckNew          bool
	CheckChange       bool
	MaxSize           int64
	MinSize           int64
	MaxAge            time.Duration
	MinAge            time.Duration
	StartTime         time.Time
	EndTime           time.Time
	Env               map[string]string `json:"-"`

	FilesFrom string

	EnableCheckpoint     bool
	CheckpointInterval   time.Duration
	CheckpointForceReset bool

	Registerer prometheus.Registerer `json:"-"`
	// contains filtered or unexported fields
}

func NewConfigFromCli

func NewConfigFromCli(c *cli.Context) *Config

type PrefixState

type PrefixState struct {
	sync.RWMutex
	ListDone      bool                     `json:"list_done"`
	LastListedKey string                   `json:"last_listed_key"`
	ListDepth     int                      `json:"list_depth"`
	PendingKeys   map[string]object.Object `json:"-"`
	FailedKeys    map[string]object.Object `json:"-"`
	// contains filtered or unexported fields
}

PrefixState maintains the state for a specific prefix

func (*PrefixState) MarshalJSON

func (s *PrefixState) MarshalJSON() ([]byte, error)

func (*PrefixState) UnmarshalJSON

func (s *PrefixState) UnmarshalJSON(data []byte) error

type Stat

type Stat struct {
	Copied           int64                            // the number of copied files
	CopiedBytes      int64                            // total amount of copied data in bytes
	Checked          int64                            // the number of checked files
	CheckedBytes     int64                            // total amount of checked data in bytes
	Deleted          int64                            // the number of deleted files
	Skipped          int64                            // the number of files skipped
	SkippedBytes     int64                            // total amount of skipped data in bytes
	Failed           int64                            // the number of files that fail to copy
	DelayDelDir      []string                         // the directories that need to be deleted
	CompletedKeys    []string                         `json:"completed_keys,omitempty"` // checkpoint: keys completed by this worker
	FailedKeys       []string                         `json:"failed_keys,omitempty"`    // checkpoint: keys failed by this worker
	MultipartUploads map[string]*multipartUploadState `json:"multipart_uploads,omitempty"`
}

Stat has the counters to represent the progress.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL