Documentation
¶
Index ¶
- Constants
- func CopyData(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error)
- func InitForCopyData()
- func ListAll(store object.ObjectStorage, prefix, start, end string, followLink bool) (<-chan object.Object, error)
- func Sync(src, dst object.ObjectStorage, config *Config) error
- type Checkpoint
- type CheckpointManager
- func (m *CheckpointManager) AddPendingKey(prefix string, obj object.Object)
- func (m *CheckpointManager) DeleteCheckpoint() error
- func (m *CheckpointManager) GetLastListedKey(prefix string) string
- func (m *CheckpointManager) GetOrCreatePrefixState(prefix string) *PrefixState
- func (m *CheckpointManager) ListPrefixes() []string
- func (m *CheckpointManager) Load() (*Checkpoint, error)
- func (m *CheckpointManager) MarkCompleted(key string)
- func (m *CheckpointManager) MarkFailed(key string)
- func (m *CheckpointManager) MarkListDone(prefix string)
- func (m *CheckpointManager) RegisterChildPrefix(childPrefix string, listDepth int)
- func (m *CheckpointManager) Reset(config *Config)
- func (m *CheckpointManager) RestorePrefix(prefix string) (objs []object.Object, listDone bool, listDepth int, found bool)
- func (m *CheckpointManager) Save(ckpt *Checkpoint) error
- func (m *CheckpointManager) SaveOnSignal()
- func (m *CheckpointManager) StartPeriodicSave(interval time.Duration)
- func (m *CheckpointManager) Stop()
- func (m *CheckpointManager) TrackKey(key, prefix string)
- func (m *CheckpointManager) UpdateLastListedKey(prefix string, obj object.Object)
- func (m *CheckpointManager) ValidateConfig(current *Config) bool
- type CheckpointStats
- type Config
- type PrefixState
- type Stat
Constants ¶
const JFS_UMASK = "JFS_UMASK"
Variables ¶
This section is empty.
Functions ¶
func InitForCopyData ¶ added in v0.1.3
func InitForCopyData()
Types ¶
type Checkpoint ¶
type Checkpoint struct {
sync.RWMutex
PrefixState map[string]*PrefixState `json:"prefix_state"`
Config *Config `json:"config"`
Stats CheckpointStats `json:"stats"`
SrcDelayDel []string `json:"src_delay_del,omitempty"`
DstDelayDel []string `json:"dst_delay_del,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
Checkpoint represents the complete checkpoint state
type CheckpointManager ¶
type CheckpointManager struct {
// contains filtered or unexported fields
}
CheckpointManager manages checkpoint persistence
func NewCheckpointManager ¶
func NewCheckpointManager(src, dst object.ObjectStorage, config *Config) *CheckpointManager
NewCheckpointManager creates a new checkpoint manager
func (*CheckpointManager) AddPendingKey ¶
func (m *CheckpointManager) AddPendingKey(prefix string, obj object.Object)
func (*CheckpointManager) DeleteCheckpoint ¶
func (m *CheckpointManager) DeleteCheckpoint() error
DeleteCheckpoint removes the checkpoint file from storage.
func (*CheckpointManager) GetLastListedKey ¶
func (m *CheckpointManager) GetLastListedKey(prefix string) string
GetLastListedKey returns the last listed key for a prefix, or "" if not tracked.
func (*CheckpointManager) GetOrCreatePrefixState ¶
func (m *CheckpointManager) GetOrCreatePrefixState(prefix string) *PrefixState
GetOrCreatePrefixState gets or creates a prefix state
func (*CheckpointManager) ListPrefixes ¶
func (m *CheckpointManager) ListPrefixes() []string
ListPrefixes returns a snapshot of all prefix keys currently tracked in checkpoint.
func (*CheckpointManager) Load ¶
func (m *CheckpointManager) Load() (*Checkpoint, error)
Load loads checkpoint from object storage
func (*CheckpointManager) MarkCompleted ¶
func (m *CheckpointManager) MarkCompleted(key string)
MarkCompleted removes a key from PendingKeys after successful completion
func (*CheckpointManager) MarkFailed ¶
func (m *CheckpointManager) MarkFailed(key string)
MarkFailed moves a key from PendingKeys to FailedKeys
func (*CheckpointManager) MarkListDone ¶
func (m *CheckpointManager) MarkListDone(prefix string)
func (*CheckpointManager) RegisterChildPrefix ¶
func (m *CheckpointManager) RegisterChildPrefix(childPrefix string, listDepth int)
RegisterChildPrefix registers a child prefix discovered during listing.
func (*CheckpointManager) Reset ¶
func (m *CheckpointManager) Reset(config *Config)
Reset discards the current checkpoint and starts fresh with the given config.
func (*CheckpointManager) RestorePrefix ¶
func (m *CheckpointManager) RestorePrefix(prefix string) (objs []object.Object, listDone bool, listDepth int, found bool)
RestorePrefix restores pending+failed keys for a prefix, merging failed into pending.
func (*CheckpointManager) Save ¶
func (m *CheckpointManager) Save(ckpt *Checkpoint) error
Save saves checkpoint to object storage
func (*CheckpointManager) SaveOnSignal ¶
func (m *CheckpointManager) SaveOnSignal()
func (*CheckpointManager) StartPeriodicSave ¶
func (m *CheckpointManager) StartPeriodicSave(interval time.Duration)
func (*CheckpointManager) Stop ¶
func (m *CheckpointManager) Stop()
func (*CheckpointManager) TrackKey ¶
func (m *CheckpointManager) TrackKey(key, prefix string)
func (*CheckpointManager) UpdateLastListedKey ¶
func (m *CheckpointManager) UpdateLastListedKey(prefix string, obj object.Object)
func (*CheckpointManager) ValidateConfig ¶
func (m *CheckpointManager) ValidateConfig(current *Config) bool
ValidateConfig checks if checkpoint config matches current config
type CheckpointStats ¶
type CheckpointStats struct {
Copied int64 `json:"copied"`
CopiedBytes int64 `json:"copied_bytes"`
Checked int64 `json:"checked"`
CheckedBytes int64 `json:"checked_bytes"`
Deleted int64 `json:"deleted"`
Skipped int64 `json:"skipped"`
SkippedBytes int64 `json:"skipped_bytes"`
Failed int64 `json:"failed"`
Handled int64 `json:"handled"`
}
CheckpointStats stores cumulative statistics
type Config ¶
type Config struct {
StorageClass string
Start string
End string
Threads int
Update bool
ForceUpdate bool
Perms bool
MaxFailure int64
Dry bool
DeleteSrc bool
DeleteSrcAfter bool
DeleteDst bool
MatchFullPath bool
Dirs bool
Exclude []string
Include []string
Existing bool
IgnoreExisting bool
Links bool
Inplace bool
Limit int64
Manager string
Workers []string
ManagerAddr string
ListThreads int
ListDepth int
BWLimit int64
TrafficControlURL string
NoHTTPS bool
Verbose bool
Quiet bool
CheckAll bool
CheckNew bool
CheckChange bool
MaxSize int64
MinSize int64
MaxAge time.Duration
MinAge time.Duration
StartTime time.Time
EndTime time.Time
Env map[string]string `json:"-"`
FilesFrom string
EnableCheckpoint bool
CheckpointInterval time.Duration
CheckpointForceReset bool
Registerer prometheus.Registerer `json:"-"`
// contains filtered or unexported fields
}
func NewConfigFromCli ¶
type PrefixState ¶
type PrefixState struct {
sync.RWMutex
ListDone bool `json:"list_done"`
LastListedKey string `json:"last_listed_key"`
ListDepth int `json:"list_depth"`
PendingKeys map[string]object.Object `json:"-"`
FailedKeys map[string]object.Object `json:"-"`
// contains filtered or unexported fields
}
PrefixState maintains the state for a specific prefix
func (*PrefixState) MarshalJSON ¶
func (s *PrefixState) MarshalJSON() ([]byte, error)
func (*PrefixState) UnmarshalJSON ¶
func (s *PrefixState) UnmarshalJSON(data []byte) error
type Stat ¶
type Stat struct {
Copied int64 // the number of copied files
CopiedBytes int64 // total amount of copied data in bytes
Checked int64 // the number of checked files
CheckedBytes int64 // total amount of checked data in bytes
Deleted int64 // the number of deleted files
Skipped int64 // the number of files skipped
SkippedBytes int64 // total amount of skipped data in bytes
Failed int64 // the number of files that fail to copy
DelayDelDir []string // the directories that need to be deleted
CompletedKeys []string `json:"completed_keys,omitempty"` // checkpoint: keys completed by this worker
FailedKeys []string `json:"failed_keys,omitempty"` // checkpoint: keys failed by this worker
}
Stat has the counters to represent the progress.