Documentation
¶
Overview ¶
Package backup writes and reads commits: a Walker backs a directory up into a set as a commit of trees, files and chunks, and a Restorer writes a commit, a subtree or one file back, downloading only what the destination lacks. Both work against an ObjectStore and an Index, which package goback opens from a storage location.
Index ¶
- Constants
- Variables
- func BlobRef(key *storekey.Key, chunk []byte) *proto.Ref
- func CheckReferences(ctx context.Context, store ObjectStore, obj *proto.Object) error
- func CollectPresence(ctx context.Context, store ObjectStore, tree *proto.Ref) (*presence.Filter, error)
- func CommitKey(ctx context.Context, store Getter, commit *proto.Commit, key *storekey.Key) (*storekey.Key, error)
- func DefaultRestoreWorkers() int
- func FileIdentity(info os.FileInfo) (ctimeNs int64, inode uint64)
- func FileParts(ctx context.Context, store Getter, file *proto.File) ([]*proto.FilePart, error)
- func FindFile(ctx context.Context, idx Index, key *storekey.Key, set, file string, ...) ([]*proto.TreeNode, error)
- func HasAll(ctx context.Context, store ObjectStore, refs []*proto.Ref) ([]bool, error)
- func HashFile(path string, key *storekey.Key) (*proto.Ref, error)
- func HashFileFS(ctx context.Context, fsys FS, path string, key *storekey.Key) (*proto.Ref, error)
- func HoldRestore(ctx context.Context, store ObjectStore, set string, ref *proto.Ref) (held context.Context, end func(), err error)
- func IndexPath(key *storekey.Key, path string) string
- func IsLockError(err error) bool
- func IsPermissionError(err error) bool
- func ListDir(ctx context.Context, lister DirLister, key *storekey.Key, set, dir string, ...) ([]*proto.TreeNode, error)
- func LiveMarkers(root string) ([]string, error)
- func LoadTree(ctx context.Context, store Getter, ref *proto.Ref) (*proto.Tree, error)
- func Missing(ctx context.Context, store ObjectStore, refs []*proto.Ref) ([]*proto.Ref, error)
- func NameToken(key *storekey.Key, parent []byte, name []byte) []byte
- func OpenBlob(key *storekey.Key, sealed *proto.Sealed) ([]byte, error)
- func OpenTree(ctx context.Context, store Getter, ref *proto.Ref, key *storekey.Key, ...) (*proto.Tree, error)
- func OwnerNames(info os.FileInfo) (string, string)
- func PartialPath(path string) string
- func PickCommit(ctx context.Context, idx Index, set string, ref *proto.Ref, notAfter time.Time) (*proto.Commit, error)
- func PutFile(ctx context.Context, store ObjectStore, key *storekey.Key, size int64, ...) (*proto.Ref, error)
- func PutParts(ctx context.Context, store ObjectStore, key *storekey.Key, ...) (*proto.Ref, error)
- func PutTree(ctx context.Context, store ObjectStore, nodes []*proto.TreeNode, ...) (*proto.Ref, error)
- func RecoverKey(ctx context.Context, escrow EscrowLister, passphrase string) (*storekey.Key, error)
- func References(obj *proto.Object) []*proto.Ref
- func SealBlob(key *storekey.Key, chunk []byte) *proto.Sealed
- func SetID(s string) (id uint64, ok bool)
- func SortNodes(nodes []*proto.TreeNode) []*proto.TreeNode
- func SubFiles(ctx context.Context, store Getter, file *proto.File, ...) error
- func TraverseTree(ctx context.Context, store ObjectStore, tree *proto.Object, workers int, ...) error
- func WithSession(ctx context.Context, s *Session) context.Context
- func WithWritten(ctx context.Context, written func(n int64)) context.Context
- type ArchiveWalker
- type BackupReader
- func (br *BackupReader) ForCommit(ctx context.Context, commit *proto.Commit) (*BackupReader, error)
- func (br *BackupReader) GetTree(ctx context.Context, ref *proto.Ref, parts []string) (*proto.Ref, []byte, error)
- func (br *BackupReader) ReadFile(ctx context.Context, ref *proto.Ref) (io.ReadSeeker, error)
- func (br *BackupReader) WalkTree(ctx context.Context, ref *proto.Ref, parent []byte, walkFn WalkFn) error
- func (br *BackupReader) WithKey(key *storekey.Key) *BackupReader
- type CommitGate
- type CommitGrant
- type CommitSizer
- type Confirmer
- type Counter
- type DamageIndex
- type DamageReport
- type DirLister
- type Eraser
- type EscrowLister
- type EscrowedKey
- type FS
- type File
- type FilePath
- type FileRead
- type FileToRestore
- type FilesReader
- type Getter
- type Growing
- type HasAller
- type HeadKeeper
- type HeaderWalker
- type Hole
- type Index
- type Keeper
- type KeyEscrow
- type Leased
- type LocalFS
- func (LocalFS) Chmod(_ context.Context, path string, mode os.FileMode) error
- func (LocalFS) Chtimes(_ context.Context, path string, mtime time.Time) error
- func (LocalFS) Join(dir, name string) string
- func (LocalFS) Lstat(_ context.Context, path string) (os.FileInfo, error)
- func (LocalFS) MkdirAll(_ context.Context, dir string) error
- func (LocalFS) Open(_ context.Context, path string) (File, error)
- func (LocalFS) OpenFile(_ context.Context, path string) (RandomFile, error)
- func (LocalFS) Owners(info os.FileInfo) (string, string)
- func (LocalFS) ReadDir(_ context.Context, dir string) ([]fs.DirEntry, error)
- func (LocalFS) Readlink(_ context.Context, path string) (string, error)
- func (LocalFS) Remove(_ context.Context, path string) error
- func (LocalFS) RemoveAll(_ context.Context, path string) error
- func (LocalFS) Rename(_ context.Context, from, to string) error
- func (LocalFS) Split(path string) (string, string)
- func (LocalFS) Stat(_ context.Context, path string) (os.FileInfo, error)
- func (LocalFS) Symlink(_ context.Context, target, path string) error
- func (LocalFS) Traits() Traits
- func (LocalFS) WriteFile(_ context.Context, path string, r io.Reader) (int64, error)
- type ObjectReceiver
- type ObjectStore
- type Outcome
- type OverwriteMode
- type PartReader
- type PolicySource
- type PresenceIndex
- type PresenceScope
- type PresenceSource
- type RandomFile
- type RandomWriteFS
- type ReIndexReport
- type RecordLocator
- type RecordReader
- type RefScope
- type RestoreLeaser
- type RestoreStats
- type RestoredFile
- type Restorer
- func (r *Restorer) ReadFile(ctx context.Context, stat *proto.FileInfo, ref *proto.Ref) ([]byte, error)
- func (r *Restorer) RestoreFile(ctx context.Context, path string, stat *proto.FileInfo, ref *proto.Ref) (Outcome, error)
- func (r *Restorer) RestoreFiles(ctx context.Context, files []FileToRestore, ...) error
- func (r *Restorer) RestorePath(ctx context.Context, idx Index, set, path string, notAfter time.Time, ...) (*proto.TreeNode, Outcome, error)
- func (r *Restorer) RestoreTree(ctx context.Context, commit *proto.Commit, dir string, opts TreeOptions) error
- func (r *Restorer) Stats() RestoreStats
- type Retention
- type Retirer
- type Revival
- type Reviver
- type SeedMap
- type Session
- type SessionStore
- type SetScope
- type StatCache
- type Stream
- type Traits
- type TraverserHandler
- type TreeFetcher
- type TreeOptions
- type TreeWriter
- type WalkFn
- type WalkResult
- type Walker
- type WriteFS
Constants ¶
const ( KindPath = "path" KindPostgresBase = "postgres-base" KindPostgresWAL = "postgres-wal" KindMySQLDump = "mysql-dump" KindMySQLBinlog = "mysql-binlog" )
The kinds of commit MetaKind names.
const DefaultProgressInterval = 15 * time.Second
DefaultProgressInterval is how often a walk reports when the caller wants progress but names no interval.
const DefaultScanWindow = 50_000
DefaultScanWindow is how many entries a walk holds for directories it has not reached yet when the caller names no window. An entry costs a couple of hundred bytes, so this is tens of megabytes whatever the root holds.
const DefaultScanWorkers = 4
DefaultScanWorkers is how many directories a walk lists ahead of itself when the caller names no number. A metadata-bound walk of 216,000 files takes 31s listing on the walk, 19s with one lister and 9s with four; eight is no better and sixteen is worse.
const DefaultWindowBytes = 64 << 20
DefaultWindowBytes bounds the chunk bytes an agent keeps for files whose assumed parts the store has not confirmed yet.
const MaxCountPeriods = 1000
MaxCountPeriods is the most periods a CountCommits range may span.
const MaxEscrow = 64 << 10
MaxEscrow is far above what an escrowed key file takes up.
const MaxFilesPerRead = 1024
MaxFilesPerRead is the most files one ReadFiles call names.
const MetaKind = "goback.kind"
MetaKind is the commit metadata key naming what made the commit, one of the Kind values.
const PreviousVersions = 3
PreviousVersions is how many live versions of a changed file the walker diffs its chunks against.
const ( // SplitFileSize is the smallest content size whose file object may be // split into sub-file objects. SplitFileSize = 4096 * chunker.MinSize )
Variables ¶
var ( // ErrEmptyBuffer is returned when trying to read from a backup file into // an empty buffer ErrEmptyBuffer = errors.New("no buffer space provided") // ErrIllegalOffset is returned when trying to seek past the end of a file ErrIllegalOffset = errors.New("illegal offset") // ErrSkipFile can be returned during backup to make the TreeWrite skip/ignore a file ErrSkipFile = errors.New("skip file") )
var ( // ErrOtherKeyEscrowed is returned when a store is given the escrowed // copy of a store key other than the one it already keeps. ErrOtherKeyEscrowed = errors.New("the store already keeps the escrowed copy of another store key") // ErrInvalidEscrow is returned for an escrowed key that is not an // armored age file of a sensible size, or has no valid key id. ErrInvalidEscrow = errors.New("not an escrowed store key") )
var ( // ErrNotImplemented is returned by a store or index that lacks the // requested feature. ErrNotImplemented = errors.New("the store doesn't implement this feature") // ErrNotFound is returned when nothing matches the given ref or set. ErrNotFound = errors.New("the requested object was not found") // ErrDanglingRef is returned when an object references objects the store // does not hold. ErrDanglingRef = errors.New("object references missing objects") // ErrSetClosed is returned for a commit into a set that is being // deleted. ErrSetClosed = errors.New("set is closed") // ErrTombstoned is returned when a commit or pin already has a // tombstone; there is no undo past that point. ErrTombstoned = errors.New("object is tombstoned") // ErrNewestCommit is returned when deleting the only live commit of an // active set; delete the set instead. ErrNewestCommit = errors.New("the only commit of a set cannot be deleted") // ErrPinned refuses the deletion of a commit a live pin holds. ErrPinned = errors.New("commit is pinned") // ErrCommitDenied is returned by BeginCommit when the store refuses the // run, with the reason. ErrCommitDenied = errors.New("commit refused") // ErrSetName refuses a set name that reads as a set id. ErrSetName = errors.New("a set name cannot be only digits, which name a set by id") // ErrInvalidCount refuses a CountCommits call with an unknown period or // time zone, an empty range, or more than MaxCountPeriods periods. ErrInvalidCount = errors.New("invalid commit count") // ErrSessionLost is returned for a commit whose session lost objects it // had already been told were stored; backing up again stores them anew. ErrSessionLost = errors.New("the session lost objects it had stored") // ErrQuotaExceeded is returned when the store refuses a read because // the caller used up what it may read; retrying does not help until // the quota is raised or renews. ErrQuotaExceeded = errors.New("read quota exceeded") )
var ErrNoSession = errors.New("no such session")
ErrNoSession is returned when a session id names no live session.
var ErrSealChanged = errors.New("the set's latest commit is sealed under another store key or none")
ErrSealChanged is returned by a Walker with Carry over a previous commit sealed under another store key or none, whose entries it cannot carry.
var SkipTree = errors.New("skip tree")
SkipTree, returned by a TraverserHandler for a directory, leaves that subtree unvisited.
Functions ¶
func CheckReferences ¶
CheckReferences fails with ErrDanglingRef when the store lacks any object the given one references. Indexes call it before accepting a Put so a stored tree always has its children.
func CollectPresence ¶
func CollectPresence(ctx context.Context, store ObjectStore, tree *proto.Ref) (*presence.Filter, error)
CollectPresence walks a tree and returns a filter over every blob ref it reaches.
func CommitKey ¶
func CommitKey(ctx context.Context, store Getter, commit *proto.Commit, key *storekey.Key) (*storekey.Key, error)
CommitKey returns the key to open commit's tree with: key for a commit sealed under a store key, nil for one written plain. A commit that does not record its key is tried with key and opened plain when that fails.
func DefaultRestoreWorkers ¶
func DefaultRestoreWorkers() int
DefaultRestoreWorkers is how much of a restore runs at once when nothing says otherwise: twice the cpu count, since a file spends most of its time waiting on the store or on the disk rather than on a core.
func FileIdentity ¶
FileIdentity returns the change time and inode, the signals the stat cache adds on top of mtime and size.
func FileParts ¶
FileParts returns a file's parts with its splits flattened, in order; an inline file has none.
func FindFile ¶
func FindFile(ctx context.Context, idx Index, key *storekey.Key, set, file string, notAfter time.Time, count int) ([]*proto.TreeNode, error)
FindFile lists up to count versions of the file at the slash-separated plaintext path in set, none newer than notAfter, newest first, with their stats opened under key.
func HasAll ¶
HasAll reports Has for each of refs, in their order, through the store's HasAller when it is one.
func HashFile ¶
HashFile computes the File ref a backup of path on the local disk would produce without storing anything.
func HashFileFS ¶
HashFileFS computes the File ref a backup of path on fsys would produce without storing anything.
func HoldRestore ¶
func HoldRestore(ctx context.Context, store ObjectStore, set string, ref *proto.Ref) (held context.Context, end func(), err error)
HoldRestore opens a restore session naming ref when store keeps sessions, so retirement and garbage collection keep it until end is called; read through held meanwhile. Without sessions it holds nothing.
func IndexPath ¶
IndexPath converts a slash-separated plaintext path into the path the index stores: name tokens computed top-down when the store is encrypted, escaped plain names otherwise.
func IsLockError ¶
IsLockError reports whether a read failed because another process holds the file; never on Unix.
func IsPermissionError ¶
IsPermissionError reports whether an open or list failed for lack of access.
func ListDir ¶
func ListDir(ctx context.Context, lister DirLister, key *storekey.Key, set, dir string, notAfter time.Time) ([]*proto.TreeNode, error)
ListDir lists what set held directly under the slash-separated plaintext dir at notAfter, as DirLister.ReadDir does, with the stats opened under key.
func LiveMarkers ¶
LiveMarkers returns the session lock files under root that another process holds, which is how a running game server shows itself.
func NameToken ¶
NameToken is the stored form of a directory entry's name, which is the parent token of everything inside that directory.
func OpenBlob ¶
OpenBlob reverses SealBlob, refusing a blob sealed under another key or one whose plaintext does not name its ref.
func OpenTree ¶
func OpenTree(ctx context.Context, store Getter, ref *proto.Ref, key *storekey.Key, parent []byte) (*proto.Tree, error)
OpenTree loads a tree and opens its names for the directory whose token is parent. Without a key it is LoadTree.
func OwnerNames ¶
OwnerNames resolves the owning user and group names, empty when unknown.
func PartialPath ¶
PartialPath is the file RestoreFile assembles path in on the local disk, next to it.
func PickCommit ¶
func PickCommit(ctx context.Context, idx Index, set string, ref *proto.Ref, notAfter time.Time) (*proto.Commit, error)
PickCommit is the commit of set that ref names, or, when ref is nil, the set's newest commit at or before notAfter.
func PutFile ¶
func PutFile(ctx context.Context, store ObjectStore, key *storekey.Key, size int64, content io.Reader) (*proto.Ref, error)
PutFile stores size bytes of content as a file, cut and sealed under key as a backup would, and returns the ref a tree node should carry.
func PutParts ¶
func PutParts(ctx context.Context, store ObjectStore, key *storekey.Key, parts []*proto.FilePart) (*proto.Ref, error)
PutParts stores a file over parts that store already holds, as a backup that cut those parts would, and returns the ref a tree node should carry. parts must not be empty; an empty file is stored with PutFile.
func PutTree ¶
func PutTree(ctx context.Context, store ObjectStore, nodes []*proto.TreeNode, key *storekey.Key, parent []byte) (*proto.Ref, error)
PutTree stores a directory's nodes as one tree object, or as split trees under a parent when the directory is large, and returns the ref of the object a parent node should reference. With a key the nodes' names are sealed for the directory whose token is parent; the given nodes are not modified.
func RecoverKey ¶
RecoverKey opens the store key with the first escrowed copy the passphrase opens.
func References ¶
References lists the refs an object points at directly: a commit's tree, a tree's nodes and splits, a file's parts, a pin's target.
func SetID ¶
SetID reads s as a set id, which a set given by digits alone is; ok is false for a set name. Digits too many for an id give id 0, which no set has.
func SubFiles ¶
func SubFiles(ctx context.Context, store Getter, file *proto.File, visit func(ref *proto.Ref) error) error
SubFiles calls visit with every file object of a split file's tree below file, loading only the ones that are split themselves.
func TraverseTree ¶
func TraverseTree(ctx context.Context, store ObjectStore, tree *proto.Object, workers int, handler TraverserHandler) error
TraverseTree visits every node under tree, calling handler from workers goroutines at once; an error other than SkipTree stops the traversal.
func WithSession ¶
WithSession attaches the session to the context.
Types ¶
type ArchiveWalker ¶
type ArchiveWalker interface {
// WalkArchives calls fn with every object of type t, loaded, in the
// archives but the finished ones skip reports true for, and returns
// the names of all the finished archives it walked or skipped.
WalkArchives(ctx context.Context, t proto.ObjectType, skip func(archive string) bool, fn ObjectReceiver) ([]string, error)
}
ArchiveWalker is implemented by stores that keep objects in archives which, once finished, never change and whose names are never reused.
type BackupReader ¶
type BackupReader struct {
// contains filtered or unexported fields
}
BackupReader reads the files and trees of a commit back from a store.
func NewBackupReader ¶
func NewBackupReader(store ObjectStore) *BackupReader
NewBackupReader returns a reader for a plaintext store; WithKey adds the store key.
func (*BackupReader) ForCommit ¶
func (br *BackupReader) ForCommit(ctx context.Context, commit *proto.Commit) (*BackupReader, error)
ForCommit returns a reader for commit's tree: one that opens it with the store key when the commit was sealed under it, and plain when it was written without one.
func (*BackupReader) GetTree ¶
func (br *BackupReader) GetTree(ctx context.Context, ref *proto.Ref, parts []string) (*proto.Ref, []byte, error)
GetTree descends from ref along the plaintext path parts and returns the tree ref at the end, with the token of the directory it names.
func (*BackupReader) ReadFile ¶
func (br *BackupReader) ReadFile(ctx context.Context, ref *proto.Ref) (io.ReadSeeker, error)
ReadFile opens the file object at ref for seekable reading.
func (*BackupReader) WalkTree ¶
func (br *BackupReader) WalkTree(ctx context.Context, ref *proto.Ref, parent []byte, walkFn WalkFn) error
WalkTree walks the tree at ref, whose directory has the given token (nil for the root of a commit). A store that can stream a subtree hands over every tree below ref in one call.
func (*BackupReader) WithKey ¶
func (br *BackupReader) WithKey(key *storekey.Key) *BackupReader
WithKey returns a reader that opens names and blobs with the store key.
type CommitGate ¶
type CommitGate interface {
// BeginCommit returns the grant for a run on set, or ErrCommitDenied.
BeginCommit(ctx context.Context, set string) (*CommitGrant, error)
}
CommitGate is asked before a run whether the caller may commit to a set. A run asks inside its session, if it has one.
type CommitGrant ¶
type CommitGrant struct {
// Policy is the store's write policy for this run; nil when the server
// holds none, which leaves the agent's key file in charge.
Policy *storekey.Policy
// Rescan asks the run to read every file again, because the store lost
// content it could not name a path for.
Rescan bool
// Damaged names the paths whose stored content the store lost. A run
// reads them again whatever its change detection says, and takes no
// chunk of theirs on trust.
Damaged []string
}
CommitGrant is the answer to an allowed BeginCommit.
type CommitSizer ¶
type CommitSizer interface {
// CommitSizes is CommitInfo with the size of each commit beside it, in
// the same order.
CommitSizes(ctx context.Context, set string, notAfter time.Time, count int) ([]*proto.Commit, []*proto.CommitSize, error)
}
CommitSizer is implemented by indexes that measure how big a set was at each commit.
type Confirmer ¶
type Confirmer interface {
// PutFile stores obj when every assumed ref is held; otherwise it
// returns the refs that are not.
PutFile(ctx context.Context, obj *proto.Object, assumed []*proto.Ref) (missing []*proto.Ref, err error)
}
Confirmer stores a file object once the store holds every part the caller assumed; otherwise it stores nothing and names the missing refs.
type Counter ¶
type Counter interface {
// Count reports how many objects the store holds, in total and distinct.
Count() (total uint64, unique uint64, err error)
}
Counter is implemented by stores that can count their objects.
type DamageIndex ¶
type DamageIndex interface {
// PathsOfFiles returns where the given file objects are stored; a file
// object no live version references any more yields nothing.
PathsOfFiles(ctx context.Context, refs []*proto.Ref) ([]FilePath, error)
// MarkDamaged records paths of the set a run must read again.
MarkDamaged(ctx context.Context, setID int64, paths []string) error
// MarkLost records versions no run can produce again, so a restore of
// the commits holding them says so rather than failing.
MarkLost(ctx context.Context, versions []FilePath) error
// MarkRescan records that a whole set must be read again.
MarkRescan(ctx context.Context, setID int64) error
}
DamageIndex records what a store lost, so the next backup of a set reads those paths again instead of trusting that they are unchanged.
type DamageReport ¶
type DamageReport struct {
// Paths are the paths marked for a re-read, by set id.
Paths map[int64][]string
// Rescan is the sets marked for a full re-read, because some loss
// could not be placed at a path.
Rescan []int64
// Unplaced are the lost objects no live path holds.
Unplaced []*proto.Ref
// Lost are the closed versions nothing can produce again; the commits
// holding them are damaged for that path.
Lost []FilePath
}
DamageReport is what ReportDamage made of a store's losses.
func ReportDamage ¶
func ReportDamage(ctx context.Context, store ObjectStore, idx DamageIndex, sets []int64, lost []*proto.Ref) (*DamageReport, error)
ReportDamage places a store's lost objects at the paths that held them and records those, so the next backup of each set reads them again. A loss it cannot place — a lost file or tree object, or a blob whose file object is gone too — marks its sets for a full re-read instead, which is the only way left to find it.
type DirLister ¶
type DirLister interface {
// ReadDir lists what the set held directly under dir at notAfter,
// sorted by name. Directory entries carry a name and a ref only, since
// the index keeps no stat for them. An unknown set lists nothing.
ReadDir(ctx context.Context, set string, dir string, notAfter time.Time) ([]*proto.TreeNode, error)
}
DirLister is implemented by indexes that can list a directory as it stood, without walking the stored trees.
type Eraser ¶
type Eraser interface {
// Erase tombstones the ref as an erasure.
Erase(context.Context, *proto.Ref) error
}
Eraser deletes as an erasure: the tombstone asks garbage collection to rewrite the archives holding the target's objects as soon as its rules allow, instead of waiting for the dead ratio or the erasure bound.
type EscrowLister ¶
type EscrowLister interface {
// EscrowedKeys returns every copy kept.
EscrowedKeys(ctx context.Context) ([]EscrowedKey, error)
}
EscrowLister reads the escrowed copies of a store's key.
type EscrowedKey ¶
EscrowedKey is a store key escrowed under a passphrase, as goback key escrow prints it, with the id of the key inside.
func (EscrowedKey) Validate ¶
func (k EscrowedKey) Validate() error
Validate reports ErrInvalidEscrow unless the key has a key id and is an armored age file of at most MaxEscrow bytes.
type FS ¶
type FS interface {
// ReadDir lists dir sorted by name, without following symlinks. Info on
// an entry may stat it; the walker only asks for entries it includes.
ReadDir(ctx context.Context, dir string) ([]fs.DirEntry, error)
// Stat stats path, following a symlink; the walker uses it on Root.
Stat(ctx context.Context, path string) (os.FileInfo, error)
Lstat(ctx context.Context, path string) (os.FileInfo, error)
Open(ctx context.Context, path string) (File, error)
Readlink(ctx context.Context, path string) (string, error)
Join(dir, name string) string
// Owners names a file's user and group; empty when unknown.
Owners(info os.FileInfo) (user, group string)
// Traits is read once as a walk starts.
Traits() Traits
}
FS is the file tree a Walker reads. Paths are the FS's own strings: Join builds them from the Walker's Root, and the walker passes them back unchanged. An error for lack of access wraps fs.ErrPermission, so the walk keeps the previous version instead of failing.
type File ¶
type File interface {
io.ReadCloser
Stat() (os.FileInfo, error)
}
File is an open file of an FS. One that is also an io.ReaderAt lets the walker re-read a single part rather than the whole file.
type FilePath ¶
type FilePath struct {
Ref *proto.Ref
SetID int64
// Set is the set's name, for reporting; SetID is what identifies it.
Set string
Path string
// Open reports whether this is the version the set's newest commit
// points at, which is the only one an agent can produce again.
Open bool
}
FilePath is where a file object is stored in a set.
type FileToRestore ¶
type FileToRestore struct {
Path string
Stat *proto.FileInfo
Ref *proto.Ref
// Written, when set, is handed the size of each part as it lands in
// the file, as WithWritten does for RestoreFile.
Written func(n int64)
}
FileToRestore is one file of a RestoreFiles call.
type FilesReader ¶
type FilesReader interface {
// ReadFiles calls object with each file's object in order, then,
// unless objectsOnly, part with every part of each file not in its
// Skip, in any order and possibly from several goroutines at once. A
// part several of the files hold is handed over once, for any of them.
ReadFiles(ctx context.Context, files []FileRead, objectsOnly bool, object func(file int, obj *proto.Object) error, part func(file, index int, obj *proto.Object) error) error
}
FilesReader reads many files in one call: their file objects and the stored objects of their parts, sealed when the store is encrypted.
type Getter ¶
type Getter interface {
// Get returns the object at the ref, or ErrNotFound.
Get(context.Context, *proto.Ref) (*proto.Object, error)
}
Getter is the read side of an ObjectStore.
type Growing ¶
type Growing struct {
// Name is the file's name in the commit's root; it replaces any entry
// of that name.
Name string
// Path is where the file is on disk.
Path string
// Size is how many bytes of it the commit holds.
Size int64
// From names the previous commit's root entry this file continues;
// empty means Name.
From string
// ModTime, when set, is recorded as the file's modification time in
// place of the one on disk, which its writer keeps changing.
ModTime time.Time
}
Growing is a file a walk includes up to a length, such as a log a receiver is still appending to. Its writer guarantees that the first Size bytes only ever grow: a walk keeps the parts of the previous version it continues, except the last, and reads only from the start of that part.
type HasAller ¶
type HasAller interface {
// HasAll reports Has for each of refs, in their order.
HasAll(ctx context.Context, refs []*proto.Ref) ([]bool, error)
}
HasAller is implemented by stores that answer Has for many refs at once.
type HeadKeeper ¶
type HeadKeeper interface {
// AdvanceHead points the commit's set at it, unless the set already
// points at one received later.
AdvanceHead(commit *proto.Object) error
}
HeadKeeper is a store that keeps each set's latest complete commit beside its objects, so a restore finds the set without an index.
type HeaderWalker ¶
type HeaderWalker interface {
// WalkHeaders calls fn for every header of type t; an error from fn
// stops the walk.
WalkHeaders(ctx context.Context, t proto.ObjectType, fn func(*proto.ObjectHeader) error) error
}
HeaderWalker visits object headers without loading bodies. Tombstones have no body and are only reachable this way.
type Hole ¶
type Hole struct {
Path string
Offset int64
Length int64
// Ref is the part that could not be read.
Ref *proto.Ref
}
A Hole is a range of a salvaged file that no source could fill; the restored file holds zeroes there.
type Index ¶
type Index interface {
// Open readies the index for use.
Open() error
// Close releases the index.
Close() error
ObjectStore
// FileInfo lists the versions of name that live commits of the set hold,
// newest first, none newer than notAfter, at most count.
FileInfo(ctx context.Context, set string, name string, notAfter time.Time, count int) ([]*proto.TreeNode, error)
// CommitInfo lists the live, complete commits of the set, newest first,
// none newer than notAfter, at most count.
CommitInfo(ctx context.Context, set string, notAfter time.Time, count int) ([]*proto.Commit, error)
// LatestCommit returns the ref of the set's newest commit, or ErrNotFound.
LatestCommit(ctx context.Context, set string) (*proto.Ref, error)
// ReIndex rebuilds the index from the store's objects.
ReIndex(ctx context.Context) (ReIndexReport, error)
}
An Index stamps a commit or pin it receives with the receipt time, and a commit with its set id, before storing it; the caller reads the ref off the stamped object afterwards. An object that already carries a receipt time is a replay and is stored as it is.
type Keeper ¶
Keeper is implemented by object stores that keep copies of what they serve. Keep hands one an object written around it, such as the commit a walker writes through the index.
type KeyEscrow ¶
type KeyEscrow interface {
// PutEscrowedKey keeps one escrowed copy, once: nothing kept is ever
// replaced. A store keeps copies of one store key only.
PutEscrowedKey(ctx context.Context, key EscrowedKey) error
EscrowLister
}
KeyEscrow keeps a store's escrowed store key beside its data, so the key survives anything the data survives. The store cannot open it.
type Leased ¶
type Leased interface {
// SessionLease is how long a session lives without a write.
SessionLease() time.Duration
}
Leased is a session store that ends sessions after a lease without a write.
type LocalFS ¶
type LocalFS struct{}
LocalFS is the local disk, the FS a Walker reads when its FS is nil. Paths are the operating system's.
func (LocalFS) Rename ¶
Rename implements WriteFS. It makes the rename durable, and replaces a read-only file, which Windows refuses to, by making it writable first.
type ObjectReceiver ¶
ObjectReceiver is called for every object a Walk visits; a non-nil error stops the walk.
type ObjectStore ¶
type ObjectStore interface {
// Put stores the object.
Put(context.Context, *proto.Object) error
// Get returns the object at the ref, or ErrNotFound.
Get(context.Context, *proto.Ref) (*proto.Object, error)
// Delete removes the object at the ref.
Delete(context.Context, *proto.Ref) error
// Walk calls the receiver for every object of the type, or of every
// type when it is INVALID; the bool says whether objects are decoded.
Walk(context.Context, bool, proto.ObjectType, ObjectReceiver) error
// Has reports whether the store holds the ref.
Has(context.Context, *proto.Ref) (bool, error)
}
ObjectStore is content-addressed storage of objects by ref.
type Outcome ¶
type Outcome int
Outcome is what RestoreFile did with one file.
const ( // OutcomeWritten means the file was written. OutcomeWritten Outcome = iota // OutcomeUnchanged means the destination already held every part and // only its stat was applied. OutcomeUnchanged // OutcomeSkipped means the destination matched by size and mtime under // OverwriteIfChanged and was left unread. OutcomeSkipped // OutcomeWouldWrite means a dry run would have written the file. OutcomeWouldWrite // OutcomeSalvaged means the file was written with holes where parts // could not be read. OutcomeSalvaged )
type OverwriteMode ¶
type OverwriteMode int
OverwriteMode says how an existing destination file is treated.
const ( // OverwriteAlways hashes every part of the existing file and rewrites // the ones that differ. OverwriteAlways OverwriteMode = iota // OverwriteIfChanged trusts a size and mtime match and leaves such a // file unread. OverwriteIfChanged )
func ParseOverwriteMode ¶
func ParseOverwriteMode(name string) (OverwriteMode, error)
ParseOverwriteMode reads "always" or "if-changed".
func (OverwriteMode) String ¶
func (m OverwriteMode) String() string
String is the mode's name, as ParseOverwriteMode reads it.
type PartReader ¶
type PartReader interface {
// ReadParts calls fn with every part of file whose index is not in
// skip, in any order and possibly from several goroutines at once.
ReadParts(ctx context.Context, file *proto.Ref, skip []int, fn func(index int, obj *proto.Object) error) error
}
PartReader streams the stored objects of a file's parts, skipping the given part indexes; the objects are as uploaded, sealed when the store is encrypted.
type PolicySource ¶
type PolicySource interface {
// StorePolicy returns the store's policy, or nil.
StorePolicy(ctx context.Context) (*storekey.Policy, error)
}
PolicySource knows the store's policy; nil means the store has none set.
type PresenceIndex ¶
type PresenceIndex interface {
// Presence returns the filters of scope; set matters for PresenceSet.
Presence(ctx context.Context, scope PresenceScope, set string) ([]*proto.PresenceFilter, error)
}
PresenceIndex serves the filters of a scope.
type PresenceScope ¶
type PresenceScope int
PresenceScope says whose commits feed the filters an agent receives.
const ( // PresenceOff hands out no filters. PresenceOff PresenceScope = iota // PresenceSet covers the agent's own set. PresenceSet // PresenceStore covers every set of the store. PresenceStore )
func ParsePresenceScope ¶
func ParsePresenceScope(s string) (PresenceScope, error)
ParsePresenceScope reads off, set or store.
func (PresenceScope) String ¶
func (s PresenceScope) String() string
String implements fmt.Stringer.
type PresenceSource ¶
type PresenceSource interface {
// Presence returns the filters for a run on set.
Presence(ctx context.Context, set string) (presence.Set, error)
}
PresenceSource hands an agent the filters of its scope.
type RandomFile ¶
type RandomFile interface {
io.ReaderAt
io.WriterAt
Truncate(size int64) error
// Sync makes what was written durable.
Sync() error
Stat() (os.FileInfo, error)
Close() error
}
RandomFile is a file of a RandomWriteFS. WriteAt may be called from several goroutines at once.
type RandomWriteFS ¶
type RandomWriteFS interface {
WriteFS
// Open opens path to read; a File that is no io.ReaderAt is read only
// front to back.
Open(ctx context.Context, path string) (File, error)
// OpenFile opens path to read and write, creating it.
OpenFile(ctx context.Context, path string) (RandomFile, error)
}
RandomWriteFS is a WriteFS whose files are read and written at any offset, so a restore takes parts from the file already there, resumes a partial file and fetches the parts of one file in parallel.
type ReIndexReport ¶
type ReIndexReport struct {
// Tied counts the commits received at the same time as their set's
// newest, indexed a microsecond after it.
Tied int
// Behind counts the commits received before their set's newest,
// left out of the index.
Behind int
// Unnamed counts the commits that name no set, indexed under a
// placeholder set.
Unnamed int
}
ReIndexReport is what a rebuild of the index found out of order.
type RecordLocator ¶
type RecordLocator interface {
// LocateRecords returns runs holding the records of refs, each
// record's index naming its ref in refs. A ref it cannot locate is in
// no run and is read the ordinary way. It authorizes nothing: the
// caller has already decided the refs are readable.
LocateRecords(ctx context.Context, refs []*proto.Ref) ([]*proto.LocatedRun, error)
}
A RecordLocator is a store that can say where the stored records of many refs sit, neighbours under one location, so a reader fetches them itself in a few range reads.
type RecordReader ¶
type RecordReader interface {
// ReadRecords returns the object of each of refs it could read, nil
// for the rest, which are read the ordinary way. It authorizes
// nothing.
ReadRecords(ctx context.Context, refs []*proto.Ref) ([]*proto.Object, error)
}
A RecordReader is a store that reads the records of many refs together, a range read for each run of neighbours.
type RefScope ¶
type RefScope interface {
// References reports whether a set of the store references ref.
References(ctx context.Context, ref *proto.Ref) (bool, error)
// ReferencesAll reports the same for each of refs, in their order.
ReferencesAll(ctx context.Context, refs []*proto.Ref) ([]bool, error)
}
RefScope is implemented by indexes that know which commit, tree and file refs the store's sets reach.
type RestoreLeaser ¶
type RestoreLeaser interface {
// RestoreLeases returns the refs of every live restore session.
RestoreLeases(ctx context.Context) ([]*proto.Ref, error)
}
RestoreLeaser reports the refs that live restore sessions are reading.
type RestoreStats ¶
type RestoreStats struct {
Files int64 `json:"files"`
Written int64 `json:"written"`
Unchanged int64 `json:"unchanged"`
Skipped int64 `json:"skipped"`
// Salvaged counts files written with holes, and MissingBytes the bytes
// of those holes.
Salvaged int64 `json:"salvaged"`
MissingBytes int64 `json:"missing_bytes"`
// Unsupported counts the symlinks left out of a tree that cannot hold
// them.
Unsupported int64 `json:"unsupported"`
BytesFromDestination int64 `json:"bytes_from_destination"`
BytesFromSeeds int64 `json:"bytes_from_seeds"`
BytesFromCache int64 `json:"bytes_from_cache"`
BytesFromStore int64 `json:"bytes_from_store"`
}
RestoreStats counts what a Restorer did and where the bytes came from.
type RestoredFile ¶
type RestoredFile struct {
Path string
Ref *proto.Ref
Size int64
// Written is how many of its bytes landed in the file; the rest came
// from the file already there.
Written int64
Outcome Outcome
Err error
}
RestoredFile is one file RestoreTree is through with.
type Restorer ¶
type Restorer struct {
Store ObjectStore
Key *storekey.Key
// FS is the tree restored into; nil is the local disk.
FS WriteFS
Seeds *SeedMap
Cache *blobcache.Cache
// Workers bounds how much of a restore runs at once, files and the
// parts of one; 0 means DefaultRestoreWorkers.
Workers int
Overwrite OverwriteMode
// Verify rehashes every written file before it is renamed into place.
Verify bool
// DryRun reports what would change and writes nothing.
DryRun bool
// Salvage writes a file whose parts cannot all be read, leaving the
// unreadable ranges as zeroes, instead of failing it.
Salvage bool
// Rechunk cuts a destination on the local disk that does not match at
// the recorded offsets with the backup's chunker, so parts that only
// moved are taken from it instead of the network. It costs a read of
// the file.
Rechunk bool
// OnHole, when set, is called for every hole a salvaged file was left
// with, possibly from several goroutines at once.
OnHole func(Hole)
// contains filtered or unexported fields
}
Restorer writes files from the store, taking every part it can from the destination file, the seeds and the blob cache before the network. Local bytes are used only after they hash to the recorded ref.
func (*Restorer) ReadFile ¶
func (r *Restorer) ReadFile(ctx context.Context, stat *proto.FileInfo, ref *proto.Ref) ([]byte, error)
ReadFile reads the file ref describes into memory, fetching it as RestoreFile does, so from a store that serves parts only to a restore too. It suits files of a few KiB.
func (*Restorer) RestoreFile ¶
func (r *Restorer) RestoreFile(ctx context.Context, path string, stat *proto.FileInfo, ref *proto.Ref) (Outcome, error)
RestoreFile puts the file ref describes at path with the recorded mode and mtime, assembling it in a partial file next to it and renaming into place. A failed restore leaves the partial file behind, and the next one takes every part from it that still hashes to its ref. On a tree that is no RandomWriteFS, the file is written front to back and neither the destination nor a partial file is read.
func (*Restorer) RestoreFiles ¶
func (r *Restorer) RestoreFiles(ctx context.Context, files []FileToRestore, done func(i int, outcome Outcome, err error) error) error
RestoreFiles restores files as RestoreFile does, Workers at a time, asking a store that is a FilesReader for the small ones in groups and restoring a group it fails file by file. done is called once for every file with its outcome or error, possibly from several goroutines at once; an error it returns stops the restore.
func (*Restorer) RestorePath ¶
func (r *Restorer) RestorePath(ctx context.Context, idx Index, set, path string, notAfter time.Time, dest string) (*proto.TreeNode, Outcome, error)
RestorePath restores to dest the version of the slash-separated file path that set held at notAfter, as the index records it, making dest's directory and holding the file while it is read. It returns the version it restored; a path the set never held at notAfter fails with ErrNotFound.
func (*Restorer) RestoreTree ¶
func (r *Restorer) RestoreTree(ctx context.Context, commit *proto.Commit, dir string, opts TreeOptions) error
RestoreTree restores the commit's tree into dir in place: directories, symlinks, and files as RestoreFiles does. Each directory stays writable for its owner until what it holds is written and then takes the recorded mode and time, deepest first; a directory already there that the tree does not hold is made writable while the restore runs and then given back its mode. A restore session holds the commit while it runs when the store keeps sessions. It refuses a tree naming a path outside dir.
func (*Restorer) Stats ¶
func (r *Restorer) Stats() RestoreStats
Stats returns what the restorer has done so far.
type Retention ¶
type Retention interface {
// DeleteCommit retires a commit into its trash window, after which
// LatestCommit no longer returns it; the only live commit of an active
// set is refused with ErrNewestCommit.
DeleteCommit(ctx context.Context, ref *proto.Ref) error
// UndeleteCommit moves a retired commit back to live; a tombstoned
// one is refused with ErrTombstoned.
UndeleteCommit(ctx context.Context, ref *proto.Ref) error
// TrashedCommits pages through the set's deleted commits that are not
// tombstoned yet, newest deleted first: those deleted before before, or
// all when it is zero, at most limit, every one when limit is zero, and
// then the others deleted at the last one's instant, so its
// DeletedAtNs is the before of the next page.
TrashedCommits(ctx context.Context, set string, before time.Time, limit int) ([]*proto.TrashedCommit, error)
// CountCommits counts the set's live complete commits received in
// [from, to), or with deleted the ones TrashedCommits lists, per period
// as it falls in the IANA zone, "" for UTC. It returns the non-empty
// periods oldest first, each starting at its first instant; the first
// may start before from.
CountCommits(ctx context.Context, set string, period proto.Period, from, to time.Time, zone string, deleted bool) ([]*proto.CommitCount, error)
// DeleteSet closes a set and retires every commit; erase uses a zero
// window.
DeleteSet(ctx context.Context, set string, erase bool) error
// UndeleteSet reopens a closed set and revives its commits.
UndeleteSet(ctx context.Context, set string) error
// Unpin tombstones a pin.
Unpin(ctx context.Context, pin *proto.Ref) error
// Pins lists the live pins.
Pins(ctx context.Context) ([]*proto.PinInfo, error)
}
Retention is implemented by indexes that keep the commit lifecycle: live, then retired, or deleted into a trash window, then tombstoned.
type Retirer ¶
type Retirer interface {
// Retire tombstones the commits whose window has passed at now and
// returns how many.
Retire(ctx context.Context, now time.Time) (int, error)
}
Retirer runs the retirement job: every retired commit past its window gets a tombstone and loses its index rows.
type Revival ¶
type Revival struct {
Commit *proto.Ref
// Missing names some of the objects the commit reaches that the store
// holds no copy of, and MissingCount counts them all.
Missing []*proto.Ref
MissingCount int
}
Revival is what Revive found under one commit.
type Reviver ¶
type Reviver interface {
// Revive checks what each commit reaches and, unless dryRun, takes back
// the tombstones of every commit missing nothing and of what it
// reaches, so collections keep them again. A commit missing anything is
// reported and left as it is.
Revive(ctx context.Context, commits []*proto.Ref, dryRun bool) ([]Revival, error)
// Revived reports whether a revival takes back the commit's tombstone.
Revived(ctx context.Context, commit *proto.Ref) (bool, error)
// Holds reports, for each commit, whether the store still holds a copy
// of the commit object, tombstoned or not; one it does not hold cannot
// be revived.
Holds(ctx context.Context, commits []*proto.Ref) ([]bool, error)
}
Reviver is a store that can take tombstoned commits back while it still holds everything they reach.
type SeedMap ¶
type SeedMap struct {
// contains filtered or unexported fields
}
SeedMap locates chunks of user-named files by the ref they would have been stored under, so a restore can take them from disk.
func NewSeedMap ¶
NewSeedMap returns an empty map; key is the store key the files were backed up under, nil for a plaintext store.
type Session ¶
type Session struct {
ID string
AgentID string
Set string
// Restore names the object a restore session reads; retirement and
// garbage collection keep it and everything it names while the session
// lives.
Restore *proto.Ref
// InitialRetention, when set, becomes Set's own retention policy if
// the set is created under the session; a set that exists keeps its
// own.
InitialRetention *retention.Policy
Started time.Time
LastSeen time.Time
}
Session is one backup or restore run of an agent against a set. Objects written under a session are visible only to that session until a commit.
type SessionStore ¶
type SessionStore interface {
// BeginSession opens s, fills in its ID and returns a context that
// carries it.
BeginSession(ctx context.Context, s *Session) (context.Context, error)
// EndSession ends the context's session.
EndSession(ctx context.Context) error
}
SessionStore opens and closes backup sessions. BeginSession fills in the session's id and returns a context carrying it; every Put and Get made with that context belongs to the session. EndSession drops whatever the session has not committed.
type SetScope ¶
type SetScope interface {
// Reachable reports whether one of the named sets references ref.
Reachable(ctx context.Context, sets []string, ref *proto.Ref) (bool, error)
}
SetScope narrows RefScope to named sets, for a server that grants a caller some of the store's sets.
type StatCache ¶
type StatCache interface {
// Lookup reports whether an entry exists for path and whether it still
// matches the file's identity.
Lookup(path string, info os.FileInfo) (present bool, matches bool)
// Store records the file's identity and the ref its read produced.
Store(path string, info os.FileInfo, ref *proto.Ref) error
}
StatCache remembers, per path, the stat data seen when a file was last read and the File ref it produced. It only ever adds a reason to re-read; a hit alone never justifies reuse.
type Stream ¶
type Stream struct {
Tar io.Reader
// Name and File, set instead of Tar, store File's content as the one
// file Name of the commit's tree.
Name string
File io.Reader
// Inspect, when set, sees every entry's header before it is stored; a
// writer it returns gets a copy of the entry's content.
Inspect func(hdr *tar.Header) io.Writer
// Finish, when set, runs once Tar is read to its end and before the
// commit is stored; an error fails the run without a commit.
Finish func() error
}
Stream is a backup read from a stream, such as a command's output, instead of from the disk: either a tar, each entry of which becomes a node of the commit's tree, or a single file.
type Traits ¶
type Traits struct {
// Resolution is the step mtimes are truncated to; 0 means exact. An FS
// must report every mtime truncated to it, the same way on every run,
// or files read with a finer time once look changed the next time.
Resolution time.Duration
// Skew bounds how far the FS's clock may be from ours.
Skew time.Duration
// Identity reports whether stat data carries ctime and inode, which
// the stat cache compares; without it the walk consults no stat cache.
Identity bool
// Reread reports whether reading a file again is cheap enough for a
// sampled re-hash; without it ForceHashPercent is ignored.
Reread bool
}
Traits say what an FS's stat data is worth to the change test.
type TraverserHandler ¶
TraverserHandler receives every visited node with its slash-joined path.
type TreeFetcher ¶
type TreeFetcher interface {
// GetTree returns the tree at ref and the trees below it, maxDepth
// levels down.
GetTree(ctx context.Context, ref *proto.Ref, maxDepth uint32) ([]*proto.Object, error)
}
TreeFetcher is implemented by stores that can stream a subtree in one round trip.
type TreeOptions ¶
type TreeOptions struct {
// From restores only the subtree at this slash-separated path of the
// commit.
From string
// Paths restores only these entries, slash-separated paths of the
// commit, files or directories, each at its own path under the
// directory. It excludes From and Delete.
Paths []string
// Name, when set, names each entry Paths names, or without Paths each
// entry the tree holds directly, given its path in the tree; what a
// renamed directory holds keeps its names. It excludes Delete.
Name func(path string, info os.FileInfo) string
// Delete removes everything under the directory the tree does not
// hold, except the partial files a later restore resumes from.
Delete bool
// Listed is handed the files and bytes the tree holds once it is
// listed, before anything is written.
Listed func(files, bytes int64)
// Written is handed the size of each part as it lands in a file,
// possibly from several goroutines at once.
Written func(n int64)
// Done is called once for every file, possibly from several goroutines
// at once. An error it returns stops the restore; returning nil for a
// file that failed passes over it. Without Done, a failed file stops
// the restore.
Done func(file RestoredFile) error
// Removing is called for each path Delete removes, before it is
// removed, or instead of removing it in a dry run.
Removing func(path string)
}
TreeOptions say what RestoreTree restores and tell its caller how it goes. Every callback is optional.
type TreeWriter ¶
type TreeWriter interface {
// File records a regular file whose content the callback writes;
// ErrSkipFile from it leaves the file out.
File(context.Context, os.FileInfo, func(io.Writer) error) error
// Tree records a subdirectory the callback fills; ErrSkipFile from it
// leaves the directory out.
Tree(context.Context, os.FileInfo, func(TreeWriter) error) error
// Node adds a finished node.
Node(*proto.TreeNode)
}
TreeWriter receives the entries of one directory being backed up.
type WalkResult ¶
type WalkResult struct {
Ref *proto.Ref
Commit *proto.Commit
Base *proto.Ref
Files int64
// Bytes is what those files hold on disk, whether or not the walk
// had to read them; Uploaded is the chunk bytes actually sent.
Bytes int64
Uploaded int64
Reused int64
Read int64
Torn int64
Unreadable int64
Skipped int64
Checkpoints int64
// Assumed counts parts skipped because a filter or the previous version
// held them; Repaired counts those the store then turned out to lack.
Assumed int64
Repaired int64
}
WalkResult summarises one run.
func (*WalkResult) Dirty ¶
func (r *WalkResult) Dirty() bool
Dirty reports whether any file was recorded from an inconsistent or failed read, which a caller should surface as a non-zero exit.
type Walker ¶
type Walker struct {
Index Index
Objects ObjectStore
Set string
AgentID string
// Root is the directory backed up, a path of FS.
Root string
// FS is the tree Root is read from; nil means LocalFS. A run on any
// other FS takes no Growing files.
FS FS
// Metadata is recorded on the commit; the store keeps it and never
// interprets it.
Metadata map[string]string
// Stream, when set, is backed up instead of Root. A stream cannot be
// read again, so a run with one does not retry a lost session.
Stream *Stream
// Carry, when set, keeps every entry of the previous commit's root that
// is gone from the disk and for which it returns true, so a set can
// accumulate files that leave the disk once committed. A run with Carry
// fails with ErrSealChanged over a commit sealed under another key or
// none.
Carry func(name string) bool
// Subdir, when set, puts Root's entries in the directory of that name
// in the commit's root instead of in the root itself. Carry then applies
// inside it, and to the previous commit's other root entries. A run with
// Subdir writes no checkpoints.
Subdir string
// Include decides per slash-separated relative path; nil includes all.
Include func(rel string) bool
// Growing are files of the root, or of Subdir when it is set, committed
// up to a length, continuing the previous commit's versions; a run with
// Stream takes none.
Growing []Growing
// Key seals names and blobs; nil backs up in the clear.
Key *storekey.Key
// Logger is where the walk reports; nil means slog.Default.
Logger *slog.Logger
// Progress, when set, is handed the counters every ProgressInterval
// while the walk runs, and once more when it ends. It is called from
// a goroutine of its own and must not block for long.
Progress func(WalkResult)
// ProgressInterval is how often Progress hears; zero means
// DefaultProgressInterval.
ProgressInterval time.Duration
// Sessions, when set, wraps the run in a session, so nothing it uploads
// is visible to others before its commit.
Sessions SessionStore
// InitialRetention, when set, is the retention policy Set starts with
// if this run creates it; it travels with the session, so a run
// without Sessions sends none.
InitialRetention *retention.Policy
// Verify reads every file, even one the change test would reuse, which
// catches a write that kept a file's mtime and size.
Verify bool
Workers int
// ForceHashPercent is the share of reused files re-hashed as a check;
// ignored on an FS whose Traits do not allow a Reread.
ForceHashPercent int
CheckpointInterval time.Duration
ReadRetries int
// LostRetries is how often a commit refused with ErrSessionLost is
// retried by walking again in the same session.
LostRetries int
// ScanWorkers is how many directories are listed ahead of the walk at
// once; zero means DefaultScanWorkers and a negative number lists every
// directory on the walk itself.
ScanWorkers int
// ScanWindow bounds the entries held for directories the walk has not
// reached yet; zero means DefaultScanWindow.
ScanWindow int
// Quiesced records that a pre hook paused the application before the
// walk; a clean run then commits as consistent.
Quiesced bool
// Cache is consulted only on an FS whose Traits carry Identity.
Cache StatCache
PrefetchDepth uint32
// WindowBytes bounds the chunk bytes kept for files whose skipped parts
// the store has not confirmed yet; 0 means DefaultWindowBytes.
WindowBytes int
// BlobCache, when set, receives every uploaded blob.
BlobCache *blobcache.Cache
// contains filtered or unexported fields
}
Walker builds a commit by diffing a directory against the set's latest commit, reading and uploading only what changed.
type WriteFS ¶
type WriteFS interface {
Join(dir, name string) string
// Split is path's directory and its last element, which Join puts back
// together.
Split(path string) (dir, name string)
Lstat(ctx context.Context, path string) (os.FileInfo, error)
// ReadDir lists dir without following symlinks; a restore lists only
// to delete what its tree does not hold.
ReadDir(ctx context.Context, dir string) ([]fs.DirEntry, error)
// MkdirAll makes dir and the directories above it that are missing.
MkdirAll(ctx context.Context, dir string) error
// WriteFile writes what r holds to path, front to back, replacing a
// file there.
WriteFile(ctx context.Context, path string, r io.Reader) (int64, error)
// Rename moves from to to, replacing a file there.
Rename(ctx context.Context, from, to string) error
Remove(ctx context.Context, path string) error
RemoveAll(ctx context.Context, path string) error
Symlink(ctx context.Context, target, path string) error
Readlink(ctx context.Context, path string) (string, error)
// Chmod sets the permission, setuid, setgid and sticky bits of mode.
Chmod(ctx context.Context, path string, mode os.FileMode) error
Chtimes(ctx context.Context, path string, mtime time.Time) error
}
WriteFS is the file tree a Restorer writes. Paths are the FS's own strings, built with Join. Symlink, Chmod and Chtimes fail with an error matching errors.ErrUnsupported where the tree cannot hold links, modes or times; a restore then goes on without them. A missing path's error matches fs.ErrNotExist.
Source Files
¶
- backup.go
- blob.go
- damage.go
- escrow.go
- file.go
- fs.go
- growing.go
- index.go
- indexpaths.go
- kind.go
- livecheck.go
- memfs.go
- names.go
- pickcommit.go
- platform_unix.go
- prefetch.go
- presence.go
- restore.go
- restorefiles.go
- restorepath.go
- restorestream.go
- restoretree.go
- scan.go
- seeds.go
- session.go
- storage.go
- stream.go
- traverser.go
- tree.go
- walker.go
- window.go
- writefs.go
Directories
¶
| Path | Synopsis |
|---|---|
|
Package blobcache keeps copies of stored objects on local disk, keyed by ref and verified on every read.
|
Package blobcache keeps copies of stored objects on local disk, keyed by ref and verified on every read. |
|
Package chunker cuts byte streams into content-defined chunks with a FastCDC-style normalised Gear hash.
|
Package chunker cuts byte streams into content-defined chunks with a FastCDC-style normalised Gear hash. |
|
Package hooks runs the shell commands a backup is wrapped in.
|
Package hooks runs the shell commands a backup is wrapped in. |
|
Package job runs each kind of backup once against a store that is already open: a directory, a Postgres base backup or WAL, a MySQL dump or binlogs.
|
Package job runs each kind of backup once against a store that is already open: a directory, a Postgres base backup or WAL, a MySQL dump or binlogs. |
|
Package lockfile takes exclusive locks on files, so a second process on the same directory refuses to start.
|
Package lockfile takes exclusive locks on files, so a second process on the same directory refuses to start. |
|
Package mysql backs up MySQL servers into goback sets: logical dumps through mysqldump into one set, and the binary logs, received as a replica, into another for point-in-time recovery.
|
Package mysql backs up MySQL servers into goback sets: logical dumps through mysqldump into one set, and the binary logs, received as a replica, into another for point-in-time recovery. |
|
Package postgres backs up Postgres clusters: base backups, and the WAL a receiver streams through a replication slot, each into a set of its own.
|
Package postgres backs up Postgres clusters: base backups, and the WAL a receiver streams through a replication slot, each into a set of its own. |
|
Package presence holds the Bloom filter an agent consults before uploading a blob: a positive answer is probably right, a negative one is certain.
|
Package presence holds the Bloom filter an agent consults before uploading a blob: a positive answer is probably right, a negative one is certain. |
|
Package retention decides which commits of a set a policy keeps.
|
Package retention decides which commits of a set a policy keeps. |
|
Package statcache is the per-machine sidecar that remembers what each file looked like when it was last read, so a run can detect changes that mtime and size alone miss and can be dropped at any time without losing correctness.
|
Package statcache is the per-machine sidecar that remembers what each file looked like when it was last read, so a run can detect changes that mtime and size alone miss and can be dropped at any time without losing correctness. |
|
Package storekey implements the client-held store key: its key file, deterministic sealing of blobs, name tokens and inline content, the digest blobs are named by, and passphrase escrow.
|
Package storekey implements the client-held store key: its key file, deterministic sealing of blobs, name tokens and inline content, the digest blobs are named by, and passphrase escrow. |