Documentation
¶
Overview ¶
Package hashfolder stores piece files in two CID-hash namespaces, open-pieces and acl-pieces. Each namespace implements HashSpace. layout-open-pieces.json and layout-acl-pieces.json at each storage root record that folder's used bytes and the hash ranges the folder owns. The space folders hold only hash-named piece files.
FirstSetup assigns ranges by presenting every drive to hashspacesolver. WriteCID creates a file only when that CID is not already stored. lib/paths does not call this package.
Index ¶
- Constants
- Variables
- func CIDHash(c cid.Cid) ([]byte, error)
- func FirstSetup(drives []Drive) (hashspacesolver.State, error)
- func WithSize(r io.Reader, n int64) io.Reader
- type Cluster
- func (c *Cluster) AbandonMoveTask(ctx context.Context, taskID int64, troubled string) error
- func (c *Cluster) AdoptLocal(storageID string, pc cid.Cid, src string) (int64, error)
- func (c *Cluster) Candidates(pc cid.Cid) ([]Location, error)
- func (c *Cluster) Close() error
- func (c *Cluster) CompleteMoveSource(ctx context.Context, m *MoveSource) error
- func (c *Cluster) CopyOne(ctx context.Context, m *MoveSource, hexHash string) error
- func (c *Cluster) DeleteCID(ctx context.Context, pieceCID string) error
- func (c *Cluster) Drop(ctx context.Context, storageID string, pc cid.Cid) error
- func (c *Cluster) DropLocal(storageID string, pc cid.Cid) error
- func (c *Cluster) FixOverlaps(ctx context.Context) (int, error)
- func (c *Cluster) HasFile(ctx context.Context, pc cid.Cid) (bool, error)
- func (c *Cluster) HasLocal(storageID string) bool
- func (c *Cluster) LocalIDs() []string
- func (c *Cluster) Locations(ctx context.Context, pieceCID string) ([]Location, error)
- func (c *Cluster) MarkMoveStarted(ctx context.Context, id int64) error
- func (c *Cluster) MaybeRebalance(ctx context.Context, storageID string) error
- func (c *Cluster) MoveSourcesByTask(ctx context.Context, taskID int64) ([]*MoveSource, error)
- func (c *Cluster) Open(ctx context.Context, storageID string, pc cid.Cid) (io.ReadCloser, error)
- func (c *Cluster) OpenLocal(storageID string, pc cid.Cid) (io.ReadCloser, error)
- func (c *Cluster) OpenLocalAt(storageID string, pc cid.Cid, offset, size int64) (io.ReadCloser, error)
- func (c *Cluster) PendingCopy(ctx context.Context, m *MoveSource) iter.Seq2[string, error]
- func (c *Cluster) PutRemote(ctx context.Context, storageID string, pc cid.Cid, r io.Reader) (int64, bool, error)
- func (c *Cluster) Refresh(ctx context.Context) error
- func (c *Cluster) RemoteAt(ctx context.Context, storageID string, pc cid.Cid, start, end int64) (io.ReadCloser, error)
- func (c *Cluster) ReturnLocal(storageID string, pc cid.Cid, dest string) error
- func (c *Cluster) ServeHTTP(w http.ResponseWriter, r *http.Request)
- func (c *Cluster) StatAt(ctx context.Context, storageID string, pc cid.Cid) (bool, error)
- func (c *Cluster) StatLocal(storageID string, pc cid.Cid) (int64, bool, error)
- func (c *Cluster) Target(ctx context.Context, digest []byte) (string, error)
- func (c *Cluster) Targets(ctx context.Context) (func(digest []byte) (string, error), error)
- func (c *Cluster) WriteLocal(storageID string, pc cid.Cid, r io.Reader) (int64, error)
- type Drive
- type HashRange
- type HashSpace
- type Layout
- type LocalDrive
- type Location
- type MoveSource
- type ReadSeekFile
- type Space
- func (s *Space) AdoptFileOn(root string, c cid.Cid, src string) (int64, error)
- func (s *Space) Close() error
- func (s *Space) DeleteCID(c cid.Cid) error
- func (s *Space) DeleteCIDOn(root string, c cid.Cid) error
- func (s *Space) Flush() error
- func (s *Space) OpenCIDAt(root string, c cid.Cid, offset, size int64) (io.ReadCloser, error)
- func (s *Space) OpenCIDOn(root string, c cid.Cid) (io.ReadCloser, error)
- func (s *Space) ReadCIDFileFrom(c cid.Cid) (ReadSeekFile, error)
- func (s *Space) ReplaceShorterHashOn(root, hexHash string, size int64) (io.WriteCloser, int64, error)
- func (s *Space) ReturnFileOn(root string, c cid.Cid, dest string) error
- func (s *Space) SetIntervalsOn(root string, version int64, owned, moveDests []HashRange) error
- func (s *Space) StatCIDOn(root string, c cid.Cid) (int64, bool, error)
- func (s *Space) Used() int64
- func (s *Space) UsedOn(root string) (int64, error)
- func (s *Space) WriteCID(c cid.Cid) (io.WriteCloser, error)
- func (s *Space) WriteCIDOn(root string, c cid.Cid) (io.WriteCloser, error)
- func (s *Space) WriteHashOn(root, hexHash string) (io.WriteCloser, error)
Constants ¶
const ( // EVENT_FULL, EVENT_ARRIVE, EVENT_ABSORB, EVENT_VACATE, EVENT_BALANCE, and // EVENT_CLAIM are hash_space_pending_event.event_kind values. EVENT_FULL = "full" EVENT_ARRIVE = "arrive" EVENT_ABSORB = "absorb" EVENT_VACATE = "vacate" EVENT_BALANCE = "balance" EVENT_CLAIM = "claim" // ROOM_RETURN_MIN is the capacity increase that asks a disk to take // bytes back after other filesystem activity frees space. ROOM_RETURN_MIN = 1 << 30 // REFRESH_INTERVAL is how often a node republishes used counters and runs a // queued event. Map edits reload the local snapshot and notify other nodes. REFRESH_INTERVAL = FLUSH_INTERVAL CAS_RETRIES = 20 // NOTIFY_TIMEOUT bounds each map-change notification to another node. NOTIFY_TIMEOUT = 5 * time.Second // LIST_PAGE is how many directory entries a rebalance scan reads at once. LIST_PAGE = 256 // MOVE_STALE is how long a move row may sit without HashSpaceMove starting // it before the row is obsolete and the destination is treated as missing. MOVE_STALE = 48 * time.Hour )
const ( // SPLIT is the folder split. The first two hex characters of the CID hash // are the directory and the rest is the file name: abcdefg is ab/cdefg. SPLIT = "2,-" // DIR_OPEN and DIR_ACL are the space folders on each storage root. DIR_OPEN = "open-pieces" DIR_ACL = "acl-pieces" // HASH_BYTES is the CID hash width used for layout ranges. HASH_BYTES = 32 // FLUSH_INTERVAL is how often a loaded space rewrites its layout. // It matches the storage heartbeat cadence. FLUSH_INTERVAL = 10 * time.Second )
const ( // MIGRATE_BATCH is how many piece-park files are renamed before their // sector_location rows are dropped in one statement. MIGRATE_BATCH = 10_000 // MIGRATE_WORKERS_PER_DISK is how many renames and copies run at once // for each disk being migrated. MIGRATE_WORKERS_PER_DISK = 5 // MIGRATE_PROGRESS_INTERVAL is how often migration progress is printed. MIGRATE_PROGRESS_INTERVAL = 5 * time.Second // MIGRATE_RETRY_MIN and MIGRATE_RETRY_MAX bound the wait before a failed // migration pass runs again. Startup waits until a pass succeeds. MIGRATE_RETRY_MIN = 5 * time.Second MIGRATE_RETRY_MAX = 2 * time.Minute // MIGRATE_SKIP_SAMPLES is how many pieces of one skip reason are logged // with their ids. Later ones of that reason are only counted. MIGRATE_SKIP_SAMPLES = 32 )
const OVERLAP_CHECK_INTERVAL = 10 * time.Minute
OVERLAP_CHECK_INTERVAL is how often a node looks for piece files held outside their range owner. A node with a has_misplaced disk checks on every refresh instead.
const SNAPSHOT_RETRIES = 3
SNAPSHOT_RETRIES bounds how often a snapshot is reloaded because the map version changed while it was being read.
Variables ¶
var ErrCrossDevice = errors.New("rename crosses filesystems")
ErrCrossDevice reports that a rename would cross filesystems, so the bytes must be copied instead.
var ErrDiskFull = errors.New("hash space destination is full")
ErrDiskFull means the destination could not store the copy because the disk is full. The move is abandoned instead of retried.
ErrSourceUnavailable means the move could not read the source. Retries continue until the last one, which abandons the move.
Functions ¶
func CIDHash ¶
CIDHash is the tree root carried by piece CID v2 c, which places c on the hash circle and names its file. Hash spaces are keyed by piece CID v2 only.
func FirstSetup ¶
func FirstSetup(drives []Drive) (hashspacesolver.State, error)
FirstSetup writes the layout for both spaces on every drive.
With no layouts, it seeds one capacity-weighted range per disk and checks that assignment with hashspacesolver.Validate. A drive added beside existing layouts is presented with EventEmptyMountConnected. Existing layouts are reloaded rather than seeded again.
Types ¶
type Cluster ¶
type Cluster struct {
// contains filtered or unexported fields
}
Cluster shares the hash-space map of every node through the database. hash_space_range covers each circle once. A stuck move may leave one range owned by both disks. The row with problem set is not a place for new files. A piece file is on that range's owner, or on both ends of a move that covers its hash.
func NewCluster ¶
func NewCluster(ctx context.Context, db *harmonydb.DB, drives []LocalDrive, auth http.Header) (*Cluster, error)
NewCluster joins drives into the cluster map, writes their layouts from it, and loads both local spaces. A node with no drives still gets both, empty. auth is sent on requests to other nodes' /hashspace routes.
func (*Cluster) AbandonMoveTask ¶
AbandonMoveTask gives each of the task's remaining intervals to both of its disks, marks the troubled owner's rows problem, and deletes those move rows. troubled is the disk that failed: the destination when it is full or missing, the source when the source cannot be read. New files in a problem range go to the other owner. When no move rows remain, queued events are solved.
func (*Cluster) AdoptLocal ¶
AdoptLocal renames src into open-pieces on a local disk. See AdoptFileOn.
func (*Cluster) Candidates ¶
Candidates lists the disks that may hold pc, local ones first, from the in-memory map: the range owner, both ends of a move covering its hash, and disks with misplaced pieces. It runs no SQL. Writers reload this map and notify the other nodes. It returns nothing when the cluster has no ranges.
func (*Cluster) CompleteMoveSource ¶
func (c *Cluster) CompleteMoveSource(ctx context.Context, m *MoveSource) error
CompleteMoveSource hands the move interval to its destination once every file still on the source is also on the destination. Source files are removed only after that ownership commit. A file deleted from the source is not required on the destination.
func (*Cluster) CopyOne ¶
CopyOne copies one piece hash of the move source onto its local destination. The source file stays until CompleteMoveSource has committed the range, so abandoning the move cannot hide the only copy. The move row still exists, so a delete during the copy still sees this disk. A source file that disappears during the copy is a delete: the destination copy is removed and the move continues. A destination file at least the source size is left in place. A shorter destination file is a partial copy and is replaced.
func (*Cluster) DeleteCID ¶
DeleteCID removes pieceCID from the disks that hold its hash: the range owner, or both ends of a move that covers it. A missing file is fine. The lookup runs twice so a move that starts during the first pass is still cleared; the second pass only contacts disks the first did not. A copy that lands after both passes drops its destination when the source file is already gone.
func (*Cluster) Drop ¶
Drop removes pc from storageID, locally or through that node's DELETE. A missing file is fine.
func (*Cluster) DropLocal ¶
DropLocal removes the open-pieces file from a local disk. Missing is fine.
func (*Cluster) FixOverlaps ¶
FixOverlaps schedules a move for every open-pieces or acl-pieces range that has files on a local disk which does not own it: files migrated from piece-park, a range a layout import took, a move whose source cleanup failed, or a duplicate copy. The move copies what the owner lacks and then drops the other disk's copies. Between each pair of disks one pass moves at most hashspacesolver.CLAIM_STEP_PERCENT of the smaller capacity; the next pass runs after those moves finish. Nothing is planned while any move is in flight, since an in-flight move holds pieces on two disks by design. A local has_misplaced disk with nothing left outside its ranges has the flag cleared. It returns the number of moves planned.
func (*Cluster) HasFile ¶
HasFile reports whether the open-pieces file is on any place that can hold it.
func (*Cluster) Locations ¶
Locations lists where pieceCID can be: the range owner, or both ends of a move that covers its hash. Local disks come first.
func (*Cluster) MarkMoveStarted ¶
MarkMoveStarted records that HashSpaceMove has begun this row. A later retry does not clear the stamp. The caller has the move row.
func (*Cluster) MaybeRebalance ¶
MaybeRebalance starts a vacate or full rebalance for storageID when that disk is leaving or is above FILL_LIMIT_PERCENT, then notices fill spread.
func (*Cluster) MoveSourcesByTask ¶
MoveSourcesByTask returns the move sources owned by a HashSpaceMove task.
func (*Cluster) Open ¶
Open reads the whole open-pieces file from storageID, locally or through that node's /hashspace route.
func (*Cluster) OpenLocalAt ¶
func (c *Cluster) OpenLocalAt(storageID string, pc cid.Cid, offset, size int64) (io.ReadCloser, error)
OpenLocalAt opens bytes [offset, offset+size) of the open-pieces file on a local disk.
func (*Cluster) PendingCopy ¶
PendingCopy yields piece hashes still on the source in the move interval, one directory page at a time. The caller copies each hash and leaves the source file until the move commits. Ranging stops without holding the rest of the interval.
func (*Cluster) PutRemote ¶
func (c *Cluster) PutRemote(ctx context.Context, storageID string, pc cid.Cid, r io.Reader) (int64, bool, error)
PutRemote streams r to the node that holds storageID. That node writes the file locally, publishes its usage, and maybe starts a rebalance. The body is not buffered. A client with no storage auth does not send the request. When r's length is known it is sent as Content-Length, and that node replaces a shorter copy of the piece. A seekable r is rewound for the next URL; otherwise the first attempt that starts sending is the only one.
func (*Cluster) RemoteAt ¶
func (c *Cluster) RemoteAt(ctx context.Context, storageID string, pc cid.Cid, start, end int64) (io.ReadCloser, error)
RemoteAt reads bytes [start, end) of the file on another node.
func (*Cluster) ReturnLocal ¶
ReturnLocal moves an open-pieces file back to dest and subtracts its size. dest must not already exist. Used to undo an adopt when placement loses the race with a delete.
func (*Cluster) ServeHTTP ¶
func (c *Cluster) ServeHTTP(w http.ResponseWriter, r *http.Request)
ServeHTTP serves /hashspace/{storage id}/{hash} for clustermates. GET reads the piece file (Range supported), PUT writes it, DELETE removes it. Query space selects open-pieces or acl-pieces; it defaults to open-pieces. POST /hashspace/notify reloads the local map after another node changed it. Callers must authenticate the request before it reaches this handler.
func (*Cluster) StatAt ¶
StatAt reports whether the open-pieces file for pc exists on storageID: a stat on a local disk, otherwise a HEAD to the node that holds it. A file's size is not compared with the CID.
func (*Cluster) Target ¶
Target returns the disk a new open-pieces file for digest belongs on: the destination of an in-flight move covering it, otherwise an owner of the range. An owner marked problem is skipped when the other owner is not.
func (*Cluster) Targets ¶
Targets snapshots the open-pieces map and returns a resolver with the same rules as Target.
func (*Cluster) WriteLocal ¶
WriteLocal copies r into open-pieces on a local disk. When r's length is known, an existing file shorter than it is a partial copy and is replaced. Otherwise an existing file is kept and its size returned.
type Drive ¶
Drive is one attached storage root. StorageID is the storage path ID from sectorstore.json and the database; it names the drive's mountpoint in the solver. Capacity, when set, is the solver disk size in bytes. When zero, capacity is the filesystem size from statfs, capped by sectorstore.json MaxStorage when that field is non-zero.
type HashRange ¶
HashRange is one half-open interval (start, end] owned by this folder. Start and end are lowercase hex of the solver hash bytes.
type HashSpace ¶
type HashSpace interface {
WriteCID(c cid.Cid) (io.WriteCloser, error)
DeleteCID(c cid.Cid) error
ReadCIDFileFrom(c cid.Cid) (ReadSeekFile, error)
}
HashSpace is the only caller-facing surface for one namespace.
type Layout ¶
type Layout struct {
// Version is the hash_space_meta.version the ranges were read at. A
// layout newer than the database is written back to it on startup.
Version int64 `json:"version,omitempty"`
Used int64 `json:"used"`
CommittedAt time.Time `json:"committed_at"`
Ranges []HashRange `json:"ranges"`
// MoveDests are intervals owned by another folder that are being moved
// here. A piece in one may be on either folder until the move ends.
MoveDests []HashRange `json:"move_dests,omitempty"`
// Claim marks a folder of pieces migrated from piece-park, holding hashes
// from the whole circle. Its disk joins with EVENT_CLAIM.
Claim bool `json:"claim,omitempty"`
}
Layout is layout-{kind}.json at the storage root, beside the space folder.
type LocalDrive ¶
LocalDrive is one storage path on this node that holds hash-space folders.
type MoveSource ¶
type MoveSource struct {
ID int64
Space string // "open-pieces" or "acl-pieces"
StartHash []byte
EndHash []byte
FromStorage string
ToStorage string
Size int64 // planned bytes of the interval
}
MoveSource is an in-flight interval copy.
type ReadSeekFile ¶
ReadSeekFile is the CID file returned by ReadCIDFileFrom.
type Space ¶
type Space struct {
// contains filtered or unexported fields
}
Space is one hash namespace, open-pieces or acl-pieces, across local storage roots.
func Load ¶
Load reads kind's layout on each root, restores used, then adds only files whose mtime is newer than the layout. kind is DIR_OPEN or DIR_ACL. An empty root list returns an empty space.
func (*Space) AdoptFileOn ¶
AdoptFileOn renames src onto the CID path on root without copying bytes and adds its size to that root's counter. It never replaces an existing CID file. A cross-filesystem rename returns ErrCrossDevice with src untouched.
func (*Space) DeleteCID ¶
DeleteCID removes the CID file and subtracts its size. A missing file returns os.ErrNotExist and does not change the counter.
func (*Space) DeleteCIDOn ¶
DeleteCIDOn removes the CID file from root only. A missing file returns os.ErrNotExist and does not change the counter.
func (*Space) ReadCIDFileFrom ¶
func (s *Space) ReadCIDFileFrom(c cid.Cid) (ReadSeekFile, error)
ReadCIDFileFrom opens the CID file on the root that owns its hash. Other local roots are probed when the file is not on that root.
func (*Space) ReplaceShorterHashOn ¶
func (s *Space) ReplaceShorterHashOn(root, hexHash string, size int64) (io.WriteCloser, int64, error)
ReplaceShorterHashOn opens a writer for hexHash on root when the piece file is missing or shorter than size. A shorter file is a partial copy: it is deleted and its bytes leave the counter. A file at least size bytes long stays, and its size is returned with a nil writer. A file another writer has open returns errWriteInProgress.
func (*Space) ReturnFileOn ¶
ReturnFileOn moves the CID file on root back to dest and subtracts its size from the counter. dest must not already exist. The source file is left in place when the rename fails.
func (*Space) SetIntervalsOn ¶
SetIntervalsOn replaces the owned ranges and move destinations of root, read at map version, and rewrites its layout.
func (*Space) StatCIDOn ¶
StatCIDOn returns the size of the CID file on root, and whether it exists.
func (*Space) WriteCID ¶
WriteCID opens a new file for a piece CID that is not already stored. The file is written to the root a move is bringing the hash to, else the root whose ranges contain it. Close adds the written byte count. A second Close does not add again. Abort deletes the partial file and leaves the counter unchanged. If the CID file already exists, WriteCID returns os.ErrExist and does not change the counter.
func (*Space) WriteCIDOn ¶
WriteCIDOn is WriteCID on the named root, whether or not that root owns the hash. Cluster placement and rebalance pick the root.
func (*Space) WriteHashOn ¶
func (s *Space) WriteHashOn(root, hexHash string) (io.WriteCloser, error)
WriteHashOn opens a new file named by the piece hash on root.