Documentation
¶
Index ¶
- Constants
- Variables
- func PortForward(p Printer, server string, port int) (*tunnel, error)
- func SetDebug(d bool)
- type ABFSClient
- type ARESClient
- type BadgerIndexStore
- func (b *BadgerIndexStore) Close() error
- func (b *BadgerIndexStore) Delete(hash entities.Hash) (*IndexKeyEntry, int64, int64, error)
- func (b *BadgerIndexStore) DeleteMany(hashes []entities.Hash) (int64, int64, error)
- func (b *BadgerIndexStore) Evict() ([]*IndexKeyEntry, int64, int64, error)
- func (b *BadgerIndexStore) ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error
- func (b *BadgerIndexStore) GCEnd() time.Time
- func (b *BadgerIndexStore) GCStart() time.Time
- func (b *BadgerIndexStore) Get(hash entities.Hash) *IndexEntry
- func (b *BadgerIndexStore) IndexCount() int64
- func (b *BadgerIndexStore) KeepAlive(key entities.Hash) error
- func (b *BadgerIndexStore) LRUSize() int64
- func (b *BadgerIndexStore) LocalCount() int64
- func (b *BadgerIndexStore) MarkObjectOffline(key entities.Hash) error
- func (b *BadgerIndexStore) MarkObjectRemote(key entities.Hash) error
- func (b *BadgerIndexStore) Open() error
- func (b *BadgerIndexStore) PeekOldest() (KeyTimestamp, error)
- func (b *BadgerIndexStore) Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)
- func (b *BadgerIndexStore) PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)
- func (b *BadgerIndexStore) ReadAll() ([]*IndexKeyEntry, error)
- func (b *BadgerIndexStore) ReadLRU() ([]KeyTimestamp, error)
- func (b *BadgerIndexStore) TotalSize() int64
- func (b *BadgerIndexStore) Use(hash entities.Hash) error
- func (b *BadgerIndexStore) UseMany(hashes []entities.Hash) error
- type BadgerMapsStore
- type BaseBadgerStore
- type BlockingQueue
- type CacheMan
- func (c *CacheMan) EnqueueNewObject(hash entities.Hash, objType protocol.RequestObjectType)
- func (c *CacheMan) ErrorJSON(w http.ResponseWriter, r *http.Request, err error, status int)
- func (c *CacheMan) FetchARI(ari string, priority float64, callback func(error))
- func (c *CacheMan) FetchCASBlob(sha256Hex string, size int64, priority float64, callback func(error))
- func (c *CacheMan) FetchMap(hash entities.Hash, priority float64, callback func(error))
- func (c *CacheMan) FetchObject(hash entities.Hash, objType protocol.RequestObjectType, priority float64, ...)
- func (c *CacheMan) GetStat(h entities.Hash) (entities.StatInfo, error)
- func (c *CacheMan) HealthStatus() int32
- func (c *CacheMan) MapResults(hash entities.Hash) []entities.Hash
- func (c *CacheMan) NewMap(from, to entities.Hash, localOnly bool)
- func (c *CacheMan) NewObject(hash entities.Hash, objType protocol.RequestObjectType)
- func (c *CacheMan) NotifyUse(hash entities.Hash)
- func (c *CacheMan) Phony(priority dagsync.RequestPriority, objects []entities.Hash, ...)
- func (c *CacheMan) Prefetch(ctx context.Context, blobs, objects []entities.Hash, priority float64) error
- func (c *CacheMan) PrefetchGitObject(hash entities.Hash, callback func(error))
- func (c *CacheMan) Printer() Printer
- func (c *CacheMan) ReadJSON(w http.ResponseWriter, r *http.Request, value any) bool
- func (c *CacheMan) RunLocalGC() error
- func (c *CacheMan) Serve(ctx context.Context) error
- func (c *CacheMan) Stats() protocol.CacheManStats
- func (c *CacheMan) Stop() error
- func (c *CacheMan) WriteJSON(w http.ResponseWriter, r *http.Request, value any)
- type CheckCredsFunc
- type Connection
- type DefaultPrinter
- type EmptyIndex
- func (e *EmptyIndex) Close() error
- func (e *EmptyIndex) Delete(hash entities.Hash) error
- func (e *EmptyIndex) DeleteMany(hashes []entities.Hash) (int64, int64, error)
- func (e *EmptyIndex) EntriesWithTotalSize() ([]*IndexKeyEntry, uint64, error)
- func (e *EmptyIndex) EvictCount() int64
- func (e *EmptyIndex) EvictEnd() time.Time
- func (e *EmptyIndex) EvictStart() time.Time
- func (e *EmptyIndex) ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error
- func (e *EmptyIndex) GCEnd() time.Time
- func (e *EmptyIndex) GCStart() time.Time
- func (e *EmptyIndex) Get(hash entities.Hash) *IndexEntry
- func (e *EmptyIndex) IndexCount() int64
- func (e *EmptyIndex) KeepAlive(key entities.Hash) error
- func (e *EmptyIndex) LRUSize() int64
- func (e *EmptyIndex) LocalCount() int64
- func (e *EmptyIndex) LockEviction() func()
- func (e *EmptyIndex) OldestUsed() time.Time
- func (e *EmptyIndex) Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)
- func (e *EmptyIndex) PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)
- func (e *EmptyIndex) ReadLRU() ([]KeyTimestamp, error)
- func (e *EmptyIndex) Sweep(timestampThreshold int64) (deletedCount int, err error)
- func (e *EmptyIndex) TotalSize() uint64
- func (e *EmptyIndex) Use(hash entities.Hash) error
- func (e *EmptyIndex) UseMany(hashes []entities.Hash) error
- type EntryType
- type FetchReq
- type FetchReqQueue
- type Index
- type IndexEntry
- type IndexKeyEntry
- type IndexOptions
- type IndexReq
- type IndexReqQueue
- type IndexStore
- type KeyTimestamp
- type Maps
- type MapsStore
- type NewARESFunc
- type NewObjectReq
- type NewObjectReqQueue
- type NotifyUseAggregator
- type PendingAddTracker
- type Printer
- type RelativeFetchReq
- type ResilientPrinter
- type StatStore
- type TunnelCommand
Constants ¶
const ( // Batch evictions for performance // If this value is too large, we may exceed transaction limits. MaxEvictBatchSize = 10000 // The time to keep evicted entries in the index. EvictTTL = 24 * time.Hour )
const ( HealthBypassFileName = "health_bypass" StatusDirName = "status" RemoteErrFileName = "remote.err" CredsErrFileName = "creds.err" )
const ( ResultWait = protocol.ResultWait ResultSuccess = protocol.ResultSuccess ResultNotFound = protocol.ResultNotFound ResultFailed = protocol.ResultFailed ResultTimeout = protocol.ResultTimeout )
const DefaultTTLHours = 8
Variables ¶
var ErrMissingEntry = errors.New("missing index entry")
var ErrProtocol = errors.New("cacheman binary protocol error")
var ErrTxnConflict = errors.New("transaction conflict")
var ErrTxnTooBig = errors.New("transaction too big")
Functions ¶
func PortForward ¶
PortForward maintains an SSH tunnel that forwards localhost:port to localhost:<rport> on the remote host, where server is "<host>:<rport>" (an "https://" prefix is ignored). The SSH destination is the bare host name, so any user, identity, or proxy settings must be configured in ~/.ssh/config, and authentication must work without prompting.
Types ¶
type ABFSClient ¶
type ABFSClient interface {
Close() error
AddMapResult(ctx context.Context, from, to entities.Hash) error
GetObject(ctx context.Context, key remote.Hash) (remote.ContentReadCloser, error)
GetRefs(ctx context.Context, server string, project string, refs []string) ([]entities.Hash, error)
Headers(ctx context.Context, keys []remote.Hash) ([]entities.ObjectHeader, error)
SHA256(ctx context.Context, keys []remote.Hash) ([]entities.SHA256, error)
MapResults(ctx context.Context, id entities.Hash) ([]entities.Hash, error)
PutObject(ctx context.Context, key remote.Hash, reader remote.ContentReadCloser) error
PutRefs(ctx context.Context, server string, project string, refs []entities.RefHash) error
SearchRefs(ctx context.Context, server string, project string, refPrefix string, limit int) ([]entities.ProjectRefHash, error)
Mark(ctx context.Context, hash entities.Hash, key, value string) (int64, error)
Version(ctx context.Context) (string, error)
WhoAmI(ctx context.Context) (string, error)
}
type ARESClient ¶
type ARESClient interface {
remote.ARESService
// Support returns whether remote execution is available
Support() bool
}
type BadgerIndexStore ¶
type BadgerIndexStore struct {
BaseBadgerStore
// contains filtered or unexported fields
}
BadgerIndexStore implements IndexStore using BadgerDB as a backing store. Note that the notion of "Use" here is external - it's not directly associated with the IndexEntry usage. Rather it's the usage of the associated localcache object.
func NewBadgerIndexStore ¶
func NewBadgerIndexStore(path string, printer Printer, evictBatchSize uint, trackUse bool) *BadgerIndexStore
NewBadgerIndexStore creates a BadgerIndexStore. On an Evict() call, up-to evictBatchSize number of keys will be evicted. The caller may need to re-call Evict() to reduce the size as desired.
func (*BadgerIndexStore) Close ¶
func (b *BadgerIndexStore) Close() error
Close implements IndexStore.
func (*BadgerIndexStore) Delete ¶
func (b *BadgerIndexStore) Delete(hash entities.Hash) (*IndexKeyEntry, int64, int64, error)
Delete implements IndexStore.
func (*BadgerIndexStore) DeleteMany ¶
DeleteMany implements IndexStore. Multiple transactions are used if hashes is large.
func (*BadgerIndexStore) Evict ¶
func (b *BadgerIndexStore) Evict() ([]*IndexKeyEntry, int64, int64, error)
Evict implements IndexStore.
func (*BadgerIndexStore) ForEach ¶
func (b *BadgerIndexStore) ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error
ForEach implements IndexStore.
func (*BadgerIndexStore) GCEnd ¶
func (b *BadgerIndexStore) GCEnd() time.Time
GCEnd implements IndexStore.
func (*BadgerIndexStore) GCStart ¶
func (b *BadgerIndexStore) GCStart() time.Time
GCStart implements IndexStore.
func (*BadgerIndexStore) Get ¶
func (b *BadgerIndexStore) Get(hash entities.Hash) *IndexEntry
Get implements IndexStore.
func (*BadgerIndexStore) IndexCount ¶
func (b *BadgerIndexStore) IndexCount() int64
IndexCount implements IndexStore.
func (*BadgerIndexStore) KeepAlive ¶
func (b *BadgerIndexStore) KeepAlive(key entities.Hash) error
KeepAlive implements IndexStore.
func (*BadgerIndexStore) LRUSize ¶
func (b *BadgerIndexStore) LRUSize() int64
LRUSize implements IndexStore.
func (*BadgerIndexStore) LocalCount ¶
func (b *BadgerIndexStore) LocalCount() int64
LocalCount implements IndexStore.
func (*BadgerIndexStore) MarkObjectOffline ¶
func (b *BadgerIndexStore) MarkObjectOffline(key entities.Hash) error
MarkObjectOffline implements IndexStore.
func (*BadgerIndexStore) MarkObjectRemote ¶
func (b *BadgerIndexStore) MarkObjectRemote(key entities.Hash) error
MarkObjectRemote implements IndexStore.
func (*BadgerIndexStore) PeekOldest ¶
func (b *BadgerIndexStore) PeekOldest() (KeyTimestamp, error)
ReadLRU implements IndexStore.
func (*BadgerIndexStore) Put ¶
func (b *BadgerIndexStore) Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)
Put implements IndexStore. There are cases where concurrent writes to the same key can occur. Put() manipulates entry:: keys. If entry is marked Remote, it also updates lru:: keys. Delete() manipulates both lru:: and entry:: keys Use() reads entry:: keys and writes lru:: keys
func (*BadgerIndexStore) PutMany ¶
func (b *BadgerIndexStore) PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)
PutMany implements IndexStore. It records multiple index entries in a single BadgerDB WriteBatch and updates LRU tracking in bulk.
func (*BadgerIndexStore) ReadAll ¶
func (b *BadgerIndexStore) ReadAll() ([]*IndexKeyEntry, error)
ReadAll implements IndexStore.
func (*BadgerIndexStore) ReadLRU ¶
func (b *BadgerIndexStore) ReadLRU() ([]KeyTimestamp, error)
ReadLRU implements IndexStore.
func (*BadgerIndexStore) TotalSize ¶
func (b *BadgerIndexStore) TotalSize() int64
TotalSize implements IndexStore.
type BadgerMapsStore ¶
type BadgerMapsStore struct {
BaseBadgerStore
// contains filtered or unexported fields
}
BadgerMapsStore implements MapsStore using BadgerDB as a backing store. It does not require any GC because the value space is all nil.
func NewBadgerMapStore ¶
func NewBadgerMapStore(path string, printer Printer, ttlHours time.Duration) *BadgerMapsStore
type BaseBadgerStore ¶
type BaseBadgerStore struct {
// contains filtered or unexported fields
}
BaseBadgerStore can be embedded in other structs for re-use.
type BlockingQueue ¶
type BlockingQueue[T any] struct { // contains filtered or unexported fields }
BlockingQueue is an unbounded thread-safe queue.
func NewBlockingQueue ¶
func NewBlockingQueue[T any]() *BlockingQueue[T]
func (*BlockingQueue[T]) Close ¶
func (q *BlockingQueue[T]) Close()
func (*BlockingQueue[T]) Pop ¶
func (q *BlockingQueue[T]) Pop() (T, bool)
func (*BlockingQueue[T]) Push ¶
func (q *BlockingQueue[T]) Push(item T)
type CacheMan ¶
type CacheMan struct {
// contains filtered or unexported fields
}
func (*CacheMan) EnqueueNewObject ¶
func (c *CacheMan) EnqueueNewObject(hash entities.Hash, objType protocol.RequestObjectType)
EnqueueNewObject pushes locally created object notifications to an asynchronous queue for background indexing and eager-push without blocking the socket reader.
func (*CacheMan) FetchCASBlob ¶
func (*CacheMan) FetchObject ¶
func (*CacheMan) HealthStatus ¶
func (*CacheMan) NewObject ¶
func (c *CacheMan) NewObject(hash entities.Hash, objType protocol.RequestObjectType)
func (*CacheMan) NotifyUse ¶
NotifyUse records a touched hash into an in-memory deduplicated aggregator, which is flushed periodically in batches by notifyUseWorker.
func (*CacheMan) Phony ¶
func (c *CacheMan) Phony(priority dagsync.RequestPriority, objects []entities.Hash, callback dagsync.Callback)
Phony - callback could be called multiple times with errors before a successful call because of retries
func (*CacheMan) PrefetchGitObject ¶
func (*CacheMan) RunLocalGC ¶
func (*CacheMan) Stats ¶
func (c *CacheMan) Stats() protocol.CacheManStats
type CheckCredsFunc ¶
type Connection ¶
type Connection struct {
// contains filtered or unexported fields
}
func (*Connection) Close ¶
func (c *Connection) Close() error
func (*Connection) Serve ¶
func (c *Connection) Serve() error
type DefaultPrinter ¶
type DefaultPrinter struct {
}
func (*DefaultPrinter) Close ¶
func (d *DefaultPrinter) Close() error
func (*DefaultPrinter) Fatal ¶
func (d *DefaultPrinter) Fatal(a ...any)
func (*DefaultPrinter) Fatalf ¶
func (d *DefaultPrinter) Fatalf(format string, a ...any)
func (*DefaultPrinter) Printf ¶
func (d *DefaultPrinter) Printf(format string, a ...any)
func (*DefaultPrinter) Println ¶
func (d *DefaultPrinter) Println(a ...any)
type EmptyIndex ¶
type EmptyIndex struct{}
func (*EmptyIndex) Close ¶
func (e *EmptyIndex) Close() error
func (*EmptyIndex) DeleteMany ¶
func (*EmptyIndex) EntriesWithTotalSize ¶
func (e *EmptyIndex) EntriesWithTotalSize() ([]*IndexKeyEntry, uint64, error)
func (*EmptyIndex) EvictCount ¶
func (e *EmptyIndex) EvictCount() int64
func (*EmptyIndex) EvictEnd ¶
func (e *EmptyIndex) EvictEnd() time.Time
func (*EmptyIndex) EvictStart ¶
func (e *EmptyIndex) EvictStart() time.Time
func (*EmptyIndex) ForEach ¶
func (e *EmptyIndex) ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error
func (*EmptyIndex) GCEnd ¶
func (e *EmptyIndex) GCEnd() time.Time
func (*EmptyIndex) GCStart ¶
func (e *EmptyIndex) GCStart() time.Time
func (*EmptyIndex) Get ¶
func (e *EmptyIndex) Get(hash entities.Hash) *IndexEntry
func (*EmptyIndex) IndexCount ¶
func (e *EmptyIndex) IndexCount() int64
func (*EmptyIndex) LRUSize ¶
func (e *EmptyIndex) LRUSize() int64
func (*EmptyIndex) LocalCount ¶
func (e *EmptyIndex) LocalCount() int64
func (*EmptyIndex) LockEviction ¶
func (e *EmptyIndex) LockEviction() func()
func (*EmptyIndex) OldestUsed ¶
func (e *EmptyIndex) OldestUsed() time.Time
func (*EmptyIndex) Put ¶
func (e *EmptyIndex) Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)
func (*EmptyIndex) PutMany ¶
func (e *EmptyIndex) PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)
func (*EmptyIndex) ReadLRU ¶
func (e *EmptyIndex) ReadLRU() ([]KeyTimestamp, error)
func (*EmptyIndex) Sweep ¶
func (e *EmptyIndex) Sweep(timestampThreshold int64) (deletedCount int, err error)
func (*EmptyIndex) TotalSize ¶
func (e *EmptyIndex) TotalSize() uint64
type EntryType ¶
type EntryType int
func AllEntryTypes ¶
func AllEntryTypes() []EntryType
func ObjectTypeToEntryType ¶
func StringToEntryType ¶
func (EntryType) MarshalJSON ¶
func (*EntryType) UnmarshalJSON ¶
type FetchReq ¶
type FetchReqQueue ¶
type FetchReqQueue struct {
// contains filtered or unexported fields
}
func NewFetchReqQueue ¶
func NewFetchReqQueue() *FetchReqQueue
func (*FetchReqQueue) Close ¶
func (q *FetchReqQueue) Close()
func (*FetchReqQueue) Pop ¶
func (q *FetchReqQueue) Pop() (FetchReq, bool)
func (*FetchReqQueue) Push ¶
func (q *FetchReqQueue) Push(req FetchReq)
type Index ¶
type Index interface {
Get(hash entities.Hash) *IndexEntry
Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)
PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)
EvictCount() int64
LockEviction() func()
Delete(hash entities.Hash) error
EntriesWithTotalSize() ([]*IndexKeyEntry, uint64, error)
TotalSize() uint64
ReadLRU() ([]KeyTimestamp, error)
Use(hash entities.Hash) error
Close() error
KeepAlive(key entities.Hash) error
Sweep(timestampThreshold int64) (deletedCount int, err error)
// Metrics and internal status getters
LRUSize() int64
LocalCount() int64
IndexCount() int64
GCStart() time.Time
GCEnd() time.Time
EvictStart() time.Time
EvictEnd() time.Time
OldestUsed() time.Time
// Internal/store methods exposed for direct usage
UseMany(hashes []entities.Hash) error
DeleteMany(hashes []entities.Hash) (int64, int64, error)
ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error
// contains filtered or unexported methods
}
func NewIndex ¶
func NewIndex(opts *IndexOptions) (Index, error)
type IndexEntry ¶
type IndexEntry struct {
EntryType EntryType `json:",omitempty"`
// Content exists as a loose file.
// We do not currently use this field,
// it is plumbing for the future.
Loose bool `json:",omitempty"`
// Object is Local.
// We do not currently use this field,
// it is plumbing for the future.
Local bool `json:",omitempty"`
// Content has been pushed to remote.
Remote bool `json:",omitempty"`
// Content should not be pushed to remote.
Offline bool `json:",omitempty"`
// Size in bytes of the content, if known.
Size uint64 `json:",omitempty"`
// It will used for garbage collection
LastAliveTime time.Time `json:",omitempty"`
}
IndexEntry is the data associated with a hash in the index.
func (*IndexEntry) Equal ¶
func (i *IndexEntry) Equal(entry *IndexEntry) bool
func (*IndexEntry) LocalCount ¶
func (i *IndexEntry) LocalCount() int64
func (*IndexEntry) LocalSize ¶
func (i *IndexEntry) LocalSize() uint64
func (*IndexEntry) ShallowCopy ¶
func (i *IndexEntry) ShallowCopy() IndexEntry
type IndexKeyEntry ¶
type IndexKeyEntry struct {
Key entities.Hash
*IndexEntry
}
func (*IndexKeyEntry) MarshalJSON ¶
func (i *IndexKeyEntry) MarshalJSON() ([]byte, error)
func (*IndexKeyEntry) UnmarshalJSON ¶
func (i *IndexKeyEntry) UnmarshalJSON(data []byte) error
type IndexOptions ¶
type IndexOptions struct {
IndexStore IndexStore
Cache *localcache.LocalCache
EvictThreshold uint64
Printer Printer
LogDir string
}
type IndexReqQueue ¶
type IndexReqQueue = BlockingQueue[IndexReq]
IndexReqQueue is an unbounded dynamic queue for offloading index registration after fetched objects have been written to disk.
func NewIndexReqQueue ¶
func NewIndexReqQueue() *IndexReqQueue
type IndexStore ¶
type IndexStore interface {
// Open must be called on any store prior to use.
Open() error
// Get returns the index entry associated with the hash, or
// nil if one cannot be found.
Get(hash entities.Hash) *IndexEntry
// Put adds an index entry to the store and returns both the updated total size of the index,
// and the updated total LRU Size.
// If the entry already exists, it is not completely replaced, but rather merged.
// This prevents clobbering of state like 'Remote'.
// Callers may want to implement a retry mechanism in case of transaction conflicts.
Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)
// PutMany adds multiple index entries to the store and returns both the updated total size of the index,
// and the updated total LRU Size.
PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)
// ForEach iterates over the keys and values and calls the provided function.
ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error
// MarkObjectRemote marks an existing entry as Remote.
MarkObjectRemote(key entities.Hash) error
// MarkObjectOffline marks an existing entry as Offline.
MarkObjectOffline(key entities.Hash) error
// Delete removes the index entry and LRU information associated with the hash.
// It returns the deleted key and the index size after the deletion.
// Returns nil on non-existent entry.
Delete(hash entities.Hash) (*IndexKeyEntry, int64, int64, error)
// DeleteMany removes multiple hashes from the index.
// It returns the index size and lru size after all the deletions.
// Non-existent entries are ignored.
DeleteMany(hashes []entities.Hash) (int64, int64, error)
// Use should be called to update LRU tracking information.
Use(hash entities.Hash) error
// UseMany can be used to "touch" multiple objects.
UseMany(hashes []entities.Hash) error
// Evict returns the evicted key entries and the size of the index after
// the deletion.
Evict() ([]*IndexKeyEntry, int64, int64, error)
// ReadAll returns all index entries.
ReadAll() ([]*IndexKeyEntry, error)
// ReadLRU returns the LRU list in oldest->newest order.
ReadLRU() ([]KeyTimestamp, error)
// PeekOldest returns the oldest item in the LRU list.
PeekOldest() (KeyTimestamp, error)
// TotalSize returns the size of the entries tracked by the index,
// not the size of the index metadata itself.
TotalSize() int64
// LRUSize returns the size of the entries tracked in LRU.
LRUSize() int64
LocalCount() int64
IndexCount() int64
GCStart() time.Time
GCEnd() time.Time
// Close closes the store.
Close() error
// KeepAlive should be called to update the last alive time of the entry.
KeepAlive(key entities.Hash) error
}
IndexStore implements the API to access index entries and support LRU functionality.
type KeyTimestamp ¶
type Maps ¶
type Maps struct {
// contains filtered or unexported fields
}
type NewARESFunc ¶
type NewObjectReq ¶
type NewObjectReq struct {
Hash entities.Hash
ObjType protocol.RequestObjectType
}
type NewObjectReqQueue ¶
type NewObjectReqQueue struct {
// contains filtered or unexported fields
}
NewObjectReqQueue is an unbounded dynamic queue for offloading local object index registration and remote eager-push after new objects are written by CASFS. It deduplicates pending requests in-memory to prevent duplicate hash conflicts.
func NewNewObjectReqQueue ¶
func NewNewObjectReqQueue() *NewObjectReqQueue
func (*NewObjectReqQueue) Close ¶
func (q *NewObjectReqQueue) Close()
func (*NewObjectReqQueue) Len ¶
func (q *NewObjectReqQueue) Len() int
func (*NewObjectReqQueue) Pop ¶
func (q *NewObjectReqQueue) Pop() (NewObjectReq, bool)
func (*NewObjectReqQueue) Push ¶
func (q *NewObjectReqQueue) Push(req NewObjectReq)
type NotifyUseAggregator ¶
type NotifyUseAggregator struct {
// contains filtered or unexported fields
}
NotifyUseAggregator coalesces and deduplicates high-frequency LRU touch notifications in-memory to prevent BadgerDB transaction thrashing on the socket connection loop.
func NewNotifyUseAggregator ¶
func NewNotifyUseAggregator() *NotifyUseAggregator
func (*NotifyUseAggregator) Add ¶
func (a *NotifyUseAggregator) Add(hash entities.Hash)
func (*NotifyUseAggregator) Flush ¶
func (a *NotifyUseAggregator) Flush() []entities.Hash
type PendingAddTracker ¶
type PendingAddTracker struct {
// contains filtered or unexported fields
}
PendingAddTracker tracks in-flight fetched objects that have been written to disk but are still awaiting asynchronous index registration.
Operations on this tracker are purely in-memory (no file stating, no disk I/O, and no BadgerDB transaction conflicts). It is therefore low latency to call synchronously on the fetch response path before unblocking CASFS.
By recording hashes here synchronously before performCallbacks runs, we eliminate the race window where a process reading the fetched file immediately writes it to a CASFS mount and triggers NewObject before indexObjectsWorker has committed the index entry.
func NewPendingAddTracker ¶
func NewPendingAddTracker() *PendingAddTracker
func (*PendingAddTracker) Add ¶
func (p *PendingAddTracker) Add(hash entities.Hash)
func (*PendingAddTracker) Remove ¶
func (p *PendingAddTracker) Remove(hash entities.Hash)
type Printer ¶
type Printer interface {
Printf(format string, a ...any)
Println(a ...any)
Fatalf(format string, a ...any)
Fatal(a ...any)
Close() error
}
func PrinterOrDefault ¶
type RelativeFetchReq ¶
type RelativeFetchReq struct {
FetchReq
FetchStat *relativeFetchStat
RequestType entities.ObjectType
}
type ResilientPrinter ¶
type ResilientPrinter struct {
// contains filtered or unexported fields
}
ResilientPrinter writes a file at a configured path and will re-create the file when it is unlinked.
func NewResilientPrinter ¶
func NewResilientPrinter(path string) (*ResilientPrinter, error)
func (*ResilientPrinter) Close ¶
func (r *ResilientPrinter) Close() error
func (*ResilientPrinter) Fatal ¶
func (r *ResilientPrinter) Fatal(a ...any)
func (*ResilientPrinter) Fatalf ¶
func (r *ResilientPrinter) Fatalf(format string, a ...any)
func (*ResilientPrinter) Printf ¶
func (r *ResilientPrinter) Printf(format string, a ...any)
func (*ResilientPrinter) Println ¶
func (r *ResilientPrinter) Println(a ...any)
type TunnelCommand ¶
TunnelCommand abstracts the execution of external commands for testing.