server

package
v0.1.23-oss Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 3, 2026 License: Apache-2.0 Imports: 62 Imported by: 0

Documentation

Index

Constants

View Source
const (

	// Batch evictions for performance
	// If this value is too large, we may exceed transaction limits.
	MaxEvictBatchSize = 10000

	// The time to keep evicted entries in the index.
	EvictTTL = 24 * time.Hour
)
View Source
const (
	HealthBypassFileName = "health_bypass"
	StatusDirName        = "status"
	RemoteErrFileName    = "remote.err"
	CredsErrFileName     = "creds.err"
)
View Source
const (
	ResultWait     = protocol.ResultWait
	ResultSuccess  = protocol.ResultSuccess
	ResultNotFound = protocol.ResultNotFound
	ResultFailed   = protocol.ResultFailed
	ResultTimeout  = protocol.ResultTimeout
)
View Source
const DefaultTTLHours = 8

Variables

View Source
var ErrMissingEntry = errors.New("missing index entry")
View Source
var ErrProtocol = errors.New("cacheman binary protocol error")
View Source
var ErrTxnConflict = errors.New("transaction conflict")
View Source
var ErrTxnTooBig = errors.New("transaction too big")

Functions

func PortForward

func PortForward(p Printer, server string, port int) (*tunnel, error)

PortForward maintains an SSH tunnel that forwards localhost:port to localhost:<rport> on the remote host, where server is "<host>:<rport>" (an "https://" prefix is ignored). The SSH destination is the bare host name, so any user, identity, or proxy settings must be configured in ~/.ssh/config, and authentication must work without prompting.

func SetDebug

func SetDebug(d bool)

Types

type ABFSClient

type ABFSClient interface {
	Close() error
	AddMapResult(ctx context.Context, from, to entities.Hash) error
	GetObject(ctx context.Context, key remote.Hash) (remote.ContentReadCloser, error)
	GetRefs(ctx context.Context, server string, project string, refs []string) ([]entities.Hash, error)
	Headers(ctx context.Context, keys []remote.Hash) ([]entities.ObjectHeader, error)
	SHA256(ctx context.Context, keys []remote.Hash) ([]entities.SHA256, error)
	MapResults(ctx context.Context, id entities.Hash) ([]entities.Hash, error)
	PutObject(ctx context.Context, key remote.Hash, reader remote.ContentReadCloser) error
	PutRefs(ctx context.Context, server string, project string, refs []entities.RefHash) error
	SearchRefs(ctx context.Context, server string, project string, refPrefix string, limit int) ([]entities.ProjectRefHash, error)
	Mark(ctx context.Context, hash entities.Hash, key, value string) (int64, error)
	Version(ctx context.Context) (string, error)
	WhoAmI(ctx context.Context) (string, error)
}

type ARESClient

type ARESClient interface {
	remote.ARESService
	// Support returns whether remote execution is available
	Support() bool
}

type BadgerIndexStore

type BadgerIndexStore struct {
	BaseBadgerStore
	// contains filtered or unexported fields
}

BadgerIndexStore implements IndexStore using BadgerDB as a backing store. Note that the notion of "Use" here is external - it's not directly associated with the IndexEntry usage. Rather it's the usage of the associated localcache object.

func NewBadgerIndexStore

func NewBadgerIndexStore(path string, printer Printer, evictBatchSize uint, trackUse bool) *BadgerIndexStore

NewBadgerIndexStore creates a BadgerIndexStore. On an Evict() call, up-to evictBatchSize number of keys will be evicted. The caller may need to re-call Evict() to reduce the size as desired.

func (*BadgerIndexStore) Close

func (b *BadgerIndexStore) Close() error

Close implements IndexStore.

func (*BadgerIndexStore) Delete

Delete implements IndexStore.

func (*BadgerIndexStore) DeleteMany

func (b *BadgerIndexStore) DeleteMany(hashes []entities.Hash) (int64, int64, error)

DeleteMany implements IndexStore. Multiple transactions are used if hashes is large.

func (*BadgerIndexStore) Evict

func (b *BadgerIndexStore) Evict() ([]*IndexKeyEntry, int64, int64, error)

Evict implements IndexStore.

func (*BadgerIndexStore) ForEach

func (b *BadgerIndexStore) ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error

ForEach implements IndexStore.

func (*BadgerIndexStore) GCEnd

func (b *BadgerIndexStore) GCEnd() time.Time

GCEnd implements IndexStore.

func (*BadgerIndexStore) GCStart

func (b *BadgerIndexStore) GCStart() time.Time

GCStart implements IndexStore.

func (*BadgerIndexStore) Get

func (b *BadgerIndexStore) Get(hash entities.Hash) *IndexEntry

Get implements IndexStore.

func (*BadgerIndexStore) IndexCount

func (b *BadgerIndexStore) IndexCount() int64

IndexCount implements IndexStore.

func (*BadgerIndexStore) KeepAlive

func (b *BadgerIndexStore) KeepAlive(key entities.Hash) error

KeepAlive implements IndexStore.

func (*BadgerIndexStore) LRUSize

func (b *BadgerIndexStore) LRUSize() int64

LRUSize implements IndexStore.

func (*BadgerIndexStore) LocalCount

func (b *BadgerIndexStore) LocalCount() int64

LocalCount implements IndexStore.

func (*BadgerIndexStore) MarkObjectOffline

func (b *BadgerIndexStore) MarkObjectOffline(key entities.Hash) error

MarkObjectOffline implements IndexStore.

func (*BadgerIndexStore) MarkObjectRemote

func (b *BadgerIndexStore) MarkObjectRemote(key entities.Hash) error

MarkObjectRemote implements IndexStore.

func (*BadgerIndexStore) Open

func (b *BadgerIndexStore) Open() error

Open implements IndexStore.

func (*BadgerIndexStore) PeekOldest

func (b *BadgerIndexStore) PeekOldest() (KeyTimestamp, error)

ReadLRU implements IndexStore.

func (*BadgerIndexStore) Put

func (b *BadgerIndexStore) Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)

Put implements IndexStore. There are cases where concurrent writes to the same key can occur. Put() manipulates entry:: keys. If entry is marked Remote, it also updates lru:: keys. Delete() manipulates both lru:: and entry:: keys Use() reads entry:: keys and writes lru:: keys

func (*BadgerIndexStore) PutMany

func (b *BadgerIndexStore) PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)

PutMany implements IndexStore. It records multiple index entries in a single BadgerDB WriteBatch and updates LRU tracking in bulk.

func (*BadgerIndexStore) ReadAll

func (b *BadgerIndexStore) ReadAll() ([]*IndexKeyEntry, error)

ReadAll implements IndexStore.

func (*BadgerIndexStore) ReadLRU

func (b *BadgerIndexStore) ReadLRU() ([]KeyTimestamp, error)

ReadLRU implements IndexStore.

func (*BadgerIndexStore) TotalSize

func (b *BadgerIndexStore) TotalSize() int64

TotalSize implements IndexStore.

func (*BadgerIndexStore) Use

func (b *BadgerIndexStore) Use(hash entities.Hash) error

Use implements IndexStore. It updates the LRU key for a hash to the current timestamp.

func (*BadgerIndexStore) UseMany

func (b *BadgerIndexStore) UseMany(hashes []entities.Hash) error

UseMany implements IndexStore.

type BadgerMapsStore

type BadgerMapsStore struct {
	BaseBadgerStore
	// contains filtered or unexported fields
}

BadgerMapsStore implements MapsStore using BadgerDB as a backing store. It does not require any GC because the value space is all nil.

func NewBadgerMapStore

func NewBadgerMapStore(path string, printer Printer, ttlHours time.Duration) *BadgerMapsStore

func (*BadgerMapsStore) Close

func (b *BadgerMapsStore) Close() error

Close implements MapsStore.

func (*BadgerMapsStore) Get

func (b *BadgerMapsStore) Get(from entities.Hash) ([]entities.Hash, error)

Get implements MapsStore.

func (*BadgerMapsStore) Open

func (b *BadgerMapsStore) Open() error

Open implements MapsStore.

func (*BadgerMapsStore) Put

func (b *BadgerMapsStore) Put(from, to entities.Hash) error

Put implements MapsStore.

type BaseBadgerStore

type BaseBadgerStore struct {
	// contains filtered or unexported fields
}

BaseBadgerStore can be embedded in other structs for re-use.

type BlockingQueue

type BlockingQueue[T any] struct {
	// contains filtered or unexported fields
}

BlockingQueue is an unbounded thread-safe queue.

func NewBlockingQueue

func NewBlockingQueue[T any]() *BlockingQueue[T]

func (*BlockingQueue[T]) Close

func (q *BlockingQueue[T]) Close()

func (*BlockingQueue[T]) Pop

func (q *BlockingQueue[T]) Pop() (T, bool)

func (*BlockingQueue[T]) Push

func (q *BlockingQueue[T]) Push(item T)

type CacheMan

type CacheMan struct {
	// contains filtered or unexported fields
}

func New

func New(inst *instance.Instance, newARES NewARESFunc) (*CacheMan, error)

func (*CacheMan) EnqueueNewObject

func (c *CacheMan) EnqueueNewObject(hash entities.Hash, objType protocol.RequestObjectType)

EnqueueNewObject pushes locally created object notifications to an asynchronous queue for background indexing and eager-push without blocking the socket reader.

func (*CacheMan) ErrorJSON

func (c *CacheMan) ErrorJSON(w http.ResponseWriter, r *http.Request, err error, status int)

func (*CacheMan) FetchARI

func (c *CacheMan) FetchARI(ari string, priority float64, callback func(error))

func (*CacheMan) FetchCASBlob

func (c *CacheMan) FetchCASBlob(sha256Hex string, size int64, priority float64, callback func(error))

func (*CacheMan) FetchMap

func (c *CacheMan) FetchMap(hash entities.Hash, priority float64, callback func(error))

func (*CacheMan) FetchObject

func (c *CacheMan) FetchObject(hash entities.Hash, objType protocol.RequestObjectType, priority float64, callback func(error))

func (*CacheMan) GetStat

func (c *CacheMan) GetStat(h entities.Hash) (entities.StatInfo, error)

func (*CacheMan) HealthStatus

func (c *CacheMan) HealthStatus() int32

func (*CacheMan) MapResults

func (c *CacheMan) MapResults(hash entities.Hash) []entities.Hash

func (*CacheMan) NewMap

func (c *CacheMan) NewMap(from, to entities.Hash, localOnly bool)

func (*CacheMan) NewObject

func (c *CacheMan) NewObject(hash entities.Hash, objType protocol.RequestObjectType)

func (*CacheMan) NotifyUse

func (c *CacheMan) NotifyUse(hash entities.Hash)

NotifyUse records a touched hash into an in-memory deduplicated aggregator, which is flushed periodically in batches by notifyUseWorker.

func (*CacheMan) Phony

func (c *CacheMan) Phony(priority dagsync.RequestPriority, objects []entities.Hash, callback dagsync.Callback)

Phony - callback could be called multiple times with errors before a successful call because of retries

func (*CacheMan) Prefetch

func (c *CacheMan) Prefetch(ctx context.Context, blobs, objects []entities.Hash, priority float64) error

func (*CacheMan) PrefetchGitObject

func (c *CacheMan) PrefetchGitObject(hash entities.Hash, callback func(error))

func (*CacheMan) Printer

func (c *CacheMan) Printer() Printer

func (*CacheMan) ReadJSON

func (c *CacheMan) ReadJSON(w http.ResponseWriter, r *http.Request, value any) bool

func (*CacheMan) RunLocalGC

func (c *CacheMan) RunLocalGC() error

func (*CacheMan) Serve

func (c *CacheMan) Serve(ctx context.Context) error

func (*CacheMan) Stats

func (c *CacheMan) Stats() protocol.CacheManStats

func (*CacheMan) Stop

func (c *CacheMan) Stop() error

func (*CacheMan) WriteJSON

func (c *CacheMan) WriteJSON(w http.ResponseWriter, r *http.Request, value any)

type CheckCredsFunc

type CheckCredsFunc func(ctx context.Context) error

type Connection

type Connection struct {
	// contains filtered or unexported fields
}

func (*Connection) Close

func (c *Connection) Close() error

func (*Connection) Serve

func (c *Connection) Serve() error

type DefaultPrinter

type DefaultPrinter struct {
}

func (*DefaultPrinter) Close

func (d *DefaultPrinter) Close() error

func (*DefaultPrinter) Fatal

func (d *DefaultPrinter) Fatal(a ...any)

func (*DefaultPrinter) Fatalf

func (d *DefaultPrinter) Fatalf(format string, a ...any)

func (*DefaultPrinter) Printf

func (d *DefaultPrinter) Printf(format string, a ...any)

func (*DefaultPrinter) Println

func (d *DefaultPrinter) Println(a ...any)

type EmptyIndex

type EmptyIndex struct{}

func (*EmptyIndex) Close

func (e *EmptyIndex) Close() error

func (*EmptyIndex) Delete

func (e *EmptyIndex) Delete(hash entities.Hash) error

func (*EmptyIndex) DeleteMany

func (e *EmptyIndex) DeleteMany(hashes []entities.Hash) (int64, int64, error)

func (*EmptyIndex) EntriesWithTotalSize

func (e *EmptyIndex) EntriesWithTotalSize() ([]*IndexKeyEntry, uint64, error)

func (*EmptyIndex) EvictCount

func (e *EmptyIndex) EvictCount() int64

func (*EmptyIndex) EvictEnd

func (e *EmptyIndex) EvictEnd() time.Time

func (*EmptyIndex) EvictStart

func (e *EmptyIndex) EvictStart() time.Time

func (*EmptyIndex) ForEach

func (e *EmptyIndex) ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error

func (*EmptyIndex) GCEnd

func (e *EmptyIndex) GCEnd() time.Time

func (*EmptyIndex) GCStart

func (e *EmptyIndex) GCStart() time.Time

func (*EmptyIndex) Get

func (e *EmptyIndex) Get(hash entities.Hash) *IndexEntry

func (*EmptyIndex) IndexCount

func (e *EmptyIndex) IndexCount() int64

func (*EmptyIndex) KeepAlive

func (e *EmptyIndex) KeepAlive(key entities.Hash) error

func (*EmptyIndex) LRUSize

func (e *EmptyIndex) LRUSize() int64

func (*EmptyIndex) LocalCount

func (e *EmptyIndex) LocalCount() int64

func (*EmptyIndex) LockEviction

func (e *EmptyIndex) LockEviction() func()

func (*EmptyIndex) OldestUsed

func (e *EmptyIndex) OldestUsed() time.Time

func (*EmptyIndex) Put

func (e *EmptyIndex) Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)

func (*EmptyIndex) PutMany

func (e *EmptyIndex) PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)

func (*EmptyIndex) ReadLRU

func (e *EmptyIndex) ReadLRU() ([]KeyTimestamp, error)

func (*EmptyIndex) Sweep

func (e *EmptyIndex) Sweep(timestampThreshold int64) (deletedCount int, err error)

func (*EmptyIndex) TotalSize

func (e *EmptyIndex) TotalSize() uint64

func (*EmptyIndex) Use

func (e *EmptyIndex) Use(hash entities.Hash) error

func (*EmptyIndex) UseMany

func (e *EmptyIndex) UseMany(hashes []entities.Hash) error

type EntryType

type EntryType int
const (
	EntryTypeInvalid EntryType = iota
	EntryTypeBlob
	EntryTypeTree
	EntryTypeObject
	EntryTypeARI
)

func AllEntryTypes

func AllEntryTypes() []EntryType

func ObjectTypeToEntryType

func ObjectTypeToEntryType(objType string) EntryType

func StringToEntryType

func StringToEntryType(str string) EntryType

func (EntryType) MarshalJSON

func (e EntryType) MarshalJSON() ([]byte, error)

func (EntryType) String

func (e EntryType) String() string

func (*EntryType) UnmarshalJSON

func (e *EntryType) UnmarshalJSON(data []byte) error

type FetchReq

type FetchReq struct {
	Hash      entities.Hash
	ARI       string
	SHA256Hex string
	Size      int64
	Priority  float64
	Time      time.Time
	ObjType   protocol.RequestObjectType
	// contains filtered or unexported fields
}

func (*FetchReq) Callback

func (f *FetchReq) Callback(err error)

type FetchReqQueue

type FetchReqQueue struct {
	// contains filtered or unexported fields
}

func NewFetchReqQueue

func NewFetchReqQueue() *FetchReqQueue

func (*FetchReqQueue) Close

func (q *FetchReqQueue) Close()

func (*FetchReqQueue) Pop

func (q *FetchReqQueue) Pop() (FetchReq, bool)

func (*FetchReqQueue) Push

func (q *FetchReqQueue) Push(req FetchReq)

type Index

type Index interface {
	Get(hash entities.Hash) *IndexEntry
	Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)
	PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)
	EvictCount() int64
	LockEviction() func()
	Delete(hash entities.Hash) error
	EntriesWithTotalSize() ([]*IndexKeyEntry, uint64, error)
	TotalSize() uint64
	ReadLRU() ([]KeyTimestamp, error)
	Use(hash entities.Hash) error
	Close() error
	KeepAlive(key entities.Hash) error
	Sweep(timestampThreshold int64) (deletedCount int, err error)

	// Metrics and internal status getters
	LRUSize() int64
	LocalCount() int64
	IndexCount() int64
	GCStart() time.Time
	GCEnd() time.Time
	EvictStart() time.Time
	EvictEnd() time.Time
	OldestUsed() time.Time

	// Internal/store methods exposed for direct usage
	UseMany(hashes []entities.Hash) error
	DeleteMany(hashes []entities.Hash) (int64, int64, error)
	ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error
	// contains filtered or unexported methods
}

func NewIndex

func NewIndex(opts *IndexOptions) (Index, error)

type IndexEntry

type IndexEntry struct {
	EntryType EntryType `json:",omitempty"`

	// Content exists as a loose file.
	// We do not currently use this field,
	// it is plumbing for the future.
	Loose bool `json:",omitempty"`

	// Object is Local.
	// We do not currently use this field,
	// it is plumbing for the future.
	Local bool `json:",omitempty"`

	// Content has been pushed to remote.
	Remote bool `json:",omitempty"`

	// Content should not be pushed to remote.
	Offline bool `json:",omitempty"`

	// Size in bytes of the content, if known.
	Size uint64 `json:",omitempty"`

	// It will used for garbage collection
	LastAliveTime time.Time `json:",omitempty"`
}

IndexEntry is the data associated with a hash in the index.

func (*IndexEntry) Equal

func (i *IndexEntry) Equal(entry *IndexEntry) bool

func (*IndexEntry) LocalCount

func (i *IndexEntry) LocalCount() int64

func (*IndexEntry) LocalSize

func (i *IndexEntry) LocalSize() uint64

func (*IndexEntry) ShallowCopy

func (i *IndexEntry) ShallowCopy() IndexEntry

type IndexKeyEntry

type IndexKeyEntry struct {
	Key entities.Hash
	*IndexEntry
}

func (*IndexKeyEntry) MarshalJSON

func (i *IndexKeyEntry) MarshalJSON() ([]byte, error)

func (*IndexKeyEntry) UnmarshalJSON

func (i *IndexKeyEntry) UnmarshalJSON(data []byte) error

type IndexOptions

type IndexOptions struct {
	IndexStore     IndexStore
	Cache          *localcache.LocalCache
	EvictThreshold uint64
	Printer        Printer
	LogDir         string
}

type IndexReq

type IndexReq struct {
	Hash      entities.Hash
	EntryType EntryType
	ObjType   protocol.RequestObjectType
	ReqStat   *protocol.QueueStat
	RawType   string
	// Size carries the known on-disk size computed during fetch to avoid redundant disk I/O in indexObjectsWorker.
	Size int64
}

type IndexReqQueue

type IndexReqQueue = BlockingQueue[IndexReq]

IndexReqQueue is an unbounded dynamic queue for offloading index registration after fetched objects have been written to disk.

func NewIndexReqQueue

func NewIndexReqQueue() *IndexReqQueue

type IndexStore

type IndexStore interface {
	// Open must be called on any store prior to use.
	Open() error

	// Get returns the index entry associated with the hash, or
	// nil if one cannot be found.
	Get(hash entities.Hash) *IndexEntry

	// Put adds an index entry to the store and returns both the updated total size of the index,
	// and the updated total LRU Size.
	// If the entry already exists, it is not completely replaced, but rather merged.
	// This prevents clobbering of state like 'Remote'.
	// Callers may want to implement a retry mechanism in case of transaction conflicts.
	Put(hash entities.Hash, entry *IndexEntry) (newSize int64, newLRUSize int64, prevEnt, newEnt *IndexEntry, err error)

	// PutMany adds multiple index entries to the store and returns both the updated total size of the index,
	// and the updated total LRU Size.
	PutMany(entries []*IndexKeyEntry) (newSize int64, newLRUSize int64, err error)

	// ForEach iterates over the keys and values and calls the provided function.
	ForEach(fn func(key entities.Hash, entry *IndexEntry, expiresAt uint64) error) error

	// MarkObjectRemote marks an existing entry as Remote.
	MarkObjectRemote(key entities.Hash) error

	// MarkObjectOffline marks an existing entry as Offline.
	MarkObjectOffline(key entities.Hash) error

	// Delete removes the index entry and LRU information associated with the hash.
	// It returns the deleted key and the index size after the deletion.
	// Returns nil on non-existent entry.
	Delete(hash entities.Hash) (*IndexKeyEntry, int64, int64, error)

	// DeleteMany removes multiple hashes from the index.
	// It returns the index size and lru size after all the deletions.
	// Non-existent entries are ignored.
	DeleteMany(hashes []entities.Hash) (int64, int64, error)

	// Use should be called to update LRU tracking information.
	Use(hash entities.Hash) error

	// UseMany can be used to "touch" multiple objects.
	UseMany(hashes []entities.Hash) error

	// Evict returns the evicted key entries and the size of the index after
	// the deletion.
	Evict() ([]*IndexKeyEntry, int64, int64, error)

	// ReadAll returns all index entries.
	ReadAll() ([]*IndexKeyEntry, error)

	// ReadLRU returns the LRU list in oldest->newest order.
	ReadLRU() ([]KeyTimestamp, error)

	// PeekOldest returns the oldest item in the LRU list.
	PeekOldest() (KeyTimestamp, error)

	// TotalSize returns the size of the entries tracked by the index,
	// not the size of the index metadata itself.
	TotalSize() int64

	// LRUSize returns the size of the entries tracked in LRU.
	LRUSize() int64

	LocalCount() int64

	IndexCount() int64

	GCStart() time.Time
	GCEnd() time.Time

	// Close closes the store.
	Close() error

	// KeepAlive should be called to update the last alive time of the entry.
	KeepAlive(key entities.Hash) error
}

IndexStore implements the API to access index entries and support LRU functionality.

type KeyTimestamp

type KeyTimestamp struct {
	Key       entities.Hash
	Timestamp int64
}

type Maps

type Maps struct {
	// contains filtered or unexported fields
}

func NewMaps

func NewMaps(mapsStore MapsStore) (*Maps, error)

func (*Maps) Close

func (m *Maps) Close() error

func (*Maps) Get

func (m *Maps) Get(key entities.Hash) ([]entities.Hash, error)

Get returns all 'to' targets for a key in non-deterministic order.

func (*Maps) Put

func (m *Maps) Put(from, to entities.Hash) error

type MapsStore

type MapsStore interface {
	// Open must be called on any store prior to use.
	Open() error

	// Get returns all mappings for 'from'.
	Get(from entities.Hash) ([]entities.Hash, error)

	// Put adds a 'from'->'to' mapping.
	Put(from, to entities.Hash) error

	// Close closes the store.
	Close() error
}

type NewARESFunc

type NewARESFunc func(address string, authInfo instance.AuthInfo) (remote.ARESService, error)

type NewObjectReq

type NewObjectReq struct {
	Hash    entities.Hash
	ObjType protocol.RequestObjectType
}

type NewObjectReqQueue

type NewObjectReqQueue struct {
	// contains filtered or unexported fields
}

NewObjectReqQueue is an unbounded dynamic queue for offloading local object index registration and remote eager-push after new objects are written by CASFS. It deduplicates pending requests in-memory to prevent duplicate hash conflicts.

func NewNewObjectReqQueue

func NewNewObjectReqQueue() *NewObjectReqQueue

func (*NewObjectReqQueue) Close

func (q *NewObjectReqQueue) Close()

func (*NewObjectReqQueue) Len

func (q *NewObjectReqQueue) Len() int

func (*NewObjectReqQueue) Pop

func (q *NewObjectReqQueue) Pop() (NewObjectReq, bool)

func (*NewObjectReqQueue) Push

func (q *NewObjectReqQueue) Push(req NewObjectReq)

type NotifyUseAggregator

type NotifyUseAggregator struct {
	// contains filtered or unexported fields
}

NotifyUseAggregator coalesces and deduplicates high-frequency LRU touch notifications in-memory to prevent BadgerDB transaction thrashing on the socket connection loop.

func NewNotifyUseAggregator

func NewNotifyUseAggregator() *NotifyUseAggregator

func (*NotifyUseAggregator) Add

func (a *NotifyUseAggregator) Add(hash entities.Hash)

func (*NotifyUseAggregator) Flush

func (a *NotifyUseAggregator) Flush() []entities.Hash

type PendingAddTracker

type PendingAddTracker struct {
	// contains filtered or unexported fields
}

PendingAddTracker tracks in-flight fetched objects that have been written to disk but are still awaiting asynchronous index registration.

Operations on this tracker are purely in-memory (no file stating, no disk I/O, and no BadgerDB transaction conflicts). It is therefore low latency to call synchronously on the fetch response path before unblocking CASFS.

By recording hashes here synchronously before performCallbacks runs, we eliminate the race window where a process reading the fetched file immediately writes it to a CASFS mount and triggers NewObject before indexObjectsWorker has committed the index entry.

func NewPendingAddTracker

func NewPendingAddTracker() *PendingAddTracker

func (*PendingAddTracker) Add

func (p *PendingAddTracker) Add(hash entities.Hash)

func (*PendingAddTracker) Has

func (p *PendingAddTracker) Has(hash entities.Hash) bool

func (*PendingAddTracker) Remove

func (p *PendingAddTracker) Remove(hash entities.Hash)

type Printer

type Printer interface {
	Printf(format string, a ...any)
	Println(a ...any)

	Fatalf(format string, a ...any)
	Fatal(a ...any)

	Close() error
}

func PrinterOrDefault

func PrinterOrDefault(printer Printer) Printer

type RelativeFetchReq

type RelativeFetchReq struct {
	FetchReq
	FetchStat   *relativeFetchStat
	RequestType entities.ObjectType
}

type ResilientPrinter

type ResilientPrinter struct {
	// contains filtered or unexported fields
}

ResilientPrinter writes a file at a configured path and will re-create the file when it is unlinked.

func NewResilientPrinter

func NewResilientPrinter(path string) (*ResilientPrinter, error)

func (*ResilientPrinter) Close

func (r *ResilientPrinter) Close() error

func (*ResilientPrinter) Fatal

func (r *ResilientPrinter) Fatal(a ...any)

func (*ResilientPrinter) Fatalf

func (r *ResilientPrinter) Fatalf(format string, a ...any)

func (*ResilientPrinter) Printf

func (r *ResilientPrinter) Printf(format string, a ...any)

func (*ResilientPrinter) Println

func (r *ResilientPrinter) Println(a ...any)

type StatStore

type StatStore interface {
	Put(key entities.Hash, stat entities.StatInfo) error
	Get(key entities.Hash) (entities.StatInfo, bool)
}

type TunnelCommand

type TunnelCommand interface {
	Start() error
	Wait() error
	Kill() error
}

TunnelCommand abstracts the execution of external commands for testing.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL