Documentation
¶
Index ¶
- Constants
- Variables
- func DialWithTimeout(ctx context.Context, addr string) (net.Conn, error)
- func FSMetadataShard(id string) int
- func GenerateFsID(name string) string
- func GetLogger() *logger
- func GetPrivateIpAddr() (string, error)
- func GetPublicIpAddr() (string, error)
- func InitLogger(debugMode bool, prettyLogs bool)
- func IsStoreHostUnavailable(err error) bool
- func MarshalFSMetadata(metadata *FSMetadata) ([]byte, error)
- func Mount(ctx context.Context, opts FSSystemOpts) (func() error, <-chan error, *fuse.Server, error)
- func RecentStubKey(workspaceID, stubID string) string
- func SHA1StringToUint64(hash string) (uint64, error)
- func ToSlice(v interface{}) []interface{}
- func ToStruct(m map[string]string, out interface{}) error
- func WithClientName(name string) func(*redis.UniversalOptions)
- type BufferPool
- type CacheChurnEvent
- type CacheChurnSink
- type CacheFS
- type CacheFSNode
- type CacheMetadataStore
- type Client
- func NewClient(ctx context.Context, cfg Config) (*Client, error)
- func NewClientWithHostDirectory(ctx context.Context, cfg Config, metadataStore CacheMetadataStore, ...) (*Client, error)
- func NewClientWithMetadataStore(ctx context.Context, cfg Config, metadataStore CacheMetadataStore, ...) (*Client, error)
- func (c *Client) AttachLocalServer(server *Server)
- func (c *Client) CacheFSMetadata(ctx context.Context, path string) (*FSMetadata, error)
- func (c *Client) Cleanup() error
- func (c *Client) ClientLocalPageFileView(hash string, offset int64, length int64, opts ClientOptions) (path string, pageOffset int64, n int, ok bool, err error)
- func (c *Client) ClientLocalPageFileViews(hash string, offset int64, length int64, opts ClientOptions) (views []ClientLocalPageFileView, err error)
- func (c *Client) ClientLocalPageFileViewsWithTrace(hash string, offset int64, length int64, opts ClientOptions) (views []ClientLocalPageFileView, trace OperationTrace, err error)
- func (c *Client) DetachLocalServer(hostID string)
- func (c *Client) GetContent(hash string, offset int64, length int64, opts struct{ ... }) ([]byte, error)
- func (c *Client) GetContentStream(hash string, offset int64, length int64, opts struct{ ... }) (chan []byte, error)
- func (c *Client) GetNearbyHosts() ([]*Host, error)
- func (c *Client) GetState() error
- func (c *Client) HostsAvailable() bool
- func (c *Client) IsCachedOnSelectedHost(hash string, routingKey string, expectedSize ...int64) (bool, error)
- func (c *Client) IsCachedReachable(hash string, routingKey string) (bool, error)
- func (c *Client) IsCachedReachableContext(ctx context.Context, hash string, routingKey string) (bool, error)
- func (c *Client) IsPathCachedReachable(ctx context.Context, path string) bool
- func (c *Client) LookupVolumeObjectContentHash(ctx context.Context, identity VolumeObjectIdentity) (string, bool, error)
- func (c *Client) MaterializeFromReplica(ctx context.Context, server *Server, hash, routingKey string, size int64) (bool, error)
- func (c *Client) PrimaryReadHost(routingKey string) (*Host, error)
- func (c *Client) RankedReadHosts(routingKey string) []*Host
- func (c *Client) ReadContentInto(ctx context.Context, hash string, offset int64, dst []byte, opts ClientOptions) (read int64, err error)
- func (c *Client) ReadContentIntoWithTrace(ctx context.Context, hash string, offset int64, dst []byte, opts ClientOptions) (read int64, trace OperationTrace, err error)
- func (c *Client) SelectedStoreHostAvailable(hash string, routingKey string) bool
- func (c *Client) StoreContent(chunks chan []byte, hash string, opts struct{ ... }) (string, error)
- func (c *Client) StoreContentAtPath(content []byte, cachePath string, opts StoreContentOptions) (string, error)
- func (c *Client) StoreContentFromFUSE(source struct{ ... }, opts struct{ ... }) (string, error)
- func (c *Client) StoreContentFromLocalFile(source LocalContentSource, opts StoreContentOptions) (string, error)
- func (c *Client) StoreContentFromLocalFileWithTrace(source LocalContentSource, opts StoreContentOptions) (hash string, trace OperationTrace, err error)
- func (c *Client) StoreContentFromLocalPath(source struct{ ... }, opts struct{ ... }) (string, error)
- func (c *Client) StoreContentFromLocalSource(source LocalContentSource, opts StoreContentOptions) (string, error)
- func (c *Client) StoreContentFromS3(source struct{ ... }, opts struct{ ... }) (string, error)
- func (c *Client) StoreContentFromS3Source(source S3ContentSource, opts StoreContentOptions) (string, error)
- func (c *Client) StoreVolumeObjectContentHash(ctx context.Context, identity VolumeObjectIdentity, hash string) error
- func (c *Client) WaitForHosts(timeout time.Duration) error
- type ClientConfig
- type ClientLocalPageFileView
- type ClientOptions
- type ClientReadTransportConfig
- type ClientRequest
- type ClientRequestType
- type Config
- type Coordinator
- func (c *Coordinator) ListHosts(ctx context.Context, poolName, locality string) ([]CoordinatorHost, error)
- func (c *Coordinator) RegisterHost(ctx context.Context, host CoordinatorHost, ttl time.Duration) error
- func (c *Coordinator) UnregisterHost(ctx context.Context, poolName, locality, logicalHostID, registrationID string) error
- type CoordinatorConfig
- type CoordinatorHost
- type CoordinatorRepository
- type DiscoveryClient
- type DiskConfig
- type DiskUsage
- type ErrNodeNotFound
- type FSConfig
- type FSMetadata
- type FSNode
- func (n *FSNode) Create(ctx context.Context, name string, flags uint32, mode uint32, ...) (inode *fs.Inode, fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno)
- func (n *FSNode) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) syscall.Errno
- func (n *FSNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno)
- func (n *FSNode) Mkdir(ctx context.Context, name string, mode uint32, out *fuse.EntryOut) (*fs.Inode, syscall.Errno)
- func (n *FSNode) OnAdd(ctx context.Context)
- func (n *FSNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno)
- func (n *FSNode) Opendir(ctx context.Context) syscall.Errno
- func (n *FSNode) Read(ctx context.Context, f fs.FileHandle, dest []byte, off int64) (fuse.ReadResult, syscall.Errno)
- func (n *FSNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno)
- func (n *FSNode) Readlink(ctx context.Context) ([]byte, syscall.Errno)
- func (n *FSNode) Rename(ctx context.Context, oldName string, newParent fs.InodeEmbedder, ...) syscall.Errno
- func (n *FSNode) Rmdir(ctx context.Context, name string) syscall.Errno
- func (n *FSNode) Unlink(ctx context.Context, name string) syscall.Errno
- type FSSystemOpts
- type FileSystem
- type FileSystemOpts
- type FileSystemStorage
- type GlobalConfig
- type Host
- type HostDirectory
- type HostMap
- func (hm *HostMap) Closest(timeout time.Duration) (*Host, error)
- func (hm *HostMap) ClosestWithCapacity(timeout time.Duration) (*Host, error)
- func (hm *HostMap) DeactivateEndpoint(host *Host) (*Host, bool)
- func (hm *HostMap) Get(hostId string) *Host
- func (hm *HostMap) GetAll() []*Host
- func (hm *HostMap) Members() mapset.Set[string]
- func (hm *HostMap) Remove(host *Host) bool
- func (hm *HostMap) RemoveLogicalHost(hostID string) (*Host, bool)
- func (hm *HostMap) Set(host *Host)
- type JuiceFSConfig
- type JuiceFsSource
- type LocalContentSource
- type MemoryConfig
- type Metadata
- func (m *Metadata) AcquireReconcileLock(ctx context.Context, locality, logicalHost, hash string, ttlS int) (bool, error)
- func (m *Metadata) AddFsNodeChild(ctx context.Context, pid, id string) error
- func (m *Metadata) AddHostToIndex(ctx context.Context, locality string, host *Host) error
- func (m *Metadata) AddRecentStub(ctx context.Context, locality, workspaceID, stubID string, ttl time.Duration) error
- func (m *Metadata) GetAvailableHosts(ctx context.Context, locality string, removeHostCallback func(host *Host)) ([]*Host, error)
- func (m *Metadata) GetFsNode(ctx context.Context, id string) (*FSMetadata, error)
- func (m *Metadata) GetFsNodeChildren(ctx context.Context, id string) ([]*FSMetadata, error)
- func (m *Metadata) GetHostIndex(ctx context.Context, locality string) ([]*Host, error)
- func (m *Metadata) ListRecentStubs(ctx context.Context, locality string, ttl time.Duration, limit int) ([]RecentStub, error)
- func (m *Metadata) ListRecentStubsAnyLocality(ctx context.Context, ttl time.Duration) ([]RecentStub, error)
- func (m *Metadata) MarkStubReported(ctx context.Context, locality, stubID string, ttl time.Duration) (bool, error)
- func (m *Metadata) RefreshReconcileLock(locality, logicalHost, hash string, ttlS int) error
- func (m *Metadata) RefreshStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- func (m *Metadata) ReleaseReconcileLock(locality, logicalHost, hash string) error
- func (m *Metadata) RemoveClientLock(ctx context.Context, clientId, hash string) error
- func (m *Metadata) RemoveFsNode(ctx context.Context, id string) error
- func (m *Metadata) RemoveFsNodeChild(ctx context.Context, pid, id string) error
- func (m *Metadata) RemoveHostFromIndex(ctx context.Context, locality string, host *Host) error
- func (m *Metadata) RemoveHostKeepAlive(ctx context.Context, locality string, host *Host) error
- func (m *Metadata) RemoveStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- func (m *Metadata) SetClientLock(ctx context.Context, clientId, hash string) error
- func (m *Metadata) SetFsNode(ctx context.Context, id string, metadata *FSMetadata) error
- func (m *Metadata) SetHostKeepAlive(ctx context.Context, locality string, host *Host) error
- func (m *Metadata) SetStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- type MetadataConfig
- type MetadataMode
- type MockCacheMetadataStore
- func (m *MockCacheMetadataStore) AcquireReconcileLock(ctx context.Context, locality, logicalHost, hash string, ttlSeconds int) (bool, error)
- func (m *MockCacheMetadataStore) AddFsNodeChild(ctx context.Context, pid, id string) error
- func (m *MockCacheMetadataStore) AddHostToIndex(ctx context.Context, locality string, host *Host) error
- func (m *MockCacheMetadataStore) AddRecentStub(ctx context.Context, locality, workspaceID, stubID string, ttl time.Duration) error
- func (m *MockCacheMetadataStore) GetAvailableHosts(ctx context.Context, locality string) ([]*Host, error)
- func (m *MockCacheMetadataStore) GetFsNode(ctx context.Context, id string) (*FSMetadata, error)
- func (m *MockCacheMetadataStore) GetFsNodeChildren(ctx context.Context, id string) ([]*FSMetadata, error)
- func (m *MockCacheMetadataStore) ListRecentStubs(ctx context.Context, locality string, ttl time.Duration, limit int) ([]RecentStub, error)
- func (m *MockCacheMetadataStore) MarkStubReported(ctx context.Context, locality, stubID string, ttl time.Duration) (bool, error)
- func (m *MockCacheMetadataStore) RefreshStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- func (m *MockCacheMetadataStore) ReleaseReconcileLock(ctx context.Context, locality, logicalHost, hash string) error
- func (m *MockCacheMetadataStore) RemoveClientLock(ctx context.Context, hash string, host string) error
- func (m *MockCacheMetadataStore) RemoveFsNode(ctx context.Context, id string) error
- func (m *MockCacheMetadataStore) RemoveFsNodeChild(ctx context.Context, pid, id string) error
- func (m *MockCacheMetadataStore) RemoveHost(ctx context.Context, locality string, host *Host) error
- func (m *MockCacheMetadataStore) RemoveStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- func (m *MockCacheMetadataStore) SetClientLock(ctx context.Context, hash string, host string) error
- func (m *MockCacheMetadataStore) SetFsNode(ctx context.Context, id string, metadata *FSMetadata) error
- func (m *MockCacheMetadataStore) SetHostKeepAlive(ctx context.Context, locality string, host *Host) error
- func (m *MockCacheMetadataStore) SetStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- type MountPointConfig
- type MountPointSource
- type OperationTrace
- type OperationTraceAttempt
- type PrefetchState
- type Prefetcher
- type ReadPrefetchConfig
- type RecentStub
- type ReconciliationConfig
- type RedisCacheMetadataStore
- func (c *RedisCacheMetadataStore) AcquireReconcileLock(ctx context.Context, locality, logicalHost, hash string, ttlSeconds int) (bool, error)
- func (c *RedisCacheMetadataStore) AddFsNodeChild(ctx context.Context, pid, id string) error
- func (c *RedisCacheMetadataStore) AddHostToIndex(ctx context.Context, locality string, host *Host) error
- func (c *RedisCacheMetadataStore) AddRecentStub(ctx context.Context, locality, workspaceID, stubID string, ttl time.Duration) error
- func (c *RedisCacheMetadataStore) GetAvailableHosts(ctx context.Context, locality string) ([]*Host, error)
- func (c *RedisCacheMetadataStore) GetFsNode(ctx context.Context, id string) (*FSMetadata, error)
- func (c *RedisCacheMetadataStore) GetFsNodeChildren(ctx context.Context, id string) ([]*FSMetadata, error)
- func (c *RedisCacheMetadataStore) ListRecentStubs(ctx context.Context, locality string, ttl time.Duration, limit int) ([]RecentStub, error)
- func (c *RedisCacheMetadataStore) ListRecentStubsAnyLocality(ctx context.Context, ttl time.Duration) ([]RecentStub, error)
- func (c *RedisCacheMetadataStore) MarkStubReported(ctx context.Context, locality, stubID string, ttl time.Duration) (bool, error)
- func (c *RedisCacheMetadataStore) RefreshStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- func (c *RedisCacheMetadataStore) ReleaseReconcileLock(_ context.Context, locality, logicalHost, hash string) error
- func (c *RedisCacheMetadataStore) RemoveClientLock(ctx context.Context, hash string, host string) error
- func (c *RedisCacheMetadataStore) RemoveFsNode(ctx context.Context, id string) error
- func (c *RedisCacheMetadataStore) RemoveFsNodeChild(ctx context.Context, pid, id string) error
- func (c *RedisCacheMetadataStore) RemoveHost(ctx context.Context, locality string, host *Host) error
- func (c *RedisCacheMetadataStore) RemoveStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- func (c *RedisCacheMetadataStore) SetClientLock(ctx context.Context, hash string, host string) error
- func (c *RedisCacheMetadataStore) SetFsNode(ctx context.Context, id string, metadata *FSMetadata) error
- func (c *RedisCacheMetadataStore) SetHostKeepAlive(ctx context.Context, locality string, host *Host) error
- func (c *RedisCacheMetadataStore) SetStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
- type RedisClient
- func (r *RedisClient) Keys(ctx context.Context, pattern string) ([]string, error)
- func (r *RedisClient) LRange(ctx context.Context, key string, start, stop int64) ([]string, error)
- func (r *RedisClient) PSubscribe(ctx context.Context, channels ...string) (<-chan *redis.Message, <-chan error, func())
- func (r *RedisClient) Publish(ctx context.Context, channel string, message interface{}) *redis.IntCmd
- func (r *RedisClient) Scan(ctx context.Context, pattern string) ([]string, error)
- func (r *RedisClient) Subscribe(ctx context.Context, channels ...string) (<-chan *redis.Message, <-chan error)
- func (r *RedisClient) ToSlice(v interface{}) []interface{}
- func (r *RedisClient) ToStruct(m map[string]string, out interface{}) error
- type RedisConfig
- type RedisLock
- type RedisLockOption
- type RedisLockOptions
- type RedisMode
- type RendezvousHasher
- type S3Client
- func (c *S3Client) BucketName() string
- func (c *S3Client) DownloadIntoBuffer(ctx context.Context, key string, buffer *bytes.Buffer) error
- func (c *S3Client) GetClient() *s3.Client
- func (c *S3Client) Head(ctx context.Context, key string) (bool, *s3.HeadObjectOutput, error)
- func (c *S3Client) Open(ctx context.Context, key string) (io.ReadCloser, error)
- func (c *S3Client) ReadRange(ctx context.Context, key string, start int64, length int64) ([]byte, error)
- func (c *S3Client) ReadRangeInto(ctx context.Context, key string, start int64, dst []byte) (int, error)
- type S3ContentSource
- type S3SourceConfig
- type Server
- func (cs *Server) AvailableDiskBytes() int64
- func (cs *Server) Close() error
- func (cs *Server) DiskMinFreeBytes() int64
- func (cs *Server) DiskPressureExceeded() bool
- func (cs *Server) Drain()
- func (cs *Server) GetContent(ctx context.Context, req *proto.CacheGetContentRequest) (*proto.CacheGetContentResponse, error)
- func (cs *Server) GetContentStream(req *proto.CacheGetContentRequest, stream proto.Cache_GetContentStreamServer) error
- func (cs *Server) GetState(ctx context.Context, req *proto.CacheGetStateRequest) (*proto.CacheGetStateResponse, error)
- func (cs *Server) HasCompleteContent(hash string, expectedSize int64) bool
- func (cs *Server) HasContent(ctx context.Context, req *proto.CacheHasContentRequest) (*proto.CacheHasContentResponse, error)
- func (cs *Server) Host() *Host
- func (cs *Server) HostID() string
- func (cs *Server) PressureEvictContent(protected map[string]struct{}, bytesToFree int64) (int, int64)
- func (cs *Server) PruneContentNotProtected(protected map[string]struct{}, ttl time.Duration) (int, int64)
- func (cs *Server) ReadContentInto(ctx context.Context, hash string, offset int64, dst []byte) (int64, error)
- func (cs *Server) RefreshDiskUsage() (DiskUsage, error)
- func (cs *Server) Serve(bindAddr string, advertiseHost string) (string, error)
- func (cs *Server) SetChurnSink(sink CacheChurnSink)
- func (cs *Server) SetProtectedContent(protected map[string]struct{})
- func (cs *Server) StartServer(port uint) error
- func (cs *Server) StoreContent(stream proto.Cache_StoreContentServer) error
- func (cs *Server) StoreContentFromSource(ctx context.Context, req *proto.CacheStoreContentFromSourceRequest) (*proto.CacheStoreContentFromSourceResponse, error)
- func (cs *Server) StoreContentFromSourceWithLock(ctx context.Context, req *proto.CacheStoreContentFromSourceRequest) (*proto.CacheStoreContentFromSourceWithLockResponse, error)
- func (cs *Server) StoreContentInCacheFS(ctx context.Context, path string, hash string, size uint64) error
- func (cs *Server) StoreReader(ctx context.Context, reader io.Reader, expectedHash string) (string, uint64, error)
- func (cs *Server) StoreSyntheticContentInCacheFS(ctx context.Context, path string, hash string, size uint64) error
- func (cs *Server) UsagePct() float64
- type ServerConfig
- type ServerOption
- type ServerOpts
- type ServerReadTransportConfig
- type Source
- type SourceConfig
- type StorageLayer
- type Store
- func (cas *Store) Add(ctx context.Context, hash string, content []byte) error
- func (cas *Store) AddPageSourceWithExpectedHash(ctx context.Context, expectedHash string, size int64, concurrency int, ...) (string, int64, error)
- func (cas *Store) AddReader(ctx context.Context, reader io.Reader) (string, int64, error)
- func (cas *Store) AddReaderWithExpectedHash(ctx context.Context, reader io.Reader, expectedHash string) (string, int64, error)
- func (cas *Store) CachedDiskAvailableBytes() int64
- func (cas *Store) CachedDiskUsagePct() float64
- func (cas *Store) Cleanup()
- func (cas *Store) ContentStatus(hash string, expectedSize ...int64) string
- func (cas *Store) Exists(hash string, expectedSize ...int64) bool
- func (cas *Store) Get(hash string, offset, length int64, dst []byte) (int64, error)
- func (cas *Store) GetDiskCacheMetrics() (int64, int64, float64, error)
- func (cas *Store) PageRegion(hash string, offset int64, length int64) (path string, pageOffset int64, n int, ok bool, err error)
- func (cas *Store) PressureEvictContent(protected map[string]struct{}, bytesToFree int64) (int, int64)
- func (cas *Store) PruneContentNotProtected(protected map[string]struct{}, ttl time.Duration) (int, int64)
- func (cas *Store) PutFullPages(hash string, offset int64, data []byte)
- func (cas *Store) PutPageRange(hash string, offset int64, data []byte)
- func (cas *Store) ReadAt(hash string, offset int64, dst []byte) (read int64, err error)
- func (cas *Store) SetChurnSink(sink CacheChurnSink)
- func (cas *Store) SetProtectedContent(protected map[string]struct{})
- func (cas *Store) StartDiskMonitor()
- func (cas *Store) WarmRange(hash string, offset int64, length int64)
- type StoreContentOptions
- type ValkeyConfig
- type ValkeyExistingPrimary
- type VolumeObjectIdentity
Constants ¶
const ( FSMetadataFormatCompact = "2" FSMetadataShardCount = 256 )
const ( SourceModeJuiceFS string = "juicefs" SourceModeMountPoint string = "mountpoint" )
const ( HostPrefix string = "cache-host" Version string = "dev" )
const ( CacheChurnOperationDiskEviction = "disk_eviction" CacheChurnStatusEvicted = "evicted" CacheChurnStatusProtectedEvicted = "protected_evicted" CacheChurnStatusNothingEvictable = "nothing_evictable" )
const (
DefaultReconcileRecentStubTTLS int = 7 * 24 * 60 * 60
)
Variables ¶
var ( BufferSize1MB = 1 * 1024 * 1024 BufferSize4MB = 4 * 1024 * 1024 BufferSize16MB = 16 * 1024 * 1024 )
Standard buffer sizes aligned with typical chunk sizes
var ( ErrInvalidHostRegistration = errors.New("invalid cache host registration") )
var ( ErrHostNotFound = errors.New("host not found") ErrUnableToReachHost = errors.New("unable to reach host") ErrInvalidHostVersion = errors.New("invalid host version") ErrContentNotFound = errors.New("content not found") ErrRawReadBusy = errors.New("raw cache read server is busy") ErrClientNotFound = errors.New("client not found") ErrCacheLockHeld = errors.New("cache lock held") ErrUnableToPopulateContent = errors.New("unable to populate content from original source") ErrFSMountFailure = errors.New("failed to mount cachefs") ErrUnableToAcquireLock = errors.New("unable to acquire lock") )
var ( ErrChannelClosed = errors.New("redis: channel closed") ErrConnectionIssue = errors.New("redis: connection issue") ErrUnknownRedisMode = errors.New("redis: unknown mode") )
var (
Logger *logger
)
var MetadataKeys = &metadataKeys{}
Functions ¶
func FSMetadataShard ¶
func GenerateFsID ¶
Generates a directory ID based on parent ID and name.
func GetPrivateIpAddr ¶
func GetPublicIpAddr ¶
func InitLogger ¶
func IsStoreHostUnavailable ¶
func MarshalFSMetadata ¶
func MarshalFSMetadata(metadata *FSMetadata) ([]byte, error)
func RecentStubKey ¶
func SHA1StringToUint64 ¶
SHA1StringToUint64 converts the first 8 bytes of a SHA-1 hash string to a uint64
func ToSlice ¶
func ToSlice(v interface{}) []interface{}
Flattens a struct using its field tags so it can be used by HSet. Struct fields must have the redis tag on them otherwise they will be ignored.
func ToStruct ¶
Copies the result of HGetAll to a provided struct. If a field cannot be parsed, we use Go's default value. Struct fields must have the redis tag on them otherwise they will be ignored.
func WithClientName ¶
func WithClientName(name string) func(*redis.UniversalOptions)
Types ¶
type BufferPool ¶
type BufferPool struct {
// contains filtered or unexported fields
}
BufferPool provides a pool of reusable byte slices to reduce allocations Optimized for 1-4MB chunks as recommended in the optimization plan
func NewBufferPool ¶
func NewBufferPool() *BufferPool
NewBufferPool creates a new buffer pool with predefined size buckets
func (*BufferPool) Get ¶
func (bp *BufferPool) Get(size int) []byte
Get retrieves a buffer of at least the requested size
type CacheChurnEvent ¶
type CacheChurnEvent struct {
Operation string
Status string
Path string
EvictedObjects int
ProtectedObjects int
FreedBytes int64
ProtectedFreedBytes int64
UsagePct float64
WatermarkPct float64
AvailableBytes uint64
ReserveBytes int64
TargetFreeBytes int64
TotalCandidates int
ProtectedCandidates int
RecentCandidates int
EligibleCandidates int
Timestamp time.Time
}
type CacheChurnSink ¶
type CacheChurnSink func(CacheChurnEvent)
type CacheFS ¶
type CacheFS struct {
MetadataStore CacheMetadataStore
Client *Client
Config ClientConfig
// contains filtered or unexported fields
}
func NewFileSystem ¶
func NewFileSystem(ctx context.Context, opts FSSystemOpts) (*CacheFS, error)
NewFileSystem initializes a new CacheFS with root metadata.
type CacheFSNode ¶
type CacheMetadataStore ¶
type CacheMetadataStore interface {
SetClientLock(ctx context.Context, hash string, host string) error
RemoveClientLock(ctx context.Context, hash string, host string) error
SetStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
RemoveStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
RefreshStoreFromContentLock(ctx context.Context, locality string, sourcePath string) error
SetFsNode(ctx context.Context, id string, metadata *FSMetadata) error
GetFsNode(ctx context.Context, id string) (*FSMetadata, error)
RemoveFsNode(ctx context.Context, id string) error
RemoveFsNodeChild(ctx context.Context, pid, id string) error
GetFsNodeChildren(ctx context.Context, id string) ([]*FSMetadata, error)
AddFsNodeChild(ctx context.Context, pid, id string) error
// Required-content reconciliation state. On workers these are brokered to
// the gateway/coordinator over gRPC; the gateway implements them against
// its Redis.
AddRecentStub(ctx context.Context, locality, workspaceID, stubID string, ttl time.Duration) error
ListRecentStubs(ctx context.Context, locality string, ttl time.Duration, limit int) ([]RecentStub, error)
MarkStubReported(ctx context.Context, locality, stubID string, ttl time.Duration) (bool, error)
AcquireReconcileLock(ctx context.Context, locality, logicalHost, hash string, ttlSeconds int) (bool, error)
ReleaseReconcileLock(ctx context.Context, locality, logicalHost, hash string) error
}
func NewRedisCacheMetadataStore ¶
func NewRedisCacheMetadataStore(_ GlobalConfig, serverConfig ServerConfig) (CacheMetadataStore, error)
func NewRedisCacheMetadataStoreWithClient ¶
func NewRedisCacheMetadataStoreWithClient(_ GlobalConfig, _ ServerConfig, client redis.UniversalClient) CacheMetadataStore
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
func NewClientWithHostDirectory ¶
func NewClientWithHostDirectory(ctx context.Context, cfg Config, metadataStore CacheMetadataStore, hostDirectory HostDirectory, locality string) (*Client, error)
func (*Client) AttachLocalServer ¶
func (*Client) CacheFSMetadata ¶
CacheFSMetadata resolves a cachefs path to its content metadata without going through the cachefs FUSE mount.
func (*Client) ClientLocalPageFileView ¶
func (*Client) ClientLocalPageFileViews ¶
func (c *Client) ClientLocalPageFileViews(hash string, offset int64, length int64, opts ClientOptions) (views []ClientLocalPageFileView, err error)
func (*Client) ClientLocalPageFileViewsWithTrace ¶
func (c *Client) ClientLocalPageFileViewsWithTrace(hash string, offset int64, length int64, opts ClientOptions) (views []ClientLocalPageFileView, trace OperationTrace, err error)
func (*Client) DetachLocalServer ¶
func (*Client) GetContent ¶
func (*Client) GetContentStream ¶
func (*Client) GetNearbyHosts ¶
func (*Client) HostsAvailable ¶
func (*Client) IsCachedOnSelectedHost ¶
func (c *Client) IsCachedOnSelectedHost(hash string, routingKey string, expectedSize ...int64) (bool, error)
IsCachedOnSelectedHost checks only the HRW-selected storage host for routingKey. This is intentionally stricter than IsCachedReachable: cache-through writers use it to avoid treating a fallback replica as proof that the primary placement is already populated.
func (*Client) IsCachedReachable ¶
IsCachedReachable reports whether hash exists on any currently reachable cache host. It checks the HRW read order first, then scans remaining known hosts as a recovery path for placement drift or host churn. Do not use it to decide whether a cache-through write can be skipped.
func (*Client) IsCachedReachableContext ¶
func (*Client) IsPathCachedReachable ¶
func (*Client) LookupVolumeObjectContentHash ¶
func (c *Client) LookupVolumeObjectContentHash(ctx context.Context, identity VolumeObjectIdentity) (string, bool, error)
LookupVolumeObjectContentHash resolves a previously verified content hash. The full immutable identity is encoded into the metadata id, so stale object versions cannot satisfy a lookup after an in-place replacement.
func (*Client) MaterializeFromReplica ¶
func (c *Client) MaterializeFromReplica(ctx context.Context, server *Server, hash, routingKey string, size int64) (bool, error)
MaterializeFromReplica streams the content for (hash, routingKey) from a reachable peer that already holds it into the given local server's store. It returns true when the content is complete locally afterward. size must be known (> 0); callers should fall back to an origin fetch otherwise.
func (*Client) PrimaryReadHost ¶
PrimaryReadHost returns the host a read for routingKey would resolve to: the highest-ranked reachable host within the read's HRW window, computed from the same hasher and window the read path uses. Reconciliation materializes content on this host so a subsequent read's HRW lookup hits it instead of missing on a host that does not hold the content.
It deliberately walks the full ranking (which retains unreachable logical-only hosts so placement stays stable during churn) and returns the first reachable host, mirroring how reads skip unavailable hosts and fall back in rank order.
func (*Client) RankedReadHosts ¶
RankedReadHosts returns the HRW-ranked hosts for routingKey, including hosts whose endpoints are currently unavailable. Callers that make placement decisions (e.g. proactive reconciliation) need the full ranking so they can apply their own liveness policy; reads should keep using PrimaryReadHost, which prefers reachable hosts.
func (*Client) ReadContentInto ¶
func (*Client) ReadContentIntoWithTrace ¶
func (c *Client) ReadContentIntoWithTrace(ctx context.Context, hash string, offset int64, dst []byte, opts ClientOptions) (read int64, trace OperationTrace, err error)
func (*Client) SelectedStoreHostAvailable ¶
func (*Client) StoreContent ¶
func (*Client) StoreContentAtPath ¶
func (*Client) StoreContentFromFUSE ¶
func (*Client) StoreContentFromLocalFile ¶
func (c *Client) StoreContentFromLocalFile(source LocalContentSource, opts StoreContentOptions) (string, error)
StoreContentFromLocalFile streams a caller-local file to the selected cache host.
func (*Client) StoreContentFromLocalFileWithTrace ¶
func (c *Client) StoreContentFromLocalFileWithTrace(source LocalContentSource, opts StoreContentOptions) (hash string, trace OperationTrace, err error)
func (*Client) StoreContentFromLocalPath ¶
func (*Client) StoreContentFromLocalSource ¶
func (c *Client) StoreContentFromLocalSource(source LocalContentSource, opts StoreContentOptions) (string, error)
StoreContentFromLocalSource asks the selected cache host to read source.Path itself. Prefer StoreContentFromLocalFile unless the source path is guaranteed to exist on cache hosts.
func (*Client) StoreContentFromS3 ¶
func (*Client) StoreContentFromS3Source ¶
func (c *Client) StoreContentFromS3Source(source S3ContentSource, opts StoreContentOptions) (string, error)
func (*Client) StoreVolumeObjectContentHash ¶
func (c *Client) StoreVolumeObjectContentHash(ctx context.Context, identity VolumeObjectIdentity, hash string) error
StoreVolumeObjectContentHash durably records a verified immutable identity-to-hash mapping. Existing conflicting mappings are rejected instead of silently changing the content address for one object identity.
type ClientConfig ¶
type ClientConfig struct {
Token string `key:"token" json:"token"`
MinRetryLengthBytes int64 `key:"minRetryLengthBytes" json:"min_retry_length_bytes"`
MaxGetContentAttempts int `key:"maxGetContentAttempts" json:"max_get_content_attempts"`
NTopHosts int `key:"nTopHosts" json:"n_top_hosts"`
CacheFS FSConfig `key:"cachefs" json:"cachefs"`
PreferLocalCacheHost bool `key:"preferLocalCacheHost" json:"prefer_local_cache_host"`
PageFDCacheSize int `key:"pageFDCacheSize" json:"page_fd_cache_size"`
ReadTransport ClientReadTransportConfig `key:"readTransport" json:"read_transport"`
Prefetch ReadPrefetchConfig `key:"prefetch" json:"prefetch"`
}
type ClientLocalPageFileView ¶
ClientLocalPageFileView describes a byte range inside a page file that is already present on this client/worker. It is intentionally not a remote address: consumers may mmap it, return it as a FUSE fd-backed response, or otherwise read it as a local file.
type ClientOptions ¶
type ClientOptions = struct {
RoutingKey string
}
type ClientReadTransportConfig ¶
type ClientReadTransportConfig struct {
Enabled bool `key:"enabled" json:"enabled"`
MaxActiveConnsPerHost int `key:"maxActiveConnsPerHost" json:"max_active_conns_per_host"`
MaxIdleConnsPerHost int `key:"maxIdleConnsPerHost" json:"max_idle_conns_per_host"`
// RequestSizeBytes bounds each raw TCP request independently of the
// server-side page prefetch part size.
RequestSizeBytes int64 `key:"requestSizeBytes" json:"request_size_bytes"`
// MaxPartsPerRead only controls fanout when a logical read is larger than
// RequestSizeBytes. A logical read that fits remains one wire request.
MaxPartsPerRead int `key:"maxPartsPerRead" json:"max_parts_per_read"`
}
type ClientRequest ¶
type ClientRequest struct {
// contains filtered or unexported fields
}
type ClientRequestType ¶
type ClientRequestType int
const ( ClientRequestTypeStorage ClientRequestType = iota ClientRequestTypeRetrieval )
type Config ¶
type Config struct {
Enabled bool `key:"enabled" json:"enabled"`
Disk DiskConfig `key:"disk" json:"disk"`
Memory MemoryConfig `key:"memory" json:"memory"`
Coordinator CoordinatorConfig `key:"coordinator" json:"coordinator"`
Server ServerConfig `key:"server" json:"server"`
Client ClientConfig `key:"client" json:"client"`
Global GlobalConfig `key:"global" json:"global"`
Reconciliation ReconciliationConfig `key:"reconciliation" json:"reconciliation"`
}
type Coordinator ¶
type Coordinator struct {
// contains filtered or unexported fields
}
func NewCoordinator ¶
func NewCoordinator(repository CoordinatorRepository) *Coordinator
func (*Coordinator) ListHosts ¶
func (c *Coordinator) ListHosts(ctx context.Context, poolName, locality string) ([]CoordinatorHost, error)
func (*Coordinator) RegisterHost ¶
func (c *Coordinator) RegisterHost(ctx context.Context, host CoordinatorHost, ttl time.Duration) error
func (*Coordinator) UnregisterHost ¶
func (c *Coordinator) UnregisterHost(ctx context.Context, poolName, locality, logicalHostID, registrationID string) error
type CoordinatorConfig ¶
type CoordinatorConfig struct {
Token string `key:"token" json:"token"`
RegistrationTTLSeconds int `key:"registrationTTLSeconds" json:"registration_ttl_seconds"`
HeartbeatIntervalSeconds int `key:"heartbeatIntervalSeconds" json:"heartbeat_interval_seconds"`
HostWatchIntervalSeconds int `key:"hostWatchIntervalSeconds" json:"host_watch_interval_seconds"`
}
type CoordinatorHost ¶
type CoordinatorHost struct {
// LogicalHostID is the stable cache routing identity. Multiple worker
// process registrations can advertise addresses for the same logical host.
LogicalHostID string
// RegistrationID identifies one live worker/cache-server process lease.
RegistrationID string
PoolName string
Locality string
NodeID string
CachePathID string
Addr string
PrivateAddr string
CapacityUsagePct float64
}
func (CoordinatorHost) LogicalOnly ¶
func (h CoordinatorHost) LogicalOnly() CoordinatorHost
type CoordinatorRepository ¶
type CoordinatorRepository interface {
SetCacheRegistration(ctx context.Context, host CoordinatorHost, ttl time.Duration) error
GetActiveCacheRegistration(ctx context.Context, logicalHostID string) (registrationID string, found bool, err error)
SetActiveCacheRegistration(ctx context.Context, logicalHostID, registrationID string, ttl time.Duration) error
ListCacheLogicalHosts(ctx context.Context, poolName, locality string) ([]string, error)
ListCacheRegistrations(ctx context.Context, logicalHostID string) ([]string, error)
GetCacheRegistration(ctx context.Context, logicalHostID, registrationID string) (CoordinatorHost, bool, error)
GetCacheLogicalHost(ctx context.Context, logicalHostID string) (CoordinatorHost, bool, error)
RemoveCacheRegistration(ctx context.Context, logicalHostID, registrationID string) error
CountCacheRegistrations(ctx context.Context, logicalHostID string) (int64, error)
RemoveCacheLogicalHost(ctx context.Context, poolName, locality, logicalHostID string) error
}
type DiscoveryClient ¶
type DiscoveryClient struct {
// contains filtered or unexported fields
}
func NewDiscoveryClient ¶
func NewDiscoveryClient(cfg GlobalConfig, hostMap *HostMap, hostDirectory HostDirectory, locality string) *DiscoveryClient
func (*DiscoveryClient) GetHostState ¶
GetHostState attempts to connect to the gRPC service and verifies its availability
type DiskConfig ¶
type ErrNodeNotFound ¶
type ErrNodeNotFound struct {
Id string
}
func (*ErrNodeNotFound) Error ¶
func (e *ErrNodeNotFound) Error() string
type FSConfig ¶
type FSConfig struct {
Enabled bool `key:"enabled" json:"enabled"`
MountPoint string `key:"mountPoint" json:"mount_point"`
MaxBackgroundTasks int `key:"maxBackgroundTasks" json:"max_background_tasks"`
MaxWriteKB int `key:"maxWriteKB" json:"max_write_kb"`
MaxReadAheadKB int `key:"maxReadAheadKB" json:"max_read_ahead_kb"`
DirectMount bool `key:"directMount" json:"direct_mount"`
DirectIO bool `key:"directIO" json:"direct_io"`
Options []string `key:"options" json:"options"`
}
type FSMetadata ¶
type FSMetadata struct {
PID string `redis:"pid" json:"pid"`
ID string `redis:"id" json:"id"`
Name string `redis:"name" json:"name"`
Path string `redis:"path" json:"path"`
Hash string `redis:"hash" json:"hash"`
Ino uint64 `redis:"ino" json:"ino"`
Size uint64 `redis:"size" json:"size"`
Blocks uint64 `redis:"blocks" json:"blocks"`
Atime uint64 `redis:"atime" json:"atime"`
Mtime uint64 `redis:"mtime" json:"mtime"`
Ctime uint64 `redis:"ctime" json:"ctime"`
Atimensec uint32 `redis:"atimensec" json:"atimensec"`
Mtimensec uint32 `redis:"mtimensec" json:"mtimensec"`
Ctimensec uint32 `redis:"ctimensec" json:"ctimensec"`
Mode uint32 `redis:"mode" json:"mode"`
Nlink uint32 `redis:"nlink" json:"nlink"`
Rdev uint32 `redis:"rdev" json:"rdev"`
Blksize uint32 `redis:"blksize" json:"blksize"`
Padding uint32 `redis:"padding" json:"padding"`
Uid uint32 `redis:"uid" json:"uid"`
Gid uint32 `redis:"gid" json:"gid"`
Gen uint64 `redis:"gen" json:"gen"`
}
func FSMetadataFromWorkerCacheProto ¶
func FSMetadataFromWorkerCacheProto(metadata *proto.WorkerCacheFSMetadata) *FSMetadata
func UnmarshalFSMetadata ¶
func UnmarshalFSMetadata(data []byte) (*FSMetadata, error)
func (*FSMetadata) ToProto ¶
func (m *FSMetadata) ToProto() *proto.CacheFSMetadata
func (*FSMetadata) ToWorkerCacheProto ¶
func (m *FSMetadata) ToWorkerCacheProto() *proto.WorkerCacheFSMetadata
type FSNode ¶
type FSSystemOpts ¶
type FSSystemOpts struct {
Verbose bool
MetadataStore CacheMetadataStore
Config ClientConfig
Client *Client
}
type FileSystem ¶
type FileSystemOpts ¶
CacheFS types
type FileSystemStorage ¶
type GlobalConfig ¶
type GlobalConfig struct {
DefaultLocality string `key:"defaultLocality" json:"default_locality"`
ServerPort uint `key:"serverPort" json:"server_port"`
DiscoveryIntervalS int `key:"discoveryIntervalS" json:"discovery_interval_s"`
DiscoveryJitterS int `key:"discoveryJitterS" json:"discovery_jitter_s"`
MaxDiscoveryConcurrency int `key:"maxDiscoveryConcurrency" json:"max_discovery_concurrency"`
HostMonitorIntervalS int `key:"hostMonitorIntervalS" json:"host_monitor_interval_s"`
RoundTripThresholdMilliseconds uint `key:"rttThresholdMilliseconds" json:"rtt_threshold_ms"`
HostStorageCapacityThresholdPct float64 `key:"hostStorageCapacityThresholdPct" json:"host_storage_capacity_threshold_pct"`
GRPCDialTimeoutS int `key:"grpcDialTimeoutS" json:"grpc_dial_timeout_s"`
GRPCMessageSizeBytes int `key:"grpcMessageSizeBytes" json:"grpc_message_size_bytes"`
GRPCInitialWindowSize int `key:"grpcInitialWindowSize" json:"grpc_initial_window_size"`
GRPCInitialConnWindowSize int `key:"grpcInitialConnWindowSize" json:"grpc_initial_conn_window_size"`
GRPCWriteBufferSize int `key:"grpcWriteBufferSize" json:"grpc_write_buffer_size"`
GRPCReadBufferSize int `key:"grpcReadBufferSize" json:"grpc_read_buffer_size"`
GRPCMaxConcurrentStreams int `key:"grpcMaxConcurrentStreams" json:"grpc_max_concurrent_streams"`
GRPCNumStreamWorkers int `key:"grpcNumStreamWorkers" json:"grpc_num_stream_workers"`
GRPCPayloadCodecV2 bool `key:"grpcPayloadCodecV2" json:"grpc_payload_codec_v2"`
GRPCPayloadCodecMinBytes int `key:"grpcPayloadCodecMinBytes" json:"grpc_payload_codec_min_bytes"`
DebugMode bool `key:"debugMode" json:"debug_mode"`
PrettyLogs bool `key:"prettyLogs" json:"pretty_logs"`
}
func (*GlobalConfig) GetLocality ¶
func (c *GlobalConfig) GetLocality() string
type Host ¶
type Host struct {
RTT time.Duration `redis:"rtt" json:"rtt"`
HostId string `redis:"host_id" json:"host_id"`
RegistrationID string `redis:"registration_id" json:"registration_id"`
PoolName string `redis:"pool_name" json:"pool_name"`
Locality string `redis:"locality" json:"locality"`
NodeID string `redis:"node_id" json:"node_id"`
CachePathID string `redis:"cache_path_id" json:"cache_path_id"`
Addr string `redis:"addr" json:"addr"`
PrivateAddr string `redis:"private_addr" json:"private_addr"`
CapacityUsagePct float64 `redis:"capacity_usage_pct" json:"capacity_usage_pct"`
}
func (*Host) HasEndpoint ¶
func (*Host) LogicalOnly ¶
type HostDirectory ¶
type HostMap ¶
type HostMap struct {
// contains filtered or unexported fields
}
func NewHostMap ¶
func NewHostMap(cfg GlobalConfig, onHostAdded func(*Host) error) *HostMap
func (*HostMap) Closest ¶
Closest finds the nearest host within a given timeout If no hosts are found, it will error out
func (*HostMap) ClosestWithCapacity ¶
ClosestWithCapacity finds the nearest host with available storage capacity within a given timeout If no hosts are found, it will error out
func (*HostMap) DeactivateEndpoint ¶
func (*HostMap) RemoveLogicalHost ¶
type JuiceFSConfig ¶
type JuiceFSConfig struct {
RedisURI string `key:"redisURI" json:"redis_uri"`
Bucket string `key:"bucket" json:"bucket"`
AccessKey string `key:"accessKey" json:"access_key"`
SecretKey string `key:"secretKey" json:"secret_key"`
CacheSize int64 `key:"cacheSize" json:"cache_size"`
BlockSize int64 `key:"blockSize" json:"block_size"`
Prefetch int64 `key:"prefetch" json:"prefetch"`
BufferSize int64 `key:"bufferSize" json:"buffer_size"`
}
type JuiceFsSource ¶
type JuiceFsSource struct {
// contains filtered or unexported fields
}
func (*JuiceFsSource) Format ¶
func (s *JuiceFsSource) Format(fsName string) error
func (*JuiceFsSource) Mount ¶
func (s *JuiceFsSource) Mount(localPath string) error
func (*JuiceFsSource) Unmount ¶
func (s *JuiceFsSource) Unmount(localPath string) error
type LocalContentSource ¶
type MemoryConfig ¶
type Metadata ¶
type Metadata struct {
// contains filtered or unexported fields
}
func NewMetadata ¶
func NewMetadata(cfg MetadataConfig) (*Metadata, error)
func NewMetadataWithRedisClient ¶
func NewMetadataWithRedisClient(client redis.UniversalClient) *Metadata
func (*Metadata) AcquireReconcileLock ¶
func (m *Metadata) AcquireReconcileLock(ctx context.Context, locality, logicalHost, hash string, ttlS int) (bool, error)
AcquireReconcileLock takes a short lifecycle lock so only one materialization runs for a (locality, logical host, hash) at a time. It returns whether the lock was acquired; contention is not an error.
func (*Metadata) AddFsNodeChild ¶
func (*Metadata) AddHostToIndex ¶
func (*Metadata) AddRecentStub ¶
func (m *Metadata) AddRecentStub(ctx context.Context, locality, workspaceID, stubID string, ttl time.Duration) error
AddRecentStub records a (workspace, stub) as recently used in a locality and prunes entries older than ttl. The index is a ZSET scored by last-seen time.
func (*Metadata) GetAvailableHosts ¶
func (*Metadata) GetFsNodeChildren ¶
func (*Metadata) GetHostIndex ¶
func (*Metadata) ListRecentStubs ¶
func (m *Metadata) ListRecentStubs(ctx context.Context, locality string, ttl time.Duration, limit int) ([]RecentStub, error)
ListRecentStubs returns recently used stubs in a locality, newest first, excluding entries older than ttl.
func (*Metadata) ListRecentStubsAnyLocality ¶
func (m *Metadata) ListRecentStubsAnyLocality(ctx context.Context, ttl time.Duration) ([]RecentStub, error)
ListRecentStubsAnyLocality returns recently used stubs across all localities from the global recent-stub index, newest first.
func (*Metadata) MarkStubReported ¶
func (m *Metadata) MarkStubReported(ctx context.Context, locality, stubID string, ttl time.Duration) (bool, error)
MarkStubReported atomically claims the one-time required-content generation for a stub. It returns true only for the first caller (cluster-wide), so the expensive enumeration + S2 write happens once per stub rather than on every container start. The marker is set once with the given TTL and is not refreshed by later callers; once it expires the content may be regenerated, which is idempotent.
func (*Metadata) RefreshReconcileLock ¶
func (*Metadata) RefreshStoreFromContentLock ¶
func (*Metadata) ReleaseReconcileLock ¶
func (*Metadata) RemoveClientLock ¶
func (*Metadata) RemoveFsNode ¶
func (*Metadata) RemoveFsNodeChild ¶
func (*Metadata) RemoveHostFromIndex ¶
func (*Metadata) RemoveHostKeepAlive ¶
func (*Metadata) RemoveStoreFromContentLock ¶
func (*Metadata) SetClientLock ¶
func (*Metadata) SetHostKeepAlive ¶
type MetadataConfig ¶
type MetadataConfig struct {
Mode MetadataMode `key:"mode" json:"mode"`
ValkeyConfig ValkeyConfig `key:"valkey" json:"valkey"`
// Default config
RedisAddr string `key:"redisAddr" json:"redis_addr"`
RedisPasswd string `key:"redisPasswd" json:"redis_passwd"`
RedisTLSEnabled bool `key:"redisTLSEnabled" json:"redis_tls_enabled"`
RedisInsecureSkipVerify bool `key:"redisInsecureSkipVerify" json:"redis_insecure_skip_verify"`
RedisMode RedisMode `key:"redisMode" json:"redis_mode"`
RedisMasterName string `key:"redisMasterName" json:"redis_master_name"`
}
type MetadataMode ¶
type MetadataMode string
const ( MetadataModeDefault MetadataMode = "default" MetadataModeLocal MetadataMode = "local" )
type MockCacheMetadataStore ¶
type MockCacheMetadataStore struct {
// contains filtered or unexported fields
}
MockCacheMetadataStore is a simple in-memory metadataStore for testing Does not require Redis or any external dependencies
func NewMockCacheMetadataStore ¶
func NewMockCacheMetadataStore() *MockCacheMetadataStore
func (*MockCacheMetadataStore) AcquireReconcileLock ¶
func (*MockCacheMetadataStore) AddFsNodeChild ¶
func (m *MockCacheMetadataStore) AddFsNodeChild(ctx context.Context, pid, id string) error
func (*MockCacheMetadataStore) AddHostToIndex ¶
func (*MockCacheMetadataStore) AddRecentStub ¶
func (*MockCacheMetadataStore) GetAvailableHosts ¶
func (*MockCacheMetadataStore) GetFsNode ¶
func (m *MockCacheMetadataStore) GetFsNode(ctx context.Context, id string) (*FSMetadata, error)
func (*MockCacheMetadataStore) GetFsNodeChildren ¶
func (m *MockCacheMetadataStore) GetFsNodeChildren(ctx context.Context, id string) ([]*FSMetadata, error)
func (*MockCacheMetadataStore) ListRecentStubs ¶
func (m *MockCacheMetadataStore) ListRecentStubs(ctx context.Context, locality string, ttl time.Duration, limit int) ([]RecentStub, error)
func (*MockCacheMetadataStore) MarkStubReported ¶
func (*MockCacheMetadataStore) RefreshStoreFromContentLock ¶
func (*MockCacheMetadataStore) ReleaseReconcileLock ¶
func (m *MockCacheMetadataStore) ReleaseReconcileLock(ctx context.Context, locality, logicalHost, hash string) error
func (*MockCacheMetadataStore) RemoveClientLock ¶
func (*MockCacheMetadataStore) RemoveFsNode ¶
func (m *MockCacheMetadataStore) RemoveFsNode(ctx context.Context, id string) error
func (*MockCacheMetadataStore) RemoveFsNodeChild ¶
func (m *MockCacheMetadataStore) RemoveFsNodeChild(ctx context.Context, pid, id string) error
func (*MockCacheMetadataStore) RemoveHost ¶
func (*MockCacheMetadataStore) RemoveStoreFromContentLock ¶
func (*MockCacheMetadataStore) SetClientLock ¶
func (*MockCacheMetadataStore) SetFsNode ¶
func (m *MockCacheMetadataStore) SetFsNode(ctx context.Context, id string, metadata *FSMetadata) error
func (*MockCacheMetadataStore) SetHostKeepAlive ¶
func (*MockCacheMetadataStore) SetStoreFromContentLock ¶
type MountPointConfig ¶
type MountPointConfig struct {
BucketName string `key:"bucketName" json:"bucket_name"`
AccessKey string `key:"accessKey" json:"access_key"`
SecretKey string `key:"secretKey" json:"secret_key"`
Region string `key:"region" json:"region"`
EndpointURL string `key:"endpointUrl" json:"endpoint_url"`
ForcePathStyle bool `key:"forcePathStyle" json:"force_path_style"`
}
type MountPointSource ¶
type MountPointSource struct {
// contains filtered or unexported fields
}
func (*MountPointSource) Format ¶
func (s *MountPointSource) Format(fsName string) error
func (*MountPointSource) Mount ¶
func (s *MountPointSource) Mount(localPath string) error
func (*MountPointSource) Unmount ¶
func (s *MountPointSource) Unmount(localPath string) error
type OperationTrace ¶
type OperationTrace struct {
Operation string `json:"operation,omitempty"`
Result string `json:"result,omitempty"`
Hash string `json:"hash,omitempty"`
RoutingKey string `json:"routing_key,omitempty"`
Offset int64 `json:"offset,omitempty"`
Length int64 `json:"length,omitempty"`
Read int64 `json:"read,omitempty"`
Bytes int64 `json:"bytes,omitempty"`
ExpectedSize int64 `json:"expected_size,omitempty"`
Views int `json:"views,omitempty"`
DurationUs int64 `json:"duration_us,omitempty"`
HostRefreshes int `json:"host_refreshes,omitempty"`
Attempts []OperationTraceAttempt `json:"attempts,omitempty"`
}
type OperationTraceAttempt ¶
type OperationTraceAttempt struct {
HostIndex int `json:"host_index"`
HostID string `json:"host_id,omitempty"`
RegistrationID string `json:"registration_id,omitempty"`
PoolName string `json:"pool_name,omitempty"`
Locality string `json:"locality,omitempty"`
NodeID string `json:"node_id,omitempty"`
CachePathID string `json:"cache_path_id,omitempty"`
Addr string `json:"addr,omitempty"`
PrivateAddr string `json:"private_addr,omitempty"`
HasEndpoint bool `json:"has_endpoint"`
Source string `json:"source"`
Result string `json:"result"`
Read int64 `json:"read,omitempty"`
Bytes int64 `json:"bytes,omitempty"`
ExpectedSize int64 `json:"expected_size,omitempty"`
ContentStatus string `json:"content_status,omitempty"`
ElapsedUs int64 `json:"elapsed_us,omitempty"`
Error string `json:"error,omitempty"`
}
type PrefetchState ¶
type PrefetchState struct {
// contains filtered or unexported fields
}
PrefetchState tracks sequential read patterns per file/hash
type Prefetcher ¶
type Prefetcher struct {
// contains filtered or unexported fields
}
Prefetcher detects sequential reads and prefetches ahead
func NewPrefetcher ¶
func NewPrefetcher(ctx context.Context, cas *Store, bufferPool *BufferPool) *Prefetcher
NewPrefetcher creates a new prefetcher instance
func (*Prefetcher) OnRead ¶
func (pf *Prefetcher) OnRead(hash string, offset, length int64)
OnRead should be called on each read to detect patterns
type ReadPrefetchConfig ¶
type ReadPrefetchConfig struct {
Enabled bool `key:"enabled" json:"enabled"`
AheadBytes int64 `key:"aheadBytes" json:"ahead_bytes"`
Workers int `key:"workers" json:"workers"`
PartLengthBytes int64 `key:"partLengthBytes" json:"part_length_bytes"`
MaxPartsPerRead int `key:"maxPartsPerRead" json:"max_parts_per_read"`
}
type RecentStub ¶
RecentStub identifies a stub recently used in a locality, tracked so cache servers can discover required-content streams to reconcile.
type ReconciliationConfig ¶
type ReconciliationConfig struct {
Enabled bool `key:"enabled" json:"enabled"`
IntervalSeconds int `key:"intervalSeconds" json:"interval_seconds"`
RecentStubTTLSeconds int `key:"recentStubTTLSeconds" json:"recent_stub_ttl_seconds"`
LockTTLSeconds int `key:"lockTTLSeconds" json:"lock_ttl_seconds"`
MaxStubsPerCycle int `key:"maxStubsPerCycle" json:"max_stubs_per_cycle"`
MaxItemsPerCycle int `key:"maxItemsPerCycle" json:"max_items_per_cycle"`
VolumeMinBytes int64 `key:"volumeMinBytes" json:"volume_min_bytes"`
OriginFallbackEnabled bool `key:"originFallbackEnabled" json:"origin_fallback_enabled"`
// MaxDiskUsagePct is the soft pressure watermark (0-1). Above it, the
// cache owner evicts lower-priority content and pauses proactive
// reconciliation. Near it, reconciliation is limited to the ranked recent
// working set so eviction and materialization do not churn.
MaxDiskUsagePct float64 `key:"maxDiskUsagePct" json:"max_disk_usage_pct"`
}
ReconciliationConfig controls the async required-content reconciliation loop. When disabled, workers do not report required content and cache servers do not run the loop, preserving prior startup behavior.
type RedisCacheMetadataStore ¶
type RedisCacheMetadataStore struct {
// contains filtered or unexported fields
}
func (*RedisCacheMetadataStore) AcquireReconcileLock ¶
func (*RedisCacheMetadataStore) AddFsNodeChild ¶
func (c *RedisCacheMetadataStore) AddFsNodeChild(ctx context.Context, pid, id string) error
func (*RedisCacheMetadataStore) AddHostToIndex ¶
func (*RedisCacheMetadataStore) AddRecentStub ¶
func (*RedisCacheMetadataStore) GetAvailableHosts ¶
func (*RedisCacheMetadataStore) GetFsNode ¶
func (c *RedisCacheMetadataStore) GetFsNode(ctx context.Context, id string) (*FSMetadata, error)
func (*RedisCacheMetadataStore) GetFsNodeChildren ¶
func (c *RedisCacheMetadataStore) GetFsNodeChildren(ctx context.Context, id string) ([]*FSMetadata, error)
func (*RedisCacheMetadataStore) ListRecentStubs ¶
func (c *RedisCacheMetadataStore) ListRecentStubs(ctx context.Context, locality string, ttl time.Duration, limit int) ([]RecentStub, error)
func (*RedisCacheMetadataStore) ListRecentStubsAnyLocality ¶
func (c *RedisCacheMetadataStore) ListRecentStubsAnyLocality(ctx context.Context, ttl time.Duration) ([]RecentStub, error)
func (*RedisCacheMetadataStore) MarkStubReported ¶
func (*RedisCacheMetadataStore) RefreshStoreFromContentLock ¶
func (*RedisCacheMetadataStore) ReleaseReconcileLock ¶
func (c *RedisCacheMetadataStore) ReleaseReconcileLock(_ context.Context, locality, logicalHost, hash string) error
func (*RedisCacheMetadataStore) RemoveClientLock ¶
func (*RedisCacheMetadataStore) RemoveFsNode ¶
func (c *RedisCacheMetadataStore) RemoveFsNode(ctx context.Context, id string) error
func (*RedisCacheMetadataStore) RemoveFsNodeChild ¶
func (c *RedisCacheMetadataStore) RemoveFsNodeChild(ctx context.Context, pid, id string) error
func (*RedisCacheMetadataStore) RemoveHost ¶
func (*RedisCacheMetadataStore) RemoveStoreFromContentLock ¶
func (*RedisCacheMetadataStore) SetClientLock ¶
func (*RedisCacheMetadataStore) SetFsNode ¶
func (c *RedisCacheMetadataStore) SetFsNode(ctx context.Context, id string, metadata *FSMetadata) error
func (*RedisCacheMetadataStore) SetHostKeepAlive ¶
func (*RedisCacheMetadataStore) SetStoreFromContentLock ¶
type RedisClient ¶
type RedisClient struct {
redis.UniversalClient
}
func NewRedisClient ¶
func NewRedisClient(config RedisConfig, options ...func(*redis.UniversalOptions)) (*RedisClient, error)
func (*RedisClient) Keys ¶
Keys gets all keys using a pattern. Prefer explicit indexes for hot paths.
func (*RedisClient) PSubscribe ¶
func (*RedisClient) ToSlice ¶
func (r *RedisClient) ToSlice(v interface{}) []interface{}
type RedisConfig ¶
type RedisConfig struct {
Addrs []string `key:"addrs" json:"addrs"`
Mode RedisMode `key:"mode" json:"mode"`
ClientName string `key:"clientName" json:"client_name"`
EnableTLS bool `key:"enableTLS" json:"enable_tls"`
InsecureSkipVerify bool `key:"insecureSkipVerify" json:"insecure_skip_verify"`
MinIdleConns int `key:"minIdleConns" json:"min_idle_conns"`
MaxIdleConns int `key:"maxIdleConns" json:"max_idle_conns"`
ConnMaxIdleTime time.Duration `key:"connMaxIdleTime" json:"conn_max_idle_time"`
ConnMaxLifetime time.Duration `key:"connMaxLifetime" json:"conn_max_lifetime"`
DialTimeout time.Duration `key:"dialTimeout" json:"dial_timeout"`
ReadTimeout time.Duration `key:"readTimeout" json:"read_timeout"`
WriteTimeout time.Duration `key:"writeTimeout" json:"write_timeout"`
MaxRedirects int `key:"maxRedirects" json:"max_redirects"`
MaxRetries int `key:"maxRetries" json:"max_retries"`
PoolSize int `key:"poolSize" json:"pool_size"`
Username string `key:"username" json:"username"`
Password string `key:"password" json:"password"`
RouteByLatency bool `key:"routeByLatency" json:"route_by_latency"`
MasterName string `key:"masterName" json:"master_name"`
SentinelPassword string `key:"sentinelPassword" json:"sentinel_password"`
}
type RedisLock ¶
type RedisLock struct {
// contains filtered or unexported fields
}
func NewRedisLock ¶
func NewRedisLock(client *RedisClient, opts ...RedisLockOption) *RedisLock
type RedisLockOption ¶
type RedisLockOption func(*RedisLock)
type RedisLockOptions ¶
type RendezvousHasher ¶
type S3Client ¶
type S3Client struct {
Client *s3.Client
Source S3SourceConfig
DownloadConcurrency int64
DownloadChunkSize int64
}
func NewS3Client ¶
func NewS3Client(ctx context.Context, sourceConfig S3SourceConfig, serverConfig ServerConfig) (*S3Client, error)
func (*S3Client) BucketName ¶
func (*S3Client) DownloadIntoBuffer ¶
type S3ContentSource ¶
type S3SourceConfig ¶
type Server ¶
type Server struct {
proto.UnimplementedCacheServer
// contains filtered or unexported fields
}
func NewServerWithOptions ¶
func (*Server) AvailableDiskBytes ¶
func (*Server) DiskMinFreeBytes ¶
func (*Server) DiskPressureExceeded ¶
func (*Server) GetContent ¶
func (cs *Server) GetContent(ctx context.Context, req *proto.CacheGetContentRequest) (*proto.CacheGetContentResponse, error)
func (*Server) GetContentStream ¶
func (cs *Server) GetContentStream(req *proto.CacheGetContentRequest, stream proto.Cache_GetContentStreamServer) error
func (*Server) GetState ¶
func (cs *Server) GetState(ctx context.Context, req *proto.CacheGetStateRequest) (*proto.CacheGetStateResponse, error)
func (*Server) HasCompleteContent ¶
HasCompleteContent reports whether the local store holds the full content for hash. When expectedSize > 0 it is validated against the stored size.
func (*Server) HasContent ¶
func (cs *Server) HasContent(ctx context.Context, req *proto.CacheHasContentRequest) (*proto.CacheHasContentResponse, error)
func (*Server) PressureEvictContent ¶
func (*Server) PruneContentNotProtected ¶
func (*Server) ReadContentInto ¶
func (cs *Server) ReadContentInto(ctx context.Context, hash string, offset int64, dst []byte) (int64, error)
ReadContentInto reads from this server's local store without cluster host selection. It is intended for in-process callers that just materialized content locally and need to consume that exact replica.
func (*Server) RefreshDiskUsage ¶
func (*Server) SetChurnSink ¶
func (cs *Server) SetChurnSink(sink CacheChurnSink)
func (*Server) SetProtectedContent ¶
func (*Server) StartServer ¶
func (*Server) StoreContent ¶
func (cs *Server) StoreContent(stream proto.Cache_StoreContentServer) error
func (*Server) StoreContentFromSource ¶
func (cs *Server) StoreContentFromSource(ctx context.Context, req *proto.CacheStoreContentFromSourceRequest) (*proto.CacheStoreContentFromSourceResponse, error)
func (*Server) StoreContentFromSourceWithLock ¶
func (cs *Server) StoreContentFromSourceWithLock(ctx context.Context, req *proto.CacheStoreContentFromSourceRequest) (*proto.CacheStoreContentFromSourceWithLockResponse, error)
func (*Server) StoreContentInCacheFS ¶
func (*Server) StoreReader ¶
func (cs *Server) StoreReader(ctx context.Context, reader io.Reader, expectedHash string) (string, uint64, error)
StoreReader stores the full contents of reader into the local store, validating against expectedHash when it is a content hash. It is used by the reconciliation loop to materialize content copied from a replica.
func (*Server) StoreSyntheticContentInCacheFS ¶
type ServerConfig ¶
type ServerConfig struct {
DiskCacheDir string `key:"diskCacheDir" json:"disk_cache_dir"`
DiskCacheMaxUsagePct float64 `key:"diskCacheMaxUsagePct" json:"disk_cache_max_usage_pct"`
// DiskCacheEvictWatermarkPct is the filesystem usage fraction (0-1) above
// which the store evicts least-recently-accessed content until usage
// falls back below the watermark. It must sit below the kubelet's
// DiskPressure thresholds so the node never reaches them. Defaults to 0.80.
DiskCacheEvictWatermarkPct float64 `key:"diskCacheEvictWatermarkPct" json:"disk_cache_evict_watermark_pct"`
ObjectTtlS int `key:"objectTtlS" json:"object_ttl_s"`
MaxCachePct int64 `key:"maxCachePct" json:"max_cache_pct"`
PageSizeBytes int64 `key:"pageSizeBytes" json:"page_size_bytes"`
PageFileBuckets int `key:"pageFileBuckets" json:"page_file_buckets"`
SmallRangeCopyThresholdBytes int64 `key:"smallRangeCopyThresholdBytes" json:"small_range_copy_threshold_bytes"`
ReadTransport ServerReadTransportConfig `key:"readTransport" json:"read_transport"`
Metadata MetadataConfig `key:"metadata" json:"metadata"`
Sources []SourceConfig `key:"sources" json:"sources"`
S3DownloadConcurrency int64 `key:"s3DownloadConcurrency" json:"s3_download_concurrency"`
S3DownloadChunkSize int64 `key:"s3DownloadChunkSize" json:"s3_download_chunk_size"`
}
type ServerOption ¶
type ServerOption func(*ServerOpts)
func WithServerAdvertiseAddr ¶
func WithServerAdvertiseAddr(addr string) ServerOption
func WithServerHostID ¶
func WithServerHostID(hostID string) ServerOption
func WithServerMetadataStore ¶
func WithServerMetadataStore(metadataStore CacheMetadataStore) ServerOption
type ServerOpts ¶
type ServerOpts struct {
HostID string
MetadataStore CacheMetadataStore
AdvertiseAddr string
}
type ServerReadTransportConfig ¶
type ServerReadTransportConfig struct {
Enabled bool `key:"enabled" json:"enabled"`
Sendfile bool `key:"sendfile" json:"sendfile"`
// MaxRequestSizeBytes is the raw-protocol cap and is independent of gRPC.
MaxRequestSizeBytes int64 `key:"maxRequestSizeBytes" json:"max_request_size_bytes"`
// MaxInflightBytes is a weighted server-wide admission budget.
MaxInflightBytes int64 `key:"maxInflightBytes" json:"max_inflight_bytes"`
// MaxConcurrentRequests also bounds requests with very small or empty ranges.
MaxConcurrentRequests int `key:"maxConcurrentRequests" json:"max_concurrent_requests"`
}
type Source ¶
type Source interface {
Mount(localPath string) error
Format(fsName string) error
Unmount(localPath string) error
}
func NewJuiceFsSource ¶
func NewJuiceFsSource(config JuiceFSConfig) (Source, error)
func NewMountPointSource ¶
func NewMountPointSource(config MountPointConfig) (Source, error)
func NewSource ¶
func NewSource(config SourceConfig) (Source, error)
type SourceConfig ¶
type SourceConfig struct {
Mode string `key:"mode" json:"mode"`
FilesystemName string `key:"fsName" json:"filesystem_name"`
FilesystemPath string `key:"fsPath" json:"filesystem_path"`
JuiceFS JuiceFSConfig `key:"juicefs" json:"juicefs"`
MountPoint MountPointConfig `key:"mountpoint" json:"mountpoint"`
}
type StorageLayer ¶
type StorageLayer interface {
}
type Store ¶
type Store struct {
// contains filtered or unexported fields
}
func (*Store) AddPageSourceWithExpectedHash ¶
func (*Store) AddReaderWithExpectedHash ¶
func (cas *Store) AddReaderWithExpectedHash(ctx context.Context, reader io.Reader, expectedHash string) (string, int64, error)
AddReaderWithExpectedHash stores a content-addressed stream into a temporary page directory, validates the full stream hash, then publishes the verified pages under the final hash path.
func (*Store) CachedDiskAvailableBytes ¶
func (*Store) CachedDiskUsagePct ¶
func (*Store) ContentStatus ¶
func (*Store) GetDiskCacheMetrics ¶
func (*Store) PageRegion ¶
func (*Store) PressureEvictContent ¶
func (*Store) PruneContentNotProtected ¶
func (cas *Store) PruneContentNotProtected(protected map[string]struct{}, ttl time.Duration) (int, int64)
PruneContentNotProtected removes content that has not been used inside ttl and is not required by any recent stub. It is intentionally driven by the embedded cache owner so non-owner workers never run prune loops.
func (*Store) SetChurnSink ¶
func (cas *Store) SetChurnSink(sink CacheChurnSink)
func (*Store) SetProtectedContent ¶
func (*Store) StartDiskMonitor ¶
func (cas *Store) StartDiskMonitor()
type StoreContentOptions ¶
type ValkeyConfig ¶
type ValkeyConfig struct {
PrimaryName string `key:"primaryName" json:"primary_name"`
Password string `key:"password" json:"password"`
TLS bool `key:"tls" json:"tls"`
Host string `key:"host" json:"host"`
Port int `key:"port" json:"port"`
ExistingPrimary ValkeyExistingPrimary `key:"existingPrimary" json:"existingPrimary"`
}
type ValkeyExistingPrimary ¶
Source Files
¶
- buffer_pool.go
- cachefs.go
- cachefs_node.go
- client.go
- coordinator.go
- discovery.go
- errors.go
- fadvise.go
- host_directory.go
- hostmap.go
- logger.go
- metadata.go
- metadata_store.go
- metrics.go
- mock_metadata_store.go
- network.go
- object_hash_registry.go
- path_stats.go
- prefetcher.go
- raw_transport.go
- redis.go
- s3_client.go
- sendfile_linux.go
- server.go
- source.go
- source_juicefs.go
- source_mountpoint.go
- storage.go
- storage_eviction.go
- store_lock.go
- trace.go
- types.go