store

package
v0.0.0-...-e9fa201 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 16, 2026 License: Apache-2.0 Imports: 17 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// ChunkTotalCount tracks the total number of chunks in the index
	ChunkTotalCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "zapfs",
		Subsystem: "storage",
		Name:      "chunks_total",
		Help:      "Total number of chunks in the index",
	})

	// ChunkTotalBytes tracks the total bytes across all chunks
	ChunkTotalBytes = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: "zapfs",
		Subsystem: "storage",
		Name:      "chunks_bytes_total",
		Help:      "Total bytes across all chunks",
	})

	// ChunkOperations tracks chunk operations by type
	ChunkOperations = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "zapfs",
		Subsystem: "storage",
		Name:      "chunk_operations_total",
		Help:      "Total number of chunk operations",
	}, []string{"operation"}) // operation: "create", "deduplicate", "delete"

	// ChunkDedupeHits tracks successful deduplication hits
	ChunkDedupeHits = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "zapfs",
		Subsystem: "storage",
		Name:      "chunk_dedupe_hits_total",
		Help:      "Number of times a chunk was deduplicated instead of stored",
	})
)

Functions

This section is empty.

Types

type Config 

type Config struct {
	IndexPath string
	IndexKind IndexKind // memory or leveldb
	Backends  []*types.Backend
	ECScheme  types.ECScheme
}

Config holds FileStore configuration

type FileStore 

type FileStore struct {
	// contains filtered or unexported fields
}

FileStore implements Store using indexed chunks on multiple backends

func NewFileStore 

func NewFileStore(cfg Config, manager *backend.Manager) (*FileStore, error)

NewFileStore creates a new FileStore

func (*FileStore) AddBackend 

func (fs *FileStore) AddBackend(b *types.Backend)

AddBackend registers a new backend

func (*FileStore) Close 

func (fs *FileStore) Close() error

func (*FileStore) DeleteChunk 

func (fs *FileStore) DeleteChunk(ctx context.Context, id types.ChunkID) error

DeleteChunk removes a chunk from the index. Note: This does NOT delete the chunk from backend storage. Use with GetBackendStorage().Delete() to fully remove a chunk.

func (*FileStore) DeleteObject 

func (fs *FileStore) DeleteObject(ctx context.Context, id uuid.UUID) error

DeleteObject marks an object as deleted

func (*FileStore) GetBackend 

func (fs *FileStore) GetBackend(id string) (*types.Backend, bool)

GetBackend returns a backend by ID

func (*FileStore) GetBackendStorage 

func (fs *FileStore) GetBackendStorage(id string) (types.BackendStorage, bool)

GetBackendStorage returns the storage interface for a backend

func (*FileStore) GetChunk 

func (fs *FileStore) GetChunk(ctx context.Context, id types.ChunkID) (io.ReadCloser, error)

GetChunk reads a chunk from storage, decompressing if needed. Returns the original (uncompressed) data.

func (*FileStore) GetChunkData 

func (fs *FileStore) GetChunkData(ctx context.Context, id types.ChunkID) ([]byte, error)

GetChunkData reads a chunk and returns the decompressed data as bytes.

func (*FileStore) GetChunkDataRange 

func (fs *FileStore) GetChunkDataRange(ctx context.Context, id types.ChunkID, offset, length int64) ([]byte, error)

GetChunkDataRange reads a range from a chunk and returns the data as bytes.

func (*FileStore) GetChunkInfo 

func (fs *FileStore) GetChunkInfo(id types.ChunkID) (*types.Chunk, error)

GetChunkInfo returns chunk metadata by ID.

func (*FileStore) GetChunkRange 

func (fs *FileStore) GetChunkRange(ctx context.Context, id types.ChunkID, offset, length int64) (io.ReadCloser, error)

GetChunkRange reads a range from a chunk, decompressing if needed. For compressed chunks, the entire chunk must be decompressed first.

func (*FileStore) GetDefaultBackend 

func (fs *FileStore) GetDefaultBackend() *types.Backend

GetDefaultBackend returns the first available backend

func (*FileStore) GetECGroup 

func (fs *FileStore) GetECGroup(id uuid.UUID) (*types.ECGroup, error)

GetECGroup returns EC group metadata by ID (for admin/debugging)

func (*FileStore) GetIndexStats 

func (fs *FileStore) GetIndexStats() (*IndexStats, error)

GetIndexStats returns statistics about the chunk index. Uses Prometheus metrics for O(1) performance instead of iterating.

func (*FileStore) GetObject 

func (fs *FileStore) GetObject(ctx context.Context, id uuid.UUID) (*types.ObjectRef, error)

GetObject retrieves object metadata by ID.

func (*FileStore) GetObjectData 

func (fs *FileStore) GetObjectData(ctx context.Context, id uuid.UUID) (io.ReadCloser, error)

GetObjectData retrieves the full object data

func (*FileStore) GetObjectRange 

func (fs *FileStore) GetObjectRange(ctx context.Context, id uuid.UUID, offset, length int64) (io.ReadCloser, error)

GetObjectRange retrieves a range of object data

func (*FileStore) IterateChunks 

func (fs *FileStore) IterateChunks(fn func(id types.ChunkID, chunk types.Chunk) error) error

IterateChunks iterates over all chunks in the index. Used by reconciliation to compare local chunks with expected chunks.

func (*FileStore) ListBackends 

func (fs *FileStore) ListBackends() map[string]*types.Backend

ListBackends returns all registered backend IDs

func (*FileStore) PutObject 

func (fs *FileStore) PutObject(ctx context.Context, obj *types.ObjectRef, reader io.Reader) error

PutObject stores an object, chunking the data and writing to backends without compression.

func (*FileStore) PutObjectWithCompression 

func (fs *FileStore) PutObjectWithCompression(ctx context.Context, obj *types.ObjectRef, reader io.Reader, algo compression.Algorithm) error

PutObjectWithCompression stores an object, chunking the data and writing to backends with optional compression. Each chunk is compressed independently using the specified algorithm.

func (*FileStore) RemoveBackend 

func (fs *FileStore) RemoveBackend(id string)

RemoveBackend removes a backend

func (*FileStore) WriteChunk 

func (fs *FileStore) WriteChunk(ctx context.Context, chunkID types.ChunkID, data []byte, backendID string) (*types.ChunkRef, error)

WriteChunk writes a chunk directly to storage (no compression). Used by migration to receive chunks from peer servers. The chunkID is provided by the caller (already known from source).

type IndexKind 

type IndexKind string

IndexKind specifies the index backend type 

const (
	IndexKindMemory  IndexKind = "memory"
	IndexKindLevelDB IndexKind = "leveldb"
)

type IndexStats 

type IndexStats struct {
	TotalChunks int64 `json:"total_chunks"`
	TotalBytes  int64 `json:"total_bytes"`
}

IndexStats holds statistics about the chunk index

type Store 

type Store interface {
	io.Closer

	// Object operations
	GetObject(ctx context.Context, id uuid.UUID) (*types.ObjectRef, error)
	PutObject(ctx context.Context, obj *types.ObjectRef, reader io.Reader) error
	DeleteObject(ctx context.Context, id uuid.UUID) error

	// Chunk operations
	GetChunk(ctx context.Context, id types.ChunkID) (io.ReadCloser, error)
	GetChunkRange(ctx context.Context, id types.ChunkID, offset, length int64) (io.ReadCloser, error)
}

Store handles chunk and object storage

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.