distributed

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 26, 2026 License: Apache-2.0 Imports: 27 Imported by: 0

Documentation

Index

Constants

View Source
const (
	TableObjects   = "objects"   // Object metadata (hash -> shard locations)
	TableBuckets   = "buckets"   // Bucket metadata
	TableMultipart = "multipart" // Multipart upload metadata (uploadID -> metadata)
	TableParts     = "parts"     // Part metadata (uploadID:partNumber -> part info)
)

Table names for global state

Variables

This section is empty.

Functions

func New

func New(config interface{}) (backend.Backend, error)

New creates a new distributed backend

func NodeToUint32

func NodeToUint32(value string) (uint32, error)

NodeToUint32 converts a node name to uint32

Types

type Backend

type Backend struct {
	// contains filtered or unexported fields
}

Backend implements the distributed storage backend with Reed-Solomon erasure coding

func (*Backend) AbortMultipartUpload

func (b *Backend) AbortMultipartUpload(ctx context.Context, bucket, key, uploadID string) error

AbortMultipartUpload aborts a multipart upload and cleans up all parts

func (*Backend) Close

func (b *Backend) Close() error

Close cleans up resources

func (*Backend) CompleteMultipartUpload

CompleteMultipartUpload completes a multipart upload by assembling all parts

func (*Backend) CreateBucket

CreateBucket creates a new bucket in the distributed store

func (*Backend) CreateMultipartUpload

CreateMultipartUpload initiates a multipart upload

func (*Backend) DB

func (b *Backend) DB() *s3db.S3DB

DB returns the local badger database (for testing/backward compatibility) Returns nil if using distributed state

func (*Backend) DataDir

func (b *Backend) DataDir() string

DataDir returns the data directory (for testing)

func (*Backend) DeleteBucket

func (b *Backend) DeleteBucket(ctx context.Context, req *backend.DeleteBucketRequest) error

DeleteBucket deletes a bucket from the distributed store

func (*Backend) DeleteObject

func (b *Backend) DeleteObject(ctx context.Context, req *backend.DeleteObjectRequest) error

DeleteObject removes an object from the distributed storage

func (*Backend) GetBucketMetadata

func (b *Backend) GetBucketMetadata(bucket string) (*backend.BucketMetadata, error)

GetBucketMetadata retrieves bucket metadata from s3db

func (*Backend) GetFromPath

func (b *Backend) GetFromPath(ctx context.Context, bucket, objectPath string, out *bytes.Buffer) error

GetFromPath retrieves an object and writes to the provided writer (used for testing)

func (*Backend) GetObject

GetObject retrieves an object using Reed-Solomon decoding. Supports byte-range requests for efficient partial reads.

func (*Backend) GlobalState

func (b *Backend) GlobalState() GlobalState

GlobalState returns the global state interface

func (*Backend) HashRing

func (b *Backend) HashRing() *consistent.Consistent

HashRing returns the hash ring (for testing)

func (*Backend) HeadBucket

HeadBucket checks if a bucket exists

func (*Backend) HeadObject

func (b *Backend) HeadObject(ctx context.Context, bucket, key string) (*backend.HeadObjectResponse, error)

HeadObject returns object metadata

func (*Backend) ListBuckets

func (b *Backend) ListBuckets(ctx context.Context, ownerID string) (*backend.ListBucketsResponse, error)

ListBuckets returns a list of buckets from both config and s3db ownerID filters to only show buckets owned by the specified user (empty = all buckets)

func (*Backend) ListObjects

ListObjects returns a list of objects in a bucket by scanning global state Objects are stored with ARN key format: arn:aws:s3:::<bucket>/<key>

func (*Backend) PutObject

PutObject stores an object using Reed-Solomon encoding across multiple nodes

func (*Backend) PutObjectFromPath

func (b *Backend) PutObjectFromPath(ctx context.Context, bucket, objectPath string) error

PutObjectFromPath stores an object from a file path (used internally and for testing)

func (*Backend) RsDataShard

func (b *Backend) RsDataShard() int

RsDataShard returns the number of data shards (for testing)

func (*Backend) RsParityShard

func (b *Backend) RsParityShard() int

RsParityShard returns the number of parity shards (for testing)

func (*Backend) SetDataDir

func (b *Backend) SetDataDir(dir string)

SetDataDir sets the data directory (for testing)

func (*Backend) Type

func (b *Backend) Type() string

Type returns the backend type identifier

func (*Backend) UploadPart

UploadPart uploads a part in a multipart upload

type BucketConfig

type BucketConfig struct {
	Name   string
	Region string
	Type   string
	Public bool
}

BucketConfig holds configuration for a bucket

type Config

type Config struct {
	// DataDir is the root directory for distributed node storage
	DataDir string

	// BadgerDir is the directory for the Badger KV database (used when DBClient is nil)
	BadgerDir string

	// Reed-Solomon configuration
	DataShards   int
	ParityShards int

	// Hash ring configuration
	PartitionCount    int
	ReplicationFactor int

	// QUIC server base port (each node uses BasePort + nodeNum)
	QuicBasePort int

	// UseQUIC enables QUIC-based shard distribution (requires QUIC servers to be running)
	UseQUIC bool

	// Nodes configuration (from cluster.toml)
	Nodes []NodeConfig

	// Buckets configuration (from cluster.toml)
	Buckets []BucketConfig

	// DBClient holds configuration for the distributed database client
	// When set, uses distributed s3db for global state instead of local BadgerDB
	DBClient *DBClientConfig
}

Config holds distributed backend configuration

type DBClientConfig

type DBClientConfig struct {
	Nodes           []string // List of DB node addresses
	AccessKeyID     string   // AWS-style access key ID
	SecretAccessKey string   // AWS-style secret access key
	Region          string   // Region for signing (default: us-east-1)
}

DBClientConfig holds configuration for connecting to the distributed database

type DeletedObjectInfo

type DeletedObjectInfo struct {
	Bucket         string   `json:"bucket"`
	Key            string   `json:"key"`
	ObjectHash     [32]byte `json:"object_hash"`
	DeletedAt      int64    `json:"deleted_at"`       // Unix timestamp
	DataShardNodes []uint32 `json:"data_shard_nodes"` // Which nodes had data shards
	ParityNodes    []uint32 `json:"parity_nodes"`     // Which nodes had parity shards
}

DeletedObjectInfo tracks a deleted object for compaction coordination

type DistributedState

type DistributedState struct {
	// contains filtered or unexported fields
}

DistributedState wraps an s3db.Client for GlobalState operations. This is used when a distributed database cluster is configured.

func NewDistributedState

func NewDistributedState(cfg *DBClientConfig) (*DistributedState, error)

NewDistributedState creates a GlobalState backed by a distributed s3db cluster

func (*DistributedState) Client

func (d *DistributedState) Client() *s3db.Client

Client returns the underlying s3db.Client

func (*DistributedState) Close

func (d *DistributedState) Close() error

Close is a no-op for the distributed client (connections are reused)

func (*DistributedState) Delete

func (d *DistributedState) Delete(table string, key []byte) error

Delete removes a key from the distributed database

func (*DistributedState) Exists

func (d *DistributedState) Exists(table string, key []byte) (bool, error)

Exists checks if a key exists in the distributed database

func (*DistributedState) Get

func (d *DistributedState) Get(table string, key []byte) ([]byte, error)

Get retrieves a value by key from the distributed database

func (*DistributedState) ListKeys

func (d *DistributedState) ListKeys(table string, prefix []byte) ([][]byte, error)

ListKeys returns all keys with the given prefix (via Scan)

func (*DistributedState) Scan

func (d *DistributedState) Scan(table string, prefix []byte, fn func(key, value []byte) error) error

Scan iterates over all keys with the given prefix

func (*DistributedState) Set

func (d *DistributedState) Set(table string, key []byte, value []byte) error

Set stores a key-value pair in the distributed database

type GlobalState

type GlobalState interface {
	// Get retrieves a value by key from the specified table
	Get(table string, key []byte) ([]byte, error)

	// Set stores a key-value pair in the specified table
	Set(table string, key []byte, value []byte) error

	// Delete removes a key from the specified table
	Delete(table string, key []byte) error

	// Exists checks if a key exists in the specified table
	Exists(table string, key []byte) (bool, error)

	// ListKeys returns all keys with the given prefix in the specified table
	ListKeys(table string, prefix []byte) ([][]byte, error)

	// Scan iterates over all keys with the given prefix
	Scan(table string, prefix []byte, fn func(key, value []byte) error) error

	// Close closes the state store
	Close() error
}

GlobalState provides an abstraction for global state storage operations. This allows the distributed backend to use either a local BadgerDB or a distributed s3db cluster for storing object metadata.

type LocalState

type LocalState struct {
	// contains filtered or unexported fields
}

LocalState wraps a local s3db.S3DB for GlobalState operations. This is used when no distributed database is configured.

func NewLocalState

func NewLocalState(dbPath string) (*LocalState, error)

NewLocalState creates a GlobalState backed by a local BadgerDB

func (*LocalState) Close

func (l *LocalState) Close() error

Close closes the underlying database

func (*LocalState) DB

func (l *LocalState) DB() *s3db.S3DB

DB returns the underlying S3DB for backward compatibility

func (*LocalState) Delete

func (l *LocalState) Delete(table string, key []byte) error

Delete removes a key

func (*LocalState) Exists

func (l *LocalState) Exists(table string, key []byte) (bool, error)

Exists checks if a key exists

func (*LocalState) Get

func (l *LocalState) Get(table string, key []byte) ([]byte, error)

Get retrieves a value by key (table is encoded in the key)

func (*LocalState) ListKeys

func (l *LocalState) ListKeys(table string, prefix []byte) ([][]byte, error)

ListKeys returns all keys with the given prefix

func (*LocalState) Scan

func (l *LocalState) Scan(table string, prefix []byte, fn func(key, value []byte) error) error

Scan iterates over all keys with the given prefix

func (*LocalState) Set

func (l *LocalState) Set(table string, key []byte, value []byte) error

Set stores a key-value pair

type NodeConfig

type NodeConfig struct {
	ID     int
	Host   string
	Port   int
	Path   string
	DB     bool
	DBPort int
	DBPath string
	Leader bool
	Epoch  int
}

NodeConfig holds configuration for a single node

type ObjectShardReader

type ObjectShardReader struct {
	File        *os.File
	WALFileInfo wal.WALFileInfo
}

ObjectShardReader provides access to a shard stored in WAL

type ObjectToShardNodes

type ObjectToShardNodes struct {
	Object           [32]byte
	Size             int64
	DataShardNodes   []uint32
	ParityShardNodes []uint32
}

ObjectToShardNodes maps an object to its shard locations

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL