Documentation
¶
Index ¶
- Constants
- func New(config interface{}) (backend.Backend, error)
- func NodeToUint32(value string) (uint32, error)
- type Backend
- func (b *Backend) AbortMultipartUpload(ctx context.Context, bucket, key, uploadID string) error
- func (b *Backend) Close() error
- func (b *Backend) CompleteMultipartUpload(ctx context.Context, req *backend.CompleteMultipartUploadRequest) (*backend.CompleteMultipartUploadResponse, error)
- func (b *Backend) CreateBucket(ctx context.Context, req *backend.CreateBucketRequest) (*backend.CreateBucketResponse, error)
- func (b *Backend) CreateMultipartUpload(ctx context.Context, req *backend.CreateMultipartUploadRequest) (*backend.CreateMultipartUploadResponse, error)
- func (b *Backend) DB() *s3db.S3DB
- func (b *Backend) DataDir() string
- func (b *Backend) DeleteBucket(ctx context.Context, req *backend.DeleteBucketRequest) error
- func (b *Backend) DeleteObject(ctx context.Context, req *backend.DeleteObjectRequest) error
- func (b *Backend) GetBucketMetadata(bucket string) (*backend.BucketMetadata, error)
- func (b *Backend) GetFromPath(ctx context.Context, bucket, objectPath string, out *bytes.Buffer) error
- func (b *Backend) GetObject(ctx context.Context, req *backend.GetObjectRequest) (*backend.GetObjectResponse, error)
- func (b *Backend) GlobalState() GlobalState
- func (b *Backend) HashRing() *consistent.Consistent
- func (b *Backend) HeadBucket(ctx context.Context, req *backend.HeadBucketRequest) (*backend.HeadBucketResponse, error)
- func (b *Backend) HeadObject(ctx context.Context, bucket, key string) (*backend.HeadObjectResponse, error)
- func (b *Backend) ListBuckets(ctx context.Context, ownerID string) (*backend.ListBucketsResponse, error)
- func (b *Backend) ListObjects(ctx context.Context, req *backend.ListObjectsRequest) (*backend.ListObjectsResponse, error)
- func (b *Backend) PutObject(ctx context.Context, req *backend.PutObjectRequest) (*backend.PutObjectResponse, error)
- func (b *Backend) PutObjectFromPath(ctx context.Context, bucket, objectPath string) error
- func (b *Backend) RsDataShard() int
- func (b *Backend) RsParityShard() int
- func (b *Backend) SetDataDir(dir string)
- func (b *Backend) Type() string
- func (b *Backend) UploadPart(ctx context.Context, req *backend.UploadPartRequest) (*backend.UploadPartResponse, error)
- type BucketConfig
- type Config
- type DBClientConfig
- type DeletedObjectInfo
- type DistributedState
- func (d *DistributedState) Client() *s3db.Client
- func (d *DistributedState) Close() error
- func (d *DistributedState) Delete(table string, key []byte) error
- func (d *DistributedState) Exists(table string, key []byte) (bool, error)
- func (d *DistributedState) Get(table string, key []byte) ([]byte, error)
- func (d *DistributedState) ListKeys(table string, prefix []byte) ([][]byte, error)
- func (d *DistributedState) Scan(table string, prefix []byte, fn func(key, value []byte) error) error
- func (d *DistributedState) Set(table string, key []byte, value []byte) error
- type GlobalState
- type LocalState
- func (l *LocalState) Close() error
- func (l *LocalState) DB() *s3db.S3DB
- func (l *LocalState) Delete(table string, key []byte) error
- func (l *LocalState) Exists(table string, key []byte) (bool, error)
- func (l *LocalState) Get(table string, key []byte) ([]byte, error)
- func (l *LocalState) ListKeys(table string, prefix []byte) ([][]byte, error)
- func (l *LocalState) Scan(table string, prefix []byte, fn func(key, value []byte) error) error
- func (l *LocalState) Set(table string, key []byte, value []byte) error
- type NodeConfig
- type ObjectShardReader
- type ObjectToShardNodes
Constants ¶
const ( TableObjects = "objects" // Object metadata (hash -> shard locations) TableBuckets = "buckets" // Bucket metadata TableMultipart = "multipart" // Multipart upload metadata (uploadID -> metadata) TableParts = "parts" // Part metadata (uploadID:partNumber -> part info) )
Table names for global state
Variables ¶
This section is empty.
Functions ¶
func NodeToUint32 ¶
NodeToUint32 converts a node name to uint32
Types ¶
type Backend ¶
type Backend struct {
// contains filtered or unexported fields
}
Backend implements the distributed storage backend with Reed-Solomon erasure coding
func (*Backend) AbortMultipartUpload ¶
AbortMultipartUpload aborts a multipart upload and cleans up all parts
func (*Backend) CompleteMultipartUpload ¶
func (b *Backend) CompleteMultipartUpload(ctx context.Context, req *backend.CompleteMultipartUploadRequest) (*backend.CompleteMultipartUploadResponse, error)
CompleteMultipartUpload completes a multipart upload by assembling all parts
func (*Backend) CreateBucket ¶
func (b *Backend) CreateBucket(ctx context.Context, req *backend.CreateBucketRequest) (*backend.CreateBucketResponse, error)
CreateBucket creates a new bucket in the distributed store
func (*Backend) CreateMultipartUpload ¶
func (b *Backend) CreateMultipartUpload(ctx context.Context, req *backend.CreateMultipartUploadRequest) (*backend.CreateMultipartUploadResponse, error)
CreateMultipartUpload initiates a multipart upload
func (*Backend) DB ¶
DB returns the local badger database (for testing/backward compatibility) Returns nil if using distributed state
func (*Backend) DeleteBucket ¶
DeleteBucket deletes a bucket from the distributed store
func (*Backend) DeleteObject ¶
DeleteObject removes an object from the distributed storage
func (*Backend) GetBucketMetadata ¶
func (b *Backend) GetBucketMetadata(bucket string) (*backend.BucketMetadata, error)
GetBucketMetadata retrieves bucket metadata from s3db
func (*Backend) GetFromPath ¶
func (b *Backend) GetFromPath(ctx context.Context, bucket, objectPath string, out *bytes.Buffer) error
GetFromPath retrieves an object and writes to the provided writer (used for testing)
func (*Backend) GetObject ¶
func (b *Backend) GetObject(ctx context.Context, req *backend.GetObjectRequest) (*backend.GetObjectResponse, error)
GetObject retrieves an object using Reed-Solomon decoding. Supports byte-range requests for efficient partial reads.
func (*Backend) GlobalState ¶
func (b *Backend) GlobalState() GlobalState
GlobalState returns the global state interface
func (*Backend) HashRing ¶
func (b *Backend) HashRing() *consistent.Consistent
HashRing returns the hash ring (for testing)
func (*Backend) HeadBucket ¶
func (b *Backend) HeadBucket(ctx context.Context, req *backend.HeadBucketRequest) (*backend.HeadBucketResponse, error)
HeadBucket checks if a bucket exists
func (*Backend) HeadObject ¶
func (b *Backend) HeadObject(ctx context.Context, bucket, key string) (*backend.HeadObjectResponse, error)
HeadObject returns object metadata
func (*Backend) ListBuckets ¶
func (b *Backend) ListBuckets(ctx context.Context, ownerID string) (*backend.ListBucketsResponse, error)
ListBuckets returns a list of buckets from both config and s3db ownerID filters to only show buckets owned by the specified user (empty = all buckets)
func (*Backend) ListObjects ¶
func (b *Backend) ListObjects(ctx context.Context, req *backend.ListObjectsRequest) (*backend.ListObjectsResponse, error)
ListObjects returns a list of objects in a bucket by scanning global state Objects are stored with ARN key format: arn:aws:s3:::<bucket>/<key>
func (*Backend) PutObject ¶
func (b *Backend) PutObject(ctx context.Context, req *backend.PutObjectRequest) (*backend.PutObjectResponse, error)
PutObject stores an object using Reed-Solomon encoding across multiple nodes
func (*Backend) PutObjectFromPath ¶
PutObjectFromPath stores an object from a file path (used internally and for testing)
func (*Backend) RsDataShard ¶
RsDataShard returns the number of data shards (for testing)
func (*Backend) RsParityShard ¶
RsParityShard returns the number of parity shards (for testing)
func (*Backend) SetDataDir ¶
SetDataDir sets the data directory (for testing)
func (*Backend) UploadPart ¶
func (b *Backend) UploadPart(ctx context.Context, req *backend.UploadPartRequest) (*backend.UploadPartResponse, error)
UploadPart uploads a part in a multipart upload
type BucketConfig ¶
BucketConfig holds configuration for a bucket
type Config ¶
type Config struct {
// DataDir is the root directory for distributed node storage
DataDir string
// BadgerDir is the directory for the Badger KV database (used when DBClient is nil)
BadgerDir string
// Reed-Solomon configuration
DataShards int
ParityShards int
// Hash ring configuration
PartitionCount int
ReplicationFactor int
// QUIC server base port (each node uses BasePort + nodeNum)
QuicBasePort int
// UseQUIC enables QUIC-based shard distribution (requires QUIC servers to be running)
UseQUIC bool
// Nodes configuration (from cluster.toml)
Nodes []NodeConfig
// Buckets configuration (from cluster.toml)
Buckets []BucketConfig
// DBClient holds configuration for the distributed database client
// When set, uses distributed s3db for global state instead of local BadgerDB
DBClient *DBClientConfig
}
Config holds distributed backend configuration
type DBClientConfig ¶
type DBClientConfig struct {
Nodes []string // List of DB node addresses
AccessKeyID string // AWS-style access key ID
SecretAccessKey string // AWS-style secret access key
Region string // Region for signing (default: us-east-1)
}
DBClientConfig holds configuration for connecting to the distributed database
type DeletedObjectInfo ¶
type DeletedObjectInfo struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
ObjectHash [32]byte `json:"object_hash"`
DeletedAt int64 `json:"deleted_at"` // Unix timestamp
DataShardNodes []uint32 `json:"data_shard_nodes"` // Which nodes had data shards
ParityNodes []uint32 `json:"parity_nodes"` // Which nodes had parity shards
}
DeletedObjectInfo tracks a deleted object for compaction coordination
type DistributedState ¶
type DistributedState struct {
// contains filtered or unexported fields
}
DistributedState wraps an s3db.Client for GlobalState operations. This is used when a distributed database cluster is configured.
func NewDistributedState ¶
func NewDistributedState(cfg *DBClientConfig) (*DistributedState, error)
NewDistributedState creates a GlobalState backed by a distributed s3db cluster
func (*DistributedState) Client ¶
func (d *DistributedState) Client() *s3db.Client
Client returns the underlying s3db.Client
func (*DistributedState) Close ¶
func (d *DistributedState) Close() error
Close is a no-op for the distributed client (connections are reused)
func (*DistributedState) Delete ¶
func (d *DistributedState) Delete(table string, key []byte) error
Delete removes a key from the distributed database
func (*DistributedState) Exists ¶
func (d *DistributedState) Exists(table string, key []byte) (bool, error)
Exists checks if a key exists in the distributed database
func (*DistributedState) Get ¶
func (d *DistributedState) Get(table string, key []byte) ([]byte, error)
Get retrieves a value by key from the distributed database
func (*DistributedState) ListKeys ¶
func (d *DistributedState) ListKeys(table string, prefix []byte) ([][]byte, error)
ListKeys returns all keys with the given prefix (via Scan)
type GlobalState ¶
type GlobalState interface {
// Get retrieves a value by key from the specified table
Get(table string, key []byte) ([]byte, error)
// Set stores a key-value pair in the specified table
Set(table string, key []byte, value []byte) error
// Delete removes a key from the specified table
Delete(table string, key []byte) error
// Exists checks if a key exists in the specified table
Exists(table string, key []byte) (bool, error)
// ListKeys returns all keys with the given prefix in the specified table
ListKeys(table string, prefix []byte) ([][]byte, error)
// Scan iterates over all keys with the given prefix
Scan(table string, prefix []byte, fn func(key, value []byte) error) error
// Close closes the state store
Close() error
}
GlobalState provides an abstraction for global state storage operations. This allows the distributed backend to use either a local BadgerDB or a distributed s3db cluster for storing object metadata.
type LocalState ¶
type LocalState struct {
// contains filtered or unexported fields
}
LocalState wraps a local s3db.S3DB for GlobalState operations. This is used when no distributed database is configured.
func NewLocalState ¶
func NewLocalState(dbPath string) (*LocalState, error)
NewLocalState creates a GlobalState backed by a local BadgerDB
func (*LocalState) DB ¶
func (l *LocalState) DB() *s3db.S3DB
DB returns the underlying S3DB for backward compatibility
func (*LocalState) Delete ¶
func (l *LocalState) Delete(table string, key []byte) error
Delete removes a key
func (*LocalState) Exists ¶
func (l *LocalState) Exists(table string, key []byte) (bool, error)
Exists checks if a key exists
func (*LocalState) Get ¶
func (l *LocalState) Get(table string, key []byte) ([]byte, error)
Get retrieves a value by key (table is encoded in the key)
func (*LocalState) ListKeys ¶
func (l *LocalState) ListKeys(table string, prefix []byte) ([][]byte, error)
ListKeys returns all keys with the given prefix
type NodeConfig ¶
type NodeConfig struct {
ID int
Host string
Port int
Path string
DB bool
DBPort int
DBPath string
Leader bool
Epoch int
}
NodeConfig holds configuration for a single node
type ObjectShardReader ¶
type ObjectShardReader struct {
File *os.File
WALFileInfo wal.WALFileInfo
}
ObjectShardReader provides access to a shard stored in WAL