Documentation
¶
Overview ¶
Package blob provides encrypted blob storage for Tier 2 data-private search. Blobs are encrypted vectors that can be stored on untrusted storage backends.
Index ¶
- Variables
- type Blob
- type BucketInfo
- type FileStore
- func (s *FileStore) BucketInfo(ctx context.Context, bucket string) (*BucketInfo, error)
- func (s *FileStore) Close() error
- func (s *FileStore) Delete(ctx context.Context, id string) error
- func (s *FileStore) DeleteBatch(ctx context.Context, ids []string) error
- func (s *FileStore) Get(ctx context.Context, id string) (*Blob, error)
- func (s *FileStore) GetBatch(ctx context.Context, ids []string) ([]*Blob, error)
- func (s *FileStore) GetBucket(ctx context.Context, bucket string) ([]*Blob, error)
- func (s *FileStore) GetBuckets(ctx context.Context, buckets []string) ([]*Blob, error)
- func (s *FileStore) GetSuperBuckets(ctx context.Context, superBucketIDs []int) ([]*Blob, error)
- func (s *FileStore) ListBuckets(ctx context.Context) ([]string, error)
- func (s *FileStore) Put(ctx context.Context, blob *Blob) error
- func (s *FileStore) PutBatch(ctx context.Context, blobs []*Blob) error
- func (s *FileStore) Stats(ctx context.Context) (*StoreStats, error)
- type MemoryStore
- func (s *MemoryStore) BucketInfo(ctx context.Context, bucket string) (*BucketInfo, error)
- func (s *MemoryStore) Clear()
- func (s *MemoryStore) Close() error
- func (s *MemoryStore) Delete(ctx context.Context, id string) error
- func (s *MemoryStore) DeleteBatch(ctx context.Context, ids []string) error
- func (s *MemoryStore) Get(ctx context.Context, id string) (*Blob, error)
- func (s *MemoryStore) GetBatch(ctx context.Context, ids []string) ([]*Blob, error)
- func (s *MemoryStore) GetBucket(ctx context.Context, bucket string) ([]*Blob, error)
- func (s *MemoryStore) GetBuckets(ctx context.Context, buckets []string) ([]*Blob, error)
- func (s *MemoryStore) GetSuperBuckets(ctx context.Context, superBucketIDs []int) ([]*Blob, error)
- func (s *MemoryStore) ListBuckets(ctx context.Context) ([]string, error)
- func (s *MemoryStore) Put(ctx context.Context, blob *Blob) error
- func (s *MemoryStore) PutBatch(ctx context.Context, blobs []*Blob) error
- func (s *MemoryStore) Stats(ctx context.Context) (*StoreStats, error)
- type ReadOnlyStore
- type Store
- type StoreStats
Constants ¶
This section is empty.
Variables ¶
var ( ErrBlobNotFound = errors.New("blob not found") ErrBucketNotFound = errors.New("bucket not found") ErrBlobExists = errors.New("blob already exists") )
Common errors for blob stores.
Functions ¶
This section is empty.
Types ¶
type Blob ¶
type Blob struct {
// ID is the unique identifier for this blob.
ID string `json:"id"`
// LSHBucket is the LSH hash bucket this vector belongs to.
// This is visible to storage (enables bucket-based retrieval).
LSHBucket string `json:"lsh_bucket"`
// Ciphertext is the encrypted vector data.
// Format: nonce (12 bytes) || ciphertext || tag (16 bytes)
Ciphertext []byte `json:"ciphertext"`
// MetadataCiphertext is optional encrypted metadata.
// Can store additional info like document title, source, etc.
MetadataCiphertext []byte `json:"metadata_ciphertext,omitempty"`
// Dimension is the vector dimension (visible, needed for decryption).
Dimension int `json:"dimension"`
// CreatedAt is when this blob was created.
CreatedAt time.Time `json:"created_at"`
// Version for future schema changes.
Version int `json:"version"`
}
Blob represents an encrypted vector with metadata. The vector contents are encrypted; only the LSH bucket and ID are visible to storage.
func Deserialize ¶
Deserialize parses JSON bytes into a Blob.
func (*Blob) WithMetadata ¶
WithMetadata adds encrypted metadata to the blob.
type BucketInfo ¶
type BucketInfo struct {
// Bucket is the LSH bucket identifier.
Bucket string `json:"bucket"`
// Count is the number of blobs in this bucket.
Count int `json:"count"`
// TotalSize is the total size of all blobs in bytes.
TotalSize int64 `json:"total_size"`
}
BucketInfo contains information about a bucket.
type FileStore ¶
type FileStore struct {
// contains filtered or unexported fields
}
FileStore implements Store using the local filesystem. Each blob is stored as a JSON file. Good for local development and testing.
Directory structure:
basePath/
├── index.json # Bucket -> blob ID mappings
└── blobs/
├── blob_id_1.json
├── blob_id_2.json
└── ...
func NewFileStore ¶
NewFileStore creates a new file-based blob store. Creates the directory structure if it doesn't exist.
func (*FileStore) BucketInfo ¶
BucketInfo returns information about a bucket.
func (*FileStore) DeleteBatch ¶
DeleteBatch removes multiple blobs.
func (*FileStore) GetBuckets ¶
GetBuckets retrieves blobs from multiple buckets.
func (*FileStore) GetSuperBuckets ¶
GetSuperBuckets retrieves all blobs from the specified super-bucket IDs.
func (*FileStore) ListBuckets ¶
ListBuckets returns all bucket identifiers.
type MemoryStore ¶
type MemoryStore struct {
// contains filtered or unexported fields
}
MemoryStore implements Store using in-memory maps. Useful for testing, demos, and small-scale deployments. Not persistent - data is lost on restart.
func NewMemoryStore ¶
func NewMemoryStore() *MemoryStore
NewMemoryStore creates a new in-memory blob store.
func (*MemoryStore) BucketInfo ¶
func (s *MemoryStore) BucketInfo(ctx context.Context, bucket string) (*BucketInfo, error)
BucketInfo returns information about a specific bucket.
func (*MemoryStore) Clear ¶
func (s *MemoryStore) Clear()
Clear removes all blobs (useful for testing).
func (*MemoryStore) Delete ¶
func (s *MemoryStore) Delete(ctx context.Context, id string) error
Delete removes a blob by ID.
func (*MemoryStore) DeleteBatch ¶
func (s *MemoryStore) DeleteBatch(ctx context.Context, ids []string) error
DeleteBatch removes multiple blobs by ID.
func (*MemoryStore) GetBuckets ¶
GetBuckets retrieves blobs from multiple buckets.
func (*MemoryStore) GetSuperBuckets ¶
GetSuperBuckets retrieves all blobs from the specified super-bucket IDs.
func (*MemoryStore) ListBuckets ¶
func (s *MemoryStore) ListBuckets(ctx context.Context) ([]string, error)
ListBuckets returns all bucket identifiers.
func (*MemoryStore) Put ¶
func (s *MemoryStore) Put(ctx context.Context, blob *Blob) error
Put stores a blob in memory.
func (*MemoryStore) PutBatch ¶
func (s *MemoryStore) PutBatch(ctx context.Context, blobs []*Blob) error
PutBatch stores multiple blobs.
func (*MemoryStore) Stats ¶
func (s *MemoryStore) Stats(ctx context.Context) (*StoreStats, error)
Stats returns overall store statistics.
type ReadOnlyStore ¶
type ReadOnlyStore interface {
Get(ctx context.Context, id string) (*Blob, error)
GetBatch(ctx context.Context, ids []string) ([]*Blob, error)
GetBucket(ctx context.Context, bucket string) ([]*Blob, error)
GetBuckets(ctx context.Context, buckets []string) ([]*Blob, error)
GetSuperBuckets(ctx context.Context, superBucketIDs []int) ([]*Blob, error)
ListBuckets(ctx context.Context) ([]string, error)
Stats(ctx context.Context) (*StoreStats, error)
}
ReadOnlyStore is a read-only view of a blob store. Useful for search operations that don't need write access.
type Store ¶
type Store interface {
// Put stores a blob. Returns ErrBlobExists if ID already exists.
Put(ctx context.Context, blob *Blob) error
// PutBatch stores multiple blobs atomically.
PutBatch(ctx context.Context, blobs []*Blob) error
// Get retrieves a blob by ID. Returns ErrBlobNotFound if not found.
Get(ctx context.Context, id string) (*Blob, error)
// GetBatch retrieves multiple blobs by ID.
// Returns nil for IDs that don't exist.
GetBatch(ctx context.Context, ids []string) ([]*Blob, error)
// GetBucket retrieves all blobs in a bucket.
// Returns empty slice if bucket doesn't exist.
GetBucket(ctx context.Context, bucket string) ([]*Blob, error)
// GetBuckets retrieves blobs from multiple buckets.
GetBuckets(ctx context.Context, buckets []string) ([]*Blob, error)
// GetSuperBuckets retrieves all blobs from the specified super-bucket IDs.
// Super-bucket keys are formatted as "XX" (2-digit zero-padded).
// This is the primary method for fetching vectors after removing sub-buckets.
GetSuperBuckets(ctx context.Context, superBucketIDs []int) ([]*Blob, error)
// Delete removes a blob by ID. No error if blob doesn't exist.
Delete(ctx context.Context, id string) error
// DeleteBatch removes multiple blobs by ID.
DeleteBatch(ctx context.Context, ids []string) error
// ListBuckets returns all bucket identifiers.
ListBuckets(ctx context.Context) ([]string, error)
// BucketInfo returns information about a specific bucket.
BucketInfo(ctx context.Context, bucket string) (*BucketInfo, error)
// Stats returns overall store statistics.
Stats(ctx context.Context) (*StoreStats, error)
// Close closes the store connection.
Close() error
}
Store is the interface for encrypted blob storage backends. Implementations can store blobs in memory, filesystem, S3, IPFS, blockchain, etc.
type StoreStats ¶
type StoreStats struct {
// TotalBlobs is the total number of blobs.
TotalBlobs int64 `json:"total_blobs"`
// TotalBuckets is the number of unique buckets.
TotalBuckets int64 `json:"total_buckets"`
// TotalSize is the total storage size in bytes.
TotalSize int64 `json:"total_size"`
// AvgBlobsPerBucket is the average number of blobs per bucket.
AvgBlobsPerBucket float64 `json:"avg_blobs_per_bucket"`
}
StoreStats contains statistics about the blob store.