blob

package
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 16, 2026 License: MIT Imports: 9 Imported by: 0

Documentation

Overview

Package blob provides encrypted blob storage for Tier 2 data-private search. Blobs are encrypted vectors that can be stored on untrusted storage backends.

Index

Constants

This section is empty.

Variables

View Source
var (
	ErrBlobNotFound   = errors.New("blob not found")
	ErrBucketNotFound = errors.New("bucket not found")
	ErrBlobExists     = errors.New("blob already exists")
)

Common errors for blob stores.

Functions

This section is empty.

Types

type Blob

type Blob struct {
	// ID is the unique identifier for this blob.
	ID string `json:"id"`

	// LSHBucket is the LSH hash bucket this vector belongs to.
	// This is visible to storage (enables bucket-based retrieval).
	LSHBucket string `json:"lsh_bucket"`

	// Ciphertext is the encrypted vector data.
	// Format: nonce (12 bytes) || ciphertext || tag (16 bytes)
	Ciphertext []byte `json:"ciphertext"`

	// MetadataCiphertext is optional encrypted metadata.
	// Can store additional info like document title, source, etc.
	MetadataCiphertext []byte `json:"metadata_ciphertext,omitempty"`

	// Dimension is the vector dimension (visible, needed for decryption).
	Dimension int `json:"dimension"`

	// CreatedAt is when this blob was created.
	CreatedAt time.Time `json:"created_at"`

	// Version for future schema changes.
	Version int `json:"version"`
}

Blob represents an encrypted vector with metadata. The vector contents are encrypted; only the LSH bucket and ID are visible to storage.

func Deserialize

func Deserialize(data []byte) (*Blob, error)

Deserialize parses JSON bytes into a Blob.

func NewBlob

func NewBlob(id, lshBucket string, ciphertext []byte, dimension int) *Blob

NewBlob creates a new encrypted blob.

func (*Blob) Serialize

func (b *Blob) Serialize() ([]byte, error)

Serialize converts the blob to JSON bytes for storage.

func (*Blob) WithMetadata

func (b *Blob) WithMetadata(metadataCiphertext []byte) *Blob

WithMetadata adds encrypted metadata to the blob.

type BucketInfo

type BucketInfo struct {
	// Bucket is the LSH bucket identifier.
	Bucket string `json:"bucket"`

	// Count is the number of blobs in this bucket.
	Count int `json:"count"`

	// TotalSize is the total size of all blobs in bytes.
	TotalSize int64 `json:"total_size"`
}

BucketInfo contains information about a bucket.

type FileStore

type FileStore struct {
	// contains filtered or unexported fields
}

FileStore implements Store using the local filesystem. Each blob is stored as a JSON file. Good for local development and testing.

Directory structure:

basePath/
├── index.json          # Bucket -> blob ID mappings
└── blobs/
    ├── blob_id_1.json
    ├── blob_id_2.json
    └── ...

func NewFileStore

func NewFileStore(basePath string) (*FileStore, error)

NewFileStore creates a new file-based blob store. Creates the directory structure if it doesn't exist.

func (*FileStore) BucketInfo

func (s *FileStore) BucketInfo(ctx context.Context, bucket string) (*BucketInfo, error)

BucketInfo returns information about a bucket.

func (*FileStore) Close

func (s *FileStore) Close() error

Close is a no-op for file store.

func (*FileStore) Delete

func (s *FileStore) Delete(ctx context.Context, id string) error

Delete removes a blob from disk.

func (*FileStore) DeleteBatch

func (s *FileStore) DeleteBatch(ctx context.Context, ids []string) error

DeleteBatch removes multiple blobs.

func (*FileStore) Get

func (s *FileStore) Get(ctx context.Context, id string) (*Blob, error)

Get retrieves a blob from disk.

func (*FileStore) GetBatch

func (s *FileStore) GetBatch(ctx context.Context, ids []string) ([]*Blob, error)

GetBatch retrieves multiple blobs.

func (*FileStore) GetBucket

func (s *FileStore) GetBucket(ctx context.Context, bucket string) ([]*Blob, error)

GetBucket retrieves all blobs in a bucket.

func (*FileStore) GetBuckets

func (s *FileStore) GetBuckets(ctx context.Context, buckets []string) ([]*Blob, error)

GetBuckets retrieves blobs from multiple buckets.

func (*FileStore) GetSuperBuckets

func (s *FileStore) GetSuperBuckets(ctx context.Context, superBucketIDs []int) ([]*Blob, error)

GetSuperBuckets retrieves all blobs from the specified super-bucket IDs.

func (*FileStore) ListBuckets

func (s *FileStore) ListBuckets(ctx context.Context) ([]string, error)

ListBuckets returns all bucket identifiers.

func (*FileStore) Put

func (s *FileStore) Put(ctx context.Context, blob *Blob) error

Put stores a blob to disk.

func (*FileStore) PutBatch

func (s *FileStore) PutBatch(ctx context.Context, blobs []*Blob) error

PutBatch stores multiple blobs.

func (*FileStore) Stats

func (s *FileStore) Stats(ctx context.Context) (*StoreStats, error)

Stats returns store statistics.

type MemoryStore

type MemoryStore struct {
	// contains filtered or unexported fields
}

MemoryStore implements Store using in-memory maps. Useful for testing, demos, and small-scale deployments. Not persistent - data is lost on restart.

func NewMemoryStore

func NewMemoryStore() *MemoryStore

NewMemoryStore creates a new in-memory blob store.

func (*MemoryStore) BucketInfo

func (s *MemoryStore) BucketInfo(ctx context.Context, bucket string) (*BucketInfo, error)

BucketInfo returns information about a specific bucket.

func (*MemoryStore) Clear

func (s *MemoryStore) Clear()

Clear removes all blobs (useful for testing).

func (*MemoryStore) Close

func (s *MemoryStore) Close() error

Close is a no-op for memory store.

func (*MemoryStore) Delete

func (s *MemoryStore) Delete(ctx context.Context, id string) error

Delete removes a blob by ID.

func (*MemoryStore) DeleteBatch

func (s *MemoryStore) DeleteBatch(ctx context.Context, ids []string) error

DeleteBatch removes multiple blobs by ID.

func (*MemoryStore) Get

func (s *MemoryStore) Get(ctx context.Context, id string) (*Blob, error)

Get retrieves a blob by ID.

func (*MemoryStore) GetBatch

func (s *MemoryStore) GetBatch(ctx context.Context, ids []string) ([]*Blob, error)

GetBatch retrieves multiple blobs by ID.

func (*MemoryStore) GetBucket

func (s *MemoryStore) GetBucket(ctx context.Context, bucket string) ([]*Blob, error)

GetBucket retrieves all blobs in a bucket.

func (*MemoryStore) GetBuckets

func (s *MemoryStore) GetBuckets(ctx context.Context, buckets []string) ([]*Blob, error)

GetBuckets retrieves blobs from multiple buckets.

func (*MemoryStore) GetSuperBuckets

func (s *MemoryStore) GetSuperBuckets(ctx context.Context, superBucketIDs []int) ([]*Blob, error)

GetSuperBuckets retrieves all blobs from the specified super-bucket IDs.

func (*MemoryStore) ListBuckets

func (s *MemoryStore) ListBuckets(ctx context.Context) ([]string, error)

ListBuckets returns all bucket identifiers.

func (*MemoryStore) Put

func (s *MemoryStore) Put(ctx context.Context, blob *Blob) error

Put stores a blob in memory.

func (*MemoryStore) PutBatch

func (s *MemoryStore) PutBatch(ctx context.Context, blobs []*Blob) error

PutBatch stores multiple blobs.

func (*MemoryStore) Stats

func (s *MemoryStore) Stats(ctx context.Context) (*StoreStats, error)

Stats returns overall store statistics.

type ReadOnlyStore

type ReadOnlyStore interface {
	Get(ctx context.Context, id string) (*Blob, error)
	GetBatch(ctx context.Context, ids []string) ([]*Blob, error)
	GetBucket(ctx context.Context, bucket string) ([]*Blob, error)
	GetBuckets(ctx context.Context, buckets []string) ([]*Blob, error)
	GetSuperBuckets(ctx context.Context, superBucketIDs []int) ([]*Blob, error)
	ListBuckets(ctx context.Context) ([]string, error)
	Stats(ctx context.Context) (*StoreStats, error)
}

ReadOnlyStore is a read-only view of a blob store. Useful for search operations that don't need write access.

type Store

type Store interface {
	// Put stores a blob. Returns ErrBlobExists if ID already exists.
	Put(ctx context.Context, blob *Blob) error

	// PutBatch stores multiple blobs atomically.
	PutBatch(ctx context.Context, blobs []*Blob) error

	// Get retrieves a blob by ID. Returns ErrBlobNotFound if not found.
	Get(ctx context.Context, id string) (*Blob, error)

	// GetBatch retrieves multiple blobs by ID.
	// Returns nil for IDs that don't exist.
	GetBatch(ctx context.Context, ids []string) ([]*Blob, error)

	// GetBucket retrieves all blobs in a bucket.
	// Returns empty slice if bucket doesn't exist.
	GetBucket(ctx context.Context, bucket string) ([]*Blob, error)

	// GetBuckets retrieves blobs from multiple buckets.
	GetBuckets(ctx context.Context, buckets []string) ([]*Blob, error)

	// GetSuperBuckets retrieves all blobs from the specified super-bucket IDs.
	// Super-bucket keys are formatted as "XX" (2-digit zero-padded).
	// This is the primary method for fetching vectors after removing sub-buckets.
	GetSuperBuckets(ctx context.Context, superBucketIDs []int) ([]*Blob, error)

	// Delete removes a blob by ID. No error if blob doesn't exist.
	Delete(ctx context.Context, id string) error

	// DeleteBatch removes multiple blobs by ID.
	DeleteBatch(ctx context.Context, ids []string) error

	// ListBuckets returns all bucket identifiers.
	ListBuckets(ctx context.Context) ([]string, error)

	// BucketInfo returns information about a specific bucket.
	BucketInfo(ctx context.Context, bucket string) (*BucketInfo, error)

	// Stats returns overall store statistics.
	Stats(ctx context.Context) (*StoreStats, error)

	// Close closes the store connection.
	Close() error
}

Store is the interface for encrypted blob storage backends. Implementations can store blobs in memory, filesystem, S3, IPFS, blockchain, etc.

type StoreStats

type StoreStats struct {
	// TotalBlobs is the total number of blobs.
	TotalBlobs int64 `json:"total_blobs"`

	// TotalBuckets is the number of unique buckets.
	TotalBuckets int64 `json:"total_buckets"`

	// TotalSize is the total storage size in bytes.
	TotalSize int64 `json:"total_size"`

	// AvgBlobsPerBucket is the average number of blobs per bucket.
	AvgBlobsPerBucket float64 `json:"avg_blobs_per_bucket"`
}

StoreStats contains statistics about the blob store.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL