Documentation
¶
Overview ¶
Package store provides a SQLite-backed memory store with FTS5 full-text search and vector search via sqlite-vec for the LLMem project.
Index ¶
- func BytesToVec(data []byte) []float32
- func DefaultRegisteredTypes() []string
- func ValidRelationTypes() []string
- func VecToBytes(vec []float32) []byte
- type AddParams
- type DuplicatePair
- type EmbeddingWithType
- type ExtractionLog
- type FindSimilarParams
- type ImportMemory
- type ListParams
- type Memory
- type MemoryStore
- func (ms *MemoryStore) Add(ctx context.Context, params AddParams) (string, error)
- func (ms *MemoryStore) AddRelation(ctx context.Context, sourceID, targetID, relationType string) (string, error)
- func (ms *MemoryStore) Close() error
- func (ms *MemoryStore) ConsolidateDuplicates(ctx context.Context, threshold float64, limit int) ([]*DuplicatePair, error)
- func (ms *MemoryStore) Count(ctx context.Context, validOnly bool) (int, error)
- func (ms *MemoryStore) CountByType(ctx context.Context, validOnly bool) (map[string]int, error)
- func (ms *MemoryStore) CountEmbeddings(ctx context.Context) (int, error)
- func (ms *MemoryStore) DB() *sql.DB
- func (ms *MemoryStore) Delete(ctx context.Context, id string) (bool, error)
- func (ms *MemoryStore) ExportAll(ctx context.Context, limit *int) ([]*Memory, error)
- func (ms *MemoryStore) FindSimilar(ctx context.Context, params FindSimilarParams) ([]*ScoredMemory, error)
- func (ms *MemoryStore) Get(ctx context.Context, id string, trackAccess bool) (*Memory, error)
- func (ms *MemoryStore) GetBatch(ctx context.Context, ids []string, validOnly bool) (map[string]*Memory, error)
- func (ms *MemoryStore) GetEmbeddingsWithTypes(ctx context.Context, limit int) ([]*EmbeddingWithType, error)
- func (ms *MemoryStore) GetRelations(ctx context.Context, memID string) ([]*Relation, error)
- func (ms *MemoryStore) GetRelationsBatch(ctx context.Context, memIDs []string) ([]*Relation, error)
- func (ms *MemoryStore) ImportMemories(ctx context.Context, memories []ImportMemory) (int, error)
- func (ms *MemoryStore) Invalidate(ctx context.Context, id string, reason string) (bool, error)
- func (ms *MemoryStore) IsExtracted(ctx context.Context, sourceType, sourceID string) (bool, error)
- func (ms *MemoryStore) ListAll(ctx context.Context, params ListParams) ([]*Memory, error)
- func (ms *MemoryStore) LogExtraction(ctx context.Context, sourceType, sourceID string, rawText *string, ...) error
- func (ms *MemoryStore) RegisterMemoryType(typeName string) error
- func (ms *MemoryStore) RemoveExtractionLog(ctx context.Context, sourceType, sourceID string) (bool, error)
- func (ms *MemoryStore) Search(ctx context.Context, params SearchParams) ([]*Memory, error)
- func (ms *MemoryStore) SearchByEmbedding(ctx context.Context, queryVec []float32, validOnly bool, limit int, ...) ([]*ScoredMemory, error)
- func (ms *MemoryStore) SearchCount(ctx context.Context, params SearchCountParams) (int, error)
- func (ms *MemoryStore) SupersedeBySource(ctx context.Context, sourceType, sourceID string) (int, error)
- func (ms *MemoryStore) Touch(ctx context.Context, id string) (bool, error)
- func (ms *MemoryStore) TouchBatch(ctx context.Context, ids []string) (int, error)
- func (ms *MemoryStore) TraverseRelations(ctx context.Context, startIDs []string, maxDepth int) ([]*TraversedRelation, error)
- func (ms *MemoryStore) Update(ctx context.Context, params UpdateParams) (bool, error)
- type Relation
- type ScoredMemory
- type SearchCountParams
- type SearchParams
- type StoreConfig
- type TraversedRelation
- type UpdateParams
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func BytesToVec ¶
BytesToVec decodes a packed float32 byte slice into a []float32. Matches Python's struct.unpack(f"{dim}f", data).
func DefaultRegisteredTypes ¶
func DefaultRegisteredTypes() []string
DefaultRegisteredTypes returns the 7 default memory types. Returns a new slice each time (defensive copy). Must match the types registered in migration 003_register_default_types.sql.
func ValidRelationTypes ¶
func ValidRelationTypes() []string
ValidRelationTypes returns the set of allowed relation types.
func VecToBytes ¶
VecToBytes encodes a []float32 into packed little-endian bytes. Matches Python's struct.pack(f"{dim}f", *vec).
Types ¶
type AddParams ¶
type AddParams struct {
ID string
Type string
Content string
Summary string
Source string
Confidence float64
ValidUntil string
Metadata map[string]any
Embedding []byte
Hints []string
}
AddParams contains the parameters for adding a new memory.
type DuplicatePair ¶
DuplicatePair represents a pair of similar memories.
type EmbeddingWithType ¶
EmbeddingWithType pairs an embedding with its memory type.
type ExtractionLog ¶
type ExtractionLog struct {
ID int64
SourceType string
SourceID string
RawText string
ExtractedCount int
CreatedAt string
}
ExtractionLog represents an extraction log entry.
type FindSimilarParams ¶
FindSimilarParams contains the parameters for finding similar memories.
type ImportMemory ¶
type ImportMemory struct {
ID string
Type string
Content string
Summary string
Source string
Confidence float64
Metadata map[string]any
Embedding []byte
Hints []string
}
ImportMemory represents a memory to be imported.
type ListParams ¶
ListParams contains the parameters for listing memories.
type Memory ¶
type Memory struct {
ID string
Type string
Content string
Summary string
Hints []string
Source string
Confidence float64
ValidFrom string
ValidUntil string
CreatedAt string
UpdatedAt string
AccessedAt string
AccessCount int
Metadata map[string]any
Embedding []byte
}
Memory represents a single memory record in the store.
type MemoryStore ¶
type MemoryStore struct {
// contains filtered or unexported fields
}
MemoryStore is a SQLite-backed memory store with FTS5 full-text search and vector search via sqlite-vec.
func NewMemoryStore ¶
func NewMemoryStore(cfg StoreConfig) (*MemoryStore, error)
NewMemoryStore creates and initializes a MemoryStore. It opens the database, applies migrations, and optionally sets up vec0. If cfg.DBPath is empty, defaults to ~/.config/llmem/memory.db. If cfg.VecDimensions is 0, defaults to 768. The constructor leaves the store in a fully usable state.
func (*MemoryStore) Add ¶
Add creates a new memory and returns its ID. Returns an error if the type is not registered or embedding dimensions don't match.
func (*MemoryStore) AddRelation ¶
func (ms *MemoryStore) AddRelation(ctx context.Context, sourceID, targetID, relationType string) (string, error)
AddRelation adds a relation between two memories. Returns the relation UUID. Validates relation_type against allowed types.
func (*MemoryStore) Close ¶
func (ms *MemoryStore) Close() error
Close closes the database connection. Safe to call multiple times.
func (*MemoryStore) ConsolidateDuplicates ¶
func (ms *MemoryStore) ConsolidateDuplicates(ctx context.Context, threshold float64, limit int) ([]*DuplicatePair, error)
ConsolidateDuplicates finds near-duplicate pairs by cosine similarity.
func (*MemoryStore) Count ¶
Count returns total number of memories, optionally limited to valid only.
func (*MemoryStore) CountByType ¶
CountByType returns a map of type to count, sorted by count descending.
func (*MemoryStore) CountEmbeddings ¶
func (ms *MemoryStore) CountEmbeddings(ctx context.Context) (int, error)
CountEmbeddings returns the count of valid memories with non-null embeddings.
func (*MemoryStore) DB ¶
func (ms *MemoryStore) DB() *sql.DB
DB returns the underlying database connection for advanced queries. Most callers should use the higher-level methods instead.
func (*MemoryStore) Delete ¶
Delete removes a memory by ID and cascades to its relations. Returns false if not found.
func (*MemoryStore) ExportAll ¶
ExportAll exports all memories ordered by created_at. If limit is nil, defaults to 10000. Pass 0 for no limit.
func (*MemoryStore) FindSimilar ¶
func (ms *MemoryStore) FindSimilar(ctx context.Context, params FindSimilarParams) ([]*ScoredMemory, error)
FindSimilar finds similar memories using vector or text search.
func (*MemoryStore) Get ¶
Get retrieves a memory by ID. Returns nil if not found. If trackAccess is true, increments access_count and updates accessed_at.
func (*MemoryStore) GetBatch ¶
func (ms *MemoryStore) GetBatch(ctx context.Context, ids []string, validOnly bool) (map[string]*Memory, error)
GetBatch retrieves multiple memories by their IDs. Returns a map keyed by memory ID. Returns empty map for empty input. If validOnly is true, only returns memories with valid_until IS NULL.
func (*MemoryStore) GetEmbeddingsWithTypes ¶
func (ms *MemoryStore) GetEmbeddingsWithTypes(ctx context.Context, limit int) ([]*EmbeddingWithType, error)
GetEmbeddingsWithTypes returns (embedding_bytes, type) tuples for valid memories with embeddings. If limit is 0, returns all rows (no limit). If limit < 0, defaults to defaultBruteForceMaxRows.
func (*MemoryStore) GetRelations ¶
GetRelations returns all relations for a memory (both source and target).
func (*MemoryStore) GetRelationsBatch ¶
GetRelationsBatch retrieves relations for multiple memory IDs.
func (*MemoryStore) ImportMemories ¶
func (ms *MemoryStore) ImportMemories(ctx context.Context, memories []ImportMemory) (int, error)
ImportMemories imports a list of memories with per-entry validation. Skips invalid entries with slog.Warn. Returns count of successfully imported memories.
func (*MemoryStore) Invalidate ¶
Invalidate sets valid_until on a memory, effectively expiring it. Also clears the embedding and updates metadata with invalidation_reason. Returns false if not found.
func (*MemoryStore) IsExtracted ¶
IsExtracted checks whether a source has been extracted.
func (*MemoryStore) ListAll ¶
func (ms *MemoryStore) ListAll(ctx context.Context, params ListParams) ([]*Memory, error)
ListAll lists memories with optional type filter. Delegates to Search with no query.
func (*MemoryStore) LogExtraction ¶
func (ms *MemoryStore) LogExtraction(ctx context.Context, sourceType, sourceID string, rawText *string, extractedCount int) error
LogExtraction upserts an extraction log entry.
func (*MemoryStore) RegisterMemoryType ¶
func (ms *MemoryStore) RegisterMemoryType(typeName string) error
RegisterMemoryType adds a custom memory type to the instance's type registry. The type name must match ^[a-z][a-z0-9_]*$ and be at most 64 characters. Returns an error for duplicate registration or invalid names.
func (*MemoryStore) RemoveExtractionLog ¶
func (ms *MemoryStore) RemoveExtractionLog(ctx context.Context, sourceType, sourceID string) (bool, error)
RemoveExtractionLog removes an extraction log entry. Returns false if not found.
func (*MemoryStore) Search ¶
func (ms *MemoryStore) Search(ctx context.Context, params SearchParams) ([]*Memory, error)
Search performs FTS5 full-text search with fallback to LIKE. Returns memories ranked by BM25 (FTS) or updated_at DESC (LIKE).
func (*MemoryStore) SearchByEmbedding ¶
func (ms *MemoryStore) SearchByEmbedding(ctx context.Context, queryVec []float32, validOnly bool, limit int, threshold float64) ([]*ScoredMemory, error)
SearchByEmbedding searches memories by vector similarity. Uses the vec virtual table if available, brute-force fallback otherwise. If limit <= 0, defaults to 20.
func (*MemoryStore) SearchCount ¶
func (ms *MemoryStore) SearchCount(ctx context.Context, params SearchCountParams) (int, error)
SearchCount returns the number of matching memories.
func (*MemoryStore) SupersedeBySource ¶
func (ms *MemoryStore) SupersedeBySource(ctx context.Context, sourceType, sourceID string) (int, error)
SupersedeBySource invalidates all memories matching source metadata.
func (*MemoryStore) Touch ¶
Touch increments access_count and updates accessed_at for a memory. Returns false if not found.
func (*MemoryStore) TouchBatch ¶
TouchBatch increments access_count and updates accessed_at for multiple memories. Returns the number of rows affected.
func (*MemoryStore) TraverseRelations ¶
func (ms *MemoryStore) TraverseRelations(ctx context.Context, startIDs []string, maxDepth int) ([]*TraversedRelation, error)
TraverseRelations performs bidirectional relation traversal using recursive CTE. Caps maxDepth at 5. Returns deduplicated results with distance and relationScore.
func (*MemoryStore) Update ¶
func (ms *MemoryStore) Update(ctx context.Context, params UpdateParams) (bool, error)
Update modifies a memory's fields. Returns false if not found, true on success. Rejects ClearEmbedding=true AND Embedding != nil (conflict).
type Relation ¶
type Relation struct {
ID string
SourceID string
TargetID string
RelationType string
CreatedAt string
}
Relation represents a relationship between two memories.
type ScoredMemory ¶
ScoredMemory represents a memory with a similarity score.
type SearchCountParams ¶
SearchCountParams contains the parameters for counting search results.
type SearchParams ¶
type SearchParams struct {
Query string
Type string
ValidOnly bool
Limit int
Offset int
FTSOnly bool
SemanticOnly bool
}
SearchParams contains the parameters for searching memories.
type StoreConfig ¶
type StoreConfig struct {
// DBPath is the path to the SQLite database file.
// If empty, defaults to ~/.config/llmem/memory.db.
DBPath string
// VecDimensions is the dimensionality for the embedding index.
// Defaults to 768 if zero. Must be positive.
VecDimensions int
// DisableVec skips vec virtual table creation and vector search.
DisableVec bool
// RegisteredTypes lists the memory types to register at construction.
// If empty, defaults to the 7 standard types from DefaultRegisteredTypes().
RegisteredTypes []string
}
StoreConfig contains the configuration for creating a MemoryStore.