Documentation
¶
Overview ¶
Package embedding provides vector embedding generation and storage for RAG (Retrieval-Augmented Generation) capabilities.
Index ¶
- type ContentResolver
- type EmbedRequest
- type EmbeddingBuffer
- type EmbeddingProvider
- type GoogleProvider
- type LocalProvider
- type OpenAIProvider
- type ProviderConfig
- type RAGResult
- type RAGService
- type Registry
- type RetrieveOptions
- type SQLiteVecStore
- func (s *SQLiteVecStore) Close() error
- func (s *SQLiteVecStore) Delete(ctx context.Context, collection string, ids []string) error
- func (s *SQLiteVecStore) Search(ctx context.Context, collection string, query []float32, limit int, ...) ([]SearchResult, error)
- func (s *SQLiteVecStore) Upsert(ctx context.Context, records []VectorRecord) error
- type SearchOptions
- type SearchResult
- type StoreResolver
- type VectorRecord
- type VectorStore
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type ContentResolver ¶
type ContentResolver interface {
ResolveContent(ctx context.Context, collection, id string) (string, error)
}
ContentResolver looks up the original text content by collection and ID.
type EmbedRequest ¶
EmbedRequest represents a request to embed and store a text.
type EmbeddingBuffer ¶
type EmbeddingBuffer struct {
// contains filtered or unexported fields
}
EmbeddingBuffer collects embed requests and processes them in batches on a background goroutine. It follows the same lifecycle pattern as memory.Buffer: Start -> Enqueue -> Stop.
func NewEmbeddingBuffer ¶
func NewEmbeddingBuffer( provider EmbeddingProvider, store VectorStore, logger *zap.SugaredLogger, ) *EmbeddingBuffer
NewEmbeddingBuffer creates a new asynchronous embedding buffer.
func (*EmbeddingBuffer) DroppedCount ¶
func (b *EmbeddingBuffer) DroppedCount() int64
DroppedCount returns the total number of dropped embed requests.
func (*EmbeddingBuffer) Enqueue ¶
func (b *EmbeddingBuffer) Enqueue(req EmbedRequest)
Enqueue submits an embed request. Non-blocking; drops if the queue is full.
func (*EmbeddingBuffer) Start ¶
func (b *EmbeddingBuffer) Start(wg *sync.WaitGroup)
Start launches the background goroutine. The WaitGroup is incremented so callers can wait for graceful shutdown.
func (*EmbeddingBuffer) Stop ¶
func (b *EmbeddingBuffer) Stop()
Stop signals the background goroutine to drain and exit.
type EmbeddingProvider ¶
type EmbeddingProvider interface {
// ID returns the unique identifier for this provider.
ID() string
// Embed generates embeddings for the given texts in a single batch.
Embed(ctx context.Context, texts []string) ([][]float32, error)
// Dimensions returns the dimensionality of the generated embeddings.
Dimensions() int
}
EmbeddingProvider generates vector embeddings from text.
type GoogleProvider ¶
type GoogleProvider struct {
// contains filtered or unexported fields
}
GoogleProvider generates embeddings using Google's Generative AI API.
func NewGoogleProvider ¶
func NewGoogleProvider(apiKey string, model string, dimensions int) (*GoogleProvider, error)
NewGoogleProvider creates a new Google embedding provider.
func (*GoogleProvider) Dimensions ¶
func (p *GoogleProvider) Dimensions() int
func (*GoogleProvider) ID ¶
func (p *GoogleProvider) ID() string
type LocalProvider ¶
type LocalProvider struct {
// contains filtered or unexported fields
}
LocalProvider generates embeddings using a local Ollama instance via the OpenAI-compatible API endpoint.
func NewLocalProvider ¶
func NewLocalProvider(baseURL, model string, dimensions int) *LocalProvider
NewLocalProvider creates a new local embedding provider (Ollama).
func (*LocalProvider) Dimensions ¶
func (p *LocalProvider) Dimensions() int
func (*LocalProvider) ID ¶
func (p *LocalProvider) ID() string
type OpenAIProvider ¶
type OpenAIProvider struct {
// contains filtered or unexported fields
}
OpenAIProvider generates embeddings using OpenAI's API.
func NewOpenAIProvider ¶
func NewOpenAIProvider(apiKey string, model string, dimensions int) *OpenAIProvider
NewOpenAIProvider creates a new OpenAI embedding provider.
func (*OpenAIProvider) Dimensions ¶
func (p *OpenAIProvider) Dimensions() int
func (*OpenAIProvider) ID ¶
func (p *OpenAIProvider) ID() string
type ProviderConfig ¶
type ProviderConfig struct {
// Provider selects the embedding backend: "openai", "google", or "local".
Provider string
// Model is the embedding model identifier.
Model string
// Dimensions is the embedding vector dimensionality.
Dimensions int
// APIKey is required for openai and google providers.
APIKey string
// BaseURL is used by the local provider (Ollama endpoint).
BaseURL string
}
ProviderConfig holds configuration for creating an embedding provider.
type RAGService ¶
type RAGService struct {
// contains filtered or unexported fields
}
RAGService provides semantic retrieval across all embedded collections.
func NewRAGService ¶
func NewRAGService( provider EmbeddingProvider, store VectorStore, resolver ContentResolver, logger *zap.SugaredLogger, ) *RAGService
NewRAGService creates a new RAG retrieval service.
func (*RAGService) Retrieve ¶
func (r *RAGService) Retrieve(ctx context.Context, query string, opts RetrieveOptions) ([]RAGResult, error)
Retrieve finds relevant context across collections for a given query.
type Registry ¶
type Registry struct {
// contains filtered or unexported fields
}
Registry manages embedding provider selection with fallback.
func NewRegistry ¶
func NewRegistry(primary ProviderConfig, fallbacks []ProviderConfig, logger *zap.SugaredLogger) (*Registry, error)
NewRegistry creates a provider registry with the given primary config. If fallback configs are provided, the first successful one becomes the fallback.
func (*Registry) Fallback ¶
func (r *Registry) Fallback() EmbeddingProvider
Fallback returns the fallback provider, or nil if none.
func (*Registry) Provider ¶
func (r *Registry) Provider() EmbeddingProvider
Provider returns the active embedding provider.
type RetrieveOptions ¶
type RetrieveOptions struct {
// Collections to search (empty means all).
Collections []string
// Maximum results to return.
Limit int
// SessionKey filters for session-specific results.
SessionKey string
// MaxDistance is the maximum cosine distance for results (0.0 = disabled).
MaxDistance float32
}
RetrieveOptions configures a RAG retrieval query.
type SQLiteVecStore ¶
type SQLiteVecStore struct {
// contains filtered or unexported fields
}
SQLiteVecStore implements VectorStore using sqlite-vec.
func NewSQLiteVecStore ¶
func NewSQLiteVecStore(db *sql.DB, dimensions int) (*SQLiteVecStore, error)
NewSQLiteVecStore creates a new sqlite-vec backed vector store. The db connection should be the same SQLite database used by ent. Connection pool settings are managed by bootstrap; WAL mode enables safe concurrent access for file-based databases.
func (*SQLiteVecStore) Close ¶
func (s *SQLiteVecStore) Close() error
Close is a no-op; the underlying sql.DB is managed externally.
func (*SQLiteVecStore) Search ¶
func (s *SQLiteVecStore) Search(ctx context.Context, collection string, query []float32, limit int, opts *SearchOptions) ([]SearchResult, error)
Search finds the most similar vectors in a collection. When opts is non-nil and MetadataFilter is set, over-fetches by 3x and post-filters.
func (*SQLiteVecStore) Upsert ¶
func (s *SQLiteVecStore) Upsert(ctx context.Context, records []VectorRecord) error
Upsert inserts or replaces vector records.
type SearchOptions ¶
type SearchOptions struct {
// MetadataFilter post-filters results by metadata key-value pairs.
MetadataFilter map[string]string
}
SearchOptions configures optional vector search filtering.
type SearchResult ¶
type SearchResult struct {
ID string
Collection string
Distance float32
Metadata map[string]string
}
SearchResult represents a single result from a vector similarity search.
type StoreResolver ¶
type StoreResolver struct {
// contains filtered or unexported fields
}
StoreResolver resolves content from knowledge and memory stores.
func NewStoreResolver ¶
func NewStoreResolver(ks *knowledge.Store, ms *memory.Store) *StoreResolver
NewStoreResolver creates a content resolver backed by the application stores.
func (*StoreResolver) ResolveContent ¶
ResolveContent looks up the original text for a given collection and ID.
type VectorRecord ¶
type VectorRecord struct {
// ID is the source entity identifier.
ID string
// Collection groups records (e.g., "knowledge", "observation", "reflection", "learning").
Collection string
// Embedding is the vector representation.
Embedding []float32
// Metadata holds filterable key-value pairs (e.g., session_key, category).
Metadata map[string]string
}
VectorRecord represents a single vector entry in the store.
type VectorStore ¶
type VectorStore interface {
// Upsert inserts or replaces vector records.
Upsert(ctx context.Context, records []VectorRecord) error
// Search finds the most similar vectors in a collection.
// Pass nil for opts to use default behavior (no filtering).
Search(ctx context.Context, collection string, query []float32, limit int, opts *SearchOptions) ([]SearchResult, error)
// Delete removes vectors by collection and source IDs.
Delete(ctx context.Context, collection string, ids []string) error
// Close releases any resources held by the store.
Close() error
}
VectorStore provides vector storage and similarity search.