Documentation
¶
Index ¶
- Constants
- type BulkIndexer
- type BulkIndexerProvider
- type CompositeSearch
- func (c *CompositeSearch) BulkIndexer() BulkIndexer
- func (c *CompositeSearch) Clear(ctx context.Context) error
- func (c *CompositeSearch) Delete(ctx context.Context, postIDs []string) error
- func (c *CompositeSearch) DeleteOrphaned(ctx context.Context, nowTime, batchSize int64) (int64, error)
- func (c *CompositeSearch) Search(ctx context.Context, query string, opts SearchOptions) ([]SearchResult, error)
- func (c *CompositeSearch) Store(ctx context.Context, docs []PostDocument) error
- type EmbeddingProvider
- type EmbeddingSearch
- type EmbeddingSearchConfig
- type PostDocument
- type SearchOptions
- type SearchResult
- type UpstreamConfig
- type VectorStore
Constants ¶
const ( ProviderTypeOpenAI = "openai" ProviderTypeOpenAICompatible = "openai-compatible" ProviderTypeBifrost = "bifrost" ProviderTypeMock = "mock" )
Provider types
const ( DefaultReindexWorkers = 4 MaxReindexWorkers = 32 DefaultReindexBatchSize = 200 MaxReindexBatchSize = 1000 )
Reindex throughput defaults and bounds. Workers are concurrent embedding pipelines; batch size is posts fetched/embedded per request. Defaults sit comfortably inside OpenAI Tier 1 rate limits.
const ( ReindexIndexStrategyMaintain = "maintain" ReindexIndexStrategyDefer = "defer" )
ReindexIndexStrategy*: maintain updates ANN during load; defer rebuilds after.
const (
SearchTypeComposite = "composite"
)
Search types
const (
VectorStoreTypePGVector = "pgvector"
)
Vector store types
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type BulkIndexer ¶
type BulkIndexer interface {
PrepareBulkIndex(ctx context.Context) error
FinalizeBulkIndex(ctx context.Context) error
VectorIndexExists(ctx context.Context) (bool, error)
}
BulkIndexer drops/rebuilds the ANN index around bulk loads.
type BulkIndexerProvider ¶
type BulkIndexerProvider interface {
// BulkIndexer returns control, or nil if unsupported.
BulkIndexer() BulkIndexer
}
BulkIndexerProvider exposes bulk index control from a search service.
type CompositeSearch ¶
type CompositeSearch struct {
// contains filtered or unexported fields
}
CompositeSearch implements EmbeddingSearch using separate vector store and embedding provider
func NewCompositeSearch ¶
func NewCompositeSearch(store VectorStore, provider EmbeddingProvider, options chunking.Options) *CompositeSearch
NewCompositeSearch creates a new CompositeSearch with required chunking options
func (*CompositeSearch) BulkIndexer ¶
func (c *CompositeSearch) BulkIndexer() BulkIndexer
BulkIndexer returns the store's bulk control, or nil.
func (*CompositeSearch) Clear ¶
func (c *CompositeSearch) Clear(ctx context.Context) error
Clear removes all documents and chunks
func (*CompositeSearch) Delete ¶
func (c *CompositeSearch) Delete(ctx context.Context, postIDs []string) error
Delete removes documents and their chunks
func (*CompositeSearch) DeleteOrphaned ¶
func (c *CompositeSearch) DeleteOrphaned(ctx context.Context, nowTime, batchSize int64) (int64, error)
DeleteOrphaned removes embeddings whose posts no longer exist or are past retention.
func (*CompositeSearch) Search ¶
func (c *CompositeSearch) Search(ctx context.Context, query string, opts SearchOptions) ([]SearchResult, error)
Search performs a semantic search and merges results from chunks of the same document
func (*CompositeSearch) Store ¶
func (c *CompositeSearch) Store(ctx context.Context, docs []PostDocument) error
Store chunks documents, generates embeddings, and stores them
type EmbeddingProvider ¶
type EmbeddingProvider interface {
// CreateEmbedding generates embedding for the given text
CreateEmbedding(ctx context.Context, text string) ([]float32, error)
// BatchCreateEmbeddings generates embeddings for any number of texts;
// implementations are responsible for splitting the batch to respect
// their provider's per-request limits
BatchCreateEmbeddings(ctx context.Context, texts []string) ([][]float32, error)
// Dimensions returns the dimensionality of the embeddings
Dimensions() int
}
EmbeddingProvider defines the interface for embedding generation
func NewMockEmbeddingProvider ¶
func NewMockEmbeddingProvider(dimensions int) EmbeddingProvider
NewMockEmbeddingProvider creates a new mock embedding provider that produces repeatable vectors.
type EmbeddingSearch ¶
type EmbeddingSearch interface {
// Store stores documents and handles embedding generation internally
Store(ctx context.Context, docs []PostDocument) error
// Search performs a similarity search using the query text
Search(ctx context.Context, query string, opts SearchOptions) ([]SearchResult, error)
// Delete removes documents
Delete(ctx context.Context, postIDs []string) error
// Clear removes all documents
Clear(ctx context.Context) error
// DeleteOrphaned removes embeddings whose posts no longer exist or are past retention.
// nowTime is the retention cutoff (Unix millis), batchSize limits rows deleted per call.
// Returns the number of rows deleted.
DeleteOrphaned(ctx context.Context, nowTime, batchSize int64) (int64, error)
}
EmbeddingSearch defines the high-level interface for storing and searching using embeddings
type EmbeddingSearchConfig ¶
type EmbeddingSearchConfig struct {
Type string `json:"type"`
VectorStore UpstreamConfig `json:"vectorStore"`
EmbeddingProvider UpstreamConfig `json:"embeddingProvider"`
Parameters json.RawMessage `json:"parameters"`
Dimensions int `json:"dimensions"`
ChunkingOptions chunking.Options `json:"chunkingOptions"`
ReindexWorkers int `json:"reindexWorkers,omitempty"`
ReindexBatchSize int `json:"reindexBatchSize,omitempty"`
ReindexIndexStrategy string `json:"reindexIndexStrategy,omitempty"`
}
ServiceConfig holds configuration for the embedding search service
func (*EmbeddingSearchConfig) EffectiveReindexIndexStrategy ¶
func (c *EmbeddingSearchConfig) EffectiveReindexIndexStrategy() string
EffectiveReindexIndexStrategy: defer if set, otherwise maintain.
func (*EmbeddingSearchConfig) GetModelName ¶
func (c *EmbeddingSearchConfig) GetModelName() string
GetModelName extracts the model name from the embedding provider parameters
func (*EmbeddingSearchConfig) GetProviderType ¶
func (c *EmbeddingSearchConfig) GetProviderType() string
GetProviderType returns the embedding provider type
func (*EmbeddingSearchConfig) GetReindexBatchSize ¶
func (c *EmbeddingSearchConfig) GetReindexBatchSize() int
GetReindexBatchSize returns the configured reindex batch size, clamped to valid bounds, with unset (<=0) falling back to the default.
func (*EmbeddingSearchConfig) GetReindexWorkers ¶
func (c *EmbeddingSearchConfig) GetReindexWorkers() int
GetReindexWorkers returns the configured reindex worker count, clamped to valid bounds, with unset (<=0) falling back to the default.
type PostDocument ¶
type PostDocument struct {
PostID string // ID of the Mattermost post
CreateAt int64 // Creation timestamp of the referenced post, not when this was indexed
TeamID string
ChannelID string
UserID string
Content string
// Embed chunk info to track if this is a chunk
chunking.ChunkInfo
}
PostDocument represents a Mattermost post with its metadata
type SearchOptions ¶
type SearchOptions struct {
Limit int
Offset int
MinScore float32
TeamID string
ChannelID string
UserID string // User ID for permission checks
CreatedAfter int64
CreatedBefore int64
}
SearchOptions contains parameters for search operations
type SearchResult ¶
type SearchResult struct {
Document PostDocument
Score float32
}
SearchResult represents a single search result with its similarity score
type UpstreamConfig ¶
type UpstreamConfig struct {
Type string `json:"type"`
Parameters json.RawMessage `json:"parameters"`
}
UpstreamConfig holds configuration for the upstream service
type VectorStore ¶
type VectorStore interface {
// Store stores documents and their embeddings
Store(ctx context.Context, docs []PostDocument, embeddings [][]float32) error
// Search performs a similarity search using the provided embedding
Search(ctx context.Context, embedding []float32, opts SearchOptions) ([]SearchResult, error)
// Delete removes documents from the vector store
Delete(ctx context.Context, postIDs []string) error
// Clear removes all documents from the vector store
Clear(ctx context.Context) error
// DeleteOrphaned removes embeddings whose posts no longer exist or are past retention.
// nowTime is the retention cutoff (Unix millis), batchSize limits rows deleted per call.
// Returns the number of rows deleted.
DeleteOrphaned(ctx context.Context, nowTime, batchSize int64) (int64, error)
}
VectorStore defines the interface for vector storage and search operations