vector

package
v0.0.0-...-dbba89f Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 14, 2026 License: MIT Imports: 7 Imported by: 0

Documentation

Overview

Package vector - AutoEmbedder implementation with provider abstraction

Package vector - Embedding provider dependencies

Package vector - Query vectorization for search pipeline

Package vector provides vector storage interfaces and implementations for embeddings. It supports Qdrant and pgvector for storing embeddings and performing similarity search.

Example (BatchVectorization)

Example_batchVectorization demonstrates batch query processing 

// Setup
cache := NewInMemoryEmbeddingCache()
embedder := NewAutoEmbedder("openai", cache)
openaiProvider := NewOpenAIEmbeddingProvider("your-api-key", "text-embedding-3-small")
embedder.AddProvider("openai", openaiProvider)

vectorizer := NewQueryVectorizer(embedder)

// Vectorize multiple queries at once
ctx := context.Background()
queries := []string{
	"What is present perfect?",
	"How to use past simple?",
	"Difference between present and past?",
}

embeddings, err := vectorizer.VectorizeBatchQueries(ctx, queries)
if err != nil {
	log.Fatalf("Failed to vectorize batch: %v", err)
}

fmt.Printf("Vectorized %d queries\n", len(embeddings))
for i, embedding := range embeddings {
	fmt.Printf("Query %d: %d dimensions\n", i+1, len(embedding))
}
Example (Caching)

Example_caching demonstrates caching behavior 

// Create embedder with caching enabled
cache := NewInMemoryEmbeddingCache()
embedder := NewAutoEmbedder("openai", cache)
openaiProvider := NewOpenAIEmbeddingProvider("your-api-key", "text-embedding-3-small")
embedder.AddProvider("openai", openaiProvider)

vectorizer := NewQueryVectorizer(embedder)
ctx := context.Background()

queryText := "What is present perfect?"

// First call - generates embedding and caches it
fmt.Println("First call (generates embedding)...")
vector1, _ := vectorizer.VectorizeQuery(ctx, queryText)

// Second call - uses cached embedding (faster)
fmt.Println("Second call (uses cache)...")
vector2, _ := vectorizer.VectorizeQuery(ctx, queryText)

// Verify they're identical
identical := true
for i := range vector1 {
	if vector1[i] != vector2[i] {
		identical = false
		break
	}
}

fmt.Printf("Vectors identical: %v\n", identical)
fmt.Printf("Cache size: %d embeddings\n", cache.GetSize())
Example (MultiProvider)

Example_multiProvider demonstrates provider fallback 

// Create AutoEmbedder with multiple providers
cache := NewInMemoryEmbeddingCache()
embedder := NewAutoEmbedder("openai", cache)

// Add primary provider (OpenAI)
openaiProvider := NewOpenAIEmbeddingProvider("your-openai-key", "text-embedding-3-small")
embedder.AddProvider("openai", openaiProvider)

// Add fallback provider (Ollama for local processing)
// ollamaProvider := NewOllamaEmbeddingProvider("http://localhost:11434", "nomic-embed-text")
// embedder.AddProvider("ollama", ollamaProvider)
// embedder.AddFallback("ollama")

vectorizer := NewQueryVectorizer(embedder)

// If OpenAI fails, it will automatically try Ollama
ctx := context.Background()
vector, err := vectorizer.VectorizeQuery(ctx, "Test query")
if err != nil {
	log.Fatalf("All providers failed: %v", err)
}

fmt.Printf("Successfully generated vector with %d dimensions\n", len(vector))
fmt.Printf("Provider used: %s\n", vectorizer.GetModel())
Example (ProcessedQuery)

Example_processedQuery demonstrates creating a ProcessedQuery 

// Setup
cache := NewInMemoryEmbeddingCache()
embedder := NewAutoEmbedder("openai", cache)
openaiProvider := NewOpenAIEmbeddingProvider("your-api-key", "text-embedding-3-small")
embedder.AddProvider("openai", openaiProvider)

vectorizer := NewQueryVectorizer(embedder)

// Create a ProcessedQuery with vector
ctx := context.Background()
queryText := "How do I use past simple tense?"

processedQuery, err := vectorizer.VectorizeProcessedQuery(ctx, queryText)
if err != nil {
	log.Fatalf("Failed to create processed query: %v", err)
}

fmt.Printf("Original text: %s\n", processedQuery.OriginalText)
fmt.Printf("Vector dimensions: %d\n", len(processedQuery.Vector))
fmt.Printf("Ready for Step 2: Hybrid Search\n")
Example (QueryVectorizer)

Example_queryVectorizer demonstrates basic query vectorization 

// Create an AutoEmbedder with OpenAI provider
cache := NewInMemoryEmbeddingCache()
embedder := NewAutoEmbedder("openai", cache)

// Add OpenAI provider (requires API key in production)
openaiProvider := NewOpenAIEmbeddingProvider("your-api-key", "text-embedding-3-small")
embedder.AddProvider("openai", openaiProvider)

// Create query vectorizer
vectorizer := NewQueryVectorizer(embedder)

// Vectorize a search query
ctx := context.Background()
queryText := "What is the present perfect tense?"

vector, err := vectorizer.VectorizeQuery(ctx, queryText)
if err != nil {
	log.Fatalf("Failed to vectorize query: %v", err)
}

fmt.Printf("Generated vector with %d dimensions\n", len(vector))
fmt.Printf("First 5 values: %.4f, %.4f, %.4f, %.4f, %.4f\n",
	vector[0], vector[1], vector[2], vector[3], vector[4])
Example (SearchPipelineIntegration)

Example_searchPipelineIntegration demonstrates integration with search pipeline 

// This example shows how query vectorization fits into the 4-step search pipeline

// Setup
cache := NewInMemoryEmbeddingCache()
embedder := NewAutoEmbedder("openai", cache)
openaiProvider := NewOpenAIEmbeddingProvider("your-api-key", "text-embedding-3-small")
embedder.AddProvider("openai", openaiProvider)

vectorizer := NewQueryVectorizer(embedder)
ctx := context.Background()

// Step 1: Input Processing & Vectorization (THIS TASK)
queryText := "Cách dùng thì Hiện tại hoàn thành mà tôi đã học là gì?"
processedQuery, err := vectorizer.VectorizeProcessedQuery(ctx, queryText)
if err != nil {
	log.Fatalf("Step 1 failed: %v", err)
}

fmt.Println("=== Step 1: Input Processing & Vectorization ===")
fmt.Printf("Query: %s\n", processedQuery.OriginalText)
fmt.Printf("Vector: [%.4f, %.4f, ...] (%d dimensions)\n",
	processedQuery.Vector[0], processedQuery.Vector[1], len(processedQuery.Vector))

// Step 2: Hybrid Search (Next task - uses the vector)
fmt.Println("\n=== Step 2: Hybrid Search ===")
fmt.Println("- Vector Search: Use processedQuery.Vector for similarity search")
fmt.Println("- Entity Search: Extract entities from processedQuery.OriginalText")
fmt.Println("- Result: Anchor nodes for graph traversal")

// Step 3: Graph Traversal (Future task)
fmt.Println("\n=== Step 3: Graph Traversal ===")
fmt.Println("- 1-hop: Direct neighbors of anchor nodes")
fmt.Println("- 2-hop: Indirect neighbors for extended context")

// Step 4: Context Assembly (Future task)
fmt.Println("\n=== Step 4: Context Assembly ===")
fmt.Println("- Rerank: Multi-factor scoring")
fmt.Println("- Build: Rich context for LLM")

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

func RegisterEmbeddingProvider 

func RegisterEmbeddingProvider(providerType EmbeddingProviderType, factory EmbeddingProviderFactory)

RegisterEmbeddingProvider registers a new embedding provider implementation

func RegisterStore 

func RegisterStore(storeType VectorStoreType, factory VectorStoreFactory)

RegisterStore registers a new vector store implementation

Types

type AutoEmbedder 

type AutoEmbedder struct {
	// contains filtered or unexported fields
}

AutoEmbedder manages multiple embedding providers with caching and fallback

func NewAutoEmbedder 

func NewAutoEmbedder(primary string, cache EmbeddingCache) *AutoEmbedder

NewAutoEmbedder creates a new AutoEmbedder

func (*AutoEmbedder) AddFallback 

func (ae *AutoEmbedder) AddFallback(name string) error

AddFallback adds a fallback provider

func (*AutoEmbedder) AddProvider 

func (ae *AutoEmbedder) AddProvider(name string, provider EmbeddingProvider)

AddProvider adds an embedding provider

func (*AutoEmbedder) GenerateBatchEmbeddings 

func (ae *AutoEmbedder) GenerateBatchEmbeddings(ctx context.Context, texts []string) ([][]float32, error)

GenerateBatchEmbeddings generates embeddings for multiple texts

func (*AutoEmbedder) GenerateEmbedding 

func (ae *AutoEmbedder) GenerateEmbedding(ctx context.Context, text string) ([]float32, error)

GenerateEmbedding generates an embedding for a single text

func (*AutoEmbedder) GetDimensions 

func (ae *AutoEmbedder) GetDimensions() int

GetDimensions returns the embedding dimensions from the primary provider

func (*AutoEmbedder) GetModel 

func (ae *AutoEmbedder) GetModel() string

GetModel returns the model name from the primary provider

func (*AutoEmbedder) Health 

func (ae *AutoEmbedder) Health(ctx context.Context) error

Health checks the health of all providers

func (*AutoEmbedder) SetPrimary 

func (ae *AutoEmbedder) SetPrimary(name string) error

SetPrimary sets the primary provider

type ChromaDBConfig 

type ChromaDBConfig struct {
	Host string `json:"host"`
	Port int    `json:"port"`

	// Authentication
	APIKey  string            `json:"api_key,omitempty"`
	Headers map[string]string `json:"headers,omitempty"`

	// Connection settings
	Timeout        time.Duration `json:"timeout"`
	MaxConnections int           `json:"max_connections"`
	EnableTLS      bool          `json:"enable_tls"`
	TLSConfig      *TLSConfig    `json:"tls_config,omitempty"`

	// ChromaDB-specific settings
	Tenant   string `json:"tenant,omitempty"`
	Database string `json:"database,omitempty"`

	// Collection settings
	DefaultCollectionName string                 `json:"default_collection_name"`
	CollectionMetadata    map[string]interface{} `json:"collection_metadata,omitempty"`

	// Embedding settings
	EmbeddingFunction string `json:"embedding_function,omitempty"`

	// Performance settings
	BatchSize         int    `json:"batch_size"`
	EnablePersistence bool   `json:"enable_persistence"`
	PersistDirectory  string `json:"persist_directory,omitempty"`
}

ChromaDB-specific configuration

type CollectionConfig 

type CollectionConfig struct {
	Dimension      int                    `json:"dimension"`
	DistanceMetric string                 `json:"distance_metric"`
	IndexType      string                 `json:"index_type"`
	IndexParams    map[string]interface{} `json:"index_params,omitempty"`
	Replicas       int                    `json:"replicas,omitempty"`
	ShardCount     int                    `json:"shard_count,omitempty"`
}

CollectionConfig configures vector collection creation

type CollectionInfo 

type CollectionInfo struct {
	Name           string                 `json:"name"`
	Dimension      int                    `json:"dimension"`
	DistanceMetric string                 `json:"distance_metric"`
	IndexType      string                 `json:"index_type"`
	VectorCount    int64                  `json:"vector_count"`
	IndexedCount   int64                  `json:"indexed_count"`
	Status         string                 `json:"status"`
	Config         map[string]interface{} `json:"config,omitempty"`
	CreatedAt      time.Time              `json:"created_at"`
	UpdatedAt      time.Time              `json:"updated_at"`
}

CollectionInfo provides information about a vector collection

type EmbeddingCache 

type EmbeddingCache interface {
	Get(ctx context.Context, key string) ([]float32, bool)
	Set(ctx context.Context, key string, embedding []float32, ttl time.Duration) error
	Delete(ctx context.Context, key string) error
	Clear(ctx context.Context) error
}

EmbeddingCache defines caching interface for embeddings

type EmbeddingData 

type EmbeddingData struct {
	ID        string                 `json:"id"`
	Embedding []float32              `json:"embedding"`
	Metadata  map[string]interface{} `json:"metadata,omitempty"`
	CreatedAt time.Time              `json:"created_at"`
	UpdatedAt time.Time              `json:"updated_at"`
}

EmbeddingData represents an embedding with metadata

type EmbeddingProvider 

type EmbeddingProvider interface {
	GenerateEmbedding(ctx context.Context, text string) ([]float32, error)
	GenerateBatchEmbeddings(ctx context.Context, texts []string) ([][]float32, error)
	GetDimensions() int
	GetModel() string
	Health(ctx context.Context) error
}

EmbeddingProvider defines the interface for generating embeddings

func NewEmbeddingProvider 

func NewEmbeddingProvider(providerType EmbeddingProviderType, config map[string]interface{}) (EmbeddingProvider, error)

NewEmbeddingProvider creates a new embedding provider using the registered factory

func NewLMStudioEmbeddingProvider 

func NewLMStudioEmbeddingProvider(endpoint, model string) EmbeddingProvider

NewLMStudioEmbeddingProvider creates an LMStudio embedding provider (legacy facade)

func NewOllamaEmbeddingProvider 

func NewOllamaEmbeddingProvider(endpoint, model string, dimensions int) EmbeddingProvider

NewOllamaEmbeddingProvider creates an Ollama embedding provider (legacy facade)

func NewOpenAIEmbeddingProvider 

func NewOpenAIEmbeddingProvider(apiKey, model string) EmbeddingProvider

NewOpenAIEmbeddingProvider creates an OpenAI embedding provider (legacy facade)

func NewOpenRouterEmbeddingProvider 

func NewOpenRouterEmbeddingProvider(config OpenRouterConfig) EmbeddingProvider

NewOpenRouterEmbeddingProvider creates an OpenRouter embedding provider (legacy facade)

type EmbeddingProviderFactory 

type EmbeddingProviderFactory func(config map[string]interface{}) (EmbeddingProvider, error)

EmbeddingProviderFactory defines the function signature for creating an EmbeddingProvider

type EmbeddingProviderType 

type EmbeddingProviderType string

EmbeddingProviderType defines supported embedding provider types 

const (
	EmbeddingProviderOpenAI     EmbeddingProviderType = "openai"
	EmbeddingProviderOllama     EmbeddingProviderType = "ollama"
	EmbeddingProviderOpenRouter EmbeddingProviderType = "openrouter"
	EmbeddingProviderLMStudio   EmbeddingProviderType = "lmstudio"
)

type InMemoryEmbeddingCache 

type InMemoryEmbeddingCache struct {
	// contains filtered or unexported fields
}

InMemoryEmbeddingCache implements EmbeddingCache using in-memory storage

func NewInMemoryEmbeddingCache 

func NewInMemoryEmbeddingCache() *InMemoryEmbeddingCache

NewInMemoryEmbeddingCache creates a new in-memory embedding cache

func (*InMemoryEmbeddingCache) Clear 

func (c *InMemoryEmbeddingCache) Clear(ctx context.Context) error

Clear removes all embeddings from cache

func (*InMemoryEmbeddingCache) Delete 

func (c *InMemoryEmbeddingCache) Delete(ctx context.Context, key string) error

Delete removes an embedding from cache

func (*InMemoryEmbeddingCache) Get 

func (c *InMemoryEmbeddingCache) Get(ctx context.Context, key string) ([]float32, bool)

Get retrieves an embedding from cache

func (*InMemoryEmbeddingCache) GetSize 

func (c *InMemoryEmbeddingCache) GetSize() int

GetSize returns the number of cached embeddings

func (*InMemoryEmbeddingCache) Set 

func (c *InMemoryEmbeddingCache) Set(ctx context.Context, key string, embedding []float32, ttl time.Duration) error

Set stores an embedding in cache

type LanceDBConfig 

type LanceDBConfig struct {
	URI         string `json:"uri"`          // File path or S3/GCS URI
	StorageType string `json:"storage_type"` // local, s3, gcs, azure

	// Local storage settings
	DataDir   string `json:"data_dir,omitempty"`
	TableName string `json:"table_name"`

	// Cloud storage settings (S3/GCS/Azure)
	Region          string `json:"region,omitempty"`
	AccessKeyID     string `json:"access_key_id,omitempty"`
	SecretAccessKey string `json:"secret_access_key,omitempty"`
	Bucket          string `json:"bucket,omitempty"`

	// LanceDB-specific settings
	BlockSize       int   `json:"block_size"`
	MaxRowsPerFile  int   `json:"max_rows_per_file"`
	MaxRowsPerGroup int   `json:"max_rows_per_group"`
	MaxBytesPerFile int64 `json:"max_bytes_per_file"`

	// Index settings
	IndexCacheSize int    `json:"index_cache_size"`
	MetricType     string `json:"metric_type"` // l2, cosine, dot

	// Performance settings
	EnableStatistics  bool `json:"enable_statistics"`
	EnableBloomFilter bool `json:"enable_bloom_filter"`
}

LanceDB-specific configuration

type OpenRouterConfig 

type OpenRouterConfig struct {
	APIKey  string `json:"api_key"`
	Model   string `json:"model,omitempty"`
	SiteURL string `json:"site_url,omitempty"`
	AppName string `json:"app_name,omitempty"`
}

OpenRouterConfig holds configuration for OpenRouter provider

type PgVectorConfig 

type PgVectorConfig struct {
	Host     string `json:"host"`
	Port     int    `json:"port"`
	Database string `json:"database"`
	Username string `json:"username"`
	Password string `json:"password"`
	SSLMode  string `json:"ssl_mode"` // disable, require, verify-ca, verify-full

	// Connection pooling
	MaxConnections    int           `json:"max_connections"`
	MinConnections    int           `json:"min_connections"`
	MaxConnLifetime   time.Duration `json:"max_conn_lifetime"`
	MaxConnIdleTime   time.Duration `json:"max_conn_idle_time"`
	HealthCheckPeriod time.Duration `json:"health_check_period"`

	// pgvector-specific settings
	VectorDimensions int                    `json:"vector_dimensions"`
	IndexType        string                 `json:"index_type"` // ivfflat, hnsw
	IndexOptions     map[string]interface{} `json:"index_options,omitempty"`

	// Table settings
	TableName      string `json:"table_name"`
	VectorColumn   string `json:"vector_column"`
	MetadataColumn string `json:"metadata_column"`

	// Performance settings
	MaintenanceWorkMem string `json:"maintenance_work_mem,omitempty"`
	MaxParallelWorkers int    `json:"max_parallel_workers,omitempty"`
	EffectiveCacheSize string `json:"effective_cache_size,omitempty"`
}

PgVector-specific configuration (PostgreSQL with pgvector extension)

type QdrantConfig 

type QdrantConfig struct {
	Host   string `json:"host"`
	Port   int    `json:"port"`
	APIKey string `json:"api_key,omitempty"`

	// Connection settings
	Timeout        time.Duration `json:"timeout"`
	MaxConnections int           `json:"max_connections"`
	EnableTLS      bool          `json:"enable_tls"`
	TLSConfig      *TLSConfig    `json:"tls_config,omitempty"`

	// Qdrant-specific settings
	Prefix           string `json:"prefix,omitempty"`
	EnableGRPC       bool   `json:"enable_grpc"`
	GRPCPort         int    `json:"grpc_port,omitempty"`
	CompressionLevel string `json:"compression_level"` // none, gzip, lz4

	// Collection settings
	DefaultVectorSize int    `json:"default_vector_size"`
	DefaultDistance   string `json:"default_distance"` // cosine, euclidean, dot
	OnDiskPayload     bool   `json:"on_disk_payload"`

	// Performance settings
	MaxSegmentSize    int64 `json:"max_segment_size"`
	MemmapThreshold   int64 `json:"memmap_threshold"`
	IndexingThreshold int64 `json:"indexing_threshold"`
}

Qdrant-specific configuration

type QueryResult 

type QueryResult struct {
	Results   []*SimilarityResult    `json:"results"`
	Total     int64                  `json:"total"`
	QueryTime time.Duration          `json:"query_time"`
	Metadata  map[string]interface{} `json:"metadata,omitempty"`
}

QueryResult represents the result of a vector query

type QueryVectorizer 

type QueryVectorizer struct {
	// contains filtered or unexported fields
}

QueryVectorizer handles query vectorization for the search pipeline This is Step 1 of the 4-step search pipeline (Input Processing)

func NewQueryVectorizer 

func NewQueryVectorizer(embedder *AutoEmbedder) *QueryVectorizer

NewQueryVectorizer creates a new query vectorizer

func (*QueryVectorizer) GetDimensions 

func (qv *QueryVectorizer) GetDimensions() int

GetDimensions returns the embedding dimensions from the underlying embedder

func (*QueryVectorizer) GetModel 

func (qv *QueryVectorizer) GetModel() string

GetModel returns the model name from the underlying embedder

func (*QueryVectorizer) Health 

func (qv *QueryVectorizer) Health(ctx context.Context) error

Health checks if the query vectorizer is operational

func (*QueryVectorizer) VectorizeBatchQueries 

func (qv *QueryVectorizer) VectorizeBatchQueries(ctx context.Context, queries []string) ([][]float32, error)

VectorizeBatchQueries vectorizes multiple queries in batch This is more efficient for processing multiple queries at once

func (*QueryVectorizer) VectorizeProcessedQuery 

func (qv *QueryVectorizer) VectorizeProcessedQuery(ctx context.Context, queryText string) (*schema.ProcessedQuery, error)

VectorizeProcessedQuery creates a ProcessedQuery with vector embedding This combines query vectorization with the ProcessedQuery structure

func (*QueryVectorizer) VectorizeQuery 

func (qv *QueryVectorizer) VectorizeQuery(ctx context.Context, queryText string) ([]float32, error)

VectorizeQuery converts a search query into a vector embedding This is the core functionality for Step 1: Input Processing & Vectorization

type RedisConfig 

type RedisConfig struct {
	Host     string `json:"host"`
	Port     int    `json:"port"`
	Password string `json:"password,omitempty"`
	Database int    `json:"database"` // Redis database number

	// Connection pooling
	PoolSize           int           `json:"pool_size"`
	MinIdleConns       int           `json:"min_idle_conns"`
	MaxConnAge         time.Duration `json:"max_conn_age"`
	PoolTimeout        time.Duration `json:"pool_timeout"`
	IdleTimeout        time.Duration `json:"idle_timeout"`
	IdleCheckFrequency time.Duration `json:"idle_check_frequency"`

	// Redis-specific
	MaxRetries      int           `json:"max_retries"`
	MinRetryBackoff time.Duration `json:"min_retry_backoff"`
	MaxRetryBackoff time.Duration `json:"max_retry_backoff"`
	DialTimeout     time.Duration `json:"dial_timeout"`
	ReadTimeout     time.Duration `json:"read_timeout"`
	WriteTimeout    time.Duration `json:"write_timeout"`

	// RedisSearch/RediSearch settings for vector search
	IndexName       string `json:"index_name"`
	VectorField     string `json:"vector_field"`
	VectorAlgorithm string `json:"vector_algorithm"` // FLAT, HNSW
	VectorType      string `json:"vector_type"`      // FLOAT32, FLOAT64

	// HNSW-specific parameters
	HNSWMaxConnections int `json:"hnsw_max_connections,omitempty"`
	HNSWEfConstruction int `json:"hnsw_ef_construction,omitempty"`
	HNSWEfRuntime      int `json:"hnsw_ef_runtime,omitempty"`

	// TLS configuration
	TLSConfig *TLSConfig `json:"tls_config,omitempty"`
}

Redis-specific configuration (for vector storage)

type SearchOptions 

type SearchOptions struct {
	Limit           int                    `json:"limit"`
	Threshold       float64                `json:"threshold"`
	Filters         map[string]interface{} `json:"filters,omitempty"`
	IncludeMetadata bool                   `json:"include_metadata"`
	IncludeVectors  bool                   `json:"include_vectors"`
	Offset          int                    `json:"offset,omitempty"`
}

SearchOptions configures similarity search behavior

func DefaultSearchOptions 

func DefaultSearchOptions() *SearchOptions

DefaultSearchOptions returns sensible defaults

type SimilarityResult 

type SimilarityResult struct {
	ID        string                 `json:"id"`
	Score     float64                `json:"score"`
	Embedding []float32              `json:"embedding,omitempty"`
	Metadata  map[string]interface{} `json:"metadata,omitempty"`
	Distance  float64                `json:"distance,omitempty"`
}

SimilarityResult represents a similarity search result

type TLSConfig 

type TLSConfig struct {
	Enabled            bool     `json:"enabled"`
	CertFile           string   `json:"cert_file,omitempty"`
	KeyFile            string   `json:"key_file,omitempty"`
	CAFile             string   `json:"ca_file,omitempty"`
	ServerName         string   `json:"server_name,omitempty"`
	InsecureSkipVerify bool     `json:"insecure_skip_verify"`
	MinVersion         string   `json:"min_version,omitempty"` // 1.0, 1.1, 1.2, 1.3
	MaxVersion         string   `json:"max_version,omitempty"`
	CipherSuites       []string `json:"cipher_suites,omitempty"`
}

TLS configuration for secure connections

type VectorConfig 

type VectorConfig struct {
	Type       VectorStoreType        `json:"type"`
	Host       string                 `json:"host,omitempty"`
	Port       int                    `json:"port,omitempty"`
	Database   string                 `json:"database,omitempty"`
	Collection string                 `json:"collection,omitempty"`
	Username   string                 `json:"username,omitempty"`
	Password   string                 `json:"password,omitempty"`
	Options    map[string]interface{} `json:"options,omitempty"`

	// Vector-specific settings
	Dimension      int    `json:"dimension"`
	DistanceMetric string `json:"distance_metric"` // cosine, euclidean, dot_product
	IndexType      string `json:"index_type"`      // hnsw, ivf, flat

	// Connection pooling
	MaxConnections int           `json:"max_connections"`
	ConnTimeout    time.Duration `json:"conn_timeout"`
	IdleTimeout    time.Duration `json:"idle_timeout"`

	// Performance settings
	BatchSize     int  `json:"batch_size"`
	EnableCaching bool `json:"enable_caching"`

	// Provider-specific configurations
	Qdrant   *QdrantConfig   `json:"qdrant,omitempty"`
	LanceDB  *LanceDBConfig  `json:"lancedb,omitempty"`
	PgVector *PgVectorConfig `json:"pgvector,omitempty"`
	ChromaDB *ChromaDBConfig `json:"chromadb,omitempty"`
	Redis    *RedisConfig    `json:"redis,omitempty"`
}

VectorConfig holds configuration for vector stores

func DefaultVectorConfig 

func DefaultVectorConfig() *VectorConfig

DefaultVectorConfig returns sensible defaults

type VectorFactory 

type VectorFactory interface {
	CreateVectorStore(config *VectorConfig) (VectorStore, error)
	ListSupportedTypes() []VectorStoreType
}

VectorFactory creates vector store instances

func NewVectorFactory 

func NewVectorFactory() VectorFactory

NewVectorFactory returns the default VectorFactory.

type VectorIndex 

type VectorIndex struct {
	Name       string                 `json:"name"`
	Type       string                 `json:"type"` // hnsw, ivf, flat, pq
	Dimension  int                    `json:"dimension"`
	Metric     string                 `json:"metric"`
	Parameters map[string]interface{} `json:"parameters,omitempty"`
	Status     string                 `json:"status"`
	CreatedAt  time.Time              `json:"created_at"`
}

VectorIndex defines indexing strategies for vector search

type VectorMetrics 

type VectorMetrics struct {
	TotalVectors     int64                      `json:"total_vectors"`
	Collections      map[string]*CollectionInfo `json:"collections"`
	StorageSize      int64                      `json:"storage_size_bytes"`
	IndexSize        int64                      `json:"index_size_bytes"`
	AverageQueryTime time.Duration              `json:"average_query_time"`
	LastUpdated      time.Time                  `json:"last_updated"`
}

VectorMetrics provides analytics about vector storage

type VectorQuery 

type VectorQuery struct {
	Vector    []float32              `json:"vector"`
	Filters   map[string]interface{} `json:"filters,omitempty"`
	Limit     int                    `json:"limit"`
	Threshold float64                `json:"threshold"`
	Offset    int                    `json:"offset,omitempty"`
	Include   []string               `json:"include,omitempty"` // metadata, vectors, distances
}

VectorQuery represents a complex vector query

type VectorStore 

type VectorStore interface {
	// Embedding operations
	StoreEmbedding(ctx context.Context, id string, embedding []float32, metadata map[string]interface{}) error
	GetEmbedding(ctx context.Context, id string) ([]float32, map[string]interface{}, error)
	UpdateEmbedding(ctx context.Context, id string, embedding []float32) error
	DeleteEmbedding(ctx context.Context, id string) error

	// Similarity search operations
	SimilaritySearch(ctx context.Context, queryEmbedding []float32, limit int, threshold float64) ([]*SimilarityResult, error)
	// SimilaritySearchWithFilter: AND trên từng cặp key→value (so khớp string, không phân biệt hoa thường).
	// Tùy chọn nâng cao: schema.VectorFilterKeyLabelsAny để lọc theo nhãn trên payload (engine Search/Think không dùng; nhãn chỉ phân loại lúc Add).
	SimilaritySearchWithFilter(ctx context.Context, queryEmbedding []float32, filters map[string]interface{}, limit int, threshold float64) ([]*SimilarityResult, error)

	// Batch operations
	StoreBatchEmbeddings(ctx context.Context, embeddings []*EmbeddingData) error
	DeleteBatchEmbeddings(ctx context.Context, ids []string) error

	// Collection/Index management
	CreateCollection(ctx context.Context, name string, dimension int, config *CollectionConfig) error
	DeleteCollection(ctx context.Context, name string) error
	ListCollections(ctx context.Context) ([]string, error)

	// Analytics and metrics
	GetCollectionInfo(ctx context.Context, name string) (*CollectionInfo, error)
	GetEmbeddingCount(ctx context.Context) (int64, error)

	// Health and lifecycle
	Health(ctx context.Context) error
	Close() error
}

VectorStore defines the interface for vector database operations

func NewInMemoryStore 

func NewInMemoryStore(dimension interface{}) VectorStore

NewInMemoryStore creates a new in-memory vector store (legacy facade)

func NewPgVectorStore 

func NewPgVectorStore(config *VectorConfig) (VectorStore, error)

NewPgVectorStore creates a new pgvector-based vector store (legacy facade)

func NewQdrantStore 

func NewQdrantStore(config *VectorConfig) (VectorStore, error)

NewQdrantStore creates a new Qdrant-based vector store (legacy facade)

func NewRedisVectorStore 

func NewRedisVectorStore(endpoint, password string, dimension int) (VectorStore, error)

NewRedisVectorStore creates a new Redis-based vector store (legacy facade)

func NewSQLiteVectorStore 

func NewSQLiteVectorStore(dbPath string, dimension int) (VectorStore, error)

NewSQLiteVectorStore creates a new SQLite-based vector store (legacy facade)

func NewVectorStore 

func NewVectorStore(config *VectorConfig) (VectorStore, error)

NewVectorStore creates a new vector store using the registered factory

type VectorStoreFactory 

type VectorStoreFactory func(config *VectorConfig) (VectorStore, error)

VectorStoreFactory defines the function signature for creating a VectorStore

type VectorStoreType 

type VectorStoreType string

VectorStoreType defines supported vector database types 

const (
	StoreTypeQdrant   VectorStoreType = "qdrant"
	StoreTypeLanceDB  VectorStoreType = "lancedb"
	StoreTypePgVector VectorStoreType = "pgvector"
	StoreTypeChromaDB VectorStoreType = "chromadb"
	StoreTypeRedis    VectorStoreType = "redis"
	StoreTypeInMemory VectorStoreType = "inmemory"
	StoreTypeSQLite   VectorStoreType = "sqlite"
)

Source Files

View all Source files

Directories

Path Synopsis
adapters
inmemory
pgvector
qdrant
redis
sqlite
embedders
lmstudio
ollama
Package ollama - Ollama native embedding provider Calls the local Ollama /api/embeddings endpoint (no OpenAI compatibility required).
 Package ollama - Ollama native embedding provider Calls the local Ollama /api/embeddings endpoint (no OpenAI compatibility required).
onnx
Package onnx - Local embedding provider for Harrier-OSS-v1-270m using ONNX Runtime.
 Package onnx - Local embedding provider for Harrier-OSS-v1-270m using ONNX Runtime.
openai
Package openai - OpenAI embedding provider implementation
 Package openai - OpenAI embedding provider implementation
openrouter
Package openrouter - OpenRouter embedding provider OpenRouter supports the OpenAI-compatible embeddings API so we reuse the OpenAI wire format and just swap the base URL + auth header.
 Package openrouter - OpenRouter embedding provider OpenRouter supports the OpenAI-compatible embeddings API so we reuse the OpenAI wire format and just swap the base URL + auth header.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.