embeddings

package
v0.66.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 14, 2026 License: Apache-2.0 Imports: 27 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	// ErrInvalidThreshold indicates the similarity threshold is out of range.
	ErrInvalidThreshold = errors.New("embeddings: similarity threshold must be between 0 and 1")

	// ErrInvalidBatchSize indicates the batch size is invalid.
	ErrInvalidBatchSize = errors.New("embeddings: batch size must be at least 1")

	// ErrInvalidCacheSize indicates the cache size is invalid.
	ErrInvalidCacheSize = errors.New("embeddings: cache size must be non-negative")

	// ErrIndexNotBuilt indicates the search index has not been built.
	ErrIndexNotBuilt = errors.New("embeddings: search index not built")

	// ErrServiceDisabled indicates the embeddings service is disabled.
	ErrServiceDisabled = errors.New("embeddings: service is disabled")

	// ErrBaseURLRequired indicates the provider base URL was not specified.
	ErrBaseURLRequired = errors.New("embeddings: provider base URL is required")

	// ErrModelRequired indicates the provider model name was not specified.
	ErrModelRequired = errors.New("embeddings: provider model name is required")

	// ErrInvalidDimension indicates the embedding dimension is invalid.
	ErrInvalidDimension = errors.New("embeddings: embedding dimension must be positive")

	// ErrProviderFailure indicates a provider request failed.
	ErrProviderFailure = errors.New("embeddings: provider request failed")

	// ErrDimensionMismatch indicates the embedding dimension does not match configuration.
	ErrDimensionMismatch = errors.New("embeddings: embedding dimension mismatch")

	// ErrInvalidResponse indicates an invalid response from the provider.
	ErrInvalidResponse = errors.New("embeddings: invalid response from provider")
)

Functions

func ContainsScript

func ContainsScript(text string, script Script) bool

ContainsScript returns true if the text contains any characters from the specified script.

func IsCrossScript

func IsCrossScript(text string) bool

IsCrossScript returns true if the text contains characters from multiple scripts. This is useful for detecting transliterated or mixed-script names.

func IsNonLatin

func IsNonLatin(text string) bool

IsNonLatin returns true if the text contains primarily non-Latin characters. This is used to determine whether to use embedding-based search (for cross-script) or Jaro-Winkler (for Latin-only queries).

Types

type Cache

type Cache interface {
	Get(ctx context.Context, text string) ([]float64, bool)
	Put(ctx context.Context, text string, embedding []float64)
}

func NewCache

func NewCache(ctx context.Context, config Config, database db.DB) (Cache, error)

type CacheConfig

type CacheConfig struct {
	// Type is the cache to be used.
	// Options: Blank (Disabled), memory, sql
	Type string `json:"type"`

	// Size is the maximum number of embeddings to cache in memory.
	// This value is ignored for sql
	Size int `json:"cacheSize"`
}

CacheConfig holds settings for the embeddings cache

type CacheStats

type CacheStats struct {
	Size     int
	Capacity int
}

Stats returns cache statistics.

type Config

type Config struct {
	// Enabled determines if embedding-based search is active.
	// When false, the service returns nil and falls back to Jaro-Winkler.
	Enabled bool `json:"enabled"`

	// Provider configuration for the embedding API.
	Provider ProviderConfig `json:"provider"`

	// Cache stores embeddings for the specified model
	Cache CacheConfig `json:"cache"`

	// CrossScriptOnly when true, embeddings are only used for non-Latin queries.
	// Latin-only queries fall back to Jaro-Winkler for better performance.
	CrossScriptOnly bool `json:"crossScriptOnly"`

	// SimilarityThreshold is the minimum cosine similarity to consider a match.
	// Range: 0.0 to 1.0
	SimilarityThreshold float64 `json:"similarityThreshold"`

	// BatchSize is the number of texts to encode in a single API call.
	// Larger batches are more efficient but use more memory.
	BatchSize int `json:"batchSize"`

	// IndexBuildTimeout is the maximum time allowed for building the index.
	IndexBuildTimeout time.Duration `json:"indexBuildTimeout"`
}

Config holds configuration for the embeddings service.

func DefaultConfig

func DefaultConfig() Config

DefaultConfig returns a Config with sensible defaults. Note: Model and Dimension must be explicitly configured when Enabled=true. No default model is provided because cross-script matching quality varies significantly between models. Users should choose based on their requirements.

func (*Config) LoadFromEnv

func (c *Config) LoadFromEnv()

LoadFromEnv applies environment variable overrides to the configuration.

func (Config) Validate

func (c Config) Validate() error

Validate checks the configuration for errors.

type MockProvider

type MockProvider struct {
	// contains filtered or unexported fields
}

MockProvider generates deterministic embeddings for testing. It produces consistent embeddings based on text hash, allowing predictable tests.

func NewMockProvider

func NewMockProvider(dimension int, opts ...MockProviderOption) *MockProvider

NewMockProvider creates a mock provider for testing.

func (*MockProvider) CallCount

func (m *MockProvider) CallCount() int

CallCount returns the number of times Embed was called.

func (*MockProvider) Close

func (m *MockProvider) Close() error

Close is a no-op for the mock provider.

func (*MockProvider) Dimension

func (m *MockProvider) Dimension() int

Dimension returns the configured embedding dimension.

func (*MockProvider) Embed

func (m *MockProvider) Embed(ctx context.Context, texts []string) ([][]float64, error)

Embed generates deterministic embeddings based on text hash.

func (*MockProvider) Name

func (m *MockProvider) Name() string

Name returns the provider name.

func (*MockProvider) Reset

func (m *MockProvider) Reset()

Reset resets the call counter.

type MockProviderOption

type MockProviderOption func(*MockProvider)

MockProviderOption configures a MockProvider.

func WithMockDelay

func WithMockDelay(delay time.Duration) MockProviderOption

WithMockDelay adds simulated latency to the mock provider.

func WithMockFailAfter

func WithMockFailAfter(n int) MockProviderOption

WithMockFailAfter causes the mock to fail after N successful calls.

type OpenRouterProvider

type OpenRouterProvider struct {
	// contains filtered or unexported fields
}

OpenRouterProvider implements Provider using OpenAI-compatible APIs. Compatible with: OpenAI, Ollama, OpenRouter, Azure OpenAI, LMStudio, etc.

func NewOpenRouterProvider

func NewOpenRouterProvider(config ProviderConfig) (*OpenRouterProvider, error)

NewOpenRouterProvider creates a new OpenAI-compatible embedding provider.

func (*OpenRouterProvider) Close

func (p *OpenRouterProvider) Close() error

func (*OpenRouterProvider) Dimension

func (p *OpenRouterProvider) Dimension() int

func (*OpenRouterProvider) Embed

func (p *OpenRouterProvider) Embed(ctx context.Context, texts []string) ([][]float64, error)

Embed generates embeddings for a batch of texts using the OpenAI-compatible API.

func (*OpenRouterProvider) Name

func (p *OpenRouterProvider) Name() string

type Provider

type Provider interface {
	// Embed generates embeddings for a batch of texts.
	// Returns L2-normalized vectors suitable for cosine similarity.
	// The returned vectors MUST be L2-normalized.
	Embed(ctx context.Context, texts []string) ([][]float64, error)

	// Dimension returns the embedding dimension for this provider.
	Dimension() int

	// Name returns the provider name for logging/telemetry.
	Name() string

	// Close releases any resources held by the provider.
	Close() error
}

Provider defines the interface for embedding generation backends. Implementations can use remote APIs (OpenAI, Ollama, OpenRouter, etc.)

type ProviderConfig

type ProviderConfig struct {
	// Name of the provider: "openai", "ollama", "openrouter", "azure"
	// All providers use OpenAI-compatible API format.
	// Default: "ollama"
	Name string `json:"name"`

	// BaseURL is the API endpoint.
	// Examples:
	//   - Ollama: "http://localhost:11434/v1"
	//   - OpenAI: "https://api.openai.com/v1"
	//   - OpenRouter: "https://openrouter.ai/api/v1"
	//   - Azure: "https://{resource}.openai.azure.com/openai/deployments/{deployment}"
	BaseURL string `json:"baseURL"`

	// APIKey for authentication. Optional for local providers (Ollama).
	// Can also be set via EMBEDDINGS_API_KEY environment variable.
	APIKey string `json:"apiKey,omitempty"`

	// Model name to use for embeddings.
	// Examples:
	//   - Ollama: "nomic-embed-text", "mxbai-embed-large"
	//   - OpenAI: "text-embedding-3-small", "text-embedding-3-large"
	//   - OpenRouter: "openai/text-embedding-3-small"
	Model string `json:"model"`

	// Dimension of the embedding vectors.
	// Must match the model's output dimension.
	// Common values: 384 (MiniLM), 768 (nomic-embed-text), 1536 (OpenAI small), 3072 (OpenAI large)
	Dimension int `json:"dimension"`

	// NormalizeVectors determines if vectors should be L2-normalized after API response.
	// Set to false if the API already returns normalized vectors (e.g., OpenAI).
	// Set to true for providers that return unnormalized vectors (e.g., some Ollama models).
	NormalizeVectors bool `json:"normalizeVectors"`

	// Timeout for API requests. Default: 30s
	Timeout time.Duration `json:"timeout"`

	// RateLimit configuration for the embedding API.
	RateLimit RateLimitConfig `json:"rateLimit"`

	// Retry configuration for failed requests.
	Retry RetryConfig `json:"retry"`

	// Headers allows adding custom HTTP headers to requests.
	// Useful for authentication or routing (e.g., OpenRouter HTTP-Referer).
	Headers map[string]string `json:"headers,omitempty"`
}

ProviderConfig holds settings for an embedding provider.

type RateLimitConfig

type RateLimitConfig struct {
	// RequestsPerSecond defines the sustained request rate.
	// Default: 10
	RequestsPerSecond float64 `json:"requestsPerSecond"`

	// Burst allows temporary exceeding of the rate limit.
	// Default: 20
	Burst int `json:"burst"`
}

RateLimitConfig controls the rate of API requests.

type RetryConfig

type RetryConfig struct {
	// MaxRetries is the maximum number of retry attempts.
	// Default: 3
	MaxRetries int `json:"maxRetries"`

	// InitialBackoff is the initial backoff duration.
	// Default: 1s
	InitialBackoff time.Duration `json:"initialBackoff"`

	// MaxBackoff is the maximum backoff duration.
	// Default: 30s
	MaxBackoff time.Duration `json:"maxBackoff"`
}

RetryConfig controls retry behavior for failed requests.

type Script

type Script string

Script represents a Unicode script/writing system.

const (
	ScriptLatin    Script = "Latin"
	ScriptArabic   Script = "Arabic"
	ScriptCyrillic Script = "Cyrillic"
	ScriptHan      Script = "Han"      // Chinese
	ScriptHangul   Script = "Hangul"   // Korean
	ScriptHiragana Script = "Hiragana" // Japanese
	ScriptKatakana Script = "Katakana" // Japanese
	ScriptThai     Script = "Thai"
	ScriptHebrew   Script = "Hebrew"
	ScriptGreek    Script = "Greek"
	ScriptUnknown  Script = "Unknown"
)

func DetectScript

func DetectScript(text string) Script

DetectScript returns the primary Unicode script of the text. If the text contains multiple scripts, returns the most common one.

type SearchResult

type SearchResult struct {
	ID    string  // Entity ID from the sanctions list
	Name  string  // Original name that was indexed
	Score float64 // Cosine similarity score (0.0 to 1.0)
}

SearchResult represents a single search result from embedding similarity search.

type Service

type Service interface {
	// Encode converts text to a normalized embedding vector.
	// The returned vector has dimensions matching the configured provider.
	Encode(ctx context.Context, text string) ([]float64, error)

	// EncodeBatch encodes multiple texts efficiently in a single API call.
	EncodeBatch(ctx context.Context, texts []string) ([][]float64, error)

	// BuildIndex creates a searchable index from entity names.
	// This must be called before Search() can be used.
	// ids and names must have the same length.
	BuildIndex(ctx context.Context, names []string, ids []string) error

	// Search finds similar names using vector similarity.
	// Returns up to k results sorted by similarity score (highest first).
	Search(ctx context.Context, query string, k int) ([]SearchResult, error)

	// Similarity computes cosine similarity between two texts.
	// Returns a value between 0.0 (dissimilar) and 1.0 (identical).
	Similarity(ctx context.Context, text1, text2 string) (float64, error)

	// ShouldUseEmbeddings determines if a query should use embeddings vs Jaro-Winkler.
	// When CrossScriptOnly is enabled, returns true only for non-Latin queries.
	ShouldUseEmbeddings(query string) bool

	// IndexSize returns the number of items in the search index.
	IndexSize() int

	// Shutdown releases resources held by the service.
	Shutdown()
}

Service provides semantic embedding-based name matching. It uses neural network embeddings to find similar names across different scripts (Arabic, Cyrillic, Chinese, etc.) and Latin text.

func NewService

func NewService(logger log.Logger, config Config, database db.DB) (Service, error)

NewService creates a new embeddings service. Returns nil if config.Enabled is false.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL