Documentation
¶
Index ¶
- Variables
- func ContainsScript(text string, script Script) bool
- func IsCrossScript(text string) bool
- func IsNonLatin(text string) bool
- type Cache
- type CacheConfig
- type CacheStats
- type Config
- type MockProvider
- type MockProviderOption
- type OpenRouterProvider
- type Provider
- type ProviderConfig
- type RateLimitConfig
- type RetryConfig
- type Script
- type SearchResult
- type Service
Constants ¶
This section is empty.
Variables ¶
var ( // ErrInvalidThreshold indicates the similarity threshold is out of range. ErrInvalidThreshold = errors.New("embeddings: similarity threshold must be between 0 and 1") // ErrInvalidBatchSize indicates the batch size is invalid. ErrInvalidBatchSize = errors.New("embeddings: batch size must be at least 1") // ErrInvalidCacheSize indicates the cache size is invalid. ErrInvalidCacheSize = errors.New("embeddings: cache size must be non-negative") // ErrIndexNotBuilt indicates the search index has not been built. ErrIndexNotBuilt = errors.New("embeddings: search index not built") // ErrServiceDisabled indicates the embeddings service is disabled. ErrServiceDisabled = errors.New("embeddings: service is disabled") // ErrBaseURLRequired indicates the provider base URL was not specified. ErrBaseURLRequired = errors.New("embeddings: provider base URL is required") // ErrModelRequired indicates the provider model name was not specified. ErrModelRequired = errors.New("embeddings: provider model name is required") // ErrInvalidDimension indicates the embedding dimension is invalid. ErrInvalidDimension = errors.New("embeddings: embedding dimension must be positive") // ErrProviderFailure indicates a provider request failed. ErrProviderFailure = errors.New("embeddings: provider request failed") // ErrDimensionMismatch indicates the embedding dimension does not match configuration. ErrDimensionMismatch = errors.New("embeddings: embedding dimension mismatch") // ErrInvalidResponse indicates an invalid response from the provider. ErrInvalidResponse = errors.New("embeddings: invalid response from provider") )
Functions ¶
func ContainsScript ¶
ContainsScript returns true if the text contains any characters from the specified script.
func IsCrossScript ¶
IsCrossScript returns true if the text contains characters from multiple scripts. This is useful for detecting transliterated or mixed-script names.
func IsNonLatin ¶
IsNonLatin returns true if the text contains primarily non-Latin characters. This is used to determine whether to use embedding-based search (for cross-script) or Jaro-Winkler (for Latin-only queries).
Types ¶
type Cache ¶
type CacheConfig ¶
type CacheConfig struct {
// Type is the cache to be used.
// Options: Blank (Disabled), memory, sql
Type string `json:"type"`
// Size is the maximum number of embeddings to cache in memory.
// This value is ignored for sql
Size int `json:"cacheSize"`
}
CacheConfig holds settings for the embeddings cache
type Config ¶
type Config struct {
// Enabled determines if embedding-based search is active.
// When false, the service returns nil and falls back to Jaro-Winkler.
Enabled bool `json:"enabled"`
// Provider configuration for the embedding API.
Provider ProviderConfig `json:"provider"`
// Cache stores embeddings for the specified model
Cache CacheConfig `json:"cache"`
// CrossScriptOnly when true, embeddings are only used for non-Latin queries.
// Latin-only queries fall back to Jaro-Winkler for better performance.
CrossScriptOnly bool `json:"crossScriptOnly"`
// SimilarityThreshold is the minimum cosine similarity to consider a match.
// Range: 0.0 to 1.0
SimilarityThreshold float64 `json:"similarityThreshold"`
// BatchSize is the number of texts to encode in a single API call.
// Larger batches are more efficient but use more memory.
BatchSize int `json:"batchSize"`
// IndexBuildTimeout is the maximum time allowed for building the index.
IndexBuildTimeout time.Duration `json:"indexBuildTimeout"`
}
Config holds configuration for the embeddings service.
func DefaultConfig ¶
func DefaultConfig() Config
DefaultConfig returns a Config with sensible defaults. Note: Model and Dimension must be explicitly configured when Enabled=true. No default model is provided because cross-script matching quality varies significantly between models. Users should choose based on their requirements.
func (*Config) LoadFromEnv ¶
func (c *Config) LoadFromEnv()
LoadFromEnv applies environment variable overrides to the configuration.
type MockProvider ¶
type MockProvider struct {
// contains filtered or unexported fields
}
MockProvider generates deterministic embeddings for testing. It produces consistent embeddings based on text hash, allowing predictable tests.
func NewMockProvider ¶
func NewMockProvider(dimension int, opts ...MockProviderOption) *MockProvider
NewMockProvider creates a mock provider for testing.
func (*MockProvider) CallCount ¶
func (m *MockProvider) CallCount() int
CallCount returns the number of times Embed was called.
func (*MockProvider) Close ¶
func (m *MockProvider) Close() error
Close is a no-op for the mock provider.
func (*MockProvider) Dimension ¶
func (m *MockProvider) Dimension() int
Dimension returns the configured embedding dimension.
type MockProviderOption ¶
type MockProviderOption func(*MockProvider)
MockProviderOption configures a MockProvider.
func WithMockDelay ¶
func WithMockDelay(delay time.Duration) MockProviderOption
WithMockDelay adds simulated latency to the mock provider.
func WithMockFailAfter ¶
func WithMockFailAfter(n int) MockProviderOption
WithMockFailAfter causes the mock to fail after N successful calls.
type OpenRouterProvider ¶
type OpenRouterProvider struct {
// contains filtered or unexported fields
}
OpenRouterProvider implements Provider using OpenAI-compatible APIs. Compatible with: OpenAI, Ollama, OpenRouter, Azure OpenAI, LMStudio, etc.
func NewOpenRouterProvider ¶
func NewOpenRouterProvider(config ProviderConfig) (*OpenRouterProvider, error)
NewOpenRouterProvider creates a new OpenAI-compatible embedding provider.
func (*OpenRouterProvider) Close ¶
func (p *OpenRouterProvider) Close() error
func (*OpenRouterProvider) Dimension ¶
func (p *OpenRouterProvider) Dimension() int
func (*OpenRouterProvider) Embed ¶
Embed generates embeddings for a batch of texts using the OpenAI-compatible API.
func (*OpenRouterProvider) Name ¶
func (p *OpenRouterProvider) Name() string
type Provider ¶
type Provider interface {
// Embed generates embeddings for a batch of texts.
// Returns L2-normalized vectors suitable for cosine similarity.
// The returned vectors MUST be L2-normalized.
Embed(ctx context.Context, texts []string) ([][]float64, error)
// Dimension returns the embedding dimension for this provider.
Dimension() int
// Name returns the provider name for logging/telemetry.
Name() string
// Close releases any resources held by the provider.
Close() error
}
Provider defines the interface for embedding generation backends. Implementations can use remote APIs (OpenAI, Ollama, OpenRouter, etc.)
type ProviderConfig ¶
type ProviderConfig struct {
// Name of the provider: "openai", "ollama", "openrouter", "azure"
// All providers use OpenAI-compatible API format.
// Default: "ollama"
Name string `json:"name"`
// BaseURL is the API endpoint.
// Examples:
// - Ollama: "http://localhost:11434/v1"
// - OpenAI: "https://api.openai.com/v1"
// - OpenRouter: "https://openrouter.ai/api/v1"
// - Azure: "https://{resource}.openai.azure.com/openai/deployments/{deployment}"
BaseURL string `json:"baseURL"`
// APIKey for authentication. Optional for local providers (Ollama).
// Can also be set via EMBEDDINGS_API_KEY environment variable.
APIKey string `json:"apiKey,omitempty"`
// Model name to use for embeddings.
// Examples:
// - Ollama: "nomic-embed-text", "mxbai-embed-large"
// - OpenAI: "text-embedding-3-small", "text-embedding-3-large"
// - OpenRouter: "openai/text-embedding-3-small"
Model string `json:"model"`
// Dimension of the embedding vectors.
// Must match the model's output dimension.
// Common values: 384 (MiniLM), 768 (nomic-embed-text), 1536 (OpenAI small), 3072 (OpenAI large)
Dimension int `json:"dimension"`
// NormalizeVectors determines if vectors should be L2-normalized after API response.
// Set to false if the API already returns normalized vectors (e.g., OpenAI).
// Set to true for providers that return unnormalized vectors (e.g., some Ollama models).
NormalizeVectors bool `json:"normalizeVectors"`
// Timeout for API requests. Default: 30s
Timeout time.Duration `json:"timeout"`
// RateLimit configuration for the embedding API.
RateLimit RateLimitConfig `json:"rateLimit"`
// Retry configuration for failed requests.
Retry RetryConfig `json:"retry"`
// Headers allows adding custom HTTP headers to requests.
// Useful for authentication or routing (e.g., OpenRouter HTTP-Referer).
Headers map[string]string `json:"headers,omitempty"`
}
ProviderConfig holds settings for an embedding provider.
type RateLimitConfig ¶
type RateLimitConfig struct {
// RequestsPerSecond defines the sustained request rate.
// Default: 10
RequestsPerSecond float64 `json:"requestsPerSecond"`
// Burst allows temporary exceeding of the rate limit.
// Default: 20
Burst int `json:"burst"`
}
RateLimitConfig controls the rate of API requests.
type RetryConfig ¶
type RetryConfig struct {
// MaxRetries is the maximum number of retry attempts.
// Default: 3
MaxRetries int `json:"maxRetries"`
// InitialBackoff is the initial backoff duration.
// Default: 1s
InitialBackoff time.Duration `json:"initialBackoff"`
// MaxBackoff is the maximum backoff duration.
// Default: 30s
MaxBackoff time.Duration `json:"maxBackoff"`
}
RetryConfig controls retry behavior for failed requests.
type Script ¶
type Script string
Script represents a Unicode script/writing system.
const ( ScriptLatin Script = "Latin" ScriptArabic Script = "Arabic" ScriptCyrillic Script = "Cyrillic" ScriptHan Script = "Han" // Chinese ScriptHangul Script = "Hangul" // Korean ScriptHiragana Script = "Hiragana" // Japanese ScriptKatakana Script = "Katakana" // Japanese ScriptThai Script = "Thai" ScriptHebrew Script = "Hebrew" ScriptGreek Script = "Greek" ScriptUnknown Script = "Unknown" )
func DetectScript ¶
DetectScript returns the primary Unicode script of the text. If the text contains multiple scripts, returns the most common one.
type SearchResult ¶
type SearchResult struct {
ID string // Entity ID from the sanctions list
Name string // Original name that was indexed
Score float64 // Cosine similarity score (0.0 to 1.0)
}
SearchResult represents a single search result from embedding similarity search.
type Service ¶
type Service interface {
// Encode converts text to a normalized embedding vector.
// The returned vector has dimensions matching the configured provider.
Encode(ctx context.Context, text string) ([]float64, error)
// EncodeBatch encodes multiple texts efficiently in a single API call.
EncodeBatch(ctx context.Context, texts []string) ([][]float64, error)
// BuildIndex creates a searchable index from entity names.
// This must be called before Search() can be used.
// ids and names must have the same length.
BuildIndex(ctx context.Context, names []string, ids []string) error
// Search finds similar names using vector similarity.
// Returns up to k results sorted by similarity score (highest first).
Search(ctx context.Context, query string, k int) ([]SearchResult, error)
// Similarity computes cosine similarity between two texts.
// Returns a value between 0.0 (dissimilar) and 1.0 (identical).
Similarity(ctx context.Context, text1, text2 string) (float64, error)
// ShouldUseEmbeddings determines if a query should use embeddings vs Jaro-Winkler.
// When CrossScriptOnly is enabled, returns true only for non-Latin queries.
ShouldUseEmbeddings(query string) bool
// IndexSize returns the number of items in the search index.
IndexSize() int
// Shutdown releases resources held by the service.
Shutdown()
}
Service provides semantic embedding-based name matching. It uses neural network embeddings to find similar names across different scripts (Arabic, Cyrillic, Chinese, etc.) and Latin text.