Documentation
¶
Overview ¶
Package vectorstore provides vector storage implementations.
Package vectorstore provides vector storage implementations.
Package vectorstore provides vector storage implementations.
Package vectorstore provides vector storage implementations.
Package vectorstore defines the interface for vector storage operations.
Package vectorstore provides vector storage implementations.
Package vectorstore provides vector storage implementations.
Package vectorstore provides vector storage implementations.
Package vectorstore provides vector storage implementations.
Index ¶
- Variables
- func ApplyTenantFilters(userFilters, tenantFilters map[string]interface{}) (map[string]interface{}, error)
- func ContextWithTenant(ctx context.Context, tenant *TenantInfo) context.Context
- func HasTenant(ctx context.Context) bool
- func IsTransientError(err error) bool
- func MergeFilters(base, override map[string]interface{}) map[string]interface{}deprecated
- func ValidateCollectionName(name string) error
- func ValidateFilterHasTenant(filters map[string]interface{}) error
- type ChromemConfig
- type ChromemStore
- func (s *ChromemStore) AddDocuments(ctx context.Context, docs []Document) ([]string, error)
- func (s *ChromemStore) Close() error
- func (s *ChromemStore) CollectionExists(ctx context.Context, collectionName string) (bool, error)
- func (s *ChromemStore) CreateCollection(ctx context.Context, collectionName string, vectorSize int) error
- func (s *ChromemStore) DeleteCollection(ctx context.Context, collectionName string) error
- func (s *ChromemStore) DeleteDocuments(ctx context.Context, ids []string) error
- func (s *ChromemStore) DeleteDocumentsFromCollection(ctx context.Context, collectionName string, ids []string) error
- func (s *ChromemStore) ExactSearch(ctx context.Context, collectionName string, query string, k int) ([]SearchResult, error)
- func (s *ChromemStore) GetCollectionInfo(ctx context.Context, collectionName string) (*CollectionInfo, error)
- func (s *ChromemStore) IsolationMode() IsolationMode
- func (s *ChromemStore) ListCollections(ctx context.Context) ([]string, error)
- func (s *ChromemStore) Search(ctx context.Context, query string, k int) ([]SearchResult, error)
- func (s *ChromemStore) SearchInCollection(ctx context.Context, collectionName string, query string, k int, ...) ([]SearchResult, error)
- func (s *ChromemStore) SearchWithFilters(ctx context.Context, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
- func (s *ChromemStore) SetIsolationMode(mode IsolationMode)
- type ChromemStoreProvider
- func (p *ChromemStoreProvider) Close() error
- func (p *ChromemStoreProvider) GetOrgStore(ctx context.Context, tenant string) (Store, error)
- func (p *ChromemStoreProvider) GetProjectStore(ctx context.Context, tenant, team, project string) (Store, error)
- func (p *ChromemStoreProvider) GetTeamStore(ctx context.Context, tenant, team string) (Store, error)
- func (p *ChromemStoreProvider) Registry() *registry.Registry
- type CollectionInfo
- type CompressionLevel
- type CompressionMetadata
- type Document
- type Embedder
- type FilesystemIsolation
- func (f *FilesystemIsolation) InjectFilter(ctx context.Context, filters map[string]interface{}) (map[string]interface{}, error)
- func (f *FilesystemIsolation) InjectMetadata(ctx context.Context, docs []Document) error
- func (f *FilesystemIsolation) Mode() string
- func (f *FilesystemIsolation) ValidateTenant(ctx context.Context) error
- type FilterBuilder
- type IsolationMode
- type MetadataBuilder
- type NoIsolation
- func (n *NoIsolation) InjectFilter(ctx context.Context, filters map[string]interface{}) (map[string]interface{}, error)
- func (n *NoIsolation) InjectMetadata(ctx context.Context, docs []Document) error
- func (n *NoIsolation) Mode() string
- func (n *NoIsolation) ValidateTenant(ctx context.Context) error
- type PayloadIsolation
- func (p *PayloadIsolation) InjectFilter(ctx context.Context, filters map[string]interface{}) (map[string]interface{}, error)
- func (p *PayloadIsolation) InjectMetadata(ctx context.Context, docs []Document) error
- func (p *PayloadIsolation) Mode() string
- func (p *PayloadIsolation) ValidateTenant(ctx context.Context) error
- type ProviderConfig
- type QdrantConfig
- type QdrantStore
- func (s *QdrantStore) AddDocuments(ctx context.Context, docs []Document) ([]string, error)
- func (s *QdrantStore) Close() error
- func (s *QdrantStore) CollectionExists(ctx context.Context, collectionName string) (bool, error)
- func (s *QdrantStore) CreateCollection(ctx context.Context, collectionName string, vectorSize int) error
- func (s *QdrantStore) DeleteCollection(ctx context.Context, collectionName string) error
- func (s *QdrantStore) DeleteDocuments(ctx context.Context, ids []string) error
- func (s *QdrantStore) DeleteDocumentsFromCollection(ctx context.Context, collectionName string, ids []string) error
- func (s *QdrantStore) ExactSearch(ctx context.Context, collectionName string, query string, k int) ([]SearchResult, error)
- func (s *QdrantStore) GetCollectionInfo(ctx context.Context, collectionName string) (*CollectionInfo, error)
- func (s *QdrantStore) IsolationMode() IsolationMode
- func (s *QdrantStore) ListCollections(ctx context.Context) ([]string, error)
- func (s *QdrantStore) Search(ctx context.Context, query string, k int) ([]SearchResult, error)
- func (s *QdrantStore) SearchInCollection(ctx context.Context, collectionName string, query string, k int, ...) ([]SearchResult, error)
- func (s *QdrantStore) SearchWithFilters(ctx context.Context, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
- func (s *QdrantStore) SetIsolationMode(mode IsolationMode)
- type SearchResult
- type Store
- type StoreOption
- type StoreProvider
- type TenantInfo
- type TroubleshootAdapter
Constants ¶
This section is empty.
Variables ¶
var ( // ErrCollectionNotFound is returned when a collection does not exist. ErrCollectionNotFound = errors.New("collection not found") // ErrCollectionExists is returned when attempting to create an existing collection. ErrCollectionExists = errors.New("collection already exists") // ErrInvalidConfig indicates invalid configuration. ErrInvalidConfig = errors.New("invalid configuration") // ErrEmptyDocuments indicates empty or nil documents. ErrEmptyDocuments = errors.New("empty or nil documents") // ErrConnectionFailed indicates gRPC connection issues. ErrConnectionFailed = errors.New("failed to connect to Qdrant") // ErrEmbeddingFailed indicates embedding generation failure. ErrEmbeddingFailed = errors.New("failed to generate embeddings") // ErrInvalidCollectionName indicates collection name validation failure. ErrInvalidCollectionName = errors.New("invalid collection name") )
Sentinel errors for vector store operations.
var ( // ErrMissingTenant is returned when tenant info is missing from context. // This triggers "fail closed" behavior - no empty results, just errors. ErrMissingTenant = errors.New("tenant info missing from context") // ErrInvalidTenant is returned when tenant identifier is invalid. ErrInvalidTenant = errors.New("invalid tenant identifier") )
Tenant isolation error types - fail closed security model.
var ErrTenantFilterInUserFilters = errors.New("user filters cannot contain tenant fields")
ErrTenantFilterInUserFilters indicates user tried to inject tenant fields.
var OptionalTenantFields = []string{"team_id", "project_id"}
OptionalTenantFields are additional scope fields that may be present.
var RequiredTenantFields = []string{"tenant_id"}
RequiredTenantFields are the fields that must be present for tenant isolation.
Functions ¶
func ApplyTenantFilters ¶ added in v0.3.0
func ApplyTenantFilters(userFilters, tenantFilters map[string]interface{}) (map[string]interface{}, error)
ApplyTenantFilters merges user filters with tenant filters, enforcing security.
This function ensures tenant filters (from isolation layer) always win and rejects attempts to inject tenant fields via user filters.
Parameters:
- userFilters: User-provided query filters (may be nil)
- tenantFilters: Tenant filters from isolation layer (may be nil)
Returns error if userFilters contains tenant_id, team_id, or project_id.
func ContextWithTenant ¶ added in v0.3.0
func ContextWithTenant(ctx context.Context, tenant *TenantInfo) context.Context
ContextWithTenant adds TenantInfo to a context.
func HasTenant ¶ added in v0.3.0
HasTenant checks if TenantInfo is present in context without error.
func IsTransientError ¶
IsTransientError checks if an error is transient (should retry). Returns true for network timeouts, temporary unavailability. Returns false for invalid config, not found, permission denied.
func MergeFilters
deprecated
added in
v0.3.0
MergeFilters combines two filter maps, with override taking precedence.
Deprecated: Use ApplyTenantFilters for tenant-aware merging with security validation. This function is kept for backward compatibility but does not enforce tenant security.
func ValidateCollectionName ¶
ValidateCollectionName validates a collection name against security rules. Pattern: ^[a-z0-9_]{1,64}$ Rejects: uppercase, special chars, path traversal, spaces.
func ValidateFilterHasTenant ¶ added in v0.3.0
ValidateFilterHasTenant checks that a filter map contains required tenant fields. Returns ErrMissingTenant if tenant_id is not present or not a string. Returns ErrInvalidTenant if tenant_id is empty.
Types ¶
type ChromemConfig ¶ added in v0.3.0
type ChromemConfig struct {
// Path is the directory for persistent storage.
// Default: "~/.config/contextd/vectorstore"
Path string
// Compress enables gzip compression for stored data.
// Note: This defaults to false (Go zero value). Set explicitly if compression is desired.
Compress bool
// DefaultCollection is the default collection name.
// Default: "contextd_default"
DefaultCollection string
// VectorSize is the expected embedding dimension.
// Must match the embedder's output dimension.
// Default: 384 (for FastEmbed bge-small-en-v1.5)
VectorSize int
// Isolation is the tenant isolation mode.
// Default: PayloadIsolation for fail-closed security.
// Set at construction time; immutable afterward to prevent race conditions.
Isolation IsolationMode
}
ChromemConfig holds configuration for chromem-go embedded vector database.
func (*ChromemConfig) ApplyDefaults ¶ added in v0.3.0
func (c *ChromemConfig) ApplyDefaults()
ApplyDefaults sets default values for unset fields.
func (*ChromemConfig) Validate ¶ added in v0.3.0
func (c *ChromemConfig) Validate() error
Validate validates the configuration.
type ChromemStore ¶ added in v0.3.0
type ChromemStore struct {
// contains filtered or unexported fields
}
ChromemStore implements the Store interface using chromem-go.
chromem-go is an embeddable vector database with zero third-party dependencies. It provides in-memory storage with optional persistence to gob files.
Key features:
- Pure Go, no CGO required
- No external database service needed
- Fast similarity search (1000 docs in 0.3ms)
- Automatic persistence to disk
- Tenant isolation via payload filtering or filesystem isolation
func NewChromemStore ¶ added in v0.3.0
func NewChromemStore(config ChromemConfig, embedder Embedder, logger *zap.Logger) (*ChromemStore, error)
NewChromemStore creates a new ChromemStore with the given configuration.
func (*ChromemStore) AddDocuments ¶ added in v0.3.0
AddDocuments adds documents to the vector store. If isolation mode is set, tenant metadata is automatically injected.
func (*ChromemStore) Close ¶ added in v0.3.0
func (s *ChromemStore) Close() error
Close closes the ChromemStore. Note: chromem-go handles persistence automatically, no explicit close needed.
func (*ChromemStore) CollectionExists ¶ added in v0.3.0
CollectionExists checks if a collection exists.
func (*ChromemStore) CreateCollection ¶ added in v0.3.0
func (s *ChromemStore) CreateCollection(ctx context.Context, collectionName string, vectorSize int) error
CreateCollection creates a new collection with the specified configuration.
func (*ChromemStore) DeleteCollection ¶ added in v0.3.0
func (s *ChromemStore) DeleteCollection(ctx context.Context, collectionName string) error
DeleteCollection deletes a collection and all its documents.
func (*ChromemStore) DeleteDocuments ¶ added in v0.3.0
func (s *ChromemStore) DeleteDocuments(ctx context.Context, ids []string) error
DeleteDocuments deletes documents by their IDs from the default collection.
func (*ChromemStore) DeleteDocumentsFromCollection ¶ added in v0.3.0
func (s *ChromemStore) DeleteDocumentsFromCollection(ctx context.Context, collectionName string, ids []string) error
DeleteDocumentsFromCollection deletes documents by their IDs from a specific collection.
func (*ChromemStore) ExactSearch ¶ added in v0.3.0
func (s *ChromemStore) ExactSearch(ctx context.Context, collectionName string, query string, k int) ([]SearchResult, error)
ExactSearch performs brute-force similarity search. Note: chromem-go always uses exact search (no HNSW), so this is the same as regular search.
func (*ChromemStore) GetCollectionInfo ¶ added in v0.3.0
func (s *ChromemStore) GetCollectionInfo(ctx context.Context, collectionName string) (*CollectionInfo, error)
GetCollectionInfo returns metadata about a collection.
func (*ChromemStore) IsolationMode ¶ added in v0.3.0
func (s *ChromemStore) IsolationMode() IsolationMode
IsolationMode returns the current isolation mode.
func (*ChromemStore) ListCollections ¶ added in v0.3.0
func (s *ChromemStore) ListCollections(ctx context.Context) ([]string, error)
ListCollections returns a list of all collection names.
func (*ChromemStore) Search ¶ added in v0.3.0
func (s *ChromemStore) Search(ctx context.Context, query string, k int) ([]SearchResult, error)
Search performs similarity search in the default collection.
func (*ChromemStore) SearchInCollection ¶ added in v0.3.0
func (s *ChromemStore) SearchInCollection(ctx context.Context, collectionName string, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
SearchInCollection performs similarity search in a specific collection. If isolation mode is set, tenant filters are automatically injected.
func (*ChromemStore) SearchWithFilters ¶ added in v0.3.0
func (s *ChromemStore) SearchWithFilters(ctx context.Context, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
SearchWithFilters performs similarity search with metadata filters.
func (*ChromemStore) SetIsolationMode ¶ added in v0.3.0
func (s *ChromemStore) SetIsolationMode(mode IsolationMode)
SetIsolationMode sets the tenant isolation mode for this store. Use NewPayloadIsolation() for multi-tenant payload filtering, or NewNoIsolation() for testing (default).
type ChromemStoreProvider ¶ added in v0.3.0
type ChromemStoreProvider struct {
// contains filtered or unexported fields
}
ChromemStoreProvider implements StoreProvider using chromem-go.
func NewChromemStoreProvider ¶ added in v0.3.0
func NewChromemStoreProvider(config ProviderConfig, embedder Embedder, logger *zap.Logger) (*ChromemStoreProvider, error)
NewChromemStoreProvider creates a new StoreProvider backed by chromem-go.
SECURITY: This provider has NO authorization checks. It is suitable for:
- Local development (single-user, localhost)
- CLI tools (trusted environment)
- Testing environments
For production multi-tenant use, wrap with AuthorizedStoreProvider.
func (*ChromemStoreProvider) Close ¶ added in v0.3.0
func (p *ChromemStoreProvider) Close() error
Close closes all managed stores.
func (*ChromemStoreProvider) GetOrgStore ¶ added in v0.3.0
GetOrgStore returns a store scoped to an org (for org-level shared collections).
func (*ChromemStoreProvider) GetProjectStore ¶ added in v0.3.0
func (p *ChromemStoreProvider) GetProjectStore(ctx context.Context, tenant, team, project string) (Store, error)
GetProjectStore returns a store scoped to a specific project.
func (*ChromemStoreProvider) GetTeamStore ¶ added in v0.3.0
func (p *ChromemStoreProvider) GetTeamStore(ctx context.Context, tenant, team string) (Store, error)
GetTeamStore returns a store scoped to a team (for shared collections).
func (*ChromemStoreProvider) Registry ¶ added in v0.3.0
func (p *ChromemStoreProvider) Registry() *registry.Registry
Registry returns the underlying registry for direct access if needed.
type CollectionInfo ¶
type CollectionInfo struct {
// Name is the collection name.
Name string `json:"name"`
// PointCount is the number of vectors in the collection.
PointCount int `json:"point_count"`
// VectorSize is the dimensionality of vectors in this collection.
VectorSize int `json:"vector_size"`
}
CollectionInfo contains metadata about a vector collection.
type CompressionLevel ¶
type CompressionLevel string
CompressionLevel represents the compression state of content.
const ( // CompressionLevelNone represents uncompressed original content. CompressionLevelNone CompressionLevel = "none" // CompressionLevelFolded represents context-folded content (partial compression). CompressionLevelFolded CompressionLevel = "folded" // CompressionLevelSummary represents summarized content (high compression). CompressionLevelSummary CompressionLevel = "summary" )
type CompressionMetadata ¶
type CompressionMetadata struct {
Level CompressionLevel `json:"compression_level"` // Current compression level
Algorithm string `json:"compression_algorithm,omitempty"` // Compression algorithm used
OriginalSize int `json:"original_size"` // Original content size (tokens/chars)
CompressedSize int `json:"compressed_size"` // Compressed content size
CompressionRatio float64 `json:"compression_ratio"` // Compression ratio (original/compressed)
CompressedAt *time.Time `json:"compressed_at,omitempty"` // When compression was applied
}
CompressionMetadata tracks compression state and metrics.
type Document ¶
type Document struct {
// ID is the unique identifier for the document
ID string
// Content is the text content of the document
Content string
// Metadata contains additional key-value pairs for filtering
// Common fields: owner, project, file, branch, timestamp
Metadata map[string]interface{}
// Collection is the target collection name for this document.
// If empty, uses the service's default collection.
//
// Collection naming convention:
// - Organization: org_{type} (e.g., org_memories)
// - Team: {team}_{type} (e.g., platform_memories)
// - Project: {team}_{project}_{type} (e.g., platform_contextd_memories)
Collection string
}
Document represents a document to be stored in the vector store.
type Embedder ¶
type Embedder interface {
// EmbedDocuments generates embeddings for multiple texts.
// Returns a slice of embeddings (one per input text) or an error.
EmbedDocuments(ctx context.Context, texts []string) ([][]float32, error)
// EmbedQuery generates an embedding for a single query.
// Some models optimize differently for queries vs documents.
EmbedQuery(ctx context.Context, text string) ([]float32, error)
}
Embedder generates vector embeddings from text.
Embeddings are dense numerical representations that capture semantic meaning, enabling similarity search. Implementations can use local models (TEI) or cloud APIs (OpenAI, Cohere).
type FilesystemIsolation ¶ added in v0.3.0
type FilesystemIsolation struct{}
FilesystemIsolation implements IsolationMode using separate stores.
In this mode:
- Each tenant/team/project gets its own database directory
- Physical filesystem isolation provides security boundary
- No metadata filtering needed (isolation is structural)
Note: This is the legacy mode used by ChromemStoreProvider. Consider migration to PayloadIsolation for simpler operations.
func NewFilesystemIsolation ¶ added in v0.3.0
func NewFilesystemIsolation() *FilesystemIsolation
NewFilesystemIsolation creates a new FilesystemIsolation mode.
func (*FilesystemIsolation) InjectFilter ¶ added in v0.3.0
func (f *FilesystemIsolation) InjectFilter(ctx context.Context, filters map[string]interface{}) (map[string]interface{}, error)
InjectFilter is a no-op for filesystem isolation (isolation is structural).
func (*FilesystemIsolation) InjectMetadata ¶ added in v0.3.0
func (f *FilesystemIsolation) InjectMetadata(ctx context.Context, docs []Document) error
InjectMetadata is optional for filesystem isolation but adds tenant info for auditability.
func (*FilesystemIsolation) Mode ¶ added in v0.3.0
func (f *FilesystemIsolation) Mode() string
Mode returns "filesystem" for this isolation mode.
func (*FilesystemIsolation) ValidateTenant ¶ added in v0.3.0
func (f *FilesystemIsolation) ValidateTenant(ctx context.Context) error
ValidateTenant checks tenant context is present and valid.
type FilterBuilder ¶ added in v0.3.0
type FilterBuilder struct {
// contains filtered or unexported fields
}
FilterBuilder provides a fluent interface for building query filters.
func NewFilterBuilder ¶ added in v0.3.0
func NewFilterBuilder() *FilterBuilder
NewFilterBuilder creates a new FilterBuilder.
func (*FilterBuilder) Build ¶ added in v0.3.0
func (b *FilterBuilder) Build() map[string]interface{}
Build returns the constructed filter map.
func (*FilterBuilder) With ¶ added in v0.3.0
func (b *FilterBuilder) With(key string, value interface{}) *FilterBuilder
With adds a key-value pair to the filter.
func (*FilterBuilder) WithMap ¶ added in v0.3.0
func (b *FilterBuilder) WithMap(m map[string]interface{}) *FilterBuilder
WithMap merges an existing filter map.
func (*FilterBuilder) WithTenant ¶ added in v0.3.0
func (b *FilterBuilder) WithTenant(tenant *TenantInfo) *FilterBuilder
WithTenant adds tenant filters from TenantInfo.
type IsolationMode ¶ added in v0.3.0
type IsolationMode interface {
// InjectFilter adds tenant filtering to search options.
// Must fail with ErrMissingTenant if tenant context is absent.
InjectFilter(ctx context.Context, filters map[string]interface{}) (map[string]interface{}, error)
// InjectMetadata adds tenant metadata to documents before storage.
// Must fail with ErrMissingTenant if tenant context is absent.
InjectMetadata(ctx context.Context, docs []Document) error
// ValidateTenant checks that tenant context is present and valid.
// Returns nil if valid, ErrMissingTenant or ErrInvalidTenant otherwise.
ValidateTenant(ctx context.Context) error
// Mode returns the isolation mode name for logging/debugging.
Mode() string
}
IsolationMode defines how tenant isolation is enforced in vector stores.
Implementations determine whether isolation is via:
- Payload filtering (single collection, metadata-based filtering)
- Filesystem isolation (separate databases per tenant/project)
- Hybrid approaches (tiered isolation)
Security: All implementations must enforce fail-closed behavior.
func IsolationModeFromString ¶ added in v0.3.0
func IsolationModeFromString(mode string) (IsolationMode, error)
IsolationModeFromString creates an IsolationMode from a string name.
type MetadataBuilder ¶ added in v0.3.0
type MetadataBuilder struct {
// contains filtered or unexported fields
}
MetadataBuilder provides a fluent interface for building document metadata.
func NewMetadataBuilder ¶ added in v0.3.0
func NewMetadataBuilder() *MetadataBuilder
NewMetadataBuilder creates a new MetadataBuilder.
func (*MetadataBuilder) Build ¶ added in v0.3.0
func (b *MetadataBuilder) Build() map[string]interface{}
Build returns the constructed metadata map.
func (*MetadataBuilder) With ¶ added in v0.3.0
func (b *MetadataBuilder) With(key string, value interface{}) *MetadataBuilder
With adds a key-value pair to the metadata.
func (*MetadataBuilder) WithMap ¶ added in v0.3.0
func (b *MetadataBuilder) WithMap(m map[string]interface{}) *MetadataBuilder
WithMap merges an existing metadata map.
func (*MetadataBuilder) WithTenant ¶ added in v0.3.0
func (b *MetadataBuilder) WithTenant(tenant *TenantInfo) *MetadataBuilder
WithTenant adds tenant metadata from TenantInfo.
type NoIsolation ¶ added in v0.3.0
type NoIsolation struct{}
NoIsolation provides no tenant isolation - for testing only.
WARNING: This mode provides no security guarantees. Use only in tests where tenant isolation is not relevant.
func NewNoIsolation ¶ added in v0.3.0
func NewNoIsolation() *NoIsolation
NewNoIsolation creates a new NoIsolation mode (testing only).
func (*NoIsolation) InjectFilter ¶ added in v0.3.0
func (n *NoIsolation) InjectFilter(ctx context.Context, filters map[string]interface{}) (map[string]interface{}, error)
InjectFilter passes through filters unchanged.
func (*NoIsolation) InjectMetadata ¶ added in v0.3.0
func (n *NoIsolation) InjectMetadata(ctx context.Context, docs []Document) error
InjectMetadata is a no-op.
func (*NoIsolation) Mode ¶ added in v0.3.0
func (n *NoIsolation) Mode() string
Mode returns "none" for this isolation mode.
func (*NoIsolation) ValidateTenant ¶ added in v0.3.0
func (n *NoIsolation) ValidateTenant(ctx context.Context) error
ValidateTenant always succeeds.
type PayloadIsolation ¶ added in v0.3.0
type PayloadIsolation struct{}
PayloadIsolation implements IsolationMode using metadata filtering.
In this mode:
- All documents in a single collection per type (e.g., "memories")
- tenant_id, team_id, project_id stored as document metadata
- All queries automatically filtered by tenant context
- Missing tenant context = error (fail closed)
Security guarantees:
- Mandatory filter injection on all queries
- No bypass possible - private methods enforce filtering
- Audit-friendly - tenant always in context
func NewPayloadIsolation ¶ added in v0.3.0
func NewPayloadIsolation() *PayloadIsolation
NewPayloadIsolation creates a new PayloadIsolation mode.
func (*PayloadIsolation) InjectFilter ¶ added in v0.3.0
func (p *PayloadIsolation) InjectFilter(ctx context.Context, filters map[string]interface{}) (map[string]interface{}, error)
InjectFilter adds tenant filters to existing query filters.
func (*PayloadIsolation) InjectMetadata ¶ added in v0.3.0
func (p *PayloadIsolation) InjectMetadata(ctx context.Context, docs []Document) error
InjectMetadata adds tenant metadata to all documents.
func (*PayloadIsolation) Mode ¶ added in v0.3.0
func (p *PayloadIsolation) Mode() string
Mode returns "payload" for this isolation mode.
func (*PayloadIsolation) ValidateTenant ¶ added in v0.3.0
func (p *PayloadIsolation) ValidateTenant(ctx context.Context) error
ValidateTenant checks tenant context is present and valid.
type ProviderConfig ¶ added in v0.3.0
type ProviderConfig struct {
// BasePath is the root directory for all vectorstore data.
// Default: ~/.config/contextd/vectorstore
BasePath string
// Compress enables gzip compression for stored data.
Compress bool
// VectorSize is the expected embedding dimension.
// Default: 384 (for FastEmbed bge-small-en-v1.5)
VectorSize int
// LocalModeAcknowledged suppresses security warnings about missing authorization.
// Set to true when you understand this provider has no auth and is for local use only.
// Alternative: Set CONTEXTD_LOCAL_MODE=1 environment variable.
LocalModeAcknowledged bool
}
ProviderConfig holds configuration for ChromemStoreProvider.
func (*ProviderConfig) ApplyDefaults ¶ added in v0.3.0
func (c *ProviderConfig) ApplyDefaults()
ApplyDefaults sets default values for unset fields.
type QdrantConfig ¶
type QdrantConfig struct {
// Host is the Qdrant server hostname or IP address.
// Default: "localhost"
Host string
// Port is the Qdrant gRPC port (NOT HTTP REST port).
// Default: 6334 (gRPC), not 6333 (HTTP)
Port int
// CollectionName is the default collection for operations.
// Format: {scope}_{type} for multi-tenancy
// Examples: org_memories, platform_memories, platform_contextd_memories
CollectionName string
// VectorSize is the dimensionality of embeddings.
// Examples: 384 (BAAI/bge-small-en-v1.5), 768 (BERT), 1536 (OpenAI)
// MUST match Embedder output dimensions.
VectorSize uint64
// Distance is the similarity metric for vector search.
// Options: Cosine (default), Euclid, Dot
Distance qdrant.Distance
// UseTLS enables TLS encryption for gRPC connection.
// Default: false (MVP), true (production)
UseTLS bool
// MaxRetries is the maximum number of retry attempts for transient failures.
// Default: 3
MaxRetries int
// RetryBackoff is the initial backoff duration for retries.
// Doubles on each retry (exponential backoff).
// Default: 1 second
RetryBackoff time.Duration
// MaxMessageSize is the maximum gRPC message size in bytes.
// Default: 50MB (to handle large documents)
MaxMessageSize int
// CircuitBreakerThreshold is the number of failures before opening circuit.
// Default: 5
CircuitBreakerThreshold int
// Isolation is the tenant isolation mode.
// Default: PayloadIsolation for fail-closed security.
// Set at construction time; immutable afterward to prevent race conditions.
Isolation IsolationMode
}
QdrantConfig holds configuration for Qdrant gRPC client.
func (*QdrantConfig) ApplyDefaults ¶
func (c *QdrantConfig) ApplyDefaults()
ApplyDefaults sets default values for unset fields.
func (QdrantConfig) Validate ¶
func (c QdrantConfig) Validate() error
Validate validates the configuration.
type QdrantStore ¶
type QdrantStore struct {
// contains filtered or unexported fields
}
QdrantStore is a Store implementation using Qdrant's native gRPC client.
This implementation bypasses Qdrant's actix-web HTTP layer, eliminating the 256kB payload limit that causes 413 errors during repository indexing.
Key features:
- Native gRPC transport (port 6334)
- Binary protobuf encoding (no JSON size limits)
- Better performance than HTTP REST
- Full Qdrant feature access
- Collection-per-project isolation
- Tenant isolation via payload filtering
func NewQdrantStore ¶
func NewQdrantStore(config QdrantConfig, embedder Embedder) (*QdrantStore, error)
NewQdrantStore creates a new QdrantStore with the given configuration.
The constructor performs the following steps:
- Validates configuration
- Creates Qdrant gRPC client
- Performs health check
- Returns ready-to-use store
Returns an error if:
- Configuration is invalid
- Connection to Qdrant fails
- Health check fails
func (*QdrantStore) AddDocuments ¶
AddDocuments adds documents to the vector store. If isolation mode is set, tenant metadata is automatically injected.
func (*QdrantStore) Close ¶
func (s *QdrantStore) Close() error
Close closes the Qdrant gRPC connection.
func (*QdrantStore) CollectionExists ¶
CollectionExists checks if a collection exists.
func (*QdrantStore) CreateCollection ¶
func (s *QdrantStore) CreateCollection(ctx context.Context, collectionName string, vectorSize int) error
CreateCollection creates a new collection with the specified configuration.
func (*QdrantStore) DeleteCollection ¶
func (s *QdrantStore) DeleteCollection(ctx context.Context, collectionName string) error
DeleteCollection deletes a collection and all its documents.
func (*QdrantStore) DeleteDocuments ¶
func (s *QdrantStore) DeleteDocuments(ctx context.Context, ids []string) error
DeleteDocuments deletes documents by their IDs from the default collection.
func (*QdrantStore) DeleteDocumentsFromCollection ¶
func (s *QdrantStore) DeleteDocumentsFromCollection(ctx context.Context, collectionName string, ids []string) error
DeleteDocumentsFromCollection deletes documents by their IDs from a specific collection.
func (*QdrantStore) ExactSearch ¶
func (s *QdrantStore) ExactSearch(ctx context.Context, collectionName string, query string, k int) ([]SearchResult, error)
ExactSearch performs brute-force similarity search without using HNSW index. This is a fallback for small datasets (<10 vectors) where HNSW index may not be built.
func (*QdrantStore) GetCollectionInfo ¶
func (s *QdrantStore) GetCollectionInfo(ctx context.Context, collectionName string) (*CollectionInfo, error)
GetCollectionInfo returns metadata about a collection.
func (*QdrantStore) IsolationMode ¶ added in v0.3.0
func (s *QdrantStore) IsolationMode() IsolationMode
IsolationMode returns the current isolation mode.
func (*QdrantStore) ListCollections ¶
func (s *QdrantStore) ListCollections(ctx context.Context) ([]string, error)
ListCollections returns a list of all collection names.
func (*QdrantStore) Search ¶
func (s *QdrantStore) Search(ctx context.Context, query string, k int) ([]SearchResult, error)
Search performs similarity search in the default collection.
func (*QdrantStore) SearchInCollection ¶
func (s *QdrantStore) SearchInCollection(ctx context.Context, collectionName string, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
SearchInCollection performs similarity search in a specific collection. If isolation mode is set, tenant filters are automatically injected.
func (*QdrantStore) SearchWithFilters ¶
func (s *QdrantStore) SearchWithFilters(ctx context.Context, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
SearchWithFilters performs similarity search with metadata filters.
func (*QdrantStore) SetIsolationMode ¶ added in v0.3.0
func (s *QdrantStore) SetIsolationMode(mode IsolationMode)
SetIsolationMode sets the tenant isolation mode for this store.
DEPRECATED: Prefer setting isolation via config at construction time (e.g., QdrantConfig.Isolation) for thread-safety. This method exists for backward compatibility but should only be called once before any operations. Calling SetIsolationMode concurrently with operations may cause race conditions.
Use NewPayloadIsolation() for multi-tenant payload filtering, NewFilesystemIsolation() for database-per-project isolation, or NewNoIsolation() for testing only.
Default is PayloadIsolation for fail-closed security.
type SearchResult ¶
type SearchResult struct {
// ID is the document identifier
ID string
// Content is the document text content
Content string
// Score is the similarity score (higher = more similar)
Score float32
// Metadata contains the document metadata
Metadata map[string]interface{}
}
SearchResult represents a search result from the vector store.
type Store ¶
type Store interface {
// AddDocuments adds documents to the vector store.
//
// Documents are embedded and stored with their metadata. The document ID
// is used as the unique identifier in the vector store.
//
// If Document.Collection is specified, the document is added to that collection.
// Otherwise, the implementation's default collection is used.
//
// Returns the IDs of added documents and an error if the operation fails.
AddDocuments(ctx context.Context, docs []Document) ([]string, error)
// Search performs similarity search in the default collection.
//
// It searches for documents similar to the query and returns up to k results
// ordered by similarity score (highest first).
//
// Returns search results with scores and metadata, or an error if search fails.
Search(ctx context.Context, query string, k int) ([]SearchResult, error)
// SearchWithFilters performs similarity search with metadata filters.
//
// Filters are applied to document metadata (e.g., {"owner": "alice"}).
// Only documents matching ALL filter conditions are returned.
//
// Returns filtered search results or an error if search fails.
SearchWithFilters(ctx context.Context, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
// SearchInCollection performs similarity search in a specific collection.
//
// This supports the hierarchical collection architecture by allowing searches
// in scope-specific collections (e.g., "org_memories", "platform_contextd_memories").
//
// Returns filtered search results from the specified collection, or an error.
SearchInCollection(ctx context.Context, collectionName string, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
// DeleteDocuments deletes documents by their IDs from the default collection.
//
// Returns an error if deletion fails.
DeleteDocuments(ctx context.Context, ids []string) error
// DeleteDocumentsFromCollection deletes documents by their IDs from a specific collection.
//
// Returns an error if deletion fails.
DeleteDocumentsFromCollection(ctx context.Context, collectionName string, ids []string) error
// CreateCollection creates a new collection with the specified configuration.
//
// Collections are namespaces for documents (e.g., project-specific collections).
// The vectorSize parameter specifies the dimensionality of embeddings.
//
// Returns an error if collection creation fails or collection already exists.
CreateCollection(ctx context.Context, collectionName string, vectorSize int) error
// DeleteCollection deletes a collection and all its documents.
//
// This is a destructive operation that cannot be undone.
//
// Returns an error if deletion fails or collection doesn't exist.
DeleteCollection(ctx context.Context, collectionName string) error
// CollectionExists checks if a collection exists.
//
// Returns true if the collection exists, false otherwise.
// Returns an error only if the check operation itself fails.
CollectionExists(ctx context.Context, collectionName string) (bool, error)
// ListCollections returns a list of all collection names.
//
// Returns collection names or an error if listing fails.
ListCollections(ctx context.Context) ([]string, error)
// GetCollectionInfo returns metadata about a collection.
//
// Returns collection info including point count and vector size.
// Returns ErrCollectionNotFound if the collection doesn't exist.
GetCollectionInfo(ctx context.Context, collectionName string) (*CollectionInfo, error)
// ExactSearch performs brute-force similarity search without using HNSW index.
//
// This is a fallback for small datasets (<10 vectors) where HNSW index
// may not be built. It performs exact cosine similarity on all vectors.
//
// Returns search results ordered by similarity score (highest first).
ExactSearch(ctx context.Context, collectionName string, query string, k int) ([]SearchResult, error)
// SetIsolationMode sets the tenant isolation mode for this store.
//
// DEPRECATED: Prefer setting isolation via config at construction time
// (e.g., ChromemConfig.Isolation) for thread-safety. This method exists
// for backward compatibility but should only be called once before any
// operations. Calling SetIsolationMode concurrently with operations may
// cause race conditions.
//
// Use NewPayloadIsolation() for multi-tenant payload filtering,
// NewFilesystemIsolation() for database-per-project isolation,
// or NewNoIsolation() for testing only.
//
// Default is PayloadIsolation for fail-closed security.
SetIsolationMode(mode IsolationMode)
// IsolationMode returns the current isolation mode.
IsolationMode() IsolationMode
// Close closes the vector store connection and releases resources.
Close() error
}
Store is the interface for vector storage operations.
This interface is transport-agnostic - implementations can use HTTP REST, gRPC, or any other protocol. The interface focuses on contextd's specific needs for document storage, search, and collection management.
Collection Naming Convention:
- Organization: org_{type} (e.g., org_memories)
- Team: {team}_{type} (e.g., platform_memories)
- Project: {team}_{project}_{type} (e.g., platform_contextd_memories)
Tenant Isolation:
Stores support two isolation modes. The preferred pattern is to set isolation via config at construction time (e.g., ChromemConfig.Isolation) for thread-safety:
PayloadIsolation: Single collection per type with metadata-based filtering. All documents include tenant_id, team_id, project_id in metadata. Queries automatically filter by tenant context from ctx. Requires: TenantInfo in context (see ContextWithTenant). Security: Fail-closed - missing tenant context returns ErrMissingTenant.
FilesystemIsolation: Database-per-project isolation (legacy). Uses StoreProvider to create separate stores per tenant/project path. Physical filesystem isolation provides security boundary.
When using PayloadIsolation, callers MUST provide tenant context:
ctx = vectorstore.ContextWithTenant(ctx, &vectorstore.TenantInfo{
TenantID: "org-123",
TeamID: "team-1", // optional
ProjectID: "proj-1", // optional
})
results, err := store.Search(ctx, query, k)
Implementations:
- ChromemStore: Embedded chromem-go (default)
- QdrantStore: External Qdrant gRPC client
func NewStore ¶ added in v0.3.0
func NewStore(cfg *config.Config, embedder Embedder, logger *zap.Logger, opts ...StoreOption) (Store, error)
NewStore creates a new Store based on the configuration.
This factory function examines the VectorStoreConfig.Provider field and creates the appropriate store implementation:
- "chromem" (default): Creates an embedded ChromemStore (no external deps)
- "qdrant": Creates a QdrantStore (requires external Qdrant server)
Tenant Isolation:
By default, stores use PayloadIsolation mode for fail-closed security. All operations require tenant context (TenantInfo in ctx) or return ErrMissingTenant. To disable isolation for testing, use the WithIsolation option:
store, err := vectorstore.NewStore(cfg, embedder, logger,
vectorstore.WithIsolation(vectorstore.NewNoIsolation())) // Testing only!
The chromem provider is recommended for most users as it requires no setup:
brew install contextd # Just works!
Example usage:
cfg := config.Load()
store, err := vectorstore.NewStore(cfg, embedder, logger)
if err != nil {
log.Fatal(err)
}
defer store.Close()
func NewStoreFromProvider ¶ added in v0.3.0
func NewStoreFromProvider(provider string, chromemCfg *ChromemConfig, qdrantCfg *QdrantConfig, embedder Embedder, logger *zap.Logger, opts ...StoreOption) (Store, error)
NewStoreFromProvider creates a store directly from provider name and specific config. This is useful when you need more control over configuration.
type StoreOption ¶ added in v0.3.0
type StoreOption func(store Store)
StoreOption configures a Store after creation.
func WithIsolation ¶ added in v0.3.0
func WithIsolation(mode IsolationMode) StoreOption
WithIsolation sets the isolation mode for a store. Use NewPayloadIsolation() for multi-tenant payload filtering, NewFilesystemIsolation() for database-per-project isolation, or NewNoIsolation() for testing only.
type StoreProvider ¶ added in v0.3.0
type StoreProvider interface {
// GetProjectStore returns a store for project-level collections.
// Path: {basePath}/{tenant}/{project}/ (direct)
// Path: {basePath}/{tenant}/{team}/{project}/ (team-scoped)
GetProjectStore(ctx context.Context, tenant, team, project string) (Store, error)
// GetTeamStore returns a store for team-level shared collections.
// Path: {basePath}/{tenant}/{team}/
GetTeamStore(ctx context.Context, tenant, team string) (Store, error)
// GetOrgStore returns a store for org-level shared collections.
// Path: {basePath}/{tenant}/
GetOrgStore(ctx context.Context, tenant string) (Store, error)
// Close closes all managed stores.
Close() error
}
StoreProvider manages chromem.DB instances per scope path.
This enables database-per-project isolation where:
- Each project gets its own chromem.DB at a unique filesystem path
- Collection names are simple ("checkpoints", "memories") not prefixed
- Physical filesystem isolation prevents data leakage
Path hierarchy:
- Project (direct): {basePath}/{tenant}/{project}/
- Project (team-scoped): {basePath}/{tenant}/{team}/{project}/
- Team shared: {basePath}/{tenant}/{team}/
- Org shared: {basePath}/{tenant}/
SECURITY NOTE: LOCAL DEVELOPMENT ONLY
The current implementation does NOT include authorization checks. Any caller can request any tenant/team/project store without verification. This is acceptable for:
- Local development (single-user, localhost)
- CLI tools (trusted environment)
- Testing environments
For multi-tenant production deployments, you MUST:
- Wrap with AuthorizedStoreProvider with session-based authentication
- Implement tenant membership verification before granting store access
- Enable audit logging for all store access
To acknowledge local-only mode and suppress warnings:
- Set CONTEXTD_LOCAL_MODE=1 environment variable, OR
- Set LocalModeAcknowledged=true in ProviderConfig
PRODUCTION MODE ENFORCEMENT:
When CONTEXTD_PRODUCTION_MODE=1 is set, the provider will FAIL FAST if authorization is not explicitly acknowledged. This prevents accidental deployment without security review.
Production mode behavior:
- Without acknowledgment: Returns error, server fails to start
- With CONTEXTD_LOCAL_MODE=1: Starts with warning (explicit override)
- With LocalModeAcknowledged=true: Starts with warning (explicit override)
Recommended production setup:
- Set CONTEXTD_PRODUCTION_MODE=1 in your deployment
- Implement AuthorizedStoreProvider wrapper (see example below)
- Never set CONTEXTD_LOCAL_MODE=1 in production
Example AuthorizedStoreProvider pattern (NOT IMPLEMENTED - reference only):
type AuthorizedStoreProvider struct {
inner StoreProvider
session *AuthSession // Contains tenant, user, permissions
}
func (a *AuthorizedStoreProvider) GetProjectStore(ctx context.Context, tenant, team, project string) (Store, error) {
if !a.session.CanAccess(tenant, project) {
return nil, fmt.Errorf("unauthorized: user cannot access tenant %s", tenant)
}
return a.inner.GetProjectStore(ctx, tenant, team, project)
}
type TenantInfo ¶ added in v0.3.0
type TenantInfo struct {
// TenantID is the organization/user identifier (required).
TenantID string
// TeamID is the team identifier (optional).
TeamID string
// ProjectID is the project identifier (optional).
ProjectID string
}
TenantInfo holds tenant context for filtering and isolation.
Multi-tenancy hierarchy:
- TenantID (required): Organization or user identifier
- TeamID (optional): Team scope within tenant
- ProjectID (optional): Project scope within team
Security: All fields are validated before use in queries.
func ExtractTenantFromFilters ¶ added in v0.3.0
func ExtractTenantFromFilters(filters map[string]interface{}) (*TenantInfo, error)
ExtractTenantFromFilters creates a TenantInfo from filter map. Returns ErrMissingTenant if required fields are missing. Returns ErrInvalidTenant if tenant_id is not a valid string.
func MustTenantFromContext ¶ added in v0.3.0
func MustTenantFromContext(ctx context.Context) *TenantInfo
MustTenantFromContext extracts TenantInfo from context or panics. Use only when tenant presence is guaranteed by middleware.
func TenantFromContext ¶ added in v0.3.0
func TenantFromContext(ctx context.Context) (*TenantInfo, error)
TenantFromContext extracts TenantInfo from a context. Returns ErrMissingTenant if not present - fail closed.
func (*TenantInfo) TenantFilter ¶ added in v0.3.0
func (t *TenantInfo) TenantFilter() map[string]interface{}
TenantFilter returns filter conditions for queries. Returns conditions that match this tenant's scope.
func (*TenantInfo) TenantMetadata ¶ added in v0.3.0
func (t *TenantInfo) TenantMetadata() map[string]interface{}
TenantMetadata returns tenant info as a metadata map for document storage.
func (*TenantInfo) Validate ¶ added in v0.3.0
func (t *TenantInfo) Validate() error
Validate checks that required fields are present and valid.
type TroubleshootAdapter ¶
type TroubleshootAdapter struct {
// contains filtered or unexported fields
}
TroubleshootAdapter adapts Store to implement troubleshoot.VectorStore interface.
func NewTroubleshootAdapter ¶
func NewTroubleshootAdapter(store Store) *TroubleshootAdapter
NewTroubleshootAdapter creates an adapter for troubleshoot service.
func (*TroubleshootAdapter) AddDocuments ¶
func (a *TroubleshootAdapter) AddDocuments(ctx context.Context, docs []Document) error
AddDocuments adds documents to the vector store. Returns nil on success (discards the returned IDs since troubleshoot doesn't need them).
func (*TroubleshootAdapter) SearchWithFilters ¶
func (a *TroubleshootAdapter) SearchWithFilters(ctx context.Context, query string, k int, filters map[string]interface{}) ([]SearchResult, error)
SearchWithFilters performs similarity search with metadata filters.