embedding

type BM25Config struct {
	// Dimensions is the output embedding dimension (default: 384 for compatibility)
	Dimensions int

	// K1 controls term frequency saturation (default: 1.5)
	// Higher values give more weight to term frequency
	K1 float64

	// B controls length normalization (default: 0.75)
	// B=1.0 means full normalization, B=0.0 means no normalization
	B float64
}

type BM25Embedder struct {
	// contains filtered or unexported fields
}

type Cache interface {
	// Get retrieves a cached embedding for the given content hash.
	//
	// Returns an error if the embedding is not found in the cache.
	Get(ctx context.Context, contentHash string) ([]float32, error)

	// Put stores an embedding in the cache with the given content hash.
	//
	// The cache should be content-addressed using a cryptographic hash
	// (e.g., SHA-256) of the text content.
	Put(ctx context.Context, contentHash string, embedding []float32) error
}

type DedupRecord struct {
	Vector         []float32 `json:"vector"`
	EntityIDs      []string  `json:"entity_ids"` // Entities sharing this content
	FirstGenerated time.Time `json:"first_generated"`
	Model          string    `json:"model,omitempty"`
	Dimensions     int       `json:"dimensions,omitempty"`
}

type Embedder interface {
	// Generate creates embeddings for the given texts (the DOCUMENT side — what is
	// embedded at ingest).
	//
	// This is the primary method - batch operations are natural for all providers.
	// For single text, pass a slice with one element.
	// Returns a slice of float32 slices, where each inner slice is an embedding vector.
	Generate(ctx context.Context, texts []string) ([][]float32, error)

	// GenerateQuery creates embeddings for QUERY-side text (semantic search),
	// as opposed to Generate which embeds documents at ingest. Asymmetric retrieval
	// models (Snowflake arctic-embed, BGE, E5) are trained to embed a query with an
	// instruction prefix while documents are embedded raw; omitting the query prefix
	// is a measured retrieval-quality cliff, not a rounding error (gh#438). Symmetric
	// embedders (BM25) implement this identically to Generate.
	GenerateQuery(ctx context.Context, texts []string) ([][]float32, error)

	// Dimensions returns the dimensionality of embeddings produced by this embedder,
	// or 0 if it is not yet known.
	//
	// For example, all-MiniLM-L6-v2 produces 384-dimensional vectors.
	//
	// Implementations that discover the width from a remote endpoint (HTTPEmbedder)
	// report 0 until the first successful response. Implementations MUST return 0
	// rather than a plausible default: callers fold this value into durable
	// content-addressed keys (DedupKey), where a guessed width silently partitions
	// the dedup bucket into pre-guess and post-guess halves. Callers must treat 0
	// as "no stable vector space yet" and skip keyed dedup entirely.
	//
	// This value must be safe to read concurrently with Generate — the graph-embedding
	// component reads it from the hop-1 watcher goroutine and from every hop-2 worker.
	Dimensions() int

	// Model returns the model identifier used by this embedder.
	//
	// This is useful for debugging and logging which model is being used.
	Model() string

	// Close releases any resources held by the embedder.
	//
	// Must be called when the embedder is no longer needed. For HTTP providers
	// this is typically a no-op, but for local ONNX models this releases GPU/CPU resources.
	Close() error
}

type EmbedderIdentity struct {
	// Type is the configured embedder kind ("bm25", "http").
	Type string
	// Model is the embedder's model identifier.
	Model string
	// Dimensions is the vector width the embedder produces.
	Dimensions int
	// MaxTextLen is the source-text truncation cap applied before generation.
	// It belongs in the key because the cap is derived from the embedder type
	// (4000 for bm25, 8000 otherwise): flipping type changes the TEXT that gets
	// embedded, not only the model that embeds it.
	MaxTextLen int
}

type FailedEntry struct {
	EntityID       string
	Reason         string
	SourceRevision uint64
}

type GeneratedCallback func(entityID string, embedding []float32)

type HTTPConfig struct {
	// BaseURL is the base URL of the embedding service.
	// Examples:
	//   - "http://localhost:8082" (TEI - Hugging Face Text Embeddings Inference)
	//   - "http://tei:8082" (TEI container by name)
	//   - "http://localhost:8080" (LocalAI)
	//   - "https://api.openai.com/v1" (OpenAI cloud)
	BaseURL string

	// Model is the embedding model to use.
	// Examples:
	//   - "all-MiniLM-L6-v2" (TEI default - 384 dims, fast)
	//   - "all-mpnet-base-v2" (TEI - 768 dims, higher quality)
	//   - "text-embedding-ada-002" (OpenAI)
	Model string

	// APIKey for authentication (optional for local services).
	// Required for OpenAI, optional for TEI/LocalAI.
	APIKey string

	// QueryPrefix is prepended to query-side text only (GenerateQuery), never to
	// documents. Set it to the model's query instruction for asymmetric retrieval
	// models — e.g. "Represent this sentence for searching relevant passages: " for
	// Snowflake arctic-embed / E5 (gh#438). Empty disables the prefix (symmetric use).
	QueryPrefix string

	// Timeout for HTTP requests (default: 30s).
	Timeout time.Duration

	// Cache for embedding results (optional but recommended).
	Cache Cache

	// Logger for error logging (optional, defaults to slog.Default()).
	Logger *slog.Logger

	// IdleConnTimeout, ResponseHeaderTimeout, and DisableKeepAlives
	// mirror the same fields on model.EndpointConfig and graph/llm's
	// OpenAIConfig — operator overrides for connection-hygiene
	// behaviour. Empty/false selects the framework default (see
	// model.NewHTTPClient).
	IdleConnTimeout       string
	ResponseHeaderTimeout string
	DisableKeepAlives     bool
}

type HTTPEmbedder struct {
	// contains filtered or unexported fields
}

type NATSCache struct {
	// contains filtered or unexported fields
}

type Record struct {
	EntityID    string    `json:"entity_id"`
	Vector      []float32 `json:"vector,omitempty"`
	ContentHash string    `json:"content_hash"`
	// SourceText is the INLINE lane's WHOLE embedding text (the identity triples
	// extracted at hop 1). It is meaningful ONLY on the inline lane (StorageRef == nil);
	// on the offloaded lane it is left EMPTY and the identity prefix travels in
	// IdentityText instead. Keeping the offloaded identity OUT of SourceText is a
	// deliberate cross-version contract: a pre-#635 worker's getSourceText is
	// SourceText-primary, so an offloaded identity stored here would make such a worker
	// embed identity-only and silently drop the body (see IdentityText, #635 retro F1).
	SourceText string `json:"source_text,omitempty"`
	// IdentityText is the inline identity PREFIX (title/.signature/.comment, per the
	// configured text suffixes) for the OFFLOADED (StorageRef) lane ONLY: hop 2 embeds it
	// AHEAD of the fetched body, identity-first, in one vector so text_suffixes takes
	// effect on offloaded entities too (D1/D2, #601). Empty on the inline lane, and empty
	// on an offloaded record with no inline identity text (body-only).
	//
	// It is a field DISTINCT from SourceText on purpose (#635 retro F1). Pending records
	// are durable and re-delivered via WatchAll, so a PRE-#635 worker can consume a record
	// this (post-#635) writer produced during a rolling upgrade or after a rollback. That
	// old worker does not know this field, ignores it, sees SourceText == "" with
	// StorageRef set, and falls back to fetching the body — the safe pre-#635 behavior. Had
	// the identity been overloaded onto SourceText (as #635 originally did), the old worker
	// would have taken its SourceText branch and embedded identity-only, silently losing the
	// body.
	IdentityText string    `json:"identity_text,omitempty"`
	Model        string    `json:"model,omitempty"`
	Dimensions   int       `json:"dimensions,omitempty"`
	GeneratedAt  time.Time `json:"generated_at,omitempty"`
	Status       Status    `json:"status"`
	ErrorMsg     string    `json:"error_msg,omitempty"` // If status=failed
	// Reason is a BOUNDED classification of a failure (status=failed), stored next to
	// the raw ErrorMsg (#613). It is the value the failures metric is labelled by — the
	// raw ErrorMsg is unbounded and must NEVER be a metric label (cardinality blowup).
	// Additive/omitempty: a record written by a pre-#613 worker carries no reason, and a
	// rolled-back worker ignores the field, so it is wire-compatible in both directions.
	// It is also what the current-failed bootstrap scan reads to seed the reason
	// breakdown after a restart.
	Reason string `json:"reason,omitempty"` // Bounded reason enum when status=failed

	// SourceRevision is the ENTITY_STATES stream revision that produced this record.
	// It is threaded from the hop-1 watcher so hop-2 can complete the embedding
	// readiness watermark at the terminal transition (ADR-066 §3), and it is now
	// PERSISTED onto generated/failed records so SaveGenerated/SaveFailed can order
	// concurrent writes by it: a record already carrying a higher SourceRevision has
	// a newer vector, so a late older-revision write is dropped (#614 part 2). 0
	// means "unknown" (a legacy record written before this field existed) — treated
	// as oldest, so any real revision wins the ordering guard, and the watermark
	// completion treats 0 as a no-op.
	SourceRevision uint64 `json:"source_revision,omitempty"`

	// ContentStorable support (Feature 008)
	// When StorageRef is set, Worker fetches content from ObjectStore
	// and uses ContentFields to extract text for embedding.
	StorageRef    *StorageRef       `json:"storage_ref,omitempty"`
	ContentFields map[string]string `json:"content_fields,omitempty"` // Role → field name
}

type ScoredEntity struct {
	EntityID   string
	Similarity float64
}

type Status string

type Storage struct {
	// contains filtered or unexported fields
}

type StorageRef struct {
	StorageInstance string `json:"storage_instance"`
	Key             string `json:"key"`
}

type StoreResolver interface {
	Streamable(instance string) (storage.StreamableStore, bool)
}

type TerminalCallback func(entityID string, sourceRevision uint64, outcome TerminalOutcome, reason string)

type TerminalOutcome int

type Worker struct {
	// contains filtered or unexported fields
}

type WorkerMetrics interface {
	// IncDedupHits increments the deduplication hits counter
	IncDedupHits()
	// IncDedupSkipped counts an embedding generated on a condition where the durable
	// dedup bucket was NOT consulted (currently: an embedder whose vector width is
	// unresolved, so no content-addressed key can be derived). It makes the
	// avoided-reuse cost visible rather than inferred (#623): the offloaded-lane
	// re-embed cost Track 0 measured, and, post-fix, its recovery.
	IncDedupSkipped(reason string)
	// IncTruncated counts one source-text truncation at the configured cap, so the
	// bytes actually embedded are discoverable rather than silently dropped (#602).
	IncTruncated()
	// IncOffloadedIdentityIncluded counts one offloaded (StorageRef) entity for which a
	// vector was STORED that included its inline identity text (title/.signature/.comment,
	// per text_suffixes) AHEAD of the body. It fires on the successful-persistence path
	// (with IncDedupHits), not at text production — a dropped save does not count (#635
	// retro F3). Paired with IncOffloadedIdentityAbsent, it makes the text-suffix effect on
	// the offloaded lane observable: a producer tuning text_suffixes confirms it took effect
	// from /metrics rather than inferring it from silence (D5/#601).
	IncOffloadedIdentityIncluded()
	// IncOffloadedIdentityAbsent counts one offloaded (StorageRef) entity for which a
	// vector was STORED from its body ALONE — no inline identity text was present. Like its
	// symmetric half it fires only on the successful-persistence path (#635 retro F3): an
	// offloaded entity whose embed failed, or one with neither identity nor body (deleted
	// before generation), counts neither. Lets a producer tell a config-effect from silence
	// (D5/#601).
	IncOffloadedIdentityAbsent()
	// IncFailed increments the failed embeddings counter
	IncFailed()
	// IncFailedReason increments the reason-labelled failures counter (#613). reason is
	// a value from the BOUNDED failure enum (see the failReason* constants) — never the
	// raw error message, which is unbounded and would blow up label cardinality. It
	// fires on the SAME terminal path as IncFailed so it stays a strict partition of the
	// total (mirrors fusion body_hydration_failures_total{reason}, inc 2).
	IncFailedReason(reason string)
	// SetPending sets the current pending embeddings gauge
	SetPending(count float64)
	// IncContentResolveError counts a body fetch that FAILED after a store was
	// resolved (an infra fault: read error, deleted bucket) — distinct from the
	// component-side content_unresolved (no store wired at all). Preserves the
	// gh#414 diagnosability the ADR-063 resolver would otherwise blur (M1).
	IncContentResolveError()
	// IncContentResolved counts a body successfully fetched from a resolved store.
	// This is the POSITIVE observable for the ADR-063 H2 behavior change: offloaded
	// bodies that configs without a store-read port previously excluded now embed —
	// a rising value is that inclusion happening, not merely content_unresolved
	// falling (the cost-ledger "make the delta observable" discipline).
	IncContentResolved()
}