rag

type ArtifactChunkCache ¶

type ArtifactChunkCache struct {
	Store  storage.ArtifactStore
	Prefix string
}

ArtifactChunkCache persists cached chunks in the configured artifact store.

func NewArtifactChunkCache(store storage.ArtifactStore) *ArtifactChunkCache

func (c *ArtifactChunkCache) GetChunks(ctx context.Context, key string) ([]Chunk, bool, error)

func (c *ArtifactChunkCache) PutChunks(ctx context.Context, key string, chunks []Chunk) error

type ArtifactEnrichmentCache ¶

type ArtifactEnrichmentCache struct {
	Store  storage.ArtifactStore
	Prefix string
}

ArtifactEnrichmentCache persists cached enrichment results in the configured artifact store.

func NewArtifactEnrichmentCache(store storage.ArtifactStore) *ArtifactEnrichmentCache

func (c *ArtifactEnrichmentCache) GetEnrichment(ctx context.Context, key string) ([]string, bool, error)

func (c *ArtifactEnrichmentCache) PutEnrichment(ctx context.Context, key string, questions []string) error

type CachedDocumentChunker ¶

type CachedDocumentChunker struct {
	Chunker DocumentChunker
	Cache   ChunkCache
}

CachedDocumentChunker wraps a DocumentChunker with a deterministic cache.

The cache key includes document bytes/text, filename, the cache schema version, and the wrapped chunker's option fingerprint when available.

func NewCachedDocumentChunker(chunker DocumentChunker, cache ChunkCache) *CachedDocumentChunker

func (c *CachedDocumentChunker) Split(ctx context.Context, text string) ([]Chunk, error)

func (c *CachedDocumentChunker) SplitDocument(ctx context.Context, doc Document) ([]Chunk, error)

type CachedEnricher ¶

type CachedEnricher struct {
	Enricher Enricher
	Cache    EnrichmentCache
}

CachedEnricher wraps an Enricher with a deterministic cache, keyed by chunk text plus the wrapped enricher's option fingerprint when available.

func NewCachedEnricher(enricher Enricher, cache EnrichmentCache) *CachedEnricher

func (c *CachedEnricher) EnrichChunks(ctx context.Context, texts []string) ([][]string, error)

type Chunk ¶

type Chunk struct {
	Key      string
	Text     string
	Position int
	Metadata map[string]string
}

type ChunkCache ¶

type ChunkCache interface {
	GetChunks(ctx context.Context, key string) ([]Chunk, bool, error)
	PutChunks(ctx context.Context, key string, chunks []Chunk) error
}

ChunkCache stores document-aware chunking outputs under a deterministic key.

type ChunkCacheKeyProvider ¶

type ChunkCacheKeyProvider interface {
	ChunkCacheKey() string
}

ChunkCacheKeyProvider lets chunkers include their own options in cache keys.

type ChunkProfile ¶

type ChunkProfile struct {
	Name          ChunkProfileName `json:"name"`
	MaxTokens     int              `json:"max_tokens"`
	OverlapTokens int              `json:"overlap_tokens,omitempty"`
}

ChunkProfile describes the chunking policy to use for document ingestion. Token limits are intentionally conservative: large-context LLMs can read more, but retrieval quality usually degrades when indexed chunks become too broad or noisy.

func DefaultChunkProfile() ChunkProfile

func NewCustomChunkProfile(maxTokens, overlapTokens int) (ChunkProfile, error)

func RecommendedChunkProfile(name ChunkProfileName) (ChunkProfile, bool)

type ChunkProfileName ¶

type ChunkProfileName string

ChunkProfileName identifies a recommended chunking profile for RAG ingestion. The profile controls indexing granularity; answer-time context expansion should still be used when a large LLM needs neighboring chunks.

const (
	ChunkProfileSmall      ChunkProfileName = "small"
	ChunkProfileMedium     ChunkProfileName = "medium"
	ChunkProfileLarge      ChunkProfileName = "large"
	ChunkProfileStructured ChunkProfileName = "structured"
	// ChunkProfileTable routes to TableChunker: table rows/structure stay
	// intact as their own chunk(s) (header row repeated on every split
	// piece) instead of a generic splitter cutting through the middle of a
	// table. See TableChunker's own doc comment.
	ChunkProfileTable ChunkProfileName = "table"
	// ChunkProfileQA routes to QAChunker: one chunk per detected
	// question/answer pair instead of arbitrary token-count splitting. See
	// QAChunker's own doc comment.
	ChunkProfileQA     ChunkProfileName = "qa"
	ChunkProfileCustom ChunkProfileName = "custom"
)

type Chunker ¶

type Chunker interface {
	Split(ctx context.Context, text string) ([]Chunk, error)
}

Chunker splits a text into indexable chunks. The context allows implementations that call remote services (e.g. SemanticChunker).

func DefaultChunker() Chunker

type Document ¶

type Document struct {
	Filename string
	Text     string
	Data     []byte
}

Document is the original source material plus its text representation. Plain text chunkers can ignore Data. Document-aware chunkers can prefer Data to preserve layout, headings, page numbers, tables, and captions.

type DocumentChunker ¶

type DocumentChunker interface {
	Chunker
	SplitDocument(ctx context.Context, doc Document) ([]Chunk, error)
}

DocumentChunker is implemented by chunkers that can split the original document, not just extracted plain text.

type Embedder ¶

type Embedder interface {
	EmbedTexts(ctx context.Context, texts []string) ([][]float32, error)
}

type Enricher ¶

type Enricher interface {
	// EnrichChunks returns, for each text in order, the synthetic questions
	// generated for it. Implementations should be best-effort per item (see
	// e.g. internal/rag/enricher.LLMEnricher) - a single chunk's failure
	// returns an empty, not necessarily nil, slice at that index rather than
	// failing the whole batch. A non-nil error here means something more
	// fundamental went wrong (e.g. ctx cancelled) and Ingest should abort,
	// the same contract Embedder.EmbedTexts already has.
	EnrichChunks(ctx context.Context, texts []string) ([][]string, error)
}

Enricher generates synthetic questions per chunk - questions that chunk's content would answer - so they can be indexed alongside it. This is a known retrieval-recall technique: chunk text is usually declarative, but users phrase queries as questions, so embedding the chunk together with questions it answers closes that vocabulary gap (see RAGFlow's own "Extractor" pipeline stage, which this mirrors). Optional: nil (the default) disables enrichment entirely - see Service.SetEnricher. Costs one LLM call per chunk, so it is opt-in, never a forced default.

type EnricherCacheKeyProvider ¶

type EnricherCacheKeyProvider interface {
	EnricherCacheKey() string
}

EnricherCacheKeyProvider lets an Enricher include its own configuration (model, questions-per-chunk, prompt version) in cache keys, so changing that configuration doesn't silently reuse stale enrichment results - the same purpose ChunkCacheKeyProvider serves for chunkers.

type EnrichmentCache ¶

type EnrichmentCache interface {
	GetEnrichment(ctx context.Context, key string) ([]string, bool, error)
	PutEnrichment(ctx context.Context, key string, questions []string) error
}

EnrichmentCache stores per-chunk enrichment results (synthetic questions) under a deterministic key - the same content-hash-keying idea as ChunkCache, so a re-ingest of unchanged chunks doesn't re-bill the LLM.

type HeadingChunker ¶

type HeadingChunker struct {
	Profile ChunkProfile
}

HeadingChunker cuts a markdown document along its own structure: a heading hierarchy (markdown headings, numbered outline, or structural words like "Chapter"/"Article"/"Capítulo"/"Artigo"), paragraphs, lists, tables and fenced code. Each chunk carries the path of headings it sits under ("Part One > Article 5 > Clause 5.2"), in its text (so embedding and BM25 relevance benefit from the section context) and in Metadata["heading_path"] (for citation and display without re-parsing the text).

A block is never cut in the middle of what it is (a paragraph is cut between sentences, a table between rows with its header repeated, code between lines, a list between items), small sections are joined to their neighbours with their headings written in the text so no chunk is a title and one line, and a paragraph that has to be cut overlaps with the piece before it by the profile's OverlapTokens. See markdownChunker for the details.

Inspired by RAGFlow's bullets_category/tree_merge (rag/nlp/__init__.py:370-387,1120-1164), not a port of it.

A document with no heading at all is simply packed block by block.

func NewHeadingChunker(profile ChunkProfile) *HeadingChunker

func (c *HeadingChunker) Split(_ context.Context, text string) ([]Chunk, error)

func (c *HeadingChunker) SplitDocument(ctx context.Context, doc Document) ([]Chunk, error)

type HybridDocumentChunker ¶

type HybridDocumentChunker struct {
	Client      documentreader.HybridChunker
	Options     documentreader.ChunkOptions
	Profile     ChunkProfile
	Fallback    Chunker
	FailOnError bool
}

HybridDocumentChunker chunks rich documents with a document-aware hybrid chunker. The backend can be local, docling-serve, seshat-intelligence, or any other document reader implementing documentreader.HybridChunker. It falls back to a plain text chunker unless FailOnError is set.

func NewHybridDocumentChunker(client documentreader.HybridChunker, opts documentreader.ChunkOptions) *HybridDocumentChunker

func NewHybridDocumentChunkerForProfile(client documentreader.HybridChunker, profile ChunkProfile, opts documentreader.ChunkOptions) *HybridDocumentChunker

func (c *HybridDocumentChunker) ChunkCacheKey() string

func (c *HybridDocumentChunker) Split(ctx context.Context, text string) ([]Chunk, error)

func (c *HybridDocumentChunker) SplitDocument(ctx context.Context, doc Document) ([]Chunk, error)

type IngestRequest ¶

type IngestRequest struct {
	CorpusID string
	// FileID, when non-empty, produces a deterministic artifact key ("rag/{CorpusID}/{FileID}").
	// This makes Upsert idempotent: re-ingesting the same file replaces its vectors in-place
	// rather than creating orphaned duplicates alongside the old ones.
	FileID   string
	Filename string
	Text     string
	// Data carries the original document bytes when ingestion can provide
	// them. Chunkers that understand document structure can use it instead of
	// splitting the already-extracted Text representation.
	Data []byte
	// ScopeID tags every chunk produced from this ingest with a permission
	// scope (a workspace ID for shared content, or an owning user ID for
	// personal content). Empty means unscoped — no scope_id metadata is
	// written, and the chunk is invisible to any search that filters on it.
	// Mutually exclusive with ScopeIDs - set at most one of the two.
	ScopeID string
	// ScopeIDs tags every chunk with multiple permission scopes at once
	// (e.g. a connector-synced document visible to several identities:
	// specific users, groups, a domain). Takes precedence over ScopeID when
	// both would apply from a caller's perspective - callers should set
	// only one. Encoded as a JSON array in the scope_id metadata field, so
	// a single "$in" filter transparently matches both the scalar (ScopeID)
	// and multi-valued (ScopeIDs) representations - see vector.matchesFilter
	// and pgvector_store.go's pgFilterClause.
	ScopeIDs []string
}

type IngestResult ¶

type IngestResult struct {
	Artifact storage.ArtifactRef
	Chunks   int
}

type MemoryChunkCache ¶

type MemoryChunkCache struct {
	// contains filtered or unexported fields
}

MemoryChunkCache is useful for tests and short-lived local runs.

func NewMemoryChunkCache() *MemoryChunkCache

func (c *MemoryChunkCache) GetChunks(_ context.Context, key string) ([]Chunk, bool, error)

func (c *MemoryChunkCache) PutChunks(_ context.Context, key string, chunks []Chunk) error

type MemoryEnrichmentCache ¶

type MemoryEnrichmentCache struct {
	// contains filtered or unexported fields
}

MemoryEnrichmentCache is useful for tests and short-lived local runs.

func NewMemoryEnrichmentCache() *MemoryEnrichmentCache

func (c *MemoryEnrichmentCache) GetEnrichment(_ context.Context, key string) ([]string, bool, error)

func (c *MemoryEnrichmentCache) PutEnrichment(_ context.Context, key string, questions []string) error

type ParagraphChunker ¶

type ParagraphChunker struct {
	MaxChunkChars int
}

ParagraphChunker splits on blank lines and hard-caps each paragraph at MaxChunkChars characters. It does not call any remote service.

func (c ParagraphChunker) Split(_ context.Context, text string) ([]Chunk, error)

type QAChunker ¶

type QAChunker struct {
	Profile  ChunkProfile
	Fallback Chunker
}

QAChunker splits a naturally question/answer-formatted document (FAQs, support docs) into one chunk per Q/A pair, instead of arbitrary token-count splitting that can separate a question from its own answer or merge several unrelated pairs into one chunk. Each pair's chunk carries the question in Metadata["qa_question"], for citation/UI display without re-parsing the text - the same idea as HeadingChunker's Metadata["heading_path"].

Two independent detection strategies are tried, in order:

1. Explicit "Q:"/"A:" labels (also "Question:"/"Answer:", numbered "Q1:", and bold "**Q:**" markdown) - the common plain-text FAQ shape.
2. Markdown headings ending in "?" (e.g. "## What is your return policy?") - the common Markdown/docling-converted FAQ shape. Reuses HeadingChunker's own heading-detection (matchHeading) and ancestor-path tracking, so a nested "Category > Question?" structure keeps its category context; a heading that does NOT end in "?" still produces an ordinary (non-QA-tagged) section chunk exactly as HeadingChunker would, rather than being silently dropped - QA tagging is additive on top of HeadingChunker's own behavior, never a narrower view of the document.

A document matching neither pattern delegates entirely to Fallback - QAChunker is additive, never worse than the fallback's own baseline, the same contract HeadingChunker/TableChunker already establish.

func NewQAChunker(profile ChunkProfile) *QAChunker

func (c *QAChunker) Split(ctx context.Context, text string) ([]Chunk, error)

func (c *QAChunker) SplitDocument(ctx context.Context, doc Document) ([]Chunk, error)

type Reranker ¶

type Reranker interface {
	Rerank(ctx context.Context, query string, docs []string, topN int) (indices []int, scores []float32, err error)
	IsConfigured() bool
}

Reranker reorders a candidate set of documents by semantic relevance. Implementations are optional — when nil the RAG service returns vector results in score order without a second-pass rerank.

Rerank returns (indices, scores) where indices[i] is the position of the i-th most relevant document in the original docs slice. Pass topN=0 to return all results.

type SearchRequest ¶

type SearchRequest struct {
	CorpusID string
	Query    string
	TopK     int
	// HybridWeight blends vector similarity with BM25 keyword search.
	// 0 (default) = pure vector; 1 = pure BM25; intermediate = linear blend.
	HybridWeight float32
	// RerankWeight blends the (normalized) reranker score with the original
	// retrieval score, when a reranker is configured - a separate knob from
	// HybridWeight, which blends at retrieval time (vector vs BM25); this
	// one blends after retrieval (retrieval score vs rerank score). Above 0
	// and up to 1, an explicit weight (1 = fully trust the reranker). Left
	// at the zero value (0, the common case for a caller with no opinion)
	// falls back to the service's configured default (see
	// Service.SetRerankWeight) instead of silently disabling the blend - a
	// caller that genuinely wants zero reranker influence should simply not
	// configure a reranker, rather than rely on this field to suppress one.
	// Ignored when no reranker is configured.
	RerankWeight float32
	// Filter restricts results by metadata key-value predicates.
	// Passed through to vector.Query.Filter unchanged.
	Filter map[string]any
}

type SearchResponse ¶

type SearchResponse struct {
	CorpusID string         `json:"corpus_id"`
	Results  []SearchResult `json:"results"`
}

type SearchResult ¶

type SearchResult struct {
	Key      string            `json:"key"`
	Text     string            `json:"text"`
	Score    float32           `json:"score"`
	Metadata map[string]string `json:"metadata,omitempty"`
}

type SemanticChunker ¶

type SemanticChunker struct {

	// Threshold below which two consecutive sentences are considered
	// semantically distinct and a new chunk is started. Default: 0.3.
	Threshold float32
	// MaxChunkChars is the hard character cap per chunk. Default: 2000.
	MaxChunkChars int
	// MinChunkChars is the minimum size a chunk must reach before it can be
	// split off. Chunks below this are merged with the next. Default: 100.
	MinChunkChars int
	// contains filtered or unexported fields
}

SemanticChunker groups sentences into chunks based on cosine similarity between consecutive sentence embeddings. A new chunk is started whenever similarity falls below Threshold. Inspired by the Max-Min algorithm (Springer 2025) as implemented in the mcp-local-rag reference.

Falls back to ParagraphChunker behaviour when the embedder returns an error, so ingest never fails due to a temporary embedding outage.

func NewSemanticChunker(embedder Embedder, threshold float32) *SemanticChunker

func (c *SemanticChunker) Split(ctx context.Context, text string) ([]Chunk, error)

type Service ¶

type Service struct {
	// contains filtered or unexported fields
}

func NewService(artifacts storage.ArtifactStore, vectors vector.Store, embedder Embedder, chunker Chunker) *Service

func (s *Service) DeleteFileChunks(ctx context.Context, namespace, artifactKey string, keepBelow int) error

func (s *Service) DeleteNamespace(ctx context.Context, namespace string) error

func (s *Service) Ingest(ctx context.Context, request IngestRequest) (IngestResult, error)

func (s *Service) Search(ctx context.Context, request SearchRequest) (SearchResponse, error)

func (s *Service) SetEnricher(e Enricher)

func (s *Service) SetRerankMargin(m float32)

func (s *Service) SetRerankWeight(w float32)

func (s *Service) SetReranker(r Reranker)

func (s *Service) Vectors() vector.Store

type TableChunker ¶

type TableChunker struct {
	Profile  ChunkProfile
	Fallback Chunker
}

TableChunker splits a Markdown/GFM document while keeping every table's rows intact as their own chunk(s), instead of letting a generic character/token splitter cut through the middle of a table (which destroys the header-to-cell association the table's meaning depends on). A table larger than the chunk profile's token budget is split by data row, but the header + separator row is repeated at the top of every resulting piece, so each chunk remains a valid, self-contained table rather than a headerless fragment.

Text outside any detected table is delegated to Fallback (default HeadingChunker via NewTableChunker, so section/heading context is preserved for the surrounding prose too, not just the tables).

A document with no GFM table at all delegates entirely to Fallback - TableChunker is additive, never worse than the fallback's own baseline for a document that has nothing for it to do, the same contract HeadingChunker already established for its own fallback.

func NewTableChunker(profile ChunkProfile) *TableChunker

func (c *TableChunker) Split(_ context.Context, text string) ([]Chunk, error)

func (c *TableChunker) SplitDocument(ctx context.Context, doc Document) ([]Chunk, error)

type VectorStore ¶

type VectorStore = vector.Store