rag

type APIAuthConfig struct {
	Type   string            `yaml:"type"`   // "bearer", "basic", "apikey", "oauth2"
	Token  string            `yaml:"token"`  // Token/API key
	User   string            `yaml:"user"`   // Username (for basic auth)
	Pass   string            `yaml:"pass"`   // Password (for basic auth)
	Header string            `yaml:"header"` // Header name (for apikey type)
	Extra  map[string]string `yaml:"extra"`  // Additional auth parameters
}

type APIEndpointConfig struct {
	Path           string            `yaml:"path"`            // API path (relative to baseURL)
	Method         string            `yaml:"method"`          // HTTP method (default: GET)
	Params         map[string]string `yaml:"params"`          // Query parameters
	Headers        map[string]string `yaml:"headers"`         // Additional headers
	Body           string            `yaml:"body"`            // Request body (for POST/PUT)
	IDField        string            `yaml:"id_field"`        // JSON field to use as document ID
	ContentField   string            `yaml:"content_field"`   // JSON field(s) to use as content (comma-separated or JSONPath)
	MetadataFields []string          `yaml:"metadata_fields"` // JSON fields to include as metadata
	UpdatedField   string            `yaml:"updated_field"`   // JSON field for last modified time
	Pagination     *PaginationConfig `yaml:"pagination"`      // Pagination configuration
	Transform      string            `yaml:"transform"`       // Optional JavaScript-like transform function (future)
}

type APISource struct {
	// contains filtered or unexported fields
}

type BinaryExtractor struct {
	// contains filtered or unexported fields
}

type Chunk struct {
	// Content is the actual text content of this chunk.
	Content string `json:"content"`

	// Index is the chunk's position within the document (0-based).
	Index int `json:"index"`

	// Total is the total number of chunks for this document.
	Total int `json:"total"`

	// StartLine is the starting line number in the source document (1-based).
	StartLine int `json:"start_line"`

	// EndLine is the ending line number in the source document (1-based).
	EndLine int `json:"end_line"`

	// StartByte is the byte offset where this chunk begins (optional).
	StartByte int `json:"start_byte,omitempty"`

	// EndByte is the byte offset where this chunk ends (optional).
	EndByte int `json:"end_byte,omitempty"`

	// Context provides semantic context for the chunk (function name, type, etc.).
	Context *ChunkContext `json:"context,omitempty"`

	// Metadata contains additional chunk-specific information.
	Metadata map[string]any `json:"metadata,omitempty"`
}

type ChunkContext struct {
	// FunctionName is the containing function/method name (for code).
	FunctionName string `json:"function_name,omitempty"`

	// TypeName is the containing type/class name (for code).
	TypeName string `json:"type_name,omitempty"`

	// FilePath is the source file path.
	FilePath string `json:"file_path,omitempty"`

	// Language is the detected programming language (for code).
	Language string `json:"language,omitempty"`

	// Section is the document section name (for prose documents).
	Section string `json:"section,omitempty"`

	// ParentID links to a parent chunk (for hierarchical retrieval).
	ParentID string `json:"parent_id,omitempty"`
}

type Chunker interface {
	// Chunk splits content into pieces.
	//
	// The content is split according to the chunker's strategy.
	// Each chunk includes position information (line numbers, byte offsets)
	// for source mapping.
	//
	// Parameters:
	//   - content: the text to split
	//   - ctx: optional context (e.g., from metadata extraction)
	//
	// Returns chunks ordered by position in the original content.
	Chunk(content string, ctx *ChunkContext) ([]Chunk, error)

	// Strategy returns the chunker strategy name.
	Strategy() ChunkerStrategy

	// Config returns the chunker configuration.
	Config() ChunkerConfig
}

type ChunkerConfig struct {
	// Strategy is the chunking strategy.
	// Values: "simple", "overlapping", "semantic"
	// Default: "simple"
	Strategy ChunkerStrategy `yaml:"strategy,omitempty"`

	// Size is the target chunk size in characters.
	// Default: 1000
	Size int `yaml:"size,omitempty"`

	// Overlap is the overlap size in characters (for overlapping strategy).
	// Default: 200
	Overlap int `yaml:"overlap,omitempty"`

	// MinSize is the minimum chunk size (chunks smaller than this are merged).
	// Default: 100
	MinSize int `yaml:"min_size,omitempty"`

	// MaxSize is the maximum chunk size (hard limit).
	// Default: 2000
	MaxSize int `yaml:"max_size,omitempty"`

	// Separators are the preferred split points for semantic chunking.
	// Default: ["\n\n", "\n", ". ", " "]
	Separators []string `yaml:"separators,omitempty"`

	// PreserveWords avoids splitting in the middle of words.
	// Default: true
	PreserveWords bool `yaml:"preserve_words,omitempty"`
}

type ChunkerStrategy string

type ChunkingError struct {
	Strategy   string // Chunking strategy
	DocumentID string // Document ID
	Message    string // Error message
	Err        error  // Underlying error
}

type CodeMetadata struct {
	Functions []FunctionInfo         `json:"functions,omitempty"`
	Types     []TypeInfo             `json:"types,omitempty"`
	Imports   []string               `json:"imports,omitempty"`
	Symbols   map[string]interface{} `json:"symbols,omitempty"`
	Custom    map[string]interface{} `json:"custom,omitempty"`
}

type CollectionSource struct {
	// contains filtered or unexported fields
}

type ContentExtractor interface {
	// Name returns the extractor name for logging/debugging.
	Name() string

	// CanExtract determines if this extractor can handle the given file.
	CanExtract(path string, mimeType string) bool

	// Extract extracts content from the file.
	Extract(ctx context.Context, path string, fileSize int64) (*ExtractedContent, error)

	// Priority returns the priority (higher = preferred when multiple extractors match).
	Priority() int
}

type DBPoolAdapter struct {
	// contains filtered or unexported fields
}

type DataSource interface {
	// Type returns the type of data source (e.g., "directory", "sql", "api", "s3")
	Type() string

	// DiscoverDocuments returns a channel of discovered documents and a channel of errors.
	// Documents are discovered asynchronously and sent through the channel.
	// For file sources, content should be read from files.
	// For SQL/API sources, content should already be populated.
	DiscoverDocuments(ctx context.Context) (<-chan Document, <-chan error)

	// ReadDocument retrieves a specific document by its ID.
	// The ID format depends on the source type (file path, SQL row ID, API endpoint, etc.)
	ReadDocument(ctx context.Context, id string) (*Document, error)

	// SupportsIncrementalIndexing indicates if this source supports incremental updates
	// based on modification timestamps or change tracking.
	SupportsIncrementalIndexing() bool

	// GetLastModified returns the last modification time for a document, if available.
	// Returns zero time if not supported or document doesn't exist.
	GetLastModified(ctx context.Context, id string) (time.Time, error)

	// Close releases any resources held by the data source.
	Close() error
}

type DirectorySource struct {
	// contains filtered or unexported fields
}

type DirectorySourceConfig struct {
	Path        string
	Include     []string
	Exclude     []string
	MaxFileSize int64 // Max file size in bytes to process (0 for no limit)
}

type Document struct {
	// ID is the unique identifier for this document.
	ID string `json:"id"`

	// Content is the text content to be indexed.
	Content string `json:"content"`

	// Title is the document title (optional).
	Title string `json:"title,omitempty"`

	// SourcePath is the path to the source file (for file-based documents).
	SourcePath string `json:"source_path,omitempty"`

	// MimeType is the content type (e.g., "text/plain", "text/markdown").
	MimeType string `json:"mime_type,omitempty"`

	// Size is the content size in bytes.
	Size int64 `json:"size"`

	// Metadata contains additional document information.
	Metadata map[string]any `json:"metadata,omitempty"`
}

type DocumentEvent struct {
	Type     DocumentEventType
	Document Document
	Error    error
}

type DocumentEventType string

type DocumentStore struct {
	// contains filtered or unexported fields
}

type DocumentStoreConfig struct {
	// Name identifies this store.
	Name string

	// Description describes the store (used by SearchTool).
	Description string

	// Source provides documents.
	Source DataSource

	// SearchEngine for indexing and search.
	SearchEngine *SearchEngine

	// Chunker for splitting documents (optional, defaults to engine's chunker).
	Chunker Chunker

	// Collection name (optional, defaults to store name).
	Collection string

	// SourcePath is the base path for checkpoints (auto-detected from directory source).
	SourcePath string

	// Watch enables file watching for automatic re-indexing.
	Watch bool

	// IncrementalIndexing only re-indexes changed documents.
	IncrementalIndexing bool

	// EnableCheckpoints enables resume capability for interrupted indexing.
	// Checkpoints are saved to .hector/checkpoints/ in the source path.
	// Default: true for directory sources
	EnableCheckpoints bool

	// EnableProgress enables progress display during indexing.
	// Default: true
	EnableProgress bool

	// Search configuration for advanced features.
	Search *SearchOptions

	// MaxConcurrentIndexing limits parallel document processing (default: NumCPU).
	// Set to 1 for sequential indexing (legacy behavior).
	MaxConcurrentIndexing int

	// RetryConfig for transient failure handling (optional).
	RetryConfig *RetryConfig
}

type DocumentStoreError struct {
	StoreName string    // Name of the document store
	Operation string    // Operation that failed
	Message   string    // Error message
	FilePath  string    // File path if applicable
	Err       error     // Underlying error
	Timestamp time.Time // When the error occurred
}

type DocumentStoreStats struct {
	Name          string  `json:"name"`
	Collection    string  `json:"collection"`
	IndexedCount  int     `json:"indexed_count"`
	WatchEnabled  bool    `json:"watch_enabled"`
	SourceType    string  `json:"source_type"`
	TotalDocs     int64   `json:"total_docs"`
	SkippedDocs   int64   `json:"skipped_docs"`
	ErrorDocs     int64   `json:"error_docs"`
	DocsPerSecond float64 `json:"docs_per_second"`
	SearchCount   int64   `json:"search_count"`
}

type ExtractedContent struct {
	Content          string            // The extracted text content
	Title            string            // Document title (if available)
	Author           string            // Document author (if available)
	Metadata         map[string]string // Additional metadata
	ProcessingTimeMs int64             // Time taken to extract
	ExtractorName    string            // Name of extractor used
}

type ExtractionError struct {
	Extractor string // Extractor name
	FilePath  string // File path
	Message   string // Error message
	Err       error  // Underlying error
}

type ExtractorRegistry struct {
	// contains filtered or unexported fields
}

type FactoryDeps struct {
	// DBPool provides database connections.
	DBPool *config.DBPool

	// VectorProviders maps provider names to instances.
	VectorProviders map[string]vector.Provider

	// Embedders maps embedder names to instances.
	Embedders map[string]embedder.Embedder

	// LLMs maps LLM names to instances.
	LLMs map[string]model.LLM

	// ToolCaller provides access to MCP tools for document parsing.
	// Optional - only needed if MCPParsers is configured.
	ToolCaller ToolCaller

	// Config is the root configuration.
	Config *config.Config
}

type FileCheckpoint struct {
	Path        string    `json:"path"`
	Hash        string    `json:"hash"`
	Size        int64     `json:"size"`
	ModTime     time.Time `json:"mod_time"`
	Status      string    `json:"status"` // "indexed", "skipped", "failed"
	ProcessedAt time.Time `json:"processed_at"`
}

type FileFilter interface {
	ShouldInclude(path string) bool
	ShouldExclude(path string) bool
}

type FileWatcher struct {
	// contains filtered or unexported fields
}

type FileWatcherConfig struct {
	BasePath      string
	Filter        FileFilter
	DebounceDelay time.Duration // Delay before processing events (default: 100ms)
}

type FunctionInfo struct {
	Name       string `json:"name"`
	Signature  string `json:"signature,omitempty"`
	StartLine  int    `json:"start_line"`
	EndLine    int    `json:"end_line"`
	Receiver   string `json:"receiver,omitempty"` // For methods
	IsExported bool   `json:"is_exported,omitempty"`
	DocComment string `json:"doc_comment,omitempty"`
}

type GoMetadataExtractor struct{}

type HealthCheck struct {
	// Component name.
	Component string `json:"component"`

	// Status of the component.
	Status HealthStatus `json:"status"`

	// Message provides details about the status.
	Message string `json:"message,omitempty"`

	// Latency of the health check.
	Latency time.Duration `json:"latency_ms"`

	// Timestamp of the check.
	Timestamp time.Time `json:"timestamp"`

	// Details contains component-specific health information.
	Details map[string]any `json:"details,omitempty"`
}

type HealthChecker interface {
	HealthCheck(ctx context.Context) HealthCheck
}

type HealthStatus string

type HyDE struct {
	// contains filtered or unexported fields
}

type IndexCheckpoint struct {
	Version        string                    `json:"version"`
	StoreName      string                    `json:"store_name"`
	SourcePath     string                    `json:"source_path"`
	StartTime      time.Time                 `json:"start_time"`
	LastUpdate     time.Time                 `json:"last_update"`
	ProcessedFiles map[string]FileCheckpoint `json:"processed_files"`
	TotalFiles     int                       `json:"total_files"`
	IndexedCount   int                       `json:"indexed_count"`
	SkippedCount   int                       `json:"skipped_count"`
	FailedCount    int                       `json:"failed_count"`
}

type IndexCheckpointManager struct {
	// contains filtered or unexported fields
}

type IndexError struct {
	StoreName  string // Document store name
	DocumentID string // Document ID
	Operation  string // Operation (e.g., "embed", "upsert", "delete")
	Message    string // Error message
	Err        error  // Underlying error
}

type IndexMetrics struct {
	// contains filtered or unexported fields
}

type IndexMetricsSnapshot struct {
	StoreName         string        `json:"store_name"`
	TotalDocs         int64         `json:"total_docs"`
	IndexedDocs       int64         `json:"indexed_docs"`
	SkippedDocs       int64         `json:"skipped_docs"`
	ErrorDocs         int64         `json:"error_docs"`
	DocsPerSecond     float64       `json:"docs_per_second"`
	StartTime         time.Time     `json:"start_time,omitempty"`
	EndTime           time.Time     `json:"end_time,omitempty"`
	SearchCount       int64         `json:"search_count"`
	AvgSearchLatency  time.Duration `json:"avg_search_latency_ns"`
	MaxSearchLatency  time.Duration `json:"max_search_latency_ns"`
	LastSearchLatency time.Duration `json:"last_search_latency_ns"`
}

type LLMQueryExpander struct {
	// contains filtered or unexported fields
}

type MCPExtractor struct {
	// contains filtered or unexported fields
}

type MCPExtractorConfig struct {
	ToolCaller      ToolCaller
	ParserToolNames []string // Tool names to try (e.g., ["parse_document", "docling_parse"])
	SupportedExts   []string // File extensions this extractor handles (empty = all)
	Priority        int      // Priority (higher = preferred)
	LocalBasePath   string   // Local base path of the document store (e.g., "/Users/user/workspace/hector/test-docs")
	PathPrefix      string   // Remote path prefix for containerized MCP services (e.g., "/docs")
}

type MetadataExtractor interface {
	// Name returns the extractor name
	Name() string

	// CanExtract determines if this extractor can handle the given language
	CanExtract(language string) bool

	// Extract extracts metadata from source code
	Extract(content string, filePath string) (*CodeMetadata, error)
}

type MetadataExtractorRegistry struct {
	// contains filtered or unexported fields
}

type MultiQueryExpander struct {
	// contains filtered or unexported fields
}

type NativeParseResult struct {
	Success          bool
	Content          string
	Title            string
	Author           string
	Metadata         map[string]string
	Error            string
	ProcessingTimeMs int64
}

type NativeParser interface {
	ParseDocument(ctx context.Context, filePath string, fileSize int64) (*NativeParseResult, error)
}

type NativeParserRegistry struct {
	// contains filtered or unexported fields
}

type NilChunker struct{}

type NilDataSource struct{}

type NilMultiQueryExpander struct{}

type NilQueryExpander struct{}

type NilReranker struct{}

type OverlappingChunker struct {
	// contains filtered or unexported fields
}

type PaginationConfig struct {
	Type      string `yaml:"type"`       // "offset", "cursor", "page", "link"
	PageParam string `yaml:"page_param"` // Query parameter name for page/offset
	SizeParam string `yaml:"size_param"` // Query parameter name for page size
	MaxPages  int    `yaml:"max_pages"`  // Maximum pages to fetch (0 = unlimited)
	PageSize  int    `yaml:"page_size"`  // Items per page
	NextField string `yaml:"next_field"` // JSON field containing next page URL/cursor
	DataField string `yaml:"data_field"` // JSON field containing array of items (if nested)
}

type PatternCache struct {
	// contains filtered or unexported fields
}

type PatternFilter struct {
	// contains filtered or unexported fields
}

type ProgressStats struct {
	TotalFiles     int64
	ProcessedFiles int64
	IndexedFiles   int64
	SkippedFiles   int64
	FailedFiles    int64
	DeletedFiles   int64
	CurrentFile    string
	ElapsedTime    time.Duration
}

type ProgressTracker struct {
	// contains filtered or unexported fields
}

type QueryExpander interface {
	// Expand generates multiple query variations from the original query.
	Expand(ctx context.Context, query string, numVariations int) ([]string, error)
}

type RankingDecision struct {
	// Index is the original result index.
	Index int `json:"index"`

	// Relevance is the LLM-assigned relevance score (1-10).
	Relevance int `json:"relevance"`

	// Reason explains why this ranking was assigned.
	Reason string `json:"reason,omitempty"`
}

type RerankResult struct {
	// Results are the reranked search results.
	Results []SearchResult

	// Rankings contains the LLM's ranking decisions.
	Rankings []RankingDecision
}

type Reranker struct {
	// contains filtered or unexported fields
}

type RetryConfig struct {
	// MaxRetries is the maximum number of retry attempts (default: 3).
	MaxRetries int

	// BaseDelay is the initial delay between retries (default: 1s).
	BaseDelay time.Duration

	// MaxDelay is the maximum delay between retries (default: 30s).
	MaxDelay time.Duration

	// JitterFactor adds randomness to delays (0.0-1.0, default: 0.1).
	JitterFactor float64

	// RetryableErrors are error substrings that indicate retryable failures.
	RetryableErrors []string
}

type RetryError struct {
	Operation   string
	Attempts    int
	LastError   error
	IsExhausted bool
}

type Retryer struct {
	// contains filtered or unexported fields
}

type SQLSource struct {
	// contains filtered or unexported fields
}

type SQLSourceOptions struct {
	DB      *sql.DB
	Driver  string
	Tables  []SQLTableConfig
	MaxRows int
}

type SQLTableConfig struct {
	Table           string   `yaml:"table"`
	Columns         []string `yaml:"columns"`          // Columns to concatenate for content
	IDColumn        string   `yaml:"id_column"`        // Primary key or unique identifier
	UpdatedColumn   string   `yaml:"updated_column"`   // Column for tracking updates (e.g., updated_at)
	WhereClause     string   `yaml:"where_clause"`     // Optional WHERE clause for filtering
	MetadataColumns []string `yaml:"metadata_columns"` // Columns to include as metadata
}

type SearchEngine struct {
	// contains filtered or unexported fields
}

type SearchEngineConfig struct {
	// Provider for vector storage and search (required).
	Provider vector.Provider

	// Embedder for generating embeddings (required).
	Embedder embedder.Embedder

	// Chunker for splitting documents (optional, defaults to simple).
	Chunker Chunker

	// Collection name for storing documents (optional, defaults to "rag_documents").
	Collection string

	// DefaultTopK is the default number of results (default: 10).
	DefaultTopK int

	// DefaultThreshold filters results below this score (default: 0.0).
	DefaultThreshold float32

	// HyDE for hypothetical document embedding (optional).
	HyDE *HyDE

	// Reranker for LLM-based result reranking (optional).
	Reranker *Reranker

	// MultiQuery for query expansion (optional).
	MultiQuery *MultiQueryExpander
}

type SearchError struct {
	Component string // Component that failed (e.g., "embedder", "vector_db", "reranker")
	Operation string // Operation that failed
	Message   string // Error message
	Query     string // Query that caused the error
	Err       error  // Underlying error
}

type SearchMetrics struct {
	// contains filtered or unexported fields
}

type SearchMetricsSnapshot struct {
	EngineName      string        `json:"engine_name"`
	TotalSearches   int64         `json:"total_searches"`
	SuccessfulHits  int64         `json:"successful_hits"`
	EmptyResults    int64         `json:"empty_results"`
	AvgLatency      time.Duration `json:"avg_latency_ns"`
	MaxLatency      time.Duration `json:"max_latency_ns"`
	MinLatency      time.Duration `json:"min_latency_ns"`
	HyDEUsage       int64         `json:"hyde_usage"`
	RerankUsage     int64         `json:"rerank_usage"`
	MultiQueryUsage int64         `json:"multi_query_usage"`
}

type SearchOptions struct {
	// Mode specifies the search mode: "vector", "keyword", "hybrid".
	Mode string `json:"mode,omitempty"`

	// EnableHyDE enables Hypothetical Document Embeddings.
	EnableHyDE bool `json:"enable_hyde,omitempty"`

	// EnableRerank enables LLM-based reranking.
	EnableRerank bool `json:"enable_rerank,omitempty"`

	// EnableMultiQuery enables query expansion.
	EnableMultiQuery bool `json:"enable_multi_query,omitempty"`

	// NumQueries is the number of query variants for multi-query.
	NumQueries int `json:"num_queries,omitempty"`
}

type SearchRequest struct {
	// Query is the search query text.
	Query string `json:"query"`

	// Collection scopes the search to a specific collection.
	Collection string `json:"collection,omitempty"`

	// TopK is the maximum number of results to return.
	TopK int `json:"top_k,omitempty"`

	// Threshold filters results below this score.
	Threshold float32 `json:"threshold,omitempty"`

	// Filter applies metadata filtering.
	Filter map[string]any `json:"filter,omitempty"`

	// Options contains search-specific options.
	Options *SearchOptions `json:"options,omitempty"`
}

type SearchResponse struct {
	// Results contains the matched documents/chunks.
	Results []SearchResult `json:"results"`

	// TotalMatches is the total number of matches (before limit).
	TotalMatches int `json:"total_matches,omitempty"`

	// SearchTimeMs is the search duration in milliseconds.
	SearchTimeMs int64 `json:"search_time_ms,omitempty"`

	// QueryExpansions contains expanded queries (if multi-query enabled).
	QueryExpansions []string `json:"query_expansions,omitempty"`
}

type SearchResult struct {
	// ID is the chunk/document identifier.
	ID string `json:"id"`

	// Content is the matched content.
	Content string `json:"content"`

	// Score represents relevance (higher is better).
	Score float32 `json:"score"`

	// DocumentID is the parent document identifier.
	DocumentID string `json:"document_id,omitempty"`

	// ChunkIndex is the chunk position within the document.
	ChunkIndex int `json:"chunk_index,omitempty"`

	// Metadata contains additional result information.
	Metadata map[string]any `json:"metadata,omitempty"`

	// Highlights contains matched text spans (optional).
	Highlights []string `json:"highlights,omitempty"`
}

type SemanticChunker struct {
	// contains filtered or unexported fields
}

type SimpleChunker struct {
	// contains filtered or unexported fields
}

type SourceDocument struct {
	// ID is a unique identifier for the document (format depends on source type)
	ID string

	// Content is the text content to be indexed.
	// For file sources, this should be populated by reading the file.
	// For SQL/API sources, this is populated during discovery.
	Content string

	// Metadata contains source-specific metadata (file path, table name, API endpoint, etc.)
	Metadata map[string]interface{}

	// LastModified is the last modification time, if available
	LastModified time.Time

	// Size is the size of the document in bytes (approximate for non-file sources)
	Size int64

	// ShouldIndex indicates whether this document should be indexed (after filtering)
	ShouldIndex bool

	// SourcePath is the original source path (file path, table name, API endpoint, etc.)
	// This is used for relative path calculations and display purposes
	SourcePath string
}

type TextExtractor struct{}

type Tool interface {
	GetInfo() ToolInfo
	Execute(ctx context.Context, args map[string]interface{}) (ToolResult, error)
}

type ToolCaller interface {
	GetTool(name string) (Tool, error)
}

type ToolInfo struct {
	Name        string
	Description string
	Parameters  []ToolParameter
}

type ToolParameter struct {
	Name        string
	Type        string
	Description string
	Required    bool
}

type ToolResult struct {
	Success  bool
	Content  string
	Error    string
	Metadata interface{}
}

type TypeInfo struct {
	Name       string   `json:"name"`
	Kind       string   `json:"kind"` // "struct", "interface", "alias", etc.
	StartLine  int      `json:"start_line"`
	EndLine    int      `json:"end_line"`
	Fields     []string `json:"fields,omitempty"`
	Methods    []string `json:"methods,omitempty"`
	IsExported bool     `json:"is_exported,omitempty"`
	DocComment string   `json:"doc_comment,omitempty"`
}