indexing

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 8, 2026 License: MIT Imports: 13 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Embedder

type Embedder interface {
	Embed(ctx context.Context, texts []string) ([][]float32, error)
}

Embedder defines the interface for embedding providers

type Indexer

type Indexer struct {
	// contains filtered or unexported fields
}

Indexer handles document indexing operations

func NewIndexer

func NewIndexer(
	parsers map[string]parser.Parser,
	defaultParser parser.Parser,
	embedder Embedder,
	store vectorstore.Store,
	metrics Metrics,
	logger Logger,
	tracer Tracer,
) *Indexer

NewIndexer creates a new indexer

func (*Indexer) AsyncBatchIndex

func (i *Indexer) AsyncBatchIndex(ctx context.Context, sources []Source) error

AsyncBatchIndex adds multiple documents to the RAG engine asynchronously

func (*Indexer) AsyncIndex

func (i *Indexer) AsyncIndex(ctx context.Context, source Source) error

AsyncIndex adds documents to the RAG engine asynchronously

func (*Indexer) AsyncIndexDirectory

func (i *Indexer) AsyncIndexDirectory(ctx context.Context, directoryPath string) error

AsyncIndexDirectory indexes all files in a directory recursively asynchronously

func (*Indexer) BatchIndex

func (i *Indexer) BatchIndex(ctx context.Context, sources []Source) error

BatchIndex adds multiple documents to the RAG engine in batch

func (*Indexer) Index

func (i *Indexer) Index(ctx context.Context, source Source) error

Index adds documents to the RAG engine

func (*Indexer) IndexDirectory

func (i *Indexer) IndexDirectory(ctx context.Context, directoryPath string) error

IndexDirectory indexes all files in a directory recursively with concurrent workers

func (*Indexer) WithConfig

func (i *Indexer) WithConfig(config IndexerConfig) *Indexer

WithConfig sets the indexer configuration

type IndexerConfig

type IndexerConfig struct {
	// WorkerCount is the number of concurrent workers for directory indexing (default: 10)
	WorkerCount int
	// FileChannelBuffer is the buffer size for the file channel in directory indexing (default: 100)
	FileChannelBuffer int
	// ErrorChannelBuffer is the buffer size for the error channel in directory indexing (default: 10)
	ErrorChannelBuffer int
}

IndexerConfig holds configurable parameters for the Indexer

func DefaultIndexerConfig

func DefaultIndexerConfig() IndexerConfig

DefaultIndexerConfig returns the default indexer configuration

type Logger

type Logger interface {
	Info(ctx context.Context, message string, fields map[string]interface{})
	Debug(ctx context.Context, message string, fields map[string]interface{})
	Error(ctx context.Context, message string, err error, fields map[string]interface{})
}

Logger defines the interface for logging

type Metrics

type Metrics interface {
	RecordErrorCount(ctx context.Context, errorType string)
	RecordIndexLatency(ctx context.Context, duration time.Duration)
	RecordIndexCount(ctx context.Context, status string)
}

Metrics defines the interface for metrics collection

type Source

type Source = core.Source

Source represents a document source for indexing

type Tracer

type Tracer interface {
	StartSpan(ctx context.Context, name string) (context.Context, observability.Span)
	Extract(ctx context.Context) (observability.Span, bool)
}

Tracer defines the interface for tracing

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL