embedding

type BatchOptions ¶ added in v1.0.1

type BatchOptions struct {
	BatchSize   int
	MaxWorkers  int
	RateLimit   time.Duration // Minimum time between requests
	RetryConfig RetryConfig
}

BatchOptions configures batch processing

func DefaultBatchOptions() BatchOptions

type BatchProcessor ¶ added in v1.0.1

type BatchProcessor struct {
	// contains filtered or unexported fields
}

BatchProcessor handles batch processing of embeddings with optimization

func NewBatchProcessor(provider Provider, opts BatchOptions) *BatchProcessor

func (bp *BatchProcessor) Close()

func (bp *BatchProcessor) Process(ctx context.Context, texts []string) ([][]float32, error)

func (bp *BatchProcessor) ProcessWithProgress(
	ctx context.Context,
	texts []string,
	callback ProgressCallback,
) ([][]float32, error)

type BatchProvider ¶ added in v1.0.1

type BatchProvider struct {
	// contains filtered or unexported fields
}

BatchProvider wraps a Provider with batch processing optimization

func NewBatchProvider(provider Provider, opts BatchOptions) *BatchProvider

func (bp *BatchProvider) Dimension() int

func (bp *BatchProvider) Embed(ctx context.Context, texts []string) ([][]float32, error)

func (bp *BatchProvider) GetProcessor() *BatchProcessor

func (bp *BatchProvider) ProcessWithProgress(
	ctx context.Context,
	texts []string,
	callback ProgressCallback,
) ([][]float32, error)

type ProgressCallback ¶ added in v1.0.1

type ProgressCallback func(processed, total int)

ProgressCallback is called during batch processing

type Provider ¶

type Provider interface {
	// Embed generates embeddings for the given texts
	//
	// Parameters:
	// - ctx: Context for cancellation and timeout
	// - texts: Slice of texts to embed
	//
	// Returns:
	// - [][]float32: Slice of embeddings, one for each text
	// - error: Error if embedding generation fails
	Embed(ctx context.Context, texts []string) ([][]float32, error)

	// Dimension returns the dimension of the embeddings generated by this provider
	//
	// Returns:
	// - int: Embedding dimension (e.g., 1536 for text-embedding-ada-002)
	Dimension() int
}

Provider defines the interface for embedding providers

This interface is implemented by all embedding model providers (OpenAI, Ollama, Cohere, Voyage) and allows the RAG engine to generate embeddings for text chunks.

Example implementation:

type OpenAIProvider struct {
    client *openai.Client
    model  string
}

func (p *OpenAIProvider) Embed(ctx context.Context, texts []string) ([][]float32, error) {
    // Call OpenAI API to generate embeddings
}

func (p *OpenAIProvider) Dimension() int {
    return 1536 // text-embedding-ada-002 dimension
}

type RetryConfig ¶ added in v1.0.1

type RetryConfig struct {
	MaxRetries int
	BaseDelay  time.Duration
	MaxDelay   time.Duration
	Multiplier float64
}

RetryConfig configures retry behavior

func DefaultRetryConfig() RetryConfig

type Stats ¶ added in v1.0.1

type Stats struct {
	TotalBatches     int
	ProcessedBatches int
	FailedBatches    int
	TotalTexts       int
	ProcessingTime   time.Duration
}

Stats returns batch processor statistics