Versions in this module Expand all Collapse all v1 v1.2.0 Jun 6, 2026 v1.1.0 Jun 3, 2026 Changes in this version + const DefaultLLMChunkMaxChars + const DefaultLLMChunkMinChars + const DefaultLLMChunkOverlapChars + const DefaultRuleChunkMaxChars + const DefaultRuleChunkMinChars + var ErrChunkFailed = errors.New("chunking failed") + var ErrChunkerNotFound = errors.New("chunker not found") + var ErrDetectionFailed = errors.New("document type detection failed") + var ErrEmptyText = errors.New("text is empty") + var ErrInvalidChunkOpt = errors.New("invalid chunk options") + var ErrInvalidConfig = errors.New("invalid configuration") + var ErrNoChunks = errors.New("no chunks produced") + var ErrProviderNotFound = errors.New("provider not found") + func List() []string + func Register(factory ChunkerFactory) error + func Supports(provider string) bool + type Chunk struct + Index int + Metadata map[string]interface{} + Text string + Title string + type ChunkOptions struct + DocumentTitle string + MaxChars int + MinChars int + OverlapChars int + PreChunkClean map[string]interface{} + type ChunkResult struct + Chunks []Chunk + DocumentType DocumentType + Duration int64 + type ChunkStats struct + AvgChunkSize int + DocumentType DocumentType + MaxChunkSize int + MinChunkSize int + OverlapChars int + TotalChars int + TotalChunks int + type Chunker interface + Chunk func(ctx context.Context, text string, opts *ChunkOptions) ([]Chunk, error) + Provider func() string + func Create(ctx context.Context, cfg *Config) (Chunker, error) + type ChunkerFactory interface + Create func(ctx context.Context, cfg *Config) (Chunker, error) + Name func() string + Supports func(provider string) bool + type Config struct + ChatModel interface{} + CustomConfig map[string]interface{} + Detector DocumentTypeDetector + MaxChars int + MinChars int + Model string + OverlapChars int + Provider string + type DefaultFactory struct + func GetFactory() *DefaultFactory + func NewDefaultFactory() *DefaultFactory + func (f *DefaultFactory) Create(ctx context.Context, cfg *Config) (Chunker, error) + func (f *DefaultFactory) List() []string + func (f *DefaultFactory) Register(factory ChunkerFactory) error + func (f *DefaultFactory) Supports(provider string) bool + type DocumentType int + const DocumentTypeStructured + const DocumentTypeTableKV + const DocumentTypeUnknown + const DocumentTypeUnstructured + type DocumentTypeDetector interface + DetectDocumentType func(ctx context.Context, text string) (DocumentType, error) + type LLMChunker struct + func NewLLMChunker(cfg *Config) *LLMChunker + func (c *LLMChunker) Chunk(ctx context.Context, text string, opts *ChunkOptions) ([]Chunk, error) + func (c *LLMChunker) Provider() string + type LLMChunkerFactory struct + func (f *LLMChunkerFactory) Create(ctx context.Context, cfg *Config) (Chunker, error) + func (f *LLMChunkerFactory) Name() string + func (f *LLMChunkerFactory) Supports(provider string) bool + type RouterChunkerFactory struct + func (f *RouterChunkerFactory) Create(ctx context.Context, cfg *Config) (Chunker, error) + func (f *RouterChunkerFactory) Name() string + func (f *RouterChunkerFactory) Supports(provider string) bool + type RoutingChunker struct + func NewRoutingChunker(cfg *Config) *RoutingChunker + func (c *RoutingChunker) Chunk(ctx context.Context, text string, opts *ChunkOptions) ([]Chunk, error) + func (c *RoutingChunker) Provider() string + type RuleBasedDocumentTypeDetector struct + func (d *RuleBasedDocumentTypeDetector) DetectDocumentType(ctx context.Context, text string) (DocumentType, error) + type StructuredChunkerFactory struct + func (f *StructuredChunkerFactory) Create(ctx context.Context, cfg *Config) (Chunker, error) + func (f *StructuredChunkerFactory) Name() string + func (f *StructuredChunkerFactory) Supports(provider string) bool + type StructuredRuleChunker struct + func NewStructuredRuleChunker(cfg *Config) *StructuredRuleChunker + func (c *StructuredRuleChunker) Chunk(ctx context.Context, text string, opts *ChunkOptions) ([]Chunk, error) + func (c *StructuredRuleChunker) Provider() string + type TableKVChunker struct + func NewTableKVChunker(cfg *Config) *TableKVChunker + func (c *TableKVChunker) Chunk(ctx context.Context, text string, opts *ChunkOptions) ([]Chunk, error) + func (c *TableKVChunker) Provider() string + type TableKVChunkerFactory struct + func (f *TableKVChunkerFactory) Create(ctx context.Context, cfg *Config) (Chunker, error) + func (f *TableKVChunkerFactory) Name() string + func (f *TableKVChunkerFactory) Supports(provider string) bool
Go to main page