Documentation
¶
Overview ¶
Package chunker defines the interface for splitting text into manageable chunks for embedding and retrieval.
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type ChunkResult ¶
type ChunkResult struct {
// Content is the text content of the chunk.
Content string `json:"content"`
// Index is the zero-based position of this chunk in the document.
Index int `json:"index"`
// StartOffset is the byte offset of the chunk start in the original text.
StartOffset int `json:"start_offset"`
// EndOffset is the byte offset of the chunk end in the original text.
EndOffset int `json:"end_offset"`
// TokenCount is the estimated number of tokens in this chunk.
TokenCount int `json:"token_count"`
// Metadata holds chunker-specific metadata.
Metadata map[string]string `json:"metadata,omitempty"`
}
ChunkResult represents a single chunk produced by a chunker.
type Chunker ¶
type Chunker interface {
// Chunk splits the given text into chunks according to the options.
Chunk(ctx context.Context, text string, opts *Options) ([]ChunkResult, error)
}
Chunker splits text into chunks for embedding.
type CodeChunker ¶
type CodeChunker struct{}
CodeChunker splits code at function and block boundaries.
func (*CodeChunker) Chunk ¶
func (c *CodeChunker) Chunk(_ context.Context, text string, opts *Options) ([]ChunkResult, error)
Chunk splits code text at function/block boundaries.
func (*CodeChunker) Describe ¶
func (*CodeChunker) Describe() weave.ComponentInfo
Describe reports the code chunker. Its offsets are approximate.
type FixedChunker ¶
type FixedChunker struct{}
FixedChunker splits text into fixed-size chunks at exact character boundaries.
func NewFixedChunker ¶
func NewFixedChunker() *FixedChunker
NewFixedChunker creates a new FixedChunker.
func (*FixedChunker) Chunk ¶
func (c *FixedChunker) Chunk(_ context.Context, text string, opts *Options) ([]ChunkResult, error)
Chunk splits text into fixed-size chunks.
func (*FixedChunker) Describe ¶
func (*FixedChunker) Describe() weave.ComponentInfo
Describe reports the fixed-size chunker.
type Options ¶
type Options struct {
// ChunkSize is the target chunk size in tokens.
ChunkSize int
// ChunkOverlap is the number of overlapping tokens between chunks.
ChunkOverlap int
// Strategy is the chunking strategy name (e.g. "recursive", "fixed").
Strategy string
}
Options configures the chunking behaviour.
type RecursiveChunker ¶
type RecursiveChunker struct {
// Separators in priority order (largest to smallest).
Separators []string
}
RecursiveChunker splits text using hierarchical separators, trying the largest separator first and falling back to smaller ones.
func NewRecursiveChunker ¶
func NewRecursiveChunker() *RecursiveChunker
NewRecursiveChunker creates a new RecursiveChunker with default separators.
func (*RecursiveChunker) Chunk ¶
func (c *RecursiveChunker) Chunk(_ context.Context, text string, opts *Options) ([]ChunkResult, error)
Chunk splits text recursively using hierarchical separators.
func (*RecursiveChunker) Describe ¶
func (*RecursiveChunker) Describe() weave.ComponentInfo
Describe reports the recursive chunker.
type SemanticChunker ¶
type SemanticChunker struct{}
SemanticChunker groups text by sentence boundaries to create semantically coherent chunks.
func NewSemanticChunker ¶
func NewSemanticChunker() *SemanticChunker
NewSemanticChunker creates a new SemanticChunker.
func (*SemanticChunker) Chunk ¶
func (c *SemanticChunker) Chunk(_ context.Context, text string, opts *Options) ([]ChunkResult, error)
Chunk splits text at sentence boundaries, grouping sentences into chunks that don't exceed the target size.
func (*SemanticChunker) Describe ¶
func (*SemanticChunker) Describe() weave.ComponentInfo
Describe reports the semantic chunker. Its offsets are approximate.
type SlidingChunker ¶
type SlidingChunker struct{}
SlidingChunker splits text using a sliding window with overlap.
func NewSlidingChunker ¶
func NewSlidingChunker() *SlidingChunker
NewSlidingChunker creates a new SlidingChunker.
func (*SlidingChunker) Chunk ¶
func (c *SlidingChunker) Chunk(_ context.Context, text string, opts *Options) ([]ChunkResult, error)
Chunk splits text using a sliding window.
func (*SlidingChunker) Describe ¶
func (*SlidingChunker) Describe() weave.ComponentInfo
Describe reports the sliding-window chunker.
Source Files