Documentation
¶
Index ¶
- func NewStructuredOptions(chunkingType types.ChunkingType) *types.ChunkingOptions
- type ChunkManager
- type ProgressCallback
- type SemanticChunker
- func (sc *SemanticChunker) Chunk(ctx context.Context, text string, options *types.ChunkingOptions, ...) error
- func (sc *SemanticChunker) ChunkFile(ctx context.Context, file string, options *types.ChunkingOptions, ...) error
- func (sc *SemanticChunker) ChunkStream(ctx context.Context, stream io.ReadSeeker, options *types.ChunkingOptions, ...) error
- type StructuredChunker
- func (chunker *StructuredChunker) Chunk(ctx context.Context, text string, options *types.ChunkingOptions, ...) error
- func (chunker *StructuredChunker) ChunkFile(ctx context.Context, file string, options *types.ChunkingOptions, ...) error
- func (chunker *StructuredChunker) ChunkStream(ctx context.Context, stream io.ReadSeeker, options *types.ChunkingOptions, ...) error
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func NewStructuredOptions ¶
func NewStructuredOptions(chunkingType types.ChunkingType) *types.ChunkingOptions
NewStructuredOptions creates a new structured chunker options by chunking type
Types ¶
type ChunkManager ¶
type ChunkManager struct {
// contains filtered or unexported fields
}
ChunkManager manages chunk hierarchy and status
func NewChunkManager ¶
func NewChunkManager() *ChunkManager
NewChunkManager creates a new chunk manager
func (*ChunkManager) AddChunk ¶
func (cm *ChunkManager) AddChunk(chunk *types.Chunk)
AddChunk adds a chunk to the manager
func (*ChunkManager) GetParents ¶
func (cm *ChunkManager) GetParents(chunk *types.Chunk) []types.Chunk
GetParents returns the parents chain for a chunk
func (*ChunkManager) UpdateChunkStatus ¶
func (cm *ChunkManager) UpdateChunkStatus(chunkID string, status types.ChunkingStatus)
UpdateChunkStatus updates a chunk's status and propagates to parents if needed
type ProgressCallback ¶
ProgressCallback represents the progress callback function
type SemanticChunker ¶
type SemanticChunker struct {
// contains filtered or unexported fields
}
SemanticChunker implements semantic-based chunking using LLM
func NewSemanticChunker ¶
func NewSemanticChunker(progressCallback ProgressCallback) *SemanticChunker
NewSemanticChunker creates a new semantic chunker
func (*SemanticChunker) Chunk ¶
func (sc *SemanticChunker) Chunk(ctx context.Context, text string, options *types.ChunkingOptions, callback types.ChunkingProgress) error
Chunk implements semantic chunking on text
func (*SemanticChunker) ChunkFile ¶
func (sc *SemanticChunker) ChunkFile(ctx context.Context, file string, options *types.ChunkingOptions, callback types.ChunkingProgress) error
ChunkFile implements semantic chunking on file
func (*SemanticChunker) ChunkStream ¶
func (sc *SemanticChunker) ChunkStream(ctx context.Context, stream io.ReadSeeker, options *types.ChunkingOptions, callback types.ChunkingProgress) error
ChunkStream implements semantic chunking on stream
type StructuredChunker ¶
type StructuredChunker struct {
// contains filtered or unexported fields
}
StructuredChunker is the chunker for structured data
func NewStructuredChunker ¶
func NewStructuredChunker() *StructuredChunker
NewStructuredChunker creates a new structured chunker
func (*StructuredChunker) Chunk ¶
func (chunker *StructuredChunker) Chunk(ctx context.Context, text string, options *types.ChunkingOptions, callback types.ChunkingProgress) error
Chunk is the main function to chunk text
func (*StructuredChunker) ChunkFile ¶
func (chunker *StructuredChunker) ChunkFile(ctx context.Context, file string, options *types.ChunkingOptions, callback types.ChunkingProgress) error
ChunkFile is the function to chunk file
func (*StructuredChunker) ChunkStream ¶
func (chunker *StructuredChunker) ChunkStream(ctx context.Context, stream io.ReadSeeker, options *types.ChunkingOptions, callback types.ChunkingProgress) error
ChunkStream is the function to chunk stream
Source Files