chunking

type Chunk struct {
	Content       string            `json:"content"`
	Type          string            `json:"type"`
	Level         int               `json:"level"`
	Title         string            `json:"title"`
	StartIndex    int               `json:"start_index"`
	EndIndex      int               `json:"end_index"`
	Metadata      map[string]string `json:"metadata"`
	ID            string            `json:"id,omitempty"`
	TokenCount    int               `json:"token_count,omitempty"`
	Relationships []string          `json:"relationships,omitempty"`
}

type ChunkType string

type Chunker interface {
	ChunkMarkdown(content string) ([]Chunk, error)
	ChunkMarkdownWithContext(ctx context.Context, content string) ([]Chunk, error)
}

type ChunkerConfig struct {
	MaxChunkSize       int
	MinChunkSize       int
	OverlapSize        int
	PreserveStructure  bool
	EnableSemantic     bool
	MergeSparseParents bool
}

type Config struct {
	// Required fields.
	MaxChunkSize int
	MinChunkSize int
	Model        string

	// Optional fields with defaults.
	OverlapSize         int
	SimilarityThreshold float64
	MaxMergeChunks      int
	EnableParallel      bool
}

type DocumentNode struct {
	Type       ast.NodeKind
	Level      int
	Title      string
	Content    string
	Children   []*DocumentNode
	StartIndex int
	EndIndex   int
	ASTNode    ast.Node
}

type NodeInfo struct {
	Title      string
	Content    string
	StartIndex int
	EndIndex   int
}

type OptimizedMarkdownChunker struct {
	// contains filtered or unexported fields
}

type Option func(*Config)

type SemanticChunker struct {
	// contains filtered or unexported fields
}