chunker

type ChunkConfig struct {
	By          string // How to split the file: "lines", "bytes", or "tokens"
	Size        int    // Chunk size (e.g., 10000 lines)
	Overlap     int    // Lines/bytes to overlap between chunks for context
	MaxChunks   int    // Limit total chunks to prevent overload
	Deduplicate bool   // If true, remove near-duplicate chunks after splitting
}

type ChunkResult struct {
	ChunkPaths    []string // Paths to the temporary chunk files
	TempDir       string   // Path to the temporary directory containing the chunks
	TotalChunks   int      // Total number of chunks created
	RemovedChunks int      // Number of duplicate chunks removed (when Deduplicate is enabled)
}