chunking

type AudioChunkOptions struct {
	// OverlapDurationMs Overlap duration in milliseconds between audio chunks (default: 0).
	OverlapDurationMs int `json:"overlap_duration_ms,omitempty,omitzero"`

	// WindowDurationMs Window duration in milliseconds for fixed-window audio chunking (default: 30000).
	WindowDurationMs int `json:"window_duration_ms,omitempty,omitzero"`
}

type BinaryContent struct {
	// Data Base64-encoded binary data (valid WAV, PNG, etc.)
	Data []byte `json:"data,omitempty,omitzero"`

	// EndTimeMs Audio: window end time in milliseconds
	EndTimeMs float32 `json:"end_time_ms,omitempty,omitzero"`

	// FrameDelayMs Animation: display delay in milliseconds
	FrameDelayMs int `json:"frame_delay_ms,omitempty,omitzero"`

	// FrameIndex Animation: frame number
	FrameIndex int `json:"frame_index,omitempty,omitzero"`

	// StartTimeMs Audio: window start time in milliseconds
	StartTimeMs float32 `json:"start_time_ms,omitempty,omitzero"`
}

type Chunk struct {
	// Id Sequence number of the chunk (0, 1, 2, ...)
	Id uint32 `json:"id"`

	// MimeType MIME type: text/plain, audio/wav, image/png, etc.
	MimeType string `json:"mime_type"`
	// contains filtered or unexported fields
}

type ChunkOptions struct {
	// Audio Options specific to audio chunking.
	Audio AudioChunkOptions `json:"audio,omitempty,omitzero"`

	// MaxChunks Maximum number of chunks to generate per document.
	MaxChunks int `json:"max_chunks,omitempty,omitzero"`

	// Text Options specific to text chunking.
	Text TextChunkOptions `json:"text,omitempty,omitzero"`

	// Threshold Confidence threshold for model-based chunking (0.0-1.0).
	Threshold float32 `json:"threshold,omitempty,omitzero"`
}

type Chunker interface {
	// Chunk splits text using the provided per-request options.
	// Options that are nil use the chunker's default values.
	Chunk(ctx context.Context, text string, opts ChunkOptions) ([]Chunk, error)
	Close() error
}

type TextChunkOptions struct {
	// OverlapTokens Number of tokens to overlap between consecutive chunks. Helps maintain context across chunk boundaries. Only used by fixed-size chunkers.
	OverlapTokens int `json:"overlap_tokens,omitempty,omitzero"`

	// Separator Separator string for splitting (e.g., '\n\n' for paragraphs). Only used by fixed-size chunkers.
	Separator string `json:"separator,omitempty,omitzero"`

	// TargetTokens Target number of tokens per chunk.
	TargetTokens int `json:"target_tokens,omitempty,omitzero"`
}

type TextContent struct {
	// EndChar Character position in original text where chunk ends (exclusive)
	EndChar int `json:"end_char"`

	// StartChar Character position in original text where chunk starts
	StartChar int `json:"start_char"`

	// Text The chunk text content
	Text string `json:"text"`
}