chunkservice

type BatchRequest struct {
	Texts             []string          `json:"texts"`
	ChunkSize         int               `json:"chunk_size"`
	Strategy          chunking.Strategy `json:"strategy,omitempty"`
	Overlap           int               `json:"overlap,omitempty"`
	TokenEncoding     TokenEncoding     `json:"token_encoding,omitempty"`
	SeparatorProfile  SeparatorProfile  `json:"separator_profile,omitempty"`
	Semantic          *SemanticOptions  `json:"semantic,omitempty"`
	MaxTokens         int               `json:"max_tokens,omitempty"`
	IncludeEmbeddings bool              `json:"include_embeddings,omitempty"`
}

type BatchResponse struct {
	Results []BatchResult `json:"results"`
}

type BatchResult struct {
	Index    int       `json:"index"`
	Response *Response `json:"response,omitempty"`
	Error    string    `json:"error,omitempty"`
}

type Chunk struct {
	ID         int    `json:"id"`
	StartChar  int    `json:"start_char"`
	EndChar    int    `json:"end_char"`
	Text       string `json:"text"`
	CharCount  int    `json:"char_count"`
	WordCount  int    `json:"word_count"`
	TokenCount int    `json:"token_count,omitempty"`
	// Path is the heading breadcrumb ("H1 Title" → "Subsection") for
	// heading-aware strategies; omitted otherwise.
	Path []string `json:"path,omitempty"`
	// Metadata carries strategy-produced annotations (for example
	// markdown_aware's heading_level, language, line_count); omitted when the
	// strategy produces none.
	Metadata map[string]string `json:"metadata,omitempty"`
	// ContentHash is the first 16 hex characters of SHA-256(Text).
	ContentHash string `json:"content_hash"`
	// Embedding is the chunk text's embedding vector, present only when the
	// request set include_embeddings.
	Embedding []float32 `json:"embedding,omitempty"`
}

type Metadata struct {
	TotalChunks   int               `json:"total_chunks"`
	TotalChars    int               `json:"total_chars"`
	TotalTokens   int               `json:"total_tokens,omitempty"`
	StrategyUsed  chunking.Strategy `json:"strategy_used"`
	TokenEncoding TokenEncoding     `json:"token_encoding,omitempty"`
	// MaxTokens reports the token ceiling that was enforced, set only when
	// the request asked for one.
	MaxTokens int `json:"max_tokens,omitempty"`
	// SeparatorProfile reports the recursive separator cascade in effect,
	// set only when the request specified one.
	SeparatorProfile SeparatorProfile `json:"separator_profile,omitempty"`
	// EmbeddingModel names the provider model that produced the chunk
	// embeddings; set only when the request set include_embeddings.
	EmbeddingModel string `json:"embedding_model,omitempty"`
	// EmbeddingDimensions is the vector length of the chunk embeddings;
	// set only when the request set include_embeddings.
	EmbeddingDimensions int `json:"embedding_dimensions,omitempty"`
}

type Request struct {
	Text      string            `json:"text"`
	ChunkSize int               `json:"chunk_size"`
	Strategy  chunking.Strategy `json:"strategy,omitempty"`
	Overlap   int               `json:"overlap,omitempty"`
	// TokenEncoding names the tiktoken encoding used for token_based
	// chunking and the MaxTokens ceiling (default cl100k_base).
	TokenEncoding TokenEncoding `json:"token_encoding,omitempty"`
	// SeparatorProfile selects the recursive strategy's separator cascade:
	// "default_text" (empty) or "cjk_thai". Only valid with strategy=recursive.
	SeparatorProfile SeparatorProfile `json:"separator_profile,omitempty"`
	// Semantic tunes the semantic strategy; zero fields fall back to library
	// defaults. Only valid with strategy=semantic.
	Semantic *SemanticOptions `json:"semantic,omitempty"`
	// MaxTokens caps every chunk's token count (measured with TokenEncoding,
	// default cl100k_base) after the base strategy runs. 0 disables the cap.
	// Chunks that exceed it are split further (sentence, then word, then
	// hard cut), so strategies can stay semantic while honoring LLM context
	// limits. It is independent of chunk_size: chunk_size bounds characters
	// (tokens for token_based); max_tokens bounds tokens on every strategy.
	MaxTokens int `json:"max_tokens,omitempty"`
	// IncludeEmbeddings attaches an embedding vector to every chunk via the
	// configured embedding provider. Works with any strategy; requires
	// embedding configuration on the service (a clear validation error
	// otherwise). Unlike the semantic strategy, a mid-request embedding
	// failure fails the request rather than falling back: the caller
	// explicitly asked for vectors.
	IncludeEmbeddings bool `json:"include_embeddings,omitempty"`
	// PreflightCounter applies a caller-supplied token budget while chunking
	// (the legacy "preflight" pipeline): boundary strategies chunk with
	// token awareness and semantic output is split where the counter says a
	// chunk exceeds budget. Unlike MaxTokens it needs no tiktoken encoding;
	// callers bring their own counter. nil disables it.
	PreflightCounter chunking.TokenCounter `json:"-"`
}

type Response struct {
	Chunks   []Chunk  `json:"chunks"`
	Metadata Metadata `json:"metadata"`
}

type SemanticOptions struct {
	BreakSensitivity float64 `json:"break_sensitivity,omitempty"`
	MinChunkChars    int     `json:"min_chunk_chars,omitempty"`
	MaxChunkChars    int     `json:"max_chunk_chars,omitempty"`
	SmoothingWindow  int     `json:"smoothing_window,omitempty"`
	CoherenceWindow  int     `json:"coherence_window,omitempty"`
}

type SeparatorProfile string

type Service struct {
	// contains filtered or unexported fields
}

type TokenEncoding string