tokenization

type Config struct {
	// Base model name for the tokenizer.
	ModelName string `json:"modelName"`
	// Number of worker goroutines for processing tokenization tasks.
	WorkersCount int `json:"workersCount"`

	UdsTokenizerConfig *UdsTokenizerConfig `json:"uds,omitempty"`
}

type MultiModalFeatures struct {
	// MMHashes maps modality (e.g. "image") to per-item content hashes.
	MMHashes map[string][]string
	// MMPlaceholders maps modality to per-item placeholder token ranges.
	MMPlaceholders map[string][]kvblock.PlaceholderRange
}

type Task struct {
	RenderReq *types.RenderChatRequest
	Prompt    string
	ModelName string
	ResultCh  chan<- tokenizationResponse // nil => fire-and-forget
}

type Tokenizer interface {
	RenderChat(*types.RenderChatRequest) ([]uint32, *MultiModalFeatures, error)
	Render(string) ([]uint32, []types.Offset, error)
	Type() string
}

type UdsTokenizer struct {
	// contains filtered or unexported fields
}

type UdsTokenizerConfig struct {
	SocketFile string `json:"socketFile"` // UDS socket path (production) or host:port for TCP (testing only)
	UseTCP     bool   `json:"useTCP"`     // If true, use TCP instead of UDS (for testing only, default: false)

	// ModelTokenizerMap maps a model name to the location of its tokenizer data.
	//
	// Each value may be either:
	//   1) A directory path that contains tokenizer files for the model (preferred), or
	//   2) A full path to a tokenizer.json file.
	//
	// Examples:
	//   {
	//     "model-a": "/mnt/models/model-a",                  // directory containing tokenizer.json, vocab, merges, etc.
	//     "model-b": "/opt/tokenizers/model-b/tokenizer.json" // explicit tokenizer.json path
	//   }
	ModelTokenizerMap map[string]string `json:"modelTokenizerMap,omitempty"`
}