tokenizer

Jan 24, 2026

Changes in this version

+ var DefaultTokenCounter = NewHeuristicTokenCounter(ModelFamilyDefault)

+ func CountTokens(text string) int

+ type HeuristicTokenCounter struct

+ func NewHeuristicTokenCounter(family ModelFamily) *HeuristicTokenCounter

+ func NewHeuristicTokenCounterWithRatio(ratio float64) *HeuristicTokenCounter

+ func (h *HeuristicTokenCounter) CountMultiple(texts []string) int

+ func (h *HeuristicTokenCounter) CountTokens(text string) int

+ func (h *HeuristicTokenCounter) Ratio() float64

+ func (h *HeuristicTokenCounter) SetRatio(ratio float64)

+ type ModelFamily string

+ const ModelFamilyClaude

+ const ModelFamilyDefault

+ const ModelFamilyGPT

+ const ModelFamilyGemini

+ const ModelFamilyLlama

+ func GetModelFamily(modelName string) ModelFamily

+ type TokenCounter interface

+ CountMultiple func(texts []string) int

+ CountTokens func(text string) int

+ func NewTokenCounterForModel(modelName string) TokenCounter