Versions in this module Expand all Collapse all v0 v0.1.0 Aug 30, 2026 Changes in this version + const DefaultMaxBufferedBytes + const Punctuations + var ErrBufferLimit = errors.New("tokenize: buffered text limit exceeded") + func HasUnclosedXMLTags(text string) bool + func Hyphenate(word string) []string + func HyphenateWord(word string) []string + func JoinTokens(tokens []TokenData) string + func TokenizeParagraphs(text string) []string + type BufferedTokenStream = TokenStream + type SentenceOptions struct + FirstTokenLength int + Language string + MaxBufferedBytes int + MaxTokenLength int + MinSentenceLength int + MinTokenLength int + OutputCapacity int + RetainFormat bool + StreamContextLength int + XMLAware bool + type SentenceStream struct + type SentenceTokenizer struct + func NewSentenceTokenizer(options ...SentenceOptions) *SentenceTokenizer + func (t *SentenceTokenizer) Options() SentenceOptions + func (t *SentenceTokenizer) Stream(_ ...string) *SentenceStream + func (t *SentenceTokenizer) Tokenize(text string, _ ...string) []string + func (t *SentenceTokenizer) TokenizeSpans(text string) []Span + type SentenceTokenizerOptions = SentenceOptions + type Span struct + End int + Start int + Text string + func SplitParagraphs(text string) []Span + func SplitSentences(text string, minLength int, retainFormat ...bool) []Span + func SplitWords(text string, ignorePunctuation ...bool) []Span + func SplitWordsWithOptions(text string, opts WordOptions) []Span + type StreamOptions struct + FirstTokenLength int + MaxBufferedBytes int + MaxTokenLength int + MinContextLength int + MinTokenLength int + OutputCapacity int + XMLAware bool + type TokenData struct + SegmentID string + Token string + type TokenStream struct + func NewBufferedTokenStream(fn func(string) []Span, opts StreamOptions) *TokenStream + func (s *TokenStream) Abort(err error) error + func (s *TokenStream) Close() error + func (s *TokenStream) Closed() bool + func (s *TokenStream) EndInput(ctx context.Context) error + func (s *TokenStream) Flush(ctx context.Context) error + func (s *TokenStream) PushText(ctx context.Context, text string) error + func (s *TokenStream) Range(ctx context.Context) iter.Seq2[TokenData, error] + func (s *TokenStream) Recv(ctx context.Context) (TokenData, error) + type WordOptions struct + DropEmptyTokens bool + KeepPunctuation bool + MaxBufferedBytes int + OutputCapacity int + RetainFormat bool + SplitCharacter bool + type WordStream struct + type WordTokenizer struct + func NewWordTokenizer(ignorePunctuation ...bool) *WordTokenizer + func NewWordTokenizerWithOptions(opts WordOptions) *WordTokenizer + func (t *WordTokenizer) FormatWords(words []string) string + func (t *WordTokenizer) Stream(_ ...string) *WordStream + func (t *WordTokenizer) Tokenize(text string, _ ...string) []string + func (t *WordTokenizer) TokenizeSpans(text string) []Span
Go to main page