tokenize

type BufferedTokenStream = TokenStream

type SentenceOptions struct {
	Language            string
	MinSentenceLength   int
	StreamContextLength int
	RetainFormat        bool
	MaxTokenLength      int
	MinTokenLength      int
	FirstTokenLength    int
	XMLAware            bool
	OutputCapacity      int
	// MaxBufferedBytes bounds un-emitted streamed input. Zero selects the
	// production default; it does not affect batch Tokenize calls.
	MaxBufferedBytes int
}

type SentenceStream struct{ *TokenStream }

type SentenceTokenizer struct {
	// contains filtered or unexported fields
}

type SentenceTokenizerOptions = SentenceOptions

type Span struct {
	Text       string
	Start, End int
}

type StreamOptions struct {
	MinTokenLength   int
	MinContextLength int
	MaxTokenLength   int
	FirstTokenLength int
	XMLAware         bool
	OutputCapacity   int
	// MaxBufferedBytes caps in-progress input plus a batched output token. Zero
	// selects DefaultMaxBufferedBytes.
	MaxBufferedBytes int
}

type TokenData struct {
	SegmentID string
	Token     string
}

type TokenStream struct {
	// contains filtered or unexported fields
}

type WordOptions struct {
	// KeepPunctuation opts out of the default punctuation stripping.
	KeepPunctuation bool
	// SplitCharacter emits CJK, Japanese, and Thai code points separately,
	// matching the Python tokenizer's non-spaced-language mode.
	SplitCharacter bool
	// RetainFormat attaches original inter-word whitespace to the next token.
	RetainFormat bool
	// DropEmptyTokens selects Python's behavior for punctuation-only spans. The
	// JS tokenizer retains those spans as empty tokens by default.
	DropEmptyTokens  bool
	OutputCapacity   int
	MaxBufferedBytes int
}

type WordStream struct{ *TokenStream }

type WordTokenizer struct {
	// contains filtered or unexported fields
}