tokenize

type AdvancedSentenceTokenizer struct {
	// contains filtered or unexported fields
}

type BasicSentenceTokenizer struct{}

type BasicWordTokenizer struct{}

type BufferedTokenStream struct {
	// contains filtered or unexported fields
}

type SentenceStream interface {
	PushText(text string) error
	Flush() error
	EndInput() error
	AClose() error
	Close() error
	Next() (*TokenData, error)
	Closed() bool
}

type SentenceTokenizer interface {
	Tokenize(text string, language string) []string
	Stream(language string) SentenceStream
}

type TokenData struct {
	SegmentID string
	Token     string
	Start     int
	End       int
}

type WordStream interface {
	PushText(text string) error
	Flush() error
	EndInput() error
	AClose() error
	Close() error
	Next() (*TokenData, error)
	Closed() bool
}

type WordTokenizer interface {
	Tokenize(text string, language string) []string
	Stream(language string) WordStream
	FormatWords(words []string) string
}