Versions in this module Expand all Collapse all v0 v0.8.8 Jul 5, 2026 v0.8.7 Jun 13, 2026 v0.8.6 Jun 11, 2026 v0.8.5 Jun 7, 2026 v0.8.4 Jun 5, 2026 Changes in this version + const RatioWord + const RatioWordFull + const Version + var StopWordMap = map[string]bool + var ToLower = true + func DictPaths(dictDir, filePath string) (files []string) + func FilterEmoji(text string) (new string) + func FilterHtml(text string) string + func FilterLang(text, lang string) (new string) + func FilterSymbol(text string) (new string) + func FindAllOccs(data []byte, searches []string) map[string][]int + func GetVersion() string + func IsJp(segText string) bool + func Join(text []Text) string + func Range(text string) (new []string) + func RangeText(text string) (new string) + func SplitNum(text string) []string + func SplitNums(text string) string + func ToSlice(segs []Segment, searchMode ...bool) (output []string) + func ToString(segs []Segment, searchMode ...bool) (output string) + type AnalyzeToken struct + End int + Freq float64 + Len int + Pos string + Position int + Start int + Text string + Type string + type Dictionary struct + Tokens []Token + func NewDict() *Dictionary + func (dict *Dictionary) AddToken(token Token) error + func (dict *Dictionary) Find(word []byte) (float64, string, bool) + func (dict *Dictionary) FindTFIDF(word []byte) (float64, float64, bool) + func (dict *Dictionary) LookupTokens(words []Text, tokens []*Token) (numOfTokens int) + func (dict *Dictionary) MaxTokenLen() int + func (dict *Dictionary) NumTokens() int + func (dict *Dictionary) RemoveToken(token Token) error + func (dict *Dictionary) TotalFreq() float64 + func (dict *Dictionary) Value(word []byte) (val, id int, err error) + type Prob struct + B map[rune]float64 + E map[rune]float64 + M map[rune]float64 + S map[rune]float64 + type SegPos struct + Pos string + Text string + func ToPos(segs []Segment, searchMode ...bool) (output []SegPos) + type Segment struct + Position int + func (s *Segment) End() int + func (s *Segment) Start() int + func (s *Segment) Token() *Token + type Segmenter struct + Alpha bool + AlphaNum bool + CorpusAverLen float64 + Dict *Dictionary + DictPath string + DictSep string + Load bool + LoadNoFreq bool + MinTokenFreq float64 + MoreLog bool + NotLoadHMM bool + NotStop bool + Num bool + SkipLog bool + SkipPos bool + StopWordMap map[string]bool + TextFreq string + func New(files ...string) (seg Segmenter, err error) + func NewEmbed(dict ...string) (seg Segmenter, err error) + func (seg *Segmenter) AddStop(text string) + func (seg *Segmenter) AddStopArr(text ...string) + func (seg *Segmenter) AddToken(text string, freq float64, pos ...string) error + func (seg *Segmenter) AddTokenForce(text string, freq float64, pos ...string) (err error) + func (seg *Segmenter) Analyze(text []string, t1 string, by ...bool) (az []AnalyzeToken) + func (seg *Segmenter) CalcToken() + func (seg *Segmenter) Cut(str string, hmm ...bool) []string + func (seg *Segmenter) CutAll(str string) []string + func (seg *Segmenter) CutDAG(str string, reg ...*regexp.Regexp) []string + func (seg *Segmenter) CutDAGNoHMM(str string) []string + func (seg *Segmenter) CutSearch(str string, hmm ...bool) []string + func (seg *Segmenter) CutStop(str string, hmm ...bool) []string + func (seg *Segmenter) CutStr(str []string, separator ...string) (r string) + func (seg *Segmenter) CutTrim(str string, hmm ...bool) []string + func (seg *Segmenter) CutTrimHtml(str string, hmm ...bool) []string + func (seg *Segmenter) CutTrimHtmls(str string, hmm ...bool) string + func (seg *Segmenter) CutUrl(str string, num ...bool) []string + func (seg *Segmenter) CutUrls(str string, num ...bool) string + func (seg *Segmenter) Dictionary() *Dictionary + func (seg *Segmenter) Empty() error + func (seg *Segmenter) EmptyStop() error + func (seg *Segmenter) Find(str string) (float64, string, bool) + func (seg *Segmenter) FindTFIDF(str string) (float64, float64, bool) + func (seg *Segmenter) GetCorpusPath(files ...string) []string + func (seg *Segmenter) GetCurrentFilePath() string + func (seg *Segmenter) GetIdfPath(files ...string) []string + func (seg *Segmenter) GetTfIdfPath(files ...string) []string + func (seg *Segmenter) HMMCut(str string, reg ...*regexp.Regexp) []string + func (seg *Segmenter) HMMCutMod(str string, prob ...map[rune]float64) []string + func (seg *Segmenter) Init() + func (seg *Segmenter) IsStop(s string) bool + func (seg *Segmenter) LoadCorpusAverLen(files ...string) (corpusTotal float64, err error) + func (seg *Segmenter) LoadDict(files ...string) error + func (seg *Segmenter) LoadDictEmbed(dict ...string) (err error) + func (seg *Segmenter) LoadDictMap(dict []map[string]string) error + func (seg *Segmenter) LoadDictStr(dict string) error + func (seg *Segmenter) LoadModel(prob ...map[rune]float64) + func (seg *Segmenter) LoadStop(files ...string) error + func (seg *Segmenter) LoadStopArr(dict []string) + func (seg *Segmenter) LoadStopEmbed(dict ...string) (err error) + func (seg *Segmenter) LoadStopStr(dict string) error + func (seg *Segmenter) LoadTFIDFDict(files []*types.LoadDictFile) error + func (seg *Segmenter) LoadTFIDFDictStr(dictFile *types.LoadDictFile) error + func (seg *Segmenter) ModeSegment(bytes []byte, searchMode ...bool) []Segment + func (seg *Segmenter) Pos(s string, searchMode ...bool) []SegPos + func (seg *Segmenter) PosStr(str []SegPos, separator ...string) (r string) + func (seg *Segmenter) PosTrim(str string, search bool, pos ...string) []SegPos + func (seg *Segmenter) PosTrimArr(str string, search bool, pos ...string) (re []string) + func (seg *Segmenter) PosTrimStr(str string, search bool, pos ...string) string + func (seg *Segmenter) ReAddToken(text string, freq float64, pos ...string) error + func (seg *Segmenter) Read(file string) error + func (seg *Segmenter) ReadCorpus(file string) (corpusAverLen float64, err error) + func (seg *Segmenter) ReadN(reader *bufio.Reader) (size int, text, freqText, pos string, fsErr error) + func (seg *Segmenter) ReadNTFIDF(reader *bufio.Reader) (size int, text, freqText, idfText string, fsErr error) + func (seg *Segmenter) ReadTFIDF(file string) error + func (seg *Segmenter) Reader(reader *bufio.Reader, files ...string) error + func (seg *Segmenter) ReaderTFIDF(reader *bufio.Reader, files ...string) error + func (seg *Segmenter) RemoveStop(text string) + func (seg *Segmenter) RemoveToken(text string) error + func (seg *Segmenter) Segment(bytes []byte) []Segment + func (seg *Segmenter) Size(size int, text, freqText string) (freq float64) + func (seg *Segmenter) Slice(s string, searchMode ...bool) []string + func (seg *Segmenter) SplitTextToWords(text Text) []Text + func (seg *Segmenter) Stop(s []string) (r []string) + func (seg *Segmenter) String(s string, searchMode ...bool) string + func (seg *Segmenter) SuggestFreq(words ...string) float64 + func (seg *Segmenter) ToToken(text string, freq float64, pos ...string) Token + func (seg *Segmenter) Trim(s []string) (r []string) + func (seg *Segmenter) TrimPos(s []SegPos) (r []SegPos) + func (seg *Segmenter) TrimPosPunct(se []SegPos) (re []SegPos) + func (seg *Segmenter) TrimPunct(s []string) (r []string) + func (seg *Segmenter) TrimSymbol(s []string) (r []string) + func (seg *Segmenter) TrimWithPos(se []SegPos, pos ...string) (re []SegPos) + func (seg *Segmenter) Value(str string) (int, int, error) + type Text []byte + func SplitWords(text Text) []Text + type Token struct + func (token *Token) Equals(str string) bool + func (token *Token) Freq() float64 + func (token *Token) Pos() string + func (token *Token) Segments() []*Segment + func (token *Token) Text() string
Go to main page