ngrams

func CalcWeights ¶

func CalcWeights(ngmap map[string]*NGram, N int)

Calculate weights for ngrams found in a map of NGrams across N documents To be used after all document NGram maps are merged

func CosineSimilarity ¶

func CosineSimilarity(ngmap map[string]*NGram)

Compute cosine similarity between all document pairs

func Count ¶

func Count(ngrams map[string]*NGram) map[string]int

Count occurences of each NGram in a map of NGrams

func FilterMeaningfulNGrams ¶

func FilterMeaningfulNGrams(ngmap map[string]*NGram, minDF int, maxDF int, minAvgTFIDF float64) []string

func Generate ¶

func Generate(tokens []lexer.Lexeme, nrange []int, path string) map[string]*NGram

Generate NGrams from a slice of document lexemes

func Merge ¶

func Merge(maps ...map[string]*NGram)

Merge 2 or more string->*NGram maps

func MergeStopWords ¶

func MergeStopWords()

Merge custom stop words from config with stop words slice

func OrderByFrequency ¶

func OrderByFrequency(m map[string]*NGram) []struct {
	Key   string
	Value float64
}

Create an slice of NGrams keywords, ordered by their weights