lm

type Config struct {
	Name        string   `json:"name"`
	NGramOrder  uint8    `json:"nGramOrder"`
	SourcePath  string   `json:"source"`
	OutputPath  string   `json:"output"`
	Alphabet    []string `json:"alphabet"`
	Separators  []string `json:"separators"`
	StartSymbol string   `json:"startSymbol"`
	EndSymbol   string   `json:"endSymbol"`
	// contains filtered or unexported fields
}

type ContextOffset = uint32

type CountTrie interface {
	// Put increments WordCount for last element of given sequence.
	Put(sentence Sentence, count WordCount)
	// Walk iterates through trie and calls walker function on each element.
	Walk(walker TrieIterator) error
}

type Indexer interface {
	// Returns the index for the word, otherwise returns UnknownWordID
	Get(token Token) (WordID, error)
	// Find a token by the given index
	Find(id WordID) (Token, error)
}

type Key uint64

type LanguageModel interface {
	// ScoreSentence scores and returns a lm weight for the given sentence
	ScoreSentence(sentence Sentence) (float64, error)
	// ScoreWordIDs scores and returns a lm weight for the given sequence of nGrams
	ScoreWordIDs(sequence []WordID) float64
	// GetWordID returns id for the given token
	GetWordID(token Token) (WordID, error)
	// Next returns the list of candidates for the given sequence
	Next(sequence []WordID) (ScorerNext, error)
}

type NGramBuilder struct {
	// contains filtered or unexported fields
}

type NGramModel interface {
	store.Marshaler
	store.Unmarshaler

	// Score returns a lm value of the given sequence of WordID
	Score(nGrams []WordID) float64
	// Next returns a list of WordID which follow after the given sequence of nGrams
	Next(nGrams []WordID) (ScorerNext, error)
}

type NGramNode struct {
	Key   Key
	Count WordCount
}

type NGramReader interface {
	// Read builds NGramModel from the given list of readers
	Read() (NGramModel, error)
}

type NGramVector interface {
	store.Marshaler
	store.Unmarshaler

	// GetCount returns WordCount and Node ContextOffset for the given pair (word, context)
	GetCount(word WordID, context ContextOffset) (WordCount, ContextOffset)
	// GetContextOffset returns the given node context offset
	GetContextOffset(word WordID, context ContextOffset) ContextOffset
	// CorpusCount returns size of all counts in the collection
	CorpusCount() WordCount
	// SubVector returns NGramVector for the given context
	SubVector(context ContextOffset) NGramVector
}

type NGramVectorBuilder interface {
	// Put adds the given sequence of nGrams and count to model
	Put(nGrams []WordID, count WordCount) error
	// Build creates new instance of NGramVector
	Build() NGramVector
}

type NGramVectorFactory func(ch <-chan NGramNode) NGramVector

type NGramWriter interface {
	// Write persists the given trie to a storage
	Write(trie CountTrie) error
}

type NGrams = [][]WordID

type ScorerNext interface {
	// ScoreNext calculates the score for the given nGram built on the parent context
	ScoreNext(nGram WordID) float64
}

type Sentence = []Token

type SentenceRetriever interface {
	// Retrieves and returns the next sentence from the source
	Retrieve() Sentence
}

type Token = analysis.Token

type TrieIterator = func(path Sentence, count WordCount) error

type WordCount = uint32

type WordID = uint32