letarette

package
v0.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 5, 2020 License: Apache-2.0 Imports: 40 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var ErrStemmerSettingsMismatch = fmt.Errorf("Config does not match index state")

ErrStemmerSettingsMismatch is returned when config and index state does not match

View Source
var Revision = ""

Revision is set the the current git sha, or "dev"

View Source
var Tag = ""

Tag is set to the current git tag, or the current date

Functions

func CheckIndex

func CheckIndex(dbo Database) error

CheckIndex runs an integrity check on the index

func CheckStemmerSettings

func CheckStemmerSettings(db Database, cfg Config) error

CheckStemmerSettings verifies that the index stemmer settings match the current config. If there are no index settings, they will be set from the provided config.

func ExposeMetrics

func ExposeMetrics(port uint16)

ExposeMetrics is a test for exposing metrics

func ForceIndexStemmerState

func ForceIndexStemmerState(state snowball.Settings, dbo Database) error

ForceIndexStemmerState resets the stemmer state stored in the database to the provided state.

func GetSpellfixLag

func GetSpellfixLag(ctx context.Context, dbo Database, minCount int) (int, error)

GetSpellfixLag returns how many words in the main index that are not yet in the spelling index.

func RebuildIndex

func RebuildIndex(dbo Database) error

RebuildIndex rebuilds the fts index from the docs table

func ResetMigration

func ResetMigration(cfg Config, version int) error

ResetMigration forces the migration version of a db. It is typically used to back out of a failed migration. Note: no migration steps are actually performed, it only sets the version and resets the dirty flag.

func SetIndexPageSize

func SetIndexPageSize(dbo Database, pageSize int) error

SetIndexPageSize sets the max page size for future index allocations.

func UpdateSpellfix

func UpdateSpellfix(ctx context.Context, dbo Database, minCount int) error

UpdateSpellfix updates the spelling table with the top terms from the fts.

func Usage

func Usage()

Usage prints usage help to stdout

Types

type Cache

type Cache struct {
	// contains filtered or unexported fields
}

Cache keeps search results for a set duration before they are thrown out. The cache is also limited in size.

func NewCache

func NewCache(timeout time.Duration, maxSize uint64) *Cache

NewCache creates cache with a given timeout and max size.

func (*Cache) Get

func (cache *Cache) Get(query string, spaces []string, limit uint16, offset uint16) (protocol.SearchResult, bool)

Get fetches cached search results

func (*Cache) Put

func (cache *Cache) Put(query string, spaces []string, limit uint16, offset uint16, res protocol.SearchResult)

Put stores search results in the cache

type Config

type Config struct {
	Nats struct {
		URLS        []string `default:"nats://localhost:4222"`
		SeedFile    string
		RootCAs     []string
		Topic       string `default:"leta"`
		SearchGroup string `ignored:"true"`
	}
	Db struct {
		Path           string `default:"letarette.db"`
		CacheSizeMB    uint32 `default:"1024" desc:"internal"` // default 1G cache
		MMapSizeMB     uint32 `default:"0" desc:"internal"`    // no mmap by default
		ToolConnection bool   `ignored:"true"`
	}
	Index struct {
		Spaces         []string `required:"true" default:"docs"`
		ChunkSize      uint16   `default:"250"`
		MaxOutstanding uint16   `split_words:"true" default:"25"`
		Wait           struct {
			Interest        time.Duration `default:"5s"`
			DocumentRefetch time.Duration `default:"1s"`
			Document        time.Duration `default:"20s"`
			Cycle           time.Duration `default:"100ms"`
			EmptyCycle      time.Duration `default:"5s"`
		}
		Disable bool `default:"false"`
	}
	Spelling struct {
		MinFrequency int `split_words:"true" default:"5"`
		MaxLag       int `split_words:"true" default:"100"`
	}
	Stemmer struct {
		Languages        []string `split_words:"true" required:"true" default:"english"`
		RemoveDiacritics bool     `split_words:"true" default:"true"`
		TokenCharacters  string
		Separators       string
	}
	Search struct {
		Timeout        time.Duration `default:"4s"`
		Cap            int           `default:"10000"`
		CacheTimeout   time.Duration `split_words:"true" default:"10m"`
		CacheMaxsizeMB uint64        `split_words:"true" default:"250"`
		Disable        bool          `default:"false"`
		Strategy       int           `default:"1" desc:"internal"`
	}
	Shardgroup      string `default:"1/1"`
	ShardgroupSize  uint16 `ignored:"true"`
	ShardgroupIndex uint16 `ignored:"true"`
	MetricsPort     uint16 `split_words:"true" default:"8000" desc:"internal"`
}

Config holds the main configuration

func LoadConfig

func LoadConfig() (cfg Config, err error)

LoadConfig loads configuration variables from the environment and returns a fully populated Config instance.

type Database

type Database interface {
	Close() error
	RawQuery(string) ([]string, error)
}

Database is a live connection to a SQLite database file, providing access methods for all db interactions.

func OpenDatabase

func OpenDatabase(cfg Config) (Database, error)

OpenDatabase connects to a new or existing database and migrates the database up to the latest version.

type IndexOptimizer

type IndexOptimizer struct {
	// contains filtered or unexported fields
}

IndexOptimizer is used to run step-wise index optimization. The instance must be closed by calling Close() to return the database connection to the pool.

func StartIndexOptimization

func StartIndexOptimization(dbo Database, pageIncrement int) (*IndexOptimizer, error)

StartIndexOptimization initiates a step-wise index optimization and returns an IndexOptimizer instance on success.

func (IndexOptimizer) Close

func (o IndexOptimizer) Close() error

Close returns the database connection to the pool.

func (IndexOptimizer) Step

func (o IndexOptimizer) Step() (bool, error)

Step runs one step of the optimizer. Returns true when optimization is complete. Stopping before done is OK.

type Indexer

type Indexer interface {
	Close()
}

Indexer continuously runs the indexing process, until Close is called.

func StartIndexer

func StartIndexer(nc *nats.Conn, db Database, cfg Config) (Indexer, error)

StartIndexer creates and starts an indexer instance. This is really a singleton in that only one instance with the same database or config can be run at the same time.

type Interest

type Interest struct {
	DocID   protocol.DocumentID `db:"docID"`
	State   InterestState
	Updated int64 `db:"updatedNanos"`
}

Interest represents one row in the interest list

type InterestListState

type InterestListState struct {
	CreatedAt        int64               `db:"listCreatedAtNanos"`
	LastUpdated      int64               `db:"lastUpdatedAtNanos"`
	LastUpdatedDocID protocol.DocumentID `db:"lastUpdatedDocID"`
}

InterestListState keeps track of where the index process is

type InterestState

type InterestState int

InterestState represents the state of an interest

type Phrase

type Phrase struct {
	Text     string
	Wildcard bool
	Exclude  bool
}

Phrase represents one parsed query phrase, with flags

func CanonicalizePhraseList

func CanonicalizePhraseList(phrases []Phrase) []Phrase

CanonicalizePhraseList turns all phrases in a phrase list to lower case, sorts it and eliminates duplicates.

func ParseQuery

func ParseQuery(query string) []Phrase

ParseQuery tokenizes a query string and returns a list of parsed phrases with exclusion and wildcard flags.

func ReducePhraseList

func ReducePhraseList(phrases []Phrase) []Phrase

ReducePhraseList removes one character phrases from a list of phrases.

func (Phrase) String

func (p Phrase) String() string

type Searcher

type Searcher interface {
	Close()
}

Searcher continuously runs the search process, until Close is called.

func StartSearcher

func StartSearcher(nc *nats.Conn, db Database, cfg Config) (Searcher, error)

StartSearcher creates and starts a searcher instance.

type Stats

type Stats struct {
	Spaces []struct {
		Name  string
		State InterestListState
	}
	CommonTerms []struct {
		Term  string
		Count int
	}
	Terms   int
	Docs    int
	Stemmer snowball.Settings
}

Stats holds statistics gathered by GetIndexStats

func GetIndexStats

func GetIndexStats(dbo Database) (Stats, error)

GetIndexStats collects statistics about the index, partly by the use of the fts4vocab virtual table.

type StatusMonitor

type StatusMonitor interface {
	Close()
}

StatusMonitor communicates worker status with the cluster

func StartStatusMonitor

func StartStatusMonitor(nc *nats.Conn, db Database, cfg Config) (StatusMonitor, error)

StartStatusMonitor creates a new StatusMonitor, listening to status broadcasts and broadcasting our status.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL