ingest

package

v0.1.2 Latest Latest Go to latest Published: Sep 27, 2022 License: BSD-3-Clause Imports: 20 Imported by: 1

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/RediSearch/RediSearchBenchmark

Links

Open Source Insights

Documentation ¶

Index ¶

func ReadDir(dirName string, pattern string, r DocumentReader, idx index.Index, ...)
func ReadFile(fileName string, r DocumentReader, idx index.Index, ac index.Autocompleter, ...) error
func ReadTerms(fileName string, r DocumentReader, idx index.Index, chunk int, ...) (finalTerms []string, err error)
type DocumentReader
type PmcReader
- func (rr *PmcReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error
type RedditReader
- func (rr *RedditReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error
type Stats
type WikipediaAbstractsReader
- func NewWikipediaAbstractsReader() *WikipediaAbstractsReader
- func (r *WikipediaAbstractsReader) LoadScores(fileName string) error
- func (wr *WikipediaAbstractsReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

func ReadDir ¶

func ReadDir(dirName string, pattern string, r DocumentReader, idx index.Index, ac index.Autocompleter,
	opts interface{}, chunk int, workers int, conns int, stats chan Stats, maxDocsToRead int)

ReadDir reads a complete directory and feeds each file it finds to a document reader

func ReadFile ¶

func ReadFile(fileName string, r DocumentReader, idx index.Index, ac index.Autocompleter,
	opts interface{}, chunk int, maxDocsToRead int, indexingWorkers int) error

IngestDocuments ingests documents into an index using a DocumentReader

func ReadTerms ¶

func ReadTerms(fileName string, r DocumentReader, idx index.Index, chunk int, maxDocsToRead int, maxTermsToProduce int, propertyName string, termStopWords []string) (finalTerms []string, err error)

IngestDocuments ingests documents into an index using a DocumentReader

Types ¶

type DocumentReader ¶

type DocumentReader interface {
	Read(io.Reader, chan index.Document, int, index.Index) error
}

DocumentReader implements parsing a data source and yielding documents

type PmcReader ¶

type PmcReader struct{}

func (*PmcReader) Read ¶

func (rr *PmcReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error

type RedditReader ¶

type RedditReader struct{}

func (*RedditReader) Read ¶

func (rr *RedditReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error

type Stats ¶

type Stats struct {
	TotalDocs             int64
	CurrentWindowDocs     int
	CurrentWindowDuration time.Duration
	CurrentWindowRate     float64
	CurrentWindowLatency  time.Duration
}

type WikipediaAbstractsReader ¶

type WikipediaAbstractsReader struct {
	// contains filtered or unexported fields
}

func NewWikipediaAbstractsReader ¶

func NewWikipediaAbstractsReader() *WikipediaAbstractsReader

func (*WikipediaAbstractsReader) LoadScores ¶

func (r *WikipediaAbstractsReader) LoadScores(fileName string) error

func (*WikipediaAbstractsReader) Read ¶

func (wr *WikipediaAbstractsReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL