preprocessor

package
v0.2.6 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 6, 2021 License: MIT Imports: 11 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ConvertDivisorToFloat32SliceTensor

func ConvertDivisorToFloat32SliceTensor(columns interface{}) (*tf.Tensor, error)

func ConvertInt32SliceToTensor

func ConvertInt32SliceToTensor(columns interface{}) (*tf.Tensor, error)

func ConvertTokenizerToFloat32SliceTensor

func ConvertTokenizerToFloat32SliceTensor(columns interface{}) (*tf.Tensor, error)

func ConvertTokenizerToInt32SliceTensor

func ConvertTokenizerToInt32SliceTensor(columns interface{}) (*tf.Tensor, error)

func ReadCsvFloat32s

func ReadCsvFloat32s(columns []string) interface{}

func ReadCsvInt32s

func ReadCsvInt32s(columns []string) interface{}

func ReadStringNop

func ReadStringNop(columns []string) interface{}

Types

type Processor

type Processor struct {
	Name       string
	LineOffset int
	// TODO: find a nicer way to capture multiple columns in a row
	DataLength  int
	RequiresFit bool
	// contains filtered or unexported fields
}

func NewProcessor

func NewProcessor(
	errorHandler *cberrors.ErrorsContainer,
	cacheDir,
	name string,
	lineOffset int,
	dataLength int,
	requiresFit bool,
	divisor *RegressionDivisor,
	tokenizer *Tokenizer,
	reader func(column []string) interface{},
	converter func(column interface{}) (*tf.Tensor, error),
) *Processor

func (*Processor) FinishFit

func (p *Processor) FinishFit() error

func (*Processor) Fit

func (p *Processor) Fit(column []string) error

func (*Processor) Load

func (p *Processor) Load() error

func (*Processor) Process

func (p *Processor) Process(columnRows []string) (*tf.Tensor, error)

type RegressionDivisor

type RegressionDivisor struct {
	// contains filtered or unexported fields
}

func NewDivisor

func NewDivisor(
	errorHandler *cberrors.ErrorsContainer,
) *RegressionDivisor

func (*RegressionDivisor) Divide

func (r *RegressionDivisor) Divide(input []float32) ([]float32, error)

func (*RegressionDivisor) Fit

func (r *RegressionDivisor) Fit(input []float32)

func (*RegressionDivisor) Load

func (r *RegressionDivisor) Load(configFile string) error

func (*RegressionDivisor) Save

func (r *RegressionDivisor) Save(configFile string) error

type Tokenizer

type Tokenizer struct {
	// contains filtered or unexported fields
}

func NewTokenizer

func NewTokenizer(
	errorHandler *cberrors.ErrorsContainer,
	maxLen int,
	numWords int,
	isCategoryTokenizer bool,
) *Tokenizer

func (*Tokenizer) FinishFit

func (t *Tokenizer) FinishFit()

func (*Tokenizer) Fit

func (t *Tokenizer) Fit(sentence string)

func (*Tokenizer) Load

func (t *Tokenizer) Load(configFile string) error

func (*Tokenizer) Save

func (t *Tokenizer) Save(configFile string) error

func (*Tokenizer) Tokenize

func (t *Tokenizer) Tokenize(sentence string) []int32

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL