etl

package
v0.0.0-debug-20260702 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 2, 2026 License: Apache-2.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const ETLMergeTask = "ETLMergeTask"

ETLMergeTask name for cron task in mo_task.sys_cron_task

View Source 
const MAX_INSERT_TIME = 3 * time.Second

Variables

This section is empty.

Functions

func GetVectorArrayLen 

func GetVectorArrayLen(ctx context.Context, vec *vector.Vector) (int, error)

func ValToString 

func ValToString(ctx context.Context, vec *vector.Vector, rowIdx int) (string, error)

Types

type BufWriter added in v0.8.0 

type BufWriter struct {
	// contains filtered or unexported fields
}

func NewBufWriter added in v0.8.0 

func NewBufWriter(ctx context.Context, writer io.Writer) *BufWriter

func (*BufWriter) Close added in v0.8.0 

func (w *BufWriter) Close() error

func (*BufWriter) Write added in v0.8.0 

func (w *BufWriter) Write(p []byte) (n int, err error)

type CSVWriter 

type CSVWriter struct {
	// contains filtered or unexported fields
}

func NewCSVWriter 

func NewCSVWriter(ctx context.Context, writer io.StringWriter) *CSVWriter

func (*CSVWriter) FlushAndClose 

func (w *CSVWriter) FlushAndClose() (int, error)

func (*CSVWriter) FlushBuffer 

func (w *CSVWriter) FlushBuffer(buf *bytes.Buffer) (int, error)

FlushBuffer flush the input buf content into file. The writer should NOT call function WriteRow, WriteStrings, FlushAndClose.

func (*CSVWriter) GetContent 

func (w *CSVWriter) GetContent() string

func (*CSVWriter) GetContentLength 

func (w *CSVWriter) GetContentLength() int

func (*CSVWriter) WriteRow 

func (w *CSVWriter) WriteRow(row *table.Row) error

func (*CSVWriter) WriteStrings 

func (w *CSVWriter) WriteStrings(record []string) error

type ContentFormatter 

type ContentFormatter struct {
	Formatter
}

ContentFormatter common File Format

type ContentWriter 

type ContentWriter struct {
	// contains filtered or unexported fields
}

ContentWriter NO do gen op, just do the flush op.

func NewContentWriter 

func NewContentWriter(ctx context.Context, tbl *table.Table, fileFlusher table.Flusher) *ContentWriter

func (*ContentWriter) FlushAndClose 

func (c *ContentWriter) FlushAndClose() (n int, err error)

func (*ContentWriter) GetContent 

func (c *ContentWriter) GetContent() string

func (*ContentWriter) GetContentLength 

func (c *ContentWriter) GetContentLength() int

func (*ContentWriter) NeedBuffer 

func (c *ContentWriter) NeedBuffer() bool

NeedBuffer implements table.BufferSettable

func (*ContentWriter) SetBuffer 

func (c *ContentWriter) SetBuffer(buf *bytes.Buffer, callback func(buffer *bytes.Buffer))

SetBuffer implements table.BufferSettable

func (*ContentWriter) SetupBackOff 

func (c *ContentWriter) SetupBackOff(backoff table.BackOff)

func (*ContentWriter) WriteRow 

func (c *ContentWriter) WriteRow(row *table.Row) error

WriteRow serialize the row into buffer It new a formatter to serialize the row.

type DefaultSqlWriter added in v0.8.0 

type DefaultSqlWriter struct {
	// contains filtered or unexported fields
}

DefaultSqlWriter SqlWriter is a writer that writes data to a SQL database.

func NewSqlWriter added in v0.8.0 

func NewSqlWriter(ctx context.Context, tbl *table.Table, csv *CSVWriter) *DefaultSqlWriter

func (*DefaultSqlWriter) FlushAndClose added in v0.8.0 

func (sw *DefaultSqlWriter) FlushAndClose() (int, error)

func (*DefaultSqlWriter) GetContent added in v0.8.0 

func (sw *DefaultSqlWriter) GetContent() string

func (*DefaultSqlWriter) GetContentLength 

func (sw *DefaultSqlWriter) GetContentLength() int

func (*DefaultSqlWriter) WriteRow added in v0.8.0 

func (sw *DefaultSqlWriter) WriteRow(row *table.Row) error

func (*DefaultSqlWriter) WriteStrings added in v0.8.0 

func (sw *DefaultSqlWriter) WriteStrings(record []string) error

type FSWriter 

type FSWriter struct {
	// contains filtered or unexported fields
}

func NewFSWriter 

func NewFSWriter(ctx context.Context, fs fileservice.FileService, opts ...FSWriterOption) *FSWriter

func (*FSWriter) Write 

func (w *FSWriter) Write(p []byte) (n int, err error)

Write implement io.Writer, Please execute in series

func (*FSWriter) WriteString 

func (w *FSWriter) WriteString(s string) (n int, err error)

WriteString implement io.StringWriter

type FSWriterOption 

type FSWriterOption func(*FSWriter)

func WithFilePath 

func WithFilePath(filepath string) FSWriterOption

func (FSWriterOption) Apply 

func (f FSWriterOption) Apply(w *FSWriter)

type FileFlusher 

type FileFlusher struct {
	// contains filtered or unexported fields
}

func NewFileContentFlusher 

func NewFileContentFlusher(writer table.Flusher) *FileFlusher

func (*FileFlusher) FlushBuffer 

func (f *FileFlusher) FlushBuffer(buf *bytes.Buffer) (int, error)

type Formatter 

type Formatter interface {
	WriteRow(*table.Row) error
	Flush()
}

func NewContentFormatter 

func NewContentFormatter(ctx context.Context, buf *bytes.Buffer) Formatter

type SQLFlusher 

type SQLFlusher struct {
	// contains filtered or unexported fields
}

func NewSQLFlusher 

func NewSQLFlusher(tbl *table.Table) *SQLFlusher

func (*SQLFlusher) FlushBuffer 

func (f *SQLFlusher) FlushBuffer(buf *bytes.Buffer) (int, error)

type TAEReader 

type TAEReader struct {
	// contains filtered or unexported fields
}

TAEReader implements the io.Reader interface for reading a tae file. Deprecated

func NewTaeReader 

func NewTaeReader(ctx context.Context, tbl *table.Table, filePath string, filesize int64, fs fileservice.FileService, mp *mpool.MPool) (*TAEReader, error)

NewTaeReader returns a TAEReader. Deprecated

func (*TAEReader) Close 

func (r *TAEReader) Close()

func (*TAEReader) ReadAll 

func (r *TAEReader) ReadAll(ctx context.Context) ([]*batch.Batch, error)

func (*TAEReader) ReadLine 

func (r *TAEReader) ReadLine() ([]string, error)

func (*TAEReader) ReadRow 

func (r *TAEReader) ReadRow(row *table.Row) error

type TAEWriter 

type TAEWriter struct {
	// contains filtered or unexported fields
}

TAEWriter implements table.RowWriter and writes data to a tae file. Deprecated

func NewTAEWriter 

func NewTAEWriter(ctx context.Context, tbl *table.Table, mp *mpool.MPool, filePath string, fs fileservice.FileService) *TAEWriter

NewTAEWriter returns a new instance of TAEWriter Deprecated

func (*TAEWriter) FlushAndClose 

func (w *TAEWriter) FlushAndClose() (int, error)

FlushAndClose implement ETLWriter

func (*TAEWriter) GetContent 

func (w *TAEWriter) GetContent() string

func (*TAEWriter) GetContentLength 

func (w *TAEWriter) GetContentLength() int

func (*TAEWriter) WriteRow 

func (w *TAEWriter) WriteRow(row *table.Row) error

WriteRow implement ETLWriter

func (*TAEWriter) WriteStrings 

func (w *TAEWriter) WriteStrings(Line []string) error

WriteStrings implement ETLWriter

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.