etl

package

v0.7.0 Latest Latest Go to latest Published: Feb 18, 2023 License: Apache-2.0 Imports: 18 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/matrixorigin/matrixone

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func GetVectorArrayLen(ctx context.Context, vec *vector.Vector) (int, error)
func Time2DatetimeString(t time.Time) string
func ValToString(ctx context.Context, vec *vector.Vector, rowIdx int) (string, error)
type CSVWriter
- func NewCSVWriter(ctx context.Context, buf *bytes.Buffer, writer io.StringWriter) *CSVWriter
- func (w *CSVWriter) FlushAndClose() (int, error)
- func (w *CSVWriter) GetContent() string
- func (w *CSVWriter) WriteRow(row *table.Row) error
- func (w *CSVWriter) WriteStrings(record []string) error
type FSWriter
- func NewFSWriter(ctx context.Context, fs fileservice.FileService, opts ...FSWriterOption) *FSWriter
- func (w *FSWriter) Write(p []byte) (n int, err error)
- func (w *FSWriter) WriteString(s string) (n int, err error)
type FSWriterOption
- func WithFilePath(filepath string) FSWriterOption
- func (f FSWriterOption) Apply(w *FSWriter)
type TAEReader
- func NewTaeReader(ctx context.Context, tbl *table.Table, filePath string, filesize int64, ...) (*TAEReader, error)
- func (r *TAEReader) Close()
- func (r *TAEReader) ReadAll(ctx context.Context) ([]*batch.Batch, error)
- func (r *TAEReader) ReadLine() ([]string, error)
- func (r *TAEReader) ReadRow(row *table.Row) error
type TAEWriter
- func NewTAEWriter(ctx context.Context, tbl *table.Table, mp *mpool.MPool, filePath string, ...) *TAEWriter
- func (w *TAEWriter) FlushAndClose() (int, error)
- func (w *TAEWriter) GetContent() string
- func (w *TAEWriter) WriteRow(row *table.Row) error
- func (w *TAEWriter) WriteStrings(Line []string) error

Constants ¶

View Source

const BatchSize = 8192

Variables ¶

View Source

var QuoteFieldFunc = func(ctx context.Context, buf *bytes.Buffer, value string, enclose rune) string {
	replaceRules := map[rune]string{
		'"':  `""`,
		'\'': `\'`,
	}
	quotedClose, hasRule := replaceRules[enclose]
	if !hasRule {
		panic(moerr.NewInternalError(ctx, "not support csv enclose: %c", enclose))
	}
	for _, c := range value {
		if c == enclose {
			buf.WriteString(quotedClose)
		} else {
			buf.WriteRune(c)
		}
	}
	return value
}

Functions ¶

func GetVectorArrayLen ¶

func GetVectorArrayLen(ctx context.Context, vec *vector.Vector) (int, error)

func Time2DatetimeString ¶

func Time2DatetimeString(t time.Time) string

func ValToString ¶

func ValToString(ctx context.Context, vec *vector.Vector, rowIdx int) (string, error)

Types ¶

type CSVWriter ¶

type CSVWriter struct {
	// contains filtered or unexported fields
}

func NewCSVWriter ¶

func NewCSVWriter(ctx context.Context, buf *bytes.Buffer, writer io.StringWriter) *CSVWriter

func (*CSVWriter) FlushAndClose ¶

func (w *CSVWriter) FlushAndClose() (int, error)

func (*CSVWriter) GetContent ¶

func (w *CSVWriter) GetContent() string

func (*CSVWriter) WriteRow ¶

func (w *CSVWriter) WriteRow(row *table.Row) error

func (*CSVWriter) WriteStrings ¶

func (w *CSVWriter) WriteStrings(record []string) error

type FSWriter ¶

type FSWriter struct {
	// contains filtered or unexported fields
}

func NewFSWriter ¶

func NewFSWriter(ctx context.Context, fs fileservice.FileService, opts ...FSWriterOption) *FSWriter

func (*FSWriter) Write ¶

func (w *FSWriter) Write(p []byte) (n int, err error)

Write implement io.Writer, Please execute in series

func (*FSWriter) WriteString ¶

func (w *FSWriter) WriteString(s string) (n int, err error)

WriteString implement io.StringWriter

type FSWriterOption ¶

type FSWriterOption func(*FSWriter)

func WithFilePath ¶

func WithFilePath(filepath string) FSWriterOption

func (FSWriterOption) Apply ¶

func (f FSWriterOption) Apply(w *FSWriter)

type TAEReader ¶

type TAEReader struct {
	// contains filtered or unexported fields
}

func NewTaeReader ¶

func NewTaeReader(ctx context.Context, tbl *table.Table, filePath string, filesize int64, fs fileservice.FileService, mp *mpool.MPool) (*TAEReader, error)

func (*TAEReader) Close ¶

func (r *TAEReader) Close()

func (*TAEReader) ReadAll ¶

func (r *TAEReader) ReadAll(ctx context.Context) ([]*batch.Batch, error)

func (*TAEReader) ReadLine ¶

func (r *TAEReader) ReadLine() ([]string, error)

func (*TAEReader) ReadRow ¶

func (r *TAEReader) ReadRow(row *table.Row) error

type TAEWriter ¶

type TAEWriter struct {
	// contains filtered or unexported fields
}

func NewTAEWriter ¶

func NewTAEWriter(ctx context.Context, tbl *table.Table, mp *mpool.MPool, filePath string, fs fileservice.FileService) *TAEWriter

func (*TAEWriter) FlushAndClose ¶

func (w *TAEWriter) FlushAndClose() (int, error)

FlushAndClose implement ETLWriter

func (*TAEWriter) GetContent ¶

func (w *TAEWriter) GetContent() string

func (*TAEWriter) WriteRow ¶

func (w *TAEWriter) WriteRow(row *table.Row) error

WriteRow implement ETLWriter

func (*TAEWriter) WriteStrings ¶

func (w *TAEWriter) WriteStrings(Line []string) error

WriteStrings implement ETLWriter

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL