external

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 29, 2022 License: Apache-2.0 Imports: 31 Imported by: 0

Documentation

Index

Constants

View Source
const NULL_FLAG = "\\N"

Variables

View Source
var (
	ONE_BATCH_MAX_ROW = 40000
)

Functions

func Call

func Call(idx int, proc *process.Process, arg any) (bool, error)

func GetBatchData

func GetBatchData(param *ExternalParam, plh *ParseLineHandler, proc *process.Process) (*batch.Batch, error)

func GetForETLWithType

func GetForETLWithType(param *tree.ExternParam, prefix string) (res fileservice.ETLFileService, readPath string, err error)

func InitS3Param

func InitS3Param(param *tree.ExternParam) error

func Prepare

func Prepare(proc *process.Process, arg any) error

func ReadDir

func ReadDir(param *tree.ExternParam) (fileList []string, err error)

func ReadFile

func ReadFile(param *tree.ExternParam) (io.ReadCloser, error)

func ScanFileData

func ScanFileData(param *ExternalParam, proc *process.Process) (*batch.Batch, error)

ScanFileData read batch data from external file

func String

func String(arg any, buf *bytes.Buffer)

Types

type Argument

type Argument struct {
	Es *ExternalParam
}

func (*Argument) Free

func (arg *Argument) Free(proc *process.Process, pipelineFailed bool)

type ExternalFileparam

type ExternalFileparam struct {
	End       bool
	FileCnt   int
	FileFin   int
	FileIndex int
}

type ExternalParam

type ExternalParam struct {
	Attrs         []string
	Cols          []*plan.ColDef
	Name2ColIndex map[string]int32
	CreateSql     string
	Ctx           context.Context

	IgnoreLine    int
	IgnoreLineTag int
	// tag indicate the fileScan is finished
	Fileparam *ExternalFileparam
	FileList  []string
	// contains filtered or unexported fields
}

Use for External table scan param

type ParseLineHandler

type ParseLineHandler struct {
	// contains filtered or unexported fields
}

func GetSimdcsvReader

func GetSimdcsvReader(param *ExternalParam) (*ParseLineHandler, error)

GetSimdcsvReader get file reader from external file

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL