importer

type ASTArgs struct {
	FileLocRef         ast.FileLocRefTp
	ColumnsAndUserVars []*ast.ColumnNameOrUserVar
	ColumnAssignments  []*ast.Assignment
	OnDuplicate        ast.OnDuplicateKeyHandlingType
	FieldsInfo         *ast.FieldsClause
	LinesInfo          *ast.LinesClause
}

type Chunk struct {
	Path         string
	FileSize     int64
	Offset       int64
	EndOffset    int64
	PrevRowIDMax int64
	RowIDMax     int64
	Type         mydump.SourceType
	Compression  mydump.Compression
	Timestamp    int64
	ParquetMeta  mydump.ParquetFileMeta
}

type ChunkProcessor interface {
	Process(ctx context.Context) error
}

type DataSourceType string

type FieldMapping struct {
	Column  *table.Column
	UserVar *ast.VariableExpr
}

type GenKVIndex struct {
	Unique bool
	// contains filtered or unexported fields
}

type ImportParameters struct {
	ColumnsAndVars string `json:"columns-and-vars,omitempty"`
	SetClause      string `json:"set-clause,omitempty"`
	// for s3 URL, AK/SK is redacted for security
	FileLocation string `json:"file-location"`
	Format       string `json:"format"`
	// only include what user specified, not include default value.
	Options map[string]any `json:"options,omitempty"`
}

type IndexRouteWriter struct {
	// contains filtered or unexported fields
}

type JobInfo struct {
	ID             int64
	CreateTime     types.Time
	StartTime      types.Time
	UpdateTime     types.Time
	EndTime        types.Time
	TableSchema    string
	TableName      string
	TableID        int64
	CreatedBy      string
	Parameters     ImportParameters
	SourceFileSize int64
	Status         string
	// Step corresponds to the `phase` field in `SHOW IMPORT JOB`
	// Here we just use the same name as in distributed framework.
	Step string
	// The summary of the job, it will store info for each step of the import and
	// will be updated when switching to a new step.
	// If the ingest step is finished, the number of ingested rows will also stored in it.
	Summary      *Summary
	ErrorMessage string
	GroupKey     string
}

type LoadDataController struct {
	*Plan
	*ASTArgs

	Table table.Table

	// how input field(or input column) from data file is mapped, either to a column or variable.
	// if there's NO column list clause in SQL statement, then it's table's columns
	// else it's user defined list.
	FieldMappings []*FieldMapping
	// InsertColumns the columns stated in the SQL statement to insert.
	// as IMPORT INTO have 2 place to state columns, in column-vars and in set clause,
	// so it's computed from both clauses:
	//  - append columns from column-vars to InsertColumns
	//  - append columns from left hand of set clause to InsertColumns
	// it's similar to InsertValues.InsertColumns.
	// Note: our behavior is different with mysql. such as for table t(a,b)
	// - "...(a,a) set a=100" is allowed in mysql, but not in tidb
	// - "...(a,b) set b=100" will set b=100 in mysql, but in tidb the set is ignored.
	// - ref columns in set clause is allowed in mysql, but not in tidb
	InsertColumns []*table.Column

	// ExecuteNodesCnt is the count of execute nodes.
	ExecuteNodesCnt int
	// contains filtered or unexported fields
}

type LoadDataReaderInfo struct {
	// Opener can be called at needed to get a io.ReadSeekCloser. It will only
	// be called once.
	Opener func(ctx context.Context) (io.ReadSeekCloser, error)
	// Remote is not nil only if load from cloud storage.
	Remote *mydump.SourceFileMeta
}

type Option func(c *LoadDataController)

type Plan struct {
	DBName string
	DBID   int64
	// TableInfo is the table info we used during import, we might change it
	// if add index by SQL is enabled(it's disabled now).
	TableInfo *model.TableInfo
	// DesiredTableInfo is the table info before import, and the desired table info
	// after import.
	DesiredTableInfo *model.TableInfo

	Path string
	// only effective when data source is file.
	Format string
	// Data interpretation is restrictive if the SQL mode is restrictive and neither
	// the IGNORE nor the LOCAL modifier is specified. Errors terminate the load
	// operation.
	// ref https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-column-assignments
	Restrictive bool

	// Location is used to convert time type for parquet, see
	// https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp
	Location *time.Location

	SQLMode mysql.SQLMode
	// Charset is the charset of the data file when file is CSV or TSV.
	// it might be nil when using LOAD DATA and no charset is specified.
	// for IMPORT INTO, it is always non-nil and default to be defaultCharacterSet.
	Charset          *string
	ImportantSysVars map[string]string

	// used for LOAD DATA and CSV format of IMPORT INTO
	FieldNullDef []string
	// this is not used in IMPORT INTO
	NullValueOptEnclosed bool
	// LinesStartingBy is not used in IMPORT INTO
	// FieldsOptEnclosed is not used in either IMPORT INTO or LOAD DATA
	plannercore.LineFieldsInfo
	IgnoreLines uint64

	DiskQuota             config.ByteSize
	Checksum              config.PostOpLevel
	ThreadCnt             int
	MaxNodeCnt            int
	MaxWriteSpeed         config.ByteSize
	SplitFile             bool
	MaxRecordedErrors     int64
	Detached              bool
	DisableTiKVImportMode bool
	MaxEngineSize         config.ByteSize
	CloudStorageURI       string
	DisablePrecheck       bool
	GroupKey              string

	// used for checksum in physical mode
	DistSQLScanConcurrency int

	// todo: remove it when load data code is reverted.
	InImportInto   bool
	DataSourceType DataSourceType
	// only initialized for IMPORT INTO, used when creating job.
	Parameters *ImportParameters `json:"-"`

	// the user who executes the statement, in the form of user@host
	// only initialized for IMPORT INTO
	User string `json:"-"`

	IsRaftKV2 bool
	// total data file size in bytes.
	TotalFileSize int64
	// used in tests to force enable merge-step when using global sort.
	ForceMergeStep bool
	// see ManualRecovery in proto.ExtraParams
	ManualRecovery bool
	// the keyspace name when submitting this job, only for import-into
	Keyspace string
	// contains filtered or unexported fields
}

type QueryChunk struct {
	Fields      []*types.FieldType
	Chk         *chunk.Chunk
	RowIDOffset int64
}

type StepSummary struct {
	Bytes  int64 `json:"input-bytes,omitempty"`
	RowCnt int64 `json:"input-rows,omitempty"`
}

type Summary struct {
	// EncodeSummary stores the bytes and rows needed to be processed in encode step.
	// Same for other summaries.
	EncodeSummary StepSummary `json:"encode-summary,omitempty"`

	MergeSummary StepSummary `json:"merge-summary,omitempty"`

	IngestSummary StepSummary `json:"ingest-summary,omitempty"`

	// ImportedRows is the number of rows imported into TiKV.
	// conflicted rows are excluded from this count if using global-sort.
	ImportedRows   int64  `json:"row-count,omitempty"`
	ConflictRowCnt uint64 `json:"conflict-row-count,omitempty"`
	// TooManyConflicts indicates there are too many conflicted rows that we
	// cannot deduplicate during collecting its checksum, so we will skip later
	// checksum step.
	TooManyConflicts bool `json:"too-many-conflicts,omitempty"`
}

type TableImporter struct {
	*LoadDataController
	// contains filtered or unexported fields
}

type TableKVEncoder struct {
	*kv.BaseKVEncoder
	// contains filtered or unexported fields
}

type WriterFactory func(indexID int64) (*external.Writer, error)