importinto

type Checksum struct {
	Sum  uint64
	KVs  uint64
	Size uint64
}

type Chunk struct {
	Path         string
	FileSize     int64
	Offset       int64
	EndOffset    int64
	PrevRowIDMax int64
	RowIDMax     int64
	Type         mydump.SourceType
	Compression  mydump.Compression
	Timestamp    int64
}

type CollectConflictsSpec struct {
	*CollectConflictsStepMeta
}

type CollectConflictsStepMeta struct {
	external.BaseExternalMeta
	Infos                   KVGroupConflictInfos `json:"infos" external:"true"`
	RecordedDataKVConflicts int64                `json:"recorded-data-kv-conflicts,omitempty"`
	// Checksum is the checksum of all conflicts rows.
	Checksum *Checksum `json:"checksum,omitempty"`
	// ConflictedRowCount is the count of all conflicted rows.
	ConflictedRowCount int64 `json:"conflicted-row-count,omitempty"`
	// ConflictedRowFilenames is the filenames of all conflicted rows.
	// Note: this file is for user to resolve conflicts manually.
	ConflictedRowFilenames []string `json:"conflicted-row-filenames,omitempty"`
	// TooManyConflictsFromIndex is true if there are too many conflicts from index.
	// if true, we will skip checksum.
	TooManyConflictsFromIndex bool `json:"too-many-conflicts-from-index,omitempty"`
}

type ConflictResolutionSpec struct {
	*ConflictResolutionStepMeta
}

type ConflictResolutionStepMeta struct {
	external.BaseExternalMeta
	Infos KVGroupConflictInfos `json:"infos" external:"true"`
}

type ImportCleanUpS3 struct {
}

type ImportSpec struct {
	*ImportStepMeta
	Plan importer.Plan
}

type ImportStepMeta struct {
	external.BaseExternalMeta
	// this is the engine ID, not the id in tidb_background_subtask table.
	ID       int32
	Chunks   []Chunk            `external:"true"`
	Checksum map[int64]Checksum // see KVGroupChecksum for definition of map key.
	Result   Result
	// MaxIDs stores the max id that have been used during encoding for each allocator type.
	// the max id is same among all allocator types for now, since we're using same base, see
	// NewPanickingAllocators for more info.
	MaxIDs map[autoid.AllocatorType]int64

	SortedDataMeta *external.SortedKVMeta `external:"true"`
	// SortedIndexMetas is a map from index id to its sorted kv meta.
	SortedIndexMetas map[int64]*external.SortedKVMeta `external:"true"`
	// it's the sum of all conflict KVs in all SortedKVMeta, we keep it here to
	// avoid get the external meta when no conflict KVs.
	RecordedConflictKVCount uint64
}

type KVGroupConflictInfos struct {
	ConflictInfos map[string]*common.ConflictInfo `json:"conflict-infos,omitempty"`
}

type LogicalPlan struct {
	JobID             int64
	Plan              importer.Plan
	Stmt              string
	EligibleInstances []*infosync.ServerInfo
	ChunkMap          map[int32][]Chunk
}

type MergeSortSpec struct {
	*MergeSortStepMeta
}

type MergeSortStepMeta struct {
	external.BaseExternalMeta
	// KVGroup is the group name of the sorted kv, either dataKVGroup or index-id.
	KVGroup                 string   `json:"kv-group"`
	DataFiles               []string `json:"data-files" external:"true"`
	external.SortedKVMeta   `external:"true"`
	RecordedConflictKVCount uint64 `json:"recorded-conflict-kv-count,omitempty"`
}

type MiniTaskExecutor interface {
	Run(ctx context.Context, dataWriter, indexWriter backend.EngineWriter) error
}

type PostProcessSpec struct {
	// for checksum request
	Schema string
	Table  string
}

type PostProcessStepMeta struct {
	// accumulated checksum of all subtasks in encode step. See KVGroupChecksum
	// for definition of map key.
	Checksum map[int64]Checksum
	// DeletedRowsChecksum is the checksum of all deleted rows due to conflicts.
	DeletedRowsChecksum Checksum
	// TooManyConflictsFromIndex is true if there are too many conflicts from index.
	// if true, the DeletedRowsChecksum might not be accurate, we will skip checksum.
	TooManyConflictsFromIndex bool `json:"too-many-conflicts-from-index,omitempty"`
	// MaxIDs of max all max-ids of subtasks in import step.
	MaxIDs map[autoid.AllocatorType]int64
}

type Result struct {
	LoadedRowCnt     uint64
	ConflictedRowCnt uint64
}

type SharedVars struct {
	TableImporter *importer.TableImporter
	DataEngine    *backend.OpenedEngine
	IndexEngine   *backend.OpenedEngine

	Checksum *verification.KVGroupChecksum

	SortedDataMeta *external.SortedKVMeta
	// SortedIndexMetas is a map from index id to its sorted kv meta.
	SortedIndexMetas        map[int64]*external.SortedKVMeta
	RecordedConflictKVCount uint64
	ShareMu                 sync.Mutex
	// contains filtered or unexported fields
}

type TaskMeta struct {
	// IMPORT INTO job id, see mysql.tidb_import_jobs.
	JobID  int64
	Plan   importer.Plan
	Stmt   string
	Result Result
	// eligible instances to run this task, we run on all instances if it's empty.
	// we only need this when run IMPORT INTO without distributed option now, i.e.
	// running on the instance that initiate the IMPORT INTO.
	EligibleInstances []*infosync.ServerInfo
	// the file chunks to import, when import from server file, we need to pass those
	// files to the framework scheduler which might run on another instance.
	// we use a map from engine ID to chunks since we need support split_file for CSV,
	// so need to split them into engines before passing to scheduler.
	ChunkMap map[int32][]Chunk
}

type WriteIngestSpec struct {
	*WriteIngestStepMeta
}

type WriteIngestStepMeta struct {
	external.BaseExternalMeta
	KVGroup                 string `json:"kv-group"`
	external.SortedKVMeta   `json:"sorted-kv-meta" external:"true"`
	RecordedConflictKVCount uint64   `json:"recorded-conflict-kv-count,omitempty"`
	DataFiles               []string `json:"data-files" external:"true"`
	StatFiles               []string `json:"stat-files" external:"true"`
	RangeJobKeys            [][]byte `json:"range-job-keys" external:"true"`
	RangeSplitKeys          [][]byte `json:"range-split-keys" external:"true"`
	TS                      uint64   `json:"ts"`

	Result Result
}