dsort

type Algorithm struct {
	// one of the `algorithms` above
	Kind string `json:"kind"`

	// used with two sorting alg-s: Alphanumeric and Content
	Decreasing bool `json:"decreasing"`

	// when sort is a random shuffle
	Seed string `json:"seed"`

	// usage: exclusively for Content sorting
	// e.g.: ".cls" containing sorting key for each record (sample) - see next
	// NOTE: not to confuse with shards "input_extension"
	Ext string `json:"extension"`

	// ditto: Content only
	// `shard.contentKeyTypes` enum values: {"int", "string", "float" }
	ContentKeyType string `json:"content_key_type"`
}

type CreationPhaseMetadata struct {
	Shards    []*shard.Shard          `msg:"shards"`
	SendOrder map[string]*shard.Shard `msg:"send_order"`
}

type JobInfo struct {
	ID                string        `json:"id"` // job ID == xact ID (aka managerUUID)
	SrcBck            cmn.Bck       `json:"src-bck"`
	DstBck            cmn.Bck       `json:"dst-bck"`
	StartedTime       time.Time     `json:"started_time,omitempty"`
	FinishTime        time.Time     `json:"finish_time,omitempty"`
	ExtractedDuration time.Duration `json:"started_meta_sorting,omitempty"`
	SortingDuration   time.Duration `json:"started_shard_creation,omitempty"`
	CreationDuration  time.Duration `json:"finished_shard_creation,omitempty"`
	Objs              int64         `json:"loc-objs,string"`  // locally processed
	Bytes             int64         `json:"loc-bytes,string"` //
	Metrics           *Metrics
	Aborted           bool `json:"aborted"`
	Archived          bool `json:"archived"`
}

type LocalExtraction struct {

	// TotalCnt is the number of shards Dsort has to process in total.
	TotalCnt int64 `json:"total_count,string"`
	// ExtractedCnt is the cumulative number of extracted shards. In the
	// end, this should be roughly equal to TotalCnt/#Targets.
	ExtractedCnt int64 `json:"extracted_count,string"`
	// ExtractedSize is uncompressed size of extracted shards.
	ExtractedSize int64 `json:"extracted_size,string"`
	// ExtractedRecordCnt - number of records extracted from all shards.
	ExtractedRecordCnt int64 `json:"extracted_record_count,string"`
	// ExtractedToDiskCnt describes number of shards extracted to the disk. To
	// compute the number shards extracted to memory just subtract it from
	// ExtractedCnt.
	ExtractedToDiskCnt int64 `json:"extracted_to_disk_count,string"`
	// ExtractedToDiskSize - uncompressed size of shards extracted to disk.
	ExtractedToDiskSize int64 `json:"extracted_to_disk_size,string"`
	// contains filtered or unexported fields
}

type Manager struct {
	// tagged fields are the only fields persisted once dsort finishes
	ManagerUUID string         `json:"manager_uuid"`
	Metrics     *Metrics       `json:"metrics"`
	Pars        *parsedReqSpec `json:"pars"`
	// contains filtered or unexported fields
}

type MetaSorting struct {

	// SentStats - time statistics about records sent to another target
	SentStats *TimeStats `json:"sent_stats,omitempty"`
	// RecvStats - time statistics about records receivied from another target
	RecvStats *TimeStats `json:"recv_stats,omitempty"`
	// contains filtered or unexported fields
}

type Metrics struct {
	Extraction *LocalExtraction `json:"local_extraction,omitempty"`
	Sorting    *MetaSorting     `json:"meta_sorting,omitempty"`
	Creation   *ShardCreation   `json:"shard_creation,omitempty"`

	// job description
	Description string `json:"description,omitempty"`

	// warnings during the run
	Warnings []string `json:"warnings,omitempty"`
	// errors, if any
	Errors []string `json:"errors,omitempty"`

	// has been aborted
	Aborted atomic.Bool `json:"aborted,omitempty"`
	// has been archived to persistent storage
	Archived atomic.Bool `json:"archived,omitempty"`
}

type ParsedReq struct {
	InputBck  cmn.Bck
	OutputBck cmn.Bck
	// contains filtered or unexported fields
}

type RemoteResponse struct {
	Record    *shard.Record    `msg:"r"`
	RecordObj *shard.RecordObj `msg:"o"`
}

type RequestSpec struct {
	// Required
	InputBck        cmn.Bck       `json:"input_bck" yaml:"input_bck"`
	InputFormat     apc.ListRange `json:"input_format" yaml:"input_format"`
	OutputFormat    string        `json:"output_format" yaml:"output_format"`
	OutputShardSize string        `json:"output_shard_size" yaml:"output_shard_size"`

	// Desirable
	InputExtension string `json:"input_extension" yaml:"input_extension"`

	// Optional
	// Default: InputExtension
	OutputExtension string `json:"output_extension" yaml:"output_extension"`
	// Default: ""
	Description string `json:"description" yaml:"description"`
	// Default: same as `bck` field
	OutputBck cmn.Bck `json:"output_bck" yaml:"output_bck"`
	// Default: alphanumeric, increasing
	Algorithm Algorithm `json:"algorithm" yaml:"algorithm"`
	// Default: ""
	EKMFileURL string `json:"ekm_file" yaml:"ekm_file"`
	// Default: "\t"
	EKMFileSep string `json:"ekm_file_sep" yaml:"ekm_file_sep"`
	// Default: "80%"
	MaxMemUsage string `json:"max_mem_usage" yaml:"max_mem_usage"`
	// Default: calcMaxLimit()
	ExtractConcMaxLimit int `json:"extract_concurrency_max_limit" yaml:"extract_concurrency_max_limit"`
	// Default: calcMaxLimit()
	CreateConcMaxLimit int `json:"create_concurrency_max_limit" yaml:"create_concurrency_max_limit"`

	// debug
	DsorterType string `json:"dsorter_type"`
	DryRun      bool   `json:"dry_run"` // Default: false

	Config cmn.DsortConf
}

type ShardCreation struct {

	// ToCreate - number of shards that to be created in this phase.
	ToCreate int64 `json:"to_create,string"`
	// CreatedCnt the number of shards that have been so far created.
	// Should match ToCreate when phase finishes.
	CreatedCnt int64 `json:"created_count,string"`
	// MovedShardCnt specifies the number of shards that have migrated from this
	// to another target. Applies only when dealing with compressed
	// data. Sometimes, rather than creating at the destination, it is faster
	// to create a shard on a specific target and send it over (to the destination).
	MovedShardCnt int64 `json:"moved_shard_count,string"`
	// RequestStats - time statistics: requests to other targets.
	RequestStats *TimeStats `json:"req_stats,omitempty"`
	// ResponseStats - time statistics: responses to other targets.
	ResponseStats *TimeStats `json:"resp_stats,omitempty"`
	// contains filtered or unexported fields
}

type TimeStats struct {
	// Total contains total number of milliseconds spend on
	// specific task.
	Total int64 `json:"total_ms,string"`
	// Count contains number of time specific task was triggered.
	Count int64 `json:"count,string"`
	MinMs int64 `json:"min_ms,string"`
	MaxMs int64 `json:"max_ms,string"`
	AvgMs int64 `json:"avg_ms,string"`
}