v1alpha1

type AppliedOverride struct {
	// index is the zero-based position of the override in spec.overrides[].
	Index int `json:"index"`

	// when is a human-readable summary of the conditions that matched.
	// Example: "gpuArchitecture=h100, platform=aws"
	When string `json:"when"`

	// patches describes what the override modifies.
	// Example: "jobTemplate, 2 dependencies"
	Patches string `json:"patches"`

	// noOp is true when the override matched but produced no changes.
	// This usually indicates a misconfigured override with wrong field paths.
	// +optional
	NoOp bool `json:"noOp,omitempty"`
}

type BandwidthMeasurement struct {
	metav1.TypeMeta `json:",inline"`

	// metadata is a standard object metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitzero"`

	// spec defines the desired state of BandwidthMeasurement
	// +required
	Spec BandwidthMeasurementSpec `json:"spec"`

	// status defines the observed state of BandwidthMeasurement
	// +optional
	Status BandwidthMeasurementStatus `json:"status,omitzero"`
}

type BandwidthMeasurementConfig struct {
	// logProfileRef is the name of the cluster-scoped LogProfile to use for log parsing.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	LogProfileRef string `json:"logProfileRef"`

	// sampleInterval is how often to sample pod logs while the Job is running.
	// Default: 60s.
	// +optional
	SampleInterval *metav1.Duration `json:"sampleInterval,omitempty"`

	// testType identifies the NCCL collective operation (e.g., "all_reduce", "alltoall").
	// Propagated to the BandwidthMeasurement spec for Prometheus metric labeling.
	// +optional
	TestType string `json:"testType,omitempty"`
}

type BandwidthMeasurementList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitzero"`
	Items           []BandwidthMeasurement `json:"items"`
}

type BandwidthMeasurementSpec struct {
	// jobRef is the reference to the NVCRE Job for which bandwidth should be measured.
	// +optional
	JobRef corev1.TypedLocalObjectReference `json:"jobRef,omitempty"`

	// logProfileRef is the name of the cluster-scoped LogProfile to use for log parsing.
	// The LogProfile defines a bandwidthResult regex pattern for extracting NCCL bandwidth data.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	LogProfileRef string `json:"logProfileRef"`

	// sampleInterval is how often to sample pod logs while the Job is running.
	// Default: 60s.
	// +optional
	SampleInterval *metav1.Duration `json:"sampleInterval,omitempty"`

	// testType identifies the NCCL collective operation being measured (e.g., "all_reduce", "alltoall").
	// Used as the "nccl_test" Prometheus label to distinguish results on dashboards.
	// +optional
	TestType string `json:"testType,omitempty"`
}

type BandwidthMeasurementStatus struct {
	// results contains per-message-size average bandwidth measurements.
	// Each entry represents the running average of all observed measurements for that size.
	// +optional
	Results []BandwidthResult `json:"results,omitempty"`

	// startTime is the time when the measurement started (when the referenced Job began running).
	// +optional
	StartTime *metav1.Time `json:"startTime,omitempty"`

	// completionTime is the time when the measurement completed (when the referenced Job reached a terminal state).
	// +optional
	CompletionTime *metav1.Time `json:"completionTime,omitempty"`

	// conditions represent the current state of the BandwidthMeasurement resource.
	//
	// Condition types:
	// - "Measuring": the measurement is in progress (referenced Job is running)
	// - "Complete": the measurement has finished (referenced Job succeeded or failed)
	//
	// The status of each condition is one of True, False, or Unknown.
	// +listType=map
	// +listMapKey=type
	// +optional
	Conditions []metav1.Condition `json:"conditions,omitempty"`
}

type BandwidthResult struct {
	// sizeBytes is the message size in bytes.
	SizeBytes int64 `json:"sizeBytes"`

	// algBW is the average algorithmic bandwidth in GB/s.
	AlgBW string `json:"algBW"`

	// busBW is the average bus bandwidth in GB/s.
	BusBW string `json:"busBW"`

	// samples is the number of measurements averaged for this size.
	Samples int `json:"samples"`
}

type CELNodeHealthCheck struct {
	// expression is the CEL expression to evaluate against Node objects.
	// Must return a boolean value. When true, the node is considered unhealthy.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	// +kubebuilder:validation:MaxLength=2048
	Expression string `json:"expression"`
}

type CategoryOptions struct {
	// nodesPerJob is the number of nodes per job for multi-node workloads.
	// When nil at both global and per-category level, the controller auto-selects:
	//   - Entries with per-node-count configs (training): largest config <= matching nodes.
	//   - All other entries: all matching nodes.
	// When set, clamped to min(nodesPerJob, matchingNodes).
	// +optional
	// +kubebuilder:validation:Minimum=1
	NodesPerJob *int32 `json:"nodesPerJob,omitempty"`

	// enableCheckpoint enables checkpointing for training workloads.
	// When true, provisions a PVC for checkpoint storage, adds checkpoint config
	// (pvcName/maxRestarts), and enables --save/--load in training scripts.
	// When false (default), uses emptyDir and disables save/load.
	// Non-training entries ignore this field.
	// +optional
	EnableCheckpoint *bool `json:"enableCheckpoint,omitempty"`

	// maxSteps sets the maximum training steps. Maps to trainer.max_steps in the
	// NeMo config. Default: 50.
	// +optional
	// +kubebuilder:validation:Minimum=-1
	MaxSteps *int32 `json:"maxSteps,omitempty"`

	// exitDurationMins sets the training duration in minutes. Maps to
	// EXIT_DURATION_MINS env var. Default: 30.
	// +optional
	// +kubebuilder:validation:Minimum=1
	ExitDurationMins *int32 `json:"exitDurationMins,omitempty"`

	// gpusPerNode optionally overrides the number of GPUs per node used by catalog
	// workloads. If not specified, the controller derives the default from the GPU
	// architecture in target.nodeSelector (e.g., 4 for GB200/GB300, 8 for H100).
	// Use this field when your hardware has a non-standard GPU count per node.
	// +optional
	// +kubebuilder:validation:Minimum=1
	GpusPerNode *int32 `json:"gpusPerNode,omitempty"`

	// mlnxPerNode overrides the auto-detected Mellanox NIC count per node.
	// Used by platforms with InfiniBand or RoCE networking (Azure, OCI, TogetherAI).
	// If not specified, derived from GPU architecture and platform via the
	// catalog's gpu-defaults.yaml (e.g., 8 for most architectures, 2 for OCI L40s).
	// +optional
	// +kubebuilder:validation:Minimum=0
	MlnxPerNode *int32 `json:"mlnxPerNode,omitempty"`

	// resources overrides the CPU and memory resources of training workload
	// containers. Training entries default to DGX-class sizing (limits:
	// cpu "128" / memory "800Gi"; requests: cpu "64" / memory "500Gi") — set
	// this on smaller nodes so training pods can schedule. Each of the four
	// values independently falls back to its default when unset. GPU count is
	// controlled by gpusPerNode. Non-training entries ignore this field.
	// Platform-specific catalog overrides (for example AWS with H100 GPUs)
	// may supersede these values.
	// +optional
	Resources *CategoryResources `json:"resources,omitempty"`

	// enableMNNVL enables Multi-Node NVLink (NCCL_MNNVL_ENABLE=1) for training
	// and communication workloads. Defaults to false (NCCL_MNNVL_ENABLE=0).
	// Enable this when running on platforms with multi-node NVLink connectivity
	// (e.g., GB300 NVL72). Can be overridden per-category via
	// categories[].options.enableMNNVL.
	// +optional
	EnableMNNVL *bool `json:"enableMNNVL,omitempty"`

	// imagePullSecrets is an optional list of references to secrets for pulling
	// container images used by catalog workloads. If not specified, the cluster's
	// default image pull configuration (e.g., ServiceAccount) is used.
	// +optional
	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`

	// storageClassName is the StorageClass to use for PersistentVolumeClaim
	// dependencies created by catalog entries. If not specified, catalog entries
	// that require PVCs will use the cluster's default StorageClass.
	// +optional
	StorageClassName *string `json:"storageClassName,omitempty"`

	// saveInterval sets the checkpoint save frequency in training steps.
	// NeMo 6: maps to --save-interval; NeMo 4: maps to every_n_train_steps.
	// Only used when enableCheckpoint is true. Default: 250.
	// +optional
	// +kubebuilder:validation:Minimum=1
	SaveInterval *int32 `json:"saveInterval,omitempty"`

	// saveRetainInterval retains checkpoints at multiples of this value,
	// deleting intermediate checkpoints (NeMo 6 only, --save-retain-interval).
	// The most recent checkpoint is always kept regardless.
	// Only used when enableCheckpoint is true. Default: 1000.
	// +optional
	// +kubebuilder:validation:Minimum=1
	SaveRetainInterval *int32 `json:"saveRetainInterval,omitempty"`

	// saveTopK keeps only the top K checkpoints by monitored metric
	// (NeMo 4 only, save_top_k). Older checkpoints that aren't the best
	// are deleted to save storage.
	// Only used when enableCheckpoint is true. Default: 1.
	// +optional
	// +kubebuilder:validation:Minimum=1
	SaveTopK *int32 `json:"saveTopK,omitempty"`

	// storageSize sets the PVC size for checkpoint storage (e.g., "10Ti", "500Gi").
	// Must be a valid Kubernetes resource quantity. Only used when enableCheckpoint
	// is true. Default: "10Ti".
	// +optional
	// +kubebuilder:validation:Pattern=`^([0-9]+(\.[0-9]+)?)(Ki|Mi|Gi|Ti|Pi|Ei)?$`
	StorageSize string `json:"storageSize,omitempty"`

	// maxRestarts sets the maximum number of checkpoint-based restarts for
	// training workloads. Maps to checkpoint.maxRestarts in the Job spec.
	// Only used when enableCheckpoint is true. Default: catalog-defined.
	// +optional
	// +kubebuilder:validation:Minimum=0
	MaxRestarts *int32 `json:"maxRestarts,omitempty"`

	// repeatCount sets the number of orchestration iterations for the Workflow.
	// Each iteration runs all groups and collects results. Multiple iterations
	// allow repeated testing for intermittent failures. Default: 1.
	// +optional
	// +kubebuilder:validation:Minimum=1
	RepeatCount *int32 `json:"repeatCount,omitempty"`

	// testScale controls the orchestration strategy for NCCL communication tests.
	// Supported values:
	//   - "intra-node": each node tested independently (nodesPerJob=1)
	//   - "intra-rack": topology-partitioned per nvidia.com/gpu.clique
	//   - "diagnose": adaptive fault isolation (topology-aware hierarchical group testing)
	//   - "full-scale": all nodes in a single group (default)
	// Non-communication entries ignore this field.
	// +optional
	// +kubebuilder:validation:Enum=intra-node;intra-rack;diagnose;full-scale
	TestScale string `json:"testScale,omitempty"`

	// maxBytes sets the maximum message size for NCCL tests (e.g., "16G", "32G").
	// Maps to the NCCL perf test `-e` flag. Default: "16G" (GB200/GB300 override: "32G").
	// +optional
	// +kubebuilder:validation:Pattern='^[0-9]+(K|M|G|T)?$'
	MaxBytes string `json:"maxBytes,omitempty"`

	// numIterations sets the number of timed iterations per message size for NCCL tests.
	// Maps to the NCCL perf test `-n` flag. Default: 100.
	// +optional
	// +kubebuilder:validation:Minimum=1
	NumIterations *int32 `json:"numIterations,omitempty"`

	// numCycles sets the number of run cycles for NCCL tests. Each cycle runs
	// numIterations iterations and prints results separately.
	// Maps to the NCCL perf test `-N` flag. Default: 10.
	// +optional
	// +kubebuilder:validation:Minimum=1
	NumCycles *int32 `json:"numCycles,omitempty"`

	// thresholds defines performance thresholds as CEL expressions.
	// Keys are metric names (e.g., "busBandwidthGBps", "goodputRatio").
	// Values are CEL expressions using a `value` variable (float64).
	// Example: {"busBandwidthGBps": "value >= 900", "avgStepTimeSec": "value <= 3.0"}
	// +optional
	Thresholds map[string]string `json:"thresholds,omitempty"`

	// maxConcurrent limits the number of simultaneously running jobs within
	// a Workflow. Useful for diagnose-mode parallel screening to avoid
	// fabric saturation. 0 means unlimited. Default: 0.
	// +optional
	MaxConcurrent *int32 `json:"maxConcurrent,omitempty"`

	// minGroupSize sets the smallest group size at which diagnose's internal
	// bisection stops splitting. Groups at this size that fail become suspects.
	// Default: 2.
	// +optional
	// +kubebuilder:validation:Minimum=1
	MinGroupSize *int32 `json:"minGroupSize,omitempty"`

	// timeoutPerJob is the maximum time to wait for each job to complete.
	// Accepts Go duration strings (e.g., "1h", "30m", "2h30m").
	// When not set, defaults to "1h" (or "15m" for diagnose test scale).
	// +optional
	// +kubebuilder:validation:Pattern=`^([0-9]+(h|m|s|ms))+$`
	TimeoutPerJob string `json:"timeoutPerJob,omitempty"`

	// measurementTimeout is the maximum time to wait after a Job succeeds for
	// measurement data (bandwidth, goodput, etc.) before failing threshold validation.
	// Accepts Go duration strings (e.g., "5m", "10m", "30m").
	// When not set, defaults to "5m".
	// +optional
	// +kubebuilder:validation:Pattern=`^([0-9]+(h|m|s|ms))+$`
	MeasurementTimeout string `json:"measurementTimeout,omitempty"`
}

type CategoryResourceList struct {
	// cpu is the CPU quantity (e.g., "6", "1500m").
	// +optional
	// +kubebuilder:validation:XIntOrString
	// +kubebuilder:validation:MaxLength=32
	// +kubebuilder:validation:XValidation:rule="quantity(string(self)).compareTo(quantity(\"0\")) >= 0",message="cpu must be a non-negative quantity"
	CPU *resource.Quantity `json:"cpu,omitempty"`

	// memory is the memory quantity (e.g., "48Gi").
	// +optional
	// +kubebuilder:validation:XIntOrString
	// +kubebuilder:validation:MaxLength=32
	// +kubebuilder:validation:XValidation:rule="quantity(string(self)).compareTo(quantity(\"0\")) >= 0",message="memory must be a non-negative quantity"
	Memory *resource.Quantity `json:"memory,omitempty"`
}

type CategoryResources struct {
	// limits overrides the container resource limits.
	// +optional
	Limits *CategoryResourceList `json:"limits,omitempty"`

	// requests overrides the container resource requests.
	// +optional
	Requests *CategoryResourceList `json:"requests,omitempty"`
}

type CertificateCategory struct {
	// Domain is the high level Domain that the certificate belongs to like training, inference etc
	// +kubebuilder:validation:Required
	Domain string `json:"domain"`

	// Variant is the lower level type such as nemotron, deepseek, nccl etc.
	// +kubebuilder:validation:Required
	Variant string `json:"variant"`

	// options provides per-category configuration overrides.
	// Fields set here take precedence over their CertificationSpec counterparts.
	// +optional
	Options *CategoryOptions `json:"options,omitempty"`
}

type Certification struct {
	metav1.TypeMeta `json:",inline"`

	// metadata is a standard object metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitzero"`

	// spec defines the desired state of Certification.
	// The entire spec is immutable after creation: the controller never updates
	// an active Workflow from an edited parent, so accepting edits would either
	// silently ignore them or apply them only to later categories. To run with
	// different inputs, delete the Certification and create a new one.
	// +required
	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable after creation"
	Spec CertificationSpec `json:"spec"`

	// status defines the observed state of Certification
	// +optional
	Status CertificationStatus `json:"status,omitzero"`
}

type CertificationCategoryStatus struct {
	// domain is the category domain.
	Domain string `json:"domain"`

	// variant is the category variant.
	Variant string `json:"variant"`

	// workflowRef references the Workflow created for this category.
	// +optional
	WorkflowRef *WorkflowReference `json:"workflowRef,omitempty"`

	// status is the current status of this category.
	// Values: "Pending", "InProgress", "Succeeded", "Failed"
	Status string `json:"status"`

	// succeededNodesRef references the ConfigMap holding this category's
	// succeeded-nodes list (comma-separated node names). Copied from the Workflow.
	// +optional
	SucceededNodesRef *corev1.TypedLocalObjectReference `json:"succeededNodesRef,omitempty"`

	// failedNodesRef references the ConfigMap holding this category's failed-nodes
	// list (name, reason, message). Copied from the Workflow.
	// +optional
	FailedNodesRef *corev1.TypedLocalObjectReference `json:"failedNodesRef,omitempty"`
}

type CertificationList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitzero"`
	Items           []Certification `json:"items"`
}

type CertificationSpec struct {
	// target specifies which nodes to include in the orchestration.
	// +kubebuilder:validation:Required
	Target TargetSpec `json:"target"`

	// Categories are the list of certificate categories required for the Target.
	// The MaxItems bound keeps the CEL rules nested under each category (the
	// resources request/limit comparison) within the API server's validation
	// cost budget; the catalog defines far fewer entries than 64.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinItems=1
	// +kubebuilder:validation:MaxItems=64
	Categories []CertificateCategory `json:"categories,omitempty"`

	// Global defaults for all categories. Per-category options override these.
	CategoryOptions `json:",inline"`
}

type CertificationStatus struct {
	// conditions represent the current state of the Certification resource.
	//
	// Condition types:
	// - "InProgress": the Certification is currently running
	// - "Succeeded": the Certification completed successfully
	// - "Failed": the Certification has failed
	//
	// The status of each condition is one of True, False, or Unknown.
	// +listType=map
	// +listMapKey=type
	// +optional
	Conditions []metav1.Condition `json:"conditions,omitempty"`

	// categoryStatuses tracks the status of each certification category.
	// +optional
	CategoryStatuses []CertificationCategoryStatus `json:"categoryStatuses,omitempty"`
}

type CheckpointConfig struct {
	// pvcName is the name of the PersistentVolumeClaim used for checkpoint storage.
	// Must match a PVC in the Workflow's dependencies.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	PVCName string `json:"pvcName"`

	// maxRestarts is the maximum number of times the Job will restart on failure
	// when a checkpoint exists. Default: 0 (no restarts).
	// +optional
	MaxRestarts *int32 `json:"maxRestarts,omitempty"`
}

type CrossBoundaryProbe struct {
	// domain is the topology domain being investigated.
	Domain string `json:"domain"`
	// halfA is the first half of the bisection that passed.
	HalfA []string `json:"halfA"`
	// halfB is the second half of the bisection that passed.
	HalfB []string `json:"halfB"`
	// probeRound is the current probe round within this investigation.
	ProbeRound int `json:"probeRound"`
}

type CrossBoundaryState struct {
	// pendingProbes are boundary investigations still in progress.
	// +optional
	PendingProbes []CrossBoundaryProbe `json:"pendingProbes,omitempty"`
	// originStage records which diagnose stage triggered the cross-boundary
	// probing, used to determine MNNVL settings for probe jobs.
	OriginStage string `json:"originStage"`
}

type DependencyResourceRef struct {
	// apiVersion of the created resource.
	APIVersion string `json:"apiVersion"`

	// kind of the created resource.
	Kind string `json:"kind"`

	// name of the created resource.
	Name string `json:"name"`

	// namespace of the created resource. Empty for cluster-scoped resources.
	// +optional
	Namespace string `json:"namespace,omitempty"`

	// scope is the lifecycle scope of this dependency (workflow, job).
	// +optional
	Scope string `json:"scope,omitempty"`

	// groupName identifies the group this dependency was created for (scope=job).
	// +optional
	GroupName string `json:"groupName,omitempty"`

	// iteration identifies the iteration this dependency was created for (scope=job).
	// +optional
	Iteration int `json:"iteration,omitempty"`
}

type DependencySpec struct {
	// Resource definition (inline, any Kubernetes resource)
	// +kubebuilder:pruning:PreserveUnknownFields
	// +kubebuilder:validation:EmbeddedResource
	runtime.RawExtension `json:",inline"`

	// when defines conditions for creating this dependency.
	// If not specified, the dependency is always created.
	// +optional
	When WhenSpec `json:"when,omitempty"`
}

type DiagnoseSpec struct {
	// minGroupSize is the smallest group at which bisection stops.
	// Nodes in failed groups of this size become suspects for confirmation.
	// +kubebuilder:validation:Minimum=1
	// +kubebuilder:default=2
	// +optional
	MinGroupSize int `json:"minGroupSize,omitempty"`

	// topologyKey is the node label key for topology-aware screening.
	// When set, Stage 1 groups nodes by this label (e.g., one group per clique).
	// When empty, nodes are grouped into disjoint chunks of ceil(sqrt(N)).
	// Example: "nvidia.com/gpu.clique"
	// +optional
	TopologyKey string `json:"topologyKey,omitempty"`
}

type DiagnoseStatus struct {
	// stage is the current stage of the diagnose algorithm.
	// Values: "intra-screening", "inter-screening", "bisection", "confirmation"
	Stage string `json:"stage"`

	// round is the total number of completed rounds across all stages.
	Round int `json:"round"`

	// healthyNodes are nodes confirmed healthy at any stage (accumulated).
	// +listType=set
	// +optional
	HealthyNodes []string `json:"healthyNodes,omitempty"`

	// suspectNodes are nodes in minGroupSize groups that failed bisection,
	// pending confirmation in Stage 3.
	// +listType=set
	// +optional
	SuspectNodes []string `json:"suspectNodes,omitempty"`

	// noNVLSuspectNodes are suspect nodes that originated from the
	// intra-screening-no-nvl stage. Confirmation tests for these nodes
	// run with NCCL_MNNVL_ENABLE=0 to isolate fabric issues.
	// +listType=set
	// +optional
	NoNVLSuspectNodes []string `json:"noNVLSuspectNodes,omitempty"`

	// representativeNodes are one healthy node per topology domain, selected
	// after intra-domain screening for inter-domain fabric testing.
	// +optional
	RepresentativeNodes []string `json:"representativeNodes,omitempty"`

	// screeningResults preserves per-domain screening outcomes across stages.
	// Key is the domain name (e.g., clique UUID), value is the domain's screening result.
	// +optional
	ScreeningResults map[string]DomainScreeningResult `json:"screeningResults,omitempty"`

	// noNVLScreeningResults preserves per-domain screening outcomes from the
	// intra-screening-no-nvl stage (MNNVL disabled). Same structure as screeningResults.
	// +optional
	NoNVLScreeningResults map[string]DomainScreeningResult `json:"noNVLScreeningResults,omitempty"`

	// infrastructureFaults records detected infrastructure-level faults where
	// bisection both-halves-pass but the full group fails. Each entry identifies
	// the node groups whose interconnecting infrastructure is suspected faulty.
	// +optional
	InfrastructureFaults []InfrastructureFault `json:"infrastructureFaults,omitempty"`

	// crossBoundaryState tracks in-progress cross-boundary probing.
	// Nil when not in cross-boundary stage.
	// +optional
	CrossBoundaryState *CrossBoundaryState `json:"crossBoundaryState,omitempty"`
}

type DomainResult struct {
	// nodeCount is the number of nodes in this domain.
	NodeCount int `json:"nodeCount"`

	// status is the overall status for this domain.
	// Examples: "Passed", "Failed", "Running"
	Status string `json:"status"`

	// averageTFLOPs is the average TFLOPs across all nodes in the domain as a string.
	// Example: "1500.5"
	// +optional
	AverageTFLOPs string `json:"averageTflops,omitempty"`
}

type DomainScreeningResult struct {
	// nodes in this domain.
	Nodes []string `json:"nodes"`
	// passed is true if the screening test succeeded.
	Passed bool `json:"passed"`
}

type EventPattern struct {
	// regex is the regular expression with Go named capture groups.
	// Uses (?P<name>...) syntax to name extracted fields.
	//
	// Well-known capture names used by the goodput calculator:
	//   Common:     timestamp
	//   Training:   globalStep, iteration, epoch, stepTiming, loss, tflops, elapsedTime
	//   Checkpoint: step, path, saveDuration
	//
	// The controller parses named captures into typed values:
	//   int:     globalStep, iteration, epoch, step
	//   float64: stepTiming, loss, tflops, elapsedTime, saveDuration
	//   string:  path
	//   time:    timestamp (parsed using LogProfile.spec.timestamp.layout)
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	Regex string `json:"regex"`

	// example is a sample log line that the regex should match.
	// Serves as documentation and is validated when the LogProfile is loaded.
	// +optional
	Example string `json:"example,omitempty"`

	// units specifies the unit for duration/timing captures.
	// The controller normalizes all durations to seconds internally.
	// Supported values: "s" (seconds), "ms" (milliseconds), "us" (microseconds).
	//
	// Only needed for duration-type captures: stepTiming, elapsedTime, saveDuration.
	// If a duration capture is not listed here, it defaults to "s" (seconds).
	//
	// Example: {"elapsedTime": "ms"} means the captured value is in milliseconds.
	// +optional
	Units map[string]string `json:"units,omitempty"`
}

type ExecFramework struct {
	// command is the entrypoint array (e.g., ["/bin/bash", "/config/train.sh"]).
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinItems=1
	Command []string `json:"command"`

	// args are arguments to the command.
	// +optional
	Args []string `json:"args,omitempty"`
}

type ExecutionSpec struct {
	// maxConcurrent is the maximum number of jobs to run concurrently.
	// 0 means unlimited. Default is 0.
	// +kubebuilder:validation:Minimum=0
	// +kubebuilder:default=0
	// +optional
	MaxConcurrent int `json:"maxConcurrent,omitempty"`

	// timeoutPerJob is the maximum time to wait for each job to complete.
	// +optional
	TimeoutPerJob *metav1.Duration `json:"timeoutPerJob,omitempty"`

	// retryFailedGroups is the number of times to retry a failed group's job.
	// After the workload controller exhausts its own restarts (backoffLimit),
	// the orchestration re-creates the entire job up to this many times.
	// 0 means no retries. Default is 0.
	// +kubebuilder:validation:Minimum=0
	// +kubebuilder:default=0
	// +optional
	RetryFailedGroups int `json:"retryFailedGroups,omitempty"`
}

type FailedNode struct {
	// name is the Kubernetes node name.
	// +kubebuilder:validation:Required
	Name string `json:"name"`

	// reason is the failure type for this node, matching the originating Job
	// condition reason.
	//   - HardwareFailureDetected: a node health check (CEL) flagged the node.
	//   - ThresholdViolation: a performance threshold (bandwidth/goodput) was not met.
	//   - WorkloadFailed: the workload exited non-zero, stalled, or otherwise failed.
	// +kubebuilder:validation:Required
	Reason NodeFailureReason `json:"reason"`

	// message is the detailed failure message for this node, matching the originating Job condition message.
	// +optional
	Message string `json:"message,omitempty"`
}

type FailureLog struct {
	// podName is the name of the pod that failed.
	PodName string `json:"podName"`
	// nodeName is the node the failed pod was running on.
	NodeName string `json:"nodeName"`
	// exitCode is the container's exit code.
	ExitCode int32 `json:"exitCode"`
	// reason is the termination reason (e.g., "OOMKilled", "Error").
	// +optional
	Reason string `json:"reason,omitempty"`
	// tail is the end of the failed container's logs, up to 32 KB.
	// When no pod was available to read, this explains why instead.
	// +optional
	Tail string `json:"tail,omitempty"`
}

type FrameworkSpec struct {
	// torch runs distributed training via torchrun.
	// Auto-generates a PyTorch TrainingRuntime with torch mlPolicy,
	// numProcPerNode, GPU resources, and shared memory.
	// +optional
	Torch *TorchFramework `json:"torch,omitempty"`

	// mpi runs MPI-based workloads (e.g., NCCL tests).
	// Auto-generates an MPI TrainingRuntime with launcher+worker pattern,
	// SSH auth, IPC_LOCK capability, and readiness probes.
	// +optional
	MPI *MPIFramework `json:"mpi,omitempty"`

	// exec runs an arbitrary command directly.
	// Auto-generates a simple TrainingRuntime with a single replicatedJob.
	// +optional
	Exec *ExecFramework `json:"exec,omitempty"`
}

type GangSchedulerSpec struct {
	// schedulerName is the name of the gang-aware scheduler to use (e.g. "kai-scheduler").
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	SchedulerName string `json:"schedulerName"`

	// queue is the scheduler queue to submit the workload to.
	// Defaults to "default-queue" if not specified.
	// When non-empty, must be a valid Kubernetes label value: at most 63 characters,
	// beginning and ending with an alphanumeric character, and containing only
	// alphanumerics, hyphens, underscores, or dots.
	// +optional
	// +kubebuilder:validation:MaxLength=63
	// +kubebuilder:validation:Pattern=`^$|^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$`
	Queue string `json:"queue,omitempty"`
}

type GoodputMeasurement struct {
	metav1.TypeMeta `json:",inline"`

	// metadata is a standard object metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitzero"`

	// spec defines the desired state of GoodputMeasurement
	// +required
	Spec GoodputMeasurementSpec `json:"spec"`

	// status defines the observed state of GoodputMeasurement
	// +optional
	Status GoodputMeasurementStatus `json:"status,omitzero"`
}

type GoodputMeasurementConfig struct {
	// logProfileRef is the name of the cluster-scoped LogProfile to use for log parsing.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	LogProfileRef string `json:"logProfileRef"`

	// sampleInterval is how often to sample pod logs while the Job is running.
	// Default: 60s.
	// +optional
	SampleInterval *metav1.Duration `json:"sampleInterval,omitempty"`
}

type GoodputMeasurementList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitzero"`
	Items           []GoodputMeasurement `json:"items"`
}

type GoodputMeasurementSpec struct {
	// jobRef is the reference to the NVCRE Job for which goodput should be calculated.
	// +optional
	JobRef corev1.TypedLocalObjectReference `json:"jobRef,omitempty"`

	// logProfileRef is the name of the cluster-scoped LogProfile to use for log parsing.
	// The LogProfile defines regex patterns for extracting training metrics from pod logs.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	LogProfileRef string `json:"logProfileRef"`

	// sampleInterval is how often to sample pod logs while the Job is running.
	// Default: 60s.
	// +optional
	SampleInterval *metav1.Duration `json:"sampleInterval,omitempty"`
}

type GoodputMeasurementStatus struct {
	// result is the current runtime goodput ratio (0.0 to 1.0) as a string.
	// Formula: (t_w - t_ch - t_rm - t_re - t_save) / (t_w - t_re)
	// where t_w=trainingTime, t_ch=lostWorkTime, t_rm=resumeTime, t_re=rescheduleTime, t_save=checkpointSaveTime
	// +optional
	Result string `json:"result,omitempty"`

	// currentStep is the latest training step observed from logs.
	// +optional
	CurrentStep int `json:"currentStep,omitempty"`

	// highestStep is the highest training step ever reached (may be > currentStep after restart).
	// +optional
	HighestStep int `json:"highestStep,omitempty"`

	// trainingTimeSec is the total training wall-clock time in seconds (t_w).
	// +optional
	TrainingTimeSec string `json:"trainingTimeSec,omitempty"`

	// lostWorkTimeSec is cumulative lost work in seconds (sum of t_ch across all interruptions).
	// +optional
	LostWorkTimeSec string `json:"lostWorkTimeSec,omitempty"`

	// rescheduleTimeSec is cumulative scheduling delay in seconds (sum of t_re).
	// +optional
	RescheduleTimeSec string `json:"rescheduleTimeSec,omitempty"`

	// resumeTimeSec is cumulative resume time in seconds (sum of t_rm: checkpoint load + resume).
	// +optional
	ResumeTimeSec string `json:"resumeTimeSec,omitempty"`

	// checkpointSaveTimeSec is cumulative checkpoint save overhead in seconds (sum of t_save).
	// +optional
	CheckpointSaveTimeSec string `json:"checkpointSaveTimeSec,omitempty"`

	// interruptionCount is the number of interruptions detected.
	// +optional
	InterruptionCount int `json:"interruptionCount,omitempty"`

	// lastCheckpointStep is the step number of the last saved checkpoint.
	// +optional
	LastCheckpointStep int `json:"lastCheckpointStep,omitempty"`

	// lastCheckpointTime is the timestamp of the last saved checkpoint.
	// Persisted so lost work time (t_ch) can be computed after controller restarts.
	// +optional
	LastCheckpointTime *metav1.Time `json:"lastCheckpointTime,omitempty"`

	// startTime is the time when the measurement started (when the referenced Job began running).
	// +optional
	StartTime *metav1.Time `json:"startTime,omitempty"`

	// completionTime is the time when the measurement completed (when the referenced Job reached a terminal state).
	// +optional
	CompletionTime *metav1.Time `json:"completionTime,omitempty"`

	// lastStepTimestamp is the timestamp of the most recently observed training step.
	// Used by the Job controller for stall detection.
	// +optional
	LastStepTimestamp *metav1.Time `json:"lastStepTimestamp,omitempty"`

	// avgStepTimeSec is the average time per training step in seconds (excluding warmup).
	// Computed from observed step timings or consecutive step timestamps.
	// +optional
	AvgStepTimeSec string `json:"avgStepTimeSec,omitempty"`

	// logInterval is the training framework's log interval from the LogProfile.
	// Persisted so the Job controller can scale the stall detection threshold
	// by the number of iterations between consecutive logged steps.
	// +optional
	LogInterval int `json:"logInterval,omitempty"`

	// warmupBaseStep is the GlobalStep of the first training step after the most recent
	// run boundary (applicationStart or checkpointRestore). Steps within warmupSteps of
	// this base are warmup. Persisted so warmup detection survives across SinceTime-based
	// incremental log reads where applicationStart is no longer in the read window.
	// +optional
	WarmupBaseStep *int `json:"warmupBaseStep,omitempty"`

	// warmupTimeSec is the cumulative time spent on warmup steps in seconds (across all runs).
	// +optional
	WarmupTimeSec string `json:"warmupTimeSec,omitempty"`

	// nonWarmupTimeSec is the cumulative time spent on non-warmup steps in seconds (across all runs).
	// +optional
	NonWarmupTimeSec string `json:"nonWarmupTimeSec,omitempty"`

	// lastNonWarmupStep is the highest step whose timing has been accumulated into nonWarmupTimeSec.
	// Used as a watermark to avoid double-counting steps across reconcile cycles.
	// +optional
	LastNonWarmupStep int `json:"lastNonWarmupStep,omitempty"`

	// priorWarmupTimeSec is the accumulated warmup time from completed runs (before the current run).
	// +optional
	PriorWarmupTimeSec string `json:"priorWarmupTimeSec,omitempty"`

	// priorNonWarmupTimeSec is the accumulated non-warmup time from completed runs (before the current run).
	// +optional
	PriorNonWarmupTimeSec string `json:"priorNonWarmupTimeSec,omitempty"`

	// avgTFLOPSPerGPU is the average TFLOPS per GPU observed from training step logs (including warmup).
	// +optional
	AvgTFLOPSPerGPU string `json:"avgTFLOPSPerGPU,omitempty"`

	// applicationStartTime is the timestamp of the first application framework log marker.
	// Persisted so startup resume time can be computed even if the log line scrolls out of the tail window.
	// +optional
	ApplicationStartTime *metav1.Time `json:"applicationStartTime,omitempty"`

	// applicationStopTime is the timestamp of the last application log line observed.
	// Used as the interruption time for goodput calculations — more accurate than
	// container termination time or wall clock for stalls and crashes.
	// +optional
	ApplicationStopTime *metav1.Time `json:"applicationStopTime,omitempty"`

	// pendingInterruption stores an in-progress interruption event that hasn't been
	// completed by a job restart yet. Persisted so state survives controller restarts.
	// +optional
	PendingInterruption *PendingInterruptionStatus `json:"pendingInterruption,omitempty"`

	// conditions represent the current state of the GoodputMeasurement resource.
	//
	// Condition types:
	// - "Measuring": the measurement is in progress (referenced Job is running)
	// - "Complete": the measurement has finished (referenced Job succeeded or failed)
	//
	// The status of each condition is one of True, False, or Unknown.
	// +listType=map
	// +listMapKey=type
	// +optional
	Conditions []metav1.Condition `json:"conditions,omitempty"`
}

type GroupIterationResult struct {
	// name is the group identifier (matches GroupStatus.Name).
	Name string `json:"name"`
	// phase is the terminal phase (Succeeded or Failed).
	Phase GroupPhase `json:"phase"`
	// jobName is the Job that ran for this group in this iteration.
	// Empty if the Job was deleted before the snapshot.
	// +optional
	JobName string `json:"jobName,omitempty"`
	// startTime when the group's job started.
	// +optional
	StartTime *metav1.Time `json:"startTime,omitempty"`
	// completionTime when the group's job finished.
	// +optional
	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
}

type GroupPhase string

type GroupStatus struct {
	// name is a unique identifier (e.g., "group-0", "group-1-overflow").
	Name string `json:"name"`

	// nodes in this group.
	Nodes []string `json:"nodes"`

	// domains lists topology domain values this group spans.
	// +optional
	Domains []string `json:"domains,omitempty"`

	// domainNodeCounts records the number of nodes per topology domain in
	// this group, keyed by domain value. Populated at partition time so
	// reports can show per-clique node counts without fetching nodes.
	// +optional
	DomainNodeCounts map[string]int `json:"domainNodeCounts,omitempty"`

	// overflow indicates this group contains borrowed nodes and must wait
	// for the jobs using those nodes to complete before launching.
	// +optional
	Overflow bool `json:"overflow,omitempty"`

	// phase is the current phase of this group's job in the current iteration.
	// Reset to Pending at the start of each iteration.
	// +kubebuilder:validation:Enum=Pending;Running;Succeeded;Failed
	Phase GroupPhase `json:"phase"`

	// retries is the number of times the current iteration's job has been retried.
	// Reset to 0 at the start of each iteration.
	// +optional
	Retries int `json:"retries,omitempty"`

	// jobRef references the current job for this group.
	// +optional
	JobRef *WorkloadReference `json:"jobRef,omitempty"`

	// startTime when this group's job started in the current iteration.
	// +optional
	StartTime *metav1.Time `json:"startTime,omitempty"`

	// completionTime when this group's job finished in the current iteration.
	// +optional
	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
}

type InfrastructureFault struct {
	// domain is the topology domain where the fault was detected.
	Domain string `json:"domain"`
	// groupA is one set of nodes on one side of the faulty boundary.
	GroupA []string `json:"groupA"`
	// groupB is the other set of nodes.
	GroupB []string `json:"groupB"`
	// stage is the diagnose stage where the fault was detected.
	Stage string `json:"stage"`
}

type IntMatchSpec struct {
	// equals checks for exact match.
	// +optional
	Equals *int `json:"equals,omitempty"`

	// greaterThan checks if value is greater than threshold.
	// +optional
	GreaterThan *int `json:"greaterThan,omitempty"`

	// lessThan checks if value is less than threshold.
	// +optional
	LessThan *int `json:"lessThan,omitempty"`
}

type IterationMetrics struct {
	// iterationNumber is the iteration index (1-based).
	IterationNumber int `json:"iterationNumber"`

	// tflops is the measured TFLOPs performance as a string.
	// Example: "1500.5"
	// +optional
	TFLOPs string `json:"tflops,omitempty"`

	// duration is the iteration duration in seconds as a string.
	// Example: "3600.5"
	// +optional
	Duration string `json:"duration,omitempty"`

	// memoryUsedGB is the peak GPU memory usage in GB as a string.
	// Example: "640.25"
	// +optional
	MemoryUsedGB string `json:"memoryUsedGB,omitempty"`

	// status is the status of this iteration.
	// Examples: "Succeeded", "Failed"
	Status string `json:"status"`
}

type IterationResult struct {
	// iteration is the 1-based iteration number.
	Iteration int `json:"iteration"`
	// groups contains the outcome of each group in this iteration.
	Groups []GroupIterationResult `json:"groups"`
}

type Job struct {
	metav1.TypeMeta `json:",inline"`

	// metadata is a standard object metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitzero"`

	// spec defines the desired state of Job
	// +required
	Spec JobSpec `json:"spec"`

	// status defines the observed state of Job
	// +optional
	Status JobStatus `json:"status,omitzero"`
}

type JobList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitzero"`
	Items           []Job `json:"items"`
}

type JobSpec struct {
	// workload defines the workload to run.
	// The workload is created as a child resource of the Job.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="workload is immutable"
	Workload WorkloadSpec `json:"workload"`

	// nodeHealthMonitor configures hardware failure detection for nodes
	// running this job's pods. When a failure is detected, the job will
	// be marked with the HardwareFailed condition.
	// +optional
	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="nodeHealthMonitor is immutable"
	NodeHealthMonitor *NodeHealthMonitor `json:"nodeHealthMonitor,omitempty"`

	// checkpoint configures checkpoint-based restart for the workload.
	// When set, the Job controller restarts the workload on failure if a
	// checkpoint was logged by an associated GoodputMeasurement. The user
	// must define the PVC volume and mounts in the workload spec directly.
	// +optional
	Checkpoint *CheckpointConfig `json:"checkpoint,omitempty"`

	// stallMultiplier configures stall detection for the workload. When set,
	// the Job controller compares the time since the last training step against
	// stallMultiplier * avgStepTime (from the associated GoodputMeasurement).
	// If the elapsed time exceeds this product, the workload is marked as Failed
	// with reason "WorkloadStalled".
	// Requires a GoodputMeasurement for this Job; if none exists, stall
	// detection is skipped.
	// Example: a value of 10 means the job is stalled if no step has been
	// logged for 10x the average step duration.
	// +optional
	// +kubebuilder:validation:Minimum=1
	StallMultiplier *int32 `json:"stallMultiplier,omitempty"`

	// startupStallTimeoutSeconds is the maximum time (in seconds) to wait after
	// the application framework starts for the first training step. If no step
	// is observed within this timeout, the workload is marked Failed with reason
	// "WorkloadStalled". Only active when stallMultiplier is set.
	// Default: 1200 (20 minutes).
	// +optional
	// +kubebuilder:validation:Minimum=1
	StartupStallTimeoutSeconds *int32 `json:"startupStallTimeoutSeconds,omitempty"`

	// goodputMeasurement configures automatic creation of a GoodputMeasurement
	// child resource that tracks training goodput metrics by parsing pod logs.
	// When absent, no measurement is created (suitable for non-training jobs).
	// Manually-created GoodputMeasurements continue to work via the existing
	// List-based lookup.
	// +optional
	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="goodputMeasurement is immutable"
	GoodputMeasurement *GoodputMeasurementConfig `json:"goodputMeasurement,omitempty"`

	// bandwidthMeasurement configures automatic creation of a BandwidthMeasurement
	// child resource that tracks NCCL bandwidth metrics by parsing pod logs.
	// When absent, no measurement is created (suitable for non-NCCL jobs).
	// +optional
	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="bandwidthMeasurement is immutable"
	BandwidthMeasurement *BandwidthMeasurementConfig `json:"bandwidthMeasurement,omitempty"`

	// thresholds maps metric names to CEL expressions for performance validation.
	// Propagated from WorkflowSpec.Validation.Performance.Thresholds by the
	// Workflow controller during Job creation. The Job controller evaluates them
	// after workload success and sets ValidationFailed if any threshold is violated.
	// Keys: "busBandwidthGBps", "goodputRatio", "avgTFLOPsPerGPU", "avgStepTimeSec", etc.
	// Values: CEL expressions using `value` variable, e.g. "value >= 900"
	// +optional
	Thresholds map[string]string `json:"thresholds,omitempty"`

	// measurementTimeout is the maximum time to wait after the Job succeeds for
	// measurement data before failing threshold validation. Propagated from the
	// Workflow's ValidationSpec by the Workflow controller during Job creation.
	// When nil, the Job controller uses its default (5m).
	// +optional
	MeasurementTimeout *metav1.Duration `json:"measurementTimeout,omitempty"`
}

type JobStatus struct {
	// conditions represent the current state of the Job.
	// Only one of the following conditions can be True at any given time:
	// - "InProgress": the Job is currently running
	// - "Succeeded": the Job has completed successfully
	// - "Failed": the Job has failed
	// - "HardwareFailed": a hardware failure was detected on a node
	//
	// The condition message contains details about the current state for debugging.
	// +listType=map
	// +listMapKey=type
	// +optional
	Conditions []metav1.Condition `json:"conditions,omitempty"`

	// failedNodes lists nodes that failed during this Job, each with the reason
	// for failure. Populated when the Job fails, a hardware failure is detected,
	// or a performance threshold is violated. A node may appear multiple times
	// with different reasons; entries are keyed by the (name, reason) pair.
	// +listType=map
	// +listMapKey=name
	// +listMapKey=reason
	// +optional
	FailedNodes []FailedNode `json:"failedNodes,omitempty"`

	// workloadRef references the workload resource created by this Job.
	// Populated after the workload is created. Used to track workload status.
	// +optional
	WorkloadRef *WorkloadReference `json:"workloadRef,omitempty"`

	// workloadStartTime is when the workload was first observed running rather
	// than pending. A workload held by an admission controller (for example a
	// TrainJob suspended by Kueue until quota is available) is pending, and its
	// queued time does not count against timeoutPerJob or stall detection:
	// timeoutPerJob is measured from this timestamp, and the stall clock never
	// starts before it. Cleared on checkpoint restart so the replacement
	// workload gets a fresh budget.
	// +optional
	WorkloadStartTime *metav1.Time `json:"workloadStartTime,omitempty"`

	// restartCount tracks the number of times the workload has been restarted from checkpoint.
	// +optional
	RestartCount int32 `json:"restartCount,omitempty"`

	// failureLog captures the tail of pod logs from the most recent workload failure.
	// Populated when the Job transitions to Failed. Only the pod that caused the
	// failure (earliest non-zero exit code) is captured. Overwritten on each retry.
	// Always set on failure: when no pod was available to read, reason and tail
	// record why rather than leaving the field unset.
	// +optional
	FailureLog *FailureLog `json:"failureLog,omitempty"`
}

type JobTemplateSpec struct {
	// metadata is standard object metadata applied to created Jobs.
	// +optional
	metav1.ObjectMeta `json:"metadata,omitzero"`

	// spec is the Job specification used as a template for each execution group.
	// +kubebuilder:validation:Required
	Spec JobSpec `json:"spec"`
}

type LogPatternSet struct {
	// trainingStep matches training iteration log lines.
	// Well-known captures: timestamp, globalStep, iteration, epoch,
	// stepTiming, loss, tflops, elapsedTime
	// +optional
	TrainingStep *EventPattern `json:"trainingStep,omitempty"`

	// checkpointSave matches checkpoint save start lines.
	// Well-known captures: timestamp, step, path, saveDuration
	// +optional
	CheckpointSave *EventPattern `json:"checkpointSave,omitempty"`

	// checkpointDone matches checkpoint save completion lines.
	// Well-known captures: timestamp, step, path
	// +optional
	CheckpointDone *EventPattern `json:"checkpointDone,omitempty"`

	// checkpointRestore matches checkpoint restore (load start) lines.
	// Well-known captures: timestamp, path, step
	// +optional
	CheckpointRestore *EventPattern `json:"checkpointRestore,omitempty"`

	// checkpointLoaded matches checkpoint load completed lines.
	// Well-known captures: timestamp, path, step
	// +optional
	CheckpointLoaded *EventPattern `json:"checkpointLoaded,omitempty"`

	// applicationStart matches a line indicating the application has started.
	// Well-known captures: timestamp
	// +optional
	ApplicationStart *EventPattern `json:"applicationStart,omitempty"`

	// warmupStep matches lines that indicate a warmup/startup iteration.
	// When a training step line also matches this pattern, the step is
	// flagged as warmup and excluded from average step time computation.
	// Well-known captures: none required (presence of match is sufficient).
	// +optional
	WarmupStep *EventPattern `json:"warmupStep,omitempty"`

	// bandwidthResult matches NCCL bandwidth test result lines.
	// Used by the BandwidthMeasurement controller (not the GoodputMeasurement controller).
	// Well-known captures: size (int, bytes), algBW (float, GB/s), busBW (float, GB/s)
	// +optional
	BandwidthResult *EventPattern `json:"bandwidthResult,omitempty"`
}

type LogProfile struct {
	metav1.TypeMeta `json:",inline"`

	// metadata is a standard object metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitzero"`

	// spec defines the log parsing configuration
	// +required
	Spec LogProfileSpec `json:"spec"`
}

type LogProfileList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitzero"`
	Items           []LogProfile `json:"items"`
}

type LogProfileSpec struct {
	// timestamp configures how to parse timestamps captured by event patterns.
	// +kubebuilder:validation:Required
	Timestamp TimestampSpec `json:"timestamp"`

	// patterns defines regex patterns for extracting training events from log lines.
	// +kubebuilder:validation:Required
	Patterns LogPatternSet `json:"patterns"`

	// workerStrategy defines how to read logs from multi-worker jobs.
	// Default: single worker (read from worker-0 only).
	// +optional
	WorkerStrategy *WorkerStrategySpec `json:"workerStrategy,omitempty"`

	// containerName is the container to read logs from.
	// If empty, the controller auto-detects the container name from the workload.
	// +optional
	ContainerName string `json:"containerName,omitempty"`

	// warmupSteps is the number of initial training steps per run to flag as warmup.
	// Steps after applicationStart or checkpointRestore up to this count are marked
	// as warmup and excluded from average step time computation.
	// +optional
	WarmupSteps *int `json:"warmupSteps,omitempty"`

	// logInterval is the training framework's log interval (e.g., 10 for --log-interval 10).
	// When set, training steps whose GlobalStep is not a multiple of this value are
	// filtered out. This removes inflated "first after boundary" log lines that include
	// startup or restart overhead. Also used as the default step delta for the first
	// step in a window when computing total training time.
	// +optional
	LogInterval *int `json:"logInterval,omitempty"`
}

type MPIFramework struct {
	// binary is the MPI binary to run (e.g., "/usr/local/bin/all_reduce_perf_mpi").
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	Binary string `json:"binary"`

	// args are arguments to the MPI binary (e.g., ["-b", "8", "-e", "32G"]).
	// +optional
	Args []string `json:"args,omitempty"`

	// mpiArgs are extra arguments to mpirun itself (e.g., ["--allow-run-as-root"]).
	// Standard args (--mca plm_rsh_args, -x NCCL vars) are auto-injected.
	// +optional
	MpiArgs []string `json:"mpiArgs,omitempty"`

	// mpiImplementation is the MPI implementation to use.
	// Default: "OpenMPI".
	// +optional
	// +kubebuilder:default="OpenMPI"
	MpiImplementation string `json:"mpiImplementation,omitempty"`

	// mpirunPath is the path to the mpirun binary in the container image.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	MpirunPath string `json:"mpirunPath"`
}

type NodeFailureReason string

type NodeHealthMonitor struct {
	// cel defines a CEL expression evaluated against Node objects.
	// The expression must return a boolean. When it evaluates to true,
	// the node is considered to have a hardware failure.
	//
	// The full corev1.Node object is available via the 'node' variable:
	// - node.metadata: ObjectMeta (name, labels, annotations, etc.)
	// - node.spec: NodeSpec (taints, unschedulable, podCIDR, etc.)
	// - node.status: NodeStatus (conditions, addresses, capacity, allocatable, nodeInfo, etc.)
	//
	// Examples:
	// - "node.spec.taints.exists(t, t.key == 'nvidia.com/gpu-unhealthy')"
	// - "'gpud.nvidia.com/unhealthy' in node.metadata.labels"
	// - "node.status.conditions.exists(c, c.type == 'GPUHealthy' && c.status == 'False')"
	// - "node.spec.unschedulable == true"
	//
	// +optional
	CEL *CELNodeHealthCheck `json:"cel,omitempty"`
}

type NodeResult struct {
	// domain is the topology domain this node belongs to.
	// For example, if using "nvidia.com/gpu.clique", this is the clique value.
	Domain string `json:"domain"`

	// group is the group name that ran on this node.
	Group string `json:"group"`

	// status is the validation status for this node.
	// Examples: "Passed", "Failed", "Running"
	Status string `json:"status"`

	// iterations contains metrics for each iteration on this node.
	// +optional
	Iterations []IterationMetrics `json:"iterations,omitempty"`
}

type OrchestrationOverrideSpec struct {
	// target overrides which nodes to include in the orchestration.
	// +optional
	Target *TargetSpec `json:"target,omitempty"`

	// topology overrides rack/zone-aware placement configuration.
	// +optional
	Topology *TopologySpec `json:"topology,omitempty"`

	// execution overrides how jobs are scheduled across groups.
	// +optional
	Execution *ExecutionSpec `json:"execution,omitempty"`

	// iterations overrides the number of times to run all jobs.
	// +optional
	Iterations *int `json:"iterations,omitempty"`
}

type OrchestrationSpec struct {
	// target specifies which nodes to include in the orchestration.
	// When nil, all nodes in the cluster are targeted.
	// +optional
	Target *TargetSpec `json:"target,omitempty"`

	// topology configures rack/zone-aware placement.
	// When set, jobs are packed into the minimum number of topology domains.
	// When nil, nodes are grouped by simple chunking (no topology awareness).
	// +optional
	Topology *TopologySpec `json:"topology,omitempty"`

	// diagnose enables adaptive fault isolation using topology-aware hierarchical
	// group testing. Stage 1a screens each topology domain in parallel, Stage 1b
	// tests inter-domain fabric, Stage 2 bisects failed groups, Stage 3 confirms
	// suspects. Identifies all faulty nodes in O(d log N) tests.
	// +optional
	Diagnose *DiagnoseSpec `json:"diagnose,omitempty"`

	// execution defines how jobs are scheduled across groups.
	// +optional
	Execution ExecutionSpec `json:"execution,omitempty"`

	// iterations is the number of times to run all jobs (one full pass = one iteration).
	// Default is 1.
	// +kubebuilder:validation:Minimum=1
	// +kubebuilder:default=1
	// +optional
	Iterations int `json:"iterations,omitempty"`
}

type OrchestrationStatus struct {
	// totalNodes discovered from target.
	TotalNodes int `json:"totalNodes"`

	// nodesPerJob auto-detected from workload template.
	NodesPerJob int `json:"nodesPerJob"`

	// totalGroups created.
	TotalGroups int `json:"totalGroups"`

	// currentIteration is the iteration currently in progress (1-based).
	// +optional
	CurrentIteration int `json:"currentIteration,omitempty"`

	// completedIterations is the number of iterations fully completed.
	// +optional
	CompletedIterations int `json:"completedIterations,omitempty"`

	// iterationHistory records per-group outcomes for each completed iteration.
	// Appended when an iteration completes and more iterations remain.
	// The final iteration's results stay in Groups (not duplicated here).
	// +optional
	IterationHistory []IterationResult `json:"iterationHistory,omitempty"`

	// detectedPlatform is the cloud platform detected from node labels.
	// Used by WhenSpec for conditional dependency creation.
	// Examples: "aws", "gcp", "onprem", "unknown"
	// +optional
	DetectedPlatform string `json:"detectedPlatform,omitempty"`

	// detectedGPUArchitecture is the GPU architecture detected from node labels.
	// Examples: "h100", "gb200", "a100", "l40s", "unknown"
	// +optional
	DetectedGPUArchitecture string `json:"detectedGPUArchitecture,omitempty"`

	// excludedNodes lists nodes that matched the target but were dropped before
	// scheduling, and exclusionReason says why. A Workflow can succeed while
	// excluding nodes, so these record what was left untested. Causes: the node
	// was cordoned, the target set has more than one GPU architecture and the
	// run continued on the primary architecture alone, or the node reports
	// fewer allocatable GPUs than the workload requests per node.
	// +optional
	ExcludedNodes []string `json:"excludedNodes,omitempty"`

	// exclusionReason explains why excludedNodes were dropped.
	// +optional
	ExclusionReason string `json:"exclusionReason,omitempty"`

	// appliedOverrides records which overrides from spec.overrides[] were applied
	// after platform and GPU architecture detection completed.
	// +optional
	AppliedOverrides []AppliedOverride `json:"appliedOverrides,omitempty"`

	// diagnose tracks adaptive fault isolation algorithm progress.
	// +optional
	Diagnose *DiagnoseStatus `json:"diagnose,omitempty"`

	// Group phases are reset to Pending at the start of each new iteration.
	// +optional
	Groups []GroupStatus `json:"groups,omitempty"`
}

type OverrideSpec struct {
	// when defines conditions for applying this override.
	// +kubebuilder:validation:Required
	When WhenSpec `json:"when"`

	// jobTemplate is a strategic merge patch applied to the base jobTemplate.
	// Maps merge recursively, named arrays (e.g., env vars) merge by key,
	// unnamed arrays (e.g., args) replace entirely.
	// Stored as raw JSON to avoid CRD validation cost budget issues.
	// +optional
	// +kubebuilder:pruning:PreserveUnknownFields
	// +kubebuilder:validation:Schemaless
	JobTemplate *apiextensionsv1.JSON `json:"jobTemplate,omitempty"`

	// jobTemplatePatch is an RFC 6902 JSON Patch applied to the base jobTemplate.
	// Use for precise operations: remove specific env vars, add at index,
	// test a value before patching (preconditions).
	// Applied AFTER the jobTemplate strategic merge patch.
	// +optional
	// +kubebuilder:pruning:PreserveUnknownFields
	// +kubebuilder:validation:Schemaless
	JobTemplatePatch *apiextensionsv1.JSON `json:"jobTemplatePatch,omitempty"`

	// dependencies are additional dependencies appended when this override matches.
	// +optional
	Dependencies []DependencySpec `json:"dependencies,omitempty"`

	// orchestration overrides orchestration settings (target, topology, execution, iterations).
	// Non-nil fields replace the base value; nil fields are preserved.
	// +optional
	Orchestration *OrchestrationOverrideSpec `json:"orchestration,omitempty"`
}

type PendingInterruptionStatus struct {
	// tCheckpoint is the time of the last checkpoint before the interruption.
	// +optional
	TCheckpoint *metav1.Time `json:"tCheckpoint,omitempty"`

	// tInterrupt is the time the job was interrupted (workload failure time).
	// +optional
	TInterrupt *metav1.Time `json:"tInterrupt,omitempty"`

	// checkpointStep is the step number of the last saved checkpoint before interruption.
	// +optional
	CheckpointStep int `json:"checkpointStep,omitempty"`

	// lastStep is the last training step observed before interruption.
	// +optional
	LastStep int `json:"lastStep,omitempty"`

	// tCh is the lost work time in seconds (time between last checkpoint and interruption).
	// +optional
	TCh string `json:"tCh,omitempty"`

	// reason is the reason for the interruption.
	// +optional
	Reason string `json:"reason,omitempty"`
}

type PerformanceValidationSpec struct {
	// enabled determines if performance validation is active.
	// +kubebuilder:default=false
	Enabled bool `json:"enabled"`

	// plugin is the name of the benchmark plugin to use.
	// Examples: "nemo-training", "nccl-tests"
	// +optional
	Plugin string `json:"plugin,omitempty"`

	// config contains plugin-specific configuration parameters.
	// +optional
	// +kubebuilder:pruning:PreserveUnknownFields
	// +kubebuilder:validation:Schemaless
	Config *apiextensionsv1.JSON `json:"config,omitempty"`

	// tracking defines what results to track.
	// +optional
	Tracking TrackingSpec `json:"tracking,omitempty"`

	// thresholds defines minimum acceptable performance thresholds.
	// When set, the Workflow controller propagates these to each Job's
	// measurement configs during creation. The Job controller evaluates
	// them after the workload succeeds and sets the ValidationFailed
	// condition if any threshold is not met.
	// +optional
	Thresholds *ThresholdSpec `json:"thresholds,omitempty"`

	// measurementTimeout is the maximum time to wait after a Job succeeds for
	// measurement data before failing threshold validation. Propagated to each
	// Job's spec during creation.
	// +optional
	MeasurementTimeout *metav1.Duration `json:"measurementTimeout,omitempty"`
}

type StringMatchSpec struct {
	// equals checks for exact match.
	// +optional
	Equals string `json:"equals,omitempty"`

	// in checks if value is in the list.
	// +optional
	In []string `json:"in,omitempty"`

	// notIn checks if value is not in the list.
	// +optional
	NotIn []string `json:"notIn,omitempty"`
}

type TaintSelector struct {
	// key is the taint key to match.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	Key string `json:"key"`

	// value is the taint value to match. If empty, matches any value.
	// +optional
	Value string `json:"value,omitempty"`

	// effect is the taint effect to match (NoSchedule, PreferNoSchedule, NoExecute).
	// If empty, matches any effect.
	// +optional
	Effect corev1.TaintEffect `json:"effect,omitempty"`
}

type TargetSpec struct {
	// nodeSelector selects nodes by label key-value pairs.
	// +optional
	NodeSelector map[string]string `json:"nodeSelector,omitempty"`

	// matchExpressions is a list of node selector requirements.
	// Supports In, NotIn, Exists, DoesNotExist operators.
	// All expressions must match (AND with nodeSelector).
	// +optional
	MatchExpressions []corev1.NodeSelectorRequirement `json:"matchExpressions,omitempty"`

	// nodeNames explicitly lists nodes to include.
	// If specified, nodeSelector is ignored.
	// +optional
	NodeNames []string `json:"nodeNames,omitempty"`

	// taintSelectors selects nodes that have ALL matching taints.
	// A node must have every taint in this list to be included.
	//
	// The controller auto-injects matching tolerations onto the workload pods so
	// they can schedule onto the selected tainted nodes. Setting this field is
	// the supported way to run training workloads (e.g. TrainJob) on tainted
	// nodes. When taintSelectors is set, the explicit tolerations replace the
	// blanket Operator: Exists toleration that MPI workloads (NCCL tests)
	// otherwise receive — declared selectors always win.
	// +optional
	TaintSelectors []TaintSelector `json:"taintSelectors,omitempty"`
}

type ThresholdSpec struct {
	// thresholds maps metric names to CEL expressions.
	// +optional
	Thresholds map[string]string `json:"thresholds,omitempty"`
}

type TimestampSpec struct {
	// layout is the Go time layout string for parsing (?P<timestamp>...) captures.
	// Examples: "2006-01-02T15:04:05.999999999Z" (NeMo/RFC3339),
	//           "2006-01-02 15:04:05.999999" (Megatron bracketed)
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	Layout string `json:"layout"`
}

type TopologyMatchSpec struct {
	// mode matches against topology mode.
	// Examples: "single-rack", "multi-rack", "multi-zone"
	// +optional
	Mode string `json:"mode,omitempty"`

	// domainCount matches against the number of topology domains (racks, zones, etc.).
	// +optional
	DomainCount *IntMatchSpec `json:"domainCount,omitempty"`
}

type TopologySpec struct {
	// topologyKey is the node label key that identifies the topology domain.
	// Examples: "nvidia.com/gpu.clique", "topology.kubernetes.io/zone"
	// Nodes with the same label value are in the same domain (rack/zone).
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	TopologyKey string `json:"topologyKey"`

	// strictDomain creates one group per topology domain at its natural size.
	// Groups never cross domain boundaries. Set by testScale=intra-rack.
	// +optional
	StrictDomain bool `json:"strictDomain,omitempty"`
}

type TorchFramework struct {
	// script is the path to run (torchrun <script>).
	// Mutually exclusive with module.
	// +optional
	// +kubebuilder:validation:MinLength=1
	Script string `json:"script,omitempty"`

	// module is the Python module to run (torchrun -m <module>).
	// Mutually exclusive with script.
	// +optional
	// +kubebuilder:validation:MinLength=1
	Module string `json:"module,omitempty"`

	// args are additional arguments passed to the script or module.
	// +optional
	Args []string `json:"args,omitempty"`
}

type TrackingSpec struct {
	// perNode tracks results for each individual node.
	// +kubebuilder:default=false
	// +optional
	PerNode bool `json:"perNode,omitempty"`

	// perDomain tracks aggregated results per topology domain.
	// Domains are defined by the topology key in orchestration.
	// For example, if using "nvidia.com/gpu.clique", results
	// are aggregated per unique clique value.
	// +kubebuilder:default=false
	// +optional
	PerDomain bool `json:"perDomain,omitempty"`
}

type ValidationSpec struct {
	// performance configures performance validation and metrics collection.
	// +optional
	Performance *PerformanceValidationSpec `json:"performance,omitempty"`
}

type WhenSpec struct {
	// gpuArchitecture matches against detected GPU architecture.
	// +optional
	GPUArchitecture *StringMatchSpec `json:"gpuArchitecture,omitempty"`

	// platform matches against detected cloud platform.
	// +optional
	Platform *StringMatchSpec `json:"platform,omitempty"`

	// workloadKind matches against the workload kind.
	// Example: "TrainJob"
	// +optional
	WorkloadKind string `json:"workloadKind,omitempty"`

	// topology matches against cluster topology characteristics.
	// +optional
	Topology *TopologyMatchSpec `json:"topology,omitempty"`

	// config matches against values in validation.performance.config.
	// +optional
	// +kubebuilder:pruning:PreserveUnknownFields
	// +kubebuilder:validation:Schemaless
	Config *apiextensionsv1.JSON `json:"config,omitempty"`

	// expression is a CEL expression that evaluates to a boolean.
	// Available variables: gpuArchitecture, platform, workloadKind, topology, config
	// Example: "gpuArchitecture == 'gb200' && platform == 'aws'"
	// +optional
	Expression string `json:"expression,omitempty"`
}

type WorkerStrategySpec struct {
	// type determines the log reading strategy.
	// "Single": read all data from worker-0.
	// "Multi": read lifecycle data from one pod, training steps from another.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:Enum=Single;Multi
	Type string `json:"type"`

	// lifecyclePod specifies which worker has lifecycle data (checkpoints, app start).
	// "first" (worker-0) or "last" (highest index). Default: "first".
	// Only used when type is "Multi".
	// +kubebuilder:validation:Enum=first;last
	// +kubebuilder:default=first
	// +optional
	LifecyclePod string `json:"lifecyclePod,omitempty"`

	// trainingStepPod specifies which worker has training step data (iterations, metrics).
	// "first" or "last". Default: "first".
	// Only used when type is "Multi".
	// +kubebuilder:validation:Enum=first;last
	// +kubebuilder:default=first
	// +optional
	TrainingStepPod string `json:"trainingStepPod,omitempty"`

	// replicatedJobName selects which JobSet replicatedJob to read logs from.
	// For MPI workloads, set to "launcher" (output goes to the launcher pod).
	// For torch workloads, use the default "trainer".
	// Only applies to TrainJob workloads that use JobSet.
	// +kubebuilder:default=trainer
	// +optional
	ReplicatedJobName string `json:"replicatedJobName,omitempty"`
}

type Workflow struct {
	metav1.TypeMeta `json:",inline"`

	// metadata is a standard object metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitzero"`

	// spec defines the desired state of Workflow
	// +required
	Spec WorkflowSpec `json:"spec"`

	// status defines the observed state of Workflow
	// +optional
	Status WorkflowStatus `json:"status,omitzero"`
}

type WorkflowList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitzero"`
	Items           []Workflow `json:"items"`
}

type WorkflowReference struct {
	// name is the name of the Workflow resource.
	Name string `json:"name"`

	// namespace is the namespace of the Workflow resource.
	// +optional
	Namespace string `json:"namespace,omitempty"`
}

type WorkflowSpec struct {
	// namespace is the target namespace for Jobs and dependencies created by this Workflow.
	// If not specified, the controller auto-generates one.
	// The controller creates the namespace if it doesn't exist and sets an owner reference
	// so it's cleaned up when the Workflow is deleted.
	// +optional
	Namespace string `json:"namespace,omitempty"`

	// jobTemplate defines the Job template used to create Jobs for each execution group.
	// +kubebuilder:validation:Required
	JobTemplate JobTemplateSpec `json:"jobTemplate"`

	// orchestration defines topology-aware orchestration strategy.
	// +kubebuilder:validation:Required
	Orchestration OrchestrationSpec `json:"orchestration"`

	// dependencies are resources required for the workload (ComputeDomain, etc.)
	// Resources are conditionally created based on "when" clauses.
	// +optional
	Dependencies []DependencySpec `json:"dependencies,omitempty"`

	// overrides defines conditional patches to the base spec.
	// Overrides are evaluated in order; all matching overrides are applied sequentially.
	// +optional
	Overrides []OverrideSpec `json:"overrides,omitempty"`

	// validation defines performance validation and metrics collection.
	// +optional
	Validation *ValidationSpec `json:"validation,omitempty"`
}

type WorkflowStatus struct {
	// conditions represent the current state of the Workflow resource.
	//
	// Condition types:
	// - "InProgress": the Workflow is currently running
	// - "Succeeded": the Workflow completed successfully
	// - "Failed": the Workflow has failed
	//
	// The status of each condition is one of True, False, or Unknown.
	// +listType=map
	// +listMapKey=type
	// +optional
	Conditions []metav1.Condition `json:"conditions,omitempty"`

	// namespace is the resolved namespace where Jobs and dependencies are created.
	// +optional
	Namespace string `json:"namespace,omitempty"`

	// succeededNodesRef references the ConfigMap holding this Workflow's
	// succeeded-nodes list (comma-separated node names). It is written on terminal
	// success and on terminal failure (capturing the nodes that passed).
	// +optional
	SucceededNodesRef *corev1.TypedLocalObjectReference `json:"succeededNodesRef,omitempty"`

	// failedNodesRef references the ConfigMap holding this Workflow's failed-nodes
	// list (name, reason, message). It is written incrementally as Jobs
	// fail.
	// +optional
	FailedNodesRef *corev1.TypedLocalObjectReference `json:"failedNodesRef,omitempty"`

	// dependencyRefs lists the dependency resources created by this Workflow.
	// +optional
	DependencyRefs []DependencyResourceRef `json:"dependencyRefs,omitempty"`

	// orchestration contains the state of orchestration execution.
	// +optional
	Orchestration *OrchestrationStatus `json:"orchestration,omitempty"`

	// nodeResults contains per-node performance metrics.
	// Only populated when spec.validation.performance.tracking.perNode is true.
	// +optional
	NodeResults map[string]NodeResult `json:"nodeResults,omitempty"`

	// domainResults contains aggregated metrics grouped by topology domain values.
	// Key is the domain value (e.g., "clique-0", "rack-1", "zone-a").
	// Only populated when spec.validation.performance.tracking.perDomain is true.
	// +optional
	DomainResults map[string]DomainResult `json:"domainResults,omitempty"`
}

type WorkloadConfig struct {
	// inline specifies config data as key-value pairs (filename -> content).
	// Creates a ConfigMap and mounts it at /config.
	// +optional
	Inline map[string]string `json:"inline,omitempty"`

	// configMapRef references an existing ConfigMap by name.
	// Mounted at /config.
	// +optional
	ConfigMapRef *corev1.LocalObjectReference `json:"configMapRef,omitempty"`
}

type WorkloadOrchestration struct {
	// testScale controls the node grouping strategy.
	//   - "intra-node": each node tested independently (numNodes=1 per job)
	//   - "intra-rack": one job per topology domain (nvidia.com/gpu.clique)
	//   - "full-scale": all nodes in a single group (default)
	// +optional
	// +kubebuilder:validation:Enum=intra-node;intra-rack;full-scale
	TestScale string `json:"testScale,omitempty"`

	// repeatCount runs the entire orchestration N times. Default: 1.
	// +optional
	// +kubebuilder:validation:Minimum=1
	RepeatCount *int32 `json:"repeatCount,omitempty"`

	// maxConcurrent limits the number of simultaneously running jobs.
	// 0 means unlimited. Default: 0.
	// +optional
	// +kubebuilder:validation:Minimum=0
	MaxConcurrent *int32 `json:"maxConcurrent,omitempty"`

	// timeoutPerJob is the maximum time to wait for each job to complete.
	// Accepts Go duration strings (e.g., "30m", "1h"). Default: "1h".
	// +optional
	TimeoutPerJob string `json:"timeoutPerJob,omitempty"`
}

type WorkloadReference struct {
	// apiVersion is the API version of the workload resource.
	// +kubebuilder:validation:Required
	APIVersion string `json:"apiVersion"`

	// kind is the kind of the workload resource.
	// +kubebuilder:validation:Required
	Kind string `json:"kind"`

	// name is the name of the workload resource.
	// +kubebuilder:validation:Required
	Name string `json:"name"`

	// namespace is the namespace of the workload resource.
	// If not specified, defaults to the Job's namespace.
	// +optional
	Namespace string `json:"namespace,omitempty"`
}

type WorkloadRun struct {
	metav1.TypeMeta `json:",inline"`

	// metadata is a standard object metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitzero"`

	// spec defines the desired state of WorkloadRun.
	// The entire spec is immutable after creation: once status.workflowRef is
	// set the controller only mirrors the existing Workflow and never rebuilds
	// it, so accepting edits would silently ignore them. To run with different
	// inputs, delete the WorkloadRun and create a new one.
	// +required
	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable after creation"
	Spec WorkloadRunSpec `json:"spec"`

	// status defines the observed state of WorkloadRun
	// +optional
	Status WorkloadRunStatus `json:"status,omitzero"`
}

type WorkloadRunCheckpoint struct {
	// storageSize is the PVC size (e.g., "10Ti", "500Gi").
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:Pattern=`^([0-9]+(\.[0-9]+)?)(Ki|Mi|Gi|Ti|Pi|Ei)?$`
	StorageSize string `json:"storageSize"`

	// storageClassName is the StorageClass to use. Defaults to cluster default.
	// +optional
	StorageClassName *string `json:"storageClassName,omitempty"`

	// maxRestarts is how many times to restart from checkpoint on failure.
	// Default: 0.
	// +optional
	// +kubebuilder:validation:Minimum=0
	MaxRestarts *int32 `json:"maxRestarts,omitempty"`
}

type WorkloadRunList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitzero"`
	Items           []WorkloadRun `json:"items"`
}

type WorkloadRunSpec struct {
	// target specifies which nodes to run the workload on.
	// When omitted, all GPU nodes (nvidia.com/gpu.present=true) are selected.
	// GPU architecture and CSP platform are auto-detected from target nodes.
	// +optional
	Target *TargetSpec `json:"target,omitempty"`

	// image is the container image for the workload.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:MinLength=1
	Image string `json:"image"`

	// framework specifies how to run the workload.
	// Exactly one of torch, mpi, or exec must be set.
	// +kubebuilder:validation:Required
	Framework FrameworkSpec `json:"framework"`

	// numNodes is the number of nodes to use for the workload.
	// +kubebuilder:validation:Required
	// +kubebuilder:validation:Minimum=1
	NumNodes int32 `json:"numNodes"`

	// config provides model configuration data.
	// When inline is specified, a ConfigMap is created and mounted at /config.
	// When configMapRef is specified, the referenced ConfigMap is mounted at /config.
	// +optional
	Config *WorkloadConfig `json:"config,omitempty"`

	// gpusPerNode overrides the auto-detected GPU count per node.
	// If not specified, derived from GPU architecture (8 for H100/A100, 4 for GB200/GB300).
	// +optional
	// +kubebuilder:validation:Minimum=1
	GpusPerNode *int32 `json:"gpusPerNode,omitempty"`

	// mlnxPerNode overrides the auto-detected Mellanox NIC count per node.
	// Used by platforms with InfiniBand or RoCE networking (Azure, OCI, TogetherAI).
	// If not specified, derived from GPU architecture and platform via the
	// catalog's gpu-defaults.yaml.
	// +optional
	// +kubebuilder:validation:Minimum=0
	MlnxPerNode *int32 `json:"mlnxPerNode,omitempty"`

	// imagePullSecrets references secrets for pulling the container image.
	// +optional
	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`

	// env specifies additional environment variables for the workload container.
	// These are merged with auto-detected NCCL/platform env vars.
	// User-specified values take precedence over auto-detected values.
	// +optional
	Env []corev1.EnvVar `json:"env,omitempty"`

	// volumes defines additional volumes for the workload pods.
	// A shared-memory volume (/dev/shm) is always auto-created.
	// +optional
	Volumes []corev1.Volume `json:"volumes,omitempty"`

	// volumeMounts defines additional volume mounts for the workload container.
	// +optional
	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`

	// enableMNNVL explicitly controls Multi-Node NVLink (NCCL_MNNVL_ENABLE).
	// If nil, auto-detected from GPU architecture (enabled for GB200/GB300).
	// +optional
	EnableMNNVL *bool `json:"enableMNNVL,omitempty"`

	// checkpoint configures checkpoint storage. When set, provisions a PVC
	// and enables checkpoint-based restart.
	// +optional
	Checkpoint *WorkloadRunCheckpoint `json:"checkpoint,omitempty"`

	// resources overrides GPU/memory/CPU resource requests and limits for the
	// workload container. If not specified, GPU resources are auto-set from gpusPerNode.
	// +optional
	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`

	// initContainers specifies init containers to run before the workload.
	// +optional
	InitContainers []corev1.Container `json:"initContainers,omitempty"`

	// orchestration controls how nodes are grouped and tested.
	// +optional
	Orchestration *WorkloadOrchestration `json:"orchestration,omitempty"`

	// thresholds defines performance pass/fail criteria as CEL expressions.
	// Keys are metric names (e.g., "busBandwidthGBps", "avgTFLOPsPerGPU").
	// Values are CEL expressions using a `value` variable (e.g., "value >= 900").
	// +optional
	Thresholds map[string]string `json:"thresholds,omitempty"`

	// goodputMeasurement enables training goodput tracking.
	// When set, a GoodputMeasurement resource is auto-created for each Job.
	// +optional
	GoodputMeasurement *GoodputMeasurementConfig `json:"goodputMeasurement,omitempty"`

	// bandwidthMeasurement enables NCCL bandwidth tracking.
	// When set, a BandwidthMeasurement resource is auto-created for each Job.
	// +optional
	BandwidthMeasurement *BandwidthMeasurementConfig `json:"bandwidthMeasurement,omitempty"`

	// overrides allows advanced users to inject additional CSP/GPU-specific patches.
	// These are appended to the auto-generated platform overrides on the Workflow.
	// +optional
	Overrides []OverrideSpec `json:"overrides,omitempty"`

	// gangScheduler opts workload pods into a gang-aware scheduler such as KAI Scheduler.
	// When set, schedulerName is injected into every pod template and the queue label
	// is applied so the scheduler holds all pods until the full gang can be placed.
	// +optional
	GangScheduler *GangSchedulerSpec `json:"gangScheduler,omitempty"`
}

type WorkloadRunStatus struct {
	// conditions represent the current state of the WorkloadRun.
	// +listType=map
	// +listMapKey=type
	// +optional
	Conditions []metav1.Condition `json:"conditions,omitempty"`

	// workflowRef references the Workflow created by this WorkloadRun.
	// +optional
	WorkflowRef *WorkflowReference `json:"workflowRef,omitempty"`

	// detectedPlatform is the auto-detected CSP platform (e.g., "aws", "gcp", "azure").
	// +optional
	DetectedPlatform string `json:"detectedPlatform,omitempty"`

	// detectedGPUArchitecture is the auto-detected GPU type (e.g., "h100", "gb200").
	// +optional
	DetectedGPUArchitecture string `json:"detectedGPUArchitecture,omitempty"`

	// resolvedGpusPerNode is the final GPU count per node used for the workload.
	// +optional
	ResolvedGpusPerNode int32 `json:"resolvedGpusPerNode,omitempty"`

	// succeededNodesRef references the ConfigMap holding the underlying Workflow's
	// succeeded-nodes list (comma-separated node names).
	// +optional
	SucceededNodesRef *corev1.TypedLocalObjectReference `json:"succeededNodesRef,omitempty"`

	// failedNodesRef references the ConfigMap holding the underlying Workflow's
	// failed-nodes list (name, reason, message).
	// +optional
	FailedNodesRef *corev1.TypedLocalObjectReference `json:"failedNodesRef,omitempty"`
}

type WorkloadSpec struct {
	// trainJob configures a Kubeflow TrainJob workload.
	// +optional
	TrainJob *trainerv1alpha1.TrainJobSpec `json:"trainJob,omitempty"`
}