Documentation
¶
Index ¶
- Constants
- func ApplyOverridesWithTracking(spec *nvcrev1alpha1.WorkflowSpec, octx OverrideContext) ([]nvcrev1alpha1.AppliedOverride, error)
- func ApplyWRPreTemplateOverrides(spec *nvcrev1alpha1.WorkloadRunSpec, overrides []platform.WorkloadRunOverride, ...)
- func CacheOptions() cache.Options
- func CondIsTrue(conditions []metav1.Condition, condType string) bool
- func CondMessage(conditions []metav1.Condition, condType string) string
- func DefaultEnableMNNVL(gpuArch string) bool
- func DetectGPUArchitecture(nodes []corev1.Node) string
- func DetectPlatform(nodes []corev1.Node) string
- func DiscoverTargetNodes(ctx context.Context, reader client.Reader, target *nvcrev1alpha1.TargetSpec) ([]corev1.Node, error)
- func NodesPerJobForScale(orch *nvcrev1alpha1.WorkloadOrchestration, numNodes int32) int32
- func RegisterFieldIndexes(ctx context.Context, indexer client.FieldIndexer) error
- func ResolveNICResourceName(field *string, platformName, gpuArch string, nodes []corev1.Node, ...) (name, refusalMessage string, detectionRan bool)
- func ResolveOptions(global *nvcrev1alpha1.CategoryOptions, override *nvcrev1alpha1.CategoryOptions) nvcrev1alpha1.CategoryOptions
- func TargetsCordonedNodes(target *nvcrev1alpha1.TargetSpec) bool
- type BandwidthMeasurementReconciler
- type CertificationReconciler
- type GoodputMeasurementReconciler
- type JobReconciler
- type OverrideContext
- type WorkflowReconciler
- type WorkloadRunReconciler
Constants ¶
const ( // ReasonNotApplicable is the default condition reason for inactive conditions. ReasonNotApplicable = "NotApplicable" // ReasonNICResourceDetection is shared by the Certification and WorkloadRun // tiers: both emit a Normal event under it when NIC resource auto-detection // ran (on-prem GB200/GB300 target with nicResourceName unset) and found // zero or multiple qualifying candidates, so nothing was injected. ReasonNICResourceDetection = "NICResourceDetection" )
const ( // ReasonWaitingForNodes marks a Certification that found no schedulable nodes // and is retrying. Without it the wait is invisible: the retry only logged, so // the CR carried no conditions at all for up to nodeDiscoveryTimeout and an // operator had nothing to explain why nothing was happening. ReasonWaitingForNodes = "WaitingForNodes" ReasonWorkflowCreated = "WorkflowCreated" ReasonWorkflowRunning = "WorkflowRunning" ReasonAllWorkflowsSucceeded = "AllWorkflowsSucceeded" ReasonWorkflowFailed = "WorkflowFailed" ReasonWorkflowValidationFailed = "WorkflowValidationFailed" ReasonCategoryNotFound = "CategoryNotFound" ReasonBuildFailed = "BuildFailed" ReasonWorkflowDeleted = "WorkflowDeleted" ReasonWorkflowSucceeded = "WorkflowSucceeded" ReasonThresholdViolation = "ThresholdViolation" // ReasonWorkflowNameCollision marks a Certification whose generated child // Workflow name is already taken by a Workflow this Certification does not // own. The foreign object is neither adopted nor recorded for cleanup. ReasonWorkflowNameCollision = "WorkflowNameCollision" // ReasonWorkflowCreationError indicates a child Workflow could not be // created. Emitted as a Warning event at the failed Create call by the // Certification and WorkloadRun reconcilers, which both create Workflows, // so the API server's rejection is visible on the parent object rather // than only in the controller log. Mirrors ReasonJobCreationError // (Workflow tier) and ReasonWorkloadCreationError (Job tier). ReasonWorkflowCreationError = "WorkflowCreationError" )
Certification tier reasons (Certification → Workflow).
const ( ReasonJobCreated = "JobCreated" ReasonJobRunning = "JobRunning" ReasonJobCompleted = "JobCompleted" ReasonJobFailed = "JobFailed" ReasonJobHardwareFailed = "JobHardwareFailed" ReasonJobCreationError = "JobCreationError" ReasonJobValidationFailed = "JobValidationFailed" // ReasonJobTimedOut marks a Job's Failed condition set by the Workflow when // the Job exceeded timeoutPerJob. Timed-out jobs are never retried. ReasonJobTimedOut = "JobTimedOut" ReasonGroupsPartitioned = "GroupsPartitioned" ReasonIterationCompleted = "IterationCompleted" ReasonIterationsFailed = "IterationsFailed" ReasonDependencyCreationError = "DependencyCreationError" ReasonNodeDiscoveryError = "NodeDiscoveryError" ReasonPartitionError = "PartitionError" // ReasonHeterogeneousPlatformStatusUpdateFailed reports inconsistent node // platforms when recording the Workflow's Failed condition also fails. // This action-failure Warning does not claim a persisted Failed phase. ReasonHeterogeneousPlatformStatusUpdateFailed = "HeterogeneousPlatformStatusUpdateFailed" // ReasonOverrideErrorStatusUpdateFailed reports an override application // error when recording the Workflow's Failed condition also fails. Both // override guards use this action-failure Warning, not a phase notification. ReasonOverrideErrorStatusUpdateFailed = "OverrideErrorStatusUpdateFailed" // ReasonJobNameCollision marks a Workflow whose generated Job name is // already taken by a Job this Workflow does not control. The foreign // object is neither adopted nor recorded for cleanup. ReasonJobNameCollision = "JobNameCollision" // ReasonDependencyNameCollision marks a Workflow whose dependency resource // name is already taken by an object this Workflow did not create. The // foreign object is neither adopted nor recorded for cleanup. ReasonDependencyNameCollision = "DependencyNameCollision" // ReasonGangSchedulingConflict marks a Workflow whose resolved Job template // is invalid after overrides. This includes workload metadata that fails // label validation even when no gang scheduler is configured, as well as a // template that no longer agrees with the gang-scheduling intent its owner // persisted. The invalid result is reported rather than repaired. ReasonGangSchedulingConflict = "GangSchedulingConflict" )
Workflow tier reasons (Workflow → Job).
const ( ReasonWorkloadCreated = "WorkloadCreated" // ReasonWorkloadPending indicates the workload exists but has not started // running — e.g. a TrainJob suspended by Kueue while it waits for quota. // The Job stays InProgress; pending time is not counted against // timeoutPerJob or stall detection. ReasonWorkloadPending = "WorkloadPending" ReasonWorkloadRunning = "WorkloadRunning" ReasonWorkloadCompleted = "WorkloadCompleted" ReasonWorkloadFailed = "WorkloadFailed" ReasonWorkloadCreationError = "WorkloadCreationError" ReasonWorkloadStalled = "WorkloadStalled" // ReasonMeasurementCreationError indicates a GoodputMeasurement or // BandwidthMeasurement child resource could not be created. Handling is // non-fatal, so this event is the operator-visible signal; a threshold that // depends on the missing measurement still fails closed via // ReasonMeasurementTimeout. ReasonMeasurementCreationError = "MeasurementCreationError" ReasonHardwareFailureDetected = "HardwareFailureDetected" )
Job tier reasons (Job → Workload).
const ( FrameworkTorch = "torch" FrameworkMPI = "mpi" FrameworkExec = "exec" )
Framework type constants for WorkloadRun.
const GPUNodeLabel = "nvidia.com/gpu.present"
GPUNodeLabel marks a node as GPU-equipped. The controllers only ever operate on GPU nodes — discoverTargetNodes filters on this label before a node can be partitioned into a group — so it also defines the Node cache boundary.
const ( // ReasonBuildFailedStatusUpdateFailed reports a WorkloadRun build error // when recording its Failed condition also fails. This action-failure // Warning does not claim that the Failed phase was persisted. ReasonBuildFailedStatusUpdateFailed = "BuildFailedStatusUpdateFailed" )
WorkloadRun tier reasons (WorkloadRun → Workflow).
Variables ¶
This section is empty.
Functions ¶
func ApplyOverridesWithTracking ¶
func ApplyOverridesWithTracking(spec *nvcrev1alpha1.WorkflowSpec, octx OverrideContext) ([]nvcrev1alpha1.AppliedOverride, error)
ApplyOverridesWithTracking is the exported version of applyOverridesWithTracking for use by CLI tools.
func ApplyWRPreTemplateOverrides ¶
func ApplyWRPreTemplateOverrides(spec *nvcrev1alpha1.WorkloadRunSpec, overrides []platform.WorkloadRunOverride, octx OverrideContext)
ApplyWRPreTemplateOverrides is the exported version of applyWRPreTemplateOverrides for use by CLI tools, so the "workloadrun render" preview bakes the same platform mpirun args into the spec that the controller bakes at reconcile time.
func CacheOptions ¶
CacheOptions returns the informer cache configuration for the manager.
Without it, controller-runtime caches every object of every watched kind, cluster-wide and in full. This controller watches Nodes, Pods and PersistentVolumes, so on a large GPU cluster the default cache holds the entire Pod population of the cluster in the controller's heap — against a 1Gi limit. Two reductions are applied:
Nodes are restricted to GPU nodes. Non-GPU nodes (control plane, CPU workers, infra) are never targets, never run burn-in pods, and never need health evaluation, so caching and watching them is pure overhead.
managedFields is stripped from every cached object. It is metadata the controllers never read and is frequently a large fraction of an object's serialized size, particularly for Pods and CRs written by several controllers.
Pods are deliberately not label-scoped. The obvious selector, nvcre.nvidia.com/job, is injected only into the worker replicatedJob's pod template — MPI launcher pods do not carry it (see workerTargetJob in pkg/workload), and the launcher is exactly the pod that NCCL bandwidth measurement and timeout log capture read from. Scoping on that label would make launchers invisible to the cache. Pod reads are instead kept cheap via field indexes; see indexes.go.
func CondIsTrue ¶
CondIsTrue returns true if the named condition has Status=True.
func CondMessage ¶
CondMessage returns the message for a condition type, or empty string if not found.
func DefaultEnableMNNVL ¶
DefaultEnableMNNVL returns whether MNNVL should be enabled for the given GPU architecture. GB200 and GB300 use Multi-Node NVLink and benefit from MNNVL.
func DetectGPUArchitecture ¶
DetectGPUArchitecture is the exported version of detectGPUArchitecture for use by CLI tools. Like the controllers, it reports the architecture that the most labeled nodes carry (gpu.MajorityArchitecture), so render, cluster info, and workloadrun previews agree with what a reconcile of the same target would detect.
func DetectPlatform ¶
DetectPlatform is the exported version of detectPlatform for use by CLI tools.
func DiscoverTargetNodes ¶
func DiscoverTargetNodes(ctx context.Context, reader client.Reader, target *nvcrev1alpha1.TargetSpec) ([]corev1.Node, error)
DiscoverTargetNodes is the exported version of discoverTargetNodes for use by CLI tools.
It drops the cordoned-node list that discoverTargetNodes also returns. Only the Workflow reconciler records coverage on status; CLI callers want the nodes a run would actually use. Widen this if a CLI ever needs to report what was skipped.
func NodesPerJobForScale ¶
func NodesPerJobForScale(orch *nvcrev1alpha1.WorkloadOrchestration, numNodes int32) int32
NodesPerJobForScale returns how many nodes a single Job should span. intra-node means each node is tested on its own, so one node per Job however many the run targets; the Workflow then makes one group per node. Anything else keeps the requested count.
It lives here so the reconcile path and the "workloadrun render" preview in pkg/workloadrun (which imports this package) apply the same rule — issue #85 was the controller not applying it at all.
func RegisterFieldIndexes ¶
func RegisterFieldIndexes(ctx context.Context, indexer client.FieldIndexer) error
RegisterFieldIndexes registers every field index the controllers rely on.
It must be called once per manager, before the controllers start, and is the single source of truth for both production (cmd/manager) and the integration harness — registering these in two places previously allowed prod and test to drift apart.
func ResolveNICResourceName ¶ added in v0.3.0
func ResolveNICResourceName( field *string, platformName, gpuArch string, nodes []corev1.Node, mlnxPerNode int32, ) (name, refusalMessage string, detectionRan bool)
ResolveNICResourceName is the exported version of resolveNICResourceName for use by the CLI dry-run paths, so "certification render --dry-run" and "workloadrun render --dry-run" resolve the NIC resource exactly as a reconcile of the same target would: field wins, detection only for on-prem GB200/GB300, and only a single candidate allocatable at the resolved mlnxPerNode count on every node is used. mlnxPerNode must be the caller's fully resolved value (field or catalog default).
refusalMessage is non-empty exactly when detection ran and refused to pick (name == ""); it is the same text the controllers emit as the NICResourceDetection event, for the CLI to print to stderr.
func ResolveOptions ¶
func ResolveOptions(global *nvcrev1alpha1.CategoryOptions, override *nvcrev1alpha1.CategoryOptions) nvcrev1alpha1.CategoryOptions
ResolveOptions merges per-category overrides with global defaults. Returns a flat CategoryOptions with all values resolved.
func TargetsCordonedNodes ¶ added in v0.5.0
func TargetsCordonedNodes(target *nvcrev1alpha1.TargetSpec) bool
TargetsCordonedNodes returns true if the given targetSpec contains a taintSelectors entry which targets the node.kubernetes.io/unschedulable taint.
Types ¶
type BandwidthMeasurementReconciler ¶
type BandwidthMeasurementReconciler struct {
client.Client
// APIReader is an uncached client used for pod discovery. Pod watch events
// from the informer cache can lag behind the API server by several seconds,
// causing the "pod not found" requeue loop to spin longer than necessary.
// A direct API read guarantees the freshest view at the cost of one extra
// API server call per reconcile cycle while waiting for pods.
APIReader client.Reader
Scheme *runtime.Scheme
Clientset *kubernetes.Clientset
Recorder events.EventRecorder
LogFetcher podlogs.PodLogFetcher // if nil, defaults to Clientset-backed fetcher
// MaxConcurrentReconciles bounds the number of BandwidthMeasurement objects reconciled concurrently.
MaxConcurrentReconciles int
// RequeueInterval is the interval between reconcile cycles when polling.
// Tests set this to 1s; production uses 15s.
RequeueInterval time.Duration
// contains filtered or unexported fields
}
BandwidthMeasurementReconciler reconciles a BandwidthMeasurement object.
func (*BandwidthMeasurementReconciler) Reconcile ¶
func (r *BandwidthMeasurementReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)
Reconcile is part of the main kubernetes reconciliation loop.
func (*BandwidthMeasurementReconciler) SetupWithManager ¶
func (r *BandwidthMeasurementReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager sets up the controller with the Manager.
type CertificationReconciler ¶
type CertificationReconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder events.EventRecorder
WorkflowRequeueInterval time.Duration
// MaxConcurrentReconciles bounds the number of Certification objects reconciled concurrently.
MaxConcurrentReconciles int
}
CertificationReconciler reconciles a Certification object
func (*CertificationReconciler) Reconcile ¶
func (r *CertificationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)
Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.
func (*CertificationReconciler) SetupWithManager ¶
func (r *CertificationReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager sets up the controller with the Manager.
type GoodputMeasurementReconciler ¶
type GoodputMeasurementReconciler struct {
client.Client
Scheme *runtime.Scheme
Clientset *kubernetes.Clientset
Recorder events.EventRecorder
LogFetcher podlogs.PodLogFetcher // if nil, defaults to Clientset-backed fetcher
// MaxConcurrentReconciles bounds the number of GoodputMeasurement objects reconciled concurrently.
MaxConcurrentReconciles int
// contains filtered or unexported fields
}
GoodputMeasurementReconciler reconciles a GoodputMeasurement object.
func (*GoodputMeasurementReconciler) Reconcile ¶
func (r *GoodputMeasurementReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)
Reconcile is part of the main kubernetes reconciliation loop.
func (*GoodputMeasurementReconciler) SetupWithManager ¶
func (r *GoodputMeasurementReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager sets up the controller with the Manager.
type JobReconciler ¶
type JobReconciler struct {
client.Client
Scheme *runtime.Scheme
Clientset *kubernetes.Clientset
NodeDiscoverer *nodemonitor.NodeDiscoverer
Recorder events.EventRecorder
// MaxConcurrentReconciles bounds the number of Job objects reconciled concurrently.
MaxConcurrentReconciles int
// WorkloadRequeueInterval is how often to poll workload status when job is in progress.
// If zero, defaults to workloadRequeueInterval (15s).
WorkloadRequeueInterval time.Duration
// MeasurementTimeout is how long after the Job succeeds to wait for measurement
// data before failing threshold validation. If zero, defaults to defaultMeasurementTimeout (5m).
MeasurementTimeout time.Duration
// contains filtered or unexported fields
}
JobReconciler reconciles a Job object
func (*JobReconciler) Reconcile ¶
Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.
func (*JobReconciler) SetupWithManager ¶
func (r *JobReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager sets up the controller with the Manager.
type OverrideContext ¶
type OverrideContext struct {
Platform string
GPUArchitecture string
WorkloadKind string
TopologyMode string
DomainCount int
Config *apiextensionsv1.JSON
}
OverrideContext holds all detected values used for WhenSpec matching.
func BuildOverrideContext ¶
func BuildOverrideContext(spec *nvcrev1alpha1.WorkflowSpec, orch *nvcrev1alpha1.OrchestrationStatus, nodes []corev1.Node) OverrideContext
BuildOverrideContext is the exported version of buildOverrideContext for use by CLI tools.
type WorkflowReconciler ¶
type WorkflowReconciler struct {
client.Client
Scheme *runtime.Scheme
Clientset *kubernetes.Clientset
Recorder events.EventRecorder
JobRequeueInterval time.Duration
// MaxConcurrentReconciles bounds the number of Workflow objects reconciled concurrently.
MaxConcurrentReconciles int
}
WorkflowReconciler reconciles a Workflow object
func (*WorkflowReconciler) Reconcile ¶
Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.
func (*WorkflowReconciler) SetupWithManager ¶
func (r *WorkflowReconciler) SetupWithManager(mgr ctrl.Manager) error
SetupWithManager sets up the controller with the Manager.
type WorkloadRunReconciler ¶
type WorkloadRunReconciler struct {
client.Client
Scheme *kruntime.Scheme
Recorder events.EventRecorder
// MaxConcurrentReconciles bounds the number of WorkloadRun objects reconciled concurrently.
MaxConcurrentReconciles int
}
WorkloadRunReconciler reconciles a WorkloadRun object.
func (*WorkloadRunReconciler) SetupWithManager ¶
func (r *WorkloadRunReconciler) SetupWithManager(mgr ctrl.Manager) error
Source Files
¶
- bandwidthmeasurement_controller.go
- cache.go
- certification_controller.go
- event_note.go
- finalizer.go
- goodputmeasurement_controller.go
- helpers.go
- indexes.go
- job_controller.go
- job_threshold_helpers.go
- metrics.go
- nic_detect.go
- node_results.go
- pod_drain.go
- status.go
- workflow_controller.go
- workflow_deps.go
- workflow_detect.go
- workflow_detect_export.go
- workloadrun_controller.go