controller

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 5, 2026 License: Apache-2.0 Imports: 68 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// ReasonNotApplicable is the default condition reason for inactive conditions.
	ReasonNotApplicable = "NotApplicable"

	// ReasonNICResourceDetection is shared by the Certification and WorkloadRun
	// tiers: both emit a Normal event under it when NIC resource auto-detection
	// ran (on-prem GB200/GB300 target with nicResourceName unset) and found
	// zero or multiple qualifying candidates, so nothing was injected.
	ReasonNICResourceDetection = "NICResourceDetection"
)
View Source
const (
	// ReasonWaitingForNodes marks a Certification that found no schedulable nodes
	// and is retrying. Without it the wait is invisible: the retry only logged, so
	// the CR carried no conditions at all for up to nodeDiscoveryTimeout and an
	// operator had nothing to explain why nothing was happening.
	ReasonWaitingForNodes          = "WaitingForNodes"
	ReasonWorkflowCreated          = "WorkflowCreated"
	ReasonWorkflowRunning          = "WorkflowRunning"
	ReasonAllWorkflowsSucceeded    = "AllWorkflowsSucceeded"
	ReasonWorkflowFailed           = "WorkflowFailed"
	ReasonWorkflowValidationFailed = "WorkflowValidationFailed"
	ReasonCategoryNotFound         = "CategoryNotFound"
	ReasonBuildFailed              = "BuildFailed"
	ReasonWorkflowDeleted          = "WorkflowDeleted"
	ReasonWorkflowSucceeded        = "WorkflowSucceeded"
	ReasonThresholdViolation       = "ThresholdViolation"

	// ReasonWorkflowNameCollision marks a Certification whose generated child
	// Workflow name is already taken by a Workflow this Certification does not
	// own. The foreign object is neither adopted nor recorded for cleanup.
	ReasonWorkflowNameCollision = "WorkflowNameCollision"

	// ReasonWorkflowCreationError indicates a child Workflow could not be
	// created. Emitted as a Warning event at the failed Create call by the
	// Certification and WorkloadRun reconcilers, which both create Workflows,
	// so the API server's rejection is visible on the parent object rather
	// than only in the controller log. Mirrors ReasonJobCreationError
	// (Workflow tier) and ReasonWorkloadCreationError (Job tier).
	ReasonWorkflowCreationError = "WorkflowCreationError"
)

Certification tier reasons (Certification → Workflow).

View Source
const (
	ReasonJobCreated          = "JobCreated"
	ReasonJobRunning          = "JobRunning"
	ReasonJobCompleted        = "JobCompleted"
	ReasonJobFailed           = "JobFailed"
	ReasonJobHardwareFailed   = "JobHardwareFailed"
	ReasonJobCreationError    = "JobCreationError"
	ReasonJobValidationFailed = "JobValidationFailed"

	// ReasonJobTimedOut marks a Job's Failed condition set by the Workflow when
	// the Job exceeded timeoutPerJob. Timed-out jobs are never retried.
	ReasonJobTimedOut = "JobTimedOut"

	ReasonGroupsPartitioned  = "GroupsPartitioned"
	ReasonIterationCompleted = "IterationCompleted"
	ReasonIterationsFailed   = "IterationsFailed"

	ReasonDependencyCreationError = "DependencyCreationError"
	ReasonNodeDiscoveryError      = "NodeDiscoveryError"
	ReasonPartitionError          = "PartitionError"

	// ReasonHeterogeneousPlatformStatusUpdateFailed reports inconsistent node
	// platforms when recording the Workflow's Failed condition also fails.
	// This action-failure Warning does not claim a persisted Failed phase.
	ReasonHeterogeneousPlatformStatusUpdateFailed = "HeterogeneousPlatformStatusUpdateFailed"

	// ReasonOverrideErrorStatusUpdateFailed reports an override application
	// error when recording the Workflow's Failed condition also fails. Both
	// override guards use this action-failure Warning, not a phase notification.
	ReasonOverrideErrorStatusUpdateFailed = "OverrideErrorStatusUpdateFailed"

	// ReasonJobNameCollision marks a Workflow whose generated Job name is
	// already taken by a Job this Workflow does not control. The foreign
	// object is neither adopted nor recorded for cleanup.
	ReasonJobNameCollision = "JobNameCollision"
	// ReasonDependencyNameCollision marks a Workflow whose dependency resource
	// name is already taken by an object this Workflow did not create. The
	// foreign object is neither adopted nor recorded for cleanup.
	ReasonDependencyNameCollision = "DependencyNameCollision"
	// ReasonGangSchedulingConflict marks a Workflow whose resolved Job template
	// is invalid after overrides. This includes workload metadata that fails
	// label validation even when no gang scheduler is configured, as well as a
	// template that no longer agrees with the gang-scheduling intent its owner
	// persisted. The invalid result is reported rather than repaired.
	ReasonGangSchedulingConflict = "GangSchedulingConflict"
)

Workflow tier reasons (Workflow → Job).

View Source
const (
	ReasonWorkloadCreated = "WorkloadCreated"
	// ReasonWorkloadAdopted indicates the Job found the workload it had
	// already created still on the API server, because the write that records
	// status.WorkloadRef was lost after the Create succeeded, and took the
	// reference back rather than treating the name as taken.
	ReasonWorkloadAdopted = "WorkloadAdopted"
	// ReasonWorkloadPending indicates the workload exists but has not started
	// running — e.g. a TrainJob suspended by Kueue while it waits for quota.
	// The Job stays InProgress; pending time is not counted against
	// timeoutPerJob or stall detection.
	ReasonWorkloadPending       = "WorkloadPending"
	ReasonWorkloadRunning       = "WorkloadRunning"
	ReasonWorkloadCompleted     = "WorkloadCompleted"
	ReasonWorkloadFailed        = "WorkloadFailed"
	ReasonWorkloadCreationError = "WorkloadCreationError"
	ReasonWorkloadStalled       = "WorkloadStalled"

	// ReasonMeasurementCreationError indicates a GoodputMeasurement or
	// BandwidthMeasurement child resource could not be created. Handling is
	// non-fatal, so this event is the operator-visible signal; a threshold that
	// depends on the missing measurement still fails closed via
	// ReasonMeasurementTimeout.
	ReasonMeasurementCreationError = "MeasurementCreationError"

	ReasonHardwareFailureDetected = "HardwareFailureDetected"
)

Job tier reasons (Job → Workload).

View Source
const (
	FrameworkTorch = "torch"
	FrameworkMPI   = "mpi"
	FrameworkExec  = "exec"
)

Framework type constants for WorkloadRun.

View Source
const GPUNodeLabel = "nvidia.com/gpu.present"

GPUNodeLabel marks a node as GPU-equipped. The controllers only ever operate on GPU nodes — discoverTargetNodes filters on this label before a node can be partitioned into a group — so it also defines the Node cache boundary.

View Source
const (
	// ReasonBuildFailedStatusUpdateFailed reports a WorkloadRun build error
	// when recording its Failed condition also fails. This action-failure
	// Warning does not claim that the Failed phase was persisted.
	ReasonBuildFailedStatusUpdateFailed = "BuildFailedStatusUpdateFailed"
)

WorkloadRun tier reasons (WorkloadRun → Workflow).

Variables

This section is empty.

Functions

func ApplyOverridesWithTracking

func ApplyOverridesWithTracking(spec *nvcrev1alpha1.WorkflowSpec, octx OverrideContext) ([]nvcrev1alpha1.AppliedOverride, error)

ApplyOverridesWithTracking is the exported version of applyOverridesWithTracking for use by CLI tools.

func ApplyWRPreTemplateOverrides

func ApplyWRPreTemplateOverrides(spec *nvcrev1alpha1.WorkloadRunSpec, overrides []platform.WorkloadRunOverride, octx OverrideContext)

ApplyWRPreTemplateOverrides is the exported version of applyWRPreTemplateOverrides for use by CLI tools, so the "workloadrun render" preview bakes the same platform mpirun args into the spec that the controller bakes at reconcile time.

func CacheOptions

func CacheOptions() cache.Options

CacheOptions returns the informer cache configuration for the manager.

Without it, controller-runtime caches every object of every watched kind, cluster-wide and in full. This controller watches Nodes, Pods and PersistentVolumes, so on a large GPU cluster the default cache holds the entire Pod population of the cluster in the controller's heap — against a 1Gi limit. Two reductions are applied:

  • Nodes are restricted to GPU nodes. Non-GPU nodes (control plane, CPU workers, infra) are never targets, never run burn-in pods, and never need health evaluation, so caching and watching them is pure overhead.

  • managedFields is stripped from every cached object. It is metadata the controllers never read and is frequently a large fraction of an object's serialized size, particularly for Pods and CRs written by several controllers.

Pods are deliberately not label-scoped. The obvious selector, nvcre.nvidia.com/job, is injected only into the worker replicatedJob's pod template — MPI launcher pods do not carry it (see workerTargetJob in pkg/workload), and the launcher is exactly the pod that NCCL bandwidth measurement and timeout log capture read from. Scoping on that label would make launchers invisible to the cache. Pod reads are instead kept cheap via field indexes; see indexes.go.

func CondIsTrue

func CondIsTrue(conditions []metav1.Condition, condType string) bool

CondIsTrue returns true if the named condition has Status=True.

func CondMessage

func CondMessage(conditions []metav1.Condition, condType string) string

CondMessage returns the message for a condition type, or empty string if not found.

func DefaultEnableMNNVL

func DefaultEnableMNNVL(gpuArch string) bool

DefaultEnableMNNVL returns whether MNNVL should be enabled for the given GPU architecture. GB200 and GB300 use Multi-Node NVLink and benefit from MNNVL.

func DetectGPUArchitecture

func DetectGPUArchitecture(nodes []corev1.Node) string

DetectGPUArchitecture is the exported version of detectGPUArchitecture for use by CLI tools. Like the controllers, it reports the architecture that the most labeled nodes carry (gpu.MajorityArchitecture), so render, cluster info, and workloadrun previews agree with what a reconcile of the same target would detect.

func DetectPlatform

func DetectPlatform(nodes []corev1.Node) string

DetectPlatform is the exported version of detectPlatform for use by CLI tools.

func DiscoverTargetNodes

func DiscoverTargetNodes(ctx context.Context, reader client.Reader, target *nvcrev1alpha1.TargetSpec) ([]corev1.Node, error)

DiscoverTargetNodes is the exported version of discoverTargetNodes for use by CLI tools.

It drops the cordoned-node list that discoverTargetNodes also returns. Only the Workflow reconciler records coverage on status; CLI callers want the nodes a run would actually use. Widen this if a CLI ever needs to report what was skipped.

reader also serves the ResourceSlice fallback List, which is safe because CLI clients are uncached client.New clients.

func NodesPerJobForScale

func NodesPerJobForScale(orch *nvcrev1alpha1.WorkloadOrchestration, numNodes int32) int32

NodesPerJobForScale returns how many nodes a single Job should span. intra-node means each node is tested on its own, so one node per Job however many the run targets; the Workflow then makes one group per node. Anything else keeps the requested count.

It lives here so the reconcile path and the "workloadrun render" preview in pkg/workloadrun (which imports this package) apply the same rule — issue #85 was the controller not applying it at all.

func RegisterFieldIndexes

func RegisterFieldIndexes(ctx context.Context, indexer client.FieldIndexer) error

RegisterFieldIndexes registers every field index the controllers rely on.

It must be called once per manager, before the controllers start, and is the single source of truth for both production (cmd/manager) and the integration harness — registering these in two places previously allowed prod and test to drift apart.

func ResolveNICResourceName added in v0.3.0

func ResolveNICResourceName(
	field *string, platformName, gpuArch string, nodes []corev1.Node, mlnxPerNode int32,
) (name, refusalMessage string, detectionRan bool)

ResolveNICResourceName is the exported version of resolveNICResourceName for use by the CLI dry-run paths, so "certification render --dry-run" and "workloadrun render --dry-run" resolve the NIC resource exactly as a reconcile of the same target would: field wins, detection only for on-prem GB200/GB300, and only a single candidate allocatable at the resolved mlnxPerNode count on every node is used. mlnxPerNode must be the caller's fully resolved value (field or catalog default).

refusalMessage is non-empty exactly when detection ran and refused to pick (name == ""); it is the same text the controllers emit as the NICResourceDetection event, for the CLI to print to stderr.

func ResolveOptions

ResolveOptions merges per-category overrides with global defaults. Returns a flat CategoryOptions with all values resolved.

func SetupStatusMetrics added in v0.6.0

func SetupStatusMetrics(mgr manager.Manager)

SetupStatusMetrics points the Certification and Workflow status collector at the manager's cache and elected channel. Collect emits nothing until mgr.Elected() is closed, so a standby replica does not start informers or double-report series.

func TargetsCordonedNodes added in v0.5.0

func TargetsCordonedNodes(target *nvcrev1alpha1.TargetSpec) bool

TargetsCordonedNodes returns true if the given targetSpec contains a taintSelectors entry which targets the node.kubernetes.io/unschedulable taint.

Types

type BandwidthMeasurementReconciler

type BandwidthMeasurementReconciler struct {
	client.Client
	// APIReader is an uncached client used for pod discovery. Pod watch events
	// from the informer cache can lag behind the API server by several seconds,
	// causing the "pod not found" requeue loop to spin longer than necessary.
	// A direct API read guarantees the freshest view at the cost of one extra
	// API server call per reconcile cycle while waiting for pods.
	APIReader  client.Reader
	Scheme     *runtime.Scheme
	Clientset  *kubernetes.Clientset
	Recorder   events.EventRecorder
	LogFetcher podlogs.PodLogFetcher // if nil, defaults to Clientset-backed fetcher
	// MaxConcurrentReconciles bounds the number of BandwidthMeasurement objects reconciled concurrently.
	MaxConcurrentReconciles int

	// RequeueInterval is the interval between reconcile cycles when polling.
	// Tests set this to 1s; production uses 15s.
	RequeueInterval time.Duration

	// FinalReadGracePeriod is the least time a failed final log read is
	// retried before the measurement completes without final results; the
	// Job's measurement timeout extends it. Zero uses
	// defaultFinalReadGracePeriod; a negative value completes on the first
	// failed read.
	FinalReadGracePeriod time.Duration
	// contains filtered or unexported fields
}

BandwidthMeasurementReconciler reconciles a BandwidthMeasurement object.

func (*BandwidthMeasurementReconciler) Reconcile

Reconcile is part of the main kubernetes reconciliation loop.

func (*BandwidthMeasurementReconciler) SetupWithManager

func (r *BandwidthMeasurementReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type CertificationReconciler

type CertificationReconciler struct {
	client.Client
	Scheme   *runtime.Scheme
	Recorder events.EventRecorder
	// APIReader reads straight from the API server, bypassing the informer
	// cache, so a cache miss on a child Workflow can be confirmed against the
	// live cluster before it is treated as a deletion (issue #384).
	APIReader               client.Reader
	WorkflowRequeueInterval time.Duration
	// MaxConcurrentReconciles bounds the number of Certification objects reconciled concurrently.
	MaxConcurrentReconciles int
}

CertificationReconciler reconciles a Certification object

func (*CertificationReconciler) Reconcile

func (r *CertificationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.

func (*CertificationReconciler) SetupWithManager

func (r *CertificationReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type GoodputMeasurementReconciler

type GoodputMeasurementReconciler struct {
	client.Client
	Scheme     *runtime.Scheme
	Clientset  *kubernetes.Clientset
	Recorder   events.EventRecorder
	LogFetcher podlogs.PodLogFetcher // if nil, defaults to Clientset-backed fetcher
	// MaxConcurrentReconciles bounds the number of GoodputMeasurement objects reconciled concurrently.
	MaxConcurrentReconciles int
	// contains filtered or unexported fields
}

GoodputMeasurementReconciler reconciles a GoodputMeasurement object.

func (*GoodputMeasurementReconciler) Reconcile

Reconcile is part of the main kubernetes reconciliation loop.

func (*GoodputMeasurementReconciler) SetupWithManager

func (r *GoodputMeasurementReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type JobReconciler

type JobReconciler struct {
	client.Client
	Scheme         *runtime.Scheme
	Clientset      *kubernetes.Clientset
	NodeDiscoverer *nodemonitor.NodeDiscoverer
	Recorder       events.EventRecorder
	// APIReader reads straight from the API server, bypassing the informer
	// cache, so a workload missing from the cache is confirmed gone before
	// anything acts on its absence.
	APIReader client.Reader
	// MaxConcurrentReconciles bounds the number of Job objects reconciled concurrently.
	MaxConcurrentReconciles int

	// WorkloadRequeueInterval is how often to poll workload status when job is in progress.
	// If zero, defaults to workloadRequeueInterval (15s).
	WorkloadRequeueInterval time.Duration

	// MeasurementTimeout is how long after the Job succeeds to wait for measurement
	// data before failing threshold validation. If zero, defaults to defaultMeasurementTimeout (5m).
	MeasurementTimeout time.Duration
	// contains filtered or unexported fields
}

JobReconciler reconciles a Job object

func (*JobReconciler) Reconcile

func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.

func (*JobReconciler) SetupWithManager

func (r *JobReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type OverrideContext

type OverrideContext struct {
	Platform        string
	GPUArchitecture string
	WorkloadKind    string
	TopologyMode    string
	DomainCount     int
	Config          *apiextensionsv1.JSON
}

OverrideContext holds all detected values used for WhenSpec matching.

func BuildOverrideContext

func BuildOverrideContext(spec *nvcrev1alpha1.WorkflowSpec, orch *nvcrev1alpha1.OrchestrationStatus, nodes []corev1.Node) OverrideContext

BuildOverrideContext is the exported version of buildOverrideContext for use by CLI tools.

type WorkflowReconciler

type WorkflowReconciler struct {
	client.Client
	Scheme    *runtime.Scheme
	Clientset *kubernetes.Clientset
	Recorder  events.EventRecorder
	// APIReader reads straight from the API server, bypassing the informer
	// cache, so a Job missing from the cache can be checked against the live
	// cluster before it is treated as deleted (issue #385).
	APIReader          client.Reader
	JobRequeueInterval time.Duration
	// MaxConcurrentReconciles bounds the number of Workflow objects reconciled concurrently.
	MaxConcurrentReconciles int
}

WorkflowReconciler reconciles a Workflow object

func (*WorkflowReconciler) Reconcile

func (r *WorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.

func (*WorkflowReconciler) SetupWithManager

func (r *WorkflowReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type WorkloadRunReconciler

type WorkloadRunReconciler struct {
	client.Client
	// APIReader is an uncached client used to confirm that a Workflow missing
	// from the informer cache is really gone. Right after Create, the
	// WorkloadRun's workflowRef write can reach the cache before the new
	// Workflow does; without a direct read that lag looks like a deletion and
	// fails the run (issue #352).
	APIReader client.Reader
	Scheme    *kruntime.Scheme
	Recorder  events.EventRecorder
	// MaxConcurrentReconciles bounds the number of WorkloadRun objects reconciled concurrently.
	MaxConcurrentReconciles int
}

WorkloadRunReconciler reconciles a WorkloadRun object.

func (*WorkloadRunReconciler) Reconcile

func (r *WorkloadRunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

func (*WorkloadRunReconciler) SetupWithManager

func (r *WorkloadRunReconciler) SetupWithManager(mgr ctrl.Manager) error

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL