controller

package
v0.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 12, 2026 License: Apache-2.0 Imports: 69 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// ReasonNotApplicable is the default condition reason for inactive conditions.
	ReasonNotApplicable = "NotApplicable"

	// ReasonNICResourceDetection is shared by the Certification and WorkloadRun
	// tiers: both emit a Normal event under it when NIC resource auto-detection
	// ran (on-prem GB200/GB300 or x86 HGX B200/B300 target with
	// nicResourceName unset) and found
	// zero or multiple qualifying candidates, so nothing was injected.
	ReasonNICResourceDetection = "NICResourceDetection"

	// ReasonGKENetworkDetection is the Certification tier's Warning event when
	// GKE TCPXO network auto-detection ran (GCP H100 target) and did not find
	// exactly the GPU NIC networks TCPXO needs on every target node, so the
	// catalog's default network names were rendered.
	ReasonGKENetworkDetection = "GKENetworkDetection"

	// ReasonTCPXOPluginDetection is the Certification tier's Warning event
	// when TCPXO plugin version detection ran (GCP H100 target) and did not
	// find one mapped plugin release on every target node, so the nearest
	// safe profile's GCP H100 workload and tcpxo-daemon images were rendered
	// (the latest mapped release for a newer tag, the minimum otherwise).
	ReasonTCPXOPluginDetection = "TCPXOPluginDetection"

	// ReasonGPUArchitectureDefaults is the Warning event both the Certification
	// and WorkloadRun tiers emit when the detected GPU architecture is not
	// listed in the catalog's gpu-defaults.yaml, so the workflow was sized
	// with the Go-side fallback (four GPUs, no NICs) instead of a known node
	// shape (ADR-094).
	ReasonGPUArchitectureDefaults = "GPUArchitectureDefaults"
)
View Source 
const (
	// ReasonWaitingForNodes marks a Certification that found no schedulable nodes
	// and is retrying. Without it the wait is invisible: the retry only logged, so
	// the CR carried no conditions at all for up to nodeDiscoveryTimeout and an
	// operator had nothing to explain why nothing was happening.
	ReasonWaitingForNodes          = "WaitingForNodes"
	ReasonWorkflowCreated          = "WorkflowCreated"
	ReasonWorkflowRunning          = "WorkflowRunning"
	ReasonAllWorkflowsSucceeded    = "AllWorkflowsSucceeded"
	ReasonWorkflowFailed           = "WorkflowFailed"
	ReasonWorkflowValidationFailed = "WorkflowValidationFailed"
	ReasonCategoryNotFound         = "CategoryNotFound"
	ReasonBuildFailed              = "BuildFailed"
	ReasonWorkflowDeleted          = "WorkflowDeleted"
	ReasonWorkflowSucceeded        = "WorkflowSucceeded"
	ReasonThresholdViolation       = "ThresholdViolation"

	// ReasonWorkflowNameCollision marks a Certification whose generated child
	// Workflow name is already taken by a Workflow this Certification does not
	// own. The foreign object is neither adopted nor recorded for cleanup.
	ReasonWorkflowNameCollision = "WorkflowNameCollision"

	// ReasonWorkflowCreationError indicates a child Workflow could not be
	// created. Emitted as a Warning event at the failed Create call by the
	// Certification and WorkloadRun reconcilers, which both create Workflows,
	// so the API server's rejection is visible on the parent object rather
	// than only in the controller log. Mirrors ReasonJobCreationError
	// (Workflow tier) and ReasonWorkloadCreationError (Job tier).
	ReasonWorkflowCreationError = "WorkflowCreationError"
)

Certification tier reasons (Certification → Workflow).

View Source 
const (
	ReasonJobCreated          = "JobCreated"
	ReasonJobRunning          = "JobRunning"
	ReasonJobCompleted        = "JobCompleted"
	ReasonJobFailed           = "JobFailed"
	ReasonJobHardwareFailed   = "JobHardwareFailed"
	ReasonJobCreationError    = "JobCreationError"
	ReasonJobValidationFailed = "JobValidationFailed"

	// ReasonJobSchedulingBlocked marks a running Workflow iteration in which
	// at least one group's Job reports WorkloadSchedulingBlocked. The message
	// relays that Job's scheduler diagnosis (ADR-083).
	ReasonJobSchedulingBlocked = "JobSchedulingBlocked"

	// ReasonJobTimedOut marks a Job's Failed condition set by the Workflow when
	// the Job exceeded timeoutPerJob. Timed-out jobs are never retried.
	ReasonJobTimedOut = "JobTimedOut"

	ReasonGroupsPartitioned  = "GroupsPartitioned"
	ReasonIterationCompleted = "IterationCompleted"
	ReasonIterationsFailed   = "IterationsFailed"

	ReasonDependencyCreationError = "DependencyCreationError"
	ReasonNodeDiscoveryError      = "NodeDiscoveryError"
	ReasonPartitionError          = "PartitionError"

	// ReasonHeterogeneousPlatformStatusUpdateFailed reports inconsistent node
	// platforms when recording the Workflow's Failed condition also fails.
	// This action-failure Warning does not claim a persisted Failed phase.
	ReasonHeterogeneousPlatformStatusUpdateFailed = "HeterogeneousPlatformStatusUpdateFailed"

	// ReasonOverrideErrorStatusUpdateFailed reports an override application
	// error when recording the Workflow's Failed condition also fails. Both
	// override guards use this action-failure Warning, not a phase notification.
	ReasonOverrideErrorStatusUpdateFailed = "OverrideErrorStatusUpdateFailed"

	// ReasonJobNameCollision marks a Workflow whose generated Job name is
	// already taken by a Job this Workflow does not control. The foreign
	// object is neither adopted nor recorded for cleanup.
	ReasonJobNameCollision = "JobNameCollision"
	// ReasonDependencyNameCollision marks a Workflow whose dependency resource
	// name is already taken by an object this Workflow did not create. The
	// foreign object is neither adopted nor recorded for cleanup.
	ReasonDependencyNameCollision = "DependencyNameCollision"
	// ReasonGangSchedulingConflict marks a Workflow whose resolved Job template
	// is invalid after overrides. This includes workload metadata that fails
	// label validation even when no gang scheduler is configured, as well as a
	// template that no longer agrees with the gang-scheduling intent its owner
	// persisted. The invalid result is reported rather than repaired.
	ReasonGangSchedulingConflict = "GangSchedulingConflict"
)

Workflow tier reasons (Workflow → Job).

View Source 
const (
	ReasonWorkloadCreated = "WorkloadCreated"
	// ReasonWorkloadAdopted indicates the Job found the workload it had
	// already created still on the API server, because the write that records
	// status.WorkloadRef was lost after the Create succeeded, and took the
	// reference back rather than treating the name as taken.
	ReasonWorkloadAdopted = "WorkloadAdopted"
	// ReasonWorkloadPending indicates the workload exists but has not started
	// running — e.g. a TrainJob suspended by Kueue while it waits for quota.
	// The Job stays InProgress; pending time is not counted against
	// timeoutPerJob or stall detection.
	ReasonWorkloadPending       = "WorkloadPending"
	ReasonWorkloadRunning       = "WorkloadRunning"
	ReasonWorkloadCompleted     = "WorkloadCompleted"
	ReasonWorkloadFailed        = "WorkloadFailed"
	ReasonWorkloadCreationError = "WorkloadCreationError"
	ReasonWorkloadStalled       = "WorkloadStalled"

	// ReasonWorkloadSchedulingBlocked indicates the workload is unsuspended
	// and admitted, but at least one of its pods cannot be scheduled
	// (PodScheduled=False/Unschedulable). The Job stays InProgress and the
	// blocked time does not count against timeoutPerJob or stall detection:
	// the same clock-neutral treatment as WorkloadPending (issue #213).
	// See ADR-083.
	ReasonWorkloadSchedulingBlocked = "WorkloadSchedulingBlocked"

	// ReasonMeasurementCreationError indicates a GoodputMeasurement or
	// BandwidthMeasurement child resource could not be created. Handling is
	// non-fatal, so this event is the operator-visible signal; a threshold that
	// depends on the missing measurement still fails closed via
	// ReasonMeasurementTimeout.
	ReasonMeasurementCreationError = "MeasurementCreationError"

	ReasonHardwareFailureDetected = "HardwareFailureDetected"
)

Job tier reasons (Job → Workload).

View Source 
const (
	FrameworkTorch = "torch"
	FrameworkMPI   = "mpi"
	FrameworkExec  = "exec"
)

Framework type constants for WorkloadRun.

View Source 
const (
	GPUArchFieldHintCertification = gpuArchFieldHintCertification
	GPUArchFieldHintWorkloadRun   = gpuArchFieldHintWorkloadRun
	GPUArchFieldHintClusterInfo   = "gpusPerNode and mlnxPerNode on the Certification or WorkloadRun"
)

Exported field hints for GPUArchFallbackMessage, one per CLI surface.

View Source 
const GPUNodeLabel = "nvidia.com/gpu.present"

GPUNodeLabel marks a node as GPU-equipped. The controllers only ever operate on GPU nodes — discoverTargetNodes filters on this label before a node can be partitioned into a group — so it also defines the Node cache boundary.

View Source 
const (
	// ReasonBuildFailedStatusUpdateFailed reports a WorkloadRun build error
	// when recording its Failed condition also fails. This action-failure
	// Warning does not claim that the Failed phase was persisted.
	ReasonBuildFailedStatusUpdateFailed = "BuildFailedStatusUpdateFailed"
)

WorkloadRun tier reasons (WorkloadRun → Workflow).

Variables

This section is empty.

Functions

func ApplyOverridesWithTracking 

func ApplyOverridesWithTracking(spec *nvcrev1alpha1.WorkflowSpec, octx OverrideContext) ([]nvcrev1alpha1.AppliedOverride, error)

ApplyOverridesWithTracking is the exported version of applyOverridesWithTracking for use by CLI tools.

func ApplyWRPreTemplateOverrides 

func ApplyWRPreTemplateOverrides(spec *nvcrev1alpha1.WorkloadRunSpec, overrides []platform.WorkloadRunOverride, octx OverrideContext)

ApplyWRPreTemplateOverrides is the exported version of applyWRPreTemplateOverrides for use by CLI tools, so the "workloadrun render" preview bakes the same platform mpirun args into the spec that the controller bakes at reconcile time.

func ArchExcludedForRender added in v0.7.0 

func ArchExcludedForRender(nodes []corev1.Node) []string

ArchExcludedForRender names the nodes the render path would drop for running a different GPU architecture, so a refusal can say which nodes an unconstrained job could have landed on.

func BuildNodeAffinity added in v0.7.0 

func BuildNodeAffinity(hostnames []string, target *nvcrev1alpha1.TargetSpec, gpuProducts []string) *corev1.NodeAffinity

BuildNodeAffinity assembles the job's required node affinity from the group's hostnames and the target.

Under Pinned, hostnames is the group's node list and the target terms are strictly redundant: every one of those hosts already satisfied the target at discovery time. They are added anyway so the constraint is re-evaluated by the scheduler at bind time rather than trusted from discovery time, which closes the window where a node's labels change in between.

Under Unpinned, hostnames is empty unless the target named explicit nodeNames, and the target terms are the only thing keeping pods inside the target set.

gpuProducts must come from GPUProductsForAffinity, not straight off status: a label synthesized from ResourceSlice attributes exists only in memory and would match nothing.

Returns nil when there is nothing to constrain, so the caller leaves the pod's affinity untouched rather than writing an empty required term that matches every node.

func BuildTolerations added in v0.7.0 

func BuildTolerations(selectors []nvcrev1alpha1.TaintSelector) []corev1.Toleration

BuildTolerations creates tolerations from taint selectors. Exported so pkg/render builds the same tolerations the controller does.

func CacheOptions 

func CacheOptions() cache.Options

CacheOptions returns the informer cache configuration for the manager.

Without it, controller-runtime caches every object of every watched kind, cluster-wide and in full. This controller watches Nodes, Pods and PersistentVolumes, so on a large GPU cluster the default cache holds the entire Pod population of the cluster in the controller's heap — against a 1Gi limit. Two reductions are applied:

  • Nodes are restricted to GPU nodes. Non-GPU nodes (control plane, CPU workers, infra) are never targets, never run burn-in pods, and never need health evaluation, so caching and watching them is pure overhead.

  • managedFields is stripped from every cached object. It is metadata the controllers never read and is frequently a large fraction of an object's serialized size, particularly for Pods and CRs written by several controllers.

Pods are deliberately not label-scoped. The obvious selector, nvcre.nvidia.com/job, is injected only into the worker replicatedJob's pod template — MPI launcher pods do not carry it (see workerTargetJob in pkg/workload), and the launcher is exactly the pod that NCCL bandwidth measurement and timeout log capture read from. Scoping on that label would make launchers invisible to the cache. Pod reads are instead kept cheap via field indexes; see indexes.go.

func CondIsTrue 

func CondIsTrue(conditions []metav1.Condition, condType string) bool

CondIsTrue returns true if the named condition has Status=True.

func CondMessage 

func CondMessage(conditions []metav1.Condition, condType string) string

CondMessage returns the message for a condition type, or empty string if not found.

func CountDRAGPUs added in v0.7.0 

func CountDRAGPUs(ctx context.Context, reader client.Reader) (map[string]int32, error)

CountDRAGPUs returns the number of full GPUs each node publishes in its gpu.nvidia.com ResourceSlices, keyed by node name. MIG and VFIO devices are excluded because they are not whole GPUs. Only the newest generation of each pool counts, since older slices of a pool are stale by the ResourcePool contract.

It serves the CLI only, as an observation to compare against the catalog default; rendering still sizes claims from gpu-defaults.yaml. No controller calls it, so it has no unexported twin in workflow_detect_export.go.

func DefaultEnableMNNVL 

func DefaultEnableMNNVL(gpuArch string) bool

DefaultEnableMNNVL returns whether MNNVL should be enabled for the given GPU architecture. GB200 and GB300 use Multi-Node NVLink and benefit from MNNVL.

func DetectGPUArchitecture 

func DetectGPUArchitecture(nodes []corev1.Node) string

DetectGPUArchitecture is the exported version of detectGPUArchitecture for use by CLI tools. Like the controllers, it reports the architecture that the most labeled nodes carry (gpu.MajorityArchitecture), so render, cluster info, and workloadrun previews agree with what a reconcile of the same target would detect.

func DetectPlatform 

func DetectPlatform(nodes []corev1.Node) string

DetectPlatform is the exported version of detectPlatform for use by CLI tools.

func DiscoverTargetNodes 

func DiscoverTargetNodes(ctx context.Context, reader client.Reader, target *nvcrev1alpha1.TargetSpec) ([]corev1.Node, error)

DiscoverTargetNodes is the exported version of discoverTargetNodes for use by CLI tools.

It drops the cordoned-node list that discoverTargetNodes also returns. Only the Workflow reconciler records coverage on status; CLI callers want the nodes a run would actually use. Widen this if a CLI ever needs to report what was skipped.

reader also serves the ResourceSlice fallback List, which is safe because CLI clients are uncached client.New clients.

func DiscoverTargetNodesWithSynthesized added in v0.7.0 

func DiscoverTargetNodesWithSynthesized(
	ctx context.Context, reader client.Reader, target *nvcrev1alpha1.TargetSpec,
) (nodes []corev1.Node, synthesizedProducts []string, err error)

DiscoverTargetNodesWithSynthesized is DiscoverTargetNodes plus the names of the nodes whose nvidia.com/gpu.product label was synthesized from ResourceSlice attributes rather than read off the stored Node.

render needs the distinction for the same reason the reconciler does: a synthesized label exists only on the returned copies, so an affinity term built from it matches nothing on a real API server.

func DistinctGPUProducts added in v0.7.0 

func DistinctGPUProducts(nodes []corev1.Node) []string

DistinctGPUProducts reads the nvidia.com/gpu.product label off the nodes that survived discovery. The values are stored verbatim because OrchestrationStatus.DetectedGPUArchitecture is normalized and lossy ("NVIDIA-H100-80GB-HBM3" becomes "h100") and so cannot be turned back into a label match.

Exported for pkg/render, which has no OrchestrationStatus to read the recorded values back from and derives them from its own discovery instead.

func GPUArchFallbackMessage added in v0.7.0 

func GPUArchFallbackMessage(gpuArch string, gpusPerNode, mlnxPerNode int32, fieldHint string) string

GPUArchFallbackMessage is the exported version of gpuArchFallbackMessage for the CLI render and cluster-info paths, so every surface that sizes a workflow on an architecture missing from gpu-defaults.yaml prints the same text the controllers emit as the GPUArchitectureDefaults event (ADR-094). gpusPerNode and mlnxPerNode are the caller's resolved counts (field or catalog default). It returns "" for an empty, unknown, or listed architecture.

func GPUProductsForAffinity added in v0.7.0 

func GPUProductsForAffinity(
	placement string, gpuProducts, synthesizedProducts, archExcluded []string,
) (products []string, ok bool)

GPUProductsForAffinity decides whether the recorded gpu.product values can be used as a scheduling constraint.

They cannot when discovery synthesized any of them from ResourceSlice attributes (ADR-082's DRA-only fallback, augmentGPUProductLabels). That label is written to the controller's in-memory Node copies and never to the API server, so an affinity term matching it selects no node and every pod stays Pending forever. Reading the label is safe for every other consumer, which is why this is the only place that has to care. The check is on whether anything was synthesized at all, not on which values: with a mixed fleet the recorded list cannot say which of its entries are real.

When nothing was synthesized the values are genuine node labels and are returned unchanged. That is the ordinary case, in both placement modes.

When something was synthesized, the term has to go. Under Pinned that costs nothing: the hostname list is the output of the architecture filter, so those hosts are already the right architecture and only the bind-time re-check is lost. Under Unpinned the term is the only thing holding the job to one architecture, so dropping it silently could put a job on the wrong GPUs. The caller gets ok=false and is expected to fail the run with an explanation rather than schedule something unsafe. That is reachable only when discovery also excluded nodes for architecture, because with nothing excluded the fleet is uniform and there is no wrong GPU to land on.

func GPUProductsForRender added in v0.7.0 

func GPUProductsForRender(
	placement string, nodes []corev1.Node, synthesizedProducts []string,
) (products []string, ok bool)

GPUProductsForRender is GPUProductsForAffinity for the render path, which has no OrchestrationStatus to read the controller's recorded decision back from and so has to repeat it against its own discovery.

It filters to the primary architecture first, exactly as discoverAndPartition does before recording the values. Without that, a preview of a mixed fleet would emit every product it found while the controller emits only the primary, and the dry-run would validate a Job that is strictly more permissive than the one the reconcile creates.

func IgnoredTopologyKey added in v0.7.0 

func IgnoredTopologyKey(orch *nvcrev1alpha1.OrchestrationSpec) string

IgnoredTopologyKey returns the topology key that Unpinned is dropping, or "" when nothing is being ignored. Callers log it so the behavior is visible rather than silent.

func JobTolerations added in v0.7.0 

func JobTolerations(
	target *nvcrev1alpha1.TargetSpec, unpinned, hasLauncher bool,
) (tolerations []corev1.Toleration, apply bool)

JobTolerations resolves the toleration precedence for a job's pods (ADR-063), returning apply=false when the controller injects nothing and the workload's own tolerations stand alone.

The order is: an explicit target.taintSelectors wins outright; otherwise only MPI workloads get anything, because only they have a launcher that has to co-locate with the tainted GPU nodes its workers run on; otherwise the blanket depends on placement.

hasLauncher is workload.HasLauncherTarget. It is passed rather than read here so this stays a pure function over the decision inputs.

func NodesPerJobForScale 

func NodesPerJobForScale(orch *nvcrev1alpha1.WorkloadOrchestration, numNodes int32) int32

NodesPerJobForScale returns how many nodes a single Job should span. intra-node means each node is tested on its own, so one node per Job however many the run targets; the Workflow then makes one group per node. Anything else keeps the requested count.

It lives here so the reconcile path and the "workloadrun render" preview in pkg/workloadrun (which imports this package) apply the same rule — issue #85 was the controller not applying it at all.

func PinnedHostnames added in v0.7.0 

func PinnedHostnames(placement string, groupNodes []string, target *nvcrev1alpha1.TargetSpec) []string

PinnedHostnames returns the hostnames a job should be constrained to.

Under Pinned that is the group's node list. Under Unpinned the scheduler chooses, so the result is empty unless the target named explicit nodeNames, in which case that list is an explicit user-supplied host constraint and is honored.

func RegisterFieldIndexes 

func RegisterFieldIndexes(ctx context.Context, indexer client.FieldIndexer) error

RegisterFieldIndexes registers every field index the controllers rely on.

It must be called once per manager, before the controllers start, and is the single source of truth for both production (cmd/manager) and the integration harness — registering these in two places previously allowed prod and test to drift apart.

func ResolveGKETCPXONetworks added in v0.7.0 

func ResolveGKETCPXONetworks(
	platformName, gpuArch string, nodes []corev1.Node,
) (names []string, refusalMessage string, detectionRan bool)

ResolveGKETCPXONetworks is the exported version of resolveGKETCPXONetworks for use by the CLI dry-run path, so "certification render --dry-run" attaches the same GKE networks a reconcile of the same target would: detection only for GCP H100, and only exactly GKETCPXONICsPerNode networks allocatable on every node are used.

names is empty when detection did not run or refused, and the catalog default is rendered. refusalMessage is non-empty exactly when detection ran and refused; it is the same text the controller emits as the GKENetworkDetection event, for the CLI to print to stderr.

func ResolveNICResourceName added in v0.3.0 

func ResolveNICResourceName(
	field *string, platformName, gpuArch string, nodes []corev1.Node, mlnxPerNode int32,
) (name, refusalMessage string, detectionRan bool)

ResolveNICResourceName is the exported version of resolveNICResourceName for use by the CLI dry-run paths, so "certification render --dry-run" and "workloadrun render --dry-run" resolve the NIC resource exactly as a reconcile of the same target would: field wins, detection only for on-prem GB200/GB300 and x86 HGX B200/B300, and only a single candidate allocatable at the resolved mlnxPerNode count on every node is used. mlnxPerNode must be the caller's fully resolved value (field or catalog default).

refusalMessage is non-empty exactly when detection ran and refused to pick (name == ""); it is the same text the controllers emit as the NICResourceDetection event, for the CLI to print to stderr.

func ResolveOptions 

func ResolveOptions(global *nvcrev1alpha1.CategoryOptions, override *nvcrev1alpha1.CategoryOptions) nvcrev1alpha1.CategoryOptions

ResolveOptions merges per-category overrides with global defaults. Returns a flat CategoryOptions with all values resolved.

func ResolveTCPXOPluginVersion added in v0.7.0 

func ResolveTCPXOPluginVersion(
	ctx context.Context, reader client.Reader, platformName, gpuArch string, nodes []corev1.Node,
) (version, fallbackMessage string, detectionRan bool)

ResolveTCPXOPluginVersion is the exported version of resolveTCPXOPluginVersion for use by the CLI dry-run path, so "certification render --dry-run" renders the same GCP H100 images a reconcile of the same target would: detection only for GCP H100, and only one plugin release tag on every node is used. reader must be able to list kube-system pods.

version is the release every target node runs, mapped or not, for catalog.BuildConfig.TCPXOPluginVersion; empty when detection did not run or found none. fallbackMessage is non-empty exactly when detection ran and found no mapped release, so a fallback profile renders; it is the same text the controller emits as the TCPXOPluginDetection event, for the CLI to print to stderr.

func SetupStatusMetrics added in v0.6.0 

func SetupStatusMetrics(mgr manager.Manager)

SetupStatusMetrics points the Certification, Workflow and Job status collector at the manager's cache and elected channel. Collect emits nothing until mgr.Elected() is closed, so a standby replica does not start informers or double-report series.

func SynthesizedProductsMessage added in v0.7.0 

func SynthesizedProductsMessage(synthesizedProducts, archExcluded []string) string

SynthesizedProductsMessage explains why an Unpinned run cannot be scheduled when its GPU architecture constraint would be built from labels that exist only in the controller's memory.

func TargetNodeAffinityTerms added in v0.7.0 

func TargetNodeAffinityTerms(target *nvcrev1alpha1.TargetSpec, gpuProducts []string) []corev1.NodeSelectorRequirement

TargetNodeAffinityTerms converts a TargetSpec into node selector requirements for the pod spec.

The target has always been a discovery-side filter only: it chose which nodes the controller considered and never reached the pods. Pinning masked that, since pods landed on hosts the filter had already approved. Unpinned removes the pin, so the target has to be carried onto the pods for real. It is applied in both modes, because the gap is a defect either way and one placement contract is easier to reason about than two.

Two discovery-side filters are deliberately omitted because the scheduler enforces them natively: cordoned nodes are unschedulable, and under-capacity nodes fail the GPU resource request. The architecture filter is not native, which is why gpuProducts is carried explicitly.

nodeNames is excluded here because Pinned already expresses it through the hostname term; the Unpinned caller adds it separately. taintSelectors is excluded because it selects nodes by taint and is answered by tolerations.

Requirements are returned for a single NodeSelectorTerm. Terms OR together while expressions within a term AND, and the target's documented semantics are AND, so the caller must not spread these across multiple terms.

func TargetsCordonedNodes added in v0.5.0 

func TargetsCordonedNodes(target *nvcrev1alpha1.TargetSpec) bool

TargetsCordonedNodes returns true if the given targetSpec contains a taintSelectors entry which targets the node.kubernetes.io/unschedulable taint.

func UnpinnedMPITolerations added in v0.7.0 

func UnpinnedMPITolerations() []corev1.Toleration

UnpinnedMPITolerations is what an MPI job tolerates under Unpinned, in place of the blanket Operator: Exists that Pinned still gets.

The blanket cannot carry over. ADR-023 justified it on the grounds that "workloads are already pinned to specific nodes via NodeAffinity", and without the pin it would let a job onto any tainted node with free GPUs, which is the whole class of node an operator taints to keep work off. But tolerating nothing is not the answer either: GPU fleets are routinely tainted to keep non-GPU work off them, and a launcher that tolerates nothing simply never schedules. That is the harder failure to diagnose of the two, because the pods sit Pending with no event naming a toleration.

So Unpinned tolerates a named set instead: the two taints the catalog's own runtime patches already list for the platforms that declare any. It is strictly narrower than Exists, so it cannot admit a node the current behavior would have refused, and it covers the taints a GPU fleet actually carries. A fleet taints its nodes some other way should name it in target.taintSelectors, which takes precedence over this.

func ValidatePlacement added in v0.7.0 

func ValidatePlacement(orch *nvcrev1alpha1.OrchestrationSpec) error

ValidatePlacement rejects orchestration settings that contradict Unpinned.

Two of the three checks reject; the third deliberately does not. diagnose and topology.strictDomain are explicit requests for a particular number of jobs, and a user who wrote either of them alongside Unpinned asked for two incompatible things. topology.topologyKey is different: the GB200/GB300 catalog override injects it (see pkg/catalog/entries/training/nemotron5-8b.yaml), and overrides are applied before this runs, so rejecting it would fail a Nemotron run on GB200 with a message about a field the user never wrote. Unpinned does not partition, so there is nothing for a topology key to do and ignoring it is both safe and the only shippable option. See ADR-089.

func ValidateWRPlacement added in v0.7.0 

func ValidateWRPlacement(orch *nvcrev1alpha1.WorkloadOrchestration) error

ValidateWRPlacement is the WorkloadRun-tier half of ValidatePlacement.

It runs before buildWROrchestration lowers testScale, which is the only reason it exists as a separate check. By the time the OrchestrationSpec exists, intra-rack has already become Topology{StrictDomain: true} and intra-node has vanished entirely into NodesPerJobForScale, so a message from the lower tier would name a field the user never wrote.

full-scale is deliberately not rejected. It has always claimed "all target nodes in a single group" and never done it; Unpinned is what finally makes that claim true, so the pair agrees rather than conflicts.

func WorkflowName added in v0.7.0 

func WorkflowName(certName string, category nvcrev1alpha1.CertificateCategory) string

WorkflowName is the single naming rule for the Workflow a category creates. Offline `nvcrectl certification render` calls it too: when the two paths disagree, render previews a Workflow under a name the controller never creates, and two categories whose options differ render documents with the same metadata.name, so `render | kubectl apply -f -` keeps only the last.

Types

type BandwidthMeasurementReconciler 

type BandwidthMeasurementReconciler struct {
	client.Client
	// APIReader is an uncached client used for pod discovery. Pod watch events
	// from the informer cache can lag behind the API server by several seconds,
	// causing the "pod not found" requeue loop to spin longer than necessary.
	// A direct API read guarantees the freshest view at the cost of one extra
	// API server call per reconcile cycle while waiting for pods.
	APIReader  client.Reader
	Scheme     *runtime.Scheme
	Clientset  *kubernetes.Clientset
	Recorder   events.EventRecorder
	LogFetcher podlogs.PodLogFetcher // if nil, defaults to Clientset-backed fetcher
	// MaxConcurrentReconciles bounds the number of BandwidthMeasurement objects reconciled concurrently.
	MaxConcurrentReconciles int

	// RequeueInterval is the interval between reconcile cycles when polling.
	// Tests set this to 1s; production uses 15s.
	RequeueInterval time.Duration

	// FinalReadGracePeriod is the least time a failed final log read is
	// retried before the measurement completes without final results; the
	// Job's measurement timeout extends it. Zero uses
	// defaultFinalReadGracePeriod; a negative value completes on the first
	// failed read.
	FinalReadGracePeriod time.Duration
	// contains filtered or unexported fields
}

BandwidthMeasurementReconciler reconciles a BandwidthMeasurement object.

func (*BandwidthMeasurementReconciler) Reconcile 

func (r *BandwidthMeasurementReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

Reconcile is part of the main kubernetes reconciliation loop.

func (*BandwidthMeasurementReconciler) SetupWithManager 

func (r *BandwidthMeasurementReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type CertificationReconciler 

type CertificationReconciler struct {
	client.Client
	Scheme   *runtime.Scheme
	Recorder events.EventRecorder
	// APIReader reads straight from the API server, bypassing the informer
	// cache, so a cache miss on a child Workflow can be confirmed against the
	// live cluster before it is treated as a deletion (issue #384).
	APIReader               client.Reader
	WorkflowRequeueInterval time.Duration
	// MaxConcurrentReconciles bounds the number of Certification objects reconciled concurrently.
	MaxConcurrentReconciles int
}

CertificationReconciler reconciles a Certification object

func (*CertificationReconciler) Reconcile 

func (r *CertificationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.

func (*CertificationReconciler) SetupWithManager 

func (r *CertificationReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type GoodputMeasurementReconciler 

type GoodputMeasurementReconciler struct {
	client.Client
	Scheme     *runtime.Scheme
	Clientset  *kubernetes.Clientset
	Recorder   events.EventRecorder
	LogFetcher podlogs.PodLogFetcher // if nil, defaults to Clientset-backed fetcher
	// MaxConcurrentReconciles bounds the number of GoodputMeasurement objects reconciled concurrently.
	MaxConcurrentReconciles int
	// contains filtered or unexported fields
}

GoodputMeasurementReconciler reconciles a GoodputMeasurement object.

func (*GoodputMeasurementReconciler) Reconcile 

func (r *GoodputMeasurementReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

Reconcile is part of the main kubernetes reconciliation loop.

func (*GoodputMeasurementReconciler) SetupWithManager 

func (r *GoodputMeasurementReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type JobReconciler 

type JobReconciler struct {
	client.Client
	Scheme         *runtime.Scheme
	Clientset      *kubernetes.Clientset
	NodeDiscoverer *nodemonitor.NodeDiscoverer
	Recorder       events.EventRecorder
	// APIReader reads straight from the API server, bypassing the informer
	// cache, so a workload missing from the cache is confirmed gone before
	// anything acts on its absence.
	APIReader client.Reader
	// MaxConcurrentReconciles bounds the number of Job objects reconciled concurrently.
	MaxConcurrentReconciles int

	// WorkloadRequeueInterval is how often to poll workload status when job is in progress.
	// If zero, defaults to workloadRequeueInterval (15s).
	WorkloadRequeueInterval time.Duration

	// MeasurementTimeout is how long after the Job succeeds to wait for measurement
	// data before failing threshold validation. If zero, defaults to defaultMeasurementTimeout (5m).
	MeasurementTimeout time.Duration
	// contains filtered or unexported fields
}

JobReconciler reconciles a Job object

func (*JobReconciler) Reconcile 

func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.

func (*JobReconciler) SetupWithManager 

func (r *JobReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type OverrideContext 

type OverrideContext struct {
	Platform        string
	GPUArchitecture string
	WorkloadKind    string
	TopologyMode    string
	DomainCount     int
	Config          *apiextensionsv1.JSON
}

OverrideContext holds all detected values used for WhenSpec matching.

func BuildOverrideContext 

func BuildOverrideContext(spec *nvcrev1alpha1.WorkflowSpec, orch *nvcrev1alpha1.OrchestrationStatus, nodes []corev1.Node) OverrideContext

BuildOverrideContext is the exported version of buildOverrideContext for use by CLI tools.

type WorkflowReconciler 

type WorkflowReconciler struct {
	client.Client
	Scheme    *runtime.Scheme
	Clientset *kubernetes.Clientset
	Recorder  events.EventRecorder
	// APIReader reads straight from the API server, bypassing the informer
	// cache, so a Job missing from the cache can be checked against the live
	// cluster before it is treated as deleted (issue #385).
	APIReader          client.Reader
	JobRequeueInterval time.Duration
	// NodeDiscoverer answers where a Job's pods were actually placed. Under
	// Unpinned the controller does not choose the nodes, so this is the only way
	// to learn which ones ran and to record them on the group.
	NodeDiscoverer *nodemonitor.NodeDiscoverer
	// MaxConcurrentReconciles bounds the number of Workflow objects reconciled concurrently.
	MaxConcurrentReconciles int
}

WorkflowReconciler reconciles a Workflow object

func (*WorkflowReconciler) Reconcile 

func (r *WorkflowReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

Reconcile is part of the main kubernetes reconciliation loop which aims to move the current state of the cluster closer to the desired state.

func (*WorkflowReconciler) SetupWithManager 

func (r *WorkflowReconciler) SetupWithManager(mgr ctrl.Manager) error

SetupWithManager sets up the controller with the Manager.

type WorkloadRunReconciler 

type WorkloadRunReconciler struct {
	client.Client
	// APIReader is an uncached client used to confirm that a Workflow missing
	// from the informer cache is really gone. Right after Create, the
	// WorkloadRun's workflowRef write can reach the cache before the new
	// Workflow does; without a direct read that lag looks like a deletion and
	// fails the run (issue #352).
	APIReader client.Reader
	Scheme    *kruntime.Scheme
	Recorder  events.EventRecorder
	// MaxConcurrentReconciles bounds the number of WorkloadRun objects reconciled concurrently.
	MaxConcurrentReconciles int
}

WorkloadRunReconciler reconciles a WorkloadRun object.

func (*WorkloadRunReconciler) Reconcile 

func (r *WorkloadRunReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error)

func (*WorkloadRunReconciler) SetupWithManager 

func (r *WorkloadRunReconciler) SetupWithManager(mgr ctrl.Manager) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.