Documentation
¶
Index ¶
- func CalculateAwayQueueName(queueName string) string
- func IsHomeJob(job *jobdb.Job, currentPool string) bool
- func PrintJobSchedulingDetails(ctx *armadacontext.Context, prefix string, evictedJctx []*JobSchedulingContext)
- func PrintJobSummary(ctx *armadacontext.Context, prefix string, jctxs []*JobSchedulingContext)
- type GangSchedulingContext
- func (gctx *GangSchedulingContext) Cardinality() int
- func (gctx *GangSchedulingContext) Fit() GangSchedulingFit
- func (gctx *GangSchedulingContext) Id() string
- func (gctx *GangSchedulingContext) IsGang() bool
- func (gctx *GangSchedulingContext) JobIds() []string
- func (gctx *GangSchedulingContext) NodeUniformityLabel() string
- func (gctx *GangSchedulingContext) PriorityClassName() string
- func (gctx *GangSchedulingContext) SetGangNodeUniformityValues(labelName, labelValue string)
- type GangSchedulingFit
- type JobSchedulingContext
- func (jctx *JobSchedulingContext) AddNodeSelector(key, value string)
- func (jctx *JobSchedulingContext) Fail(unschedulableReason string)
- func (jctx *JobSchedulingContext) GetAssignedNode() *internaltypes.Node
- func (jctx *JobSchedulingContext) GetAssignedNodeId() string
- func (jctx *JobSchedulingContext) IsHomeJob(currentPool string) bool
- func (jctx *JobSchedulingContext) IsSuccessful() bool
- func (jctx *JobSchedulingContext) SchedulingKey() (internaltypes.SchedulingKey, bool)
- func (jctx *JobSchedulingContext) SetAssignedNode(assignedNode *internaltypes.Node)
- func (jctx *JobSchedulingContext) String() string
- type PodSchedulingContext
- type PreemptionType
- type QueueSchedulingContext
- func (qctx *QueueSchedulingContext) ClearJobSpecs()
- func (qctx *QueueSchedulingContext) GetAllocation() internaltypes.ResourceList
- func (qctx *QueueSchedulingContext) GetAllocationInclShortJobPenalty() internaltypes.ResourceList
- func (qctx *QueueSchedulingContext) GetBillableResource() internaltypes.ResourceList
- func (qctx *QueueSchedulingContext) GetWeight() float64
- func (qctx *QueueSchedulingContext) ReportString(verbosity int32) string
- func (qctx *QueueSchedulingContext) SetBillableResource()
- func (qctx *QueueSchedulingContext) String() string
- type SchedulingContext
- func (sctx *SchedulingContext) AddGangSchedulingContext(gctx *GangSchedulingContext) (bool, error)
- func (sctx *SchedulingContext) AddJobSchedulingContext(jctx *JobSchedulingContext) (bool, error)
- func (sctx *SchedulingContext) AddQueueSchedulingContext(queue string, weight float64, rawWeight float64, ...) error
- func (sctx *SchedulingContext) AllocatedByQueueAndPriority() map[string]map[string]internaltypes.ResourceList
- func (sctx *SchedulingContext) CalculateTheoreticalShare(priority float64) float64
- func (sctx *SchedulingContext) ClearJobSpecs()
- func (sctx *SchedulingContext) ClearUnfeasibleSchedulingKeys()
- func (sctx *SchedulingContext) EvictGang(gctx *GangSchedulingContext) (bool, error)
- func (sctx *SchedulingContext) EvictJob(jctx *JobSchedulingContext) (bool, error)
- func (sctx *SchedulingContext) FairnessError() float64
- func (sctx *SchedulingContext) GetQueue(queue string) (fairness.Queue, bool)
- func (sctx *SchedulingContext) GetSpotPrice() float64
- func (sctx *SchedulingContext) PreemptJob(jctx *JobSchedulingContext) (bool, error)
- func (sctx *SchedulingContext) QueueContextExists(job *jobdb.Job) bool
- func (sctx *SchedulingContext) ReportString(verbosity int32) string
- func (sctx *SchedulingContext) String() string
- func (sctx *SchedulingContext) SuccessfulJobSchedulingContexts() []*JobSchedulingContext
- func (sctx *SchedulingContext) UpdateFairShares()
- type SchedulingType
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CalculateAwayQueueName ¶ added in v0.14.0
func PrintJobSchedulingDetails ¶ added in v0.16.6
func PrintJobSchedulingDetails(ctx *armadacontext.Context, prefix string, evictedJctx []*JobSchedulingContext)
PrintJobSchedulingDetails prints details of where jobs were scheduled It will log the first 100 at info level, and all at debug level
func PrintJobSummary ¶
func PrintJobSummary(ctx *armadacontext.Context, prefix string, jctxs []*JobSchedulingContext)
PrintJobSummary logs a summary of the job scheduling context It will log a high level summary at Info level, and a list of all queues + jobs affected at debug level
Types ¶
type GangSchedulingContext ¶
type GangSchedulingContext struct {
Created time.Time
IsGangJob bool
GangId string
NodeUniformity string
PriorityClass string
Queue string
JobSchedulingContexts []*JobSchedulingContext
TotalResourceRequests internaltypes.ResourceList
AllJobsEvicted bool
RequestsFloatingResources bool
}
func NewGangSchedulingContext ¶
func NewGangSchedulingContext(jctxs []*JobSchedulingContext) *GangSchedulingContext
func (*GangSchedulingContext) Cardinality ¶
func (gctx *GangSchedulingContext) Cardinality() int
Cardinality returns the number of jobs in the gang.
func (*GangSchedulingContext) Fit ¶
func (gctx *GangSchedulingContext) Fit() GangSchedulingFit
func (*GangSchedulingContext) Id ¶ added in v0.19.13
func (gctx *GangSchedulingContext) Id() string
Id returns the id of the gang
func (*GangSchedulingContext) IsGang ¶ added in v0.19.13
func (gctx *GangSchedulingContext) IsGang() bool
func (*GangSchedulingContext) JobIds ¶
func (gctx *GangSchedulingContext) JobIds() []string
JobIds returns a sliced composed of the ids of the jobs that make up the gang.
func (*GangSchedulingContext) NodeUniformityLabel ¶ added in v0.19.13
func (gctx *GangSchedulingContext) NodeUniformityLabel() string
NodeUniformityLabel returns the label used to ensure scheduling unfiormity for the gang
func (*GangSchedulingContext) PriorityClassName ¶ added in v0.19.13
func (gctx *GangSchedulingContext) PriorityClassName() string
func (*GangSchedulingContext) SetGangNodeUniformityValues ¶ added in v0.20.16
func (gctx *GangSchedulingContext) SetGangNodeUniformityValues(labelName, labelValue string)
SetGangNodeUniformityValues sets the gang node uniformity label name and value for all jobs in the gang.
type GangSchedulingFit ¶
type GangSchedulingFit struct {
// The number of jobs in the gang that were successfully scheduled.
NumScheduled int
// The mean PreemptedAtPriority among successfully scheduled pods in the gang.
MeanPreemptedAtPriority float64
}
func (GangSchedulingFit) Less ¶
func (f GangSchedulingFit) Less(other GangSchedulingFit) bool
type JobSchedulingContext ¶
type JobSchedulingContext struct {
// Time at which this context was created.
Created time.Time
// Id of the job this pod corresponds to.
JobId string
// Indicates whether this context is for re-scheduling an evicted job.
IsEvicted bool
// Job spec.
Job *jobdb.Job
// Scheduling requirements of this job.
// We currently require that each job contains exactly one pod spec.
PodRequirements *internaltypes.PodRequirements
// Resource requirements in an efficient internaltypes.ResourceList
KubernetesResourceRequirements internaltypes.ResourceList
// Node selectors to consider in addition to those included with the PodRequirements.
// These are added as part of scheduling to further constrain where nodes are scheduled,
// e.g., to ensure evicted jobs are re-scheduled onto the same node.
//
// If some key appears in both PodRequirements.NodeSelector and AdditionalNodeSelectors,
// the value in AdditionalNodeSelectors trumps that of PodRequirements.NodeSelector.
AdditionalNodeSelectors map[string]string
// Tolerations to consider in addition to those included with the PodRequirements.
// These are added as part of scheduling to expand the set of nodes a job can be scheduled on.
AdditionalTolerations []v1.Toleration
// Reason for why the job could not be scheduled.
// Empty if the job was scheduled successfully.
UnschedulableReason string
// Pod scheduling contexts for the individual pods that make up the job.
PodSchedulingContext *PodSchedulingContext
// The number of active jobs in the gang this jctx belongs to
// This may differ from the jobs GangInfo cardinality, as it finished jobs are not included in this count
CurrentGangCardinality int
// This is the node the pod is assigned to.
// This is only set for evicted jobs and is set alongside adding an additionalNodeSelector for the node
AssignedNode *internaltypes.Node
// job that preempted this pod
PreemptingJob *jobdb.Job
// The type of preemption used to preempt this job (i.e fairshare, urgency)
PreemptionType PreemptionType
// Description of the cause of preemption
PreemptionDescription string
// If this job context should contribute to the billable resource of the queue
Billable bool
// Gang node uniformity label name (e.g., "rack") - only set for gang jobs with uniformity requirements
GangNodeUniformityLabelName string
// Gang node uniformity label value (e.g., "rack-1") - the actual value selected during scheduling
GangNodeUniformityLabelValue string
}
JobSchedulingContext is created by the scheduler and contains information about the decision made by the scheduler for a particular job.
func JobSchedulingContextFromJob ¶
func JobSchedulingContextFromJob(job *jobdb.Job) *JobSchedulingContext
func JobSchedulingContextsFromJobs ¶
func JobSchedulingContextsFromJobs[J *jobdb.Job](jobs []J) []*JobSchedulingContext
func (*JobSchedulingContext) AddNodeSelector ¶
func (jctx *JobSchedulingContext) AddNodeSelector(key, value string)
func (*JobSchedulingContext) Fail ¶
func (jctx *JobSchedulingContext) Fail(unschedulableReason string)
func (*JobSchedulingContext) GetAssignedNode ¶ added in v0.15.5
func (jctx *JobSchedulingContext) GetAssignedNode() *internaltypes.Node
func (*JobSchedulingContext) GetAssignedNodeId ¶
func (jctx *JobSchedulingContext) GetAssignedNodeId() string
func (*JobSchedulingContext) IsHomeJob ¶ added in v0.14.0
func (jctx *JobSchedulingContext) IsHomeJob(currentPool string) bool
func (*JobSchedulingContext) IsSuccessful ¶
func (jctx *JobSchedulingContext) IsSuccessful() bool
func (*JobSchedulingContext) SchedulingKey ¶
func (jctx *JobSchedulingContext) SchedulingKey() (internaltypes.SchedulingKey, bool)
SchedulingKey returns the scheduling key of the embedded job. If the jctx contains additional node selectors or tolerations, the key is invalid and the second return value is false.
func (*JobSchedulingContext) SetAssignedNode ¶ added in v0.15.5
func (jctx *JobSchedulingContext) SetAssignedNode(assignedNode *internaltypes.Node)
func (*JobSchedulingContext) String ¶
func (jctx *JobSchedulingContext) String() string
type PodSchedulingContext ¶
type PodSchedulingContext struct {
// Time at which this context was created.
Created time.Time
// ID of the node that the pod was assigned to, or empty.
NodeId string
// If set, indicates that the pod was scheduled on a specific node type.
WellKnownNodeTypeName string
// Priority this pod was most recently attempted to be scheduled at.
// If scheduling was successful, resources were marked as allocated to the job at this priority.
ScheduledAtPriority int32
// Maximum priority that this pod preempted other pods at.
PreemptedAtPriority int32
// Total number of nodes in the cluster when trying to schedule.
NumNodes int
// Number of nodes excluded by reason.
NumExcludedNodesByReason map[string]int
// If this pod was scheduled as an away job
ScheduledAway bool
// The method of scheduling that was used to schedule this job
SchedulingMethod SchedulingType
}
PodSchedulingContext is returned by SelectAndBindNodeToPod and contains detailed information on the scheduling decision made for this pod.
func (*PodSchedulingContext) IsSuccessful ¶
func (pctx *PodSchedulingContext) IsSuccessful() bool
func (*PodSchedulingContext) String ¶
func (pctx *PodSchedulingContext) String() string
type PreemptionType ¶ added in v0.16.10
type PreemptionType string
const ( Unknown PreemptionType = "unknown" UnknownGangJob PreemptionType = "unknown-gang" PreemptedWithUrgencyPreemption PreemptionType = "urgency" PreemptedWithOptimiserPreemption PreemptionType = "optimiser" PreemptedViaApi PreemptionType = "api" PreemptedViaNodeReconciler PreemptionType = "reconciler" )
type QueueSchedulingContext ¶
type QueueSchedulingContext struct {
// The scheduling context to which this QueueSchedulingContext belongs.
SchedulingContext *SchedulingContext
// Time at which this context was created.
Created time.Time
// Queue name.
Queue string
// Determines the fair share of this queue relative to other queues.
Weight float64
// Raw Weight of the queue before any priority boosts.
// This is purely informational as all scheduling decisions are made using Weight
RawWeight float64
// Limits job scheduling rate for this queue.
// Use the "Started" time to ensure limiter state remains constant within each scheduling round.
Limiter *rate.Limiter
// Total resources assigned to the queue across all clusters by priority class priority.
// Includes jobs scheduled during this invocation of the scheduler.
Allocated internaltypes.ResourceList
// Resource which should be charged for when using market driven scheduling
BillableResource internaltypes.ResourceList
// Used to penalize short jobs by pretending they are still running
// if they started recently but then exited.
ShortJobPenalty internaltypes.ResourceList
// Total demand from this queue. This is essentially the cumulative resources of all non-terminal jobs at the
// start of the scheduling cycle
Demand internaltypes.ResourceList
// Constrained demand for this queue. This differs from Demand in that it takes into account any constraints that we have
// placed on the queue
ConstrainedDemand internaltypes.ResourceList
FairShare float64
// not capped by this queue's demand, so includes any fairshare unused by this queue. It's effectively the share this queue would get if it had infinite demand.
// This measure is designed to not punish queues for being undemanding.
UncappedAdjustedFairShare float64
// capped by this queue's demand, so does not include any fairshare unused by this queue.
DemandCappedAdjustedFairShare float64
// IdealisedValue is the total value of jobs that would be scheduled from this queue if there was no fragmentation.
// This only applies if the pool was market priced
IdealisedValue float64
// IdealisedAllocated are the resources that would be allocated from this queue if there was no fragmentation.
// This only applies if the pool was market priced
IdealisedAllocated internaltypes.ResourceList
// RealisedValue is the total value of jobs that were actually scheduled. Note that this us only populated
// on market driven pools
RealisedValue float64
// Total resources assigned to the queue across all clusters by priority class.
// Includes jobs scheduled during this invocation of the scheduler.
AllocatedByPriorityClass map[string]internaltypes.ResourceList
// Resources assigned to this queue during this scheduling cycle.
ScheduledResourcesByPriorityClass map[string]internaltypes.ResourceList
// Resources evicted from this queue during this scheduling cycle.
EvictedResourcesByPriorityClass map[string]internaltypes.ResourceList
// Resources preempted from this queue during this scheduling cycle.
PreemptedByOptimiserResourceByPriorityClass map[string]internaltypes.ResourceList
// Job scheduling contexts associated with successful scheduling attempts.
SuccessfulJobSchedulingContexts map[string]*JobSchedulingContext
// Job scheduling contexts associated with rescheduled jobs.
RescheduledJobSchedulingContexts map[string]*JobSchedulingContext
// Job scheduling contexts associated with preempted jobs.
PreemptedByOptimiserJobSchedulingContexts map[string]*JobSchedulingContext
// Job scheduling contexts associated with unsuccessful scheduling attempts.
UnsuccessfulJobSchedulingContexts map[string]*JobSchedulingContext
// Jobs evicted in this round.
EvictedJobsById map[string]bool
}
QueueSchedulingContext captures the decisions made by the scheduler during one invocation for a particular queue.
func (*QueueSchedulingContext) ClearJobSpecs ¶
func (qctx *QueueSchedulingContext) ClearJobSpecs()
ClearJobSpecs zeroes out job specs to reduce memory usage.
func (*QueueSchedulingContext) GetAllocation ¶
func (qctx *QueueSchedulingContext) GetAllocation() internaltypes.ResourceList
GetAllocation is necessary to implement the fairness.Queue interface.
func (*QueueSchedulingContext) GetAllocationInclShortJobPenalty ¶ added in v0.19.4
func (qctx *QueueSchedulingContext) GetAllocationInclShortJobPenalty() internaltypes.ResourceList
func (*QueueSchedulingContext) GetBillableResource ¶ added in v0.19.9
func (qctx *QueueSchedulingContext) GetBillableResource() internaltypes.ResourceList
func (*QueueSchedulingContext) GetWeight ¶
func (qctx *QueueSchedulingContext) GetWeight() float64
GetWeight is necessary to implement the fairness.Queue interface.
func (*QueueSchedulingContext) ReportString ¶
func (qctx *QueueSchedulingContext) ReportString(verbosity int32) string
func (*QueueSchedulingContext) SetBillableResource ¶ added in v0.19.9
func (qctx *QueueSchedulingContext) SetBillableResource()
func (*QueueSchedulingContext) String ¶
func (qctx *QueueSchedulingContext) String() string
type SchedulingContext ¶
type SchedulingContext struct {
// Time at which the scheduling cycle started.
Started time.Time
// Time at which the scheduling cycle finished.
Finished time.Time
// Pool for which we're currently scheduling jobs.
Pool string
// Determines how fairness is computed.
FairnessCostProvider fairness.FairnessCostProvider
// Limits job scheduling rate globally across all queues.
// Use the "Started" time to ensure limiter state remains constant within each scheduling round.
Limiter *rate.Limiter
// Sum of queue weights across all queues.
WeightSum float64
// Per-queue scheduling contexts.
QueueSchedulingContexts map[string]*QueueSchedulingContext
// Total resources across all clusters in this pool available at the start of the scheduling cycle.
TotalResources internaltypes.ResourceList
// Allocated resources across all clusters in this pool
Allocated internaltypes.ResourceList
// Resources assigned across all queues during this scheduling cycle.
ScheduledResources internaltypes.ResourceList
// Resources evicted across all queues during this scheduling cycle.
EvictedResources internaltypes.ResourceList
// Total number of successfully scheduled jobs.
NumScheduledJobs int
// Total number of successfully scheduled gangs.
NumScheduledGangs int
// Total number of evicted jobs.
NumEvictedJobs int
// TODO(reports): Count the number of evicted gangs.
// Reason for why the scheduling round finished.
TerminationReason string
// Used to efficiently generate scheduling keys.
SchedulingKeyGenerator *internaltypes.SchedulingKeyGenerator
// Record of job scheduling requirements known to be unfeasible.
// Used to immediately reject new jobs with identical reqirements.
// Maps to the JobSchedulingContext of a previous job attempted to schedule with the same key.
UnfeasibleSchedulingKeys map[internaltypes.SchedulingKey]*JobSchedulingContext
SpotPrice *float64
}
SchedulingContext contains information necessary for scheduling and records what happened in a scheduling round.
func NewSchedulingContext ¶
func NewSchedulingContext( pool string, fairnessCostProvider fairness.FairnessCostProvider, limiter *rate.Limiter, totalResources internaltypes.ResourceList, ) *SchedulingContext
func (*SchedulingContext) AddGangSchedulingContext ¶
func (sctx *SchedulingContext) AddGangSchedulingContext(gctx *GangSchedulingContext) (bool, error)
func (*SchedulingContext) AddJobSchedulingContext ¶
func (sctx *SchedulingContext) AddJobSchedulingContext(jctx *JobSchedulingContext) (bool, error)
AddJobSchedulingContext adds a job scheduling context. Automatically updates scheduled resources.
func (*SchedulingContext) AddQueueSchedulingContext ¶
func (sctx *SchedulingContext) AddQueueSchedulingContext( queue string, weight float64, rawWeight float64, initialAllocatedByPriorityClass map[string]internaltypes.ResourceList, demand internaltypes.ResourceList, constrainedDemand internaltypes.ResourceList, shortJobPenalty internaltypes.ResourceList, limiter *rate.Limiter, ) error
func (*SchedulingContext) AllocatedByQueueAndPriority ¶
func (sctx *SchedulingContext) AllocatedByQueueAndPriority() map[string]map[string]internaltypes.ResourceList
AllocatedByQueueAndPriority returns map from queue name and priority to resources allocated.
func (*SchedulingContext) CalculateTheoreticalShare ¶ added in v0.15.8
func (sctx *SchedulingContext) CalculateTheoreticalShare(priority float64) float64
CalculateTheoreticalShare calculates the maximum potential adjustedFairShare share of a new queue at a given priority.
func (*SchedulingContext) ClearJobSpecs ¶
func (sctx *SchedulingContext) ClearJobSpecs()
ClearJobSpecs zeroes out job specs to reduce memory usage.
func (*SchedulingContext) ClearUnfeasibleSchedulingKeys ¶
func (sctx *SchedulingContext) ClearUnfeasibleSchedulingKeys()
func (*SchedulingContext) EvictGang ¶
func (sctx *SchedulingContext) EvictGang(gctx *GangSchedulingContext) (bool, error)
func (*SchedulingContext) EvictJob ¶
func (sctx *SchedulingContext) EvictJob(jctx *JobSchedulingContext) (bool, error)
func (*SchedulingContext) FairnessError ¶
func (sctx *SchedulingContext) FairnessError() float64
FairnessError returns the cumulative delta between adjusted fair share and actual share for all users who are below their fair share
func (*SchedulingContext) GetQueue ¶
func (sctx *SchedulingContext) GetQueue(queue string) (fairness.Queue, bool)
GetQueue is necessary to implement the fairness.QueueRepository interface.
func (*SchedulingContext) GetSpotPrice ¶ added in v0.19.6
func (sctx *SchedulingContext) GetSpotPrice() float64
func (*SchedulingContext) PreemptJob ¶ added in v0.16.10
func (sctx *SchedulingContext) PreemptJob(jctx *JobSchedulingContext) (bool, error)
func (*SchedulingContext) QueueContextExists ¶ added in v0.14.1
func (sctx *SchedulingContext) QueueContextExists(job *jobdb.Job) bool
QueueContextExists returns true if we know about the queue associated with the job. An example of when this can return false is when a job is running on a node
func (*SchedulingContext) ReportString ¶
func (sctx *SchedulingContext) ReportString(verbosity int32) string
func (*SchedulingContext) String ¶
func (sctx *SchedulingContext) String() string
func (*SchedulingContext) SuccessfulJobSchedulingContexts ¶
func (sctx *SchedulingContext) SuccessfulJobSchedulingContexts() []*JobSchedulingContext
func (*SchedulingContext) UpdateFairShares ¶
func (sctx *SchedulingContext) UpdateFairShares()
UpdateFairShares updates FairShare/DemandCappedAdjustedFairShare/UncappedAdjustedFairShare for every QueueSchedulingContext associated with this SchedulingContext. This works by calculating a FairShare as queue_weight/sum_of_all_queue_weights then DemandCappedAdjustedFairShare/UncappedAdjustedFairShare by resharing any unused capacity (as determined by a queue's demand).
type SchedulingType ¶ added in v0.15.4
type SchedulingType string
const ( None SchedulingType = "none" Rescheduled SchedulingType = "rescheduled" ScheduledWithoutPreemption SchedulingType = "no-preemption" ScheduledWithUrgencyBasedPreemption SchedulingType = "urgency" ScheduledAsAwayJob SchedulingType = "away" ScheduledWithFairnessOptimiser SchedulingType = "optimiser" )