scheduler

package
v0.26.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 15, 2026 License: Apache-2.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source
const (
	EntityEvent = "event"

	ISODateFormat = "2006-01-02T15:04:05Z"

	EventCategorySLAMiss    EventCategory = "sla_miss"
	EventCategoryJobFailure EventCategory = "failure"
	EventCategoryJobSuccess EventCategory = "job_success"
	EventCategoryReplay     EventCategory = "replay_lifecycle"

	SLAMissEvent    JobEventType = "sla_miss"
	JobFailureEvent JobEventType = "failure"
	JobSuccessEvent JobEventType = "job_success"
	ReplayEvent     JobEventType = "replay_lifecycle"

	TaskStartEvent   JobEventType = "task_start"
	TaskRetryEvent   JobEventType = "task_retry"
	TaskFailEvent    JobEventType = "task_fail"
	TaskSuccessEvent JobEventType = "task_success"

	HookStartEvent   JobEventType = "hook_start"
	HookRetryEvent   JobEventType = "hook_retry"
	HookFailEvent    JobEventType = "hook_fail"
	HookSuccessEvent JobEventType = "hook_success"

	SensorStartEvent   JobEventType = "sensor_start"
	SensorRetryEvent   JobEventType = "sensor_retry"
	SensorFailEvent    JobEventType = "sensor_fail"
	SensorSuccessEvent JobEventType = "sensor_success"

	OperatorStartEvent   JobEventType = "operator_start"
	OperatorRetryEvent   JobEventType = "operator_retry"
	OperatorFailEvent    JobEventType = "operator_fail"
	OperatorSuccessEvent JobEventType = "operator_success"

	StatusFiring   EventStatus = "firing"
	StatusResolved EventStatus = "resolved"
)
View Source
const (
	EntityJobRun           = "jobRun"
	EntityThirdPartySensor = "thirdPartySensor"

	OperatorTask   OperatorType = "task"
	OperatorSensor OperatorType = "sensor"
	OperatorHook   OperatorType = "hook"

	UpstreamTypeStatic   = "static"
	UpstreamTypeInferred = "inferred"
)
View Source
const (
	MetricNotificationQueue         = "notification_queue_total"
	MetricNotificationWorkerBatch   = "notification_worker_batch_total"
	MetricNotificationWorkerSendErr = "notification_worker_send_err_total"
	MetricNotificationSend          = "notification_worker_send_total"
)
View Source
const (
	// MaxLineageDepth is a safeguard to avoid infinite recursion in case of unexpected cycles
	// generally we don't expect lineage to be deeper than 20 levels
	MaxLineageDepth = 25
)

Variables

Functions

func GetSLADuration added in v0.22.4

func GetSLADuration(alerts []Alert) (int64, error)

func GroupJobsByTenant

func GroupJobsByTenant(j []*JobWithDetails) map[tenant.Tenant][]*JobWithDetails

Types

type Alert

type Alert struct {
	On       EventCategory
	Channels []string
	Config   map[string]string
	Severity string
	Team     string
}

type AlertAttrs added in v0.11.7

type AlertAttrs struct {
	Owner         string
	JobURN        string
	Title         string
	SchedulerHost string
	Status        EventStatus
	JobEvent      *Event

	JobWithDetails *JobWithDetails

	AlertManager AlertManagerConfig
}

type AlertManagerConfig added in v0.21.3

type AlertManagerConfig struct {
	Endpoint string
}

AlertManagerConfig holds the configuration for the AlertManager endpoint. we can add more fields in the future if needed, such as dashboard url or console url

type BackfilConfig added in v0.25.0

type BackfilConfig struct {
	Dstart      time.Time
	Dend        time.Time
	JobConfig   map[string]string
	Assets      map[string]string
	Description string
	Category    string
	ApprovalID  string
	UserID      string
}

func NewBackfillConfig added in v0.25.0

func NewBackfillConfig(startTime, endTime time.Time, jobConfig, assets map[string]string, description, category, approvalID, userID string) *BackfilConfig

type Backfill added in v0.25.0

type Backfill struct {
	SchedulerRunID string
	CanceledBy     string
	// contains filtered or unexported fields
}

func NewBackfill added in v0.25.0

func NewBackfill(id uuid.UUID, jobName JobName, tenant tenant.Tenant, config *BackfilConfig, state BackfillState, createdAt, updatedAt time.Time, message string) *Backfill

func NewBackfillRequest added in v0.25.0

func NewBackfillRequest(jobName JobName, tenant tenant.Tenant, config *BackfilConfig, state BackfillState, message string) *Backfill

func (*Backfill) Config added in v0.25.0

func (r *Backfill) Config() *BackfilConfig

func (*Backfill) CreatedAt added in v0.25.0

func (r *Backfill) CreatedAt() time.Time

func (*Backfill) GetApprovalID added in v0.25.0

func (r *Backfill) GetApprovalID() string

func (*Backfill) GetCanceledBy added in v0.25.0

func (r *Backfill) GetCanceledBy() string

func (*Backfill) GetCategory added in v0.25.0

func (r *Backfill) GetCategory() string

func (*Backfill) GetEndTime added in v0.25.0

func (r *Backfill) GetEndTime() time.Time

func (*Backfill) GetJobConfig added in v0.25.0

func (r *Backfill) GetJobConfig() map[string]string

func (*Backfill) GetStartTime added in v0.25.0

func (r *Backfill) GetStartTime() time.Time

func (*Backfill) GetUserID added in v0.25.0

func (r *Backfill) GetUserID() string

func (*Backfill) ID added in v0.25.0

func (r *Backfill) ID() uuid.UUID

func (*Backfill) IsTerminated added in v0.25.0

func (r *Backfill) IsTerminated() bool

func (*Backfill) JobName added in v0.25.0

func (r *Backfill) JobName() JobName

func (*Backfill) Message added in v0.25.0

func (r *Backfill) Message() string

func (*Backfill) SetJobConfig added in v0.25.0

func (r *Backfill) SetJobConfig(config map[string]string)

func (*Backfill) State added in v0.25.0

func (r *Backfill) State() BackfillState

func (*Backfill) Tenant added in v0.25.0

func (r *Backfill) Tenant() tenant.Tenant

func (*Backfill) UpdatedAt added in v0.25.0

func (r *Backfill) UpdatedAt() time.Time

type BackfillState added in v0.25.0

type BackfillState string
const (
	// BackfillStateCreated is an initial state which indicates the backfil has been created but not picked up yet
	BackfillStateCreated BackfillState = "created"

	// BackfillStateInProgress indicates the backfil is being executed
	BackfillStateInProgress BackfillState = "in progress"

	// BackfillStateSuccess is a terminal state which occurs when the backfil execution finished with successful job runs
	BackfillStateSuccess BackfillState = "success"

	BackfillStateTimeout BackfillState = "timeout"

	// BackfillStateFailed is a terminal state which occurs when the backfil execution failed, timed out, or finished with one of the run fails
	BackfillStateFailed BackfillState = "failed"

	// BackfillStateCancelled is a terminal state which occurs when the backfil is cancelled by user
	BackfillStateCancelled BackfillState = "cancelled"

	EntityBackfill = "backfill"
)

func (BackfillState) String added in v0.25.0

func (j BackfillState) String() string

type Change added in v0.22.12

type Change struct {
	Property string
	Diff     ChangeDiff
}

func (Change) NewValue added in v0.22.12

func (ch Change) NewValue() string

func (Change) OldValue added in v0.22.12

func (ch Change) OldValue() string

type ChangeDiff added in v0.22.12

type ChangeDiff string

type Changelog added in v0.22.12

type Changelog struct {
	Change    []Change
	Type      string
	CreatedAt time.Time
}

func (*Changelog) GetAttributeChange added in v0.22.12

func (c *Changelog) GetAttributeChange(attributeName string) *Change

type ChangelogFilter added in v0.22.12

type ChangelogFilter struct {
	ProjectName tenant.ProjectName
	Name        string
	StartTime   time.Time
	EndTime     time.Time
}

type ConfigMap

type ConfigMap map[string]string

type DagRun added in v0.22.0

type DagRun struct {
	RunID         string     `json:"run_id"`
	JobName       string     `json:"dag_id"`
	ScheduledAt   time.Time  `json:"scheduled_at"`
	ExecutionDate time.Time  `json:"execution_date"`
	StartTime     time.Time  `json:"start_date"`
	EndTime       *time.Time `json:"end_date,omitempty"`
}

type DamperFactor added in v0.26.0

type DamperFactor struct {
	BaseValue float64
	Alpha     float64
	MinValue  float64
	Method    DamperFactorMethod
}

func (DamperFactor) InitialDamper added in v0.26.0

func (d DamperFactor) InitialDamper() float64

func (DamperFactor) NextDamper added in v0.26.0

func (d DamperFactor) NextDamper(current float64) float64

type DamperFactorMethod added in v0.26.0

type DamperFactorMethod int
const (
	DamperFactorMethodExponential DamperFactorMethod = iota
	DamperFactorMethodDecrement
	DamperFactorMethodConstant
)

type DataCompletenessByDate added in v0.22.14

type DataCompletenessByDate struct {
	Date       time.Time
	IsComplete bool
}

type DataCompletenessStatus added in v0.22.14

type DataCompletenessStatus struct {
	IsComplete             bool
	DataCompletenessByDate []*DataCompletenessByDate
}

type Event

type Event struct {
	JobName        JobName
	URN            string
	Tenant         tenant.Tenant
	Type           JobEventType
	EventTime      time.Time
	OperatorName   string
	Status         State
	JobScheduledAt time.Time
	Values         map[string]any
	SLAObjectList  []*SLAObject

	EventContext *EventContext

	SkipAlerting bool
}

func EventFrom

func EventFrom(eventTypeName string, eventValues map[string]any, jobName JobName, tenent tenant.Tenant) (*Event, error)

func (Event) String added in v0.19.0

func (e Event) String() string

type EventCategory added in v0.18.0

type EventCategory string

func (EventCategory) String added in v0.18.0

func (e EventCategory) String() string

type EventContext added in v0.22.0

type EventContext struct {
	Tenant              tenant.Tenant
	Type                JobEventType        `json:"event_type"`
	OperatorType        OperatorType        `json:"operator_type"`
	OperatorRunInstance OperatorRunInstance `json:"task_instance"`
	DagRun              DagRun              `json:"dag_run"`
	Task                OperatorObj         `json:"task"`
	EventReason         *string             `json:"event_reason,omitempty"`
}

func EventContextFrom added in v0.22.0

func EventContextFrom(eventType JobEventType, rawEventContext any) (*EventContext, error)

type EventName

type EventName string

type EventStatus added in v0.11.7

type EventStatus string

type Executor

type Executor struct {
	Name string
	Type ExecutorType
}

func ExecutorFrom

func ExecutorFrom(name string, executorType ExecutorType) (Executor, error)

func ExecutorFromEnum

func ExecutorFromEnum(name, enum string) (Executor, error)

type ExecutorInput

type ExecutorInput struct {
	Configs ConfigMap
	Secrets ConfigMap
	Files   ConfigMap
}

type ExecutorType

type ExecutorType string
const (
	ExecutorTask ExecutorType = "task"
	ExecutorHook ExecutorType = "hook"
)

func ExecutorTypeFrom

func ExecutorTypeFrom(val string) (ExecutorType, error)

func (ExecutorType) String

func (e ExecutorType) String() string

type HTTPUpstreams

type HTTPUpstreams struct {
	Name    string
	URL     string
	Headers map[string]string
	Params  map[string]string
}

type Hook

type Hook struct {
	Name        string               `json:"name,omitempty"`
	Version     string               `json:"version,omitempty"`
	Config      map[string]string    `json:"config,omitempty"`
	AlertConfig *OperatorAlertConfig `json:"alert_config,omitempty"`
}

type Job

type Job struct {
	ID     uuid.UUID
	Name   JobName
	Tenant tenant.Tenant

	Destination resource.URN
	Task        *Task
	Hooks       []*Hook

	WindowConfig window.Config
	Assets       map[string]string
}

func (*Job) GetHook

func (j *Job) GetHook(hookName string) (*Hook, error)

func (*Job) GetHookAlertConfigByName added in v0.22.0

func (j *Job) GetHookAlertConfigByName(hookName string) *OperatorAlertConfig

func (*Job) GetOperatorAlertConfigByName added in v0.22.0

func (j *Job) GetOperatorAlertConfigByName(operatorType OperatorType, operatorName string) *OperatorAlertConfig

func (*Job) GetTaskAlertConfig added in v0.22.0

func (j *Job) GetTaskAlertConfig() *OperatorAlertConfig

func (*Job) IsDryRun added in v0.19.0

func (j *Job) IsDryRun() bool

func (*Job) URN added in v0.11.7

func (j *Job) URN() string

type JobEventType

type JobEventType string

func FromStringToEventType

func FromStringToEventType(name string) (JobEventType, error)

func (JobEventType) IsOfType

func (event JobEventType) IsOfType(category EventCategory) bool

func (JobEventType) IsOperatorType added in v0.22.2

func (event JobEventType) IsOperatorType() bool

func (JobEventType) String added in v0.7.0

func (event JobEventType) String() string

type JobExecutionSummary added in v0.22.4

type JobExecutionSummary struct {
	JobName JobName
	SLA     SLAConfig
	// Level marks the distance from the original job in question
	Level              int
	JobRunSummary      *JobRunSummary
	DownstreamPathName string
	DelaySummary       *JobRunDelaySummary
	HistoricalSummary  JobHistoricalDuration
}

JobExecutionSummary is a flattened version of JobLineageSummary

type JobHistoricalDuration added in v0.24.0

type JobHistoricalDuration struct {
	TaskDuration time.Duration
	HookDuration time.Duration
}

type JobLineageSummary added in v0.22.4

type JobLineageSummary struct {
	JobName   JobName
	IsEnabled bool
	Upstreams []*JobLineageSummary

	Tenant           tenant.Tenant
	ScheduleInterval string
	SLA              SLAConfig
	Window           *window.Config

	// JobRuns contain the mapping of downstream's job name to their respective job run summaries
	JobRuns map[JobName]*JobRunSummary
}

func (*JobLineageSummary) GenerateLineageExecutionSummary added in v0.23.19

func (j *JobLineageSummary) GenerateLineageExecutionSummary(maxUpstreamsPerLevel, maxDepth int) *JobRunLineage

func (*JobLineageSummary) GetFlattenedSummaries added in v0.23.19

func (j *JobLineageSummary) GetFlattenedSummaries(maxUpstreamsPerLevel, maxDepth int) []*JobExecutionSummary

func (*JobLineageSummary) GetRunForJob added in v0.23.5

func (j *JobLineageSummary) GetRunForJob(jobName JobName) *JobRunSummary

type JobMetadata

type JobMetadata struct {
	Version     int
	Owner       string
	Description string
	Labels      map[string]string
}

type JobName

type JobName string

func JobNameFrom

func JobNameFrom(name string) (JobName, error)

func (JobName) GetJobURN added in v0.17.0

func (n JobName) GetJobURN(tnnt tenant.Tenant) string

func (JobName) String

func (n JobName) String() string

type JobRun

type JobRun struct {
	ID uuid.UUID

	JobName       JobName
	Tenant        tenant.Tenant
	State         State
	ScheduledAt   time.Time
	SLAAlert      bool
	StartTime     time.Time
	EndTime       *time.Time
	WindowStart   *time.Time
	WindowEnd     *time.Time
	SLADefinition int64

	Monitoring map[string]any
}

func (*JobRun) HasSLABreached added in v0.9.7

func (j *JobRun) HasSLABreached() bool

type JobRunDelaySummary added in v0.23.19

type JobRunDelaySummary struct {
	ScheduledWayTooLateSeconds   int64
	SystemSchedulingDelaySeconds int64
}

type JobRunDetailsList added in v0.17.0

type JobRunDetailsList []*JobRunWithDetails

func (JobRunDetailsList) FilterRunsManagedByReplay added in v0.17.0

func (j JobRunDetailsList) FilterRunsManagedByReplay(runs []*JobRunStatus) JobRunDetailsList

func (JobRunDetailsList) GetSortedRunsByStates added in v0.17.0

func (j JobRunDetailsList) GetSortedRunsByStates(states []State) []*JobRunWithDetails

type JobRunID

type JobRunID uuid.UUID

func JobRunIDFromString

func JobRunIDFromString(runID string) (JobRunID, error)

func (JobRunID) IsEmpty

func (i JobRunID) IsEmpty() bool

func (JobRunID) UUID

func (i JobRunID) UUID() uuid.UUID

type JobRunIdentifier added in v0.22.4

type JobRunIdentifier struct {
	JobName     JobName
	ScheduledAt time.Time
}

type JobRunLineage added in v0.22.4

type JobRunLineage struct {
	JobName          JobName
	ScheduledAt      time.Time
	JobRuns          []*JobExecutionSummary
	ExecutionSummary *LineageExecutionSummary
}

func (*JobRunLineage) GetAllJobNames added in v0.24.0

func (j *JobRunLineage) GetAllJobNames() []JobName

func (*JobRunLineage) GroupJobsInLineageByHookNames added in v0.24.0

func (j *JobRunLineage) GroupJobsInLineageByHookNames() map[string][]JobName

type JobRunMeta added in v0.11.4

type JobRunMeta struct {
	Labels         map[string]string
	DestinationURN resource.URN
}

type JobRunStatus

type JobRunStatus struct {
	ScheduledAt time.Time
	State       State
}

func JobRunStatusFrom

func JobRunStatusFrom(scheduledAt time.Time, state string) (JobRunStatus, error)

func (JobRunStatus) GetLogicalTime added in v0.7.0

func (j JobRunStatus) GetLogicalTime(jobCron *cron.ScheduleSpec) time.Time

func (JobRunStatus) GetScheduledAt added in v0.17.0

func (j JobRunStatus) GetScheduledAt() time.Time

func (JobRunStatus) GetState added in v0.17.0

func (j JobRunStatus) GetState() State

type JobRunStatusList added in v0.7.0

type JobRunStatusList []*JobRunStatus

func (JobRunStatusList) GetJobRunStatusSummary added in v0.11.3

func (j JobRunStatusList) GetJobRunStatusSummary() string

func (JobRunStatusList) GetLatestPendingRun added in v0.22.12

func (j JobRunStatusList) GetLatestPendingRun() *JobRunStatus

func (JobRunStatusList) GetOnlyDifferedRuns added in v0.18.1

func (j JobRunStatusList) GetOnlyDifferedRuns(runsComparator []*JobRunStatus) JobRunStatusList

func (JobRunStatusList) GetSortedRunsByScheduledAt added in v0.8.0

func (j JobRunStatusList) GetSortedRunsByScheduledAt() []*JobRunStatus

func (JobRunStatusList) GetSortedRunsByStates added in v0.7.0

func (j JobRunStatusList) GetSortedRunsByStates(states []State) []*JobRunStatus

func (JobRunStatusList) GetSuccessRuns added in v0.19.0

func (j JobRunStatusList) GetSuccessRuns() int

func (JobRunStatusList) IsAllTerminated added in v0.11.0

func (j JobRunStatusList) IsAllTerminated() bool

func (JobRunStatusList) IsAnyFailure added in v0.11.0

func (j JobRunStatusList) IsAnyFailure() bool

func (JobRunStatusList) MergeWithUpdatedRuns added in v0.7.0

func (j JobRunStatusList) MergeWithUpdatedRuns(updatedRunMap map[time.Time]State) []*JobRunStatus

func (JobRunStatusList) OverrideWithStatus added in v0.9.0

func (j JobRunStatusList) OverrideWithStatus(status State) []*JobRunStatus

func (JobRunStatusList) String added in v0.22.8

func (j JobRunStatusList) String() string

func (JobRunStatusList) ToRunStatusMap added in v0.7.0

func (j JobRunStatusList) ToRunStatusMap() map[time.Time]State

type JobRunSummary added in v0.22.4

type JobRunSummary struct {
	JobName     JobName
	ScheduledAt time.Time
	SLATime     *time.Time

	JobStartTime  *time.Time
	JobEndTime    *time.Time
	JobStatus     string
	WaitStartTime *time.Time
	WaitEndTime   *time.Time
	TaskStartTime *time.Time
	TaskEndTime   *time.Time
	HookStartTime *time.Time
	HookEndTime   *time.Time

	SensorName *string
	TaskName   *string
	HookName   *string
}

func (*JobRunSummary) GetActualEndTime added in v0.23.19

func (j *JobRunSummary) GetActualEndTime() *time.Time

func (*JobRunSummary) GetHookDuration added in v0.23.19

func (j *JobRunSummary) GetHookDuration() time.Duration

func (*JobRunSummary) GetTaskDuration added in v0.23.19

func (j *JobRunSummary) GetTaskDuration() time.Duration

type JobRunWithDetails added in v0.17.0

type JobRunWithDetails struct {
	ScheduledAt     time.Time
	State           State
	RunType         string
	ExternalTrigger bool
	DagRunID        string
	DagID           string
}

func (JobRunWithDetails) GetState added in v0.17.0

func (j JobRunWithDetails) GetState() State

type JobRunsCriteria

type JobRunsCriteria struct {
	Name        string
	StartDate   time.Time
	EndDate     time.Time
	Filter      []string
	OnlyLastRun bool
}

JobRunsCriteria represents the filter condition to get run status from scheduler

func (*JobRunsCriteria) ExecutionEndDate

func (c *JobRunsCriteria) ExecutionEndDate(jobCron *cron.ScheduleSpec) time.Time

func (*JobRunsCriteria) ExecutionStart

func (c *JobRunsCriteria) ExecutionStart(cron *cron.ScheduleSpec) time.Time

type JobSLAState added in v0.26.0

type JobSLAState struct {
	EstimatedDuration *time.Duration
	InferredSLA       *time.Time
}

type JobSchedule added in v0.22.4

type JobSchedule struct {
	JobName     JobName
	ScheduledAt time.Time
}

type JobState added in v0.26.0

type JobState struct {
	JobSLAState
	JobName       JobName
	JobRun        JobRunSummary
	Tenant        tenant.Tenant
	RelativeLevel int
	Status        SLABreachCause
}

type JobSummary added in v0.22.4

type JobSummary struct {
	JobName          JobName
	IsEnabled        bool
	Tenant           tenant.Tenant
	ScheduleInterval string
	SLA              SLAConfig
	Window           window.Config
}

type JobUpstream

type JobUpstream struct {
	JobName        string
	Host           string
	TaskName       string        // TODO: remove after airflow migration
	DestinationURN resource.URN  //- bigquery://pilot.playground.table
	Tenant         tenant.Tenant // Current or external tenant
	Type           string
	External       bool
	State          string
}

type JobWithDetails

type JobWithDetails struct {
	Name JobName

	Job           *Job
	JobMetadata   *JobMetadata
	Schedule      *Schedule
	Retry         Retry
	Alerts        []Alert
	Webhook       []Webhook
	RuntimeConfig RuntimeConfig
	Priority      int
	Upstreams     Upstreams
}

JobWithDetails contains the details for a job

func (*JobWithDetails) GetLabelsAsString

func (j *JobWithDetails) GetLabelsAsString() string

func (*JobWithDetails) GetName

func (j *JobWithDetails) GetName() string

func (*JobWithDetails) GetSafeLabels added in v0.10.2

func (j *JobWithDetails) GetSafeLabels() map[string]string

func (*JobWithDetails) GetUniqueLabelValues added in v0.8.0

func (j *JobWithDetails) GetUniqueLabelValues() []string

func (*JobWithDetails) SLADuration

func (j *JobWithDetails) SLADuration() (int64, error)

type JobWithTaskDuration added in v0.23.19

type JobWithTaskDuration struct {
	JobName      JobName
	ScheduledAt  time.Time
	TaskDuration time.Duration
	Level        int
}

type Kubernetes added in v0.23.15

type Kubernetes struct {
	ServiceAccount string
}

type LineageDelaySummary added in v0.23.19

type LineageDelaySummary struct {
	JobName             JobName
	ScheduledAt         time.Time
	UpstreamJobName     JobName
	UpstreamScheduledAt time.Time
	DelayDuration       int64
}

type LineageExecutionSummary added in v0.23.19

type LineageExecutionSummary struct {
	TotalScheduledWayTooLateSeconds     int64
	TotalSystemSchedulingDelaySeconds   int64
	AverageSystemSchedulingDelaySeconds int64

	TotalLineageDelaySeconds    int64
	TotalLineageDurationSeconds int64

	LargestScheduledWayTooLateJob   LineageDelaySummary
	LargestSystemSchedulingDelayJob LineageDelaySummary

	TopLongestTaskDurationJobs []JobWithTaskDuration
	TopLongestHookDurationJobs []JobWithTaskDuration
}

type NotifyAttrs

type NotifyAttrs struct {
	Owner    string
	JobEvent *Event
	Route    string
	Secret   string
}

type OperatorAlertConfig added in v0.22.0

type OperatorAlertConfig struct {
	SLAAlertConfigs []*SLAAlertConfig `json:"sla_alert_configs,omitempty"`
	Team            string            `json:"team,omitempty"`
}

func (OperatorAlertConfig) GetSLAOperatorAlertConfigByTag added in v0.22.0

func (o OperatorAlertConfig) GetSLAOperatorAlertConfigByTag(alertTag string) *SLAAlertConfig

type OperatorObj added in v0.22.0

type OperatorObj struct {
	DownstreamTaskIDs []string `json:"downstream_task_ids"`
}

type OperatorRun

type OperatorRun struct {
	ID           uuid.UUID
	Name         string
	JobRunID     uuid.UUID
	OperatorType OperatorType
	Status       State
	StartTime    time.Time
	EndTime      *time.Time
}

type OperatorRunInstance added in v0.22.0

type OperatorRunInstance struct {
	MaxTries     int        `json:"max_tries"`
	OperatorName string     `json:"task_id"`
	StartTime    time.Time  `json:"start_date"`
	OperatorKey  string     `json:"task_instance_key_str"`
	TryNumber    int        `json:"attempt"`
	State        string     `json:"status"`
	EndTime      *time.Time `json:"end_date,omitempty"`
	LogURL       string     `json:"log_url"`
}

func (*OperatorRunInstance) IsTerminated added in v0.22.0

func (o *OperatorRunInstance) IsTerminated() bool

type OperatorSLAAlertAttrs added in v0.22.0

type OperatorSLAAlertAttrs struct {
	Team               string
	Project            string
	Namespace          string
	JobName            string
	OperatorName       string
	OperatorType       string
	Message            string
	Severity           string
	ScheduledAt        time.Time
	StartTime          time.Time
	ExpectedSLAEndTime time.Time
	CurrentState       State

	AlertManager AlertManagerConfig
}

type OperatorType

type OperatorType string

func NewOperatorType added in v0.22.0

func NewOperatorType(op string) (OperatorType, error)

func (OperatorType) String

func (o OperatorType) String() string

type OperatorsSLA added in v0.22.0

type OperatorsSLA struct {
	ID           uuid.UUID
	JobName      JobName
	ProjectName  tenant.ProjectName
	OperatorName string
	RunID        string
	OperatorType OperatorType
	SLATime      time.Time

	AlertTag string

	ScheduledAt       time.Time
	OperatorStartTime time.Time

	WorkerSignature string
	WorkerLockUntil time.Time
}

func (OperatorsSLA) String added in v0.22.2

func (o OperatorsSLA) String() string

type PotentialSLABreachAttrs added in v0.22.4

type PotentialSLABreachAttrs struct {
	TeamName string
	Projects []SLABreachProject
}

PotentialSLABreachAttrs is a single consolidated alert for one team. It is routed to the team owning the root-cause (upstream) jobs, and its body is organized as project -> SLA group (with severity) -> target -> causes.

type Replay added in v0.7.0

type Replay struct {
	// contains filtered or unexported fields
}

func NewReplay added in v0.7.0

func NewReplay(id uuid.UUID, jobName JobName, tenant tenant.Tenant, config *ReplayConfig, state ReplayState, createdAt, updatedAt time.Time, message string) *Replay

func NewReplayRequest added in v0.7.0

func NewReplayRequest(jobName JobName, tenant tenant.Tenant, config *ReplayConfig, state ReplayState) *Replay

func (*Replay) Config added in v0.7.0

func (r *Replay) Config() *ReplayConfig

func (*Replay) CreatedAt added in v0.7.0

func (r *Replay) CreatedAt() time.Time

func (*Replay) GetEndTime added in v0.25.0

func (r *Replay) GetEndTime() time.Time

func (*Replay) GetJobConfig added in v0.25.0

func (r *Replay) GetJobConfig() map[string]string

func (*Replay) GetStartTime added in v0.25.0

func (r *Replay) GetStartTime() time.Time

func (*Replay) ID added in v0.7.0

func (r *Replay) ID() uuid.UUID

func (*Replay) IsTerminated added in v0.11.3

func (r *Replay) IsTerminated() bool

func (*Replay) JobName added in v0.7.0

func (r *Replay) JobName() JobName

func (*Replay) Message added in v0.7.0

func (r *Replay) Message() string

func (*Replay) State added in v0.7.0

func (r *Replay) State() ReplayState

func (*Replay) Tenant added in v0.7.0

func (r *Replay) Tenant() tenant.Tenant

func (*Replay) UpdatedAt added in v0.17.0

func (r *Replay) UpdatedAt() time.Time

type ReplayConfig added in v0.7.0

type ReplayConfig struct {
	StartTime   time.Time
	EndTime     time.Time
	Parallel    bool
	JobConfig   map[string]string
	Description string
	Category    string
	ApprovalID  string
	UserID      string
}

func NewReplayConfig added in v0.7.0

func NewReplayConfig(startTime, endTime time.Time, parallel bool, jobConfig map[string]string, description, category, approvalID, userID string) *ReplayConfig

type ReplayNotificationAttrs added in v0.16.2

type ReplayNotificationAttrs struct {
	JobName  string
	ReplayID string
	Tenant   tenant.Tenant
	JobURN   string
	State    ReplayState

	JobWithDetails *JobWithDetails

	AlertManager AlertManagerConfig
}

type ReplayState added in v0.7.0

type ReplayState string // contract status for business layer
const (
	// ReplayStateCreated is an initial state which indicates the replay has been created but not picked up yet
	ReplayStateCreated ReplayState = "created"

	// ReplayStateInProgress indicates the replay is being executed
	ReplayStateInProgress    ReplayState = "in progress"
	ReplayRunStateInProgress ReplayState = "in_progress"

	// ReplayStateSuccess is a terminal state which occurs when the replay execution finished with successful job runs
	ReplayStateSuccess ReplayState = "success"

	ReplayStateTimeout ReplayState = "timeout"

	// ReplayStateFailed is a terminal state which occurs when the replay execution failed, timed out, or finished with one of the run fails
	ReplayStateFailed ReplayState = "failed"

	// ReplayStateCancelled is a terminal state which occurs when the replay is cancelled by user
	ReplayStateCancelled ReplayState = "cancelled"

	EntityReplay = "replay"
)

func ReplayStateFromString added in v0.7.0

func ReplayStateFromString(state string) (ReplayState, error)

func (ReplayState) String added in v0.7.0

func (j ReplayState) String() string

type ReplayUserState added in v0.8.1

type ReplayUserState string // contract status for presentation layer

func (ReplayUserState) String added in v0.8.1

func (j ReplayUserState) String() string

type ReplayWithRun added in v0.7.0

type ReplayWithRun struct {
	Replay *Replay
	Runs   []*JobRunStatus // TODO: JobRunStatus does not have `message/log`
}

func (*ReplayWithRun) GetFirstExecutableRun added in v0.7.0

func (r *ReplayWithRun) GetFirstExecutableRun() *JobRunStatus

func (*ReplayWithRun) GetLastExecutableRun added in v0.7.0

func (r *ReplayWithRun) GetLastExecutableRun() *JobRunStatus

type Resource

type Resource struct {
	Request *ResourceConfig
	Limit   *ResourceConfig
}

type ResourceConfig

type ResourceConfig struct {
	CPU    string
	Memory string
}

type Retry

type Retry struct {
	ExponentialBackoff bool
	Count              int
	Delay              int32
}

type RunConfig

type RunConfig struct {
	Executor Executor

	ScheduledAt time.Time
	JobRunID    JobRunID
	DagRunID    string
	RunMode     RunMode
}

func RunConfigFrom

func RunConfigFrom(executor Executor, scheduledAt time.Time, runID, dagRunID string) (RunConfig, error)

func (*RunConfig) IsDryRun added in v0.26.1

func (r *RunConfig) IsDryRun() bool

func (*RunConfig) WithRunMode added in v0.26.1

func (r *RunConfig) WithRunMode(m RunMode)

type RunMode added in v0.26.1

type RunMode string
const (
	DryRun    RunMode = "DryRun"
	Execution RunMode = "Execution"
)

type RuntimeConfig

type RuntimeConfig struct {
	Resource   *Resource
	Scheduler  map[string]string
	Kubernetes *Kubernetes
}

type SLAAlertConfig added in v0.22.0

type SLAAlertConfig struct {
	DurationThreshold time.Duration `json:"duration_threshold,omitempty"`
	Severity          Severity      `json:"severity,omitempty"`
	Team              string        `json:"team,omitempty"`
	AutoThreshold     bool          `json:"auto_threshold,omitempty"`
}

func (SLAAlertConfig) Tag added in v0.22.0

func (s SLAAlertConfig) Tag() string

type SLABreachCause added in v0.26.0

type SLABreachCause string
const (
	SLABreachCauseNotStarted  SLABreachCause = "NOT_STARTED"
	SLABreachCauseRunningLate SLABreachCause = "RUNNING_LATE"
)

type SLABreachCombo added in v0.26.0

type SLABreachCombo struct {
	ProjectName tenant.ProjectName
	JobNames    []JobName
	Labels      map[string]string
	GroupName   string // display name for the SLA group; derived from labels if empty
}

SLABreachCombo is a single (project, label-group) unit of work. The batch entrypoint evaluates the cross product of projects x label-groups as combos and consolidates the results into one alert per team.

type SLABreachGroup added in v0.26.0

type SLABreachGroup struct {
	Name     string
	Severity string
	Targets  []SLABreachTarget
}

SLABreachGroup groups targets that share the same SLA target (label group), carrying its own severity so different SLA buckets can be distinguished within a single team's message.

type SLABreachProject added in v0.26.0

type SLABreachProject struct {
	Name   string
	Groups []SLABreachGroup
}

SLABreachProject groups the breaching targets by the target job's project.

type SLABreachTarget added in v0.26.0

type SLABreachTarget struct {
	JobName string
	Causes  []UpstreamAttrs
}

SLABreachTarget is a single SLA-bearing (target) job that may breach, along with the upstream cause jobs owned by the alerted team.

type SLAConfig added in v0.22.4

type SLAConfig struct {
	Duration time.Duration
}

type SLAObject

type SLAObject struct {
	JobName        JobName
	JobScheduledAt time.Time
}

func (*SLAObject) String

func (s *SLAObject) String() string

type Schedule

type Schedule struct {
	DependsOnPast bool
	CatchUp       bool
	StartDate     time.Time
	EndDate       *time.Time
	Interval      string
}

func (*Schedule) GetLogicalStartTime added in v0.19.0

func (s *Schedule) GetLogicalStartTime() (time.Time, error)

func (*Schedule) GetNextSchedule added in v0.22.4

func (s *Schedule) GetNextSchedule(after time.Time) (time.Time, error)

func (*Schedule) GetPreviousSchedule added in v0.22.10

func (s *Schedule) GetPreviousSchedule(before time.Time) (time.Time, error)

func (*Schedule) GetScheduleStartTime added in v0.19.0

func (s *Schedule) GetScheduleStartTime() (time.Time, error)

type Severity added in v0.22.0

type Severity string
const (
	Critical Severity = "CRITICAL"
	Warning  Severity = "WARNING"
	Info     Severity = "INFO"
)

func SeverityFromString added in v0.22.0

func SeverityFromString(severity string) (Severity, error)

func (Severity) String added in v0.22.0

func (s Severity) String() string

type State

type State string
const (
	StatePending State = "pending"

	StateAccepted State = "accepted"
	StateRunning  State = "running"
	StateQueued   State = "queued"
	StateCanceled State = "canceled"

	StateRetry State = "retried"

	StateSuccess State = "success"
	StateFailed  State = "failed"

	StateNotScheduled State = "waiting_to_schedule"
	StateWaitUpstream State = "wait_upstream"
	StateInProgress   State = "in_progress"
	StateUpForRetry   State = "up_for_retry"
	StateRestarting   State = "restarting"
	StateMissing      State = "missing"
)

func StateFromString

func StateFromString(state string) (State, error)

func (State) String

func (j State) String() string

type TargetBreach added in v0.26.0

type TargetBreach struct {
	TargetProject string
	TargetJobName JobName
	Upstreams     map[JobName]*JobState
}

TargetBreach is the per-target result returned to the caller for building the API response. It is project-qualified to avoid clashes across projects.

type Task

type Task struct {
	Name        string               `json:"name,omitempty"`
	Version     string               `json:"version,omitempty"`
	Config      map[string]string    `json:"config,omitempty"`
	AlertConfig *OperatorAlertConfig `json:"alert_config,omitempty"`
}

type TeamBreachAggregator added in v0.26.0

type TeamBreachAggregator struct {
	// contains filtered or unexported fields
}

TeamBreachAggregator accumulates breaches into a deterministic, insertion- ordered team -> project -> group -> target -> causes structure and builds one PotentialSLABreachAttrs per team.

func NewTeamBreachAggregator added in v0.26.0

func NewTeamBreachAggregator() *TeamBreachAggregator

func (*TeamBreachAggregator) Add added in v0.26.0

func (a *TeamBreachAggregator) Add(team, project, group, severity, target string, cause UpstreamAttrs)

func (*TeamBreachAggregator) Build added in v0.26.0

type ThirdPartyUpstream added in v0.22.14

type ThirdPartyUpstream struct {
	Type       string
	Identifier string
	Config     map[string]string
}

type UpstreamAttrs added in v0.22.4

type UpstreamAttrs struct {
	JobName       string
	RelativeLevel int
	Status        string
}

type Upstreams

type Upstreams struct {
	HTTP         []*HTTPUpstreams
	UpstreamJobs []*JobUpstream
	ThirdParty   []*ThirdPartyUpstream
}

type Webhook added in v0.11.4

type Webhook struct {
	On        EventCategory
	Endpoints []WebhookEndPoint
}

type WebhookAttrs added in v0.11.4

type WebhookAttrs struct {
	Owner    string
	JobEvent *Event
	Meta     *JobRunMeta
	Route    string
	Headers  map[string]string
}

type WebhookEndPoint added in v0.11.4

type WebhookEndPoint struct {
	URL     string
	Headers map[string]string
}

Directories

Path Synopsis
handler

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL