dagrun

package
v2.18.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 4, 2026 License: GPL-3.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source
const DefaultStaleLeaseThreshold = 90 * time.Second

DefaultStaleLeaseThreshold is the default interval after which an unobserved distributed run is considered stale.

View Source
const MaxPushBackInputsSize = 16 << 10

MaxPushBackInputsSize is the largest JSON encoding of the inputs one push-back may record. Rewound steps receive the inputs as environment variables, and a single variable is limited to 32,767 characters on Windows and 128 KiB on Linux.

Variables

View Source
var (
	ErrDAGRunIDNotFound    = errors.New("dag-run ID not found")
	ErrDAGRunIDEmpty       = errors.New("dag-run ID is empty")
	ErrDAGRunAlreadyExists = errors.New("dag-run already exists")
	ErrDAGRunActive        = errors.New("dag-run is active")
	ErrNoStatusData        = errors.New("no status data")
	ErrCorruptedStatusData = errors.New("corrupted status data")
)

Errors related to dag-run management

View Source
var (
	ErrRetryStepNotFound = errors.New("retry step not found")
	ErrInvalidRetryPath  = errors.New("retry path is invalid")
	// ErrRepeatingStepTarget indicates the target child DAG run was invoked by a
	// repeating step. Such child runs carry non-reproducible IDs, so only the
	// repeating step itself can be retried.
	ErrRepeatingStepTarget = errors.New("child DAG runs of a repeating step cannot be retried individually")
)
View Source
var (
	ErrStateNotFound      = errors.New("dag state: not found")
	ErrStateConflict      = errors.New("dag state: conflict")
	ErrInvalidStateRef    = errors.New("dag state: invalid ref")
	ErrInvalidStateValue  = errors.New("dag state: invalid value")
	ErrStateValueTooLarge = errors.New("dag state: value too large")
)
View Source
var ErrFailedAutoRetryCancelStateChanged = errors.New(
	"dag-run state changed before failed auto-retry cancel could be applied",
)
View Source
var ErrNoopAttemptNotSupported = errors.New("operation not supported by no-op DAG run attempt")

ErrNoopAttemptNotSupported is returned when an operation is not supported by a no-op attempt.

Functions

func ApplyPushBack added in v2.17.1

func ApplyPushBack(status *ir.DAGRunStatus, source *ir.Node, pb PushBack) (int, error)

ApplyPushBack resets the target step and every step that depends on it, directly or transitively, for another execution. Each reset step records the push-back iteration, the feedback, the source step's push-back history, and its own previous stdout. The new iteration exceeds the iteration of the source and of every reset step, and is returned.

func CanCancelFailedAutoRetryPendingRun

func CanCancelFailedAutoRetryPendingRun(status *ir.DAGRunStatus) bool

CanCancelFailedAutoRetryPendingRun returns true when a failed DAG-run is a root run and still has remaining DAG-level auto-retry budget.

func ClonePushBackHistory

func ClonePushBackHistory(src []ir.PushBackEntry) []ir.PushBackEntry

ClonePushBackHistory returns a deep copy of push-back history entries.

func FilterPushBackInputs

func FilterPushBackInputs(allowed []string, inputs map[string]string) map[string]string

FilterPushBackInputs returns only declared push-back inputs. If no allowlist is provided, the stored inputs are preserved as-is.

func HashStateValue

func HashStateValue(value json.RawMessage) string

HashStateValue returns the SHA-256 hash of a normalized state value.

func IsLeaseActive

func IsLeaseActive(status *ir.DAGRunStatus, staleThreshold time.Duration) bool

IsLeaseActive reports whether the coordinator has observed the run within the stale threshold. A run without a lease timestamp is stale.

func NormalizePushBackHistory

func NormalizePushBackHistory(
	allowed []string,
	iteration int,
	latestInputs map[string]string,
	history []ir.PushBackEntry,
) []ir.PushBackEntry

NormalizePushBackHistory returns the history a step with the given input allowlist observes, seeded from legacy state when only the latest iteration/input pair is available. Human-task feedback is never filtered.

func NormalizeStateValue

func NormalizeStateValue(data []byte) (json.RawMessage, error)

NormalizeStateValue validates and compacts a JSON value before storage.

func RevertPushBack added in v2.17.1

func RevertPushBack(latest, original, applied *ir.DAGRunStatus) error

RevertPushBack restores in latest every step that differs between original, the status before a push-back, and applied, the status the push-back produced. It fails without changing latest when any of those steps changed after the push-back.

func ValidatePushBackInputsSize added in v2.17.1

func ValidatePushBackInputsSize(inputs map[string]string) error

ValidatePushBackInputsSize reports an error when inputs exceed MaxPushBackInputsSize.

func VisiblePushBackInputs added in v2.17.1

func VisiblePushBackInputs(allowed []string, latest map[string]string, history []ir.PushBackEntry) map[string]string

VisiblePushBackInputs returns the latest push-back inputs a step with the given input allowlist receives. Feedback from a human task is never filtered because it holds only declared feedback properties.

Types

type Attempt added in v2.14.0

type Attempt interface {
	// ID returns the identifier for the attempt that is unique within the dag-run.
	ID() string
	// Open prepares the attempt for writing status updates
	Open(ctx context.Context) error
	// Write updates the status of the attempt
	Write(ctx context.Context, status ir.DAGRunStatus) error
	// Close finalizes writing to the attempt
	Close(ctx context.Context) error
	// ReadStatus retrieves the current status of the attempt
	ReadStatus(ctx context.Context) (*ir.DAGRunStatus, error)
	// ReadStatusUncached retrieves the current status without retaining it in a shared cache.
	ReadStatusUncached(ctx context.Context) (*ir.DAGRunStatus, error)
	// ReadDAG reads the DAG associated with this run attempt
	ReadDAG(ctx context.Context) (*ir.DAG, error)
	// SetDAG sets the DAG for this attempt (must be called before Open for DAG to be persisted)
	SetDAG(dag *ir.DAG)
	// Abort requests aborting the attempt
	Abort(ctx context.Context) error
	// IsAborting checks if an abort has been requested for the attempt
	IsAborting(ctx context.Context) (bool, error)
	// Hide marks the attempt as hidden from normal operations.
	// This is useful for preserving previous state visibility when dequeuing.
	Hide(ctx context.Context) error
	// Hidden returns true if the attempt is hidden from normal operations.
	Hidden() bool
	// WriteOutputs writes the collected step outputs for the dag-run.
	// Does nothing if outputs is nil or has no output entries.
	WriteOutputs(ctx context.Context, outputs *ir.DAGRunOutputs) error
	// ReadOutputs reads the collected step outputs for the dag-run.
	// Returns nil when outputs are unavailable or use the legacy representation.
	ReadOutputs(ctx context.Context) (*ir.DAGRunOutputs, error)
	// WriteStepMessages writes LLM messages for a single step.
	WriteStepMessages(ctx context.Context, stepName string, messages []ir.LLMMessage) error
	// ReadStepMessages reads LLM messages for a single step.
	// Returns nil if no messages exist for the step.
	ReadStepMessages(ctx context.Context, stepName string) ([]ir.LLMMessage, error)
}

Attempt represents a single execution of a dag-run to record the status and execution details.

func NewNoopAttempt added in v2.14.0

func NewNoopAttempt(id string, dag *ir.DAG) Attempt

NewNoopAttempt creates a no-op attempt for remote worker execution.

type FailedAutoRetryCancelEligibility

type FailedAutoRetryCancelEligibility int

FailedAutoRetryCancelEligibility describes whether a failed DAG-run can be canceled before the scheduler issues the next DAG-level auto-retry.

const (
	FailedAutoRetryCancelEligible FailedAutoRetryCancelEligibility = iota
	FailedAutoRetryCancelMissingStatus
	FailedAutoRetryCancelNotRoot
	FailedAutoRetryCancelNotPending
)

func FailedAutoRetryCancelEligibilityOf

func FailedAutoRetryCancelEligibilityOf(status *ir.DAGRunStatus) FailedAutoRetryCancelEligibility

FailedAutoRetryCancelEligibilityOf classifies whether the provided status can be canceled while it is failed and still waiting for a DAG-level auto-retry.

type FailedAutoRetryCancelStateChangedError

type FailedAutoRetryCancelStateChangedError struct {
	CurrentStatus *ir.DAGRunStatus
}

FailedAutoRetryCancelStateChangedError reports the latest observed status when another actor changed the latest attempt before the cancel CAS completed.

func (*FailedAutoRetryCancelStateChangedError) Error

func (*FailedAutoRetryCancelStateChangedError) Unwrap

type PushBack added in v2.17.1

type PushBack struct {
	// TargetName is the step that runs again first.
	TargetName string
	// AllowedInputs restricts which inputs are recorded. Empty keeps them all.
	AllowedInputs []string
	// Inputs holds the reviewer feedback.
	Inputs map[string]string
	// By, ByID, and At identify who pushed the step back and when.
	By   string
	ByID string
	At   string
}

PushBack describes one push-back of a waiting manual step.

type RetryHop

type RetryHop struct {
	Step  string `json:"step"`
	RunID string `json:"runId"`
}

RetryHop identifies one parent-to-child invocation.

type RetryPath

type RetryPath struct {
	Hops []RetryHop `json:"hops"`
	Step string     `json:"step"`
}

RetryPath identifies a step in a persisted child DAG run.

func ParseRetryPath

func ParseRetryPath(value string) (RetryPath, error)

ParseRetryPath parses an internal retry path.

func (RetryPath) Advance

func (p RetryPath) Advance() RetryPath

Advance returns the path to pass into the selected child run.

func (RetryPath) Current

func (p RetryPath) Current() (RetryHop, bool)

Current returns the child invocation owned by the current DAG level.

func (RetryPath) Encode

func (p RetryPath) Encode() string

Encode serializes the path for internal transport.

func (RetryPath) NextStep

func (p RetryPath) NextStep() string

NextStep returns the step that the selected child run must retry.

func (RetryPath) RootStep

func (p RetryPath) RootStep() string

RootStep returns the root DAG step that contains the target child run.

type StateEntry

type StateEntry struct {
	StateRef
	Value     json.RawMessage    `json:"value"`
	Version   int64              `json:"version"`
	Hash      string             `json:"hash"`
	CreatedAt time.Time          `json:"created_at"`
	UpdatedAt time.Time          `json:"updated_at"`
	UpdatedBy *StateUpdateSource `json:"updated_by,omitempty"`
}

StateEntry is a versioned JSON value stored for a state reference.

func (*StateEntry) Clone

func (e *StateEntry) Clone() *StateEntry

Clone returns a deep copy of the entry.

type StateListOptions

type StateListOptions struct {
	Scope     StateScope
	Namespace string
	KeyPrefix string
	Limit     int
}

StateListOptions filters state entries by scope, namespace, and key prefix.

func (StateListOptions) Validate

func (o StateListOptions) Validate() error

Validate rejects malformed list filters before store access.

type StatePutOptions

type StatePutOptions struct {
	ExpectedVersion *int64
	CreateOnly      bool
	UpdatedBy       *StateUpdateSource
}

StatePutOptions controls optimistic concurrency and audit metadata for writes.

type StateRef

type StateRef struct {
	Scope     StateScope `json:"scope"`
	Namespace string     `json:"namespace"`
	Key       string     `json:"key"`
}

StateRef identifies one persistent DAG state entry.

func (StateRef) Validate

func (r StateRef) Validate() error

Validate rejects malformed state references.

type StateScope

type StateScope string

StateScope identifies the namespace strategy for a state entry.

const (
	// StateScopeDAG stores state under the current DAG name.
	StateScopeDAG StateScope = "dag"
	// StateScopeRootDAG stores state under the root DAG name for nested runs.
	StateScopeRootDAG StateScope = "root_dag"
	// StateScopeGlobal stores state in a process-wide namespace.
	StateScopeGlobal StateScope = "global"
	// StateScopeCustom stores state in an explicitly provided namespace.
	StateScopeCustom StateScope = "custom"

	// DefaultGlobalStateNamespace is used when global state does not need a user namespace.
	DefaultGlobalStateNamespace = "_"
	// MaxStateValueBytes is the maximum normalized JSON payload size for one state entry.
	MaxStateValueBytes = 1 << 20
)

func (StateScope) Valid

func (s StateScope) Valid() bool

Valid reports whether the scope is supported.

type StateStore

type StateStore interface {
	Get(ctx context.Context, ref StateRef) (*StateEntry, error)
	Put(ctx context.Context, ref StateRef, value json.RawMessage, opts StatePutOptions) (*StateEntry, error)
	Delete(ctx context.Context, ref StateRef) (bool, error)
	List(ctx context.Context, opts StateListOptions) ([]*StateEntry, error)
}

StateStore persists JSON state entries across DAG runs.

type StateUpdateSource

type StateUpdateSource struct {
	DAGName   string `json:"dag_name,omitempty"`
	DAGRunID  string `json:"dag_run_id,omitempty"`
	AttemptID string `json:"attempt_id,omitempty"`
	StepName  string `json:"step_name,omitempty"`
}

StateUpdateSource records the DAG run and step that last updated an entry.

func (*StateUpdateSource) Clone

Clone returns a copy of the update source.

type WorkDirRef added in v2.14.0

type WorkDirRef struct {
	RootDAGRun ir.DAGRunRef
	DAGRun     ir.DAGRunRef
}

WorkDirRef identifies the execution work directory belonging to a DAG run.

type WorkDirStore added in v2.14.0

type WorkDirStore interface {
	Materialize(ctx context.Context, ref WorkDirRef) (string, error)
	Snapshot(ctx context.Context, ref WorkDirRef, localDir string) error
	Remove(ctx context.Context, ref WorkDirRef) error
}

WorkDirStore manages durable execution work directories for DAG runs.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL