Documentation
¶
Index ¶
- Constants
- Variables
- func AbortQueuedDAGRun(ctx context.Context, dagRunStore DAGRunStore, dagRun DAGRunRef) error
- func AttemptKeyForStatus(status *DAGRunStatus, fallbackAttemptID string) string
- func CanCancelFailedAutoRetryPendingRun(status *DAGRunStatus) bool
- func CancelFailedAutoRetryPendingRun(ctx context.Context, dagRunStore latestAttemptStatusSwapper, ...) error
- func DecodeSearchCursor(raw string, dest any) error
- func DistributedLeaseExpiredReason(workerID string) string
- func EncodeSearchCursor(payload any) string
- func EnqueueRetry(ctx context.Context, dagRunStore DAGRunStore, queueStore QueueStore, ...) (bool, error)
- func FilterPushBackInputs(allowed []string, inputs map[string]string) map[string]string
- func FormatTime(val time.Time) string
- func GenerateAttemptKey(rootName, rootID, dagName, dagRunID, attemptID string) string
- func IsLeaseActive(status *DAGRunStatus, staleThreshold time.Duration) bool
- func IsQueuedCatchup(status *DAGRunStatus) bool
- func IsRemoteWorkerID(workerID string) bool
- func LeaseIdentityMatchesStatus(lease *DAGRunLease, status *DAGRunStatus, fallbackAttemptID string) bool
- func LeaseMatchesStatus(lease *DAGRunLease, status *DAGRunStatus, fallbackAttemptID string, ...) bool
- func NewContext(ctx context.Context, dag *core.DAG, dagRunID string, logFile string, ...) context.Context
- func NewDAGRunID() (string, error)
- func NormalizeDAGRunConditions(status *DAGRunStatus)
- func PreservedQueueTriggerType(status *DAGRunStatus) core.TriggerType
- func ResolveRetryPath(ctx context.Context, store DAGRunStore, root DAGRunRef, targetRunID string, ...) (RetryPath, *DAGRunStatus, error)
- func ValidateDAGRunID(dagRunID string) error
- func WithContext(ctx context.Context, rCtx Context) context.Context
- func WorkspaceNameFromLabels(labels core.Labels) (string, bool)
- type ActiveDistributedRun
- type ActiveDistributedRunStore
- type ClaimedDispatchTask
- type CompareAndSwapStatusOption
- type CompareAndSwapStatusOptions
- type Context
- type ContextOption
- func WithArtifactDir(dir string) ContextOption
- func WithAttemptID(attemptID string) ContextOption
- func WithCoordinator(cli Dispatcher) ContextOption
- func WithDAGRunArtifactDir(dir string) ContextOption
- func WithDAGRunLogDir(dir string) ContextOption
- func WithDAGRunStore(store DAGRunStore) ContextOption
- func WithDatabase(db Database) ContextOption
- func WithDefaultEnvVars(envs ...string) ContextOption
- func WithDefaultExecMode(mode config.ExecutionMode) ContextOption
- func WithDefaultSecrets(secrets []string) ContextOption
- func WithEnvVars(envs ...string) ContextOption
- func WithLogEncoding(charset string) ContextOption
- func WithLogWriterFactory(factory LogWriterFactory) ContextOption
- func WithParams(params []string) ContextOption
- func WithQueueStore(store QueueStore) ContextOption
- func WithRetryPath(path RetryPath) ContextOption
- func WithRootDAGRun(ref DAGRunRef) ContextOption
- func WithRunStartedAt(startedAt string) ContextOption
- func WithRuntimeProfile(name, resolvedAt string, entries []RuntimeProfileEntry) ContextOption
- func WithScheduleTime(scheduleTime string) ContextOption
- func WithSecrets(secrets []string) ContextOption
- func WithStateStore(store dagstate.Store) ContextOption
- func WithTriggerActor(actor string) ContextOption
- func WithTriggerType(triggerType core.TriggerType) ContextOption
- func WithWorkDir(dir string) ContextOption
- type CoordinatorEndpoint
- type CursorResult
- type DAGRunAttempt
- type DAGRunCondition
- func MergeDAGRunConditions(conditions []DAGRunCondition, observations ...DAGRunCondition) []DAGRunCondition
- func NewDAGRunCondition(conditionType, status, reason, message string, checkedAt time.Time) DAGRunCondition
- func UpsertDAGRunCondition(conditions []DAGRunCondition, condition DAGRunCondition) []DAGRunCondition
- type DAGRunLease
- type DAGRunLeaseStore
- type DAGRunNotQueuedError
- type DAGRunOutputs
- type DAGRunRef
- type DAGRunStatus
- func (st *DAGRunStatus) DAGRun() DAGRunRef
- func (s DAGRunStatus) EffectiveClaimKey() string
- func (st *DAGRunStatus) Errors() []error
- func (st *DAGRunStatus) NodeByName(name string) (*Node, error)
- func (st *DAGRunStatus) NodesInRunOrder() []*Node
- func (s DAGRunStatus) Tags() []string
- func (st *DAGRunStatus) UnmarshalJSON(data []byte) error
- type DAGRunStatusPage
- type DAGRunStatusResult
- type DAGRunStore
- type DAGStore
- type Database
- type DispatchAdmissionBindRequest
- type DispatchAdmissionDecision
- type DispatchAdmissionRejectReason
- type DispatchAdmissionRequest
- type DispatchAdmissionStore
- type DispatchOperation
- type DispatchRequest
- type DispatchTask
- type DispatchTaskClaim
- type DispatchTaskStore
- type Dispatcher
- type EnqueueRetryOptions
- type FailedAutoRetryCancelEligibility
- type FailedAutoRetryCancelStateChangedError
- type GrepDAGsResult
- type HostInfo
- type LLMMessage
- type LLMMessageMetadata
- type ListDAGRunStatusesOption
- func WithAllHistory() ListDAGRunStatusesOption
- func WithCursor(cursor string) ListDAGRunStatusesOption
- func WithDAGRunID(dagRunID string) ListDAGRunStatusesOption
- func WithExactName(name string) ListDAGRunStatusesOption
- func WithFrom(from TimeInUTC) ListDAGRunStatusesOption
- func WithLabels(labels []string) ListDAGRunStatusesOption
- func WithLimit(limit int) ListDAGRunStatusesOption
- func WithName(name string) ListDAGRunStatusesOption
- func WithStatuses(statuses []core.Status) ListDAGRunStatusesOption
- func WithTags(tags []string) ListDAGRunStatusesOption
- func WithTo(to TimeInUTC) ListDAGRunStatusesOption
- func WithWorkspaceFilter(filter *WorkspaceFilter) ListDAGRunStatusesOption
- func WithoutLimit() ListDAGRunStatusesOption
- type ListDAGRunStatusesOptions
- type ListDAGsOptions
- type ListDAGsResult
- type LogWriterFactory
- type Match
- type MockDAGRunAttempt
- func (m *MockDAGRunAttempt) Abort(ctx context.Context) error
- func (m *MockDAGRunAttempt) Close(ctx context.Context) error
- func (m *MockDAGRunAttempt) Hidden() bool
- func (m *MockDAGRunAttempt) Hide(ctx context.Context) error
- func (m *MockDAGRunAttempt) ID() string
- func (m *MockDAGRunAttempt) IsAborting(ctx context.Context) (bool, error)
- func (m *MockDAGRunAttempt) Open(ctx context.Context) error
- func (m *MockDAGRunAttempt) ReadDAG(ctx context.Context) (*core.DAG, error)
- func (m *MockDAGRunAttempt) ReadOutputs(ctx context.Context) (*DAGRunOutputs, error)
- func (m *MockDAGRunAttempt) ReadStatus(ctx context.Context) (*DAGRunStatus, error)
- func (m *MockDAGRunAttempt) ReadStepMessages(ctx context.Context, stepName string) ([]LLMMessage, error)
- func (m *MockDAGRunAttempt) SetDAG(dag *core.DAG)
- func (m *MockDAGRunAttempt) WorkDir() string
- func (m *MockDAGRunAttempt) Write(ctx context.Context, status DAGRunStatus) error
- func (m *MockDAGRunAttempt) WriteOutputs(ctx context.Context, outputs *DAGRunOutputs) error
- func (m *MockDAGRunAttempt) WriteStepMessages(ctx context.Context, stepName string, messages []LLMMessage) error
- type MockQueueStore
- func (m *MockQueueStore) All(ctx context.Context) ([]QueuedItemData, error)
- func (m *MockQueueStore) DeleteByItemIDs(ctx context.Context, name string, itemIDs []string) (int, error)
- func (m *MockQueueStore) DequeueByDAGRunID(ctx context.Context, name string, dagRun DAGRunRef) ([]QueuedItemData, error)
- func (m *MockQueueStore) DequeueByName(ctx context.Context, name string) (QueuedItemData, error)
- func (m *MockQueueStore) Enqueue(ctx context.Context, name string, priority QueuePriority, dagRun DAGRunRef) error
- func (m *MockQueueStore) Len(ctx context.Context, name string) (int, error)
- func (m *MockQueueStore) List(ctx context.Context, name string) ([]QueuedItemData, error)
- func (m *MockQueueStore) ListByDAGName(ctx context.Context, name, dagName string) ([]QueuedItemData, error)
- func (m *MockQueueStore) ListCursor(ctx context.Context, name, cursor string, limit int) (CursorResult[QueuedItemData], error)
- func (m *MockQueueStore) QueueList(ctx context.Context) ([]string, error)
- func (m *MockQueueStore) QueueWatcher(ctx context.Context) QueueWatcher
- type NewDAGRunAttemptOptions
- type Node
- type OutputsMetadata
- type PID
- type PageRange
- type PaginatedResult
- type Paginator
- type PendingStepRetry
- type ProcEntry
- type ProcEntryID
- type ProcHandle
- type ProcHeartbeat
- type ProcMeta
- type ProcStore
- type PushBackEntry
- type QueuePriority
- type QueueStore
- type QueueWatcher
- type QueuedItem
- type QueuedItemData
- type RemoveDAGRunOption
- type RemoveDAGRunOptions
- type RemoveOldDAGRunsOption
- type RemoveOldDAGRunsOptions
- type RetryHop
- type RetryPath
- type RunStatus
- type RunningTask
- type RuntimeProfileEntry
- type SearchDAGMatchesOptions
- type SearchDAGResult
- type SearchDAGsOptions
- type ServiceName
- type ServiceRegistry
- type ServiceStatus
- type StaleQueueDispatchError
- type SubDAGRun
- type TimeInUTC
- type ToolCall
- type ToolCallFunction
- type ToolDefinition
- type WorkerHeartbeatRecord
- type WorkerHeartbeatStore
- type WorkerStats
- type WorkspaceFilter
- type WorkspaceLabelState
Constants ¶
const ( // StreamTypeStdout indicates stdout stream StreamTypeStdout = 1 // StreamTypeStderr indicates stderr stream StreamTypeStderr = 2 )
Stream type constants for LogWriterFactory.NewStepWriter
const ( // EnvKeyDAGName holds the name of the currently executing DAG. EnvKeyDAGName = "DAG_NAME" // EnvKeyDAGRunID holds the unique identifier for the current DAG run. EnvKeyDAGRunID = "DAG_RUN_ID" // EnvKeyDAGRunLogFile holds the path to the main log file for the DAG run. EnvKeyDAGRunLogFile = "DAG_RUN_LOG_FILE" // EnvKeyDAGRunStepName holds the name of the currently executing step. EnvKeyDAGRunStepName = "DAG_RUN_STEP_NAME" // EnvKeyDAGRunStepStdoutFile holds the path to the stdout log file for the current step. EnvKeyDAGRunStepStdoutFile = "DAG_RUN_STEP_STDOUT_FILE" // EnvKeyDAGRunStepStderrFile holds the path to the stderr log file for the current step. EnvKeyDAGRunStepStderrFile = "DAG_RUN_STEP_STDERR_FILE" // EnvKeyDAGUOutputFile holds the file path used for declared step outputs. EnvKeyDAGUOutputFile = "DAGU_OUTPUT_FILE" // EnvKeyDAGRunStatus holds the current status of the DAG run (e.g., "running", "success", "failed"). EnvKeyDAGRunStatus = "DAG_RUN_STATUS" // EnvKeyDAGWaitingSteps holds comma-separated step names that require manual action. EnvKeyDAGWaitingSteps = "DAG_WAITING_STEPS" // EnvKeyDAGParamsJSON exposes the resolved parameters encoded as JSON. // When params were provided as JSON, the original payload is preserved. EnvKeyDAGParamsJSON = "DAG_PARAMS_JSON" // EnvKeyDAGParamsJSONCompat exposes the resolved parameters encoded as JSON. EnvKeyDAGParamsJSONCompat = "DAGU_PARAMS_JSON" // EnvKeyDAGRunWorkDir holds the path to the per-DAG-run working directory. EnvKeyDAGRunWorkDir = "DAG_RUN_WORK_DIR" // EnvKeyDAGRunArtifactsDir holds the path to the per-DAG-run artifacts directory. EnvKeyDAGRunArtifactsDir = "DAG_RUN_ARTIFACTS_DIR" // EnvKeyDAGPushBack exposes the current push-back iteration and history as JSON. EnvKeyDAGPushBack = "DAG_PUSHBACK" // EnvKeyDAGPushBackIteration exposes the current push-back iteration as a plain value. EnvKeyDAGPushBackIteration = "DAG_PUSHBACK_ITERATION" // EnvKeyDAGPushBackPreviousStdoutFile exposes the previous stdout log path for a pushed-back step. EnvKeyDAGPushBackPreviousStdoutFile = "DAG_PUSHBACK_PREVIOUS_STDOUT_FILE" // EnvKeyExternalStepRetry enables parent-managed step retries for sub-DAG runs. // When set, retriable step failures transition to a queued retry state instead of // sleeping inline inside the child DAG process. EnvKeyExternalStepRetry = "DAGU_EXTERNAL_STEP_RETRY" // EnvKeyQueueDispatchRetry marks an internal retry invocation that is consuming // an already-queued run from the scheduler/worker queue dispatch path. EnvKeyQueueDispatchRetry = "DAGU_QUEUE_DISPATCH_RETRY" )
Environment variable keys that are automatically set by Dagu during execution.
const ( RoleSystem = core.LLMRoleSystem RoleUser = core.LLMRoleUser RoleAssistant = core.LLMRoleAssistant RoleTool = core.LLMRoleTool )
LLM message role constants - aliases for core package constants.
const ( // DefaultStaleLeaseThreshold is the shared default for considering a // distributed run lease stale when the coordinator has not observed recent // liveness for that run. DefaultStaleLeaseThreshold = 30 * time.Second )
const WorkspaceLabelKey = "workspace"
Variables ¶
var ( ErrDAGAlreadyExists = errors.New("DAG already exists") ErrDAGNotFound = errors.New("DAG is not found") )
Errors for DAG file operations
var ( ErrDAGRunIDNotFound = errors.New("dag-run ID not found") ErrDAGRunAlreadyExists = errors.New("dag-run already exists") ErrDAGRunActive = errors.New("dag-run is active") ErrNoStatusData = errors.New("no status data") ErrCorruptedStatusFile = errors.New("corrupted status file") // Status file exists but contains no valid data or is corrupted ErrInvalidQueryCursor = errors.New("dagrun: invalid query cursor") )
Errors related to dag-run management
var ( ErrDispatchTaskNotFound = errors.New("dispatch task claim not found") ErrDispatchAdmissionNotFound = errors.New("dispatch admission not found") ErrDispatchAdmissionConflict = errors.New("dispatch admission conflict") ErrDispatchAdmissionLivenessNotConfigured = errors.New("dispatch admission liveness not configured") ErrDAGRunLeaseNotFound = errors.New("dag-run lease not found") ErrActiveRunNotFound = errors.New("active distributed run not found") ErrWorkerHeartbeatNotFound = errors.New("worker heartbeat not found") )
var ( ErrQueueEmpty = errors.New("queue is empty") ErrQueueItemNotFound = errors.New("queue item not found") )
Errors for the queue
var ( ErrRetryStepNotFound = errors.New("retry step not found") ErrInvalidRetryPath = errors.New("retry path is invalid") // ErrRepeatingStepTarget indicates the target child DAG run was invoked by a // repeating step. Such child runs carry non-reproducible IDs, so only the // repeating step itself can be retried. ErrRepeatingStepTarget = errors.New("child DAG runs of a repeating step cannot be retried individually") )
var ErrFailedAutoRetryCancelStateChanged = errors.New(
"dag-run state changed before failed auto-retry cancel could be applied",
)
var (
ErrInvalidCursor = errors.New("invalid cursor")
)
var (
ErrInvalidRunRefFormat = errors.New("invalid dag-run reference format")
)
Errors for RunRef parsing
var ErrNoopAttemptNotSupported = errors.New("operation not supported by no-op DAG run attempt")
ErrNoopAttemptNotSupported is returned when an operation is not supported by a no-op attempt.
var ErrRetryStaleLatest = errors.New("retry target is no longer the latest attempt")
ErrRetryStaleLatest indicates the caller tried to retry a non-latest attempt.
Functions ¶
func AbortQueuedDAGRun ¶
func AbortQueuedDAGRun(ctx context.Context, dagRunStore DAGRunStore, dagRun DAGRunRef) error
AbortQueuedDAGRun marks the latest visible queued attempt as aborted, hides it, and removes the dag-run record only when no visible attempts remain.
func AttemptKeyForStatus ¶
func AttemptKeyForStatus(status *DAGRunStatus, fallbackAttemptID string) string
AttemptKeyForStatus resolves the authoritative attempt key for a persisted status. When older statuses omit AttemptKey, it is regenerated from the stored DAG-run identity and attempt ID.
func CanCancelFailedAutoRetryPendingRun ¶
func CanCancelFailedAutoRetryPendingRun(status *DAGRunStatus) bool
CanCancelFailedAutoRetryPendingRun returns true when a failed DAG-run is a root run and still has remaining DAG-level auto-retry budget.
func CancelFailedAutoRetryPendingRun ¶
func CancelFailedAutoRetryPendingRun( ctx context.Context, dagRunStore latestAttemptStatusSwapper, status *DAGRunStatus, ) error
CancelFailedAutoRetryPendingRun atomically marks the latest failed attempt as aborted so the retry scanner stops treating it as pending auto-retry.
func DecodeSearchCursor ¶
DecodeSearchCursor deserializes an opaque search cursor from base64url JSON.
func DistributedLeaseExpiredReason ¶
DistributedLeaseExpiredReason is the canonical failure reason for a distributed attempt that lost authoritative worker ownership.
func EncodeSearchCursor ¶
EncodeSearchCursor serializes an opaque search cursor as base64url JSON.
func EnqueueRetry ¶
func EnqueueRetry( ctx context.Context, dagRunStore DAGRunStore, queueStore QueueStore, dag *core.DAG, status *DAGRunStatus, opts EnqueueRetryOptions, ) (bool, error)
EnqueueRetry queues a DAG run for retry and records its Queued status. It restores the previous status if enqueueing fails and reports whether this call added the queue item.
func FilterPushBackInputs ¶
FilterPushBackInputs returns only declared push-back inputs. If no allowlist is provided, the stored inputs are preserved as-is.
func FormatTime ¶
FormatTime formats a time.Time or returns empty string if it's the zero value. This is a convenience wrapper around stringutil.FormatTime.
func GenerateAttemptKey ¶
GenerateAttemptKey creates a globally unique attempt identifier for cancellation tracking. Format: FNV1a64 hash of hierarchy + ":" + attemptId (e.g., "a1b2c3d4e5f67890:abc123").
func IsLeaseActive ¶
func IsLeaseActive(status *DAGRunStatus, staleThreshold time.Duration) bool
IsLeaseActive reports whether the run's lease is fresh (i.e. a worker is still actively executing and the coordinator has observed recent liveness). A zero LeaseAt is treated as stale.
func IsQueuedCatchup ¶
func IsQueuedCatchup(status *DAGRunStatus) bool
IsQueuedCatchup reports whether the queued status belongs to a catchup run.
func IsRemoteWorkerID ¶
IsRemoteWorkerID reports whether the status originated from a distributed worker instead of the local process runtime.
func LeaseIdentityMatchesStatus ¶
func LeaseIdentityMatchesStatus( lease *DAGRunLease, status *DAGRunStatus, fallbackAttemptID string, ) bool
LeaseIdentityMatchesStatus reports whether the lease belongs to the same persisted distributed attempt as status, independent of freshness.
func LeaseMatchesStatus ¶
func LeaseMatchesStatus( lease *DAGRunLease, status *DAGRunStatus, fallbackAttemptID string, now time.Time, staleThreshold time.Duration, ) bool
LeaseMatchesStatus reports whether the lease still authoritatively belongs to the exact distributed attempt represented by the persisted status.
func NewContext ¶
func NewContext( ctx context.Context, dag *core.DAG, dagRunID string, logFile string, opts ...ContextOption, ) context.Context
NewContext creates a new context with DAG execution metadata. Required: ctx, dag, dagRunID, logFile Optional: use ContextOption functions (WithDatabase, WithParams, etc.)
func NewDAGRunID ¶
NewDAGRunID returns a compact UUIDv7-derived DAG-run ID.
func NormalizeDAGRunConditions ¶
func NormalizeDAGRunConditions(status *DAGRunStatus)
NormalizeDAGRunConditions clears runtime conditions from non-queued statuses.
func PreservedQueueTriggerType ¶
func PreservedQueueTriggerType(status *DAGRunStatus) core.TriggerType
PreservedQueueTriggerType returns the trigger type that must be preserved when consuming a queued item. Queued retry records still execute as retries; initial queued runs keep the trigger that originally enqueued them.
func ResolveRetryPath ¶
func ResolveRetryPath( ctx context.Context, store DAGRunStore, root DAGRunRef, targetRunID string, stepName string, ) (RetryPath, *DAGRunStatus, error)
ResolveRetryPath resolves the ancestry of a persisted child DAG run.
func ValidateDAGRunID ¶
ValidateDAGRunID checks that the dag-run ID contains only safe characters (alphanumeric, hyphens, underscores) and does not exceed the max length. Returns nil if the ID is valid. Returns an error if the ID is empty, contains invalid characters, or is too long.
func WithContext ¶
WithContext returns a new context with the given DAGContext. This is useful for tests that need to set up a DAGContext directly.
func WorkspaceNameFromLabels ¶
WorkspaceNameFromLabels returns a valid single workspace label value.
Missing, invalid, or conflicting workspace labels return ok=false. Use WorkspaceLabelFromLabels when callers must distinguish missing from malformed labels.
Types ¶
type ActiveDistributedRun ¶
type ActiveDistributedRun struct {
AttemptKey string `json:"attemptKey"`
DAGRun DAGRunRef `json:"dagRun"`
Root DAGRunRef `json:"root,omitzero"`
AttemptID string `json:"attemptId"`
WorkerID string `json:"workerId"`
Status core.Status `json:"status"`
UpdatedAt int64 `json:"updatedAt"`
}
ActiveDistributedRun is the durable active-set record for a remote attempt.
type ActiveDistributedRunStore ¶
type ActiveDistributedRunStore interface {
Upsert(ctx context.Context, record ActiveDistributedRun) error
Delete(ctx context.Context, attemptKey string) error
Get(ctx context.Context, attemptKey string) (*ActiveDistributedRun, error)
ListAll(ctx context.Context) ([]ActiveDistributedRun, error)
}
ActiveDistributedRunStore persists the coordinator-owned active distributed attempt index used by the zombie detector.
type ClaimedDispatchTask ¶
type ClaimedDispatchTask struct {
Task *DispatchTask
ClaimToken string
ClaimedAt time.Time
WorkerID string
PollerID string
Owner CoordinatorEndpoint
}
ClaimedDispatchTask is a shared pending task that has been claimed by a specific worker poller and must be acknowledged before execution begins.
type CompareAndSwapStatusOption ¶
type CompareAndSwapStatusOption func(*CompareAndSwapStatusOptions)
CompareAndSwapStatusOption configures CompareAndSwapLatestAttemptStatus.
func WithCompareAndSwapExpectedAttemptKey ¶
func WithCompareAndSwapExpectedAttemptKey(attemptKey string) CompareAndSwapStatusOption
WithCompareAndSwapExpectedAttemptKey requires the current status attempt key to match.
func WithCompareAndSwapRootDAGRun ¶
func WithCompareAndSwapRootDAGRun(root DAGRunRef) CompareAndSwapStatusOption
WithCompareAndSwapRootDAGRun routes CompareAndSwapLatestAttemptStatus through a root dag-run when the target dag-run is stored as a sub-DAG attempt.
type CompareAndSwapStatusOptions ¶
CompareAndSwapStatusOptions configures additional identity guards for CompareAndSwapLatestAttemptStatus.
func NewCompareAndSwapStatusOptions ¶
func NewCompareAndSwapStatusOptions(opts ...CompareAndSwapStatusOption) CompareAndSwapStatusOptions
NewCompareAndSwapStatusOptions applies CompareAndSwapLatestAttemptStatus options.
type Context ¶
type Context struct {
DAGRunID string
RootDAGRun DAGRunRef
RetryPath RetryPath
AttemptID string
TriggerType core.TriggerType
TriggerActor string
RunStartedAt string
ScheduleTime string
DAG *core.DAG
DB Database
BaseEnv *config.BaseEnv
EnvScope *cmnvalue.EnvScope // Unified environment scope for runtime variables
CoordinatorCli Dispatcher
DAGRunStore DAGRunStore
QueueStore QueueStore
StateStore dagstate.Store
DAGRunLogDir string
DAGRunArtifactDir string
ProfileName string
ProfileResolvedAt string
ProfileEntries []RuntimeProfileEntry
Shell string // Default shell for this DAG (from DAG.Shell)
LogEncodingCharset string // Character encoding for log files (e.g., "utf-8", "shift_jis", "euc-jp")
LogWriterFactory LogWriterFactory // For remote log streaming (nil = use local files)
DefaultExecMode config.ExecutionMode // Server-level default execution mode (local or distributed)
}
Context contains the execution metadata for a dag-run.
func GetContext ¶
GetContext retrieves the DAGContext from the context.
func LookupContext ¶
LookupContext returns the DAGContext when one is present in ctx.
func (Context) AllEnvs ¶
AllEnvs returns every environment variable as "key=value" strings. Uses EnvScope as the single source of truth for all env vars.
func (Context) UserEnvsMap ¶
UserEnvsMap returns only user-defined environment variables as a map, excluding OS environment (BaseEnv). Use this for isolated execution environments.
type ContextOption ¶
type ContextOption func(*contextOptions)
ContextOption configures optional parameters for NewContext.
func WithArtifactDir ¶
func WithArtifactDir(dir string) ContextOption
WithArtifactDir sets the per-DAG-run artifacts directory path.
func WithAttemptID ¶
func WithAttemptID(attemptID string) ContextOption
WithAttemptID sets the DAG-run attempt identifier for value resolution.
func WithCoordinator ¶
func WithCoordinator(cli Dispatcher) ContextOption
WithCoordinator sets the coordinator dispatcher for distributed execution.
func WithDAGRunArtifactDir ¶
func WithDAGRunArtifactDir(dir string) ContextOption
WithDAGRunArtifactDir sets the base artifact directory for newly persisted DAG runs.
func WithDAGRunLogDir ¶
func WithDAGRunLogDir(dir string) ContextOption
WithDAGRunLogDir sets the base log directory for newly persisted DAG runs.
func WithDAGRunStore ¶
func WithDAGRunStore(store DAGRunStore) ContextOption
WithDAGRunStore sets the dag-run store for executors that persist DAG runs.
func WithDatabase ¶
func WithDatabase(db Database) ContextOption
WithDatabase sets the database interface.
func WithDefaultEnvVars ¶
func WithDefaultEnvVars(envs ...string) ContextOption
WithDefaultEnvVars sets low-precedence inherited environment variables.
func WithDefaultExecMode ¶
func WithDefaultExecMode(mode config.ExecutionMode) ContextOption
WithDefaultExecMode sets the server-level default execution mode.
func WithDefaultSecrets ¶
func WithDefaultSecrets(secrets []string) ContextOption
WithDefaultSecrets sets low-precedence inherited secret environment variables.
func WithEnvVars ¶
func WithEnvVars(envs ...string) ContextOption
WithEnvVars sets additional execution-scoped environment variables.
func WithLogEncoding ¶
func WithLogEncoding(charset string) ContextOption
WithLogEncoding sets the log file character encoding.
func WithLogWriterFactory ¶
func WithLogWriterFactory(factory LogWriterFactory) ContextOption
WithLogWriterFactory sets the log writer factory for remote log streaming. When set, logs are streamed to the coordinator instead of written to local files.
func WithParams ¶
func WithParams(params []string) ContextOption
WithParams sets runtime parameters.
func WithQueueStore ¶
func WithQueueStore(store QueueStore) ContextOption
WithQueueStore sets the queue store for executors that enqueue DAG runs.
func WithRetryPath ¶
func WithRetryPath(path RetryPath) ContextOption
WithRetryPath sets the persisted child DAG path for a targeted retry.
func WithRootDAGRun ¶
func WithRootDAGRun(ref DAGRunRef) ContextOption
WithRootDAGRun sets the root DAG run reference for sub-DAG execution.
func WithRunStartedAt ¶
func WithRunStartedAt(startedAt string) ContextOption
WithRunStartedAt sets the recorded DAG-run start timestamp for value resolution.
func WithRuntimeProfile ¶
func WithRuntimeProfile(name, resolvedAt string, entries []RuntimeProfileEntry) ContextOption
WithRuntimeProfile sets the selected profile metadata for this run context.
func WithScheduleTime ¶
func WithScheduleTime(scheduleTime string) ContextOption
WithScheduleTime sets the logical schedule time for value resolution.
func WithSecrets ¶
func WithSecrets(secrets []string) ContextOption
WithSecrets sets secret environment variables.
func WithStateStore ¶
func WithStateStore(store dagstate.Store) ContextOption
WithStateStore sets the persistent DAG state store for state actions.
func WithTriggerActor ¶
func WithTriggerActor(actor string) ContextOption
WithTriggerActor sets the attributable trigger actor for value resolution.
func WithTriggerType ¶
func WithTriggerType(triggerType core.TriggerType) ContextOption
WithTriggerType sets the DAG-run trigger type for value resolution.
func WithWorkDir ¶
func WithWorkDir(dir string) ContextOption
WithWorkDir sets the per-DAG-run working directory path.
type CoordinatorEndpoint ¶
type CoordinatorEndpoint struct {
ID string `json:"id,omitempty"`
Host string `json:"host,omitempty"`
Port int `json:"port,omitempty"`
}
CoordinatorEndpoint identifies a coordinator instance that owns a distributed task or run.
func CoordinatorEndpointFromHostInfo ¶
func CoordinatorEndpointFromHostInfo(info HostInfo) CoordinatorEndpoint
CoordinatorEndpointFromHostInfo converts a host record to an endpoint.
func (CoordinatorEndpoint) HostInfo ¶
func (e CoordinatorEndpoint) HostInfo() HostInfo
HostInfo converts the endpoint to the existing service registry host shape.
type CursorResult ¶
CursorResult contains a bounded result window and continuation state.
type DAGRunAttempt ¶
type DAGRunAttempt interface {
// ID returns the identifier for the attempt that is unique within the dag-run.
ID() string
// Open prepares the attempt for writing status updates
Open(ctx context.Context) error
// Write updates the status of the attempt
Write(ctx context.Context, status DAGRunStatus) error
// Close finalizes writing to the attempt
Close(ctx context.Context) error
// ReadStatus retrieves the current status of the attempt
ReadStatus(ctx context.Context) (*DAGRunStatus, error)
// ReadDAG reads the DAG associated with this run attempt
ReadDAG(ctx context.Context) (*core.DAG, error)
// SetDAG sets the DAG for this attempt (must be called before Open for DAG to be persisted)
SetDAG(dag *core.DAG)
// Abort requests aborting the attempt
Abort(ctx context.Context) error
// IsAborting checks if an abort has been requested for the attempt
IsAborting(ctx context.Context) (bool, error)
// Hide marks the attempt as hidden from normal operations.
// This is useful for preserving previous state visibility when dequeuing.
Hide(ctx context.Context) error
// Hidden returns true if the attempt is hidden from normal operations.
Hidden() bool
// WriteOutputs writes the collected step outputs for the dag-run.
// Does nothing if outputs is nil or has no output entries.
WriteOutputs(ctx context.Context, outputs *DAGRunOutputs) error
// ReadOutputs reads the collected step outputs for the dag-run.
// Returns nil if no outputs file exists or if the file is in v1 format.
ReadOutputs(ctx context.Context) (*DAGRunOutputs, error)
// WriteStepMessages writes LLM messages for a single step.
WriteStepMessages(ctx context.Context, stepName string, messages []LLMMessage) error
// ReadStepMessages reads LLM messages for a single step.
// Returns nil if no messages exist for the step.
ReadStepMessages(ctx context.Context, stepName string) ([]LLMMessage, error)
// WorkDir returns the path to the per-DAG-run working directory.
// Returns "" if the attempt does not support local storage.
WorkDir() string
}
DAGRunAttempt represents a single execution of a dag-run to record the status and execution details.
func NewNoopDAGRunAttempt ¶
func NewNoopDAGRunAttempt(id string, dag *core.DAG) DAGRunAttempt
NewNoopDAGRunAttempt creates a no-op attempt for remote worker execution.
type DAGRunCondition ¶
type DAGRunCondition struct {
Type string `json:"type"`
Status string `json:"status"`
Reason string `json:"reason"`
Message string `json:"message"`
CheckedAt string `json:"checkedAt"`
}
DAGRunCondition describes an observed runtime condition for a DAG-run.
func MergeDAGRunConditions ¶
func MergeDAGRunConditions(conditions []DAGRunCondition, observations ...DAGRunCondition) []DAGRunCondition
MergeDAGRunConditions merges observations into a type-keyed current-state list.
func NewDAGRunCondition ¶
func NewDAGRunCondition(conditionType, status, reason, message string, checkedAt time.Time) DAGRunCondition
NewDAGRunCondition creates a runtime condition for a DAG-run.
func UpsertDAGRunCondition ¶
func UpsertDAGRunCondition(conditions []DAGRunCondition, condition DAGRunCondition) []DAGRunCondition
UpsertDAGRunCondition merges a condition into a type-keyed current-state list.
type DAGRunLease ¶
type DAGRunLease struct {
AttemptKey string `json:"attemptKey"`
DAGRun DAGRunRef `json:"dagRun"`
Root DAGRunRef `json:"root,omitzero"`
AttemptID string `json:"attemptId"`
QueueName string `json:"queueName"`
WorkerID string `json:"workerId"`
Owner CoordinatorEndpoint `json:"owner"`
ClaimedAt int64 `json:"claimedAt"`
LastHeartbeatAt int64 `json:"lastHeartbeatAt"`
}
DAGRunLease is the shared liveness record for an accepted worker claim.
func (DAGRunLease) ClaimedTime ¶
func (l DAGRunLease) ClaimedTime() time.Time
ClaimedTime returns the lease creation time.
func (DAGRunLease) LastHeartbeatTime ¶
func (l DAGRunLease) LastHeartbeatTime() time.Time
LastHeartbeatTime returns the last run heartbeat time.
func (*DAGRunLease) MatchesClaim ¶
func (l *DAGRunLease) MatchesClaim(claimKey, workerID string) bool
MatchesClaim reports whether the lease identifies claimKey without conflicting with workerID. An empty worker ID on either side is treated as unspecified.
type DAGRunLeaseStore ¶
type DAGRunLeaseStore interface {
Upsert(ctx context.Context, lease DAGRunLease) error
Touch(ctx context.Context, attemptKey string, observedAt time.Time) error
Delete(ctx context.Context, attemptKey string) error
Get(ctx context.Context, attemptKey string) (*DAGRunLease, error)
ListByQueue(ctx context.Context, queueName string) ([]DAGRunLease, error)
ListAll(ctx context.Context) ([]DAGRunLease, error)
}
DAGRunLeaseStore persists live worker-claim leases.
type DAGRunNotQueuedError ¶
DAGRunNotQueuedError reports that the latest visible attempt is no longer queued.
func (*DAGRunNotQueuedError) Error ¶
func (e *DAGRunNotQueuedError) Error() string
type DAGRunOutputs ¶
type DAGRunOutputs struct {
Metadata OutputsMetadata `json:"metadata"`
Outputs map[string]string `json:"outputs"`
}
DAGRunOutputs represents the full outputs file structure with metadata.
type DAGRunRef ¶
DAGRunRef represents a reference to a dag-run
func NewDAGRunRef ¶
NewDAGRunRef creates a new reference to dag-run with the given DAG name and run ID. It is used to identify a specific dag-run.
func ParseDAGRunRef ¶
ParseDAGRunRef parses a string into a DAGRunRef. The expected format is "name:runId". If the format is invalid, it returns an error.
type DAGRunStatus ¶
type DAGRunStatus struct {
Root DAGRunRef `json:"root,omitzero"`
Parent DAGRunRef `json:"parent,omitzero"`
Name string `json:"name"`
DAGRunID string `json:"dagRunId"`
AttemptID string `json:"attemptId"`
AttemptKey string `json:"attemptKey,omitempty"` // Globally unique attempt identifier
ClaimKey string `json:"claimKey,omitempty"` // Worker claim that executes this attempt
Status core.Status `json:"status"`
Conditions []DAGRunCondition `json:"conditions,omitempty"`
TriggerType core.TriggerType `json:"triggerType,omitempty"`
TriggerActor string `json:"triggerActor,omitempty"`
WorkerID string `json:"workerId,omitempty"`
PID PID `json:"pid,omitempty"`
PIDStartedAt int64 `json:"pidStartedAt,omitempty"`
Nodes []*Node `json:"nodes,omitempty"`
OnInit *Node `json:"onInit,omitempty"`
OnExit *Node `json:"onExit,omitempty"`
OnSuccess *Node `json:"onSuccess,omitempty"`
OnFailure *Node `json:"onFailure,omitempty"`
OnAbort *Node `json:"onAbort,omitempty"`
OnWait *Node `json:"onWait,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
QueuedAt string `json:"queuedAt,omitempty"`
ScheduleTime string `json:"scheduleTime,omitempty"`
StartedAt string `json:"startedAt,omitempty"`
FinishedAt string `json:"finishedAt,omitempty"`
AutoRetryCount int `json:"autoRetryCount,omitempty"`
AutoRetryLimit int `json:"autoRetryLimit,omitempty"`
// AutoRetryInterval is stored as a duration snapshot for retry scanner decisions.
AutoRetryInterval time.Duration `json:"autoRetryInterval,omitempty"`
AutoRetryBackoff float64 `json:"autoRetryBackoff,omitempty"`
// AutoRetryMaxInterval is stored as a duration snapshot for retry scanner decisions.
AutoRetryMaxInterval time.Duration `json:"autoRetryMaxInterval,omitempty"`
ProcGroup string `json:"procGroup,omitempty"`
SuspendFlagName string `json:"suspendFlagName,omitempty"`
Log string `json:"log,omitempty"`
WorkingDir string `json:"workingDir,omitempty"`
ArchiveDir string `json:"archiveDir,omitempty"`
Error string `json:"error,omitempty"`
Params string `json:"params,omitempty"`
ParamsList []string `json:"paramsList,omitempty"`
ProfileName string `json:"profileName,omitempty"`
ProfileResolvedAt string `json:"profileResolvedAt,omitempty"`
ProfileEntries []RuntimeProfileEntry `json:"profileEntries,omitempty"`
PendingStepRetries []PendingStepRetry `json:"pendingStepRetries"`
Preconditions []*core.Condition `json:"preconditions,omitempty"`
Labels []string `json:"labels,omitempty"`
LeaseAt int64 `json:"leaseAt,omitempty"` // Unix millis; stamped by coordinator on observed run liveness
}
DAGRunStatus represents the complete execution state of a dag-run.
func InitialStatus ¶
func InitialStatus(dag *core.DAG) DAGRunStatus
InitialStatus creates an initial Status object for the given DAG
func StatusFromJSON ¶
func StatusFromJSON(s string) (*DAGRunStatus, error)
StatusFromJSON deserializes a JSON string into a Status object
func (*DAGRunStatus) DAGRun ¶
func (st *DAGRunStatus) DAGRun() DAGRunRef
DAGRun returns a reference to the dag-run associated with this status
func (DAGRunStatus) EffectiveClaimKey ¶
func (s DAGRunStatus) EffectiveClaimKey() string
EffectiveClaimKey returns ClaimKey, falling back to AttemptKey when no claim is recorded.
func (*DAGRunStatus) Errors ¶
func (st *DAGRunStatus) Errors() []error
Errors returns a slice of errors for the current status
func (*DAGRunStatus) NodeByName ¶
func (st *DAGRunStatus) NodeByName(name string) (*Node, error)
NodeByName returns the node with the specified name. For handlers, it matches on both the handler label (e.g., "onSuccess") and the step name within the handler.
func (*DAGRunStatus) NodesInRunOrder ¶
func (st *DAGRunStatus) NodesInRunOrder() []*Node
NodesInRunOrder returns the run's step nodes together with the lifecycle handler nodes that were configured, ordered by when they run. Handlers that the DAG does not declare are omitted.
func (DAGRunStatus) Tags ¶
func (s DAGRunStatus) Tags() []string
Tags returns labels under their deprecated name. Deprecated: use Labels directly.
func (*DAGRunStatus) UnmarshalJSON ¶
func (st *DAGRunStatus) UnmarshalJSON(data []byte) error
UnmarshalJSON keeps legacy onCancel and tags status files readable while normalizing canonical handler/metadata names in memory.
type DAGRunStatusPage ¶
type DAGRunStatusPage struct {
Items []*DAGRunStatus
NextCursor string
}
DAGRunStatusPage is one forward-only page of DAG-run statuses.
type DAGRunStatusResult ¶
type DAGRunStatusResult struct {
Found bool
Status *DAGRunStatus
}
DAGRunStatusResult is a distributed status lookup result.
type DAGRunStore ¶
type DAGRunStore interface {
// CreateAttempt creates a new execution record for a dag-run.
CreateAttempt(ctx context.Context, dag *core.DAG, ts time.Time, dagRunID string, opts NewDAGRunAttemptOptions) (DAGRunAttempt, error)
// RecentAttempts returns the most recent dag-run's attempt for the DAG name, limited by itemLimit
RecentAttempts(ctx context.Context, name string, itemLimit int) []DAGRunAttempt
// LatestAttempt returns the most recent dag-run's attempt for the DAG name.
LatestAttempt(ctx context.Context, name string) (DAGRunAttempt, error)
// ListStatuses returns a list of statuses.
ListStatuses(ctx context.Context, opts ...ListDAGRunStatusesOption) ([]*DAGRunStatus, error)
// ListStatusesPage returns one forward-only page of statuses in canonical list order.
ListStatusesPage(ctx context.Context, opts ...ListDAGRunStatusesOption) (DAGRunStatusPage, error)
// CompareAndSwapLatestAttemptStatus atomically updates the latest attempt status
// when both the latest attempt ID and status still match the expected values.
CompareAndSwapLatestAttemptStatus(
ctx context.Context,
dagRun DAGRunRef,
expectedAttemptID string,
expectedStatus core.Status,
mutate func(*DAGRunStatus) error,
opts ...CompareAndSwapStatusOption,
) (*DAGRunStatus, bool, error)
// FindAttempt finds the latest attempt for the dag-run.
FindAttempt(ctx context.Context, dagRun DAGRunRef) (DAGRunAttempt, error)
// FindSubAttempt finds a sub dag-run record by dag-run ID.
FindSubAttempt(ctx context.Context, dagRun DAGRunRef, subDAGRunID string) (DAGRunAttempt, error)
// CreateSubAttempt creates a new sub dag-run attempt under the root dag-run.
// This is used for distributed sub-DAG execution where the coordinator needs
// to create the attempt directory before the worker reports status.
CreateSubAttempt(ctx context.Context, rootRef DAGRunRef, subDAGRunID string) (DAGRunAttempt, error)
// RemoveOldDAGRuns deletes dag-run records older than retentionDays, by absolute
// cutoff (WithOlderThan), or by run count (WithRetentionRuns).
// If retentionDays is negative and OlderThan is not set, it won't delete any records.
// If retentionDays is zero, it will delete all records for the DAG name.
// But it will not delete the records with non-final statuses (e.g., running, queued).
// Returns a list of dag-run IDs that were removed (or would be removed in dry-run mode).
RemoveOldDAGRuns(ctx context.Context, name string, retentionDays int, opts ...RemoveOldDAGRunsOption) ([]string, error)
// RemoveDAGRun removes a dag-run record by its reference.
RemoveDAGRun(ctx context.Context, dagRun DAGRunRef, opts ...RemoveDAGRunOption) error
}
DAGRunStore provides an interface for interacting with the underlying database for storing and retrieving dag-run data. It abstracts the details of the storage mechanism, allowing for different implementations (e.g., file-based, in-memory, etc.) to be used interchangeably.
type DAGStore ¶
type DAGStore interface {
// Create stores a new DAG definition with the given name and returns its file name
Create(ctx context.Context, fileName string, spec []byte) error
// Delete removes a DAG definition by name
Delete(ctx context.Context, fileName string) error
// List returns a paginated list of DAG definitions with filtering options
List(ctx context.Context, params ListDAGsOptions) (PaginatedResult[*core.DAG], []string, error)
// GetMetadata retrieves only the metadata of a DAG definition (faster than full load)
GetMetadata(ctx context.Context, fileName string) (*core.DAG, error)
// GetDetails retrieves the complete DAG definition including all fields
GetDetails(ctx context.Context, fileName string, opts ...spec.LoadOption) (*core.DAG, error)
// Grep searches for a pattern in all DAG definitions and returns matching results
Grep(ctx context.Context, pattern string) (ret []*GrepDAGsResult, errs []string, err error)
// SearchCursor returns lightweight, cursor-based search hits for DAG definitions.
SearchCursor(ctx context.Context, opts SearchDAGsOptions) (*CursorResult[SearchDAGResult], []string, error)
// SearchMatches returns cursor-based match snippets for a specific DAG definition.
SearchMatches(ctx context.Context, fileName string, opts SearchDAGMatchesOptions) (*CursorResult[*Match], error)
// Rename changes a DAG's identifier from oldID to newID
Rename(ctx context.Context, oldID, newID string) error
// GetSpec retrieves the raw YAML specification of a DAG
GetSpec(ctx context.Context, fileName string) (string, error)
// UpdateSpec modifies the specification of an existing DAG
UpdateSpec(ctx context.Context, fileName string, spec []byte) error
// LoadSpec loads a DAG from a YAML file and returns the DAG object
LoadSpec(ctx context.Context, spec []byte, opts ...spec.LoadOption) (*core.DAG, error)
// LabelList returns all unique labels across all DAGs with any errors encountered
LabelList(ctx context.Context) ([]string, []string, error)
// ToggleSuspend changes the suspension state of a DAG by ID
ToggleSuspend(ctx context.Context, fileName string, suspend bool) error
// IsSuspended checks if a DAG is currently suspended
IsSuspended(ctx context.Context, fileName string) bool
}
DAGStore is an interface for interacting with underlying DAG storage systems. It allows for different implementations (e.g., local file system, database) to be used interchangeably.
type Database ¶
type Database interface {
// GetDAG retrieves a DAG by its name.
GetDAG(ctx context.Context, name string) (*core.DAG, error)
// GetSubDAGRunStatus retrieves the status of a sub dag-run by its ID and the root dag-run reference.
GetSubDAGRunStatus(ctx context.Context, dagRunID string, rootDAGRun DAGRunRef) (*RunStatus, error)
// IsSubDAGRunCompleted checks if a sub dag-run has completed.
IsSubDAGRunCompleted(ctx context.Context, dagRunID string, rootDAGRun DAGRunRef) (bool, error)
// RequestChildCancel requests cancellation of a sub dag-run.
RequestChildCancel(ctx context.Context, dagRunID string, rootDAGRun DAGRunRef) error
}
Database is the interface for accessing the database to retrieve DAGs and dag-run statuses. This interface abstracts the underlying storage mechanism, allowing for different implementations (e.g., SQL, NoSQL, in-memory).
type DispatchAdmissionBindRequest ¶
type DispatchAdmissionBindRequest struct {
ReservationToken string
Task *DispatchTask
}
DispatchAdmissionBindRequest describes a coordinator bind for a reservation.
type DispatchAdmissionDecision ¶
type DispatchAdmissionDecision struct {
Reserved bool
Reason DispatchAdmissionRejectReason
ReservationToken string
}
DispatchAdmissionDecision is the durable queue admission result.
type DispatchAdmissionRejectReason ¶
type DispatchAdmissionRejectReason string
DispatchAdmissionRejectReason identifies why a reservation was not granted.
const ( DispatchAdmissionRejectedDuplicate DispatchAdmissionRejectReason = "duplicate" DispatchAdmissionRejectedNoCapacity DispatchAdmissionRejectReason = "no_capacity" )
type DispatchAdmissionRequest ¶
type DispatchAdmissionRequest struct {
QueueName string
MaxConcurrency int
NonAdmissionOccupancy int
AttemptKey string
AttemptID string
DAGRun DAGRunRef
StaleThreshold time.Duration
}
DispatchAdmissionRequest describes a queue admission reservation request.
type DispatchAdmissionStore ¶
type DispatchAdmissionStore interface {
ReserveAdmission(ctx context.Context, req DispatchAdmissionRequest) (*DispatchAdmissionDecision, error)
BindAdmission(ctx context.Context, req DispatchAdmissionBindRequest) error
ReleaseAdmissionToken(ctx context.Context, reservationToken string) error
FinalizeAdmissionAttempt(ctx context.Context, attemptKey string) error
CleanupAdmissions(ctx context.Context, staleThreshold time.Duration) error
}
DispatchAdmissionStore reserves queue capacity and binds reservations to tasks.
type DispatchOperation ¶
type DispatchOperation int32
DispatchOperation identifies the operation requested for a distributed DAG run.
const ( DispatchOperationUnspecified DispatchOperation = iota DispatchOperationStart DispatchOperationRetry )
func (DispatchOperation) String ¶
func (o DispatchOperation) String() string
type DispatchRequest ¶
type DispatchRequest struct {
Task *DispatchTask
AdmissionReservationToken string
}
DispatchRequest describes a distributed dispatch call.
type DispatchTask ¶
type DispatchTask struct {
RootDAGRunName string
RootDAGRunID string
ParentDAGRunName string
ParentDAGRunID string
Operation DispatchOperation
DAGRunID string
Target string
Definition string
AttemptID string
AttemptKey string
Step string
Params string
QueueName string
WorkerID string
ProfileName string
TriggerActor string
PreviousStatus *DAGRunStatus
BaseConfig string
Labels string
ScheduleTime string
SourceFile string
WorkerSelector map[string]string
ExternalStepRetry bool
RetryPath string
WorkspaceBundleDigest string
WorkspaceBundleSize int64
WorkspaceBundleDAGPath string
WorkspaceBundleOriginalRef string
WorkspaceBundleResolvedRef string
Owner CoordinatorEndpoint
ClaimToken string
}
DispatchTask describes a DAG run request for a distributed executor.
type DispatchTaskClaim ¶
type DispatchTaskClaim struct {
WorkerID string
PollerID string
Labels map[string]string
Owner CoordinatorEndpoint
ClaimTimeout time.Duration
}
DispatchTaskClaim describes a worker poller's attempt to claim a shared distributed task.
type DispatchTaskStore ¶
type DispatchTaskStore interface {
Enqueue(ctx context.Context, task *DispatchTask) error
ClaimNext(ctx context.Context, claim DispatchTaskClaim) (*ClaimedDispatchTask, error)
GetClaim(ctx context.Context, claimToken string) (*ClaimedDispatchTask, error)
ReleaseClaim(ctx context.Context, claimToken string) error
DeleteClaim(ctx context.Context, claimToken string) error
CountOutstandingByQueue(ctx context.Context, queueName string, claimTimeout time.Duration) (int, error)
HasOutstandingAttempt(ctx context.Context, attemptKey string, claimTimeout time.Duration) (bool, error)
}
DispatchTaskStore manages the shared distributed dispatch queue.
type Dispatcher ¶
type Dispatcher interface {
Dispatch(ctx context.Context, req DispatchRequest) error
Cleanup(ctx context.Context) error
GetDAGRunStatus(ctx context.Context, dagName, dagRunID string, rootRef *DAGRunRef) (*DAGRunStatusResult, error)
RequestCancel(ctx context.Context, dagName, dagRunID string, rootRef *DAGRunRef) error
}
Dispatcher defines distributed DAG run operations.
type EnqueueRetryOptions ¶
type EnqueueRetryOptions struct {
// AutoRetry marks scheduler-issued DAG auto-retries. These consume the
// DAG-level retry budget at enqueue time.
AutoRetry bool
// TriggerActor replaces the attributable actor for a user-issued retry.
// Nil preserves the actor already recorded on the run.
TriggerActor *string
}
EnqueueRetryOptions configure a retry enqueue.
type FailedAutoRetryCancelEligibility ¶
type FailedAutoRetryCancelEligibility int
FailedAutoRetryCancelEligibility describes whether a failed DAG-run can be canceled before the scheduler issues the next DAG-level auto-retry.
const ( FailedAutoRetryCancelEligible FailedAutoRetryCancelEligibility = iota FailedAutoRetryCancelMissingStatus FailedAutoRetryCancelNotRoot FailedAutoRetryCancelNotPending )
func FailedAutoRetryCancelEligibilityOf ¶
func FailedAutoRetryCancelEligibilityOf(status *DAGRunStatus) FailedAutoRetryCancelEligibility
FailedAutoRetryCancelEligibilityOf classifies whether the provided status can be canceled while it is failed and still waiting for a DAG-level auto-retry.
type FailedAutoRetryCancelStateChangedError ¶
type FailedAutoRetryCancelStateChangedError struct {
CurrentStatus *DAGRunStatus
}
FailedAutoRetryCancelStateChangedError reports the latest observed status when another actor changed the latest attempt before the cancel CAS completed.
func (*FailedAutoRetryCancelStateChangedError) Error ¶
func (e *FailedAutoRetryCancelStateChangedError) Error() string
func (*FailedAutoRetryCancelStateChangedError) Unwrap ¶
func (e *FailedAutoRetryCancelStateChangedError) Unwrap() error
type GrepDAGsResult ¶
type GrepDAGsResult struct {
Name string // Name of the DAG
DAG *core.DAG // The DAG object
Matches []*Match // Matching lines and their context
}
GrepDAGsResult represents the result of a pattern search within a DAG definition
type HostInfo ¶
type HostInfo struct {
// ID is a unique identifier for the host
ID string
// Host is the hostname or IP address
Host string
// Port is the port number (0 if not applicable)
Port int
// Status is the operational status of the service instance
Status ServiceStatus
// StartedAt is when the service instance was started
StartedAt time.Time
}
HostInfo contains information about a host in the service registry system
type LLMMessage ¶
type LLMMessage struct {
// Role is the message role (system, user, assistant, tool).
Role core.LLMRole `json:"role"`
// Content is the message content.
Content string `json:"content"`
// ToolCallID is the ID of the tool call this message is responding to.
// Only set when Role is "tool".
ToolCallID string `json:"tool_call_id,omitempty"`
// ToolCalls contains tool calls made by the assistant.
// Only set when Role is "assistant" and the model requested tool calls.
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
// Metadata contains API call metadata (only set for assistant responses).
Metadata *LLMMessageMetadata `json:"metadata,omitempty"`
}
LLMMessage represents a single message in the session.
func DeduplicateSystemMessages ¶
func DeduplicateSystemMessages(messages []LLMMessage) []LLMMessage
DeduplicateSystemMessages keeps only the first system message.
type LLMMessageMetadata ¶
type LLMMessageMetadata struct {
// Provider is the LLM provider used (openai, anthropic, etc.).
Provider string `json:"provider,omitempty"`
// Model is the model identifier used.
Model string `json:"model,omitempty"`
// PromptTokens is the number of tokens in the prompt.
PromptTokens int `json:"promptTokens,omitempty"`
// CompletionTokens is the number of tokens in the completion.
CompletionTokens int `json:"completionTokens,omitempty"`
// TotalTokens is the sum of prompt and completion tokens.
TotalTokens int `json:"totalTokens,omitempty"`
// Cost is the estimated USD cost for this API call.
Cost float64 `json:"cost,omitempty"`
}
LLMMessageMetadata contains metadata about an LLM API call.
type ListDAGRunStatusesOption ¶
type ListDAGRunStatusesOption func(*ListDAGRunStatusesOptions)
ListRunsOption is a functional option for configuring ListRunsOptions
func WithAllHistory ¶
func WithAllHistory() ListDAGRunStatusesOption
WithAllHistory disables the default implicit "today only" time window when no explicit range is supplied.
func WithCursor ¶
func WithCursor(cursor string) ListDAGRunStatusesOption
WithCursor sets the opaque cursor for forward-only DAG-run pagination.
func WithDAGRunID ¶
func WithDAGRunID(dagRunID string) ListDAGRunStatusesOption
WithDAGRunID sets the dag-run ID for listing dag-runs
func WithExactName ¶
func WithExactName(name string) ListDAGRunStatusesOption
WithExactName sets the name for listing dag-runs
func WithFrom ¶
func WithFrom(from TimeInUTC) ListDAGRunStatusesOption
WithFrom sets the start time for listing dag-runs
func WithLabels ¶
func WithLabels(labels []string) ListDAGRunStatusesOption
WithLabels sets the labels filter for listing dag-runs (AND logic - all labels must match)
func WithLimit ¶
func WithLimit(limit int) ListDAGRunStatusesOption
WithLimit sets the maximum number of results to return when listing dag-runs
func WithName ¶
func WithName(name string) ListDAGRunStatusesOption
WithName sets the name for listing dag-runs
func WithStatuses ¶
func WithStatuses(statuses []core.Status) ListDAGRunStatusesOption
WithStatuses sets the statuses for listing dag-runs
func WithTags ¶
func WithTags(tags []string) ListDAGRunStatusesOption
WithTags sets the labels filter for listing dag-runs. Deprecated: use WithLabels.
func WithTo ¶
func WithTo(to TimeInUTC) ListDAGRunStatusesOption
WithTo sets the end time for listing dag-runs
func WithWorkspaceFilter ¶
func WithWorkspaceFilter(filter *WorkspaceFilter) ListDAGRunStatusesOption
WithWorkspaceFilter sets the workspace visibility filter for listing dag-runs.
func WithoutLimit ¶
func WithoutLimit() ListDAGRunStatusesOption
WithoutLimit disables the default 1000-item cap for internal callers that need to scan the full recent result set.
type ListDAGRunStatusesOptions ¶
type ListDAGRunStatusesOptions struct {
DAGRunID string
Name string
ExactName string
From TimeInUTC
To TimeInUTC
Statuses []core.Status
Limit int
Cursor string
Labels []string // Filter by DAG labels (AND logic - all labels must match)
WorkspaceFilter *WorkspaceFilter
Unlimited bool
AllHistory bool
}
ListDAGRunStatusesOptions contains options for listing runs
type ListDAGsOptions ¶
type ListDAGsOptions struct {
Paginator *Paginator
Name string // Optional search filter for DAG name or file name
Labels []string // Optional labels filter (AND logic - all labels must match)
Sort string // Optional sort field (name, updated_at, created_at, nextRun)
Order string // Optional sort order (asc, desc)
Time *time.Time // Optional reference time for nextRun sorting/projection (defaults to time.Now())
NextRunProjection func(*core.DAG, time.Time) time.Time // Optional scheduler-aware nextRun projector used when Sort == "nextRun"
WorkspaceFilter *WorkspaceFilter // Optional workspace visibility filter
}
ListDAGsOptions contains parameters for paginated DAG listing
type ListDAGsResult ¶
type ListDAGsResult struct {
DAGs []*core.DAG // The list of DAGs for the current page
Count int // Total count of DAGs matching the filter
Errors []string // Any errors encountered during listing
}
ListDAGsResult contains the result of a paginated DAG listing operation
type LogWriterFactory ¶
type LogWriterFactory interface {
// NewStepWriter creates a writer for a step's log output.
// stepName identifies the step, streamType should be StreamTypeStdout or StreamTypeStderr.
NewStepWriter(ctx context.Context, stepName string, streamType int) io.WriteCloser
}
LogWriterFactory creates log writers for step stdout/stderr. It abstracts where logs are written, allowing for: - Local file-based storage (default) - Remote streaming to coordinator
type MockDAGRunAttempt ¶
type MockDAGRunAttempt struct {
mock.Mock
// Status can be set for tests that need to return a specific status without mock setup
Status *DAGRunStatus
}
MockDAGRunAttempt is a mock implementation of DAGRunAttempt for testing.
func (*MockDAGRunAttempt) Hidden ¶
func (m *MockDAGRunAttempt) Hidden() bool
func (*MockDAGRunAttempt) ID ¶
func (m *MockDAGRunAttempt) ID() string
func (*MockDAGRunAttempt) IsAborting ¶
func (m *MockDAGRunAttempt) IsAborting(ctx context.Context) (bool, error)
func (*MockDAGRunAttempt) ReadOutputs ¶
func (m *MockDAGRunAttempt) ReadOutputs(ctx context.Context) (*DAGRunOutputs, error)
func (*MockDAGRunAttempt) ReadStatus ¶
func (m *MockDAGRunAttempt) ReadStatus(ctx context.Context) (*DAGRunStatus, error)
func (*MockDAGRunAttempt) ReadStepMessages ¶
func (m *MockDAGRunAttempt) ReadStepMessages(ctx context.Context, stepName string) ([]LLMMessage, error)
func (*MockDAGRunAttempt) SetDAG ¶
func (m *MockDAGRunAttempt) SetDAG(dag *core.DAG)
func (*MockDAGRunAttempt) WorkDir ¶
func (m *MockDAGRunAttempt) WorkDir() string
func (*MockDAGRunAttempt) Write ¶
func (m *MockDAGRunAttempt) Write(ctx context.Context, status DAGRunStatus) error
func (*MockDAGRunAttempt) WriteOutputs ¶
func (m *MockDAGRunAttempt) WriteOutputs(ctx context.Context, outputs *DAGRunOutputs) error
func (*MockDAGRunAttempt) WriteStepMessages ¶
func (m *MockDAGRunAttempt) WriteStepMessages(ctx context.Context, stepName string, messages []LLMMessage) error
type MockQueueStore ¶
MockQueueStore is a mock implementation of QueueStore for testing.
func (*MockQueueStore) All ¶
func (m *MockQueueStore) All(ctx context.Context) ([]QueuedItemData, error)
func (*MockQueueStore) DeleteByItemIDs ¶
func (*MockQueueStore) DequeueByDAGRunID ¶
func (m *MockQueueStore) DequeueByDAGRunID(ctx context.Context, name string, dagRun DAGRunRef) ([]QueuedItemData, error)
func (*MockQueueStore) DequeueByName ¶
func (m *MockQueueStore) DequeueByName(ctx context.Context, name string) (QueuedItemData, error)
func (*MockQueueStore) Enqueue ¶
func (m *MockQueueStore) Enqueue(ctx context.Context, name string, priority QueuePriority, dagRun DAGRunRef) error
func (*MockQueueStore) List ¶
func (m *MockQueueStore) List(ctx context.Context, name string) ([]QueuedItemData, error)
func (*MockQueueStore) ListByDAGName ¶
func (m *MockQueueStore) ListByDAGName(ctx context.Context, name, dagName string) ([]QueuedItemData, error)
func (*MockQueueStore) ListCursor ¶
func (m *MockQueueStore) ListCursor(ctx context.Context, name, cursor string, limit int) (CursorResult[QueuedItemData], error)
func (*MockQueueStore) QueueList ¶
func (m *MockQueueStore) QueueList(ctx context.Context) ([]string, error)
func (*MockQueueStore) QueueWatcher ¶
func (m *MockQueueStore) QueueWatcher(ctx context.Context) QueueWatcher
type NewDAGRunAttemptOptions ¶
type NewDAGRunAttemptOptions struct {
// RootDAGRun is the root dag-run reference for this attempt.
RootDAGRun *DAGRunRef
// Retry indicates whether this is a retry of a previous run.
Retry bool
// AttemptID is an optional attempt ID. If set, this ID is used instead of generating a new one.
// This is used when the coordinator has already created an attempt and wants the worker
// to use the same ID for consistency.
AttemptID string
}
NewDAGRunAttemptOptions contains options for creating a new run record
type Node ¶
type Node struct {
Step core.Step `json:"step,omitzero"`
Stdout string `json:"stdout"` // standard output log file path
Stderr string `json:"stderr"` // standard error log file path
WorkingDir string `json:"workingDir,omitempty"`
StartedAt string `json:"startedAt"`
FinishedAt string `json:"finishedAt"`
Status core.NodeStatus `json:"status"`
RetriedAt string `json:"retriedAt,omitempty"`
RetryCount int `json:"retryCount,omitempty"`
DoneCount int `json:"doneCount,omitempty"`
Repeated bool `json:"repeated,omitempty"` // indicates if the node has been repeated
SkippedByRetry bool `json:"skippedByRetry,omitempty"`
Error string `json:"error,omitempty"`
SubRuns []SubDAGRun `json:"children,omitempty"`
SubRunsRepeated []SubDAGRun `json:"childrenRepeated,omitempty"` // repeated sub DAG runs
OutputVariables *collections.SyncMap `json:"outputVariables,omitempty"`
OutputValue *string `json:"outputValue,omitempty"`
OutputsValue *string `json:"outputsValue,omitempty"`
StepOutputsValue *string `json:"stepOutputsValue,omitempty"`
HumanTaskInput json.RawMessage `json:"humanTaskInput,omitempty"`
// ControllerState stores the goal progress of a controller DAG's controller
// step, so a suspended run resumes with its task list intact.
ControllerState json.RawMessage `json:"controllerState,omitempty"`
// HumanTaskCompletedBy records who completed this human task.
HumanTaskCompletedBy string `json:"humanTaskCompletedBy,omitempty"`
// HumanTaskCompletedByID records the subject ID that completed this human task.
HumanTaskCompletedByID string `json:"humanTaskCompletedById,omitempty"`
// ApprovedAt records when this wait step was approved
ApprovedAt string `json:"approvedAt,omitempty"`
// ApprovalInputs stores key-value parameters provided during approval
ApprovalInputs map[string]string `json:"approvalInputs,omitempty"`
// ApprovedBy records who approved this wait step (username)
ApprovedBy string `json:"approvedBy,omitempty"`
// ApprovedByID records the subject ID that approved this wait step.
ApprovedByID string `json:"approvedById,omitempty"`
// RejectedAt records when this wait step was rejected
RejectedAt string `json:"rejectedAt,omitempty"`
// RejectedBy records who rejected this wait step (username)
RejectedBy string `json:"rejectedBy,omitempty"`
// RejectedByID records the subject ID that rejected this wait step.
RejectedByID string `json:"rejectedById,omitempty"`
// RejectionReason stores the optional reason for rejection
RejectionReason string `json:"rejectionReason,omitempty"`
// ApprovalIteration tracks how many times this step has been pushed back.
ApprovalIteration int `json:"approvalIteration,omitempty"`
// PushBackInputs stores the inputs from the last push-back.
// These are injected as environment variables when the step re-executes.
PushBackInputs map[string]string `json:"pushBackInputs,omitempty"`
// PushBackHistory stores the chronological push-back inputs for this step.
PushBackHistory []PushBackEntry `json:"pushBackHistory,omitempty"`
// PushBackPreviousStdout stores the stdout log path from the execution that
// was reset by the latest push-back.
PushBackPreviousStdout string `json:"pushBackPreviousStdout,omitempty"`
// ChatMessages stores the session messages for chat/LLM steps.
// This field is populated during execution and synced via status updates
// from workers.
ChatMessages []LLMMessage `json:"chatMessages,omitempty"`
// ToolDefinitions stores the tool definitions that were available to the LLM.
// This enables debugging visibility into what tools and schemas were sent.
ToolDefinitions []ToolDefinition `json:"toolDefinitions,omitempty"`
}
Node represents a DAG step with its execution state for persistence
func NewNodeFromStep ¶
NewNodeFromStep creates a new Node with default status values for the given step.
func NewNodeOrNil ¶
NewNodeOrNil creates a Node from a Step or returns nil if the step is nil.
func NewNodesFromSteps ¶
NewNodesFromSteps converts a list of DAG steps to persistence Node objects.
type OutputsMetadata ¶
type OutputsMetadata struct {
DAGName string `json:"dagName"`
DAGRunID string `json:"dagRunId"`
AttemptID string `json:"attemptId"`
Status string `json:"status"`
CompletedAt string `json:"completedAt"`
Params string `json:"params,omitempty"` // JSON-serialized parameters
}
OutputsMetadata contains execution context for the outputs.
type PaginatedResult ¶
type PaginatedResult[T any] struct { Items []T CurrentPage int TotalPages int TotalCount int Offset int HasNextPage bool HasPrevPage bool NextPage int PrevPage int }
func NewPaginatedResult ¶
func NewPaginatedResult[T any](items []T, total int, pg Paginator) PaginatedResult[T]
func (PaginatedResult[T]) Data ¶
func (r PaginatedResult[T]) Data() []T
func (PaginatedResult[T]) PageRange ¶
func (r PaginatedResult[T]) PageRange(size int) PageRange
func (PaginatedResult[T]) RangeEnd ¶
func (r PaginatedResult[T]) RangeEnd() int
func (PaginatedResult[T]) RangeStart ¶
func (r PaginatedResult[T]) RangeStart() int
type Paginator ¶
type Paginator struct {
// contains filtered or unexported fields
}
func DefaultPaginator ¶
func DefaultPaginator() Paginator
func NewPaginator ¶
type PendingStepRetry ¶
type PendingStepRetry struct {
StepName string `json:"stepName"`
Interval time.Duration `json:"interval"`
}
SubDAGRunStatus is an interface that represents the status of a sub dag-run.
func PendingStepRetriesFromNodes ¶
func PendingStepRetriesFromNodes(nodes []*Node) []PendingStepRetry
PendingStepRetriesFromNodes extracts pending parent-managed step retries from a DAG status snapshot.
func PendingStepRetriesFromStatus ¶
func PendingStepRetriesFromStatus(status *DAGRunStatus) []PendingStepRetry
PendingStepRetriesFromStatus returns the persisted pending step retries when present and falls back to deriving them from node state for older statuses that predate the field.
func (PendingStepRetry) MarshalJSON ¶
func (p PendingStepRetry) MarshalJSON() ([]byte, error)
MarshalJSON emits Interval as a Go duration string while keeping the surrounding shape stable for callers.
func (*PendingStepRetry) UnmarshalJSON ¶
func (p *PendingStepRetry) UnmarshalJSON(data []byte) error
UnmarshalJSON accepts both the current string encoding and the legacy numeric nanosecond encoding for backward compatibility with persisted data.
type ProcEntry ¶
type ProcEntry struct {
GroupName string
Identity ProcEntryID
Meta ProcMeta
LastHeartbeatAt int64
Fresh bool
}
ProcEntry represents a storage-independent proc heartbeat observation.
func (ProcEntry) AttemptKey ¶
AttemptKey returns a stable identifier for the exact proc-backed attempt.
func (ProcEntry) RunScopeKey ¶
RunScopeKey returns a stable identifier for the DAG-run scope across attempts.
type ProcEntryID ¶
type ProcEntryID struct {
// contains filtered or unexported fields
}
ProcEntryID is an opaque identity returned by ProcStore for exact stale-entry removal. Callers must not interpret it as a filesystem path or record key.
func NewProcEntryID ¶
func NewProcEntryID(token string) ProcEntryID
NewProcEntryID creates an opaque proc entry identity token.
func (ProcEntryID) IsZero ¶
func (id ProcEntryID) IsZero() bool
IsZero reports whether the identity is empty.
func (ProcEntryID) String ¶
func (id ProcEntryID) String() string
String returns an opaque display token. Callers must not parse it.
type ProcHandle ¶
type ProcHandle interface {
// Stop stops the heartbeat for the process.
Stop(ctx context.Context) error
// GetMeta retrieves the metadata for the process.
GetMeta() ProcMeta
}
ProcHandle represents a process that is associated with a dag-run.
type ProcHeartbeat ¶
type ProcHeartbeat struct {
GroupName string
DAGRun DAGRunRef
AttemptID string
StartedAt int64
LastHeartbeatAt int64
ObservedAt time.Time
Fresh bool
}
ProcHeartbeat is a storage-independent observation of a proc heartbeat.
func (ProcHeartbeat) AdvancedSince ¶
func (h ProcHeartbeat) AdvancedSince(previous ProcHeartbeat) bool
AdvancedSince reports whether this observation is newer than previous.
type ProcMeta ¶
type ProcMeta struct {
StartedAt int64
Name string
DAGRunID string
AttemptID string
RootName string
RootDAGRunID string
}
ProcMeta is a struct that holds metadata for a process.
type ProcStore ¶
type ProcStore interface {
// Lock try to lock process group return error if it's held by another process
Lock(ctx context.Context, groupName string) error
// UnLock unlocks process group
Unlock(ctx context.Context, groupName string)
// Acquire creates a new process entry for a given group name and execution metadata.
// It automatically starts the heartbeat for the process.
Acquire(ctx context.Context, groupName string, meta ProcMeta) (ProcHandle, error)
// CountAlive retrieves the number of processes associated with a group name.
CountAlive(ctx context.Context, groupName string) (int, error)
// CountAlive retrieves the number of processes associated with a group name.
CountAliveByDAGName(ctx context.Context, groupName, dagName string) (int, error)
// IsRunAlive checks if a specific DAG run is currently alive.
IsRunAlive(ctx context.Context, groupName string, dagRun DAGRunRef) (bool, error)
// IsAttemptAlive checks if a specific DAG-run attempt is currently alive.
IsAttemptAlive(ctx context.Context, groupName string, dagRun DAGRunRef, attemptID string) (bool, error)
// ListAlive returns list of running DAG runs by the group name.
ListAlive(ctx context.Context, groupName string) ([]DAGRunRef, error)
// ListAllAlive returns all running DAG runs across all groups.
// Returns a map where key is the group name and value is list of DAG runs.
ListAllAlive(ctx context.Context) (map[string][]DAGRunRef, error)
// ListEntries returns all proc entries for a group, including stale entries.
ListEntries(ctx context.Context, groupName string) ([]ProcEntry, error)
// LatestFreshEntryByDAGName returns the freshest proc entry for the DAG in the group.
LatestFreshEntryByDAGName(ctx context.Context, groupName, dagName string) (*ProcEntry, error)
// LatestHeartbeat returns the latest heartbeat observation for a DAG-run in the group.
LatestHeartbeat(ctx context.Context, groupName string, dagRun DAGRunRef) (*ProcHeartbeat, error)
// ListAllEntries returns all proc entries across all groups, including stale entries.
ListAllEntries(ctx context.Context) ([]ProcEntry, error)
// RemoveIfStale removes the exact proc entry if it is still stale and unchanged.
RemoveIfStale(ctx context.Context, entry ProcEntry) error
}
ProcStore is an interface for managing process storage.
type PushBackEntry ¶
type PushBackEntry struct {
Iteration int `json:"iteration"`
By string `json:"by,omitempty"`
ByID string `json:"byId,omitempty"`
At string `json:"at,omitempty"`
Inputs map[string]string `json:"inputs,omitempty"`
}
PushBackEntry records one push-back event for a step approval cycle.
func ClonePushBackHistory ¶
func ClonePushBackHistory(src []PushBackEntry) []PushBackEntry
ClonePushBackHistory returns a deep copy of push-back history entries.
func NormalizePushBackHistory ¶
func NormalizePushBackHistory( allowed []string, iteration int, latestInputs map[string]string, history []PushBackEntry, ) []PushBackEntry
NormalizePushBackHistory ensures stored history is filtered and seeded from legacy state when only the latest iteration/input pair is available.
type QueuePriority ¶
type QueuePriority int
QueuePriority represents the priority of a queued item
const ( QueuePriorityHigh QueuePriority = iota QueuePriorityLow )
type QueueStore ¶
type QueueStore interface {
// Enqueue adds an item to the queue
Enqueue(ctx context.Context, name string, priority QueuePriority, dagRun DAGRunRef) error
// DequeueByName retrieves an item from the queue and removes it
DequeueByName(ctx context.Context, name string) (QueuedItemData, error)
// DequeueByDAGRunID retrieves items from the queue by dag-run reference and removes them
DequeueByDAGRunID(ctx context.Context, name string, dagRun DAGRunRef) ([]QueuedItemData, error)
// DeleteByItemIDs removes the exact queued items identified by their queue item IDs.
DeleteByItemIDs(ctx context.Context, name string, itemIDs []string) (int, error)
// Len returns the number of items in the queue
Len(ctx context.Context, name string) (int, error)
// List returns all items in the queue with the given name
List(ctx context.Context, name string) ([]QueuedItemData, error)
// ListCursor returns one forward-only page of queued items for a specific queue.
ListCursor(ctx context.Context, name, cursor string, limit int) (CursorResult[QueuedItemData], error)
// All returns all items in the queue
All(ctx context.Context) ([]QueuedItemData, error)
// ListByDAGName returns all items that has a specific DAG name
ListByDAGName(ctx context.Context, name, dagName string) ([]QueuedItemData, error)
// QueueList lists all queue names that have at least one item in the queue
QueueList(ctx context.Context) ([]string, error)
// Watcher returns a QueueWatcher for the queue data
QueueWatcher(ctx context.Context) QueueWatcher
}
QueueStore provides an interface for interacting with the underlying database for storing and retrieving queued dag-run items.
type QueueWatcher ¶
type QueueWatcher interface {
// Start start swatching queue data and signal when a queue state changed
Start(ctx context.Context) (<-chan struct{}, error)
// Stop stops watching queue data
Stop(ctx context.Context)
}
QueueWatcher watches the queue state
type QueuedItem ¶
type QueuedItem struct {
QueuedItemData
}
QueuedItem is a wrapper for QueuedItemData
func NewQueuedItem ¶
func NewQueuedItem(data QueuedItemData) *QueuedItem
NewQueuedItem creates a new QueuedItem
type QueuedItemData ¶
type QueuedItemData interface {
// ID returns the ID of the queued item
ID() string
// Data returns the data of the queued item
Data() (*DAGRunRef, error)
}
QueuedItemData represents a dag-run reference that is queued for execution.
type RemoveDAGRunOption ¶
type RemoveDAGRunOption func(*RemoveDAGRunOptions)
RemoveDAGRunOption is a functional option for configuring RemoveDAGRunOptions.
func WithRejectActiveDAGRun ¶
func WithRejectActiveDAGRun() RemoveDAGRunOption
WithRejectActiveDAGRun refuses to remove dag-runs that are still active.
type RemoveDAGRunOptions ¶
type RemoveDAGRunOptions struct {
// RejectActive if true, refuses to remove dag-runs with an active status.
RejectActive bool
}
RemoveDAGRunOptions contains options for removing a dag-run.
type RemoveOldDAGRunsOption ¶
type RemoveOldDAGRunsOption func(*RemoveOldDAGRunsOptions)
RemoveOldDAGRunsOption is a functional option for configuring RemoveOldDAGRunsOptions
func WithDryRun ¶
func WithDryRun() RemoveOldDAGRunsOption
WithDryRun sets the dry-run mode for removing old dag-runs
func WithOlderThan ¶ added in v2.11.3
func WithOlderThan(t time.Time) RemoveOldDAGRunsOption
WithOlderThan deletes dag-runs older than the given cutoff time. A zero cutoff removes no dag-runs. When set, the retentionDays argument to RemoveOldDAGRuns is ignored.
func WithRetentionRuns ¶
func WithRetentionRuns(runs int) RemoveOldDAGRunsOption
WithRetentionRuns keeps the most recent number of dag-runs.
type RemoveOldDAGRunsOptions ¶
type RemoveOldDAGRunsOptions struct {
// DryRun if true, only returns the paths that would be removed without actually deleting
DryRun bool
// RetentionRuns keeps the most recent number of dag-runs when set.
RetentionRuns *int
// OlderThan when set, deletes dag-runs whose recorded time is strictly before this
// cutoff. When set, the retentionDays argument is ignored.
OlderThan *time.Time
}
RemoveOldDAGRunsOptions contains options for removing old dag-runs
type RetryPath ¶
RetryPath identifies a step in a persisted child DAG run.
func ParseRetryPath ¶
ParseRetryPath parses an internal retry path.
type RunStatus ¶
type RunStatus struct {
// Name represents the name of the executed DAG.
Name string
// DAGRunID is the ID of the dag-run.
DAGRunID string
// Params is the parameters of the DAG.
Params string
// Outputs is the outputs of the dag-run.
Outputs map[string]string
// OutputValues contains typed outputs published through stdout.outputs or outputs.write.
OutputValues map[string]any
// Status is the execution status of the dag-run.
Status core.Status
// PendingStepRetries contains any step retries that are waiting to be scheduled
// by the parent executor.
PendingStepRetries []PendingStepRetry
}
func (*RunStatus) MarshalJSON ¶
MarshalJSON implements the json.Marshaler interface for RunStatus.
type RunningTask ¶
type RunningTask struct {
DAGRunID string `json:"dagRunId,omitempty"`
DAGName string `json:"dagName,omitempty"`
StartedAt int64 `json:"startedAt,omitempty"`
RootDAGRunName string `json:"rootDagRunName,omitempty"`
RootDAGRunID string `json:"rootDagRunId,omitempty"`
ParentDAGRunName string `json:"parentDagRunName,omitempty"`
ParentDAGRunID string `json:"parentDagRunId,omitempty"`
AttemptKey string `json:"attemptKey,omitempty"`
}
RunningTask describes one task currently executing on a worker.
type RuntimeProfileEntry ¶
type RuntimeProfileEntry struct {
// Key is the injected environment variable name.
Key string `json:"key"`
// Kind is the profile entry type, such as variable or secret.
Kind string `json:"kind"`
}
RuntimeProfileEntry is non-secret metadata about a profile key injected into a run.
type SearchDAGMatchesOptions ¶
type SearchDAGMatchesOptions struct {
Cursor string
Limit int
Query string
Labels []string
WorkspaceFilter *WorkspaceFilter
}
SearchDAGMatchesOptions contains parameters for cursor-based snippet loading.
type SearchDAGResult ¶
type SearchDAGResult struct {
Name string
FileName string
Workspace string
Matches []*Match
HasMoreMatches bool
NextMatchesCursor string
}
SearchDAGResult represents a lightweight DAG search hit for paginated UIs.
type SearchDAGsOptions ¶
type SearchDAGsOptions struct {
Cursor string
Limit int
Query string
MatchLimit int
Labels []string
WorkspaceFilter *WorkspaceFilter
}
SearchDAGsOptions contains parameters for cursor-based DAG search.
type ServiceName ¶
type ServiceName string
ServiceName represents the name of a service in the service registry system
const ( // ServiceNameCoordinator is the name of the coordinator service ServiceNameCoordinator ServiceName = "coordinator" // ServiceNameScheduler is the name of the scheduler service ServiceNameScheduler ServiceName = "scheduler" )
type ServiceRegistry ¶
type ServiceRegistry interface {
// Register registers services for the given service name and host info.
// It returns an error if the registry failed to start heartbeat.
Register(ctx context.Context, serviceName ServiceName, hostInfo HostInfo) error
// Unregister un-registers current service.
Unregister(ctx context.Context)
// GetServiceMembers returns the list of active hosts for the given service.
// This method combines service resolution and member lookup.
GetServiceMembers(ctx context.Context, serviceName ServiceName) ([]HostInfo, error)
// UpdateStatus updates the status of the current registered instance
UpdateStatus(ctx context.Context, serviceName ServiceName, status ServiceStatus) error
}
ServiceRegistry is responsible for registering and persisting running service information.
type ServiceStatus ¶
type ServiceStatus int
ServiceStatus represents the operational status of a service instance
const ( // ServiceStatusUnknown indicates unknown status ServiceStatusUnknown ServiceStatus = iota // ServiceStatusActive indicates the service is active (e.g., scheduler holds lock) ServiceStatusActive // ServiceStatusInactive indicates the service is inactive (e.g., scheduler waiting for lock) ServiceStatusInactive )
func (ServiceStatus) String ¶
func (s ServiceStatus) String() string
String returns the string representation of the service status
type StaleQueueDispatchError ¶
type StaleQueueDispatchError struct {
Reason string
}
StaleQueueDispatchError reports that a scheduler-owned queued dispatch is no longer valid for the latest visible attempt.
func ParseStaleQueueDispatchError ¶
func ParseStaleQueueDispatchError(msg string) (*StaleQueueDispatchError, bool)
ParseStaleQueueDispatchError reconstructs a stale queue-dispatch error from a transport-safe string representation.
func (*StaleQueueDispatchError) Error ¶
func (e *StaleQueueDispatchError) Error() string
type SubDAGRun ¶
type SubDAGRun struct {
DAGRunID string `json:"dagRunId,omitempty"`
Params string `json:"params,omitempty"`
// DAGName is the name of the executed sub-DAG.
// For chat tool calls, this is the tool DAG name.
// This field enables UI drill-down when step.call is not set.
DAGName string `json:"dagName,omitempty"`
}
SubDAGRun represents a sub DAG run associated with a node
type ToolCall ¶
type ToolCall struct {
// ID is a unique identifier for this tool call.
ID string `json:"id"`
// Type is always "function" for function calls.
Type string `json:"type"`
// Function contains the function call details.
Function ToolCallFunction `json:"function"`
}
ToolCall represents an LLM's request to call a tool. This mirrors llmpkg.ToolCall for use in exec layer.
type ToolCallFunction ¶
type ToolCallFunction struct {
// Name is the name of the function to call.
Name string `json:"name"`
// Arguments is a JSON string containing the function arguments.
Arguments string `json:"arguments"`
}
ToolCallFunction contains the details of a function call.
type ToolDefinition ¶
type ToolDefinition struct {
// Name is the tool/function name as presented to the LLM.
Name string `json:"name"`
// Description describes what the tool does.
Description string `json:"description,omitempty"`
// Parameters is the JSON Schema describing the tool's parameters.
Parameters map[string]any `json:"parameters,omitempty"`
}
ToolDefinition represents a tool that was available to the LLM. This is stored alongside messages to provide visibility into what tool definitions were sent to the LLM during execution.
type WorkerHeartbeatRecord ¶
type WorkerHeartbeatRecord struct {
WorkerID string `json:"workerId"`
Labels map[string]string `json:"labels,omitempty"`
Stats *WorkerStats `json:"stats,omitempty"`
LastHeartbeatAt int64 `json:"lastHeartbeatAt"`
}
WorkerHeartbeatRecord is the shared presence record for a worker.
func (WorkerHeartbeatRecord) LastHeartbeatTime ¶
func (r WorkerHeartbeatRecord) LastHeartbeatTime() time.Time
LastHeartbeatTime returns the last heartbeat as a time.
type WorkerHeartbeatStore ¶
type WorkerHeartbeatStore interface {
Upsert(ctx context.Context, record WorkerHeartbeatRecord) error
Get(ctx context.Context, workerID string) (*WorkerHeartbeatRecord, error)
List(ctx context.Context) ([]WorkerHeartbeatRecord, error)
DeleteStale(ctx context.Context, before time.Time) (int, error)
}
WorkerHeartbeatStore persists shared worker presence across coordinators.
type WorkerStats ¶
type WorkerStats struct {
TotalPollers int32 `json:"totalPollers,omitempty"`
BusyPollers int32 `json:"busyPollers,omitempty"`
RunningTasks []*RunningTask `json:"runningTasks,omitempty"`
}
WorkerStats describes worker poller capacity and running distributed tasks.
type WorkspaceFilter ¶
WorkspaceFilter restricts list/search results to allowed workspaces.
func (*WorkspaceFilter) MatchesLabels ¶
func (f *WorkspaceFilter) MatchesLabels(labels core.Labels) bool
MatchesLabels reports whether labels are visible under the filter.
type WorkspaceLabelState ¶
type WorkspaceLabelState int
WorkspaceLabelState describes whether labels contain a usable workspace.
const ( // WorkspaceLabelMissing means no workspace label is present. WorkspaceLabelMissing WorkspaceLabelState = iota // WorkspaceLabelValid means exactly one valid workspace label value is present. WorkspaceLabelValid // WorkspaceLabelInvalid means a workspace label is present but malformed or ambiguous. WorkspaceLabelInvalid )
func WorkspaceLabelFromLabels ¶
func WorkspaceLabelFromLabels(labels core.Labels) (string, WorkspaceLabelState)
Source Files
¶
- context.go
- context_env.go
- dag.go
- dagrun.go
- dagrun_attempt.go
- dispatch.go
- distributed.go
- enqueue_retry.go
- env.go
- failed_auto_retry_cancel.go
- messages.go
- node.go
- noop_attempt.go
- outputs.go
- paginator.go
- proc.go
- pushback.go
- queue.go
- queue_abort.go
- queued_status.go
- registry.go
- retry_path.go
- runid.go
- runstatus.go
- search_cursor.go
- stale_queue_dispatch.go
- utc.go
- workspace.go