core

package
v1.30.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 4, 2026 License: GPL-3.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// TypeGraph is the default execution type using dependency-based execution
	TypeGraph = "graph"
	// TypeChain executes steps sequentially in the order they are defined
	TypeChain = "chain"
	// TypeAgent is reserved for future agent-based execution
	TypeAgent = "agent"
)

Execution type constants

View Source
const (
	// ExecutorTypeDAG is the executor type for a sub DAG.
	ExecutorTypeDAG = "dag"

	// ExecutorTypeParallel is the executor type for parallel steps.
	ExecutorTypeParallel = "parallel"

	// ExecutorTypeHITL is the executor type for HITL (Human In The Loop) steps.
	ExecutorTypeHITL = "hitl"
)
View Source
const DAGNameMaxLen = 40

DAGNameMaxLen defines the maximum allowed length for a DAG name.

View Source
const DefaultMaxConcurrent = 10

DefaultMaxConcurrent is the default maximum concurrent executions for parallel steps

View Source
const ExecutorTypeChat = "chat"

ExecutorTypeChat is the executor type for chat steps.

View Source
const ParallelItemVariable = "ITEM"

ParallelItemVariable is the special variable name that represents the current item in parallel execution

Variables

View Source
var (
	ErrNameTooLong                         = errors.New("name must be less than 40 characters")
	ErrNameInvalidChars                    = errors.New("name must only contain alphanumeric characters, dashes, dots, and underscores")
	ErrInvalidSchedule                     = errors.New("invalid schedule")
	ErrScheduleMustBeStringOrArray         = errors.New("schedule must be a string or an array of strings")
	ErrInvalidScheduleType                 = errors.New("invalid schedule type")
	ErrInvalidKeyType                      = errors.New("invalid key type")
	ErrExecutorConfigMustBeString          = errors.New("executor config key must be string")
	ErrDuplicateFunction                   = errors.New("duplicate function")
	ErrFuncParamsMismatch                  = errors.New("func params and args given to func command do not match")
	ErrInvalidStepData                     = errors.New("invalid step data")
	ErrStepNameRequired                    = errors.New("step name must be specified")
	ErrStepNameDuplicate                   = errors.New("step name must be unique")
	ErrStepNameTooLong                     = errors.New("step name must be less than 40 characters")
	ErrStepCommandIsRequired               = errors.New("step command is required")
	ErrStepCommandIsEmpty                  = errors.New("step command is empty")
	ErrStepCommandMustBeArrayOrString      = errors.New("step command must be an array of strings or a string")
	ErrInvalidParamValue                   = errors.New("invalid parameter value")
	ErrCallFunctionNotFound                = errors.New("call must specify a functions that exists")
	ErrNumberOfParamsMismatch              = errors.New("the number of parameters defined in the function does not match the number of parameters given")
	ErrRequiredParameterNotFound           = errors.New("required parameter not found")
	ErrScheduleKeyMustBeString             = errors.New("schedule key must be a string")
	ErrInvalidSignal                       = errors.New("invalid signal")
	ErrInvalidEnvValue                     = errors.New("env config should be map of strings or array of key=value formatted string")
	ErrArgsMustBeConvertibleToIntOrString  = errors.New("args must be convertible to either int or string")
	ErrExecutorTypeMustBeString            = errors.New("executor.type value must be string")
	ErrExecutorConfigValueMustBeMap        = errors.New("executor.config value must be a map")
	ErrExecutorHasInvalidKey               = errors.New("executor has invalid key")
	ErrExecutorConfigMustBeStringOrMap     = errors.New("executor config must be string or map")
	ErrDotEnvMustBeStringOrArray           = errors.New("dotenv must be a string or an array of strings")
	ErrPreconditionMustBeArrayOrString     = errors.New("precondition must be a string or an array of strings")
	ErrPreconditionValueMustBeString       = errors.New("precondition value must be a string")
	ErrPreconditionHasInvalidKey           = errors.New("precondition has invalid key")
	ErrContinueOnOutputMustBeStringOrArray = errors.New("continueOn.Output must be a string or an array of strings")
	ErrContinueOnExitCodeMustBeIntOrArray  = errors.New("continueOn.ExitCode must be an int or an array of ints")
	ErrDependsMustBeStringOrArray          = errors.New("depends must be a string or an array of strings")
	ErrStepsMustBeArrayOrMap               = errors.New("steps must be an array or a map")
)

errors on building a DAG.

Functions

func InitializeDefaults

func InitializeDefaults(d *DAG)

InitializeDefaults exposes initializeDefaults for packages that prepare DAGs before execution.

func NewValidationError

func NewValidationError(field string, value any, err error) error

NewValidationError wraps an error with field context so other packages can build LoadError instances.

func RegisterExecutorCapabilities added in v1.29.0

func RegisterExecutorCapabilities(executorType string, caps ExecutorCapabilities)

RegisterExecutorCapabilities registers capabilities for an executor type.

func RegisterStepValidator

func RegisterStepValidator(executorType string, validator StepValidator)

func SockAddr

func SockAddr(name, dagRunID string) string

SockAddr returns the unix socket address for the DAG. The address is used to communicate with the agent process.

func SupportsCommand added in v1.29.0

func SupportsCommand(executorType string) bool

SupportsCommand returns whether the executor type supports the command field.

func SupportsContainer added in v1.29.0

func SupportsContainer(executorType string) bool

SupportsContainer returns whether the executor type supports step-level container config.

func SupportsLLM added in v1.30.0

func SupportsLLM(executorType string) bool

SupportsLLM returns whether the executor type supports the llm field.

func SupportsMultipleCommands added in v1.29.0

func SupportsMultipleCommands(executorType string) bool

SupportsMultipleCommands returns whether the executor type supports multiple commands.

func SupportsScript added in v1.29.0

func SupportsScript(executorType string) bool

SupportsScript returns whether the executor type supports the script field.

func SupportsShell added in v1.29.0

func SupportsShell(executorType string) bool

SupportsShell returns whether the executor type uses shell configuration.

func SupportsSubDAG added in v1.29.0

func SupportsSubDAG(executorType string) bool

SupportsSubDAG returns whether the executor type can execute sub-DAGs.

func SupportsWorkerSelector added in v1.29.0

func SupportsWorkerSelector(executorType string) bool

SupportsWorkerSelector returns whether the executor type supports worker selection.

func ValidateDAGName

func ValidateDAGName(name string) error

ValidateDAGName validates a DAG name according to shared rules. - Empty name is allowed (caller may provide one via context or filename). - Non-empty name must satisfy length and allowed character constraints.

func ValidateSteps

func ValidateSteps(dag *DAG) error

ValidateSteps exposes validateSteps for packages that need to perform validation during DAG construction.

Types

type AuthConfig

type AuthConfig struct {
	// Username for registry authentication
	Username string `json:"username,omitempty"`
	// Password for registry authentication
	Password string `json:"password,omitempty"`
	// Auth can be used instead of username/password for pre-encoded credentials
	// This should be base64(username:password)
	Auth string `json:"auth,omitempty"`
}

AuthConfig represents Docker registry authentication configuration. This is a simplified structure for user convenience that will be converted to Docker's registry.AuthConfig format when needed.

type CommandEntry added in v1.29.0

type CommandEntry struct {
	// Command is the executable name or path.
	Command string `json:"command"`
	// Args contains the arguments for the command.
	Args []string `json:"args,omitempty"`
	// CmdWithArgs is the original command string for display purposes.
	CmdWithArgs string `json:"cmdWithArgs,omitempty"`
}

CommandEntry represents a single command in a multi-command step. Each entry contains a parsed command with its arguments.

func (CommandEntry) String added in v1.29.0

func (c CommandEntry) String() string

String returns a display string for the command entry.

type Condition

type Condition struct {
	Condition string // Condition to evaluate
	Expected  string // Expected value
	Negate    bool   // Negate the condition result (run when condition does NOT match)
	// contains filtered or unexported fields
}

Condition contains a condition and the expected value. Conditions are evaluated and compared to the expected value. The condition can be a command substitution or an environment variable. The expected value must be a string without any substitutions.

func (*Condition) GetErrorMessage

func (c *Condition) GetErrorMessage() string

func (*Condition) MarshalJSON

func (c *Condition) MarshalJSON() ([]byte, error)

func (*Condition) SetErrorMessage

func (c *Condition) SetErrorMessage(msg string)

func (*Condition) UnmarshalJSON

func (c *Condition) UnmarshalJSON(data []byte) error

func (*Condition) Validate

func (c *Condition) Validate() error

type Container

type Container struct {
	// Exec specifies an existing container name to exec into.
	// Mutually exclusive with Image.
	Exec string `yaml:"exec,omitempty"`
	// Name is the container name to use. If empty, Docker generates a random name.
	Name string `yaml:"name,omitempty"`
	// Image is the container image to use.
	Image string `yaml:"image,omitempty"`
	// PullPolicy is the policy to pull the image (e.g., "Always", "IfNotPresent").
	PullPolicy PullPolicy `yaml:"pullPolicy,omitempty"`
	// Env specifies environment variables for the container.
	Env []string `yaml:"env,omitempty"` // List of environment variables in "key=value" format
	// Volumes specifies the volumes to mount in the container.
	Volumes []string `yaml:"volumes,omitempty"` // Map of volume names to volume definitions
	// User is the user to run the container as.
	User string `yaml:"user,omitempty"` // User to run the container as
	// WorkingDir is the working directory inside the container.
	WorkingDir string `yaml:"workingDir,omitempty"` // Working directory inside the container
	// WorkDir is the working directory inside the container.
	// Deprecated: use workingDir instead
	WorkDir string `yaml:"workDir,omitempty"` // Working directory inside the container
	// Platform specifies the platform for the container (e.g., "linux/amd64").
	Platform string `yaml:"platform,omitempty"` // Platform for the container
	// Ports specifies the ports to expose from the container.
	Ports []string `yaml:"ports,omitempty"` // List of ports to expose
	// Network is the network configuration for the container.
	Network string `yaml:"network,omitempty"` // Network configuration for the container
	// KeepContainer is the flag to keep the container after the DAG run.
	KeepContainer bool `yaml:"keepContainer,omitempty"` // Keep the container after the DAG run
	// Startup determines how the DAG-level container starts up.
	// One of: "keepalive" (default), "entrypoint", "command".
	Startup ContainerStartup `yaml:"startup,omitempty"`
	// Command is used when Startup == "command".
	Command []string `yaml:"command,omitempty"`
	// WaitFor determines readiness gate before steps run: "running" (default) or "healthy".
	WaitFor ContainerWaitFor `yaml:"waitFor,omitempty"`
	// LogPattern optionally waits for a regex to appear in container logs before proceeding.
	LogPattern string `yaml:"logPattern,omitempty"`
	// RestartPolicy applies Docker restart policy for long-running containers ("no", "always", or "unless-stopped").
	RestartPolicy string `yaml:"restartPolicy,omitempty"`
}

Container defines the container configuration for the DAG.

func (Container) GetWorkingDir

func (ct Container) GetWorkingDir() string

GetWorkingDir returns workdir or working dir (backward compatibility)

func (Container) IsExecMode added in v1.30.0

func (ct Container) IsExecMode() bool

IsExecMode returns true if this container is configured to exec into an existing container

type ContainerStartup

type ContainerStartup string

ContainerStartup is an enum for DAG-level container startup modes.

const (
	StartupKeepalive  ContainerStartup = "keepalive"
	StartupEntrypoint ContainerStartup = "entrypoint"
	StartupCommand    ContainerStartup = "command"
)

type ContainerWaitFor

type ContainerWaitFor string

ContainerWaitFor is an enum for container readiness conditions.

const (
	WaitForRunning ContainerWaitFor = "running"
	WaitForHealthy ContainerWaitFor = "healthy"
)

type ContinueOn

type ContinueOn struct {
	Failure     bool     `json:"failure,omitempty"`     // Failure is the flag to continue to the next step on failure.
	Skipped     bool     `json:"skipped,omitempty"`     // Skipped is the flag to continue to the next step on skipped.
	ExitCode    []int    `json:"exitCode,omitempty"`    // ExitCode is the list of exit codes to continue to the next step.
	Output      []string `json:"output,omitempty"`      // Output is the list of output (stdout/stderr) to continue to the next step.
	MarkSuccess bool     `json:"markSuccess,omitempty"` // MarkSuccess is the flag to mark the step as success when the condition is met.
}

ContinueOn contains the conditions to continue on failure or skipped. Failure is the flag to continue to the next step on failure. Skipped is the flag to continue to the next step on skipped. A step can be skipped when the preconditions are not met. Then if the ContinueOn.Skip is set, the step will continue to the next step.

type DAG

type DAG struct {
	// WorkingDir is the working directory to run the DAG.
	// Default value is the directory of DAG file.
	// If the source is not a DAG file, current directory when it's created.
	WorkingDir string `json:"workingDir,omitempty"`
	// Location is the absolute path to the DAG file.
	// It is used to generate unix socket name and can be blank
	Location string `json:"location,omitempty"`
	// Group is the group name of the DAG. This is optional.
	Group string `json:"group,omitempty"`
	// Name is the name of the DAG. The default is the filename without the extension.
	Name string `json:"name,omitempty"`
	// Type is the execution type (graph, chain, or agent). Default is graph.
	Type string `json:"type,omitempty"`
	// Shell is the default shell to use for all steps in this DAG.
	// If not specified, the system default shell is used.
	// Can be overridden at the step level.
	Shell string `json:"shell,omitempty"`
	// ShellArgs contains additional arguments to pass to the shell.
	// These are populated when Shell is specified as a string with arguments
	// (e.g., "bash -e") or as an array (e.g., ["bash", "-e"]).
	ShellArgs []string `json:"shellArgs,omitempty"`
	// Dotenv is the path to the dotenv file. This is optional.
	Dotenv []string `json:"dotenv,omitempty"`
	// Tags contains the list of tags for the DAG. This is optional.
	Tags []string `json:"tags,omitempty"`
	// Description is the description of the DAG. This is optional.
	Description string `json:"description,omitempty"`
	// Schedule configuration for starting, stopping, and restarting the DAG.
	Schedule []Schedule `json:"schedule,omitempty"`
	// StopSchedule contains the cron expressions for stopping the DAG.
	StopSchedule []Schedule `json:"stopSchedule,omitempty"`
	// RestartSchedule contains the cron expressions for restarting the DAG.
	RestartSchedule []Schedule `json:"restartSchedule,omitempty"`
	// SkipIfSuccessful indicates whether to skip the DAG if it was successful previously.
	// E.g., when the DAG has already been executed manually before the scheduled time.
	SkipIfSuccessful bool `json:"skipIfSuccessful,omitempty"`
	// Env contains a list of environment variables to be set before running the DAG.
	Env []string `json:"env,omitempty"`
	// LogDir is the directory where the logs are stored.
	LogDir string `json:"logDir,omitempty"`
	// LogOutput specifies how stdout and stderr are handled in log files.
	// Can be "separate" (default) for separate .out and .err files,
	// or "merged" for a single combined .log file.
	LogOutput LogOutputMode `json:"logOutput,omitempty"`
	// DefaultParams contains the default parameters to be passed to the DAG.
	DefaultParams string `json:"defaultParams,omitempty"`
	// Params contains the list of parameters to be passed to the DAG.
	Params []string `json:"params,omitempty"`
	// ParamsJSON contains the JSON representation of the resolved parameters.
	// When params were supplied as JSON, the original payload is preserved.
	// Steps can consume this via the DAGU_PARAMS_JSON environment variable.
	ParamsJSON string `json:"paramsJSON,omitempty"`
	// Steps contains the list of steps in the DAG.
	Steps []Step `json:"steps,omitempty"`
	// HandlerOn contains the steps to be executed on different events.
	HandlerOn HandlerOn `json:"handlerOn,omitzero"`
	// Preconditions contains the conditions to be met before running the DAG.
	Preconditions []*Condition `json:"preconditions,omitempty"`
	// SMTP contains the SMTP configuration.
	SMTP *SMTPConfig `json:"smtp,omitempty"`
	// ErrorMail contains the mail configuration for errors.
	ErrorMail *MailConfig `json:"errorMail,omitempty"`
	// InfoMail contains the mail configuration for informational messages.
	InfoMail *MailConfig `json:"infoMail,omitempty"`
	// WaitMail contains the mail configuration for wait status.
	WaitMail *MailConfig `json:"waitMail,omitempty"`
	// MailOn contains the conditions to send mail.
	MailOn *MailOn `json:"mailOn,omitempty"`
	// Timeout specifies the maximum execution time of the DAG task.
	Timeout time.Duration `json:"timeout,omitempty"`
	// Delay is the delay before starting the DAG.
	Delay time.Duration `json:"delay,omitempty"`
	// RestartWait is the time to wait before restarting the DAG.
	RestartWait time.Duration `json:"restartWait,omitempty"`
	// MaxActiveSteps specifies the maximum concurrent steps to run in an execution.
	MaxActiveSteps int `json:"maxActiveSteps,omitempty"`
	// MaxActiveRuns specifies the maximum number of concurrent dag-runs.
	MaxActiveRuns int `json:"maxActiveRuns,omitempty"`
	// MaxCleanUpTime is the maximum time to wait for cleanup when the DAG is stopped.
	MaxCleanUpTime time.Duration `json:"maxCleanUpTime,omitempty"`
	// HistRetentionDays is the number of days to keep the history of dag-runs.
	HistRetentionDays int `json:"histRetentionDays,omitempty"`
	// Queue is the name of the queue to assign this DAG to.
	Queue string `json:"queue,omitempty"`
	// WorkerSelector defines labels required for worker selection in distributed execution.
	// If specified, the DAG will only run on workers with matching tag.
	WorkerSelector map[string]string `json:"workerSelector,omitempty"`
	// MaxOutputSize is the maximum size of step output to capture in bytes.
	// Default is 1MB. Output exceeding this will return an error.
	MaxOutputSize int `json:"maxOutputSize,omitempty"`
	// OTel contains the OpenTelemetry configuration for the DAG.
	OTel *OTelConfig `json:"otel,omitempty"`
	// BuildErrors contains any errors encountered while building the DAG.
	BuildErrors []error `json:"-"`
	// BuildWarnings contains non-fatal warnings detected while building the DAG.
	BuildWarnings []string `json:"-"`
	// LocalDAGs contains DAGs defined in the same file, keyed by DAG name
	LocalDAGs map[string]*DAG `json:"localDAGs,omitempty"`
	// YamlData contains the raw YAML data of the DAG.
	YamlData []byte `json:"yamlData,omitempty"`
	// Container contains the container definition for the DAG.
	Container *Container `json:"container,omitempty"`
	// RunConfig contains configuration for controlling user interactions during DAG runs.
	RunConfig *RunConfig `json:"runConfig,omitempty"`
	// RegistryAuths maps registry hostnames to authentication configs.
	// Optional: If not specified, falls back to DOCKER_AUTH_CONFIG or docker config.
	RegistryAuths map[string]*AuthConfig `json:"registryAuths,omitempty"`
	// SSH contains the default SSH configuration for the DAG.
	SSH *SSHConfig `json:"ssh,omitempty"`
	// LLM contains the default LLM configuration for the DAG.
	// Steps with type: chat inherit this configuration if they don't specify their own llm field.
	LLM *LLMConfig `json:"llm,omitempty"`
	// Secrets contains references to external secrets to be resolved at runtime.
	Secrets []SecretRef `json:"secrets,omitempty"`
}

DAG contains all information about a DAG.

func (*DAG) FileName

func (d *DAG) FileName() string

FileName returns the filename of the DAG without the extension.

func (*DAG) GetName

func (d *DAG) GetName() string

GetName returns the name of the DAG. If the name is not set, it returns the default name (filename without extension).

func (*DAG) HasHITLSteps added in v1.30.0

func (d *DAG) HasHITLSteps() bool

HasHITLSteps returns true if the DAG contains any HITL executor steps. DAGs with HITL steps cannot be dispatched to workers because HITL steps require local storage access for approval.

func (*DAG) HasTag

func (d *DAG) HasTag(tag string) bool

HasTag checks if the DAG has the given tag.

func (*DAG) LoadDotEnv

func (d *DAG) LoadDotEnv(ctx context.Context)

LoadDotEnv loads all dotenv files in order, with later files overriding earlier ones.

func (*DAG) NextRun

func (d *DAG) NextRun(now time.Time) time.Time

NextRun returns the next scheduled run time based on the DAG's schedules.

func (*DAG) ParamsMap

func (d *DAG) ParamsMap() map[string]string

ParamsMap returns the parameters as a map.

func (*DAG) ProcGroup

func (d *DAG) ProcGroup() string

ProcGroup returns the name of the process group for this DAG. The process group name is used to identify and manage related DAG executions.

Returns:

  • If Queue is set: returns the Queue value
  • If Queue is empty: returns the DAG name as the default

The process group name is used by the process store to:

  1. Manage heartbeat files for active DAG runs
  2. Enforce concurrency limits (max concurrent runs) across DAGs in the same group

This allows the scheduler to control how many DAGs can run simultaneously within the same process group.

func (*DAG) SockAddr

func (d *DAG) SockAddr(dagRunID string) string

SockAddr returns the unix socket address for the DAG. The address is used to communicate with the agent process.

func (*DAG) SockAddrForSubDAGRun added in v1.24.0

func (d *DAG) SockAddrForSubDAGRun(dagRunID string) string

SockAddrForSubDAGRun returns the unix socket address for a specific dag-run ID. This is used to control sub dag-runs.

func (*DAG) String

func (d *DAG) String() string

String implements the Stringer interface. String returns a formatted string representation of the DAG

func (*DAG) Validate

func (d *DAG) Validate() error

Validate performs basic validation of the DAG structure

type ErrorList

type ErrorList []error

ErrorList is just a list of errors. It is used to collect multiple errors in building a DAG.

func (ErrorList) Error

func (e ErrorList) Error() string

Error implements the error interface. It returns a string with all the errors separated by a semicolon.

func (*ErrorList) ToStringList

func (e *ErrorList) ToStringList() []string

ToStringList returns the list of errors as a slice of strings.

func (ErrorList) Unwrap

func (e ErrorList) Unwrap() []error

Unwrap implements the errors.Unwrap interface.

type ExecutorCapabilities added in v1.29.0

type ExecutorCapabilities struct {
	// Command indicates whether the executor supports the command field.
	Command bool
	// MultipleCommands indicates whether the executor supports multiple commands.
	MultipleCommands bool
	// Script indicates whether the executor supports the script field.
	Script bool
	// Shell indicates whether the executor uses shell/shellArgs/shellPackages.
	Shell bool
	// Container indicates whether the executor supports step-level container config.
	Container bool
	// SubDAG indicates whether the executor can execute sub-DAGs.
	SubDAG bool
	// WorkerSelector indicates whether the executor supports worker selection.
	WorkerSelector bool
	// LLM indicates whether the executor supports the llm field.
	LLM bool
	// GetEvalOptions returns eval options for command argument evaluation.
	// If nil, default evaluation is used.
	GetEvalOptions func(ctx context.Context, step Step) []cmdutil.EvalOption
}

ExecutorCapabilities defines what an executor can do.

type ExecutorConfig

type ExecutorConfig struct {
	// Type represents one of the registered executors.
	// See `executor.Register` in `internal/executor/executor.go`.
	Type   string         `json:"type,omitempty"`
	Config map[string]any `json:"config,omitempty"` // Config contains executor-specific configuration.
	// Metadata contains additional metadata for the executor that is not passed to the executor itself.
	// This is used internally for optimization purposes.
	Metadata map[string]any `json:"metadata,omitempty"`
}

ExecutorConfig contains the configuration for the executor.

func (ExecutorConfig) IsCommand

func (e ExecutorConfig) IsCommand() bool

IsCommand returns true if the executor is a command

type HandlerOn

type HandlerOn struct {
	Init    *Step `json:"init,omitempty"`
	Failure *Step `json:"failure,omitempty"`
	Success *Step `json:"success,omitempty"`
	Cancel  *Step `json:"cancel,omitempty"`
	Exit    *Step `json:"exit,omitempty"`
	Wait    *Step `json:"wait,omitempty"`
}

HandlerOn contains the steps to be executed on different events in the DAG.

type HandlerType

type HandlerType string

HandlerType is the type of the handler.

const (
	HandlerOnInit    HandlerType = "onInit"
	HandlerOnSuccess HandlerType = "onSuccess"
	HandlerOnFailure HandlerType = "onFailure"
	HandlerOnCancel  HandlerType = "onCancel"
	HandlerOnExit    HandlerType = "onExit"
	HandlerOnWait    HandlerType = "onWait"
)

func (HandlerType) String

func (h HandlerType) String() string

type LLMConfig added in v1.30.0

type LLMConfig struct {
	// Provider is the LLM provider (openai, anthropic, gemini, openrouter, local).
	Provider string `json:"provider,omitempty"`
	// Model is the model to use (e.g., gpt-4o, claude-sonnet-4-20250514).
	Model string `json:"model,omitempty"`
	// System is the default system prompt for conversations.
	System string `json:"system,omitempty"`
	// Temperature controls randomness (0.0-2.0).
	Temperature *float64 `json:"temperature,omitempty"`
	// MaxTokens is the maximum number of tokens to generate.
	MaxTokens *int `json:"maxTokens,omitempty"`
	// TopP is the nucleus sampling parameter.
	TopP *float64 `json:"topP,omitempty"`
	// BaseURL is a custom API endpoint.
	BaseURL string `json:"baseURL,omitempty"`
	// APIKeyName is the name of the environment variable containing the API key.
	// If not specified, the default environment variable for the provider is used.
	APIKeyName string `json:"apiKeyName,omitempty"`
	// Stream enables or disables streaming output.
	// Default is true.
	Stream *bool `json:"stream,omitempty"`
	// Thinking enables extended thinking/reasoning mode.
	// Provider-specific: Anthropic uses budget_tokens, OpenAI uses reasoning.effort,
	// Gemini uses thinkingLevel/thinkingBudget, OpenRouter normalizes across providers.
	Thinking *ThinkingConfig `json:"thinking,omitempty"`
}

LLMConfig contains the configuration for LLM-based executors (chat, agent, etc.).

func (*LLMConfig) StreamEnabled added in v1.30.0

func (c *LLMConfig) StreamEnabled() bool

StreamEnabled returns true if streaming is enabled. Default is true if Stream is nil.

type LLMMessage added in v1.30.0

type LLMMessage struct {
	// Role is the message role (system, user, assistant, tool).
	Role LLMRole `json:"role,omitempty"`
	// Content is the message content. Supports variable substitution with ${VAR}.
	Content string `json:"content,omitempty"`
}

LLMMessage represents a message in the LLM conversation.

type LLMRole added in v1.30.0

type LLMRole string

LLMRole represents the role of a message sender in a conversation.

const (
	LLMRoleSystem    LLMRole = "system"
	LLMRoleUser      LLMRole = "user"
	LLMRoleAssistant LLMRole = "assistant"
	LLMRoleTool      LLMRole = "tool"
)

LLM message role constants.

func ParseLLMRole added in v1.30.0

func ParseLLMRole(s string) (LLMRole, error)

ParseLLMRole validates and returns an LLMRole from a string. Returns error for invalid or empty role values.

type LogOutputMode added in v1.29.0

type LogOutputMode string

LogOutputMode represents the mode for log output handling. It determines how stdout and stderr are written to log files.

const (
	// LogOutputSeparate keeps stdout and stderr in separate files (.out and .err).
	// This is the default behavior for backward compatibility.
	LogOutputSeparate LogOutputMode = "separate"

	// LogOutputMerged combines stdout and stderr into a single log file (.log).
	// Both streams are interleaved in the order they are written.
	LogOutputMerged LogOutputMode = "merged"
)

func EffectiveLogOutput added in v1.29.0

func EffectiveLogOutput(dag *DAG, step *Step) LogOutputMode

EffectiveLogOutput returns the effective log output mode for a step. It resolves the inheritance chain: step-level overrides DAG-level, and if neither is set, returns the default (LogOutputSeparate).

type MailConfig

type MailConfig struct {
	From       string   `json:"from,omitempty"`
	To         []string `json:"to,omitempty"`
	Prefix     string   `json:"prefix,omitempty"`
	AttachLogs bool     `json:"attachLogs,omitempty"`
}

MailConfig contains the mail configuration.

type MailOn

type MailOn struct {
	Failure bool `json:"failure,omitempty"`
	Success bool `json:"success,omitempty"`
	Wait    bool `json:"wait,omitempty"`
}

MailOn contains the conditions to send mail.

type NodeStatus

type NodeStatus int

NodeStatus represents the canonical lifecycle phases for an individual node.

const (
	NodeNotStarted NodeStatus = iota
	NodeRunning
	NodeFailed
	NodeAborted
	NodeSucceeded
	NodeSkipped
	NodePartiallySucceeded
	NodeWaiting
	NodeRejected
)

func (NodeStatus) IsSuccess

func (s NodeStatus) IsSuccess() bool

IsSuccess checks if the node status indicates a successful execution.

func (NodeStatus) IsWaiting added in v1.30.0

func (s NodeStatus) IsWaiting() bool

IsWaiting checks if the node status is waiting for human approval.

func (NodeStatus) String

func (s NodeStatus) String() string

String returns the canonical lowercase token for the node lifecycle phase.

type OTelConfig

type OTelConfig struct {
	Enabled  bool              `json:"enabled,omitempty"`
	Endpoint string            `json:"endpoint,omitempty"`
	Headers  map[string]string `json:"headers,omitempty"`
	Insecure bool              `json:"insecure,omitempty"`
	Timeout  time.Duration     `json:"timeout,omitempty"`
	Resource map[string]any    `json:"resource,omitempty"`
}

OTelConfig contains the OpenTelemetry configuration.

type ParallelConfig

type ParallelConfig struct {
	// Variable is the name of a variable that contains the json array of items to process in parallel.
	Variable string `json:"variable,omitempty"`

	// Items is the array of items to process in parallel.
	// Can be a direct array or a reference to a variable containing an array.
	Items []ParallelItem `json:"items,omitempty"`

	// MaxConcurrent is the maximum number of parallel executions.
	// Default is 10 if not specified.
	MaxConcurrent int `json:"maxConcurrent,omitempty"`
}

ParallelConfig contains the configuration for parallel execution of a step. MVP version supports basic parallel execution with maxConcurrent control.

type ParallelItem

type ParallelItem struct {
	// Value is used for simple string items or variable references
	// E.g. "item1", "item2", "${ITEM_VAR}"
	Value string `json:"value,omitempty"`

	// Params is used for key-value pairs that will be passed as parameters
	// E.g. {"SOURCE": "s3://customers", "TYPE": "csv"}
	// Uses DeterministicMap to ensure consistent JSON marshaling for hashing
	Params collections.DeterministicMap `json:"params,omitempty"`
}

ParallelItem represents a single item to be processed in parallel. It can be either a simple value or a set of parameters.

type ParamType

type ParamType int

ParamType identifies which field in Params is active

const (
	ParamTypeUnknown ParamType = iota
	ParamTypeString            // Simple map[string]string
	ParamTypeAny               // Rich map[string]any
	ParamTypeRaw               // Lazy json.RawMessage
)

func (ParamType) String

func (t ParamType) String() string

String returns the string representation of ParamType

type Params

type Params struct {
	// Simple params (backward compatible)
	Simple map[string]string `json:"simple,omitempty"`

	// Rich params with type preservation
	Rich map[string]any `json:"rich,omitempty"`

	// Raw params for lazy parsing
	Raw json.RawMessage `json:"raw,omitempty"`
}

Params holds parameter data in one of several formats Only one of Simple, Rich, or Raw should be non-nil

func NewRawParams

func NewRawParams(raw json.RawMessage) Params

NewRawParams creates params from json.RawMessage

func NewRichParams

func NewRichParams(data map[string]any) Params

NewRichParams creates params from map[string]any

func NewSimpleParams

func NewSimpleParams(data map[string]string) Params

NewSimpleParams creates params from map[string]string

func ParseParams

func ParseParams(input any) (Params, error)

ParseParams creates Params from any input type

func (*Params) AsStringMap

func (p *Params) AsStringMap() (map[string]string, error)

AsStringMap returns all parameters as map[string]string Non-string values are converted using fmt.Sprintf

func (*Params) IsEmpty

func (p *Params) IsEmpty() bool

IsEmpty returns true if no params are set

func (*Params) MarshalJSON

func (p *Params) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler

func (*Params) Type

func (p *Params) Type() ParamType

Type returns which param type is active

func (*Params) UnmarshalJSON

func (p *Params) UnmarshalJSON(data []byte) error

UnmarshalJSON implements json.Unmarshaler

type PullPolicy

type PullPolicy int

PullPolicy defines image pull policy for a container execution

const (
	PullPolicyAlways PullPolicy = iota
	PullPolicyNever
	PullPolicyMissing
)

func ParsePullPolicy

func ParsePullPolicy(raw any) (PullPolicy, error)

ParsePullPolicy parses a pull policy from a raw value.

func (PullPolicy) String

func (p PullPolicy) String() string

type RepeatMode

type RepeatMode string

RepeatMode is the type for the repeat mode.

const (
	// RepeatModeWhile repeats the step while the condition is met.
	RepeatModeWhile RepeatMode = "while"
	// RepeatModeUntil repeats the step until the condition is met.
	RepeatModeUntil RepeatMode = "until"
)

type RepeatPolicy

type RepeatPolicy struct {
	// RepeatMode determines if and how the step should be repeated.
	// It can be 'while' or 'until'.
	RepeatMode RepeatMode `json:"repeatMode,omitempty"`
	// Interval is the time to wait between repeats.
	Interval time.Duration `json:"interval,omitempty"`
	// Limit is the maximum number of times to repeat the step.
	Limit int `json:"limit,omitempty"`
	// Backoff is the exponential backoff multiplier (e.g., 2.0 for doubling).
	Backoff float64 `json:"backoff,omitempty"`
	// MaxInterval is the maximum interval cap for exponential backoff.
	MaxInterval time.Duration `json:"maxInterval,omitempty"`
	// Condition is the condition object to be met for the repeat.
	Condition *Condition `json:"condition,omitempty"`
	// ExitCode is the list of exit codes that should trigger a repeat.
	ExitCode []int `json:"exitCode,omitempty"`
}

RepeatPolicy contains the repeat policy for a step.

func (*RepeatPolicy) UnmarshalJSON

func (r *RepeatPolicy) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for RepeatPolicy. It handles the legacy boolean repeat field and the new string repeat modes.

type RetryPolicy

type RetryPolicy struct {
	// Limit is the number of retries allowed.
	Limit int `json:"limit,omitempty"`
	// Interval is the time to wait between retries.
	Interval time.Duration `json:"interval,omitempty"`
	// LimitStr is the string representation of the limit.
	LimitStr string `json:"limitStr,omitempty"`
	// IntervalSecStr is the string representation of the interval.
	IntervalSecStr string `json:"intervalSecStr,omitempty"`
	// ExitCodes is the list of exit codes that should trigger a retry.
	ExitCodes []int `json:"exitCode,omitempty"`
	// Backoff is the exponential backoff multiplier (e.g., 2.0 for doubling).
	Backoff float64 `json:"backoff,omitempty"`
	// MaxInterval is the maximum interval cap for exponential backoff.
	MaxInterval time.Duration `json:"maxInterval,omitempty"`
}

RetryPolicy contains the retry policy for a step.

type RunConfig

type RunConfig struct {
	// DisableParamEdit when set to true, prevents users from editing parameters when starting a DAG.
	DisableParamEdit bool `json:"disableParamEdit,omitempty"`
	// DisableRunIdEdit when set to true, prevents users from specifying custom run IDs.
	DisableRunIdEdit bool `json:"disableRunIdEdit,omitempty"`
}

RunConfig contains configuration for controlling user interactions during DAG runs.

type SMTPConfig

type SMTPConfig struct {
	Host     string `json:"host,omitempty"`
	Port     string `json:"port,omitempty"`
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
}

SMTPConfig contains the SMTP configuration.

type SSHConfig

type SSHConfig struct {
	// User is the SSH user.
	User string `json:"user,omitempty"`
	// Host is the SSH host.
	Host string `json:"host,omitempty"`
	// Port is the SSH port. Default is "22".
	Port string `json:"port,omitempty"`
	// Key is the path to the SSH private key.
	Key string `json:"key,omitempty"`
	// Password is the SSH password.
	Password string `json:"password,omitempty"`
	// StrictHostKey enables strict host key checking. Defaults to true.
	StrictHostKey bool `json:"strictHostKey,omitempty"`
	// KnownHostFile is the path to the known_hosts file. Defaults to ~/.ssh/known_hosts.
	KnownHostFile string `json:"knownHostFile,omitempty"`
	// Shell is the shell to use for remote command execution.
	// If not specified, commands are executed directly without shell wrapping.
	Shell string `json:"shell,omitempty"`
	// ShellArgs contains additional arguments that should be passed to the shell executable.
	ShellArgs []string `json:"shellArgs,omitempty"`
}

SSHConfig contains the SSH configuration for the DAG.

type Schedule

type Schedule struct {
	// Expression is the cron expression.
	Expression string `json:"expression"`
	// Parsed is the parsed cron schedule.
	Parsed cron.Schedule `json:"-"`
}

Schedule contains the cron expression and the parsed cron schedule.

func (Schedule) MarshalJSON

func (s Schedule) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface.

func (*Schedule) UnmarshalJSON

func (s *Schedule) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface. and also parses the cron expression to populate the Parsed field.

type SecretRef

type SecretRef struct {
	// Name is the environment variable name to set (required).
	Name string `json:"name"`
	// Provider specifies the secret backend (e.g., "env", "file", "gcp-secrets") (required).
	Provider string `json:"provider"`
	// Key is the provider-specific identifier for the secret (required).
	Key string `json:"key"`
	// Options contains provider-specific configuration (optional).
	Options map[string]string `json:"options,omitempty"`
}

SecretRef represents a reference to an external secret. Secrets are resolved at DAG execution time and never persisted to disk.

type Status

type Status int

Status represents the canonical lifecycle phases for a DAG execution.

const (
	NotStarted Status = iota
	Running
	Failed
	Aborted
	Succeeded
	Queued
	PartiallySucceeded
	Waiting
	Rejected
)

func (Status) IsActive

func (s Status) IsActive() bool

IsActive checks if the status is active (not yet completed). This includes Running, Queued, and Wait (waiting for human approval).

func (Status) IsSuccess

func (s Status) IsSuccess() bool

IsSuccess checks if the status indicates a successful execution.

func (Status) IsWaiting added in v1.30.0

func (s Status) IsWaiting() bool

IsWaiting checks if the status is waiting for human approval.

func (Status) String

func (s Status) String() string

String returns the canonical lowercase token used across APIs, logs, and environment variables.

type Step

type Step struct {
	// ID is the optional unique identifier for the step.
	ID string `json:"id,omitempty"`
	// Name is the name of the step.
	Name string `json:"name"`
	// Description is the description of the step. This is optional.
	Description string `json:"description,omitempty"`
	// Shell is the shell program to execute the command. This is optional.
	Shell string `json:"shell,omitempty"`
	// ShellPackages is the list of packages to install. This is used only when the shell is `nix-shell`.
	ShellPackages []string `json:"shellPackages,omitempty"`
	// ShellArgs is the list of arguments for the shell program.
	ShellArgs []string `json:"shellArgs,omitempty"`
	// Dir is the working directory for the step.
	Dir string `json:"dir,omitempty"`
	// ExecutorConfig contains the configuration for the executor.
	ExecutorConfig ExecutorConfig `json:"executorConfig,omitzero"`
	// CmdWithArgs is the command with arguments for display purposes.
	// Deprecated: Use Commands[0].CmdWithArgs instead. Kept for JSON backward compatibility.
	CmdWithArgs string `json:"cmdWithArgs,omitempty"`
	// CmdArgsSys is the command with arguments for the system.
	// Deprecated: Kept for JSON backward compatibility.
	CmdArgsSys string `json:"cmdArgsSys,omitempty"`
	// Command specifies only the command without arguments.
	// Deprecated: Use Commands field instead. Kept for JSON backward compatibility.
	Command string `json:"command,omitempty"`
	// ShellCmdArgs is the shell command with arguments.
	ShellCmdArgs string `json:"shellCmdArgs,omitempty"`
	// Script is the script to be executed.
	Script string `json:"script,omitempty"`
	// Args contains the arguments for the command.
	// Deprecated: Use Commands field instead. Kept for JSON backward compatibility.
	Args []string `json:"args,omitempty"`
	// Commands is the source of truth for commands to execute.
	// Each entry represents a command to be executed sequentially.
	// For single commands, this will contain exactly one entry.
	Commands []CommandEntry `json:"commands,omitempty"`
	// Stdout is the file to store the standard output.
	Stdout string `json:"stdout,omitempty"`
	// Stderr is the file to store the standard error.
	Stderr string `json:"stderr,omitempty"`
	// LogOutput specifies how stdout and stderr are handled in log files for this step.
	// Overrides the DAG-level LogOutput setting. Empty string means inherit from DAG.
	LogOutput LogOutputMode `json:"logOutput,omitempty"`
	// Output is the variable name to store the output.
	Output string `json:"output,omitempty"`
	// OutputKey is the custom key for the output in outputs.json.
	// If empty, the Output name is converted from UPPER_CASE to camelCase.
	OutputKey string `json:"outputKey,omitempty"`
	// OutputOmit excludes this output from outputs.json when true.
	OutputOmit bool `json:"outputOmit,omitempty"`
	// Depends contains the list of step names to depend on.
	Depends []string `json:"depends,omitempty"`
	// ExplicitlyNoDeps indicates the depends field was explicitly set to empty
	ExplicitlyNoDeps bool `json:"-"`
	// ContinueOn contains the conditions to continue on failure or skipped.
	ContinueOn ContinueOn `json:"continueOn,omitzero"`
	// RetryPolicy contains the retry policy for the step.
	RetryPolicy RetryPolicy `json:"retryPolicy,omitzero"`
	// RepeatPolicy contains the repeat policy for the step.
	RepeatPolicy RepeatPolicy `json:"repeatPolicy,omitzero"`
	// MailOnError is the flag to send mail on error.
	MailOnError bool `json:"mailOnError,omitempty"`
	// Preconditions contains the conditions to be met before running the step.
	Preconditions []*Condition `json:"preconditions,omitempty"`
	// SignalOnStop is the signal to send on stop.
	SignalOnStop string `json:"signalOnStop,omitempty"`
	// SubDAG contains the information about a sub DAG to be executed.
	SubDAG *SubDAG `json:"childDag,omitempty"`
	// WorkerSelector specifies required worker labels for execution.
	WorkerSelector map[string]string `json:"workerSelector,omitempty"`
	// Parallel contains the configuration for parallel execution.
	Parallel *ParallelConfig `json:"parallel,omitempty"`
	// Env contains environment variables for the step.
	Env []string `json:"env,omitempty"`
	// Params contains parameters/inputs for the step (e.g., action inputs for GitHub Actions).
	Params Params `json:"params,omitzero"`
	// Timeout specifies the maximum execution time for the step.
	// If set, this timeout takes precedence over the DAG-level timeout for this step.
	Timeout time.Duration `json:"timeout,omitempty"`
	// Container specifies the container configuration for this step.
	// If set, the step runs in its own container instead of the DAG-level container.
	// This uses the same configuration format as the DAG-level container field.
	Container *Container `json:"container,omitempty"`
	// LLM contains the configuration for LLM-based executors (chat, agent, etc.).
	// Used with explicit type: chat (or future type: agent).
	LLM *LLMConfig `json:"llm,omitempty"`
	// Messages contains the conversation messages for chat executor.
	// Only used when type is "chat".
	Messages []LLMMessage `json:"messages,omitempty"`
}

Step contains the runtime information for a step in a DAG. A step is created from parsing a DAG file written in YAML. It marshals/unmarshals to/from JSON when it is saved in the execution history.

func (Step) EvalOptions added in v1.30.0

func (s Step) EvalOptions(ctx context.Context) []cmdutil.EvalOption

EvalOptions returns eval options for this step's executor type. Returns nil if no special eval options are needed.

func (*Step) HasMultipleCommands added in v1.29.0

func (s *Step) HasMultipleCommands() bool

HasMultipleCommands returns true if the step has multiple commands to execute.

func (*Step) String

func (s *Step) String() string

String returns a formatted string representation of the step

func (*Step) UnmarshalJSON added in v1.29.0

func (s *Step) UnmarshalJSON(data []byte) error

UnmarshalJSON implements json.Unmarshaler for backward compatibility. It handles old JSON format where command/args fields were used instead of commands.

type StepValidator

type StepValidator func(step Step) error

StepValidator is a function type for validating step configurations.

type SubDAG added in v1.24.0

type SubDAG struct {
	Name   string `json:"name,omitempty"`
	Params string `json:"params,omitempty"`
}

SubDAG contains information about a sub DAG to be executed.

type ThinkingConfig added in v1.30.0

type ThinkingConfig struct {
	// Enabled activates thinking mode for supported models.
	Enabled bool `json:"enabled,omitempty"`
	// Effort controls reasoning depth: low, medium, high, xhigh.
	// Maps to provider-specific parameters.
	Effort ThinkingEffort `json:"effort,omitempty"`
	// BudgetTokens sets explicit token budget (provider-specific).
	// For Anthropic: minimum 1024, max 128K.
	// For Gemini 2.5: range 128-32768.
	BudgetTokens *int `json:"budgetTokens,omitempty"`
	// IncludeInOutput includes thinking blocks in stdout.
	// Default is false for consistency across providers.
	IncludeInOutput bool `json:"includeInOutput,omitempty"`
}

ThinkingConfig contains configuration for extended thinking/reasoning.

type ThinkingEffort added in v1.30.0

type ThinkingEffort string

ThinkingEffort represents the reasoning depth level for thinking mode.

const (
	ThinkingEffortLow    ThinkingEffort = "low"
	ThinkingEffortMedium ThinkingEffort = "medium"
	ThinkingEffortHigh   ThinkingEffort = "high"
	ThinkingEffortXHigh  ThinkingEffort = "xhigh"
)

ThinkingEffort constants for reasoning/thinking depth.

func ParseThinkingEffort added in v1.30.0

func ParseThinkingEffort(s string) (ThinkingEffort, error)

ParseThinkingEffort validates and returns a ThinkingEffort from a string. Returns empty string for empty input (no effort specified). Returns error for invalid effort values.

type ValidationError

type ValidationError struct {
	Field string
	Value any
	Err   error
}

ValidationError represents an error in a specific field of the configuration

func (*ValidationError) Error

func (e *ValidationError) Error() string

func (*ValidationError) Unwrap

func (e *ValidationError) Unwrap() error

Directories

Path Synopsis
types
Package types provides typed union types for YAML fields that accept multiple formats.
Package types provides typed union types for YAML fields that accept multiple formats.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL