Documentation
¶
Index ¶
- Constants
- Variables
- func BuildTaskPayload(job *models.Job) *corndogs.TaskPayload
- func BuilderSidecarName(jobID string) string
- func ComputeWorkflowStatus(nodes []models.WorkflowNode) string
- func ContainerPathInsideJob(path string) string
- func DefaultJobCodeDir(codeDir string) string
- func DefaultJobDir(codeDir, jobDir string) string
- func DefaultRunAsUser(user string) (string, error)
- func HasCapability(caps []string, want string) bool
- func HasEnvRefs(s string) bool
- func HasSecretRefs(s string) bool
- func IsBackendImplemented(backend string) bool
- func IsBackendSupported(backend string) bool
- func IsKubernetesEnvironment() bool
- func IsPodStartupError(err error) bool
- func IsRetryable(err error) bool
- func LoadJobSpecWithOverlays(jobPath string, overlayPaths []string) (*JobSpec, []SecretOverride, error)
- func MergeJobSpecs(base *JobSpec, overlays []*JobSpec, overlayFiles []string) (*JobSpec, []SecretOverride)
- func NormalizeRunAsUser(user string) (string, error)
- func ParseCommand(cmd string) []string
- func ParseCommandWithPrefix(cmd, prefix string) []string
- func ResolveEnvInMap(env map[string]string) map[string]string
- func ResolveEnvRefs(value string) string
- func ResolveSecretRefs(value string, getSecret func(path, key string) (string, error)) (string, error)
- func ResolveSecretsInEnv(env map[string]string, getSecret func(path, key string) (string, error)) (map[string]string, []string, error)
- func RetryWithBackoff(ctx context.Context, config *RetryConfig, operation string, fn func() error) error
- func RetryWithBackoffCounter(ctx context.Context, config *RetryConfig, operation string, ...) error
- type BuilderConfig
- type Config
- type ContainerdRunner
- func (cr *ContainerdRunner) Cleanup(ctx context.Context, containerID string) error
- func (cr *ContainerdRunner) SpawnJob(ctx context.Context, config *JobConfig) (string, error)
- func (cr *ContainerdRunner) Stop(ctx context.Context, containerID string, grace time.Duration) error
- func (cr *ContainerdRunner) StreamLogs(ctx context.Context, containerID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
- func (cr *ContainerdRunner) WaitForCompletion(ctx context.Context, containerID string) (int, error)
- type CornDogsWorker
- type DockerRunner
- func (dr *DockerRunner) Cleanup(ctx context.Context, containerID string) error
- func (dr *DockerRunner) SpawnJob(ctx context.Context, config *JobConfig) (string, error)
- func (dr *DockerRunner) Stop(ctx context.Context, containerID string, grace time.Duration) error
- func (dr *DockerRunner) StreamLogs(ctx context.Context, containerID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
- func (dr *DockerRunner) WaitForCompletion(ctx context.Context, containerID string) (int, error)
- type HeartbeatFunc
- type JobConfig
- type JobContext
- type JobExecutionContext
- type JobProcessor
- func NewJobProcessor(store store.Store, runner JobRunner, dryRun bool) *JobProcessor
- func NewJobProcessorWithConfig(store store.Store, runner JobRunner, dryRun bool, config *JobProcessorConfig) *JobProcessor
- func NewJobProcessorWithRetryConfig(store store.Store, runner JobRunner, dryRun bool, retryConfig *RetryConfig) *JobProcessor
- type JobProcessorConfig
- type JobProcessorInterface
- type JobResult
- type JobRunner
- type JobSpec
- type KubernetesRunner
- func (kr *KubernetesRunner) Cleanup(ctx context.Context, jobName string) error
- func (kr *KubernetesRunner) GetJobStatus(ctx context.Context, jobName string) (status string, failureReason string, err error)
- func (kr *KubernetesRunner) SpawnJob(ctx context.Context, config *JobConfig) (string, error)
- func (kr *KubernetesRunner) Stop(ctx context.Context, jobName string, grace time.Duration) error
- func (kr *KubernetesRunner) StreamLogs(ctx context.Context, jobName string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
- func (kr *KubernetesRunner) WaitForCompletion(ctx context.Context, jobName string) (int, error)
- type KubernetesRunnerConfig
- type LifecycleManager
- func (lm *LifecycleManager) GetActiveJobs() []string
- func (lm *LifecycleManager) GracefulShutdown(ctx context.Context) error
- func (lm *LifecycleManager) IsShuttingDown() bool
- func (lm *LifecycleManager) JobCleanupOnFailure(jobID string, err error)
- func (lm *LifecycleManager) RecoverJobs(ctx context.Context, workerID string) error
- func (lm *LifecycleManager) RegisterJob(job *models.Job, workDir string, cancel context.CancelFunc)
- func (lm *LifecycleManager) SetupSignalHandlers(ctx context.Context, cancel context.CancelFunc)
- func (lm *LifecycleManager) UnregisterJob(jobID string)
- type LogEntry
- type LogShipper
- type LogShipperConfig
- type PodStartupError
- type ResourceMetrics
- type ResourceMonitor
- func (rm *ResourceMonitor) GetMetrics() ResourceMetrics
- func (rm *ResourceMonitor) IsHealthy() bool
- func (rm *ResourceMonitor) LogMetricsSummary()
- func (rm *ResourceMonitor) RecordJobComplete(jobID string, success bool)
- func (rm *ResourceMonitor) RecordJobStart(jobID string)
- func (rm *ResourceMonitor) SetThresholds(cpu, memory float64)
- func (rm *ResourceMonitor) Start(ctx context.Context)
- func (rm *ResourceMonitor) Stop()
- type RetryConfig
- type RetryableError
- type RunAsSpec
- type RunLocalSpec
- type RunnerBackend
- type SecretOverride
- type SecretResolutionResult
- type SourceSpec
- type TriggerProcessor
- func (tp *TriggerProcessor) EvaluateWorkflow(ctx context.Context, wf *models.WorkflowInstance) ([]string, error)
- func (tp *TriggerProcessor) ProcessTriggers(ctx context.Context, workspaceDir string, parentJob *models.Job) error
- func (tp *TriggerProcessor) ProcessTriggersFromData(ctx context.Context, data []byte, workspaceDir string, parentJob *models.Job) ([]string, error)
- func (tp *TriggerProcessor) ProcessWorkflowCompletion(ctx context.Context, workspaceDir string, job *models.Job) error
- func (tp *TriggerProcessor) ProcessWorkflowJobStarted(ctx context.Context, job *models.Job) error
- func (tp *TriggerProcessor) SetStatusUpdater(u vcs.JobStatusUpdaterInterface)
- type VCSAuthConfig
- type Worker
Constants ¶
const ( // CapabilityDocker provides access to a docker CLI for running ad-hoc // containers (e.g. integration tests). Implementations historically // privilege the job container and expose a runtime socket. Prefer // CapabilityBuilder for building images — that leaves the job container // unprivileged. CapabilityDocker = "docker" // CapabilityBuilder provisions a buildkitd sidecar and injects // BUILDKIT_HOST=tcp://localhost:1234 into the job, so jobs can build and // push container images without privileged access or host runtime // coupling. The sidecar's registry policy (insecure registries, CA // bundles, etc.) comes from operator-provided buildkitd.toml, not from // the job. CapabilityBuilder = "builder" // CapabilityGPU provides access to GPU resources. // NOT YET IMPLEMENTED - placeholder for future development. // DockerRunner: would use --gpus all flag // KubernetesRunner: would add nvidia.com/gpu resource request CapabilityGPU = "gpu" )
Capability constants for job requirements
const ( RunnerUser = "1001:1001" RootUser = "0:0" )
const BuilderHostEnv = "BUILDKIT_HOST"
BuilderHostEnv is the env var set on the job container when CapabilityBuilder is present, pointing buildctl at the sidecar.
const BuilderLogPrefix = "[builder] "
BuilderLogPrefix is prepended to every line of sidecar log output so operators can tell at a glance which container produced which line. The callers that apply secret masking read line-by-line, so a deterministic line-level prefix preserves masker correctness.
const BuilderSidecarPort = 1234
BuilderSidecarPort is the TCP port buildkitd listens on inside its sidecar container. Jobs reach it via BUILDKIT_HOST=tcp://localhost:1234 when they share netns with the sidecar.
const DefaultBuilderImage = "moby/buildkit:v0.17.3"
DefaultBuilderImage is the buildkitd image used for builder sidecars when REACTORCIDE_BUILDER_IMAGE is not set.
const DefaultCancelGrace = 60 * time.Second
DefaultCancelGrace is the fallback grace period used when JobProcessorConfig.CancelGrace is unset (zero). Mirrors REACTORCIDE_CANCEL_GRACE_SECONDS' own default in internal/config.
const DefaultRunnerImage = "containers.catalystsquad.com/public/reactorcide/runnerbase:dev"
DefaultRunnerImage is the default container image for job execution
Variables ¶
var EnvRefPattern = regexp.MustCompile(`\$\{env:([^}]+)\}`)
EnvRefPattern matches ${env:VAR_NAME} references in strings This allows job YAMLs to reference host environment variables
var SecretRefPattern = regexp.MustCompile(`\$\{secret:([^:}]+):([^}]+)\}`)
SecretRefPattern matches ${secret:path:key} references in strings
Functions ¶
func BuildTaskPayload ¶
func BuildTaskPayload(job *models.Job) *corndogs.TaskPayload
BuildTaskPayload is the exported, receiver-free form of buildTaskPayload: it depends only on the job, not on any TriggerProcessor field, so it's safe to call from other packages that need to mirror the exact submission shape trigger_processor.go/workflow_runtime.go use — currently internal/jobcontrol.RetryJob, which resubmits a cloned job the same way a freshly triggered or workflow-node job is submitted.
func BuilderSidecarName ¶
BuilderSidecarName returns the deterministic sidecar container/pod-container name for a given job id. Runners use this so cleanup can find the sidecar without extra state.
func ComputeWorkflowStatus ¶
func ComputeWorkflowStatus(nodes []models.WorkflowNode) string
ComputeWorkflowStatus is the exported form of computeWorkflowStatus, used by internal/jobcontrol (CancelWorkflow) so the workflow-cancel cascade and the normal per-node completion path (refreshWorkflowStatus, above) agree on exactly one status-derivation rule instead of maintaining two.
func ContainerPathInsideJob ¶
func DefaultJobCodeDir ¶
func DefaultJobDir ¶
func DefaultRunAsUser ¶
func HasCapability ¶
HasCapability returns true if caps contains the given capability.
func HasEnvRefs ¶
HasEnvRefs checks if a string contains environment variable references
func HasSecretRefs ¶
HasSecretRefs checks if a string contains secret references
func IsBackendImplemented ¶
IsBackendImplemented checks if a backend is fully implemented (not just stubbed)
func IsBackendSupported ¶
IsBackendSupported checks if a backend is supported (though may not be fully implemented)
func IsKubernetesEnvironment ¶
func IsKubernetesEnvironment() bool
IsKubernetesEnvironment checks if the code is running inside a Kubernetes cluster
func IsPodStartupError ¶
IsPodStartupError checks if an error is a pod startup failure Uses errors.As to handle wrapped errors
func LoadJobSpecWithOverlays ¶
func LoadJobSpecWithOverlays(jobPath string, overlayPaths []string) (*JobSpec, []SecretOverride, error)
LoadJobSpecWithOverlays loads a job spec and applies overlay files in order. The overlay files are specified from highest to lowest priority (first file wins). Returns the merged spec and any warnings about secret overrides.
func MergeJobSpecs ¶
func MergeJobSpecs(base *JobSpec, overlays []*JobSpec, overlayFiles []string) (*JobSpec, []SecretOverride)
MergeJobSpecs merges overlay specs onto a base spec. Overlays are applied in order, with later overlays taking precedence. Returns the merged spec and any warnings about secret overrides.
func NormalizeRunAsUser ¶
NormalizeRunAsUser converts a job run_as.user value into a runtime user string. Deployed workers intentionally do not support "host" because there is no stable host user to map to across VM and Kubernetes runtimes.
func ParseCommand ¶
ParseCommand splits a command string for container execution. Uses default "sh -c" prefix for multiline commands.
func ParseCommandWithPrefix ¶
ParseCommandWithPrefix converts a command string to []string for container execution. For multiline commands, wraps with the specified prefix (default "sh -c"). For single-line commands, splits on whitespace respecting basic quoting.
func ResolveEnvInMap ¶
ResolveEnvInMap resolves ${env:VAR_NAME} references in all values of a map
func ResolveEnvRefs ¶
ResolveEnvRefs resolves ${env:VAR_NAME} references in a string using os.Getenv to get values from the host environment
func ResolveSecretRefs ¶
func ResolveSecretRefs(value string, getSecret func(path, key string) (string, error)) (string, error)
ResolveSecretRefs resolves ${secret:path:key} references in a string using the provided getter function
func ResolveSecretsInEnv ¶
func ResolveSecretsInEnv(env map[string]string, getSecret func(path, key string) (string, error)) (map[string]string, []string, error)
ResolveSecretsInEnv resolves all secret references in environment variables Returns a new map with resolved values, a list of resolved secret values for masking, and a list of env var names that contained secrets. Note: ${env:VAR} references should be resolved first using ResolveEnvInMap
func RetryWithBackoff ¶
func RetryWithBackoff(ctx context.Context, config *RetryConfig, operation string, fn func() error) error
RetryWithBackoff executes a function with exponential backoff retry logic
func RetryWithBackoffCounter ¶
func RetryWithBackoffCounter(ctx context.Context, config *RetryConfig, operation string, fn func(attempt int) error) error
RetryWithBackoffCounter executes a function with exponential backoff retry logic and provides attempt counter
Types ¶
type BuilderConfig ¶
type BuilderConfig struct {
// Image is the buildkitd container image. Defaults to DefaultBuilderImage.
Image string
// ConfigPath is a host file path to a buildkitd.toml that will be bind-
// mounted at /etc/buildkit/buildkitd.toml inside the sidecar. Empty means
// the sidecar uses its image's built-in defaults.
ConfigPath string
// RegistryAuthPath is a host file path to a docker registry auth file
// (config.json format) mounted into the sidecar at /root/.docker/config.json.
// Buildkit uses this when pulling base images. Push credentials are
// typically supplied client-side by the job via buildctl.
RegistryAuthPath string
// CacheVolume is an optional named docker volume (or k8s PVC name) mounted
// at buildkit's state dir for cross-job layer cache. Empty means no cache
// volume — each sidecar starts clean.
CacheVolume string
}
BuilderConfig holds operator-level configuration for the buildkitd sidecar launched for jobs with CapabilityBuilder. Runners pick the fields they need and ignore the rest; the zero value is a valid "no operator overrides" state in which the sidecar runs with its image defaults.
func LoadBuilderConfig ¶
func LoadBuilderConfig() BuilderConfig
LoadBuilderConfig resolves BuilderConfig from environment variables. This is the single source of truth for operator knobs across all runners.
Env vars:
- REACTORCIDE_BUILDER_IMAGE (default DefaultBuilderImage)
- REACTORCIDE_BUILDER_CONFIG_PATH (optional, no default)
- REACTORCIDE_BUILDER_REGISTRY_AUTH_PATH (optional, no default)
- REACTORCIDE_BUILDER_CACHE_VOLUME (optional, no default)
type Config ¶
type Config struct {
QueueName string
PollInterval time.Duration
Concurrency int
DryRun bool
Store store.Store
WorkerID string // Unique identifier for this worker instance
ContainerRuntime string // Container runtime backend: "docker", "containerd", or "kubernetes"
// Log shipping configuration
ObjectStore objects.ObjectStore // Object store for logs and artifacts
LogChunkInterval time.Duration // Interval for uploading log chunks (default: 3 seconds)
// Heartbeat configuration
HeartbeatInterval time.Duration // Interval for sending heartbeats to Corndogs (default: 30 seconds)
HeartbeatTimeout time.Duration // Timeout extension for each heartbeat (default: 10 minutes)
// CancelGrace is how long a graceful cancel waits between SIGTERM
// (JobRunner.Stop) and a forced Cleanup, checked on the heartbeat tick
// (default: 60 seconds). Not used for kill (immediate, no grace).
CancelGrace time.Duration
}
Config holds the configuration for the worker
type ContainerdRunner ¶
type ContainerdRunner struct {
// contains filtered or unexported fields
}
ContainerdRunner implements JobRunner using nerdctl CLI This approach lets nerdctl handle networking (CNI) automatically
func NewContainerdRunner ¶
func NewContainerdRunner() (*ContainerdRunner, error)
NewContainerdRunner creates a new nerdctl-based job runner
func (*ContainerdRunner) Cleanup ¶
func (cr *ContainerdRunner) Cleanup(ctx context.Context, containerID string) error
Cleanup removes the container and associated resources
func (*ContainerdRunner) Stop ¶
func (cr *ContainerdRunner) Stop(ctx context.Context, containerID string, grace time.Duration) error
Stop requests a graceful shutdown of the job container via `nerdctl stop`, which sends SIGTERM to PID 1 and waits up to `grace` seconds before sending SIGKILL. grace == 0 requests immediate termination (nerdctl stop -t 0 sends SIGTERM immediately followed by SIGKILL with no wait).
func (*ContainerdRunner) StreamLogs ¶
func (cr *ContainerdRunner) StreamLogs(ctx context.Context, containerID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
StreamLogs returns stdout and stderr readers for a container
func (*ContainerdRunner) WaitForCompletion ¶
WaitForCompletion waits for the container to exit and returns the exit code
type CornDogsWorker ¶
type CornDogsWorker struct {
// contains filtered or unexported fields
}
CornDogsWorker represents a job processing worker that uses Corndogs for task management
func NewCornDogsWorker ¶
func NewCornDogsWorker(config *Config, corndogsClient corndogs.ClientInterface, statusUpdater vcs.JobStatusUpdaterInterface) *CornDogsWorker
NewCornDogsWorker creates a new worker that uses Corndogs for task management. statusUpdater is optional; if nil, VCS status updates are silently skipped.
func NewCornDogsWorkerWithProcessor ¶
func NewCornDogsWorkerWithProcessor(config *Config, corndogsClient corndogs.ClientInterface, processor JobProcessorInterface, triggerProcessor *TriggerProcessor, statusUpdater vcs.JobStatusUpdaterInterface) *CornDogsWorker
NewCornDogsWorkerWithProcessor creates a new worker with a custom processor (for testing). statusUpdater is optional; if nil, VCS status updates are silently skipped.
func (*CornDogsWorker) SetPublisher ¶
func (w *CornDogsWorker) SetPublisher(p *pubsub.Publisher)
SetPublisher wires a pubsub.Publisher so job-status transitions and log chunk flushes get broadcast to WebSocket subscribers across replicas. Safe to call with nil (disables live broadcasts).
type DockerRunner ¶
type DockerRunner struct {
// contains filtered or unexported fields
}
DockerRunner implements JobRunner using the Docker daemon
func NewDockerRunner ¶
func NewDockerRunner() (*DockerRunner, error)
NewDockerRunner creates a new Docker-based job runner Uses the default Docker socket (unix:///var/run/docker.sock or npipe on Windows)
func NewDockerRunnerWithClient ¶
func NewDockerRunnerWithClient(cli *client.Client) *DockerRunner
NewDockerRunnerWithClient creates a DockerRunner with a custom Docker client Useful for testing or custom configurations
func (*DockerRunner) Cleanup ¶
func (dr *DockerRunner) Cleanup(ctx context.Context, containerID string) error
Cleanup removes the container and any builder sidecar launched for it.
func (*DockerRunner) Stop ¶
Stop requests a graceful shutdown of the job container: the Docker daemon sends SIGTERM to PID 1 and waits up to `grace` before sending SIGKILL. grace == 0 skips the wait and kills immediately (Docker's ContainerStop with Timeout=0 still sends SIGTERM first, but does not wait for it to take effect before following up with SIGKILL).
Any builder sidecar is left running; the caller's subsequent Cleanup() call tears down both the job container and its sidecar together.
func (*DockerRunner) StreamLogs ¶
func (dr *DockerRunner) StreamLogs(ctx context.Context, containerID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
StreamLogs streams stdout and stderr from the container
func (*DockerRunner) WaitForCompletion ¶
WaitForCompletion waits for the container to exit and returns the exit code
type HeartbeatFunc ¶
HeartbeatFunc is a function that sends a heartbeat It should extend the timeout for the currently executing task
type JobConfig ¶
type JobConfig struct {
// Container image to use (e.g., "reactorcide/runner:latest")
Image string
// Command to execute in the container. The container's entrypoint is
// cleared, so this is the full command (e.g., ["sh", "-c", "make build"])
Command []string
// Environment variables to inject into the container
Env map[string]string
// WorkspaceDir is the host directory to mount into the container at /job
WorkspaceDir string
// SourceDir is an optional host directory to mount at /job/src in the container.
// Used by run-local to mount user's source into the standard production layout.
SourceDir string
// SourceMountPath is the container path where SourceDir is mounted. Defaults
// to /job/src when empty.
SourceMountPath string
// WorkingDir is the working directory inside the container (default: /job)
WorkingDir string
// Capabilities declares what the job needs from the runtime environment.
// Each runner interprets these appropriately for its environment.
// See CapabilityDocker, CapabilityGPU constants.
Capabilities []string
// RunAsUser optionally overrides the container user ("uid:gid"). When
// empty, runners default to RunnerUser. Capabilities provision runtime
// services/privileges but do not implicitly change the job user.
// run-local sets this to the host uid so bind-mounted sources are writable.
RunAsUser string
// ExtraMounts are additional bind mounts in "hostpath:containerpath[:ro]"
// format. Used by run-local to inject a synthetic /etc/passwd and
// /etc/group for the host uid so tools like ssh find a valid user entry.
ExtraMounts []string
// VCSAuth contains per-job checkout credential files. Docker/containerd
// jobs read these from WorkspaceDir; Kubernetes jobs materialize them as
// a short-lived Secret copied into an emptyDir.
VCSAuth *VCSAuthConfig
// Timeout for the job execution (0 = no timeout)
TimeoutSeconds int
// Resource limits
CPULimit string // e.g., "1.0" for 1 CPU
MemoryLimit string // e.g., "512Mi" or "1Gi"
// Job metadata (for labeling/tagging)
JobID string
QueueName string
}
JobConfig contains all the configuration needed to spawn a job container. The container's entrypoint is always cleared - the Command field contains the full command to execute. Users can run their own commands or invoke runnerlib for source preparation and lifecycle hooks.
type JobContext ¶
type JobContext struct {
Job *models.Job
StartTime time.Time
WorkDir string
Cancel context.CancelFunc
}
JobContext tracks the context of an active job
type JobExecutionContext ¶
type JobExecutionContext struct {
HeartbeatFunc HeartbeatFunc // Optional: function to call for sending heartbeats
}
JobExecutionContext holds context for job execution
type JobProcessor ¶
type JobProcessor struct {
// contains filtered or unexported fields
}
JobProcessor handles the execution of individual jobs
func NewJobProcessor ¶
func NewJobProcessor(store store.Store, runner JobRunner, dryRun bool) *JobProcessor
NewJobProcessor creates a new job processor
func NewJobProcessorWithConfig ¶
func NewJobProcessorWithConfig(store store.Store, runner JobRunner, dryRun bool, config *JobProcessorConfig) *JobProcessor
NewJobProcessorWithConfig creates a new job processor with configuration
func NewJobProcessorWithRetryConfig ¶
func NewJobProcessorWithRetryConfig(store store.Store, runner JobRunner, dryRun bool, retryConfig *RetryConfig) *JobProcessor
NewJobProcessorWithRetryConfig creates a new job processor with custom retry configuration
func (*JobProcessor) ProcessJob ¶
ProcessJob executes a job using runnerlib
func (*JobProcessor) ProcessJobWithContext ¶
func (jp *JobProcessor) ProcessJobWithContext(ctx context.Context, job *models.Job, execCtx *JobExecutionContext) *JobResult
ProcessJobWithContext executes a job with optional execution context (e.g., heartbeat function)
func (*JobProcessor) SetPublisher ¶
func (jp *JobProcessor) SetPublisher(p *pubsub.Publisher)
SetPublisher threads a pubsub.Publisher through the processor's config so per-job log shippers can emit log_available NOTIFYs.
type JobProcessorConfig ¶
type JobProcessorConfig struct {
ObjectStore objects.ObjectStore
LogChunkInterval time.Duration
HeartbeatInterval time.Duration
HeartbeatTimeout time.Duration
RetryConfig *RetryConfig
// CancelGrace is how long a graceful cancel (JobRunner.Stop) waits
// between SIGTERM and the runner's own forced kill, checked on each
// heartbeat tick alongside the cancel-status poll (default: 60s, see
// DefaultCancelGrace).
CancelGrace time.Duration
// Publisher, if non-nil, is threaded into each LogShipper so chunk
// flushes trigger NOTIFY events to WebSocket subscribers.
Publisher *pubsub.Publisher
// Optional: Callbacks for log updates and heartbeats
OnLogUpdate func(jobID, objectKey string, bytesWritten int64) error
// Secrets configuration
// SecretsKeyManager is the master key manager for decrypting org keys
SecretsKeyManager *secrets.MasterKeyManager
// SecretsStorageType determines where secrets are stored: "database" (default), "local", or "none"
SecretsStorageType string
// SecretsLocalPath is the path for local secrets storage (only used when SecretsStorageType="local")
SecretsLocalPath string
// SecretsLocalPassword is the password for local secrets storage
SecretsLocalPassword string
}
JobProcessorConfig holds configuration for the job processor
type JobProcessorInterface ¶
type JobProcessorInterface interface {
ProcessJob(ctx context.Context, job *models.Job) *JobResult
ProcessJobWithContext(ctx context.Context, job *models.Job, execCtx *JobExecutionContext) *JobResult
}
JobProcessorInterface defines the interface for job processing
type JobResult ¶
type JobResult struct {
ExitCode int
Output string
Error string
LogsObjectKey string
ArtifactsObjectKey string
Duration time.Duration
RetryCount int // Number of retry attempts made
Retryable bool // Whether the failure was retryable
WorkspaceDir string // Host path to workspace directory; caller must clean up via os.RemoveAll
// Cancelled is true when the cancel-poll observed the job's DB status
// flip to "cancelling" during execution and drove the container to stop
// (gracefully or via kill) itself, rather than the job command exiting
// on its own. Callers should set the job's terminal status to
// "cancelled" rather than deriving it from ExitCode, which is not
// meaningful for a runner-initiated stop (see JobRunner.Stop, Killed).
Cancelled bool
// Killed is true when Cancelled is true and the cancel was an admin
// kill (immediate force-Cleanup, no SIGTERM grace) rather than a
// graceful cancel (JobRunner.Stop).
Killed bool
}
JobResult represents the result of job execution
type JobRunner ¶
type JobRunner interface {
// SpawnJob creates and starts a job container with the specified configuration
// Returns a unique job ID/handle and any error encountered
SpawnJob(ctx context.Context, config *JobConfig) (string, error)
// StreamLogs streams stdout/stderr from a running job container
// Returns separate readers for stdout and stderr
StreamLogs(ctx context.Context, jobID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
// WaitForCompletion blocks until the job container exits
// Returns the exit code and any error encountered
WaitForCompletion(ctx context.Context, jobID string) (int, error)
// Stop requests a graceful shutdown of a running job: SIGTERM to the
// container's PID 1 (giving runnerlib's SIGTERM trap a chance to run
// PluginPhase.CLEANUP/ON_ERROR and cleanup_vcs_auth), then a forced kill
// if the container/pod hasn't exited within grace. Stop does not remove
// the container/pod — callers still call Cleanup afterward (WaitForCompletion
// unblocking is what tells the caller it's safe to do so).
//
// grace == 0 requests immediate forced termination (no SIGTERM wait) —
// used by the admin "kill" path when it needs an unambiguous immediate
// stop rather than going through Cleanup directly (e.g. to unblock a
// stuck WaitForCompletion caller). Cleanup remains the primary "kill"
// primitive at the job_processor/corndogs_worker layer (see AGENTS.md /
// UI_AUTH_PLAN.md Cancel vs Kill) — Stop(grace=0) exists so runner
// backends have a single, testable primitive for "terminate now".
//
// Calling Stop on a container/pod that has already exited or been
// removed must be a safe no-op (implementations should tolerate
// "not found" style errors rather than returning them as failures),
// since the worker's cancel-poll and the container's own natural
// completion can race.
Stop(ctx context.Context, jobID string, grace time.Duration) error
// Cleanup removes the job container and associated resources
// Should be called after the job completes (success or failure)
Cleanup(ctx context.Context, jobID string) error
}
JobRunner defines the interface for container runtime backends This abstraction allows the worker to spawn job containers using different runtimes (Docker, containerd, Kubernetes) without changing the core worker logic.
func NewJobRunner ¶
NewJobRunner creates a new JobRunner based on the specified backend Supported backends: "docker", "containerd", "kubernetes", "auto" "auto" will detect if running in Kubernetes and use that, otherwise Docker
func NewJobRunnerAuto ¶
NewJobRunnerAuto automatically detects the best runner backend It checks if running in Kubernetes first, then falls back to Docker
type JobSpec ¶
type JobSpec struct {
// Name is a human-readable name for the job
Name string `json:"name" yaml:"name"`
// Command is the full command to execute.
// Single-line commands are parsed and split on whitespace.
// Multiline commands are wrapped with "sh -c" by default (see CommandPrefix).
Command string `json:"command" yaml:"command"`
// CommandPrefix overrides the default shell wrapper for multiline commands.
// Default is "sh -c". Examples: "bash -c", "zsh -c", "/bin/ash -c"
// Only applies to multiline commands that don't already start with a shell invocation.
CommandPrefix string `json:"command_prefix" yaml:"command_prefix"`
// Image is the container image to use (defaults to DefaultRunnerImage)
Image string `json:"image" yaml:"image"`
// Environment variables to set in the container
// Values can contain ${secret:path:key} references that get resolved
Environment map[string]string `json:"environment" yaml:"environment"`
// Source defines how to prepare the source code
Source *SourceSpec `json:"source" yaml:"source"`
// WorkingDir is the working directory inside the container (default: /job)
WorkingDir string `json:"working_dir" yaml:"working_dir"`
// CodeDir is where source code is expected inside the job container.
// run-local mounts local code here; runnerlib checks out remote source here.
CodeDir string `json:"code_dir" yaml:"code_dir"`
// JobDir is the directory runnerlib treats as the job working directory.
// Defaults to CodeDir when empty.
JobDir string `json:"job_dir" yaml:"job_dir"`
// Capabilities declares what the job needs from the runtime environment.
// The runner interprets these based on its environment:
// - "docker": Access to build/push container images
// DockerRunner: mounts /var/run/docker.sock
// KubernetesRunner: uses DinD sidecar or hostPath
// - "gpu": Access to GPU resources (future - not yet implemented)
// DockerRunner: --gpus all
// KubernetesRunner: nvidia.com/gpu resource request
Capabilities []string `json:"capabilities" yaml:"capabilities"`
// Timeout in seconds (0 = no timeout)
TimeoutSeconds int `json:"timeout_seconds" yaml:"timeout_seconds"`
// Resource limits
CPULimit string `json:"cpu_limit" yaml:"cpu_limit"`
MemoryLimit string `json:"memory_limit" yaml:"memory_limit"`
// RunAs controls the container user for deployed workers. run-local uses
// this only as a fallback after run_local and CLI overrides.
RunAs *RunAsSpec `json:"run_as" yaml:"run_as"`
// DisableRunLocal marks this job as remote-only. run-local refuses to
// execute jobs with this set — used for jobs that fundamentally need CI
// state (PR diff base, push-back-to-remote with a PAT, etc.).
DisableRunLocal bool `json:"disable_run_local" yaml:"disable_run_local"`
// RunLocal holds settings that only affect `reactorcide run-local`. The
// worker ignores this block entirely (ToJobConfig never reads it), so it's
// the place to pin local-only behavior — e.g. the container uid — without
// changing how the job runs in CI. Defining it here makes a job behave
// consistently across local invocations without per-command flags.
RunLocal *RunLocalSpec `json:"run_local" yaml:"run_local"`
}
JobSpec represents a job definition that can be loaded from YAML/JSON files or constructed programmatically. This is the canonical representation of a job that gets converted to JobConfig for execution.
func LoadJobSpec ¶
LoadJobSpec reads a job specification from a YAML or JSON file. Supports both flat format (image/command at top level) and eval format (image/command nested under a "job" block with triggers/description).
func LoadJobSpecOverlay ¶
LoadJobSpecOverlay reads a partial job specification from a YAML/JSON file Unlike LoadJobSpec, this doesn't require command or set defaults
func (*JobSpec) ToJobConfig ¶
ToJobConfig converts a JobSpec to a JobConfig for execution workspaceDir is the host directory to mount into the container jobID is a unique identifier for this job execution
type KubernetesRunner ¶
type KubernetesRunner struct {
// contains filtered or unexported fields
}
KubernetesRunner implements JobRunner using Kubernetes Jobs It creates K8s Job resources and streams logs directly via the K8s API to ensure secret masking is applied before logs reach any aggregator.
func NewKubernetesRunner ¶
func NewKubernetesRunner() (*KubernetesRunner, error)
NewKubernetesRunner creates a new Kubernetes-based job runner It automatically detects if running in-cluster and uses the appropriate config
func NewKubernetesRunnerWithConfig ¶
func NewKubernetesRunnerWithConfig(cfg KubernetesRunnerConfig) (*KubernetesRunner, error)
NewKubernetesRunnerWithConfig creates a KubernetesRunner with custom configuration
func (*KubernetesRunner) Cleanup ¶
func (kr *KubernetesRunner) Cleanup(ctx context.Context, jobName string) error
Cleanup removes the Kubernetes Job resource
func (*KubernetesRunner) GetJobStatus ¶
func (kr *KubernetesRunner) GetJobStatus(ctx context.Context, jobName string) (status string, failureReason string, err error)
GetJobStatus returns the current status of a Kubernetes job Returns: running, succeeded, failed, or pending Also returns an error message if the job failed
func (*KubernetesRunner) Stop ¶
Stop requests a graceful shutdown of the job's pod(s) by deleting them with an explicit grace period. Deleting a pod (rather than patching terminationGracePeriodSeconds, which is immutable post-creation) is how Kubernetes lets a caller override the grace period per-delete: DeleteOptions.GracePeriodSeconds controls how long the kubelet waits between sending SIGTERM and SIGKILL to the pod's containers, independent of the pod spec's default. grace == 0 requests immediate SIGKILL (the `kubectl delete --grace-period=0 --force` equivalent).
SpawnJob sets BackoffLimit=0 on the owning Job resource, so when the kubelet-terminated pod disappears the Job controller counts it as a failure and transitions the Job to JobFailed without creating a replacement pod — exactly the condition WaitForCompletion's watch is looking for. The pod is deleted (and possibly fully gone) by the time WaitForCompletion tries to read its exit code via getPodExitCode, so callers should expect -1 rather than a real container exit code for a Stop-initiated termination; the caller identifies "this was a stop, not a real failure" out-of-band (the cancel-poll result), not from the exit code.
func (*KubernetesRunner) StreamLogs ¶
func (kr *KubernetesRunner) StreamLogs(ctx context.Context, jobName string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
StreamLogs streams stdout and stderr from the job pod Logs are streamed directly via K8s API, bypassing any file-based log collectors
func (*KubernetesRunner) WaitForCompletion ¶
WaitForCompletion waits for the Kubernetes Job to complete and returns the exit code
type KubernetesRunnerConfig ¶
type KubernetesRunnerConfig struct {
Namespace string // Namespace for job pods (default: current namespace)
ServiceAccount string // Service account for job pods (default: "default")
RunnerImage string // Default runner image if job doesn't specify one
DindImage string // Docker-in-Docker sidecar image (default: "docker:27-dind")
ImagePullSecrets []string // Image pull secrets for private registries
NodeName string // Node to schedule job pods on (for workspace sharing via HostPath)
}
KubernetesRunnerConfig holds configuration for the K8s runner
type LifecycleManager ¶
type LifecycleManager struct {
// contains filtered or unexported fields
}
LifecycleManager handles worker lifecycle events
func NewLifecycleManager ¶
func NewLifecycleManager(store store.Store) *LifecycleManager
NewLifecycleManager creates a new lifecycle manager
func (*LifecycleManager) GetActiveJobs ¶
func (lm *LifecycleManager) GetActiveJobs() []string
GetActiveJobs returns a list of currently active job IDs
func (*LifecycleManager) GracefulShutdown ¶
func (lm *LifecycleManager) GracefulShutdown(ctx context.Context) error
GracefulShutdown handles graceful shutdown of the worker
func (*LifecycleManager) IsShuttingDown ¶
func (lm *LifecycleManager) IsShuttingDown() bool
IsShuttingDown checks if the lifecycle manager is shutting down
func (*LifecycleManager) JobCleanupOnFailure ¶
func (lm *LifecycleManager) JobCleanupOnFailure(jobID string, err error)
JobCleanupOnFailure handles cleanup when a job fails
func (*LifecycleManager) RecoverJobs ¶
func (lm *LifecycleManager) RecoverJobs(ctx context.Context, workerID string) error
RecoverJobs checks for jobs that were running when the worker stopped
func (*LifecycleManager) RegisterJob ¶
func (lm *LifecycleManager) RegisterJob(job *models.Job, workDir string, cancel context.CancelFunc)
RegisterJob registers a job as active
func (*LifecycleManager) SetupSignalHandlers ¶
func (lm *LifecycleManager) SetupSignalHandlers(ctx context.Context, cancel context.CancelFunc)
SetupSignalHandlers sets up OS signal handlers for graceful shutdown
func (*LifecycleManager) UnregisterJob ¶
func (lm *LifecycleManager) UnregisterJob(jobID string)
UnregisterJob removes a job from active tracking
type LogEntry ¶
type LogEntry struct {
Timestamp string `json:"timestamp"`
Stream string `json:"stream"`
Level string `json:"level,omitempty"`
Message string `json:"message"`
}
LogEntry represents a single log line in JSON format
type LogShipper ¶
type LogShipper struct {
// contains filtered or unexported fields
}
LogShipper handles streaming logs to object storage in chunks
func NewLogShipper ¶
func NewLogShipper(config LogShipperConfig, masker *secrets.Masker) *LogShipper
NewLogShipper creates a new log shipper
func (*LogShipper) GetObjectKey ¶
func (ls *LogShipper) GetObjectKey() string
GetObjectKey returns the object key where logs are being stored
func (*LogShipper) GetTotalBytes ¶
func (ls *LogShipper) GetTotalBytes() int64
GetTotalBytes returns the total number of bytes written
func (*LogShipper) StreamAndShip ¶
func (ls *LogShipper) StreamAndShip(ctx context.Context, reader io.ReadCloser) (string, int64, error)
StreamAndShip reads from the input stream, masks secrets, and ships logs to object storage Returns the object key where logs were stored and total bytes written
type LogShipperConfig ¶
type LogShipperConfig struct {
ObjectStore objects.ObjectStore
JobID string
StreamType string // "stdout" or "stderr"
ChunkInterval time.Duration
OnChunkUploaded func(objectKey string, bytesWritten int64) error // Callback for chunk uploads
Publisher *pubsub.Publisher // optional: NOTIFY WS clients when a chunk is flushed
}
LogShipperConfig holds configuration for log shipping
type PodStartupError ¶
PodStartupError represents a pod that failed to start
func (*PodStartupError) Error ¶
func (e *PodStartupError) Error() string
type ResourceMetrics ¶
type ResourceMetrics struct {
Timestamp time.Time `json:"timestamp"`
// CPU metrics
CPUPercent float64 `json:"cpu_percent"`
CPUCores int `json:"cpu_cores"`
GoRoutines int `json:"go_routines"`
// Memory metrics
MemoryUsedMB uint64 `json:"memory_used_mb"`
MemoryTotalMB uint64 `json:"memory_total_mb"`
MemoryPercent float64 `json:"memory_percent"`
HeapAllocMB uint64 `json:"heap_alloc_mb"`
HeapSysMB uint64 `json:"heap_sys_mb"`
// Job metrics
ActiveJobs int `json:"active_jobs"`
MaxConcurrency int `json:"max_concurrency"`
JobsProcessed int64 `json:"jobs_processed"`
JobsFailed int64 `json:"jobs_failed"`
// Worker metrics
WorkerID string `json:"worker_id"`
Uptime time.Duration `json:"uptime"`
LastJobTime *time.Time `json:"last_job_time"`
}
ResourceMetrics holds current resource usage metrics
type ResourceMonitor ¶
type ResourceMonitor struct {
// contains filtered or unexported fields
}
ResourceMonitor monitors system resources and worker metrics
func NewResourceMonitor ¶
func NewResourceMonitor(workerID string, maxConcurrency int) (*ResourceMonitor, error)
NewResourceMonitor creates a new resource monitor
func (*ResourceMonitor) GetMetrics ¶
func (rm *ResourceMonitor) GetMetrics() ResourceMetrics
GetMetrics returns the current resource metrics
func (*ResourceMonitor) IsHealthy ¶
func (rm *ResourceMonitor) IsHealthy() bool
IsHealthy checks if the worker is operating within healthy parameters
func (*ResourceMonitor) LogMetricsSummary ¶
func (rm *ResourceMonitor) LogMetricsSummary()
LogMetricsSummary logs a summary of current metrics
func (*ResourceMonitor) RecordJobComplete ¶
func (rm *ResourceMonitor) RecordJobComplete(jobID string, success bool)
RecordJobComplete records that a job has completed
func (*ResourceMonitor) RecordJobStart ¶
func (rm *ResourceMonitor) RecordJobStart(jobID string)
RecordJobStart records that a job has started
func (*ResourceMonitor) SetThresholds ¶
func (rm *ResourceMonitor) SetThresholds(cpu, memory float64)
SetThresholds sets the resource monitoring thresholds
func (*ResourceMonitor) Start ¶
func (rm *ResourceMonitor) Start(ctx context.Context)
Start begins monitoring resources
type RetryConfig ¶
type RetryConfig struct {
MaxRetries int // Maximum number of retry attempts
InitialDelay time.Duration // Initial delay between retries
MaxDelay time.Duration // Maximum delay between retries
BackoffFactor float64 // Exponential backoff factor (e.g., 2.0)
JitterFraction float64 // Fraction of delay to add as random jitter (0.0-1.0)
}
RetryConfig holds configuration for retry logic
func DefaultRetryConfig ¶
func DefaultRetryConfig() *RetryConfig
DefaultRetryConfig returns the default retry configuration
type RetryableError ¶
RetryableError represents an error that can be retried
func ClassifyExecutionError ¶
func ClassifyExecutionError(err error, exitCode int) *RetryableError
ClassifyExecutionError classifies an execution error as retryable or not
func (*RetryableError) Error ¶
func (e *RetryableError) Error() string
func (*RetryableError) Unwrap ¶
func (e *RetryableError) Unwrap() error
type RunAsSpec ¶
type RunAsSpec struct {
// User accepts "runner", "root", or numeric "uid[:gid]". "host" is
// intentionally run-local-only and is not accepted by deployed workers.
User string `json:"user" yaml:"user"`
}
RunAsSpec controls the user identity for deployed job containers.
type RunLocalSpec ¶
type RunLocalSpec struct {
// AsRunner runs the job container as the image's conventional runner uid
// (1001:1001), matching the worker, instead of the host uid. This gives
// sudo and HOME parity for jobs that rely on the image's runner user.
AsRunner bool `json:"as_runner" yaml:"as_runner"`
// User pins an explicit uid[:gid] for the job container (e.g. "1001:1001").
// Takes precedence over AsRunner when both are set. Empty means unset.
User string `json:"user" yaml:"user"`
}
RunLocalSpec holds run-local-only overrides. The worker never reads this; it exists so a job can declare how it should be executed on a laptop.
type RunnerBackend ¶
type RunnerBackend string
RunnerBackend represents the container runtime backend to use
const ( // BackendDocker uses the Docker daemon BackendDocker RunnerBackend = "docker" // BackendContainerd uses containerd via nerdctl BackendContainerd RunnerBackend = "containerd" // BackendKubernetes uses Kubernetes Jobs BackendKubernetes RunnerBackend = "kubernetes" // BackendAuto automatically detects the best backend BackendAuto RunnerBackend = "auto" )
func GetSupportedBackends ¶
func GetSupportedBackends() []RunnerBackend
GetSupportedBackends returns a list of all supported runner backends
type SecretOverride ¶
type SecretOverride struct {
Key string
OldValue string // The ${secret:...} reference
NewValue string // The plaintext value
OverlayFile string
}
SecretOverride represents a case where a secret reference was overridden
type SecretResolutionResult ¶
type SecretResolutionResult struct {
// Resolved contains all environment variables with secrets resolved
Resolved map[string]string
// SecretValues contains the actual secret values for masking
SecretValues []string
// SecretEnvNames contains the names of env vars that contained secret references
SecretEnvNames []string
}
SecretResolutionResult holds the results of resolving secrets in environment variables
func ResolveSecretsInEnvFull ¶
func ResolveSecretsInEnvFull(env map[string]string, getSecret func(path, key string) (string, error)) (*SecretResolutionResult, error)
ResolveSecretsInEnvFull resolves all secret references and returns full result including env var names
type SourceSpec ¶
type SourceSpec struct {
Type string `json:"type" yaml:"type"` // git, copy, none
URL string `json:"url" yaml:"url"`
Ref string `json:"ref" yaml:"ref"`
Path string `json:"path" yaml:"path"` // for copy type
}
SourceSpec defines source code preparation
type TriggerProcessor ¶
type TriggerProcessor struct {
// contains filtered or unexported fields
}
TriggerProcessor handles reading triggers.json from completed eval jobs and creating/submitting the triggered jobs to Corndogs.
func NewTriggerProcessor ¶
func NewTriggerProcessor(store store.Store, corndogsClient corndogs.ClientInterface) *TriggerProcessor
NewTriggerProcessor creates a new TriggerProcessor.
func (*TriggerProcessor) EvaluateWorkflow ¶
func (tp *TriggerProcessor) EvaluateWorkflow(ctx context.Context, wf *models.WorkflowInstance) ([]string, error)
EvaluateWorkflow is the exported form of evaluateWorkflow, used by internal/jobcontrol.RetryWorkflow to drive initial node submission for a freshly created workflow instance exactly the way ProcessTriggersFromData drives it for a brand-new one (same dependency/condition evaluation, same submitWorkflowNode path, same refreshWorkflowStatus at the end) — the alternative would be reimplementing that evaluation loop a second time in jobcontrol, which risks drifting from the worker's own semantics. The caller must pass a store implementing this package's full workflowStore interface (postgres_store.PostgresDbStore does); against a narrower store this returns ErrWorkflowsUnsupported-equivalent errors the same way ProcessTriggersFromData's own callers would see.
func (*TriggerProcessor) ProcessTriggers ¶
func (tp *TriggerProcessor) ProcessTriggers(ctx context.Context, workspaceDir string, parentJob *models.Job) error
ProcessTriggers reads triggers.json from the workspace directory of a completed eval job, creates the triggered jobs in the database, and submits them to Corndogs.
func (*TriggerProcessor) ProcessTriggersFromData ¶
func (tp *TriggerProcessor) ProcessTriggersFromData(ctx context.Context, data []byte, workspaceDir string, parentJob *models.Job) ([]string, error)
ProcessTriggersFromData processes raw trigger JSON data, creates the triggered jobs in the database, submits them to Corndogs, and returns the created job IDs. workspaceDir is the host workspace directory used to resolve job_file references.
func (*TriggerProcessor) ProcessWorkflowCompletion ¶
func (*TriggerProcessor) ProcessWorkflowJobStarted ¶
func (*TriggerProcessor) SetStatusUpdater ¶
func (tp *TriggerProcessor) SetStatusUpdater(u vcs.JobStatusUpdaterInterface)
SetStatusUpdater wires a VCS status updater so that newly-created child jobs get registered as pending checks on their commit the moment they exist in the database, before the worker picks them up.
type VCSAuthConfig ¶
type VCSAuthConfig struct {
ContainerDir string
GitConfig string
Credentials string
SecretValues []string
}
VCSAuthConfig contains runtime Git auth material for source preparation. Secret values must not be logged or exposed as environment values.
Source Files
¶
- builder_config.go
- containerd_runner.go
- corndogs_worker.go
- docker_runner.go
- interfaces.go
- job_processor.go
- job_spec.go
- kubernetes_runner.go
- lifecycle.go
- log_muxer.go
- log_shipper.go
- monitor.go
- retry.go
- run_as.go
- runner_factory.go
- secret_authorization.go
- trigger_processor.go
- vcs_checkout_auth.go
- worker.go
- workflow_runtime.go