Documentation
¶
Index ¶
- Constants
- Variables
- func AuthorizeSecretAccess(ctx context.Context, grantStore SecretGrantStore, job *models.Job, ...) error
- func BuildJobEnv(job *models.Job) map[string]string
- func BuildTaskPayload(job *models.Job) *corndogs.TaskPayload
- func BuilderSidecarName(jobID string) string
- func CombineImagePullSecrets(global, jobLevel []string) []string
- func ComputeWorkflowStatus(nodes []models.WorkflowNode) string
- func ConfigureVMPlainHTTPRegistries(hosts []string)
- func ContainerPathInsideJob(path string) string
- func DefaultJobCodeDir(codeDir string) string
- func DefaultJobDir(codeDir, jobDir string) string
- func DefaultRunAsUser(user string) (string, error)
- func EncodeWorkflowVars(values map[string]models.JSONB) ([]byte, error)
- func EnforceImagePullSecretAllowlist(requested, global, allowed []string) error
- func FinalizeJobSpec(spec *JobSpec) error
- func HasCapability(caps []string, want string) bool
- func HasEnvRefs(s string) bool
- func HasSecretRefs(s string) bool
- func IsBackendImplemented(backend string) bool
- func IsBackendSupported(backend string) bool
- func IsInvalidTriggerError(err error) bool
- func IsKubernetesEnvironment() bool
- func IsPodStartupError(err error) bool
- func IsRetryable(err error) bool
- func LoadJobSpecWithOverlays(jobPath string, overlayPaths []string) (*JobSpec, []SecretOverride, error)
- func MergeJobSpecs(base *JobSpec, overlays []*JobSpec, overlayFiles []string) (*JobSpec, []SecretOverride)
- func NormalizeRunAsUser(user string) (string, error)
- func ParseCommand(cmd string) []string
- func ParseCommandWithPrefix(cmd, prefix string) []string
- func ResolveEnvInMap(env map[string]string) map[string]string
- func ResolveEnvRefs(value string) string
- func ResolveSecretRefs(value string, getSecret func(path, key string) (string, error)) (string, error)
- func ResolveSecretsInEnv(env map[string]string, getSecret func(path, key string) (string, error)) (map[string]string, []string, error)
- func RetryWithBackoff(ctx context.Context, config *RetryConfig, operation string, fn func() error) error
- func RetryWithBackoffCounter(ctx context.Context, config *RetryConfig, operation string, ...) error
- func SecretPathAllowed(patterns []string, value string) bool
- func ValidateImagePullSecretNames(names []string) error
- type BuilderConfig
- type CheckoutSpec
- type ContainerdRunner
- func (cr *ContainerdRunner) Cleanup(ctx context.Context, containerID string) error
- func (cr *ContainerdRunner) SampleResources(ctx context.Context, jobID string, options ResourceSampleOptions) (ResourceSnapshot, error)
- func (cr *ContainerdRunner) SpawnJob(ctx context.Context, config *JobConfig) (string, error)
- func (cr *ContainerdRunner) Stop(ctx context.Context, containerID string, grace time.Duration) error
- func (cr *ContainerdRunner) StreamLogs(ctx context.Context, containerID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
- func (cr *ContainerdRunner) WaitForCompletion(ctx context.Context, containerID string) (int, error)
- type DockerRunner
- func (dr *DockerRunner) Cleanup(ctx context.Context, containerID string) error
- func (dr *DockerRunner) SampleResources(ctx context.Context, jobID string, options ResourceSampleOptions) (ResourceSnapshot, error)
- func (dr *DockerRunner) SpawnJob(ctx context.Context, config *JobConfig) (string, error)
- func (dr *DockerRunner) Stop(ctx context.Context, containerID string, grace time.Duration) error
- func (dr *DockerRunner) StreamLogs(ctx context.Context, containerID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
- func (dr *DockerRunner) WaitForCompletion(ctx context.Context, containerID string) (int, error)
- type InvalidTriggerError
- type JobConfig
- type JobRunner
- type JobSpec
- type KubernetesRunner
- func (kr *KubernetesRunner) Cleanup(ctx context.Context, jobName string) error
- func (kr *KubernetesRunner) GetJobStatus(ctx context.Context, jobName string) (status string, failureReason string, err error)
- func (kr *KubernetesRunner) SampleResources(ctx context.Context, jobName string, options ResourceSampleOptions) (ResourceSnapshot, error)
- func (kr *KubernetesRunner) SpawnJob(ctx context.Context, config *JobConfig) (string, error)
- func (kr *KubernetesRunner) Stop(ctx context.Context, jobName string, grace time.Duration) error
- func (kr *KubernetesRunner) StreamLogs(ctx context.Context, jobName string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
- func (kr *KubernetesRunner) TakeWorkflowOutput(jobName string) (string, bool)
- func (kr *KubernetesRunner) WaitForCompletion(ctx context.Context, jobName string) (int, error)
- type KubernetesRunnerConfig
- type LogEntry
- type PodStartupError
- type ResourceSampleOptions
- type ResourceSnapshot
- type RetryConfig
- type RetryableError
- type RunAsSpec
- type RunLocalSpec
- type RunnerBackend
- type SecretGrantStore
- type SecretOverride
- type SecretResolutionResult
- type SourceSpec
- type TriggerProcessingResult
- type TriggerProcessor
- func (tp *TriggerProcessor) EvaluateWorkflow(ctx context.Context, wf *models.WorkflowInstance) ([]string, error)
- func (tp *TriggerProcessor) ProcessTriggers(ctx context.Context, workspaceDir string, parentJob *models.Job) error
- func (tp *TriggerProcessor) ProcessTriggersFromData(ctx context.Context, data []byte, workspaceDir string, parentJob *models.Job) ([]string, error)
- func (tp *TriggerProcessor) ProcessTriggersFromDataWithOutcomes(ctx context.Context, data []byte, workspaceDir string, parentJob *models.Job) (TriggerProcessingResult, error)
- func (tp *TriggerProcessor) ProcessWorkflowCompletion(ctx context.Context, workspaceDir string, job *models.Job) error
- func (tp *TriggerProcessor) ProcessWorkflowCompletionData(ctx context.Context, data []byte, job *models.Job) error
- func (tp *TriggerProcessor) ProcessWorkflowJobStarted(ctx context.Context, job *models.Job) error
- func (tp *TriggerProcessor) SetStatusUpdater(u vcs.JobStatusUpdaterInterface)
- type TriggerWorkflowOutcome
- type VCSAuthConfig
- type VMConfig
- type VMImageCacheManager
- type WorkflowOutputReader
Constants ¶
const ( // CapabilityDocker provides access to a docker CLI for running ad-hoc // containers (e.g. integration tests). Implementations historically // privilege the job container and expose a runtime socket. Prefer // CapabilityBuilder for building images — that leaves the job container // unprivileged. CapabilityDocker = "docker" // CapabilityBuilder provisions a buildkitd sidecar and injects // BUILDKIT_HOST=tcp://localhost:1234 into the job, so jobs can build and // push container images without privileged access or host runtime // coupling. The sidecar's registry policy (insecure registries, CA // bundles, etc.) comes from operator-provided buildkitd.toml, not from // the job. CapabilityBuilder = "builder" // CapabilityGPU provides access to GPU resources. NOT YET IMPLEMENTED - // placeholder for future development. DockerRunner: would use --gpus all // flag KubernetesRunner: would add nvidia.com/gpu resource request CapabilityGPU = "gpu" )
Capability constants for job requirements
const ( RunnerUser = "1001:1001" RootUser = "0:0" )
const BuilderHostEnv = "BUILDKIT_HOST"
BuilderHostEnv is the env var set on the job container when CapabilityBuilder is present, pointing buildctl at the sidecar.
const BuilderLogPrefix = "[builder] "
BuilderLogPrefix is prepended to every line of sidecar log output so operators can tell at a glance which container produced which line. The callers that apply secret masking read line-by-line, so a deterministic line-level prefix preserves masker correctness.
const BuilderSidecarPort = 1234
BuilderSidecarPort is the TCP port buildkitd listens on inside its sidecar container. Jobs reach it via BUILDKIT_HOST=tcp://localhost:1234 when they share netns with the sidecar.
const DefaultBuilderImage = "moby/buildkit:v0.17.3"
DefaultBuilderImage is the buildkitd image used for builder sidecars when REACTORCIDE_BUILDER_IMAGE is not set.
const DefaultCancelGrace = 60 * time.Second
DefaultCancelGrace is the fallback grace period used when a caller (e.g. internal/workerapi's lease construction) doesn't have an explicit operator-configured cancel grace. Mirrors REACTORCIDE_CANCEL_GRACE_SECONDS' own default in internal/config.
const DefaultRunnerImage = "containers.catalystsquad.com/public/reactorcide/runnerbase:dev"
DefaultRunnerImage is the default container image for job execution
Variables ¶
var EnvRefPattern = regexp.MustCompile(`\$\{env:([^}]+)\}`)
EnvRefPattern matches ${env:VAR_NAME} references in strings This allows job YAMLs to reference host environment variables
var SecretRefPattern = regexp.MustCompile(`\$\{secret:([^:}]+):([^}]+)\}`)
SecretRefPattern matches ${secret:path:key} references in strings
Functions ¶
func AuthorizeSecretAccess ¶
func AuthorizeSecretAccess(ctx context.Context, grantStore SecretGrantStore, job *models.Job, path, key string) error
AuthorizeSecretAccess is the free-function core of secret-grant authorization: a job-scoped secret path (isJobScopedSecret) is always allowed; anything else requires a matching models.SecretGrant row (looked up via grantStore, or denied outright if grantStore is nil -- e.g. a store that doesn't implement SecretGrantStore). internal/workerapi's RequestJob calls this directly so a coordinator-mediated worker's job is authorized identically to (the now-removed) local worker's decision.
func BuildJobEnv ¶
BuildJobEnv builds a job's base environment map (system REACTORCIDE_*/ RC_WF_* vars, source/CI-source config, API credentials, and the job's own JobEnvVars). Exported so internal/workerapi's coordinator-mediated RequestJob can build an identical base env for a lease without duplicating this logic...} refs out of this map (see AuthorizeSecretAccess and ResolveSecretsInEnvFull).
func BuildTaskPayload ¶
func BuildTaskPayload(job *models.Job) *corndogs.TaskPayload
BuildTaskPayload is the exported, receiver-free form of buildTaskPayload: it depends only on the job, not on any TriggerProcessor field, so it's safe to call from other packages that need to mirror the exact submission shape trigger_processor.go/workflow_runtime.go use — currently internal/jobcontrol.RetryJob, which resubmits a cloned job the same way a freshly triggered or workflow-node job is submitted.
func BuilderSidecarName ¶
BuilderSidecarName returns the deterministic sidecar container/pod-container name for a given job id. Runners use this so cleanup can find the sidecar without extra state.
func CombineImagePullSecrets ¶
CombineImagePullSecrets returns the operator's global list followed by the job-level names, first occurrence wins, order preserved.
func ComputeWorkflowStatus ¶
func ComputeWorkflowStatus(nodes []models.WorkflowNode) string
ComputeWorkflowStatus is the exported form of computeWorkflowStatus, used by internal/jobcontrol (CancelWorkflow) so the workflow-cancel cascade and the normal per-node completion path (refreshWorkflowStatus, above) agree on exactly one status-derivation rule instead of maintaining two.
func ConfigureVMPlainHTTPRegistries ¶
func ConfigureVMPlainHTTPRegistries(hosts []string)
ConfigureVMPlainHTTPRegistries supplies explicit CLI-only registry exceptions before the worker constructs its VM backend.
func ContainerPathInsideJob ¶
func DefaultJobCodeDir ¶
func DefaultJobDir ¶
func DefaultRunAsUser ¶
func EncodeWorkflowVars ¶
EncodeWorkflowVars returns the JSON object that runnerlib receives. The database uses a JSON object wrapper because WorkflowVar.Value is a JSONB map. This function removes that storage-only wrapper for every value.
func EnforceImagePullSecretAllowlist ¶
EnforceImagePullSecretAllowlist rejects any requested name that is in neither the operator's global list (applied to every job pod) nor the job-level allowlist. Secure default: with both lists empty, every request is rejected.
func FinalizeJobSpec ¶
FinalizeJobSpec applies built-in defaults and validates a resolved job.
func HasCapability ¶
HasCapability returns true if caps contains the given capability.
func HasEnvRefs ¶
HasEnvRefs checks if a string contains environment variable references
func HasSecretRefs ¶
HasSecretRefs checks if a string contains secret references
func IsBackendImplemented ¶
IsBackendImplemented checks if a backend is fully implemented (not just stubbed)
func IsBackendSupported ¶
IsBackendSupported checks if a backend is supported (though may not be fully implemented)
func IsInvalidTriggerError ¶
IsInvalidTriggerError reports whether trigger processing rejected the request shape before it started durable workflow work.
func IsKubernetesEnvironment ¶
func IsKubernetesEnvironment() bool
IsKubernetesEnvironment checks if the code is running inside a Kubernetes cluster
func IsPodStartupError ¶
IsPodStartupError checks if an error is a pod startup failure Uses errors.As to handle wrapped errors
func LoadJobSpecWithOverlays ¶
func LoadJobSpecWithOverlays(jobPath string, overlayPaths []string) (*JobSpec, []SecretOverride, error)
LoadJobSpecWithOverlays loads a job spec and applies overlay files in order. The overlay files are specified from highest to lowest priority (first file wins). Returns the merged spec and any warnings about secret overrides.
func MergeJobSpecs ¶
func MergeJobSpecs(base *JobSpec, overlays []*JobSpec, overlayFiles []string) (*JobSpec, []SecretOverride)
MergeJobSpecs merges overlay specs onto a base spec. Overlays are applied in order, with later overlays taking precedence. Returns the merged spec and any warnings about secret overrides.
func NormalizeRunAsUser ¶
NormalizeRunAsUser converts a job run_as.user value into a runtime user string. Deployed workers intentionally do not support "host" because there is no stable host user to map to across VM and Kubernetes runtimes.
func ParseCommand ¶
ParseCommand splits a command string for container execution. Uses default "sh -c" prefix for multiline commands.
func ParseCommandWithPrefix ¶
ParseCommandWithPrefix converts a command string to []string for container execution. For multiline commands, wraps with the specified prefix (default "sh -c"). For single-line commands, splits on whitespace respecting basic quoting.
func ResolveEnvInMap ¶
ResolveEnvInMap resolves ${env:VAR_NAME} references in all values of a map
func ResolveEnvRefs ¶
ResolveEnvRefs resolves ${env:VAR_NAME} references in a string using os.Getenv to get values from the host environment
func ResolveSecretRefs ¶
func ResolveSecretRefs(value string, getSecret func(path, key string) (string, error)) (string, error)
ResolveSecretRefs resolves ${secret:path:key} references in a string using the provided getter function
func ResolveSecretsInEnv ¶
func ResolveSecretsInEnv(env map[string]string, getSecret func(path, key string) (string, error)) (map[string]string, []string, error)
ResolveSecretsInEnv resolves all secret references in environment variables Returns a new map with resolved values, a list of resolved secret values for masking, and a list of env var names that contained secrets. Note: ${env:VAR} references should be resolved first using ResolveEnvInMap
func RetryWithBackoff ¶
func RetryWithBackoff(ctx context.Context, config *RetryConfig, operation string, fn func() error) error
RetryWithBackoff executes a function with exponential backoff retry logic
func RetryWithBackoffCounter ¶
func RetryWithBackoffCounter(ctx context.Context, config *RetryConfig, operation string, fn func(attempt int) error) error
RetryWithBackoffCounter executes a function with exponential backoff retry logic and provides attempt counter
func SecretPathAllowed ¶
SecretPathAllowed applies an execution profile secret-path allowlist.
func ValidateImagePullSecretNames ¶
ValidateImagePullSecretNames rejects empty, over-length, non-DNS-subdomain, and duplicate names. It runs at every parse/submit boundary; the worker re-validates before Kubernetes Job creation so coordinator validation is not load-bearing.
Types ¶
type BuilderConfig ¶
type BuilderConfig struct {
// Image is the buildkitd container image. Defaults to DefaultBuilderImage.
Image string
// ConfigPath is a host file path to a buildkitd.toml that will be bind-
// mounted at /etc/buildkit/buildkitd.toml inside the sidecar. Empty means
// the sidecar uses its image's built-in defaults.
ConfigPath string
// RegistryAuthPath is a host file path to a docker registry auth file
// (config.json format) mounted into the sidecar at /root/.docker/config.json.
// Buildkit uses this when pulling base images. Push credentials are
// typically supplied client-side by the job via buildctl.
RegistryAuthPath string
// CacheVolume is an optional named docker volume (or k8s PVC name) mounted
// at buildkit's state dir for cross-job layer cache. Empty means no cache
// volume — each sidecar starts clean.
CacheVolume string
}
BuilderConfig holds operator-level configuration for the buildkitd sidecar launched for jobs with CapabilityBuilder. Runners pick the fields they need and ignore the rest; the zero value is a valid "no operator overrides" state in which the sidecar runs with its image defaults.
func LoadBuilderConfig ¶
func LoadBuilderConfig() BuilderConfig
LoadBuilderConfig resolves BuilderConfig from environment variables. This is the single source of truth for operator knobs across all runners.
Env vars:
- REACTORCIDE_BUILDER_IMAGE (default DefaultBuilderImage)
- REACTORCIDE_BUILDER_CONFIG_PATH (optional, no default)
- REACTORCIDE_BUILDER_REGISTRY_AUTH_PATH (optional, no default)
- REACTORCIDE_BUILDER_CACHE_VOLUME (optional, no default)
type CheckoutSpec ¶
type CheckoutSpec struct {
Mode string `json:"mode" yaml:"mode"`
}
type ContainerdRunner ¶
type ContainerdRunner struct {
// contains filtered or unexported fields
}
ContainerdRunner implements JobRunner using nerdctl CLI This approach lets nerdctl handle networking (CNI) automatically
func NewContainerdRunner ¶
func NewContainerdRunner() (*ContainerdRunner, error)
NewContainerdRunner creates a new nerdctl-based job runner
func (*ContainerdRunner) Cleanup ¶
func (cr *ContainerdRunner) Cleanup(ctx context.Context, containerID string) error
Cleanup removes the container and associated resources
func (*ContainerdRunner) SampleResources ¶
func (cr *ContainerdRunner) SampleResources(ctx context.Context, jobID string, options ResourceSampleOptions) (ResourceSnapshot, error)
func (*ContainerdRunner) Stop ¶
func (cr *ContainerdRunner) Stop(ctx context.Context, containerID string, grace time.Duration) error
Stop requests a graceful shutdown of the job container via `nerdctl stop`, which sends SIGTERM to PID 1 and waits up to `grace` seconds before sending SIGKILL. grace == 0 requests immediate termination (nerdctl stop -t 0 sends SIGTERM immediately followed by SIGKILL with no wait).
func (*ContainerdRunner) StreamLogs ¶
func (cr *ContainerdRunner) StreamLogs(ctx context.Context, containerID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
StreamLogs returns stdout and stderr readers for a container
func (*ContainerdRunner) WaitForCompletion ¶
WaitForCompletion waits for the container to exit and returns the exit code
type DockerRunner ¶
type DockerRunner struct {
// contains filtered or unexported fields
}
DockerRunner implements JobRunner using the Docker daemon
func NewDockerRunner ¶
func NewDockerRunner() (*DockerRunner, error)
NewDockerRunner creates a new Docker-based job runner Uses the default Docker socket (unix:///var/run/docker.sock or npipe on Windows)
func (*DockerRunner) Cleanup ¶
func (dr *DockerRunner) Cleanup(ctx context.Context, containerID string) error
Cleanup removes the container and any builder sidecar launched for it.
func (*DockerRunner) SampleResources ¶
func (dr *DockerRunner) SampleResources(ctx context.Context, jobID string, options ResourceSampleOptions) (ResourceSnapshot, error)
func (*DockerRunner) Stop ¶
Stop requests a graceful shutdown of the job container: the Docker daemon sends SIGTERM to PID 1 and waits up to `grace` before sending SIGKILL. grace == 0 skips the wait and kills immediately (Docker's ContainerStop with Timeout=0 still sends SIGTERM first, but does not wait for it to take effect before following up with SIGKILL).
Any builder sidecar is left running; the caller's subsequent Cleanup() call tears down both the job container and its sidecar together.
func (*DockerRunner) StreamLogs ¶
func (dr *DockerRunner) StreamLogs(ctx context.Context, containerID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
StreamLogs streams stdout and stderr from the container
func (*DockerRunner) WaitForCompletion ¶
WaitForCompletion waits for the container to exit and returns the exit code
type InvalidTriggerError ¶
type InvalidTriggerError struct {
// contains filtered or unexported fields
}
InvalidTriggerError identifies a request that the client can correct.
func (*InvalidTriggerError) Error ¶
func (e *InvalidTriggerError) Error() string
func (*InvalidTriggerError) Unwrap ¶
func (e *InvalidTriggerError) Unwrap() error
type JobConfig ¶
type JobConfig struct {
// Container image to use (e.g., "reactorcide/runner:latest")
Image string
// Command to execute in the container. The container's entrypoint is
// cleared, so this is the full command (e.g., ["sh", "-c", "make build"])
Command []string
// Environment variables to inject into the container
Env map[string]string
// WorkspaceDir is the host directory to mount into the container at /job
WorkspaceDir string
// SourceDir is an optional host directory to mount at /job/src in the
// container. Used by run-local to mount user's source into the standard
// production layout.
SourceDir string
// SourceMountPath is the container path where SourceDir is mounted.
// Defaults to /job/src when empty.
SourceMountPath string
// WorkingDir is the working directory inside the container (default:
// /job)
WorkingDir string
// Capabilities declares what the job needs from the runtime environment.
// Each runner interprets these appropriately for its environment. See
// CapabilityDocker, CapabilityGPU constants.
Capabilities []string
// ImagePullSecrets lists names of Kubernetes Secrets the job requested
// for pulling its image. Names only, never values. KubernetesRunner
// enforces its allowlist and adds approved names to the pod spec's
// imagePullSecrets; other runners preserve the field and never read a
// Kubernetes Secret.
ImagePullSecrets []string
// RunAsUser optionally overrides the container user ("uid:gid"). When
// empty, runners default to RunnerUser. Capabilities provision runtime
// services/privileges but do not implicitly change the job user.
// run-local sets this to the host uid so bind-mounted sources are
// writable.
RunAsUser string
// ExtraMounts are additional bind mounts in "hostpath:containerpath[:ro]"
// format. Used by run-local to inject a synthetic /etc/passwd and
// /etc/group for the host uid so tools like ssh find a valid user entry.
ExtraMounts []string
// VCSAuth contains per-job checkout credential files. Docker/containerd
// jobs read these from WorkspaceDir; Kubernetes jobs materialize them as
// a short-lived Secret copied into an emptyDir.
VCSAuth *VCSAuthConfig
// Timeout for the job execution (0 = no timeout)
TimeoutSeconds int
// Resource requests/limits. Values are Kubernetes-style quantity strings
// (https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/,
// validated by k8s.io/apimachinery/pkg/api/resource -- see
// internal/resources.ParseResources): "1", "2", "500m" for CPU; "4Gi",
// "512Mi" for memory. Memory is limit-only -- there is no MemoryRequest,
// mirroring the Job model and DB schema (memory is a pure ceiling, not a
// reservation). CPURequest is advisory for runners that have no native
// CPU reservation concept (see DockerRunner: mapped to --cpu-shares);
// KubernetesRunner sets it as a real requests.cpu.
CPURequest string // e.g., "1" or "500m"; advisory outside Kubernetes
CPULimit string // e.g., "1", "2", or "500m"
MemoryLimit string // e.g., "512Mi" or "4Gi"
// Job metadata (for labeling/tagging)
JobID string
QueueName string
}
JobConfig contains all the configuration needed to spawn a job container. The container's entrypoint is always cleared - the Command field contains the full command to execute. Users can run their own commands or invoke runnerlib for source preparation and lifecycle hooks.
type JobRunner ¶
type JobRunner interface {
// SpawnJob creates and starts a job container with the specified
// configuration Returns a unique job ID/handle and any error encountered
SpawnJob(ctx context.Context, config *JobConfig) (string, error)
// StreamLogs streams stdout/stderr from a running job container Returns
// separate readers for stdout and stderr
StreamLogs(ctx context.Context, jobID string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
// WaitForCompletion blocks until the job container exits Returns the exit
// code and any error encountered
WaitForCompletion(ctx context.Context, jobID string) (int, error)
// Stop requests a graceful shutdown of a running job: SIGTERM to the
// container's PID 1 (giving runnerlib's SIGTERM trap a chance to run
// PluginPhase.CLEANUP/ON_ERROR and cleanup_vcs_auth), then a forced kill
// if the container/pod hasn't exited within grace. Stop does not remove
// the container/pod — callers still call Cleanup afterward
// (WaitForCompletion unblocking is what tells the caller it's safe to do
// so).
//
// grace == 0 requests immediate forced termination (no SIGTERM wait) —
// used by the admin "kill" path when it needs an unambiguous immediate
// stop rather than going through Cleanup directly (e.g. to unblock a
// stuck WaitForCompletion caller). Cleanup remains the primary "kill"
// primitive at the caller layer — Stop(grace=0) exists so runner backends
// have a single, testable primitive for "terminate now".
//
// Calling Stop on a container/pod that has already exited or been removed
// must be a safe no-op (implementations should tolerate "not found" style
// errors rather than returning them as failures), since the worker's
// cancel-poll and the container's own natural completion can race.
Stop(ctx context.Context, jobID string, grace time.Duration) error
// Cleanup removes the job container and associated resources Should be
// called after the job completes (success or failure)
Cleanup(ctx context.Context, jobID string) error
// SampleResources returns one resource snapshot for a running job. A
// backend omits unavailable values and returns safe availability reasons.
SampleResources(ctx context.Context, jobID string, options ResourceSampleOptions) (ResourceSnapshot, error)
}
JobRunner defines the interface for container runtime backends This abstraction allows the worker to spawn job containers using different runtimes (Docker, containerd, Kubernetes) without changing the core worker logic.
func NewJobRunner ¶
NewJobRunner creates a new JobRunner based on the specified backend Supported backends: "docker", "containerd", "kubernetes", "vm", "auto" "auto" will detect if running in Kubernetes and use that, otherwise Docker
func NewJobRunnerAuto ¶
NewJobRunnerAuto automatically detects the best runner backend It checks if running in Kubernetes first, then falls back to Docker
type JobSpec ¶
type JobSpec struct {
// Name is a human-readable name for the job
Name string `json:"name" yaml:"name"`
// Command is the full command to execute. Single-line commands are parsed
// and split on whitespace. Multiline commands are wrapped with "sh -c" by
// default (see CommandPrefix).
Command string `json:"command" yaml:"command"`
// CommandPrefix overrides the default shell wrapper for multiline
// commands. Default is "sh -c". Examples: "bash -c", "zsh -c", "/bin/ash
// -c" Only applies to multiline commands that don't already start with a
// shell invocation.
CommandPrefix string `json:"command_prefix" yaml:"command_prefix"`
// Image is the container image to use (defaults to DefaultRunnerImage)
Image string `json:"image" yaml:"image"`
// Environment variables to set in the container Values can contain
// ${secret:path:key} references that get resolved
Environment map[string]string `json:"environment" yaml:"environment"`
// Source defines how to prepare the source code
Source *SourceSpec `json:"source" yaml:"source"`
// Checkout selects the runnerlib checkout layout. It is also supported by
// overlay files. Isolated keeps separate clones. Shared uses one filtered
// object store for pull-request evaluation.
Checkout *CheckoutSpec `json:"checkout" yaml:"checkout"`
// WorkingDir is the working directory inside the container (default: /job)
WorkingDir string `json:"working_dir" yaml:"working_dir"`
// CodeDir is where source code is expected inside the job container.
// run-local mounts local code here; runnerlib checks out remote source
// here.
CodeDir string `json:"code_dir" yaml:"code_dir"`
// JobDir is the directory runnerlib treats as the job working directory.
// Defaults to CodeDir when empty.
JobDir string `json:"job_dir" yaml:"job_dir"`
// Capabilities declares what the job needs from the runtime environment.
// The runner interprets these based on its environment:
// - "docker": Access to build/push container images
// DockerRunner: mounts /var/run/docker.sock
// KubernetesRunner: uses DinD sidecar or hostPath
// - "gpu": Access to GPU resources (future - not yet implemented)
// DockerRunner: --gpus all
// KubernetesRunner: nvidia.com/gpu resource request
Capabilities []string `json:"capabilities" yaml:"capabilities"`
// ImagePullSecrets lists the NAMES of Kubernetes Secrets (type
// kubernetes.io/dockerconfigjson, in the job namespace) the job pod may
// use to pull its image. Names only — never credentials. The worker
// enforces its operator allowlist before Kubernetes Job creation; the
// Docker/containerd/local runners preserve the field but never read a
// Kubernetes Secret.
ImagePullSecrets []string `json:"image_pull_secrets" yaml:"image_pull_secrets"`
// Timeout in seconds (0 = no timeout)
TimeoutSeconds int `json:"timeout_seconds" yaml:"timeout_seconds"`
// Resource limits
CPULimit string `json:"cpu_limit" yaml:"cpu_limit"`
MemoryLimit string `json:"memory_limit" yaml:"memory_limit"`
// RunAs controls the container user for deployed workers. run-local uses
// this only as a fallback after run_local and CLI overrides.
RunAs *RunAsSpec `json:"run_as" yaml:"run_as"`
// DisableRunLocal marks this job as remote-only. run-local refuses to
// execute jobs with this set — used for jobs that fundamentally need CI
// state (PR diff base, push-back-to-remote with a PAT, etc.).
DisableRunLocal bool `json:"disable_run_local" yaml:"disable_run_local"`
// RunLocal holds settings that only affect `reactorcide run-local`. The
// worker ignores this block entirely (ToJobConfig never reads it), so
// it's the place to pin local-only behavior — e.g. the container uid —
// without changing how the job runs in CI. Defining it here makes a job
// behave consistently across local invocations without per-command flags.
RunLocal *RunLocalSpec `json:"run_local" yaml:"run_local"`
// Characteristics routes this job to a queue when submitted remotely; raw
// map, validated by internal/characteristics.ParseJobCharacteristics at
// submit time. Not consumed by run-local/ToJobConfig -- queue routing
// only applies to jobs submitted through the coordinator API (see
// cmd/submit.go).
Characteristics map[string]interface{} `json:"characteristics" yaml:"characteristics"`
WorkerClass string `json:"worker_class" yaml:"worker_class"`
// Resources declares per-job compute resource cpu.request/cpu.limit/
// memory.limit; raw map, validated by internal/resources.ParseResources
// at submit time. Not consumed by run-local/ToJobConfig -- see the flat
// CPULimit/MemoryLimit fields above for that path.
Resources map[string]interface{} `json:"resources" yaml:"resources"`
}
JobSpec represents a job definition that can be loaded from YAML/JSON files or constructed programmatically. This is the canonical representation of a job that gets converted to JobConfig for execution.
func LoadJobSpec ¶
LoadJobSpec reads a job specification from a YAML or JSON file. Supports both flat format (image/command at top level) and eval format (image/command nested under a "job" block with triggers/description).
func LoadJobSpecOverlay ¶
LoadJobSpecOverlay reads a partial job specification from a YAML/JSON file Unlike LoadJobSpec, this doesn't require command or set defaults
func LoadJobSpecPartial ¶
LoadJobSpecPartial reads a job without applying the built-in image or command requirements. Local contexts use this form before they apply project defaults and command-line overlays.
func (*JobSpec) ToJobConfig ¶
ToJobConfig converts a JobSpec to a JobConfig for execution workspaceDir is the host directory to mount into the container jobID is a unique identifier for this job execution
type KubernetesRunner ¶
type KubernetesRunner struct {
// contains filtered or unexported fields
}
KubernetesRunner implements JobRunner using Kubernetes Jobs It creates K8s Job resources and streams logs directly via the K8s API to ensure secret masking is applied before logs reach any aggregator.
func NewKubernetesRunner ¶
func NewKubernetesRunner() (*KubernetesRunner, error)
NewKubernetesRunner creates a new Kubernetes-based job runner It automatically detects if running in-cluster and uses the appropriate config
func NewKubernetesRunnerWithConfig ¶
func NewKubernetesRunnerWithConfig(cfg KubernetesRunnerConfig) (*KubernetesRunner, error)
NewKubernetesRunnerWithConfig creates a KubernetesRunner with custom configuration
func (*KubernetesRunner) Cleanup ¶
func (kr *KubernetesRunner) Cleanup(ctx context.Context, jobName string) error
Cleanup removes the Kubernetes Job resource
func (*KubernetesRunner) GetJobStatus ¶
func (kr *KubernetesRunner) GetJobStatus(ctx context.Context, jobName string) (status string, failureReason string, err error)
GetJobStatus returns the current status of a Kubernetes job Returns: running, succeeded, failed, or pending Also returns an error message if the job failed
func (*KubernetesRunner) SampleResources ¶
func (kr *KubernetesRunner) SampleResources(ctx context.Context, jobName string, options ResourceSampleOptions) (ResourceSnapshot, error)
func (*KubernetesRunner) Stop ¶
Stop requests a graceful shutdown of the job's pod(s) by deleting them with an explicit grace period. Deleting a pod (rather than patching terminationGracePeriodSeconds, which is immutable post-creation) is how Kubernetes lets a caller override the grace period per-delete: DeleteOptions.GracePeriodSeconds controls how long the kubelet waits between sending SIGTERM and SIGKILL to the pod's containers, independent of the pod spec's default. grace == 0 requests immediate SIGKILL (the `kubectl delete --grace-period=0 --force` equivalent).
SpawnJob sets BackoffLimit=0 on the owning Job resource, so when the kubelet-terminated pod disappears the Job controller counts it as a failure and transitions the Job to JobFailed without creating a replacement pod — exactly the condition WaitForCompletion's watch is looking for. The pod is deleted (and possibly fully gone) by the time WaitForCompletion tries to read its exit code via getPodExitCode, so callers should expect -1 rather than a real container exit code for a Stop-initiated termination; the caller identifies "this was a stop, not a real failure" out-of-band (the cancel-poll result), not from the exit code.
func (*KubernetesRunner) StreamLogs ¶
func (kr *KubernetesRunner) StreamLogs(ctx context.Context, jobName string) (stdout io.ReadCloser, stderr io.ReadCloser, err error)
StreamLogs streams stdout and stderr from the job pod Logs are streamed directly via K8s API, bypassing any file-based log collectors
func (*KubernetesRunner) TakeWorkflowOutput ¶
func (kr *KubernetesRunner) TakeWorkflowOutput(jobName string) (string, bool)
TakeWorkflowOutput returns and removes output captured from the pod-local workspace. The bool is true when capture completed, including when the job did not create an output file.
func (*KubernetesRunner) WaitForCompletion ¶
WaitForCompletion waits for the main job container to exit. The workflow output reader stays alive so the worker can copy workflow-output.json from the pod-local /job volume before Cleanup removes the pod.
type KubernetesRunnerConfig ¶
type KubernetesRunnerConfig struct {
Namespace string // Namespace for job pods (default: current namespace)
ServiceAccount string // Service account for job pods (default: "default")
RunnerImage string // Default runner image if job doesn't specify one
DindImage string // Docker-in-Docker sidecar image (default: "docker:27-dind")
ImagePullSecrets []string // Image pull secrets added to every job pod
// AllowedJobImagePullSecrets lists Secret names a job may request via
// image_pull_secrets in addition to ImagePullSecrets above.
AllowedJobImagePullSecrets []string
NodeName string // Node to schedule job pods on (for workspace sharing via HostPath)
}
KubernetesRunnerConfig holds configuration for the K8s runner
type LogEntry ¶
type LogEntry struct {
Timestamp string `json:"timestamp"`
Stream string `json:"stream"`
Level string `json:"level,omitempty"`
Message string `json:"message"`
}
LogEntry is one line of job output as stored in the object store and returned by the log endpoints. Workers ship these as JSON arrays under logs/{job_id}/{stream}.json.
type PodStartupError ¶
PodStartupError represents a pod that failed to start
func (*PodStartupError) Error ¶
func (e *PodStartupError) Error() string
type ResourceSampleOptions ¶
type ResourceSampleOptions struct {
IncludeStorage bool
}
ResourceSampleOptions selects the resource groups for one snapshot.
type ResourceSnapshot ¶
type ResourceSnapshot struct {
ObservedAt time.Time
Series []jobtelemetry.SeriesDefinition
Values []jobtelemetry.Value
}
ResourceSnapshot is one backend-neutral set of metrics at ObservedAt.
type RetryConfig ¶
type RetryConfig struct {
MaxRetries int // Maximum number of retry attempts
InitialDelay time.Duration // Initial delay between retries
MaxDelay time.Duration // Maximum delay between retries
BackoffFactor float64 // Exponential backoff factor (e.g., 2.0)
JitterFraction float64 // Fraction of delay to add as random jitter (0.0-1.0)
}
RetryConfig holds configuration for retry logic
func DefaultRetryConfig ¶
func DefaultRetryConfig() *RetryConfig
DefaultRetryConfig returns the default retry configuration
type RetryableError ¶
RetryableError represents an error that can be retried
func ClassifyExecutionError ¶
func ClassifyExecutionError(err error, exitCode int) *RetryableError
ClassifyExecutionError classifies an execution error as retryable or not
func (*RetryableError) Error ¶
func (e *RetryableError) Error() string
func (*RetryableError) Unwrap ¶
func (e *RetryableError) Unwrap() error
type RunAsSpec ¶
type RunAsSpec struct {
// User accepts "runner", "root", or numeric "uid[:gid]". "host" is
// intentionally run-local-only and is not accepted by deployed workers.
User string `json:"user" yaml:"user"`
}
RunAsSpec controls the user identity for deployed job containers.
type RunLocalSpec ¶
type RunLocalSpec struct {
// AsRunner runs the job container as the image's conventional runner uid
// (1001:1001), matching the worker, instead of the host uid. This gives
// sudo and HOME parity for jobs that rely on the image's runner user.
AsRunner bool `json:"as_runner" yaml:"as_runner"`
// User pins an explicit uid[:gid] for the job container (e.g.
// "1001:1001"). Takes precedence over AsRunner when both are set. Empty
// means unset.
User string `json:"user" yaml:"user"`
}
RunLocalSpec holds run-local-only overrides. The worker never reads this; it exists so a job can declare how it should be executed on a laptop.
type RunnerBackend ¶
type RunnerBackend string
RunnerBackend represents the container runtime backend to use
const ( // BackendDocker uses the Docker daemon BackendDocker RunnerBackend = "docker" // BackendContainerd uses containerd via nerdctl BackendContainerd RunnerBackend = "containerd" // BackendKubernetes uses Kubernetes Jobs BackendKubernetes RunnerBackend = "kubernetes" // BackendAuto automatically detects the best backend BackendAuto RunnerBackend = "auto" // BackendVM uses ephemeral guest VMs to run native macOS/Windows jobs. It // is a recognized backend name on every OS, but only actually implemented // on darwin and windows. BackendVM RunnerBackend = "vm" )
func GetSupportedBackends ¶
func GetSupportedBackends() []RunnerBackend
GetSupportedBackends returns a list of all supported runner backends
type SecretGrantStore ¶
type SecretGrantStore interface {
ListSecretGrantsForJob(ctx context.Context, userID string, projectID *string, jobName string) ([]models.SecretGrant, error)
}
SecretGrantStore is the narrow store capability secret-grant authorization needs. Exported (alongside AuthorizeSecretAccess) so internal/workerapi's coordinator-mediated RequestJob can run the grant- authorization decision without reimplementing it.
type SecretOverride ¶
type SecretOverride struct {
Key string
OldValue string // The ${secret:...} reference
NewValue string // The plaintext value
OverlayFile string
}
SecretOverride represents a case where a secret reference was overridden
type SecretResolutionResult ¶
type SecretResolutionResult struct {
// Resolved contains all environment variables with secrets resolved
Resolved map[string]string
// SecretValues contains the actual secret values for masking
SecretValues []string
// SecretEnvNames contains the names of env vars that contained secret references
SecretEnvNames []string
}
SecretResolutionResult holds the results of resolving secrets in environment variables
func ResolveSecretsInEnvFull ¶
func ResolveSecretsInEnvFull(env map[string]string, getSecret func(path, key string) (string, error)) (*SecretResolutionResult, error)
ResolveSecretsInEnvFull resolves all secret references and returns full result including env var names
type SourceSpec ¶
type SourceSpec struct {
Type string `json:"type" yaml:"type"` // git, copy, none
URL string `json:"url" yaml:"url"`
Ref string `json:"ref" yaml:"ref"`
Path string `json:"path" yaml:"path"` // for copy type
}
SourceSpec defines source code preparation
type TriggerProcessingResult ¶
type TriggerProcessingResult struct {
CreatedJobIDs []string `json:"created_job_ids"`
Workflows []TriggerWorkflowOutcome `json:"workflows,omitempty"`
}
TriggerProcessingResult is the durable result of one trigger request.
type TriggerProcessor ¶
type TriggerProcessor struct {
// contains filtered or unexported fields
}
TriggerProcessor handles reading triggers.json from completed eval jobs and creating/submitting the triggered jobs to Corndogs.
func NewTriggerProcessor ¶
func NewTriggerProcessor(store store.Store, corndogsClient corndogs.ClientInterface) *TriggerProcessor
NewTriggerProcessor creates a new TriggerProcessor.
func (*TriggerProcessor) EvaluateWorkflow ¶
func (tp *TriggerProcessor) EvaluateWorkflow(ctx context.Context, wf *models.WorkflowInstance) ([]string, error)
EvaluateWorkflow is the exported form of evaluateWorkflow, used by internal/jobcontrol.RetryWorkflow to drive initial node submission for a freshly created workflow instance exactly the way ProcessTriggersFromData drives it for a brand-new one (same dependency/condition evaluation, same submitWorkflowNode path, same refreshWorkflowStatus at the end) — the alternative would be reimplementing that evaluation loop a second time in jobcontrol, which risks drifting from the worker's own semantics. The caller must pass a store implementing this package's full workflowStore interface (postgres_store.PostgresDbStore does); against a narrower store this returns ErrWorkflowsUnsupported-equivalent errors the same way ProcessTriggersFromData's own callers would see.
func (*TriggerProcessor) ProcessTriggers ¶
func (tp *TriggerProcessor) ProcessTriggers(ctx context.Context, workspaceDir string, parentJob *models.Job) error
ProcessTriggers reads triggers.json from the workspace directory of a completed eval job, creates the triggered jobs in the database, and submits them to Corndogs.
func (*TriggerProcessor) ProcessTriggersFromData ¶
func (tp *TriggerProcessor) ProcessTriggersFromData(ctx context.Context, data []byte, workspaceDir string, parentJob *models.Job) ([]string, error)
ProcessTriggersFromData processes raw trigger JSON data, creates the triggered jobs in the database, submits them to Corndogs, and returns the created job IDs. workspaceDir is the host workspace directory used to resolve job_file references.
func (*TriggerProcessor) ProcessTriggersFromDataWithOutcomes ¶
func (tp *TriggerProcessor) ProcessTriggersFromDataWithOutcomes(ctx context.Context, data []byte, workspaceDir string, parentJob *models.Job) (TriggerProcessingResult, error)
ProcessTriggersFromDataWithOutcomes processes triggers and also returns a durable outcome for each workflow. A recorded workflow refusal is an outcome, not a request error, so other workflows continue.
func (*TriggerProcessor) ProcessWorkflowCompletion ¶
func (*TriggerProcessor) ProcessWorkflowCompletionData ¶
func (tp *TriggerProcessor) ProcessWorkflowCompletionData(ctx context.Context, data []byte, job *models.Job) error
ProcessWorkflowCompletionData advances a workflow and merges an output document returned by a remote worker.
func (*TriggerProcessor) ProcessWorkflowJobStarted ¶
func (*TriggerProcessor) SetStatusUpdater ¶
func (tp *TriggerProcessor) SetStatusUpdater(u vcs.JobStatusUpdaterInterface)
SetStatusUpdater wires a VCS status updater so that newly-created child jobs get registered as pending checks on their commit the moment they exist in the database, before the worker picks them up.
type TriggerWorkflowOutcome ¶
type TriggerWorkflowOutcome struct {
WorkflowID string `json:"workflow_id"`
WorkflowSecurityID string `json:"workflow_security_id,omitempty"`
Name string `json:"name"`
Status string `json:"status"`
Disposition string `json:"disposition"`
Reason string `json:"reason,omitempty"`
CreatedJobIDs []string `json:"created_job_ids"`
}
TriggerWorkflowOutcome reports the durable result for one workflow in a multi-workflow trigger request. Disposition separates admission from job execution state so clients do not report a refused workflow as a build.
type VCSAuthConfig ¶
type VCSAuthConfig struct {
ContainerDir string
GitConfig string
Credentials string
SecretValues []string
}
VCSAuthConfig contains runtime Git auth material for source preparation. Secret values must not be logged or exposed as environment values.
type VMConfig ¶
type VMConfig struct {
// ImageSource selects the ImageSource implementation: "local" (default,
// vmrunner.LocalImageSource) or "oci" (vmrunner.OCIImageSource, pulling
// base images from an OCI registry).
ImageSource string
// ImageDir is the directory relative image references resolve under
// when ImageSource is "local" (see vmrunner.LocalImageSource).
ImageDir string
// OCIImageCacheDir is the local content-addressed cache directory
// vmrunner.OCIImageSource pulls base images into when ImageSource is
// "oci" (see vmrunner.NewOCIImageSource).
OCIImageCacheDir string
// OCIRegistryAuthFile is a Docker-compatible credential file. One file can
// hold credentials for several registry hosts.
OCIRegistryAuthFile string
// OCIPlainHTTPRegistries lists registry hosts (as they appear in an
// image reference, e.g. "registry.internal:5000") to reach over plain
// HTTP instead of HTTPS when ImageSource is "oci". Each host must come
// from an explicit command-line option.
OCIPlainHTTPRegistries []string
// SSHUser is the guest OS account SSHTransport authenticates as.
SSHUser string
// SSHPassword authenticates via SSH password auth when set.
SSHPassword string
// SSHPrivateKeyPEM authenticates via SSH public-key auth when set,
// taking precedence over SSHPassword. Loaded from the file at
// REACTORCIDE_VM_SSH_PRIVATE_KEY_FILE, never itself taken directly from
// an env var so its contents can't leak into a printed environment.
SSHPrivateKeyPEM []byte
// SSHHostPublicKey pins the guest SSH server key when configured.
SSHHostPublicKey []byte
MetricsDir string
MetricsInterval time.Duration
}
VMConfig holds operator-level configuration for the "vm" JobRunner backend: which ImageSource to use and where its images live/cache (LocalImageSource's pre-placed ImageDir, or the OCI-backed OCIImageSource) and the SSH credentials used to reach a booted guest. Mirrors BuilderConfig's env-var-driven, zero-value-is-valid shape.
func LoadVMConfig ¶
type VMImageCacheManager ¶
type VMImageCacheManager interface {
PrefetchImages(ctx context.Context, imageRefs []string) error
PruneImages(ctx context.Context, maxUnused time.Duration, now time.Time) (int, error)
}
VMImageCacheManager is an optional capability implemented by an OCI-backed VM runner. The coordinator-mediated worker uses it for startup prefetch and periodic retention without coupling other runners to VM images.
type WorkflowOutputReader ¶
WorkflowOutputReader is an optional runner capability for backends whose job workspace is not mounted on the worker host. The coordinator worker calls TakeWorkflowOutput after WaitForCompletion and before Cleanup. Implementations must not log the returned document.
Source Files
¶
- builder_config.go
- containerd_metrics.go
- containerd_runner.go
- docker_metrics.go
- docker_runner.go
- image_pull_secrets.go
- interfaces.go
- job_env.go
- job_spec.go
- kubernetes_metrics.go
- kubernetes_runner.go
- log_entry.go
- log_muxer.go
- retry.go
- run_as.go
- runner_factory.go
- secret_authorization.go
- trigger_processor.go
- vm_adapter.go
- workflow_publishing_store.go
- workflow_runtime.go
Directories
¶
| Path | Synopsis |
|---|---|
|
Package vmrunner implements the "vm" JobRunner backend: ephemeral, per-job guest VMs used to run native macOS/Windows jobs on host workers.
|
Package vmrunner implements the "vm" JobRunner backend: ephemeral, per-job guest VMs used to run native macOS/Windows jobs on host workers. |