worker

package
v0.12.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 9, 2026 License: MIT Imports: 52 Imported by: 0

Documentation

Overview

Package worker contains code which executes a task.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func DownloadInputs

func DownloadInputs(pctx context.Context, inputs []*tes.Input, store storage.Storage, ev *events.TaskWriter, parallelLimit int) error

DownloadInputs downloads the given inputs.

func FlattenInputs

func FlattenInputs(ctx context.Context, inputs []*tes.Input, store storage.Storage, ev *events.TaskWriter) ([]*tes.Input, error)

FlattenInputs flattens any directory inputs into a list of file inputs. A warning event will be generated if an input directory is empty.

func FlattenOutputs

func FlattenOutputs(ctx context.Context, outputs []*tes.Output, store storage.Storage, ev *events.TaskWriter) ([]*tes.Output, error)

FlattenOutputs flattens output directories into a list of files. A warning event will be generated if an output directory is empty.

func UploadOutputs

func UploadOutputs(ctx context.Context, outputs []*tes.Output, store storage.Storage, ev *events.TaskWriter, parallelLimit int) ([]*tes.OutputFileLog, error)

UploadOutputs uploads the outputs.

Types

type Base64TaskReader

type Base64TaskReader struct {
	// contains filtered or unexported fields
}

Base64TaskReader reads a task from a base64 encoded string.

func NewBase64TaskReader

func NewBase64TaskReader(raw string) (*Base64TaskReader, error)

NewBase64TaskReader creates a new Base64TaskReader.

func (*Base64TaskReader) Close

func (f *Base64TaskReader) Close()

Close the Base64TaskReader

func (*Base64TaskReader) State

func (f *Base64TaskReader) State(ctx context.Context) (tes.State, error)

State returns the task state. Due to some quirks in the implementation of this reader, and since there is no online database to connect to, this will always return QUEUED.

func (*Base64TaskReader) Task

func (f *Base64TaskReader) Task(ctx context.Context) (*tes.Task, error)

Task returns the task. A random ID will be generated.

type Command

type Command struct {
	Image        string
	ShellCommand []string
	Volumes      []workspace.Volume
	Workdir      string
	Env          map[string]string
	Stdin        io.Reader
	Stdout       io.Writer
	Stderr       io.Writer
	Event        *events.ExecutorWriter
	TaskCommand
}

func (*Command) GetStderr

func (c *Command) GetStderr() io.Writer

func (*Command) GetStdout

func (c *Command) GetStdout() io.Writer

func (*Command) SetStderr

func (c *Command) SetStderr(w io.Writer)

func (*Command) SetStdin

func (c *Command) SetStdin(r io.Reader)

func (*Command) SetStdout

func (c *Command) SetStdout(w io.Writer)

type DefaultWorker

type DefaultWorker struct {
	Executor         Executor
	Conf             *config.Worker
	Store            storage.Storage
	TaskReader       TaskReader
	ExecutionClaimer ExecutionClaimer
	EventWriter      events.Writer
	Command
}

DefaultWorker is the default task worker, which follows a basic, sequential process of task initialization, execution, finalization, and logging.

func (*DefaultWorker) Close

func (r *DefaultWorker) Close()

func (*DefaultWorker) Run

func (r *DefaultWorker) Run(pctx context.Context) (runerr error)

Run runs the Worker.

type DockerCommand

type DockerCommand struct {
	Id               string
	Name             string
	Volumes          []workspace.Volume
	Workdir          string
	RemoveContainer  bool
	Event            *events.ExecutorWriter
	DriverCommand    string
	RunCommand       string // template string
	PullCommand      string // template string
	StopCommand      string // template string
	EnableTags       bool
	Tags             map[string]string
	Resources        *tes.Resources
	EnforceResources bool
	ComputeBackend   string
	Command
}

DockerCommand is responsible for configuring and running a docker container.

func (DockerCommand) GPUArgs

func (docker DockerCommand) GPUArgs() (string, error)

GPUArgs returns the Docker flags for the task's requested GPUs. AWS Batch must use the exact devices assigned to its worker so the nested container cannot select GPUs allocated to another job.

func (DockerCommand) GetEnvArgs

func (docker DockerCommand) GetEnvArgs() string

func (DockerCommand) GetImage

func (docker DockerCommand) GetImage() string

func (*DockerCommand) GetStderr

func (docker *DockerCommand) GetStderr() io.Writer

func (*DockerCommand) GetStdout

func (docker *DockerCommand) GetStdout() io.Writer

func (DockerCommand) GetTagArgs

func (docker DockerCommand) GetTagArgs() string

GetTagArgs formats opted-in task tags as literal container label arguments.

func (*DockerCommand) InspectContainer

func (docker *DockerCommand) InspectContainer(ctx context.Context) DockerCommand

inspectContainer inspects the docker container for metadata.

func (DockerCommand) MemoryMB

func (docker DockerCommand) MemoryMB() int64

MemoryMB returns the task's requested RAM as an integer megabyte value suitable for passing to nerdctl/docker --memory flag (e.g. "2048m"). Returns 0 if no memory limit is set.

func (DockerCommand) NeedsTmpfs

func (docker DockerCommand) NeedsTmpfs() bool

NeedsTmpfs reports whether the container needs Funnel's executor-local /tmp mount. A task-provided mount at /tmp or one of its ancestors takes precedence so explicitly requested storage can persist between executors.

func (DockerCommand) Run

func (docker DockerCommand) Run(ctx context.Context) error

Run runs the Docker command and blocks until done.

func (DockerCommand) Stop

func (docker DockerCommand) Stop() error

Stop stops the container.

func (*DockerCommand) SyncAPIVersion

func (docker *DockerCommand) SyncAPIVersion() error

SyncDockerAPIVersion ensures that the client uses the same API version as the server.

type DockerVersion

type DockerVersion struct {
	Client string
	Server string
}

type ExecutionClaimer

type ExecutionClaimer = events.ExecutionClaimer

ExecutionClaimer atomically admits one worker to execute a task.

type Executor

type Executor struct {
	StoragePlan *storageplan.Plan
	// "docker" or "kubernetes"
	Backend string
	// ComputeBackend identifies the configured task compute backend, such as
	// local or aws-batch. GPU handling differs between these backends.
	ComputeBackend string
	// Kubernetes executor template
	Template string
	// Kubernetes persistent volume template
	PVTemplate string
	// Kubernetes persistent volume claim template
	PVCTemplate string
	// Funnel Server namespace
	Namespace string
	// Funnel Worker + Executor namespace
	JobsNamespace string
	// ExecutionOwner identifies the worker pod that owns the task execution claim.
	ExecutionOwner *events.ExecutionOwner
	// Kubernetes service account name
	ServiceAccount string
	// Kubernetes service account template
	ServiceAccountTemplate string
	// Kubernetes role template
	RoleTemplate string
	// Kubernetes role binding template
	RoleBindingTemplate string
	// NodeSelector for scheduling jobs onto specific nodes
	NodeSelector map[string]string
	// Tolerations for scheduling jobs onto specific nodes
	Tolerations []map[string]interface{}
	// Resources specifies the default resource requirements for Kubernetes jobs.
	Resources *config.KubernetesResources
}

Configuration of the task executor.

type FileTaskReader

type FileTaskReader struct {
	// contains filtered or unexported fields
}

FileTaskReader provides a TaskReader implementation from a task file.

func NewFileTaskReader

func NewFileTaskReader(path string) (*FileTaskReader, error)

NewFileTaskReader creates a new FileTaskReader.

func (*FileTaskReader) Close

func (f *FileTaskReader) Close()

Close the FileTaskReader

func (*FileTaskReader) State

func (f *FileTaskReader) State(ctx context.Context) (tes.State, error)

State returns the task state. Due to some quirks in the implementation of this reader, and since there is no online database to connect to, this will always return QUEUED.

func (*FileTaskReader) Task

func (f *FileTaskReader) Task(ctx context.Context) (*tes.Task, error)

Task returns the task. A random ID will be generated.

type GenericTaskReader

type GenericTaskReader struct {
	// contains filtered or unexported fields
}

GenericTaskReader provides read access to tasks.

func NewGenericTaskReader

func NewGenericTaskReader(get func(ctx context.Context, in *tes.GetTaskRequest) (*tes.Task, error), taskID string, close func()) *GenericTaskReader

NewGenericTaskReader returns a new generic task reader.

func (*GenericTaskReader) Close

func (r *GenericTaskReader) Close()

func (*GenericTaskReader) State

func (r *GenericTaskReader) State(ctx context.Context) (tes.State, error)

State returns the current state of the task.

func (*GenericTaskReader) Task

func (r *GenericTaskReader) Task(ctx context.Context) (*tes.Task, error)

Task returns the task descriptor.

type K8sExecutorErr

type K8sExecutorErr struct {
	ExitCode int
	Reason   string
	Message  string
	JobName  string
}

func (*K8sExecutorErr) Error

func (e *K8sExecutorErr) Error() string

type K8sSystemErr

type K8sSystemErr struct {
	Reason  string
	Message string
	Err     error
	// contains filtered or unexported fields
}

func (*K8sSystemErr) Error

func (e *K8sSystemErr) Error() string

func (*K8sSystemErr) Unwrap

func (e *K8sSystemErr) Unwrap() error

type KubernetesCommand

type KubernetesCommand struct {
	StoragePlan     *storageplan.Plan
	TaskId          string
	JobId           int
	StdinFile       string
	StdoutFile      string
	StderrFile      string
	TaskTemplate    string
	Namespace       string // Funnel Server Namespace
	JobsNamespace   string // Funnel Worker + Executor Namespace (default: Namespace)
	NodeSelector    map[string]string
	Tolerations     []map[string]interface{}
	Resources       *tes.Resources
	ResourceLimits  *tes.Resources
	GPUCount        int64
	GPUResource     string
	GPUNodeSelector map[string]string
	ServiceAccount  string
	ImagePullPolicy corev1.PullPolicy
	NeedsPVC        bool
	Clientset       kubernetes.Interface
	Command
}

KubernetesCommand is responsible for configuring and running a task in a Kubernetes cluster.

func (KubernetesCommand) GetStderr

func (kcmd KubernetesCommand) GetStderr() io.Writer

func (KubernetesCommand) GetStdout

func (kcmd KubernetesCommand) GetStdout() io.Writer

func (KubernetesCommand) Run

func (kcmd KubernetesCommand) Run(ctx context.Context) error

Create the Executor K8s job from kubernetes-executor-template.yaml Funnel Worker job is created in compute/kubernetes/backend.go#CreateResources

func (KubernetesCommand) Stop

func (kcmd KubernetesCommand) Stop() error

Deletes the job running the task.

type RPCTaskReader

type RPCTaskReader struct {
	// contains filtered or unexported fields
}

RPCTaskReader provides read access to tasks from the funnel server over gRPC.

func NewRPCTaskReader

func NewRPCTaskReader(ctx context.Context, conf *config.RPCClient, taskID string) (*RPCTaskReader, error)

NewRPCTaskReader returns a new RPC-based task reader.

func (*RPCTaskReader) ClaimExecution

func (r *RPCTaskReader) ClaimExecution(ctx context.Context, taskID string, owner *events.ExecutionOwner) (bool, error)

ClaimExecution asks the server to atomically admit this worker for the task.

func (*RPCTaskReader) Close

func (r *RPCTaskReader) Close()

Close closes the connection.

func (*RPCTaskReader) State

func (r *RPCTaskReader) State(ctx context.Context) (tes.State, error)

State returns the current state of the task.

func (*RPCTaskReader) Task

func (r *RPCTaskReader) Task(ctx context.Context) (*tes.Task, error)

Task returns the task descriptor.

type TaskCommand

type TaskCommand interface {
	Run(context.Context) error
	Stop() error
	GetStdout() io.Writer
	GetStderr() io.Writer
	SetStdout(io.Writer)
	SetStderr(io.Writer)
	SetStdin(io.Reader)
}

type TaskReader

type TaskReader interface {
	Task(ctx context.Context) (*tes.Task, error)
	State(ctx context.Context) (tes.State, error)
	Close()
}

TaskReader is a type which reads task information during task execution.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL