Documentation
¶
Overview ¶
Package worker contains code which executes a task.
Index ¶
- func DownloadInputs(pctx context.Context, inputs []*tes.Input, store storage.Storage, ...) error
- func FlattenInputs(ctx context.Context, inputs []*tes.Input, store storage.Storage, ...) ([]*tes.Input, error)
- func FlattenOutputs(ctx context.Context, outputs []*tes.Output, store storage.Storage, ...) ([]*tes.Output, error)
- func UploadOutputs(ctx context.Context, outputs []*tes.Output, store storage.Storage, ...) ([]*tes.OutputFileLog, error)
- type Base64TaskReader
- type Command
- type DefaultWorker
- type DockerCommand
- func (docker DockerCommand) GPUArgs() (string, error)
- func (docker DockerCommand) GetEnvArgs() string
- func (docker DockerCommand) GetImage() string
- func (docker *DockerCommand) GetStderr() io.Writer
- func (docker *DockerCommand) GetStdout() io.Writer
- func (docker DockerCommand) GetTagArgs() string
- func (docker *DockerCommand) InspectContainer(ctx context.Context) DockerCommand
- func (docker DockerCommand) MemoryMB() int64
- func (docker DockerCommand) NeedsTmpfs() bool
- func (docker DockerCommand) Run(ctx context.Context) error
- func (docker DockerCommand) Stop() error
- func (docker *DockerCommand) SyncAPIVersion() error
- type DockerVersion
- type ExecutionClaimer
- type Executor
- type FileTaskReader
- type GenericTaskReader
- type K8sExecutorErr
- type K8sSystemErr
- type KubernetesCommand
- type RPCTaskReader
- type TaskCommand
- type TaskReader
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func DownloadInputs ¶
func DownloadInputs(pctx context.Context, inputs []*tes.Input, store storage.Storage, ev *events.TaskWriter, parallelLimit int) error
DownloadInputs downloads the given inputs.
func FlattenInputs ¶
func FlattenInputs(ctx context.Context, inputs []*tes.Input, store storage.Storage, ev *events.TaskWriter) ([]*tes.Input, error)
FlattenInputs flattens any directory inputs into a list of file inputs. A warning event will be generated if an input directory is empty.
func FlattenOutputs ¶
func FlattenOutputs(ctx context.Context, outputs []*tes.Output, store storage.Storage, ev *events.TaskWriter) ([]*tes.Output, error)
FlattenOutputs flattens output directories into a list of files. A warning event will be generated if an output directory is empty.
func UploadOutputs ¶
func UploadOutputs(ctx context.Context, outputs []*tes.Output, store storage.Storage, ev *events.TaskWriter, parallelLimit int) ([]*tes.OutputFileLog, error)
UploadOutputs uploads the outputs.
Types ¶
type Base64TaskReader ¶
type Base64TaskReader struct {
// contains filtered or unexported fields
}
Base64TaskReader reads a task from a base64 encoded string.
func NewBase64TaskReader ¶
func NewBase64TaskReader(raw string) (*Base64TaskReader, error)
NewBase64TaskReader creates a new Base64TaskReader.
type Command ¶
type DefaultWorker ¶
type DefaultWorker struct {
Executor Executor
Conf *config.Worker
Store storage.Storage
TaskReader TaskReader
ExecutionClaimer ExecutionClaimer
EventWriter events.Writer
Command
}
DefaultWorker is the default task worker, which follows a basic, sequential process of task initialization, execution, finalization, and logging.
func (*DefaultWorker) Close ¶
func (r *DefaultWorker) Close()
type DockerCommand ¶
type DockerCommand struct {
Id string
Name string
Volumes []workspace.Volume
Workdir string
RemoveContainer bool
Event *events.ExecutorWriter
DriverCommand string
RunCommand string // template string
PullCommand string // template string
StopCommand string // template string
EnableTags bool
Tags map[string]string
Resources *tes.Resources
EnforceResources bool
ComputeBackend string
Command
}
DockerCommand is responsible for configuring and running a docker container.
func (DockerCommand) GPUArgs ¶
func (docker DockerCommand) GPUArgs() (string, error)
GPUArgs returns the Docker flags for the task's requested GPUs. AWS Batch must use the exact devices assigned to its worker so the nested container cannot select GPUs allocated to another job.
func (DockerCommand) GetEnvArgs ¶
func (docker DockerCommand) GetEnvArgs() string
func (DockerCommand) GetImage ¶
func (docker DockerCommand) GetImage() string
func (*DockerCommand) GetStderr ¶
func (docker *DockerCommand) GetStderr() io.Writer
func (*DockerCommand) GetStdout ¶
func (docker *DockerCommand) GetStdout() io.Writer
func (DockerCommand) GetTagArgs ¶
func (docker DockerCommand) GetTagArgs() string
GetTagArgs formats opted-in task tags as literal container label arguments.
func (*DockerCommand) InspectContainer ¶
func (docker *DockerCommand) InspectContainer(ctx context.Context) DockerCommand
inspectContainer inspects the docker container for metadata.
func (DockerCommand) MemoryMB ¶
func (docker DockerCommand) MemoryMB() int64
MemoryMB returns the task's requested RAM as an integer megabyte value suitable for passing to nerdctl/docker --memory flag (e.g. "2048m"). Returns 0 if no memory limit is set.
func (DockerCommand) NeedsTmpfs ¶
func (docker DockerCommand) NeedsTmpfs() bool
NeedsTmpfs reports whether the container needs Funnel's executor-local /tmp mount. A task-provided mount at /tmp or one of its ancestors takes precedence so explicitly requested storage can persist between executors.
func (DockerCommand) Run ¶
func (docker DockerCommand) Run(ctx context.Context) error
Run runs the Docker command and blocks until done.
func (*DockerCommand) SyncAPIVersion ¶
func (docker *DockerCommand) SyncAPIVersion() error
SyncDockerAPIVersion ensures that the client uses the same API version as the server.
type DockerVersion ¶
type ExecutionClaimer ¶
type ExecutionClaimer = events.ExecutionClaimer
ExecutionClaimer atomically admits one worker to execute a task.
type Executor ¶
type Executor struct {
StoragePlan *storageplan.Plan
// "docker" or "kubernetes"
Backend string
// ComputeBackend identifies the configured task compute backend, such as
// local or aws-batch. GPU handling differs between these backends.
ComputeBackend string
// Kubernetes executor template
Template string
// Kubernetes persistent volume template
PVTemplate string
// Kubernetes persistent volume claim template
PVCTemplate string
// Funnel Server namespace
Namespace string
// Funnel Worker + Executor namespace
JobsNamespace string
// ExecutionOwner identifies the worker pod that owns the task execution claim.
ExecutionOwner *events.ExecutionOwner
// Kubernetes service account name
ServiceAccount string
// Kubernetes service account template
ServiceAccountTemplate string
// Kubernetes role template
RoleTemplate string
// Kubernetes role binding template
RoleBindingTemplate string
// NodeSelector for scheduling jobs onto specific nodes
NodeSelector map[string]string
// Tolerations for scheduling jobs onto specific nodes
Tolerations []map[string]interface{}
// Resources specifies the default resource requirements for Kubernetes jobs.
Resources *config.KubernetesResources
}
Configuration of the task executor.
type FileTaskReader ¶
type FileTaskReader struct {
// contains filtered or unexported fields
}
FileTaskReader provides a TaskReader implementation from a task file.
func NewFileTaskReader ¶
func NewFileTaskReader(path string) (*FileTaskReader, error)
NewFileTaskReader creates a new FileTaskReader.
type GenericTaskReader ¶
type GenericTaskReader struct {
// contains filtered or unexported fields
}
GenericTaskReader provides read access to tasks.
func NewGenericTaskReader ¶
func NewGenericTaskReader(get func(ctx context.Context, in *tes.GetTaskRequest) (*tes.Task, error), taskID string, close func()) *GenericTaskReader
NewGenericTaskReader returns a new generic task reader.
func (*GenericTaskReader) Close ¶
func (r *GenericTaskReader) Close()
type K8sExecutorErr ¶
func (*K8sExecutorErr) Error ¶
func (e *K8sExecutorErr) Error() string
type K8sSystemErr ¶
type K8sSystemErr struct {
Reason string
Message string
Err error
// contains filtered or unexported fields
}
func (*K8sSystemErr) Error ¶
func (e *K8sSystemErr) Error() string
func (*K8sSystemErr) Unwrap ¶
func (e *K8sSystemErr) Unwrap() error
type KubernetesCommand ¶
type KubernetesCommand struct {
StoragePlan *storageplan.Plan
TaskId string
JobId int
StdinFile string
StdoutFile string
StderrFile string
TaskTemplate string
Namespace string // Funnel Server Namespace
JobsNamespace string // Funnel Worker + Executor Namespace (default: Namespace)
NodeSelector map[string]string
Tolerations []map[string]interface{}
Resources *tes.Resources
ResourceLimits *tes.Resources
GPUCount int64
GPUResource string
GPUNodeSelector map[string]string
ServiceAccount string
ImagePullPolicy corev1.PullPolicy
NeedsPVC bool
Clientset kubernetes.Interface
Command
}
KubernetesCommand is responsible for configuring and running a task in a Kubernetes cluster.
func (KubernetesCommand) GetStderr ¶
func (kcmd KubernetesCommand) GetStderr() io.Writer
func (KubernetesCommand) GetStdout ¶
func (kcmd KubernetesCommand) GetStdout() io.Writer
func (KubernetesCommand) Run ¶
func (kcmd KubernetesCommand) Run(ctx context.Context) error
Create the Executor K8s job from kubernetes-executor-template.yaml Funnel Worker job is created in compute/kubernetes/backend.go#CreateResources
func (KubernetesCommand) Stop ¶
func (kcmd KubernetesCommand) Stop() error
Deletes the job running the task.
type RPCTaskReader ¶
type RPCTaskReader struct {
// contains filtered or unexported fields
}
RPCTaskReader provides read access to tasks from the funnel server over gRPC.
func NewRPCTaskReader ¶
func NewRPCTaskReader(ctx context.Context, conf *config.RPCClient, taskID string) (*RPCTaskReader, error)
NewRPCTaskReader returns a new RPC-based task reader.
func (*RPCTaskReader) ClaimExecution ¶
func (r *RPCTaskReader) ClaimExecution(ctx context.Context, taskID string, owner *events.ExecutionOwner) (bool, error)
ClaimExecution asks the server to atomically admit this worker for the task.