jobrunner

package
v4.59.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 29, 2025 License: Apache-2.0 Imports: 10 Imported by: 0

Documentation

Overview

Example (Jobrunner_RunJob_BadConfig) 
os.Setenv(JobConfigEnvVar, "{ \"Some\": \"unfinished JSON }")
fmt.Printf("%v\n", RunJob(false))

Output:

Failed to parse env var JOB_CONFIG: unexpected end of JSON input
Example (Jobrunner_RunJob_BadInputLocalPath) 
cfg := JobConfig{
	JobId:         "Job001",
	RequiredFiles: []JobFilePath{{LocalPath: "", RemoteBucket: "test-piquant", RemotePath: "jobs/Job001/input.csv"}},
	Command:       "ls",
}

cfgJSON, err := json.Marshal(cfg)
fmt.Printf("cfgErr: %v\n", err)

os.Setenv(JobConfigEnvVar, string(cfgJSON))
fmt.Printf("Job: %v\n", RunJob(false))

Output:

cfgErr: <nil>
INFO: Preparing job: Job001
Config: {"JobId":"Job001","RequiredFiles":[{"RemoteBucket":"test-piquant","RemotePath":"jobs/Job001/input.csv","LocalPath":""}],"Command":"ls","Args":null,"OutputFiles":null}
INFO: Job config struct: jobrunner.JobConfig{JobId:"Job001", RequiredFiles:[]jobrunner.JobFilePath{jobrunner.JobFilePath{RemoteBucket:"test-piquant", RemotePath:"jobs/Job001/input.csv", LocalPath:""}}, Command:"ls", Args:[]string(nil), OutputFiles:[]jobrunner.JobFilePath(nil)}
INFO: AWS S3 setup...
INFO: Downloading files...
INFO: Download "s3://test-piquant/jobs/Job001/input.csv" -> "":
Job: No localPath specified
Example (Jobrunner_RunJob_BadInputRemotePath) 
cfg := JobConfig{
	JobId:         "Job001",
	RequiredFiles: []JobFilePath{{LocalPath: "input.csv", RemoteBucket: "test-piquant", RemotePath: "jobs/Job001/input.csv"}},
	Command:       "ls",
}

cfgJSON, err := json.Marshal(cfg)
fmt.Printf("cfgErr: %v\n", err)

os.Setenv(JobConfigEnvVar, string(cfgJSON))
fmt.Printf("Job: %v\n", RunJob(false))

Output:

cfgErr: <nil>
INFO: Preparing job: Job001
Config: {"JobId":"Job001","RequiredFiles":[{"RemoteBucket":"test-piquant","RemotePath":"jobs/Job001/input.csv","LocalPath":"input.csv"}],"Command":"ls","Args":null,"OutputFiles":null}
INFO: Job config struct: jobrunner.JobConfig{JobId:"Job001", RequiredFiles:[]jobrunner.JobFilePath{jobrunner.JobFilePath{RemoteBucket:"test-piquant", RemotePath:"jobs/Job001/input.csv", LocalPath:"input.csv"}}, Command:"ls", Args:[]string(nil), OutputFiles:[]jobrunner.JobFilePath(nil)}
INFO: AWS S3 setup...
INFO: Downloading files...
INFO: Download "s3://test-piquant/jobs/Job001/input.csv" -> "input.csv":
INFO:  Local file will be written to working dir
Job: Failed to download s3://test-piquant/jobs/Job001/input.csv: Not found
Example (Jobrunner_RunJob_DownloadUploadOK) 
// Before we do anything, ensure the file is in S3
sess, err := awsutil.GetSession()
fmt.Printf("GetSession: %v\n", err)
s3, err := awsutil.GetS3(sess)
fmt.Printf("GetS3: %v\n", err)
remoteFS := fileaccess.MakeS3Access(s3)

fmt.Printf("Write S3 input.csv: %v\n", remoteFS.WriteObject("test-piquant", "RunnerTest/input.csv", []byte("hello")))
fmt.Printf("Write S3 input2.csv: %v\n", remoteFS.WriteObject("test-piquant", "RunnerTest/input2.csv", []byte("hello2")))
fmt.Printf("Write local data.txt: %v\n", os.WriteFile("data.txt", []byte("hello"), dirperm))

cfg := JobConfig{
	JobId: "Job001",
	RequiredFiles: []JobFilePath{
		{LocalPath: "inputfile.csv", RemoteBucket: "test-piquant", RemotePath: "RunnerTest/input.csv"},
		{LocalPath: "second.csv", RemoteBucket: "test-piquant", RemotePath: "RunnerTest/input2.csv"},
	},
	Command: NoOpCommand,
	OutputFiles: []JobFilePath{
		{LocalPath: "stdout", RemoteBucket: "test-piquant", RemotePath: "RunnerTest/Output/stdout"},
		{LocalPath: "data.txt", RemoteBucket: "test-piquant", RemotePath: "RunnerTest/Output/file.csv"},
	},
}

cfgJSON, err := json.Marshal(cfg)
fmt.Printf("cfgErr: %v\n", err)

os.Setenv("JOB_CONFIG", string(cfgJSON))
fmt.Printf("Job: %v\n", RunJob(false))

// Clean up files
os.Remove("inputfile.csv")
os.Remove("second.csv")
os.Remove("data.txt")

Output:

GetSession: <nil>
GetS3: <nil>
Write S3 input.csv: <nil>
Write S3 input2.csv: <nil>
Write local data.txt: <nil>
cfgErr: <nil>
INFO: Preparing job: Job001
Config: {"JobId":"Job001","RequiredFiles":[{"RemoteBucket":"test-piquant","RemotePath":"RunnerTest/input.csv","LocalPath":"inputfile.csv"},{"RemoteBucket":"test-piquant","RemotePath":"RunnerTest/input2.csv","LocalPath":"second.csv"}],"Command":"noop","Args":null,"OutputFiles":[{"RemoteBucket":"test-piquant","RemotePath":"RunnerTest/Output/stdout","LocalPath":"stdout"},{"RemoteBucket":"test-piquant","RemotePath":"RunnerTest/Output/file.csv","LocalPath":"data.txt"}]}
INFO: Job config struct: jobrunner.JobConfig{JobId:"Job001", RequiredFiles:[]jobrunner.JobFilePath{jobrunner.JobFilePath{RemoteBucket:"test-piquant", RemotePath:"RunnerTest/input.csv", LocalPath:"inputfile.csv"}, jobrunner.JobFilePath{RemoteBucket:"test-piquant", RemotePath:"RunnerTest/input2.csv", LocalPath:"second.csv"}}, Command:"noop", Args:[]string(nil), OutputFiles:[]jobrunner.JobFilePath{jobrunner.JobFilePath{RemoteBucket:"test-piquant", RemotePath:"RunnerTest/Output/stdout", LocalPath:"stdout"}, jobrunner.JobFilePath{RemoteBucket:"test-piquant", RemotePath:"RunnerTest/Output/file.csv", LocalPath:"data.txt"}}}
INFO: AWS S3 setup...
INFO: Downloading files...
INFO: Download "s3://test-piquant/RunnerTest/input.csv" -> "inputfile.csv":
INFO:  Local file will be written to working dir
INFO:  Downloaded 5 bytes
INFO:  Wrote file: inputfile.csv
INFO: Download "s3://test-piquant/RunnerTest/input2.csv" -> "second.csv":
INFO:  Local file will be written to working dir
INFO:  Downloaded 6 bytes
INFO:  Wrote file: second.csv
DEBUG: exec.Command starting "noop", args: []
INFO: Job Job001 runtime was 0 sec
INFO: Uploaded stdout log to: s3://test-piquant/RunnerTest/Output/stdout
INFO: Upload data.txt -> s3://test-piquant/RunnerTest/Output/file.csv
Job: <nil>
Example (Jobrunner_RunJob_NoCommand) 
cfg := JobConfig{
	JobId: "Job001",
}

cfgJSON, err := json.Marshal(cfg)
fmt.Printf("cfgErr: %v\n", err)

os.Setenv(JobConfigEnvVar, string(cfgJSON))
fmt.Printf("Job: %v\n", RunJob(false))

Output:

cfgErr: <nil>
INFO: Preparing job: Job001
Config: {"JobId":"Job001","RequiredFiles":null,"Command":"","Args":null,"OutputFiles":null}
INFO: Job config struct: jobrunner.JobConfig{JobId:"Job001", RequiredFiles:[]jobrunner.JobFilePath(nil), Command:"", Args:[]string(nil), OutputFiles:[]jobrunner.JobFilePath(nil)}
Job: No command specified
Example (Jobrunner_RunJob_NoConfig) 
os.Setenv(JobConfigEnvVar, "")
fmt.Printf("%v\n", RunJob(false))

Output:

JOB_CONFIG env var not set
Example (Jobrunner_RunJob_UploadNotThere) 
cfg := JobConfig{
	JobId:   "Job001",
	Command: NoOpCommand,
	OutputFiles: []JobFilePath{
		{LocalPath: "nofile.txt", RemoteBucket: "test-piquant", RemotePath: "RunnerTest/Output/file.csv"},
	},
}

cfgJSON, err := json.Marshal(cfg)
fmt.Printf("cfgErr: %v\n", err)

os.Setenv(JobConfigEnvVar, string(cfgJSON))
fmt.Printf("Job: %v\n", RunJob(false))

Output:

cfgErr: <nil>
INFO: Preparing job: Job001
Config: {"JobId":"Job001","RequiredFiles":null,"Command":"noop","Args":null,"OutputFiles":[{"RemoteBucket":"test-piquant","RemotePath":"RunnerTest/Output/file.csv","LocalPath":"nofile.txt"}]}
INFO: Job config struct: jobrunner.JobConfig{JobId:"Job001", RequiredFiles:[]jobrunner.JobFilePath(nil), Command:"noop", Args:[]string(nil), OutputFiles:[]jobrunner.JobFilePath{jobrunner.JobFilePath{RemoteBucket:"test-piquant", RemotePath:"RunnerTest/Output/file.csv", LocalPath:"nofile.txt"}}}
INFO: AWS S3 setup...
INFO: Downloading files...
DEBUG: exec.Command starting "noop", args: []
INFO: Job Job001 runtime was 0 sec
ERROR: Job Job001 did not generate expected output file: nofile.txt
Job: Job Job001 failed to generate/upload output files: nofile.txt

Index

Examples

Constants

This section is empty.

Variables

View Source 
var JobConfigEnvVar = "JOB_CONFIG"
View Source 
var NoOpCommand = "noop"

Functions

func RunJob 

func RunJob(logWD bool) error

Downloads files required for job to run. These are read from an env var Parameters:

  • logWD should be false for testing - we don't want to log the working dir if we're running as a test because it may differ on different machines and fail the test...

Types

type JobConfig 

type JobConfig struct {
	// The job id
	JobId string

	// What files are required to be present when running the job?
	RequiredFiles []JobFilePath

	// What command to execute
	Command string
	Args    []string

	// What to upload on completion (if file doesn't exist, it can be ignored with a warning)
	OutputFiles []JobFilePath
}

func (JobConfig) Copy 

func (c JobConfig) Copy() JobConfig

type JobFilePath 

type JobFilePath struct {
	// The remote file info
	RemoteBucket string
	RemotePath   string

	// Local copy
	LocalPath string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.