agent

package
v3.52.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 17, 2023 License: MIT Imports: 66 Imported by: 2

Documentation

Overview

Package agent provides the key agent components - workers, worker pool, job runner, log streamer, artifact up/downloaders, etc.

It is intended for internal use of buildkite-agent only.

Index

Constants

View Source
const (
	ArtifactPathDelimiter    = ";"
	ArtifactFallbackMimeType = "binary/octet-stream"
)
View Source
const (
	// BuildkiteMessageMax is the maximum length of "BUILDKITE_MESSAGE=...\0"
	// environment entry passed to bootstrap, beyond which it will be truncated
	// to avoid exceeding the system limit. Note that it includes the variable
	// name, equals sign, and null terminator.
	//
	// The true limit varies by system and may be shared with other env/argv
	// data. We'll settle on an arbitrary generous but reasonable value, and
	// adjust it if issues arise.
	//
	// macOS 10.15:    256 KiB shared by environment & argv
	// Linux 4.19:     128 KiB per k=v env
	// Windows 10:  16,384 KiB shared
	// POSIX:            4 KiB minimum shared
	BuildkiteMessageMax = 64 * 1024

	// BuildkiteMessageName is the env var name of the build/commit message.
	BuildkiteMessageName = "BUILDKITE_MESSAGE"

	VerificationBehaviourWarn  = "warn"
	VerificationBehaviourBlock = "block"
)
View Source
const (
	SignalReasonAgentRefused      = "agent_refused"
	SignalReasonAgentStop         = "agent_stop"
	SignalReasonCancel            = "cancel"
	SignalReasonSignatureRejected = "signature_rejected"
	SignalReasonProcessRunError   = "process_run_error"
)

Variables

View Source
var ArtifactPathVariableRegex = regexp.MustCompile("\\$\\{artifact\\:path\\}")
View Source
var ErrNoSignature = errors.New("job had no signature to verify")
View Source
var ProtectedEnv = map[string]struct{}{
	"BUILDKITE_AGENT_ENDPOINT":           {},
	"BUILDKITE_AGENT_ACCESS_TOKEN":       {},
	"BUILDKITE_AGENT_DEBUG":              {},
	"BUILDKITE_AGENT_PID":                {},
	"BUILDKITE_BIN_PATH":                 {},
	"BUILDKITE_CONFIG_PATH":              {},
	"BUILDKITE_BUILD_PATH":               {},
	"BUILDKITE_GIT_MIRRORS_PATH":         {},
	"BUILDKITE_GIT_MIRRORS_SKIP_UPDATE":  {},
	"BUILDKITE_HOOKS_PATH":               {},
	"BUILDKITE_PLUGINS_PATH":             {},
	"BUILDKITE_SSH_KEYSCAN":              {},
	"BUILDKITE_GIT_SUBMODULES":           {},
	"BUILDKITE_COMMAND_EVAL":             {},
	"BUILDKITE_PLUGINS_ENABLED":          {},
	"BUILDKITE_LOCAL_HOOKS_ENABLED":      {},
	"BUILDKITE_GIT_CLONE_FLAGS":          {},
	"BUILDKITE_GIT_FETCH_FLAGS":          {},
	"BUILDKITE_GIT_CLONE_MIRROR_FLAGS":   {},
	"BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT": {},
	"BUILDKITE_GIT_CLEAN_FLAGS":          {},
	"BUILDKITE_SHELL":                    {},
}

Certain env can only be set by agent configuration. We show the user a warning in the bootstrap if they use any of these at a job level.

Functions

func FetchTags

func FetchTags(ctx context.Context, l logger.Logger, conf FetchTagsConfig) []string

FetchTags loads tags from a variety of sources

func K8sTagsFromEnv added in v3.45.0

func K8sTagsFromEnv(envn []string) (map[string]string, error)

func NewJobRunner

func NewJobRunner(l logger.Logger, apiClient APIClient, conf JobRunnerConfig) (jobRunner, error)

Initializes the job runner

func NewS3Client added in v3.39.0

func NewS3Client(l logger.Logger, bucket string) (*s3.S3, error)

func ParseArtifactoryDestination

func ParseArtifactoryDestination(destination string) (repo string, path string)

func ParseGSDestination

func ParseGSDestination(destination string) (name string, path string)

func ParseS3Destination

func ParseS3Destination(destination string) (string, string)

func Register

Register takes an api.Agent and registers it with the Buildkite API and populates the result of the register call

Types

type APIClient

type APIClient interface {
	AcceptJob(context.Context, *api.Job) (*api.Job, *api.Response, error)
	AcquireJob(context.Context, string, ...api.Header) (*api.Job, *api.Response, error)
	Annotate(context.Context, string, *api.Annotation) (*api.Response, error)
	AnnotationRemove(context.Context, string, string) (*api.Response, error)
	Config() api.Config
	Connect(context.Context) (*api.Response, error)
	CreateArtifacts(context.Context, string, *api.ArtifactBatch) (*api.ArtifactBatchCreateResponse, *api.Response, error)
	Disconnect(context.Context) (*api.Response, error)
	ExistsMetaData(context.Context, string, string, string) (*api.MetaDataExists, *api.Response, error)
	FinishJob(context.Context, *api.Job) (*api.Response, error)
	FromAgentRegisterResponse(*api.AgentRegisterResponse) *api.Client
	FromPing(*api.Ping) *api.Client
	GetJobState(context.Context, string) (*api.JobState, *api.Response, error)
	GetMetaData(context.Context, string, string, string) (*api.MetaData, *api.Response, error)
	Heartbeat(context.Context) (*api.Heartbeat, *api.Response, error)
	MetaDataKeys(context.Context, string, string) ([]string, *api.Response, error)
	OIDCToken(context.Context, *api.OIDCTokenRequest) (*api.OIDCToken, *api.Response, error)
	Ping(context.Context) (*api.Ping, *api.Response, error)
	PipelineUploadStatus(context.Context, string, string, ...api.Header) (*api.PipelineUploadStatus, *api.Response, error)
	Register(context.Context, *api.AgentRegisterRequest) (*api.AgentRegisterResponse, *api.Response, error)
	SaveHeaderTimes(context.Context, string, *api.HeaderTimes) (*api.Response, error)
	SearchArtifacts(context.Context, string, *api.ArtifactSearchOptions) ([]*api.Artifact, *api.Response, error)
	SetMetaData(context.Context, string, *api.MetaData) (*api.Response, error)
	StartJob(context.Context, *api.Job) (*api.Response, error)
	StepExport(context.Context, string, *api.StepExportRequest) (*api.StepExportResponse, *api.Response, error)
	StepUpdate(context.Context, string, *api.StepUpdate) (*api.Response, error)
	UpdateArtifacts(context.Context, string, map[string]string) (*api.Response, error)
	UploadChunk(context.Context, string, *api.Chunk) (*api.Response, error)
	UploadPipeline(context.Context, string, *api.PipelineChange, ...api.Header) (*api.Response, error)
}

APIClient is an interface generated for "github.com/buildkite/agent/v3/api.Client".

type AgentConfiguration

type AgentConfiguration struct {
	ConfigPath            string
	BootstrapScript       string
	BuildPath             string
	HooksPath             string
	SocketsPath           string
	GitMirrorsPath        string
	GitMirrorsLockTimeout int
	GitMirrorsSkipUpdate  bool
	PluginsPath           string
	GitCheckoutFlags      string
	GitCloneFlags         string
	GitCloneMirrorFlags   string
	GitCleanFlags         string
	GitFetchFlags         string
	GitSubmodules         bool
	SSHKeyscan            bool
	CommandEval           bool
	PluginsEnabled        bool
	PluginValidation      bool
	LocalHooksEnabled     bool
	StrictSingleHooks     bool
	RunInPty              bool

	JobSigningJWKSPath  string // Where to find the key to sign jobs with (passed through to jobs, they might be uploading pipelines)
	JobSigningAlgorithm string // The algorithm to sign jobs with
	JobSigningKeyID     string // The key ID to sign jobs with

	JobVerificationJWKS                     jwk.Set // The set of keys to verify jobs with
	JobVerificationNoSignatureBehavior      string  // What to do if a job has no signature (either block or warn)
	JobVerificationInvalidSignatureBehavior string  // What to do if a job has an invalid signature (either block or warn)

	ANSITimestamps             bool
	TimestampLines             bool
	HealthCheckAddr            string
	DisconnectAfterJob         bool
	DisconnectAfterIdleTimeout int
	CancelGracePeriod          int
	SignalGracePeriod          time.Duration
	EnableJobLogTmpfile        bool
	JobLogPath                 string
	WriteJobLogsToStdout       bool
	LogFormat                  string
	Shell                      string
	Profile                    string
	RedactedVars               []string
	AcquireJob                 string
	TracingBackend             string
	TracingServiceName         string
}

AgentConfiguration is the run-time configuration for an agent that has been loaded from the config file and command-line params

type AgentPool

type AgentPool struct {
	// contains filtered or unexported fields
}

AgentPool manages multiple parallel AgentWorkers

func NewAgentPool

func NewAgentPool(workers []*AgentWorker) *AgentPool

NewAgentPool returns a new AgentPool

func (*AgentPool) Start

func (r *AgentPool) Start(ctx context.Context) error

Start kicks off the parallel AgentWorkers and waits for them to finish

func (*AgentPool) Stop

func (r *AgentPool) Stop(graceful bool)

type AgentWorker

type AgentWorker struct {
	// contains filtered or unexported fields
}

func NewAgentWorker

Creates the agent worker and initializes its API Client

func (*AgentWorker) AcceptAndRunJob

func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, job *api.Job) error

Accepts a job and runs it, only returns an error if something goes wrong

func (*AgentWorker) AcquireAndRunJob

func (a *AgentWorker) AcquireAndRunJob(ctx context.Context, jobId string) error

AcquireAndRunJob attempts to acquire a job an run it. It will retry at after the server determined interval (from the Retry-After response header) if the job is in the waiting state. If the job is in an unassignable state, it will return an error immediately. Otherwise, it will retry every 3s for 30 s. The whole operation will timeout after 5 min.

func (*AgentWorker) Connect

func (a *AgentWorker) Connect(ctx context.Context) error

Connects the agent to the Buildkite Agent API, retrying up to 30 times if it fails.

func (*AgentWorker) Disconnect

func (a *AgentWorker) Disconnect(ctx context.Context) error

Disconnect notifies the Buildkite API that this agent worker/session is permanently disconnecting. Don't spend long retrying, because we want to disconnect as fast as possible.

func (*AgentWorker) Heartbeat

func (a *AgentWorker) Heartbeat(ctx context.Context) error

Performs a heatbeat

func (*AgentWorker) Ping

func (a *AgentWorker) Ping(ctx context.Context) (*api.Job, error)

Performs a ping that checks Buildkite for a job or action to take Returns a job, or nil if none is found

func (*AgentWorker) RunJob

func (a *AgentWorker) RunJob(ctx context.Context, acceptResponse *api.Job) error

func (*AgentWorker) Start

func (a *AgentWorker) Start(ctx context.Context, idleMonitor *IdleMonitor) error

Starts the agent worker

func (*AgentWorker) Stop

func (a *AgentWorker) Stop(graceful bool)

Stops the agent from accepting new work and cancels any current work it's running

type AgentWorkerConfig

type AgentWorkerConfig struct {
	// Whether to set debug in the job
	Debug bool

	// Whether to set debugHTTP in the job
	DebugHTTP bool

	// What signal to use for worker cancellation
	CancelSignal process.Signal

	// Time wait between sending the CancelSignal and SIGKILL to the process
	// groups that the executor starts
	SignalGracePeriod time.Duration

	// The index of this agent worker
	SpawnIndex int

	// The configuration of the agent from the CLI
	AgentConfiguration AgentConfiguration

	// Stdout of the parent agent process. Used for job log stdout writing arg, for simpler containerized log collection.
	AgentStdout io.Writer
}

type ArtifactBatchCreator

type ArtifactBatchCreator struct {
	// contains filtered or unexported fields
}

func (*ArtifactBatchCreator) Create

func (a *ArtifactBatchCreator) Create(ctx context.Context) ([]*api.Artifact, error)

type ArtifactBatchCreatorConfig

type ArtifactBatchCreatorConfig struct {
	// The ID of the Job that these artifacts belong to
	JobID string

	// All the artifacts that need to be created
	Artifacts []*api.Artifact

	// Where the artifacts are being uploaded to on the command line
	UploadDestination string

	// CreateArtifactsTimeout, sets a context.WithTimeout around the CreateArtifacts API.
	// If it's zero, there's no context timeout and the default HTTP timeout will prevail.
	CreateArtifactsTimeout time.Duration
}

type ArtifactDownloader

type ArtifactDownloader struct {
	// contains filtered or unexported fields
}

func (*ArtifactDownloader) Download

func (a *ArtifactDownloader) Download(ctx context.Context) error

type ArtifactDownloaderConfig

type ArtifactDownloaderConfig struct {
	// The ID of the Build
	BuildID string

	// The query used to find the artifacts
	Query string

	// Which step should we look at for the jobs
	Step string

	// Whether to include artifacts from retried jobs in the search
	IncludeRetriedJobs bool

	// Where we'll be downloading artifacts to
	Destination string

	// Whether to show HTTP debugging
	DebugHTTP bool
}

type ArtifactSearcher

type ArtifactSearcher struct {
	// contains filtered or unexported fields
}

func NewArtifactSearcher

func NewArtifactSearcher(l logger.Logger, ac APIClient, buildID string) *ArtifactSearcher

func (*ArtifactSearcher) Search

func (a *ArtifactSearcher) Search(ctx context.Context, query, scope string, includeRetriedJobs, includeDuplicates bool) ([]*api.Artifact, error)

type ArtifactUploader

type ArtifactUploader struct {
	// contains filtered or unexported fields
}

func (*ArtifactUploader) Collect

func (a *ArtifactUploader) Collect() (artifacts []*api.Artifact, err error)

func (*ArtifactUploader) Upload

func (a *ArtifactUploader) Upload(ctx context.Context) error

type ArtifactUploaderConfig

type ArtifactUploaderConfig struct {
	// The ID of the Job
	JobID string

	// The path of the uploads
	Paths string

	// Where we'll be uploading artifacts
	Destination string

	// A specific Content-Type to use for all artifacts
	ContentType string

	// Whether to show HTTP debugging
	DebugHTTP bool

	// Whether to follow symbolic links when resolving globs
	GlobResolveFollowSymlinks bool

	// Whether to not upload symlinks
	UploadSkipSymlinks bool
}

type ArtifactoryDownloader

type ArtifactoryDownloader struct {
	// contains filtered or unexported fields
}

func (ArtifactoryDownloader) RepositoryFileLocation

func (d ArtifactoryDownloader) RepositoryFileLocation() string

func (ArtifactoryDownloader) RepositoryName

func (d ArtifactoryDownloader) RepositoryName() string

func (ArtifactoryDownloader) RepositoryPath

func (d ArtifactoryDownloader) RepositoryPath() string

func (ArtifactoryDownloader) Start

type ArtifactoryDownloaderConfig

type ArtifactoryDownloaderConfig struct {
	// The Artifactory repository name and the path, for example, rt://my-repo-name/foo/bar
	Repository string

	// The root directory of the download
	Destination string

	// The relative path that should be preserved in the download folder,
	// also its location in the repo
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

type ArtifactoryUploader

type ArtifactoryUploader struct {
	// The artifactory bucket path set from the destination
	Path string

	// The artifactory bucket name set from the destination
	Repository string
	// contains filtered or unexported fields
}

func (*ArtifactoryUploader) URL

func (u *ArtifactoryUploader) URL(artifact *api.Artifact) string

func (*ArtifactoryUploader) Upload

func (u *ArtifactoryUploader) Upload(artifact *api.Artifact) error

type ArtifactoryUploaderConfig

type ArtifactoryUploaderConfig struct {
	// The destination which includes the Artifactory bucket name and the path.
	// e.g artifactory://my-repo-name/foo/bar
	Destination string

	// Whether or not HTTP calls should be debugged
	DebugHTTP bool
}

type Download

type Download struct {
	// contains filtered or unexported fields
}

func NewDownload

func NewDownload(l logger.Logger, client *http.Client, c DownloadConfig) *Download

func (Download) Start

func (d Download) Start(ctx context.Context) error

type DownloadConfig

type DownloadConfig struct {
	// The actual URL to get the file from
	URL string

	// The root directory of the download
	Destination string

	// Optional Headers to append to the request
	Headers map[string]string

	// The relative path that should be preserved in the download folder
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

type EC2MetaData

type EC2MetaData struct {
}

func (EC2MetaData) Get

func (e EC2MetaData) Get() (map[string]string, error)

func (EC2MetaData) GetPaths

func (e EC2MetaData) GetPaths(paths map[string]string) (map[string]string, error)

Takes a map of tags and meta-data paths to get, returns a map of tags and fetched values.

type EC2Tags

type EC2Tags struct {
}

func (EC2Tags) Get

func (e EC2Tags) Get() (map[string]string, error)

type ECSMetadata added in v3.43.0

type ECSMetadata struct {
}

func (ECSMetadata) Get added in v3.43.0

func (e ECSMetadata) Get() (map[string]string, error)

type Error

type Error struct {
	Status  int    `json:"status"`  // Error code
	Message string `json:"message"` // Message describing the error.
}

An Error reports more details on an individual error in an ErrorResponse.

type FetchTagsConfig

type FetchTagsConfig struct {
	Tags []string

	TagsFromEC2MetaData       bool
	TagsFromEC2MetaDataPaths  []string
	TagsFromEC2Tags           bool
	TagsFromECSMetaData       bool
	TagsFromGCPMetaData       bool
	TagsFromGCPMetaDataPaths  []string
	TagsFromGCPLabels         bool
	TagsFromHost              bool
	WaitForEC2TagsTimeout     time.Duration
	WaitForEC2MetaDataTimeout time.Duration
	WaitForECSMetaDataTimeout time.Duration
	WaitForGCPLabelsTimeout   time.Duration
}

type FormUploader

type FormUploader struct {
	// contains filtered or unexported fields
}

func NewFormUploader

func NewFormUploader(l logger.Logger, c FormUploaderConfig) *FormUploader

func (*FormUploader) URL

func (u *FormUploader) URL(artifact *api.Artifact) string

The FormUploader doens't specify a URL, as one is provided by Buildkite after uploading

func (*FormUploader) Upload

func (u *FormUploader) Upload(artifact *api.Artifact) error

type FormUploaderConfig

type FormUploaderConfig struct {
	// Whether or not HTTP calls should be debugged
	DebugHTTP bool
}

type GCPLabels

type GCPLabels struct{}

func (GCPLabels) Get

func (e GCPLabels) Get(ctx context.Context) (map[string]string, error)

type GCPMetaData

type GCPMetaData struct {
}

func (GCPMetaData) Get

func (e GCPMetaData) Get() (map[string]string, error)

func (GCPMetaData) GetPaths

func (e GCPMetaData) GetPaths(paths map[string]string) (map[string]string, error)

Takes a map of tags and meta-data paths to get, returns a map of tags and fetched values.

type GSDownloader

type GSDownloader struct {
	// contains filtered or unexported fields
}

func NewGSDownloader

func NewGSDownloader(l logger.Logger, c GSDownloaderConfig) *GSDownloader

func (GSDownloader) BucketFileLocation

func (d GSDownloader) BucketFileLocation() string

func (GSDownloader) BucketName

func (d GSDownloader) BucketName() string

func (GSDownloader) BucketPath

func (d GSDownloader) BucketPath() string

func (GSDownloader) Start

func (d GSDownloader) Start(ctx context.Context) error

type GSDownloaderConfig

type GSDownloaderConfig struct {
	// The name of the bucket
	Bucket string

	// The root directory of the download
	Destination string

	// The relative path that should be preserved in the download folder,
	// also its location in the bucket
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

type GSUploader

type GSUploader struct {
	// The gs bucket path set from the destination
	BucketPath string

	// The gs bucket name set from the destination
	BucketName string
	// contains filtered or unexported fields
}

func NewGSUploader

func NewGSUploader(l logger.Logger, c GSUploaderConfig) (*GSUploader, error)

func (*GSUploader) URL

func (u *GSUploader) URL(artifact *api.Artifact) string

func (*GSUploader) Upload

func (u *GSUploader) Upload(artifact *api.Artifact) error

type GSUploaderConfig

type GSUploaderConfig struct {
	// The destination which includes the GS bucket name and the path.
	// gs://my-bucket-name/foo/bar
	Destination string

	// Whether or not HTTP calls shoud be debugged
	DebugHTTP bool
}

type IdleMonitor

type IdleMonitor struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewIdleMonitor

func NewIdleMonitor(totalAgents int) *IdleMonitor

func (*IdleMonitor) Idle

func (i *IdleMonitor) Idle() bool

func (*IdleMonitor) MarkBusy

func (i *IdleMonitor) MarkBusy(agentUUID string)

func (*IdleMonitor) MarkIdle

func (i *IdleMonitor) MarkIdle(agentUUID string)

type JobRunner

type JobRunner struct {

	// How the JobRunner should respond in various signature failure modes
	InvalidSignatureBehavior string
	NoSignatureBehavior      string
	// contains filtered or unexported fields
}

func (*JobRunner) Cancel

func (r *JobRunner) Cancel() error

func (*JobRunner) CancelAndStop added in v3.21.0

func (r *JobRunner) CancelAndStop() error

func (*JobRunner) Run

func (r *JobRunner) Run(ctx context.Context) error

Runs the job

type JobRunnerConfig

type JobRunnerConfig struct {
	// The configuration of the agent from the CLI
	AgentConfiguration AgentConfiguration

	// How often to check if the job has been cancelled
	JobStatusInterval time.Duration

	// The JSON Web Keyset for verifying the job
	JWKS jwk.Set

	// A scope for metrics within a job
	MetricsScope *metrics.Scope

	// The job to run
	Job *api.Job

	// What signal to use for worker cancellation
	CancelSignal process.Signal

	// Whether to set debug in the job
	Debug bool

	// Whether to set debug HTTP Requests in the job
	DebugHTTP bool

	// Stdout of the parent agent process. Used for job log stdout writing arg, for simpler containerized log collection.
	AgentStdout io.Writer
}

type LogStreamer

type LogStreamer struct {
	// contains filtered or unexported fields
}

func NewLogStreamer

Creates a new instance of the log streamer

func (*LogStreamer) FailedChunks

func (ls *LogStreamer) FailedChunks() int

func (*LogStreamer) Process

func (ls *LogStreamer) Process(output []byte) error

Process streams the output.

func (*LogStreamer) Start

func (ls *LogStreamer) Start(ctx context.Context) error

Spins up x number of log streamer workers

func (*LogStreamer) Stop

func (ls *LogStreamer) Stop() error

Waits for all the chunks to be uploaded, then shuts down all the workers

type LogStreamerChunk

type LogStreamerChunk struct {
	// The contents of the chunk
	Data []byte

	// The sequence number of this chunk
	Order uint64

	// The byte offset of this chunk
	Offset uint64

	// The byte size of this chunk
	Size uint64
}

type LogStreamerConfig

type LogStreamerConfig struct {
	// How many log streamer workers are running at any one time
	Concurrency int

	// The maximum size of each chunk
	MaxChunkSizeBytes uint64

	// The maximum size of the log
	MaxSizeBytes uint64
}

type LogWriter added in v3.32.0

type LogWriter struct {
	// contains filtered or unexported fields
}

func (LogWriter) Write added in v3.32.0

func (w LogWriter) Write(bytes []byte) (int, error)

type PipelineUploader added in v3.44.0

type PipelineUploader struct {
	Client         APIClient
	Change         *api.PipelineChange
	JobID          string
	RetrySleepFunc func(time.Duration)
}

PipelineUploader contains the data needed to upload a pipeline to Buildkite

func (*PipelineUploader) Upload added in v3.45.0

func (u *PipelineUploader) Upload(ctx context.Context, l logger.Logger) error

Upload will first attempt to perform an async pipeline upload and, depending on the API's response, it will poll for the upload's status.

There are 3 "routes" that are relevant 1. Async Route: /jobs/:job_uuid/pipelines?async=true 2. Sync Route: /jobs/:job_uuid/pipelines 3. Status Route: /jobs/:job_uuid/pipelines/:upload_uuid

In this method, the agent will first upload the pipeline to the Async Route. Then, depending on the response it will behave differetly

1. The Async Route responds 202: poll the Status Route until the upload has beed "applied" 2. The Async Route responds with other 2xx: exit, the upload succeeded synchronously (possibly after retry) 3. The Async Route responds with other xxx: retry uploading the pipeline to the Async Route

Note that the Sync Route is not used by this version of the agent at all. Typically, the Aysnc Route will return 202 whether or not the pipeline upload has been processed.

However, the API has the option to treat the Async Route as if it were the Sync Route by returning a 2xx that's not a 202. This will tigger option 2. While the API currently does not do this, we want to maintain the flexbitity to do so in the future. If that is implemented, the Status Route will not be polled, and either the Async Route will be retried until a (non 202) 2xx is returned from the API, or the method will exit early with no error. This reiterates option 2.

If, during a retry loop in option 3, the API returns a 2xx that is a 202, then we assume the API changed to supporting Async Uploads between retries and option 1 will be taken.

type S3Downloader

type S3Downloader struct {
	// contains filtered or unexported fields
}

func NewS3Downloader

func NewS3Downloader(l logger.Logger, c S3DownloaderConfig) *S3Downloader

func (S3Downloader) BucketFileLocation

func (d S3Downloader) BucketFileLocation() string

func (S3Downloader) BucketName

func (d S3Downloader) BucketName() string

func (S3Downloader) BucketPath

func (d S3Downloader) BucketPath() string

func (S3Downloader) Start

func (d S3Downloader) Start(ctx context.Context) error

type S3DownloaderConfig

type S3DownloaderConfig struct {
	// The client for interacting with S3
	S3Client *s3.S3

	// The S3 bucket name and the path, for example, s3://my-bucket-name/foo/bar
	S3Path string

	// The root directory of the download
	Destination string

	// The relative path that should be preserved in the download folder,
	// also its location in the bucket
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

type S3Uploader

type S3Uploader struct {
	// The s3 bucket path set from the destination
	BucketPath string

	// The s3 bucket name set from the destination
	BucketName string
	// contains filtered or unexported fields
}

func NewS3Uploader

func NewS3Uploader(l logger.Logger, c S3UploaderConfig) (*S3Uploader, error)

func (*S3Uploader) URL

func (u *S3Uploader) URL(artifact *api.Artifact) string

func (*S3Uploader) Upload

func (u *S3Uploader) Upload(artifact *api.Artifact) error

type S3UploaderConfig

type S3UploaderConfig struct {
	// The destination which includes the S3 bucket name and the path.
	// For example, s3://my-bucket-name/foo/bar
	Destination string

	// Whether or not HTTP calls should be debugged
	DebugHTTP bool
}

type Uploader

type Uploader interface {
	// The Artifact.URL property is populated with what ever is returned
	// from this method prior to uploading.
	URL(*api.Artifact) string

	// The actual uploading of the file
	Upload(*api.Artifact) error
}

Directories

Path Synopsis
Package plugin provides types for managing agent plugins.
Package plugin provides types for managing agent plugins.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL