Documentation
¶
Overview ¶
Package controller is the wiring between everything else: it owns the reconcile loop, the GitHub client cache, webhook ingest, the agent task queue, the log relay and the background housekeeping.
Nothing in here decides how many runners a pool should have -- that is internal/scheduler, and it is a pure function -- and nothing in here writes SQL, which is internal/store. What this package does is turn a decision into GitHub calls, database rows and tasks for an agent, in an order that leaves the fleet in a defensible state when any single step fails.
The controller never dials an agent. Agents connect outbound, long-poll for tasks and POST their results, which is why the task queue and the log relay are shaped the way they are.
Index ¶
- Constants
- Variables
- func IsDemoID(id string) bool
- type BackendInfoView
- type CapacityDemandEvent
- type Controller
- func (c *Controller) AcceptLogStream(hostID, streamID string, r io.Reader) error
- func (c *Controller) ApplyMigration(ctx context.Context, req MigrationApplyRequest) (*MigrationOutcome, error)
- func (c *Controller) Auth() *auth.Service
- func (c *Controller) Backends() *backend.Registry
- func (c *Controller) CancelJobWorkflow(ctx context.Context, jobID string, force bool) (*store.Job, error)
- func (c *Controller) ClientFor(ctx context.Context, installationID string) (github.Client, error)
- func (c *Controller) ClientForRepo(ctx context.Context, repoFullName string) (github.Client, error)
- func (c *Controller) Config() *config.Config
- func (c *Controller) ControlProvisioning(ctx context.Context, ids []string, action string) ([]store.ProvisioningResult, error)
- func (c *Controller) DeleteHost(ctx context.Context, id string) error
- func (c *Controller) DeleteInstallation(ctx context.Context, id string) error
- func (c *Controller) DeletePool(ctx context.Context, id string) error
- func (c *Controller) DrainRunner(ctx context.Context, runnerID, reason string, confirmed bool) (*store.Runner, error)
- func (c *Controller) EmbeddedAgent() *agent.Agent
- func (c *Controller) EmbeddedTransport() agent.Transport
- func (c *Controller) Events() *events.Bus
- func (c *Controller) ExplainJob(ctx context.Context, jobID string) (*JobExplanation, error)
- func (c *Controller) Fenced() store.RecoveryFence
- func (c *Controller) Forget(installationID string)
- func (c *Controller) HandleWebhook(w http.ResponseWriter, r *http.Request)
- func (c *Controller) Heartbeat(ctx context.Context, hostID string, req agent.HeartbeatRequest) (*agent.HeartbeatResponse, error)
- func (c *Controller) HostFit(ctx context.Context, p *store.Pool) (HostFit, error)
- func (c *Controller) HostView(h *store.Host) HostView
- func (c *Controller) JobEvents(ctx context.Context, jobID string) ([]*store.JobEvent, error)
- func (c *Controller) JobRenderer(ctx context.Context) (*JobRenderer, error)
- func (c *Controller) Join(ctx context.Context, req agent.JoinRequest, ip string) (*agent.JoinResponse, error)
- func (c *Controller) LastPollAt() time.Time
- func (c *Controller) LoadFence(ctx context.Context) error
- func (c *Controller) LoopPanics() map[string]loopPanic
- func (c *Controller) NoteAgentSession(ctx context.Context, hostID, sessionID string)
- func (c *Controller) Now() time.Time
- func (c *Controller) Nudge()
- func (c *Controller) OpenLogStream(ctx context.Context, runnerID string, opts backend.LogOptions) (<-chan []byte, func(), error)
- func (c *Controller) PlanMigration(ctx context.Context, req MigrationPlanRequest) (*MigrationPlan, error)
- func (c *Controller) PollTasks(ctx context.Context, hostID string, wait time.Duration) (*agent.TaskBatch, error)
- func (c *Controller) PollerEnabled() bool
- func (c *Controller) PollingOnly() bool
- func (c *Controller) PoolCapacityProblems() []Problem
- func (c *Controller) PoolCountsByInstallation(ctx context.Context) (map[string]int, error)
- func (c *Controller) PoolRenderer(ctx context.Context) (*PoolRenderer, error)
- func (c *Controller) PoolRunnerGroupProblems() []Problem
- func (c *Controller) PrewarmPool(ctx context.Context, p *store.Pool) (int, error)
- func (c *Controller) ProbeInstallation(ctx context.Context, installationID string) (*github.AppInfo, error)
- func (c *Controller) Problems(ctx context.Context) ([]Problem, error)
- func (c *Controller) PublishHost(h *store.Host)
- func (c *Controller) PublishHostDeleted(id string)
- func (c *Controller) PublishInstallation(ctx context.Context, inst *store.Installation)
- func (c *Controller) PublishInstallationDeleted(id string)
- func (c *Controller) PublishPool(ctx context.Context, kind events.Kind, p *store.Pool)
- func (c *Controller) PublishPoolDeleted(id string)
- func (c *Controller) Reconcile(ctx context.Context) error
- func (c *Controller) Registry() *prometheus.Registry
- func (c *Controller) RemoveRunner(ctx context.Context, runnerID, reason string, force, confirmed bool) (*store.Runner, error)
- func (c *Controller) ReportResult(ctx context.Context, hostID string, res agent.TaskResult) error
- func (c *Controller) ReportRunners(ctx context.Context, hostID string, reports []agent.RunnerReport) error
- func (c *Controller) RunnerImage(p *store.Pool) string
- func (c *Controller) RunnerRenderer(ctx context.Context, runners []*store.Runner) (*RunnerRenderer, error)
- func (c *Controller) Samples(ctx context.Context, since time.Time) ([]store.FleetSample, error)
- func (c *Controller) SeedDemo(ctx context.Context) error
- func (c *Controller) SeedStuck(ctx context.Context) error
- func (c *Controller) Start(ctx context.Context) error
- func (c *Controller) StartEmbeddedAgent(ctx context.Context, cfg *config.Config) error
- func (c *Controller) Stats(ctx context.Context, window time.Duration) (*Stats, error)
- func (c *Controller) Stop(ctx context.Context) error
- func (c *Controller) Store() *store.Store
- func (c *Controller) Unfence(ctx context.Context) error
- func (c *Controller) UpdateConfig(fn func(*config.Config)) *config.Config
- type Fence
- type HostExclusion
- type HostFit
- type HostStats
- type HostView
- type InstallationView
- type JobExplanation
- type JobRenderer
- type JobView
- type MigrationApplyRequest
- type MigrationOutcome
- type MigrationPlan
- type MigrationPlanRequest
- type MigrationPoolOption
- type MigrationResult
- type Options
- type PoolCountsView
- type PoolRenderer
- type PoolStats
- type PoolView
- type Problem
- type ProblemsView
- type RunnerRenderer
- type RunnerStats
- type RunnerView
- type ScopedJobStats
- type Stats
Constants ¶
const ( ExcludedBackend = "backend" ExcludedPlatform = "platform" ExcludedSize = "size" )
The reasons a host the selector reached cannot run the pool.
const ( // MaxPlanRepos is the most repositories one plan will read workflows from. // The wizard scans in pages; an operator with more repositories than this // migrates them in batches, which is how anyone would want to review them // anyway. MaxPlanRepos = 50 // MaxApplyRepos is the most pull requests one call will open. It is lower // than the plan's limit on purpose: fifty pull requests landing at once in // somebody's review queue is not a migration, it is an incident. MaxApplyRepos = 25 )
Both halves fan out into GitHub's API on a quota the scheduler shares, so each bounds what one request will do.
const ( BadgeAdded = "added" BadgePresent = "present" BadgeNoReadme = "no_readme" BadgeUnread = "unread" BadgeOff = "off" )
The values MigrationResult.Badge takes.
const ( // LeaseTTL is how long a controller's claim on the database stands without // being renewed. It is comfortably longer than the renewal interval, so a // slow disk or a paused process does not lose a lease it still holds, and // short enough that a controller killed rather than stopped does not keep // a fleet waiting for long. LeaseTTL = 90 * time.Second )
const ManagedRunnerGroupName = "zoomies"
ManagedRunnerGroupName is the organisation runner group Zoomies provisions and uses for pools that do not explicitly choose another one.
const SeedEnvVar = "ZOOMIES_SEED_DEMO"
SeedEnvVar names the environment variable that turns demo seeding on. The Playwright suite sets it (see web/tests/support/serve.mjs) so the UI has a fixture fleet to render.
const StuckSeedEnvVar = "ZOOMIES_SEED_STUCK"
StuckSeedEnvVar names the environment variable that adds the diagnostics fixture on top of the demo fleet.
It is separate from ZOOMIES_SEED_DEMO on purpose, and it is the opposite of what a demo wants. The demo deliberately keeps its two starting runners young (see freshenDemoRunners) so an instance left open never reports a fault in a fleet that has no agent to have one. Every diagnostic this project has for a fleet in trouble -- the problems drawer, the stuck-runner shapes, the blocked pool, the held job -- is therefore unreachable from the demo, which is why they went untested until this existed.
Nothing here is for an operator. It exists so a test can open the pages an operator opens on their worst day.
const UnmatchedPool = "unmatched"
UnmatchedPool is the `pool` label for work no pool here claims.
A literal rather than an empty string, because Prometheus has no notion of an absent label value and a blank one is indistinguishable from a bug. A real pool named `unmatched` would merge with it; that is a price worth naming in the docs rather than designing around.
Variables ¶
var ( ErrWorkflowCancellationDisabled = errors.New("workflow cancellation is disabled") ErrJobAlreadyCompleted = errors.New("the job has already completed") )
var ( // ErrNoMigrationPool means the installation has no enabled pool, so there // is nowhere to send a job and nothing to plan. ErrNoMigrationPool = errors.New("there is nowhere to migrate to: this installation has no enabled pool") // ErrNothingMapped means the apply request maps no hosted label to a pool // and points no single job at one either, so no workflow would change. ErrNothingMapped = errors.New("nothing would change: no hosted label is mapped to a pool, and no job is pointed at one by name") // ErrNotAWorkflow means the apply request named a file GitHub would never // run, which would be a pull request that quietly changed nothing. ErrNotAWorkflow = errors.New("only files directly under .github/workflows can be migrated") // ErrBadOverride means an override named something other than one job in // one workflow file in one repository, so there is no single place for it // to apply to. ErrBadOverride = errors.New("an override names one job in one workflow file in one repository") )
var ErrConfirmationRequired = errors.New("this runner is running a job, and draining it will end that job")
ErrConfirmationRequired is an operator asking to drain a runner that is running a job, without having said they accept losing it.
A drain is not the gentle option it reads as. The stop task carries agent.DefaultStopTimeout, so the runner gets five minutes and is then killed; a twenty-minute job drained at minute one dies at minute six and GitHub marks it failed. That is the behaviour -- an operator draining a host for maintenance needs the machine to actually empty -- but it must not be something they get by accident, so the paths an operator reaches ask first.
The scheduler is deliberately not subject to this. It reaches a drain through the unexported drainRunnerID, and never proposes one for a busy runner in the first place; automatic scale-down must not be waiting on anybody to confirm.
var ErrDemoFixture = errors.New("this is a seeded demo installation with no GitHub App behind it")
ErrDemoFixture is returned by the demo installation for anything that would have to reach GitHub to be true.
var ErrPrewarmUnsupported = errors.New("the process backend does not support image prewarming; its runners install the runner archive themselves")
PrewarmPool queues one idempotent image preparation task on every matching healthy host. A failure is recorded per host and never affects scheduling. ErrPrewarmUnsupported is returned for a pool whose backend has no image to pull ahead of time. It is a refusal the caller can act on, kept apart from a failure to list hosts so the API can answer the two differently.
var ErrStreamUnknown = errors.New("controller: no log stream is open with that ID")
ErrStreamUnknown is returned when an agent posts a log stream nobody is waiting for, which happens whenever the last viewer closed the tab between the task being issued and the agent acting on it.
Functions ¶
func IsDemoID ¶
IsDemoID reports whether an identifier belongs to the seeded demo fixtures.
The fixtures have no GitHub behind them, so the two places that would otherwise reach out on their behalf -- the credential prober and the registration reaper -- check this and skip. Without it a demo or a UI test run fills the problems drawer with "this installation is not usable" and the log with parse failures, none of which says anything about the fleet.
A fixture's identifier begins "demo" after its prefix and is never the shape store.NewID produces. Both halves matter: the prefix alone also matched a real identifier whose random part happened to start with those four letters, which one row in a million does, and such an installation was then skipped by the prober, the poller and the reap, and such a host was never reclaimed. A test in this package holds every fixture literal to the rule.
Types ¶
type BackendInfoView ¶
type BackendInfoView struct {
Kind store.BackendKind `json:"kind"`
Available bool `json:"available"`
Version string `json:"version,omitempty"`
Rootless bool `json:"rootless"`
Endpoint string `json:"endpoint,omitempty"`
Detail string `json:"detail,omitempty"`
DinD bool `json:"supports_dind"`
}
BackendInfoView describes one backend a host offers.
type CapacityDemandEvent ¶
type CapacityDemandEvent struct {
SchemaVersion int `json:"schema_version"`
EventID string `json:"event_id"`
Type string `json:"type"`
Timestamp time.Time `json:"timestamp"`
PoolID string `json:"pool_id"`
HostSelector store.StringMap `json:"host_selector"`
Backend store.BackendKind `json:"backend"`
RequiredRunnerSlots int `json:"required_runner_slots"`
CurrentCapacity int `json:"current_capacity"`
QueuedJobCount int `json:"queued_job_count"`
OldestQueueAgeSeconds int64 `json:"oldest_queue_age_seconds"`
}
CapacityDemandEvent is the stable wire contract consumed by provisioners. RequiredRunnerSlots is positive for demand and negative for removable idle capacity. It is a reading, not an increment: the pool should have CurrentCapacity+RequiredRunnerSlots slots, and a receiver that adds RequiredRunnerSlots to whatever it already asked for adds the same shortfall again on every re-delivery, because each attempt after the cooldown carries a fresh EventID for what may be the same unmet demand.
type Controller ¶
type Controller struct {
// contains filtered or unexported fields
}
Controller owns the control plane's moving parts and their lifecycles.
func New ¶
func New(opts Options) (*Controller, error)
New validates the options and builds a controller that is not yet running.
func (*Controller) AcceptLogStream ¶
func (c *Controller) AcceptLogStream(hostID, streamID string, r io.Reader) error
AcceptLogStream consumes the agent's outbound chunked POST and fans it out. It returns when the agent closes the body or the stream is torn down.
hostID is the host the request authenticated as, and it must be the host the stream was opened for: whatever arrives here is shown to operators as that runner's output, so a stream another agent could write into would let one host put words in another's mouth. Every other agent endpoint checks the same thing against the runner's host; this one checks it against the stream's.
func (*Controller) ApplyMigration ¶
func (c *Controller) ApplyMigration(ctx context.Context, req MigrationApplyRequest) (*MigrationOutcome, error)
ApplyMigration opens the pull requests a plan described, one repository at a time.
It re-reads and re-plans rather than trusting a plan the browser sends back. The workflows may have changed since the review step, and the alternative -- committing content the browser supplied -- would make this a way to write arbitrary files into any repository the App can reach. The caller has already bounded req.Repos to MaxApplyRepos; an empty mapping is ErrNothingMapped.
func (*Controller) Auth ¶
func (c *Controller) Auth() *auth.Service
Auth returns the authentication and audit service.
func (*Controller) Backends ¶
func (c *Controller) Backends() *backend.Registry
Backends returns the backend registry, or nil in a controller that runs no embedded agent.
func (*Controller) CancelJobWorkflow ¶
func (c *Controller) CancelJobWorkflow(ctx context.Context, jobID string, force bool) (*store.Job, error)
CancelJobWorkflow asks GitHub to cancel the workflow run containing jobID. Zoomies deliberately leaves the local job pending until GitHub confirms the terminal state through a webhook or the fallback poller.
func (*Controller) ClientFor ¶
ClientFor returns a GitHub client for one installation, identified by its Zoomies installation ID ("ins_...").
func (*Controller) ClientForRepo ¶
ClientForRepo returns the client for whichever installation covers a repository, preferring a repo-scoped installation over the owning org.
func (*Controller) Config ¶
func (c *Controller) Config() *config.Config
Config returns the configuration as it currently stands: what the controller was built with, as changed since by UpdateConfig. It is a snapshot and must not be written through; the next UpdateConfig would silently discard the write, and until then every loop would be reading it unsynchronised.
func (*Controller) ControlProvisioning ¶
func (c *Controller) ControlProvisioning(ctx context.Context, ids []string, action string) ([]store.ProvisioningResult, error)
func (*Controller) DeleteHost ¶
func (c *Controller) DeleteHost(ctx context.Context, id string) error
DeleteHost removes a host and announces its runner rows and then the host.
func (*Controller) DeleteInstallation ¶
func (c *Controller) DeleteInstallation(ctx context.Context, id string) error
DeleteInstallation removes an installation with its pools and their runner rows, forgets its GitHub client, and announces each thing that went in the order a page wants them: runners, pools, then the installation.
func (*Controller) DeletePool ¶
func (c *Controller) DeletePool(ctx context.Context, id string) error
DeletePool removes a pool and announces everything that went with it: each runner row, then the pool. The runners go first so a page that drops them has nothing left to explain when the pool disappears.
func (*Controller) DrainRunner ¶
func (c *Controller) DrainRunner(ctx context.Context, runnerID, reason string, confirmed bool) (*store.Runner, error)
DrainRunner asks a runner to finish its current job and exit.
confirmed is the operator saying they accept that a job in flight will be ended: see ErrConfirmationRequired. It is ignored for a runner that is not busy, because there is nothing to lose.
func (*Controller) EmbeddedAgent ¶
func (c *Controller) EmbeddedAgent() *agent.Agent
EmbeddedAgent returns the in-process agent, or nil when this controller runs without one.
func (*Controller) EmbeddedTransport ¶
func (c *Controller) EmbeddedTransport() agent.Transport
EmbeddedTransport returns an agent.Transport that calls this controller directly. It is what makes the single-VM case one process: no listener, no TLS, no token on the wire.
func (*Controller) Events ¶
func (c *Controller) Events() *events.Bus
Events returns the bus the API's SSE endpoint subscribes to.
func (*Controller) ExplainJob ¶
func (c *Controller) ExplainJob(ctx context.Context, jobID string) (*JobExplanation, error)
ExplainJob works out why a job is where it is.
It reads rather than decides: the scheduler has already said why it could not place a runner, and repeating that decision here would give an operator two answers that drift apart. What this adds is the surrounding facts, which the plan does not carry -- whose runner it is, and whether that runner's host is still alive.
func (*Controller) Fenced ¶
func (c *Controller) Fenced() store.RecoveryFence
Fenced reports whether this fleet is held for recovery, and why.
func (*Controller) Forget ¶
func (c *Controller) Forget(installationID string)
Forget drops the cached client for an installation. The API calls it after deleting or re-keying one, so the next call rebuilds from the stored row.
func (*Controller) HandleWebhook ¶
func (c *Controller) HandleWebhook(w http.ResponseWriter, r *http.Request)
HandleWebhook is the endpoint GitHub delivers to, mounted by the API at config.GitHub.WebhookPath.
It answers quickly on purpose. GitHub gives a webhook ten seconds before it records the delivery as failed, so this verifies, writes one row and returns; the scheduling it triggers happens on the reconcile loop, which nothing here waits for.
func (*Controller) Heartbeat ¶
func (c *Controller) Heartbeat(ctx context.Context, hostID string, req agent.HeartbeatRequest) (*agent.HeartbeatResponse, error)
Heartbeat records that a host is alive, merges anything its agent observed, and tells the agent whether the controller still recognises it.
func (*Controller) HostFit ¶
HostFit counts the hosts that could run a pool, using the scheduler's own placement rule so the wizard can never disagree with the fleet.
func (*Controller) HostView ¶
func (c *Controller) HostView(h *store.Host) HostView
HostView renders a host as the API returns it.
backend_info is the agent's own probe, which includes the backends it could not use and the sentence explaining why: that sentence is the whole answer to "this host is connected, so why is nothing running on it?". A host that joined an older controller has no probe stored, so its available kinds are rendered as the bare list they are, and nothing is invented about the backends it never reported on.
func (*Controller) JobRenderer ¶
func (c *Controller) JobRenderer(ctx context.Context) (*JobRenderer, error)
JobRenderer builds the index a page of jobs is rendered from.
func (*Controller) Join ¶
func (c *Controller) Join(ctx context.Context, req agent.JoinRequest, ip string) (*agent.JoinResponse, error)
Join enrols an agent: it redeems the single-use join token, creates or updates the host row, and mints the long-lived agent token that every later request carries. The plaintext token is returned exactly once.
func (*Controller) LastPollAt ¶
func (c *Controller) LastPollAt() time.Time
LastPollAt is when the fallback poller last completed a sweep, or the zero time if it has not completed one since this controller started.
func (*Controller) LoadFence ¶
func (c *Controller) LoadFence(ctx context.Context) error
LoadFence reads the recovery fence from the database into this controller.
It is called at start and again whenever the fence is changed, rather than on every pass: the fence changes once in the life of a recovery, and a database read in the hot path of a loop that runs every few seconds would cost more than it could ever save.
func (*Controller) LoopPanics ¶
func (c *Controller) LoopPanics() map[string]loopPanic
LoopPanics returns the crashes recorded against each background loop since the process started, for the problems drawer.
func (*Controller) NoteAgentSession ¶
func (c *Controller) NoteAgentSession(ctx context.Context, hostID, sessionID string)
NoteAgentSession folds the session an agent identified itself with into its host's record, and warns when the host has seen that session before.
Seeing a session a host has already moved on from is the duplicate-agent signal: an agent mints its session at start-up, so restarting moves the id forward and never back, and only two agents sharing one host's credentials -- a cloned VM, or a copied state directory -- hand the same pair back and forth. Counting plain changes instead would flag every ordinary restart.
Detect only, by decision. Refusing the older session would be a coin toss over which of two live agents keeps the host, and both are running real work; the problems panel names it and an operator decides which machine should not be there.
func (*Controller) Now ¶
func (c *Controller) Now() time.Time
Now returns the controller's clock, always in UTC.
func (*Controller) Nudge ¶
func (c *Controller) Nudge()
Nudge wakes the reconcile loop immediately.
It never blocks and never queues: the channel holds a single token, so a burst of webhooks costs one reconcile pass rather than one per delivery.
func (*Controller) OpenLogStream ¶
func (c *Controller) OpenLogStream(ctx context.Context, runnerID string, opts backend.LogOptions) (<-chan []byte, func(), error)
OpenLogStream subscribes to a runner's output and returns the channel the API's SSE handler reads, plus the function that unsubscribes.
The channel is closed when the stream ends, so a handler can range over it. Bytes are dropped rather than queued without limit when a reader falls behind: a wedged browser tab must not hold a runner's output in memory.
func (*Controller) PlanMigration ¶
func (c *Controller) PlanMigration(ctx context.Context, req MigrationPlanRequest) (*MigrationPlan, error)
PlanMigration works out what a migration would change. It creates nothing.
It answers store.ErrNotFound for an installation that does not exist, ErrNoMigrationPool for one with nowhere to migrate to, and GitHub's own errors when the App cannot see or read the repositories.
func (*Controller) PollTasks ¶
func (c *Controller) PollTasks(ctx context.Context, hostID string, wait time.Duration) (*agent.TaskBatch, error)
PollTasks blocks until this host has work or wait elapses.
It waits on a per-host channel rather than sleeping in a loop, so a task enqueued by a reconcile reaches the agent in the same instant rather than up to a poll interval later.
func (*Controller) PollerEnabled ¶
func (c *Controller) PollerEnabled() bool
PollerEnabled reports whether the fallback poller is running at all. It is off by configuration, never by failure, so a fleet with no safety net is a choice somebody made and the UI should say which.
func (*Controller) PollingOnly ¶
func (c *Controller) PollingOnly() bool
PollingOnly reports whether scaling is running on the fallback poller because no webhook has ever been received.
func (*Controller) PoolCapacityProblems ¶
func (c *Controller) PoolCapacityProblems() []Problem
func (*Controller) PoolCountsByInstallation ¶
PoolCountsByInstallation answers "how much depends on this installation?", which is what makes the delete confirmation honest.
func (*Controller) PoolRenderer ¶
func (c *Controller) PoolRenderer(ctx context.Context) (*PoolRenderer, error)
PoolRenderer gathers the per-pool counts, installation targets and queue depths in three queries rather than three per pool.
func (*Controller) PoolRunnerGroupProblems ¶
func (c *Controller) PoolRunnerGroupProblems() []Problem
PoolRunnerGroupProblems reports the pools whose runners went into GitHub's default runner group because the group they asked for could not be resolved.
It is a warning rather than a log line because runner-group policy can make a runner either broader than intended or completely ineligible. A pool put into a named group to fence its runners off and quietly placed in Default is wider than its operator asked for; a group that blocks public repositories leaves their matching jobs queued while its runners look healthy and idle.
func (*Controller) PrewarmPool ¶
func (*Controller) ProbeInstallation ¶
func (c *Controller) ProbeInstallation(ctx context.Context, installationID string) (*github.AppInfo, error)
ProbeInstallation verifies one installation's credentials end to end and records the outcome, publishing it so the UI updates.
It returns the App information on success and the operator-facing error on failure; the error is also stored, because "why is nothing scaling?" is answered on the Installations page.
func (*Controller) PublishHost ¶
func (c *Controller) PublishHost(h *store.Host)
PublishHost announces a host an operator changed: its capacity, its labels, or whether it is cordoned.
func (*Controller) PublishHostDeleted ¶
func (c *Controller) PublishHostDeleted(id string)
PublishHostDeleted announces that a host is gone.
func (*Controller) PublishInstallation ¶
func (c *Controller) PublishInstallation(ctx context.Context, inst *store.Installation)
PublishInstallation announces an installation an operator added or edited.
func (*Controller) PublishInstallationDeleted ¶
func (c *Controller) PublishInstallationDeleted(id string)
PublishInstallationDeleted announces that an installation is gone.
func (*Controller) PublishPool ¶
PublishPool announces a pool an operator created or changed, in the shape GET /pools/{id} returns. The API calls it after its own write: the operator who made the change is looking at the response, but every other open dashboard learns about it from here.
func (*Controller) PublishPoolDeleted ¶
func (c *Controller) PublishPoolDeleted(id string)
PublishPoolDeleted announces that a pool is gone.
func (*Controller) Reconcile ¶
func (c *Controller) Reconcile(ctx context.Context) error
Reconcile runs exactly one scheduling pass: snapshot the fleet, decide, and apply. It is exported so that tests and the API can force a pass and know it has finished, which Nudge deliberately cannot promise.
func (*Controller) Registry ¶
func (c *Controller) Registry() *prometheus.Registry
Registry returns the Prometheus registry the API serves at the configured metrics path.
func (*Controller) RemoveRunner ¶
func (c *Controller) RemoveRunner(ctx context.Context, runnerID, reason string, force, confirmed bool) (*store.Runner, error)
RemoveRunner tears a runner down now and deletes its GitHub registration. Without force it drains instead, which still ends a running job after the stop timeout -- so the unforced path asks for the same acknowledgement a direct drain does.
func (*Controller) ReportResult ¶
func (c *Controller) ReportResult(ctx context.Context, hostID string, res agent.TaskResult) error
ReportResult applies the outcome of one task and clears its lease.
func (*Controller) ReportRunners ¶
func (c *Controller) ReportRunners(ctx context.Context, hostID string, reports []agent.RunnerReport) error
ReportRunners merges an agent's observations outside the heartbeat cycle, so a runner coming up is visible immediately rather than up to a beat later.
func (*Controller) RunnerImage ¶
func (c *Controller) RunnerImage(p *store.Pool) string
RunnerImage returns the image a pool's runners use.
Three answers in order. A pool that names an image gets it, unchanged: an operator who has built their own is not second-guessed. Otherwise the pool's platform picks the variant, so a pool called zoomies-4vcpu-debian-12 boots the Debian 12 image without anyone keeping the two in step by hand. A pool that names neither falls back to the instance default.
Whichever it lands on, a pool that gives its jobs a daemon then runs that image's Docker variant. The API already writes the swap into the pool when it is saved; it is made again here because this is the one place that sees the other two answers, and because what runs should be decided where the runner is made rather than trusted to every path that ever wrote a pool row.
func (*Controller) RunnerRenderer ¶
func (c *Controller) RunnerRenderer(ctx context.Context, runners []*store.Runner) (*RunnerRenderer, error)
RunnerRenderer builds the index a page of runners is rendered from.
func (*Controller) Samples ¶
func (c *Controller) Samples(ctx context.Context, since time.Time) ([]store.FleetSample, error)
Samples returns the Overview's sparkline points since a cutoff.
func (*Controller) SeedDemo ¶
func (c *Controller) SeedDemo(ctx context.Context) error
SeedDemo writes a deterministic fixture fleet: one installation, two pools, a dozen runners spread across the state machine, fifty jobs with plausible queue waits and outcomes, three hosts, some scaling history and an audit trail. It is what ZOOMIES_SEED_DEMO turns on for the Playwright suite and for a demo instance.
It is idempotent -- a second call does nothing -- and it refuses to run at all if this instance has any pool that is not one of its own, because a fixture fleet appearing in a real deployment would be indistinguishable from a compromise.
The installation it creates carries a dummy private key. Nothing ever calls GitHub with it: the fixtures are written straight to the database.
func (*Controller) SeedStuck ¶
func (c *Controller) SeedStuck(ctx context.Context) error
SeedStuck ages the demo's starting runners into the two stuck shapes and adds the two fixtures the demo has no room for: a pool nothing can place, and a job GitHub is holding.
It runs after SeedDemo and depends on it, because a fleet in trouble is only legible against one that is working: the point of the problems drawer is that it names the three things wrong among the twenty that are right.
func (*Controller) Start ¶
func (c *Controller) Start(ctx context.Context) error
Start launches the background loops and returns as soon as they are running. It does not block until they finish; Stop does that.
func (*Controller) StartEmbeddedAgent ¶
StartEmbeddedAgent runs an agent inside the controller process, so a single VM needs exactly one process and one systemd unit.
It joins itself with a short-lived token it mints on the spot: the embedded agent is not a remote party, and requiring an operator to paste a token into their own controller would be ceremony without a threat model behind it.
func (*Controller) Stop ¶
func (c *Controller) Stop(ctx context.Context) error
Stop shuts the loops down gracefully and flushes what is worth keeping.
It deliberately does not tear down runners. Restarting a controller must not kill jobs: the runners keep working, their agents keep reporting, and the next reconcile picks up where this one left off.
func (*Controller) Store ¶
func (c *Controller) Store() *store.Store
Store returns the database handle the API reads through.
func (*Controller) Unfence ¶
func (c *Controller) Unfence(ctx context.Context) error
Unfence lifts the fence and lets the fleet act again.
It writes the database first and the in-memory copy second, so a failed write leaves a fenced controller rather than one that believes it is free.
func (*Controller) UpdateConfig ¶
func (c *Controller) UpdateConfig(fn func(*config.Config)) *config.Config
UpdateConfig changes the running configuration.
fn is applied to a copy of the current snapshot, which then replaces it, so a loop in the middle of a pass finishes on the values it started with and the next pass sees the new ones. Anything built once from the configuration -- the log level's gate, the scheduler's and poller's timers -- is retuned here, which is what lets PATCH /settings promise that an accepted change is in effect and not merely recorded.
type Fence ¶
type Fence = store.RecoveryFence
Fence is the recovery fence as the API renders it. It is an alias so the transport layer names a controller type rather than reaching into the store.
type HostExclusion ¶
type HostExclusion struct {
Host string `json:"host"`
// Code is which of the placement rules turned this host down, for a caller
// that wants to group or colour them: unavailable, backend, platform, size.
Code string `json:"code"`
Reason string `json:"reason"`
}
HostExclusion is one host a pool's selector reaches that could not run it, and why. The reason is a sentence about the host with the host's name left out, so a caller can put the name where its own layout wants it.
type HostFit ¶
type HostFit struct {
Count int
Detail string
// Alternatives are the backends offered by the hosts that match this pool
// in every way except its backend, in the order a pool would move to them.
// The wizard turns them into the second half of its warning, in the same
// words the scheduler uses once the pool is real.
Alternatives []string
// PlatformMismatch counts the hosts that fit in every other way but are
// not the machine the pool asked for. It is called out separately because
// it is the one mismatch no backend change can fix: the answer is a
// different host, or a different platform on the pool.
PlatformMismatch int
// Selected is how many hosts the pool's host selector reaches at all,
// whatever became of them afterwards. It is the number the wizard's
// placement step shows while a selector is being typed, and Count is the
// number its review step shows -- so when the two differ, something
// happened between them that an operator has to be told about.
Selected int
// Excluded is that difference, host by host: everything the selector
// reaches and the fleet cannot run, with the reason in the operator's
// terms. It is what turns "2 hosts match" followed by "1 host can run
// this pool" from a contradiction into an explanation.
Excluded []HostExclusion
}
HostFit is what the pool wizard asks before a pool exists: how many hosts could run it as configured, and, when none can, the first host's own account of why and the backends those hosts do offer.
Zero is worth saying out loud before a pool is created: a pool whose selector matches nothing looks completely healthy and never starts a runner. The explanation matters as much as the count, because the usual cause is not a missing machine but a daemon the agent on an existing one could not reach, and the backends those same hosts do offer are the other way out.
type HostStats ¶
type HostStats struct {
Total int `json:"total"`
Healthy int `json:"healthy"`
Cordoned int `json:"cordoned"`
Capacity int `json:"capacity"`
Used int `json:"used"`
}
HostStats summarises the agents and the room they have left.
type HostView ¶
type HostView struct {
Usage *store.HostUsage `json:"usage,omitempty"`
UsageFresh bool `json:"usage_fresh"`
AdmissionReason string `json:"admission_reason,omitempty"`
Connection string `json:"connection"`
ID string `json:"id"`
Name string `json:"name"`
Address string `json:"address,omitempty"`
Embedded bool `json:"embedded"`
Capacity int `json:"capacity"`
ActiveRunners int `json:"active_runners"`
Free int `json:"free"`
Backends []string `json:"backends"`
BackendInfo []BackendInfoView `json:"backend_info"`
Labels map[string]string `json:"labels"`
OS string `json:"os,omitempty"`
Distro string `json:"distro,omitempty"`
OSVersion string `json:"os_version,omitempty"`
Arch string `json:"arch,omitempty"`
// CPUs and MemoryMB are how much machine this host is, as its agent
// reported it: the daemon's view of the machine where that is larger than
// the agent's own share, because a container runner runs beside the agent
// rather than inside its cgroup.
CPUs int `json:"cpus,omitempty"`
MemoryMB int64 `json:"memory_mb,omitempty"`
// DiskTotalMB and DiskFreeMB are the filesystem holding the agent's work
// directory, which is where a runner's checkout and its caches land -- so
// it is the disk that decides whether a job has anywhere to go, and the
// answer to "the host has slots free, why did nothing start?".
//
// Free is what a runner may write to rather than what is unused; the two
// differ by the reserve the filesystem keeps for root, and a runner is not
// root. Zero on both is "not measured" -- an agent too old to report it,
// or a platform with no portable way to ask -- and is not a full disk.
DiskTotalMB int64 `json:"disk_total_mb,omitempty"`
DiskFreeMB int64 `json:"disk_free_mb,omitempty"`
// ReserveCPUs, ReserveMemoryMB and ReserveDiskMB are what the operator has
// held back from placement, for the machine's own sake rather than for any
// pool's. They are the operator's alone: a heartbeat never writes them,
// exactly as it never writes capacity.
ReserveCPUs int `json:"reserve_cpus,omitempty"`
ReserveMemoryMB int64 `json:"reserve_memory_mb,omitempty"`
ReserveDiskMB int64 `json:"reserve_disk_mb,omitempty"`
// AllocatableCPUs, AllocatableMemoryMB and AllocatableDiskMB are the
// machine less its reserve: what the scheduler may actually place onto,
// which is the figure a "how full is this host" question is asked against.
// The floors under the reserve -- 512 MB of memory and 2 GB of disk -- are
// applied here too, so what is shown is what is used.
AllocatableCPUs float64 `json:"allocatable_cpus,omitempty"`
AllocatableMemoryMB int64 `json:"allocatable_memory_mb,omitempty"`
AllocatableDiskMB int64 `json:"allocatable_disk_mb,omitempty"`
// ReservedCPUs and ReservedMemoryMB are what the runners already on this
// host have promised away, as of the last scheduling pass and from the
// snapshot that pass decided on. Disk is deliberately absent: free disk is
// a measurement of the filesystem as it is now, so what those runners have
// written is in DiskFreeMB already, and adding their reservations would
// charge the same bytes twice.
//
// ReservedKnown is false until a pass has run, because zero would read as
// an idle machine rather than as an unanswered question.
ReservedCPUs float64 `json:"reserved_cpus,omitempty"`
ReservedMemoryMB int64 `json:"reserved_memory_mb,omitempty"`
ReservedKnown bool `json:"reserved_known"`
// ResourcesKnown is whether this host has reported what machine it is at
// all. An agent too old to say is placed by slots alone, which is what
// keeps an upgrade from emptying a fleet -- and the page has to say so,
// because a host showing no figures looks broken rather than old.
ResourcesKnown bool `json:"resources_known"`
// Platform is what this host is in the terms a pool asks in, and
// PlatformLabel is the same thing as a sentence: "Ubuntu 24.04, arm64".
Platform store.Platform `json:"platform"`
PlatformLabel string `json:"platform_label,omitempty"`
// CanonicalName is the name this machine would be given today. It is shown
// beside a host called something that says nothing, so an operator can see
// what renaming it would buy them.
CanonicalName string `json:"canonical_name,omitempty"`
Version string `json:"version,omitempty"`
VersionChannel string `json:"version_channel,omitempty"`
Cordoned bool `json:"cordoned"`
// ProtocolVersion is the agent protocol this host reported, and
// Incompatible whether this controller can work with it. An incompatible
// host is excluded from placement exactly as a cordoned one is, so the
// Hosts page has to say which of the two it is: nothing is broken, the
// host is up and heartbeating, and it has quietly stopped taking work.
ProtocolVersion int `json:"protocol_version,omitempty"`
Incompatible bool `json:"incompatible"`
IncompatibleReason string `json:"incompatible_reason,omitempty"`
// VersionSkew is how this host's release stands to the controller's:
// "behind", "ahead", "differs", or absent when they match. It is computed
// here rather than in the browser because the controller is the side that
// knows its own version, and because the agent's own warning comes from
// the same comparison -- the two used to disagree for two builds of one
// tag, and a badge that contradicts a log line is worse than neither.
VersionSkew string `json:"version_skew,omitempty"`
UpgradeCommand string `json:"upgrade_command,omitempty"`
UpgradeVersion string `json:"upgrade_version,omitempty"`
UpgradeNote string `json:"upgrade_note,omitempty"`
Healthy bool `json:"healthy"`
LastHeartbeat time.Time `json:"last_heartbeat"`
CreatedAt time.Time `json:"created_at"`
}
HostView is one agent host and the room it has left.
type InstallationView ¶
type InstallationView struct {
ID string `json:"id"`
AppID int64 `json:"app_id"`
InstallationID int64 `json:"installation_id"`
Target string `json:"target"`
TargetType store.TargetType `json:"target_type"`
APIBaseURL string `json:"api_base_url"`
AppSlug string `json:"app_slug,omitempty"`
WebURL string `json:"web_url,omitempty"`
// SettingsURL is the App's own settings page on GitHub. It is carried on
// every installation, not only on the one the connect flow just created,
// because the one thing a manifest cannot do is set the App's avatar --
// GitHub takes it as an upload -- and an operator who missed that step
// during setup has nowhere else to be told about it. Empty when the slug
// is unknown, which is what a hand-added installation looks like.
SettingsURL string `json:"settings_url,omitempty"`
Enterprise bool `json:"enterprise"`
Healthy bool `json:"healthy"`
LastError string `json:"last_error,omitempty"`
LastCheckedAt *time.Time `json:"last_checked_at"`
PoolCount int `json:"pool_count"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
InstallationView is a GitHub App installation as the API returns it.
The private key and the webhook secret are absent, and there is no field they could be put in: the store keeps them sealed and tagged `json:"-"`, and this type names every field explicitly so that adding one to the domain model cannot leak it here by accident.
func NewInstallationView ¶
func NewInstallationView(i *store.Installation, pools int) InstallationView
NewInstallationView renders an installation, given how many pools depend on it.
type JobExplanation ¶
type JobExplanation struct {
JobID string `json:"job_id"`
State store.JobState `json:"state"`
// Summary is the sentence. It is always set, including for a job that is
// running or finished: "nothing is wrong" is an answer to the question, and
// a caller that has to special-case an empty string will print nothing on
// the one page an operator opened to be told something.
Summary string `json:"summary"`
// Detail is what the summary leaves out, in the scheduler's own words
// where it has any.
Detail string `json:"detail,omitempty"`
// Fix is what to do. Empty means there is nothing to do, which is the
// normal case and is different from not knowing.
Fix string `json:"fix,omitempty"`
// Waiting is whether the job is still waiting on something. Blocked is
// whether waiting will not on its own end it: a fleet that is merely busy
// clears, and a pool nothing can place never will.
Waiting bool `json:"waiting"`
Blocked bool `json:"blocked"`
// PoolID, RunnerID and HostID are what the explanation is about, so a
// caller can link to them rather than parse them back out of the prose.
PoolID string `json:"pool_id,omitempty"`
RunnerID string `json:"runner_id,omitempty"`
HostID string `json:"host_id,omitempty"`
ComputedAt time.Time `json:"computed_at"`
}
JobExplanation is the answer to "why is this job not running?", computed once, on the side that knows.
The browser used to work this out for itself from the pool's live counts, and the CLI could only say "unmatched": two answers to one question, neither of which could see the scheduler's own reason for refusing to place a runner. This is that reason plus the four facts around it -- the last plan, the pool's warnings, the runner's stamps and its host's heartbeat -- so the drawer, the CLI and the support bundle read from one place and cannot disagree.
type JobRenderer ¶
type JobRenderer struct {
// contains filtered or unexported fields
}
JobRenderer names pools without a query per job.
type JobView ¶
type JobView struct {
Provisioning string `json:"provisioning"`
ProvisionNow bool `json:"provision_now"`
ID string `json:"id"`
GitHubJobID int64 `json:"github_job_id"`
GitHubRunID int64 `json:"github_run_id"`
Repo string `json:"repo"`
Workflow string `json:"workflow"`
JobName string `json:"job_name"`
Labels []string `json:"labels"`
State store.JobState `json:"state"`
Conclusion string `json:"conclusion,omitempty"`
// InstallationID is the GitHub App installation covering this job's
// repository. It is read-only and derived at ingest; a pool only ever runs
// work in its own installation's target, so this is half of why a job is
// or is not claimed.
InstallationID string `json:"installation_id,omitempty"`
PoolID string `json:"pool_id,omitempty"`
PoolName string `json:"pool_name,omitempty"`
RunnerID string `json:"runner_id,omitempty"`
RunnerName string `json:"runner_name,omitempty"`
HTMLURL string `json:"html_url,omitempty"`
Matched bool `json:"matched"`
// EligibleAt is when this fleet could first have acted on the job: the
// moment an enabled pool claimed its labels. It is the start of the
// scheduling-latency interval, and it is not queued_at -- the wait
// before anything could run the job is not this fleet's to answer for.
// Absent on a job nothing has claimed, and on one recorded before the
// column existed.
EligibleAt *time.Time `json:"eligible_at,omitempty"`
// Hosted is true when every label names a runner somebody else operates:
// GitHub's own or a hosted-runner vendor's. Such a job is theirs to run,
// so it being unmatched here is expected rather than a job going nowhere.
Hosted bool `json:"hosted"`
HeadBranch string `json:"head_branch,omitempty"`
HeadSHA string `json:"head_sha,omitempty"`
RunAttempt int `json:"run_attempt,omitempty"`
Steps []store.JobStep `json:"steps"`
// FailedStep is the step a failed job stopped at, worked out here from the
// steps so that the grid and the drawer name the same one. Null when the
// job did not fail on a step it ran.
FailedStep *store.JobStep `json:"failed_step"`
// RunnerFault is set when the runner executing this job stopped before
// GitHub reported the job over: the fleet's own explanation of a failure
// GitHub records like any other.
RunnerFault string `json:"runner_fault,omitempty"`
QueuedAt time.Time `json:"queued_at"`
StartedAt *time.Time `json:"started_at"`
CompletedAt *time.Time `json:"completed_at"`
QueueWaitMS int64 `json:"queue_wait_ms"`
DurationMS int64 `json:"duration_ms"`
}
JobView is one job with its pool named and its waits measured.
type MigrationApplyRequest ¶
type MigrationApplyRequest struct {
InstallationID string `json:"installation_id"`
Repos []string `json:"repos"`
Mapping map[string]string `json:"mapping"`
// Overrides are the exceptions to Mapping. They sit alongside Workflows
// rather than inside it: Workflows says which files move at all, and an
// override says where one job in one of them lands.
Overrides []migrate.Override `json:"overrides"`
// Workflows narrows a repository to the workflow files named here, keyed by
// repository. A repository absent from the map gets every file the mapping
// would change, which is what a client that does not know about the field
// asks for. It exists because a repository with a dozen workflow files is
// rarely one where all dozen should move at once.
Workflows map[string][]string `json:"workflows"`
// Title, Body and CommitMessage override the defaults. They are here
// because the pull request lands in somebody else's repository, and an
// organisation with a pull-request template or a commit convention should
// not have to accept ours.
Title string `json:"title"`
Body string `json:"body"`
CommitMessage string `json:"commit_message"`
// Badge says whether each pull request also adds the "CI has the
// Zoomies" badge to the repository's README. It is a pointer because
// the default is yes: a client that does not know the field asks for the
// badge, and only an explicit false leaves the README alone. The badge is
// how the people who read a README find out what runs its CI, and a
// checkbox is a fair price for a line in somebody else's front page.
Badge *bool `json:"badge"`
}
MigrationApplyRequest opens the pull requests a plan described.
func (MigrationApplyRequest) WantsBadge ¶
func (r MigrationApplyRequest) WantsBadge() bool
WantsBadge reports whether the request asks for the badge, which it does unless it said not to.
type MigrationOutcome ¶
type MigrationOutcome struct {
Results []MigrationResult `json:"results"`
Opened int `json:"opened"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
// Branch is the branch every pull request was opened from, for the audit
// row; each result carries its own for the wire.
Branch string `json:"-"`
}
MigrationOutcome is what applying a migration did.
type MigrationPlan ¶
type MigrationPlan struct {
InstallationID string `json:"installation_id"`
Target string `json:"target"`
// Repositories is every repository looked at, whether or not it changed.
// A repository with nothing to do is information: it means its workflows
// are already somewhere deliberate.
Repositories []migrate.RepoPlan `json:"repositories"`
// HostedLabels is every hosted-runner label found across them -- GitHub's
// own and the vendors' -- which is what the mapping step lists.
HostedLabels []string `json:"hosted_labels"`
// Mapping is what was applied -- the request's, or the proposal the server
// made from the pools that exist.
Mapping map[string]string `json:"mapping"`
// Overrides is what was applied on top of it, echoed back so that a plan
// describes itself: every `to` under Repositories came either from Mapping
// or from one of these.
Overrides []migrate.Override `json:"overrides"`
// Unmapped are the hosted labels no pool was proposed for. They are the
// operator's decision, and the reason a plan can be empty.
Unmapped []string `json:"unmapped"`
// Pools is what the mapping step chooses between.
Pools []MigrationPoolOption `json:"pools"`
Counts migrate.Counts `json:"counts"`
// Truncated says there are more repositories after this page. It is kept
// alongside NextCursor because it is what a client reads to know the
// listing is incomplete, whether or not it can page.
Truncated bool `json:"truncated"`
// NextCursor is what to send as Cursor to read the next page. Empty means
// this was the last one.
NextCursor string `json:"next_cursor,omitempty"`
// TotalRepos is how many repositories the installation can see, so a wizard
// showing a page can say what fraction of the whole it is.
TotalRepos int `json:"total_repos"`
// MissingPermissions is what the App still needs before the apply step can
// work. It is reported here, in the step before, because discovering it
// halfway through opening pull requests leaves half of them open.
MissingPermissions []string `json:"missing_permissions"`
// PermissionHint is the sentence that fixes MissingPermissions.
PermissionHint string `json:"permission_hint,omitempty"`
// SettingsURL is where to go and fix it.
SettingsURL string `json:"settings_url,omitempty"`
}
MigrationPlan is the whole review step.
type MigrationPlanRequest ¶
type MigrationPlanRequest struct {
InstallationID string `json:"installation_id"`
// Repos limits the plan to these repositories. Empty means every
// repository the installation can see, up to the limit.
Repos []string `json:"repos"`
// Mapping is hosted label -> the runs-on value that replaces it. Empty asks
// the server to propose one from the pools that exist.
Mapping map[string]string `json:"mapping"`
// Overrides are the exceptions to Mapping, each naming one job in one file
// in one repository. Empty is the common case: most fleets want one answer
// per hosted-runner label everywhere.
Overrides []migrate.Override `json:"overrides"`
// Cursor asks for the page of repositories after this one. It is the full
// name of the last repository the previous page returned, because the
// listing is sorted by name: a name is stable when the App gains or loses
// a repository between two pages, and an offset is not.
Cursor string `json:"cursor"`
}
MigrationPlanRequest asks what a migration would change.
type MigrationPoolOption ¶
type MigrationPoolOption struct {
ID string `json:"id"`
Name string `json:"name"`
Labels []string `json:"labels"`
// RunsOn is what a workflow writes to reach this pool: the value the
// mapping should hold.
RunsOn string `json:"runs_on"`
Enabled bool `json:"enabled"`
}
MigrationPoolOption is one pool the wizard can map a hosted label to.
type MigrationResult ¶
type MigrationResult struct {
Repo string `json:"repo"`
// Status is "opened", "skipped" or "failed".
Status string `json:"status"`
// PullRequestURL and PullRequestNumber are set when Status is "opened".
PullRequestURL string `json:"pull_request_url,omitempty"`
PullRequestNumber int `json:"pull_request_number,omitempty"`
Branch string `json:"branch,omitempty"`
// Workflows is how many files the pull request changed.
Workflows int `json:"workflows"`
// Jobs is how many runs-on lines it rewrote.
Jobs int `json:"jobs"`
// Reason explains a skip or a failure.
Reason string `json:"reason,omitempty"`
// Badge says what became of the README badge on a pull request that was
// opened: "added", "present" when the README already carried it,
// "no_readme" when there was nowhere Markdown to put it, "unread" when
// GitHub would not hand the README over, and "off" when the request
// asked for none. BadgeReason says which of the no_readme and unread
// cases it was. A badge that could not be added never stops the
// migration: the runs-on lines are the work, and the badge is the thanks.
Badge string `json:"badge,omitempty"`
BadgeReason string `json:"badge_reason,omitempty"`
}
MigrationResult is what happened to one repository.
type Options ¶
type Options struct {
// Store is the only writer of persistent state.
Store *store.Store
// Config is the validated configuration; the controller reads the
// scheduler tunables, the GitHub settings and the retention windows.
Config *config.Config
// Key unseals the GitHub App private keys and webhook secrets held in the
// database. Without it no installation can be used.
Key *cryptox.Key
// Auth mints agent tokens, redeems join tokens and records audit rows.
Auth *auth.Service
// Events carries every state change to the UI's SSE stream.
Events *events.Bus
// GitHub builds a client per installation; nil uses the real GitHub App
// factory, and tests pass one backed by github.NewFake.
GitHub github.Factory
// Backends is only needed when this process also runs the embedded agent.
Backends *backend.Registry
Logger *slog.Logger
// Clock is injectable so tests can freeze time.
Clock func() time.Time
// HTTPClient delivers capacity-demand events. Tests may inject a transport.
HTTPClient *http.Client
// LogLevel is the gate the process logger is filtered at, when the caller
// built one that can move. UpdateConfig sets it from log.level, so that a
// level changed through PATCH /settings or SIGHUP is the level the process
// actually logs at, not merely the one the settings page shows.
LogLevel *slog.LevelVar
// Lease is this controller's claim on the database, taken before the
// controller was built. A nil lease means nothing renews and nothing is
// reported, which is what an embedded or test controller wants.
Lease *store.ControllerLease
}
Options are the collaborators the controller needs. Everything except the store and the configuration has a defensible default, because a caller that forgets the event bus should get a working controller rather than a nil dereference three loops later.
type PoolCountsView ¶
type PoolCountsView struct {
Provisioning int `json:"provisioning"`
Registering int `json:"registering"`
Idle int `json:"idle"`
Busy int `json:"busy"`
Draining int `json:"draining"`
Failed int `json:"failed"`
Live int `json:"live"`
}
PoolCountsView is a pool's live runner tally, in the shape the OpenAPI document's Pool.counts has.
type PoolRenderer ¶
type PoolRenderer struct {
// contains filtered or unexported fields
}
PoolRenderer is everything needed to render pools without one query per pool.
type PoolStats ¶
type PoolStats struct {
PoolID string `json:"pool_id"`
PoolName string `json:"pool_name"`
Min int `json:"min"`
Max int `json:"max"`
Live int `json:"live"`
Busy int `json:"busy"`
Idle int `json:"idle"`
// Queued is the number of queued jobs this pool has claimed.
Queued int `json:"queued"`
Utilisation float64 `json:"utilisation"`
}
PoolStats is one row of the Overview's per-pool utilisation.
type PoolView ¶
type PoolView struct {
ID string `json:"id"`
Name string `json:"name"`
InstallationID string `json:"installation_id"`
InstallationTarget string `json:"installation_target,omitempty"`
Labels []string `json:"labels"`
RunnerGroup string `json:"runner_group,omitempty"`
Backend store.BackendKind `json:"backend"`
// Platform is the machine this pool's runners need. It picks the runner
// image and restricts which hosts the scheduler may place them on.
Platform store.Platform `json:"platform"`
Image string `json:"image"`
// EffectiveImage is the image runners will actually boot: Image when the
// pool names one, otherwise the variant its platform selects, otherwise
// the instance default. A pool page that showed a blank image field and
// nothing else would leave an operator guessing at the single most
// important thing about their runners.
EffectiveImage string `json:"effective_image"`
PullPolicy store.PullPolicy `json:"pull_policy"`
RunnerVersion string `json:"runner_version,omitempty"`
MinRunners int `json:"min_runners"`
MaxRunners int `json:"max_runners"`
Priority int `json:"priority"`
// RepositoryScaleUpLimit and CostPerRunnerHour are accepted on the way in,
// so they are rendered on the way out: a field the API takes but never
// shows again is a field an operator cannot check, edit or explain.
RepositoryScaleUpLimit int `json:"repository_scale_up_limit"`
CostPerRunnerHour *float64 `json:"cost_per_runner_hour"`
IdleTimeout store.Duration `json:"idle_timeout"`
Ephemeral bool `json:"ephemeral"`
DockerMode store.DockerMode `json:"docker_mode"`
Resources store.Resources `json:"resources"`
Cache store.CacheConfig `json:"cache"`
HostSelector map[string]string `json:"host_selector"`
Env map[string]string `json:"env"`
RunAsRoot bool `json:"run_as_root"`
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Counts PoolCountsView `json:"counts"`
QueuedJobs int `json:"queued_jobs"`
Utilisation float64 `json:"utilisation"`
Warnings []Problem `json:"warnings,omitempty"`
}
PoolView is a pool plus what an operator needs to see next to it: which installation it belongs to, how many runners it has in each state, how much of itself it is using, and every dangerous setting it has in effect.
func (PoolView) WithoutEnvValues ¶
WithoutEnvValues returns the view with every environment value blanked and every key left in place.
Env is injected into every runner a pool creates, which makes it the one field on this view an operator may reasonably have put a registry password or a proxy credential in -- and reading a pool is a viewer action while setting one is an operator action, so the two were not the same audience. Viewer is documented as reading everything except secrets; this is what makes that true of pools.
The keys stay because they are what the pool page renders -- it lists names and never values -- so a viewer still sees which variables a pool sets, and only the operator who could have written them can read them back.
type Problem ¶
type Problem struct {
// Code is a stable identifier such as "host.unhealthy", suitable for
// grouping or suppressing in an alerting rule.
Code string `json:"code"`
Severity config.Severity `json:"severity"`
// Setting names the configuration key involved, when there is one.
Setting string `json:"setting,omitempty"`
Title string `json:"title"`
Detail string `json:"detail,omitempty"`
Fix string `json:"fix,omitempty"`
// TargetKind and TargetID let the UI link a problem to the pool, host,
// runner or installation it is about.
TargetKind string `json:"target_kind,omitempty"`
TargetID string `json:"target_id,omitempty"`
// Alternatives are the choices the fix leaves open, when the fix is a
// choice: for a pool no host can run, the backends its hosts do offer. They
// are carried apart from the prose so the UI can put the change one click
// away rather than leaving an operator to find the pool's edit form.
Alternatives []string `json:"alternatives,omitempty"`
// Since is when the situation started, where that is knowable.
Since *time.Time `json:"since,omitempty"`
}
Problem is one thing an operator should know about, in the shape the API's /problems endpoint returns.
It carries the same four sentences a config.Finding does -- what is true, why it matters, and what to change -- because an operator reading the problems drawer should never have to go and look up what a code means.
func PoolWarnings ¶
func PoolWarnings(p *store.Pool, inst *store.Installation) []Problem
PoolWarnings renders a pool's dangerous settings as problems.
They are the same sentences the UI's problems drawer shows, because an operator should not have to learn that "host-socket" on the pool page and "any job on this pool can become root on the host" on the Overview are the same fact.
The installation is there for the one risk a pool cannot see in itself, the repository cache below. Nil when it is unknown, which validation reports on its own.
type ProblemsView ¶
ProblemsView carries the drawer's own "nothing is wrong" flag rather than leaving the UI to infer it from an empty array, so that "we checked and all is well" and "we have not looked yet" cannot be rendered the same way.
func NewProblemsView ¶
func NewProblemsView(items []Problem) ProblemsView
NewProblemsView wraps a problem list. A nil list renders as an empty array, never as null, because the UI reads "nothing needs your attention" from exactly that.
type RunnerRenderer ¶
type RunnerRenderer struct {
// contains filtered or unexported fields
}
RunnerRenderer names pools and hosts without a query per runner.
func (*RunnerRenderer) View ¶
func (v *RunnerRenderer) View(r *store.Runner) RunnerView
View renders one runner.
type RunnerStats ¶
type RunnerStats struct {
Provisioning int `json:"provisioning"`
Registering int `json:"registering"`
Idle int `json:"idle"`
Busy int `json:"busy"`
Draining int `json:"draining"`
Failed int `json:"failed"`
Total int `json:"total"`
}
RunnerStats counts the fleet by runner state.
type RunnerView ¶
type RunnerView struct {
ID string `json:"id"`
Name string `json:"name"`
PoolID string `json:"pool_id"`
PoolName string `json:"pool_name,omitempty"`
HostID string `json:"host_id"`
HostName string `json:"host_name,omitempty"`
State store.RunnerState `json:"state"`
GitHubRunnerID int64 `json:"github_runner_id,omitempty"`
ContainerID string `json:"container_id,omitempty"`
Ephemeral bool `json:"ephemeral"`
Labels []string `json:"labels"`
Image string `json:"image,omitempty"`
ImageDigest string `json:"image_digest,omitempty"`
RunnerVersion string `json:"runner_version,omitempty"`
CurrentJobID string `json:"current_job_id,omitempty"`
CurrentJob *JobView `json:"current_job,omitempty"`
Message string `json:"message,omitempty"`
JobsHandled int `json:"jobs_handled"`
CPUPercent float64 `json:"cpu_percent,omitempty"`
MemoryBytes int64 `json:"memory_bytes,omitempty"`
CreatedAt time.Time `json:"created_at"`
// ContainerStartedAt and RegisteredAt are the two halves of coming up, and
// they are on the view because the gap between them is the whole diagnosis
// of a runner stuck in `registering`: a container that never started is a
// backend or image problem on the host, and a container that started and
// never registered is a credential, network or GitHub one. Without both, a
// stuck runner has one symptom and two unrelated causes.
CreateTaskIssuedAt *time.Time `json:"create_task_issued_at,omitempty"`
HostRemovedAt *time.Time `json:"host_removed_at,omitempty"`
RegistrationDeletedAt *time.Time `json:"registration_deleted_at,omitempty"`
CleanupEstimatedAt *time.Time `json:"cleanup_estimated_at,omitempty"`
ContainerStartedAt *time.Time `json:"container_started_at,omitempty"`
RegisteredAt *time.Time `json:"registered_at,omitempty"`
StartedAt *time.Time `json:"started_at"`
LastIdleAt *time.Time `json:"last_idle_at"`
FinishedAt *time.Time `json:"finished_at"`
// CleanupError and its companions describe a runner Zoomies could not
// finish taking away. An empty error is the normal case; a non-empty one
// means something is still on a host or on GitHub.
CleanupError string `json:"cleanup_error,omitempty"`
CleanupFailedAt *time.Time `json:"cleanup_failed_at,omitempty"`
CleanupAttempts int `json:"cleanup_attempts,omitempty"`
// CleanedUpAt is the end of the runner's life: nothing of it left, on the
// host or on GitHub.
CleanedUpAt *time.Time `json:"cleaned_up_at,omitempty"`
}
RunnerView is one runner, with the pool and host named rather than only referenced: a runner grid that shows two opaque IDs per row is a grid nobody can read.
type ScopedJobStats ¶
type ScopedJobStats struct {
QueuedJobs int `json:"queued_jobs"`
RunningJobs int `json:"running_jobs"`
Completed int `json:"completed"`
Succeeded int `json:"succeeded"`
Failed int `json:"failed"`
Cancelled int `json:"cancelled"`
Unknown int `json:"unknown"`
MedianWaitMS int64 `json:"median_wait_ms"`
P95WaitMS int64 `json:"p95_wait_ms"`
}
ScopedJobStats is the job half of Stats over one scope.
GitHub reports every job in an installed repository, and on an organisation that also uses hosted runners most of them are somebody else's. A queue depth that counts those answers "why is my fleet slow?" with a number nobody here can act on, and a median wait computed from them is somebody else's queue.
type Stats ¶
type Stats struct {
// Window is the period the completed counts and the wait percentiles
// cover. Completed is Succeeded + Failed + Cancelled + Unknown; a rate
// computed from Completed and Failed alone would count a job GitHub
// stopped reporting as a success, which is why the split is here.
Window string `json:"window"`
QueuedJobs int `json:"queued_jobs"`
RunningJobs int `json:"running_jobs"`
Completed int `json:"completed"`
Succeeded int `json:"succeeded"`
Failed int `json:"failed"`
Cancelled int `json:"cancelled"`
Unknown int `json:"unknown"`
MedianWaitMS int64 `json:"median_wait_ms"`
P95WaitMS int64 `json:"p95_wait_ms"`
P50StartupMS int64 `json:"p50_startup_ms"`
P95StartupMS int64 `json:"p95_startup_ms"`
P50RegistrationMS int64 `json:"p50_registration_ms"`
P95RegistrationMS int64 `json:"p95_registration_ms"`
// Fleet is every job figure above, narrowed to the jobs this fleet has a
// hand in. Both are carried in one payload rather than chosen by a query
// parameter because the same numbers arrive over the event stream, which
// is one frame for every viewer: a per-request scope would be correct
// until the next frame overwrote it, a second or two later.
Fleet ScopedJobStats `json:"fleet"`
Runners RunnerStats `json:"runners"`
Hosts HostStats `json:"hosts"`
Pools []PoolStats `json:"pools"`
}
Stats is the Overview payload: what the queue is doing, what the fleet is doing, and how long jobs are waiting.
Source Files
¶
- agents.go
- background.go
- capacity_demand.go
- clients.go
- controller.go
- democlient.go
- derived.go
- explain.go
- host_cleanup.go
- host_upgrade.go
- hostfit.go
- jobs.go
- lease.go
- logs.go
- metrics.go
- migrations.go
- poller.go
- problems.go
- provisioning.go
- reconcile.go
- runner_readiness.go
- seed.go
- seed_stuck.go
- stats.go
- timings.go
- updates.go
- views.go
- webhooks.go