Documentation
¶
Overview ¶
Package backend provides the WorkerBackend interface that abstracts how the check worker loop (internal/checkworker) talks to the master: DirectBackend calls the database and services in-process (the production server path), WSBackend (agent mode) speaks the WebSocket agent protocol to a remote server. The lease/lane loop, budgets, and express path in CheckWorker run identically on top of either.
Index ¶
- Constants
- Variables
- type DirectBackend
- func (b *DirectBackend) ClaimJobs(ctx context.Context, workerUID string, region *string, fastLimit int, ...) ([]*models.CheckJob, time.Duration, error)
- func (b *DirectBackend) ClaimJobsForCheck(ctx context.Context, workerUID string, region *string, checkUID string) ([]*models.CheckJob, error)
- func (b *DirectBackend) DeferRateLimited(ctx context.Context, job *models.CheckJob, workerUID string, ...) error
- func (b *DirectBackend) Heartbeat(ctx context.Context, workerUID string, capabilities []string, version string) error
- func (b *DirectBackend) Hints() <-chan string
- func (b *DirectBackend) LastResults(ctx context.Context, orgUID string, checkUIDs []string) (map[string]*models.Result, error)
- func (b *DirectBackend) Register(ctx context.Context, worker *models.Worker) (*models.Worker, error)
- func (b *DirectBackend) SubmitResult(ctx context.Context, job *models.CheckJob, workerUID string, ...) error
- type Identity
- type Option
- type SchedulingState
- type SubmitResultRequest
- type WSBackend
- func (b *WSBackend) ClaimJobs(ctx context.Context, _ string, _ *string, fastLimit int, _ int, ...) ([]*models.CheckJob, time.Duration, error)
- func (b *WSBackend) ClaimJobsForCheck(ctx context.Context, _ string, _ *string, checkUID string) ([]*models.CheckJob, error)
- func (b *WSBackend) DeferRateLimited(_ context.Context, _ *models.CheckJob, _ string, _ time.Time) error
- func (b *WSBackend) Heartbeat(_ context.Context, _ string, _ []string, _ string) error
- func (b *WSBackend) Hints() <-chan string
- func (b *WSBackend) Identity() Identity
- func (b *WSBackend) LastResults(_ context.Context, _ string, _ []string) (map[string]*models.Result, error)
- func (b *WSBackend) Register(ctx context.Context, _ *models.Worker) (*models.Worker, error)
- func (b *WSBackend) ResolveTunnel(ctx context.Context, _, tunnelCheckUID string) (*sshtunnel.Dialer, io.Closer, error)
- func (b *WSBackend) SubmitResult(ctx context.Context, job *models.CheckJob, _ string, req *SubmitResultRequest) error
- type WorkerBackend
Constants ¶
const CloseProtocolError websocket.StatusCode = 4400
CloseProtocolError mirrors the server's protocol-error close code.
Variables ¶
var ( ErrNotEnrolled = errors.New("agent is not enrolled yet") ErrEnrollNoToken = errors.New("no enrollment token configured (SP_AGENT_ENROLLMENT_TOKEN)") ErrRequestTimeout = errors.New("agent request timed out") ErrServerError = errors.New("server rejected the request") ErrPassiveUnsupported = errors.New("passive checks are not supported on deported agents") // ErrConnLost fails an in-flight request the moment its connection is // retired, instead of waiting out the full requestTimeout. ErrConnLost = errors.New("agent connection lost") // ErrSealedForOthers is surfaced as the job error when a sealed blob cannot // be decrypted by this agent — the fix is a credentials re-save. ErrSealedForOthers = errors.New( "credentials not sealed for this agent — re-save the check's credentials") // ErrTunnelSealedForOthers is surfaced as the job error when a tunnel block's // sealed envelope cannot be decrypted by this agent (spec 2026-07-18-07). The // SSH check is not, in fact, sealed to this agent's region. ErrTunnelSealedForOthers = errors.New( "ssh tunnel credentials not sealed for this agent — " + "allocate the SSH check to this agent's region and re-save its credentials") // ErrTunnelNoCredentials is surfaced when an unsealed tunnel block has no // usable auth material (no username, or no password/private_key). ErrTunnelNoCredentials = errors.New( "ssh tunnel check needs a username and a password or private_key") )
Errors returned by WSBackend.
var ErrUploadRejected = errors.New("attachment upload rejected")
ErrUploadRejected is returned when the server refuses an attachment upload.
Functions ¶
This section is empty.
Types ¶
type DirectBackend ¶
type DirectBackend struct {
// contains filtered or unexported fields
}
DirectBackend implements WorkerBackend by calling the database and services directly. This is the production path when the worker runs in the same process as the API server; its SubmitResult mirrors the exact save-result → process-incidents → release-lease sequence CheckWorker performed inline before the WorkerBackend refactor.
func NewDirectBackend ¶
func NewDirectBackend( dbService db.Service, checkJobSvc checkjobsvc.Service, incidentSvc *incidents.Service, events notifier.EventNotifier, creds credentials.Service, ) *DirectBackend
NewDirectBackend creates a DirectBackend. creds may be nil (tests, or a deployment with no master key); jobs carrying an encrypted envelope then fail loudly rather than running without their secrets.
func (*DirectBackend) ClaimJobs ¶
func (b *DirectBackend) ClaimJobs( ctx context.Context, workerUID string, region *string, fastLimit int, slowLimit int, maxAhead time.Duration, ) ([]*models.CheckJob, time.Duration, error)
ClaimJobs claims up to fastLimit jobs for the given worker with the slow lane bounded by slowLimit (spec 2026-07-01-03 D3). Claimed jobs come back with their secrets already merged — see mergeClaimedSecrets.
func (*DirectBackend) ClaimJobsForCheck ¶
func (b *DirectBackend) ClaimJobsForCheck( ctx context.Context, workerUID string, region *string, checkUID string, ) ([]*models.CheckJob, error)
ClaimJobsForCheck claims any due job rows for one check (express path).
func (*DirectBackend) DeferRateLimited ¶
func (b *DirectBackend) DeferRateLimited( ctx context.Context, job *models.CheckJob, workerUID string, nextScheduledAt time.Time, ) error
DeferRateLimited releases the lease and reschedules without writing a result, preserving effective_scheduled_at so the deferred job sorts ahead of its on-time org siblings next window (spec 2026-08-26-02).
func (*DirectBackend) Heartbeat ¶
func (b *DirectBackend) Heartbeat( ctx context.Context, workerUID string, capabilities []string, version string, ) error
Heartbeat updates the worker's last_active_at timestamp and the reported capability set and build version.
func (*DirectBackend) Hints ¶
func (b *DirectBackend) Hints() <-chan string
Hints subscribes to check.created events (the in-process express hint).
func (*DirectBackend) LastResults ¶
func (b *DirectBackend) LastResults( ctx context.Context, orgUID string, checkUIDs []string, ) (map[string]*models.Result, error)
LastResults returns the latest result per check (passive checks).
func (*DirectBackend) Register ¶
func (b *DirectBackend) Register( ctx context.Context, worker *models.Worker, ) (*models.Worker, error)
Register registers or updates a worker in the database.
func (*DirectBackend) SubmitResult ¶
func (b *DirectBackend) SubmitResult( ctx context.Context, job *models.CheckJob, workerUID string, req *SubmitResultRequest, ) error
SubmitResult saves the result row (with status tracking), processes incidents, and releases the lease — with scheduling state when provided, plain otherwise. Incident processing only runs after a successful save (a result that never landed must not drive the incident state machine); the lease is released regardless so the job never wedges behind a failed write.
type Identity ¶
type Identity struct {
agents.AgentKeys
AgentUID string `json:"agentUid,omitempty"`
Region string `json:"region,omitempty"`
}
Identity is the persisted agent identity: its two keypairs plus the server-assigned UID and bound region learned at enrollment.
type Option ¶
type Option func(*WSBackend)
Option customizes a WSBackend at construction.
func WithCaptureCache ¶
func WithCaptureCache(cache *capturecache.Cache) Option
WithCaptureCache overrides the local capture cache (tests use tighter bounds and a driven clock).
func WithLogger ¶
WithLogger overrides the default logger (used in tests to capture output).
func WithPingInterval ¶
WithPingInterval overrides the keepalive ping cadence (used in tests to force fast half-open detection).
type SchedulingState ¶
type SchedulingState struct {
CostEWMAMs float64 `json:"costEwmaMs"`
DelayEWMAMs float64 `json:"delayEwmaMs"`
EffectiveScheduledAt time.Time `json:"effectiveScheduledAt"`
Lane uint8 `json:"lane"`
}
SchedulingState carries the post-exec cost/lane write folded into the lease release (specs 2026-06-30-09 / 2026-07-01-03). Nil on paths without a fresh cost sample (rate-limit deferral, passive checks, error results), which use a plain release instead.
type SubmitResultRequest ¶
type SubmitResultRequest struct {
Status int `json:"status"`
Duration float32 `json:"duration"` // milliseconds
Metrics map[string]any `json:"metrics,omitempty"`
Output map[string]any `json:"output,omitempty"`
// Diagnostics carries the opt-in failure capture (spec 2026-08-20-01).
// Deliberately separate from Output: Output is persisted on the raw result
// row, this is persisted only if the result opens/reopens an incident.
Diagnostics *checkerdef.Diagnostics `json:"diagnostics,omitempty"`
// Region is the resolved region for the result row (job region, falling
// back to the worker's own region).
Region *string `json:"region,omitempty"`
// NextScheduledAt is the worker-computed next tick (phase-locked).
NextScheduledAt time.Time `json:"nextScheduledAt"`
// ExecStart is when the outbound probe actually began. The in-process path
// folds it into Sched itself; a remote transport ships it so the SERVER can
// compute the same delay sample (spec 2026-07-27-01). Zero on paths with no
// probe (error results, rate-limit deferral).
ExecStart time.Time `json:"execStart,omitempty"`
// Sched, when non-nil, releases the lease with the updated scheduling
// state; nil uses the plain release.
Sched *SchedulingState `json:"sched,omitempty"`
}
SubmitResultRequest is the terminal write for one executed job: the result row plus the lease release/reschedule, submitted as a single backend call so a remote transport can carry it in one frame.
type WSBackend ¶
type WSBackend struct {
// contains filtered or unexported fields
}
WSBackend implements WorkerBackend over the WebSocket agent protocol. It owns the connection lifecycle: enrollment on first run, signed-header reconnects afterwards, correlation-id request/response, jobs-available hint fan-out, and unsealing of region-sealed credentials (in memory only).
func NewWSBackend ¶
func NewWSBackend( serverURL, enrollmentToken, name string, identity *Identity, onIdentityChange func(*Identity), opts ...Option, ) *WSBackend
NewWSBackend creates a WSBackend. identity must carry the agent's keypairs; AgentUID/Region may be empty (enrollment fills them, then onIdentityChange fires so the caller can persist).
func (*WSBackend) ClaimJobs ¶
func (b *WSBackend) ClaimJobs( ctx context.Context, _ string, _ *string, fastLimit int, _ int, _ time.Duration, ) ([]*models.CheckJob, time.Duration, error)
ClaimJobs claims jobs over the WS transport (single limit — the lane reservation is an in-process pool concern; the server orders cost-aware). Sealed credentials are unsealed here with the agent's X25519 identity and merged into the job config, in memory only; a blob this agent cannot decrypt is reported as a clear job error and dropped from the batch.
func (*WSBackend) ClaimJobsForCheck ¶
func (b *WSBackend) ClaimJobsForCheck( ctx context.Context, _ string, _ *string, checkUID string, ) ([]*models.CheckJob, error)
ClaimJobsForCheck is the agent express path: a claim pinned to one check. The next-eligible hint is dropped — the express goroutine doesn't poll.
func (*WSBackend) DeferRateLimited ¶
func (b *WSBackend) DeferRateLimited(_ context.Context, _ *models.CheckJob, _ string, _ time.Time) error
DeferRateLimited is unused agent-side (the entitlements gate is enforced at the server's dispatch, in agentws.handleClaim, which calls checkjobsvc.DeferLeaseRateLimited directly); a lease the agent abandons simply expires.
func (*WSBackend) Heartbeat ¶
Heartbeat is a no-op: the server refreshes last_seen_at on pings and frames, and the agent's capability set and build version ride the claim frame (see claim) for the same reason.
func (*WSBackend) Identity ¶
Identity returns a copy of the current identity (for persistence/logging).
func (*WSBackend) LastResults ¶
func (b *WSBackend) LastResults(_ context.Context, _ string, _ []string) (map[string]*models.Result, error)
LastResults is unsupported: passive checks (heartbeat/email) are driven by inbound signals to the server and never dispatch to private regions' agents.
func (*WSBackend) Register ¶
Register satisfies WorkerBackend. The server registers the worker row during the WS handshake; this just ensures a connection exists and returns the server-assigned identity.
func (*WSBackend) ResolveTunnel ¶
func (b *WSBackend) ResolveTunnel( ctx context.Context, _, tunnelCheckUID string, ) (*sshtunnel.Dialer, io.Closer, error)
ResolveTunnel is the agent-side sshtunnel.Resolver (wired into sshtunnel.ResolverFunc at agent startup). It looks up the unsealed snapshot for the tunnel check and dials a fresh SSH session through it — the exact dial/handshake/host-key/classification machinery the server path uses. A missing snapshot yields decision 7's clearer error, wrapped as a tunnel *Error so the worker still classifies the execution as a tunnel failure.
orgUID is ignored: an agent is single-org and single-region, and the tunnel check UID is unique within it.
func (*WSBackend) SubmitResult ¶
func (b *WSBackend) SubmitResult( ctx context.Context, job *models.CheckJob, _ string, req *SubmitResultRequest, ) error
SubmitResult submits one result frame and waits for the ack. The server performs the save/incident/release sequence; the agent-side scheduling state is advisory only (the server recomputes the next tick itself).
type WorkerBackend ¶
type WorkerBackend interface {
// Register registers or updates the worker identity and returns the
// persisted record.
Register(ctx context.Context, worker *models.Worker) (*models.Worker, error)
// Heartbeat updates the worker's last_active_at timestamp and refreshes the
// self-reported egress families (spec 2026-08-15-11). Reporting on the
// heartbeat rather than at process start is what lets a host that gains or
// loses IPv6 converge within one beat instead of needing a restart. The
// value is advertised as a hint only — it never gates execution.
// Heartbeat refreshes liveness and, when the executor reported one, its
// capability set. A nil set means "not reported" and leaves the stored set
// untouched; an empty non-nil set is a real report of "none". version is
// this worker's self-reported build version (spec 2026-08-19-07); an
// empty string means "not reported" and leaves the stored value untouched
// — a real version is never the empty string, so this sentinel is safe.
Heartbeat(ctx context.Context, workerUID string, capabilities []string, version string) error
// ClaimJobs claims due jobs with per-lane reservation (fastLimit is the
// total capacity, slowLimit the slow-lane budget — see
// checkjobsvc.Service.ClaimJobs). The second return is the next-eligible
// hint: how long until the earliest still-unleased job in this worker's
// scope becomes claimable (0 = none known). The fetcher sleeps on it
// instead of its flat fallback poll, which is what keeps sub-minute
// periods honest on an otherwise idle worker or deported agent.
ClaimJobs(
ctx context.Context,
workerUID string,
region *string,
fastLimit int,
slowLimit int,
maxAhead time.Duration,
) ([]*models.CheckJob, time.Duration, error)
// ClaimJobsForCheck claims any due job rows for one check (the express
// path for freshly created checks).
ClaimJobsForCheck(
ctx context.Context,
workerUID string,
region *string,
checkUID string,
) ([]*models.CheckJob, error)
// SubmitResult persists a finished execution: saves the result row,
// processes incidents (always server-side), and releases the lease with
// the given schedule (and scheduling state when provided).
SubmitResult(
ctx context.Context,
job *models.CheckJob,
workerUID string,
req *SubmitResultRequest,
) error
// DeferRateLimited releases a job's lease and reschedules it without writing
// a result, for the one caller that needs it: the per-org
// MaxChecksPerMinute gate turning a job away before its probe runs.
//
// The name is deliberate. The underlying write preserves the job's
// effective_scheduled_at (the claim ordering key) so a deferred job keeps
// accumulating overdue-ness and wins the next contended slot — the
// anti-starvation rotation of spec 2026-08-26-02. A generic "release the
// lease" here is what previously re-anchored that key and starved the same
// checks forever.
DeferRateLimited(
ctx context.Context,
job *models.CheckJob,
workerUID string,
nextScheduledAt time.Time,
) error
// LastResults returns the latest result per check, used by passive checks
// (heartbeat/email) to inspect inbound signal recency. Remote backends may
// not support it — passive checks are a server-side concern.
LastResults(ctx context.Context, orgUID string, checkUIDs []string) (map[string]*models.Result, error)
// Hints returns a fresh channel signaled when new jobs may be available
// (check.created events in-process; jobs-available frames over WS). Each
// call returns an independent subscription.
Hints() <-chan string
}
WorkerBackend abstracts how a check worker communicates with the master. CheckWorker consumes exactly this interface, so the same loop runs in-process (DirectBackend) and inside a deported agent (WSBackend).