Help us shape the future of Go! Take the 2026 Go Developer Survey

cluster

package
v0.0.0-...-d7687b9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 1, 2026 License: Apache-2.0 Imports: 10 Imported by: 0

Documentation

Overview

Package cluster keeps the registry of manent executors (workers): heartbeats, shutdown, claiming the workflows of a lost node and leases for the tasks only one worker at a time must perform (automatic recovery, retention, reconciliation). It also stores the state read from the targets by the reconciliation.

The registry lives in the DBOS database, in its own tables: without DBOS Conductor it is the only way to know which executors are still alive.

Index

Constants

View Source
const ClaimTimeout = 2 * time.Minute

ClaimTimeout is the time after which a recovery not marked as done (e.g. because whoever ran it crashed) no longer blocks the executor's registration.

Variables

View Source
var ErrPolicyConflict = errors.New("another policy revision was activated meanwhile")

ErrPolicyConflict means another revision was activated meanwhile.

View Source
var ErrReplaced = errors.New("another process registered with the same executor_id")

ErrReplaced means another startup registered with the same executor_id.

Functions

This section is empty.

Types

type Executor

type Executor struct {
	ID string `json:"id"`
	// Version is the application version of the workflows (app.version): a worker only resumes
	// workflows of its own version.
	Version string `json:"version"`
	// Build is the manent binary version.
	Build    string `json:"build,omitempty"`
	Hostname string `json:"hostname,omitempty"`
	// Region is the worker's region (dbos.region), if configured.
	Region      string     `json:"region,omitempty"`
	StartedAt   time.Time  `json:"started_at"`
	HeartbeatAt time.Time  `json:"heartbeat_at"`
	StoppedAt   *time.Time `json:"stopped_at,omitempty"`
	// RecoveredBy is who claimed the executor's workflows after it was lost.
	RecoveredBy string     `json:"recovered_by,omitempty"`
	RecoveredAt *time.Time `json:"recovered_at,omitempty"`
	// Queues are the DBOS queues the executor dequeues from.
	Queues []string `json:"queues,omitempty"`
	// Resources is the last resource sample sent by the executor, in JSON
	// (service.WorkerResources): the registry stores it without interpreting it.
	Resources json.RawMessage `json:"resources,omitempty"`
}

Executor is a registered worker.

func (Executor) Stale

func (e Executor) Stale(now time.Time, after time.Duration) bool

Stale reports an executor that has sent no heartbeat for at least after, or that stopped.

type FreezeRecord

type FreezeRecord struct {
	Name      string
	Data      string
	CreatedBy string
	CreatedAt time.Time
}

FreezeRecord is a freeze window created at runtime; Data is the window in JSON.

type InUseError

type InUseError struct {
	ID, Hostname           string
	StartedAt, HeartbeatAt time.Time
}

InUseError means an executor_id already registered by another live process.

func (*InUseError) Error

func (e *InUseError) Error() string

type PolicyRecord

type PolicyRecord struct {
	Revision    int64
	SHA256      string
	Status      string
	Author      string
	Comment     string
	ReviewedBy  string
	CreatedAt   time.Time
	ActivatedAt *time.Time
	// Tests is the test report (JSON), Files the files of the revision (JSON object); Files
	// is empty in listings.
	Tests string
	Files string
}

PolicyRecord is a governance policy revision stored in the database.

type PolicyStore

type PolicyStore interface {
	// CreatePolicy stores a draft revision and returns its number.
	CreatePolicy(ctx context.Context, r PolicyRecord) (int64, error)
	// Policies lists the revisions, newest first, without files.
	Policies(ctx context.Context) ([]PolicyRecord, error)
	// Policy returns a revision with its files, nil if missing.
	Policy(ctx context.Context, revision int64) (*PolicyRecord, error)
	// ActivePolicy returns the active revision, 0 if none.
	ActivePolicy(ctx context.Context) (int64, error)
	// ActivatePolicy makes revision active if the active one is still expected (0 = none):
	// otherwise ErrPolicyConflict. reviewer, if set, is recorded as who reviewed a draft.
	ActivatePolicy(ctx context.Context, revision, expected int64, reviewer string) error
	CreateFreeze(ctx context.Context, f FreezeRecord) error
	// DeleteFreeze returns false if the window does not exist.
	DeleteFreeze(ctx context.Context, name string) (bool, error)
	ListFreezes(ctx context.Context) ([]FreezeRecord, error)
	// NotifyPolicy tells the other processes that policy or freezes changed (Postgres NOTIFY;
	// nothing on SQLite, where they poll).
	NotifyPolicy(ctx context.Context) error
	// ListenPolicy calls fn at every notification until ctx ends; on SQLite it returns at once.
	ListenPolicy(ctx context.Context, fn func()) error
}

PolicyStore stores the governance policy revisions and the runtime freeze windows.

type Registry

type Registry interface {
	// Register records the executor's startup and clears previous shutdown and claim.
	// It returns false if a recovery of its workflows is running: retry later.
	// It returns *InUseError if the same ID is registered by another startup that has not
	// stopped and sent a heartbeat less than staleAfter ago: two processes with the same ID would
	// resume each other's workflows. Call it before starting DBOS.
	Register(ctx context.Context, e Executor, staleAfter time.Duration) (bool, error)
	// Heartbeat updates the executor started at startedAt. It returns who claimed its workflows,
	// if that happened while it was unreachable, or ErrReplaced if another startup registered
	// with the same ID.
	Heartbeat(ctx context.Context, id string, startedAt time.Time) (recoveredBy string, err error)
	// Report stores the resource sample (JSON) of the executor started at startedAt.
	Report(ctx context.Context, id string, startedAt time.Time, resources json.RawMessage) error
	// Stopped records the orderly shutdown of the executor started at startedAt.
	Stopped(ctx context.Context, id string, startedAt time.Time) error
	// List returns the registered executors, ordered by ID.
	List(ctx context.Context) ([]Executor, error)
	// Claim claims the workflows of a lost executor: it succeeds only if the executor stopped or
	// has sent no heartbeat since before staleBefore, and nobody claimed it already (or the claim
	// stayed unfinished beyond ClaimTimeout). With force it always succeeds, even on an executor
	// never registered.
	Claim(ctx context.Context, id, by string, staleBefore time.Time, force bool) (bool, error)
	// ClaimDone marks the recovery of the executor's workflows as done.
	ClaimDone(ctx context.Context, id string) error
	// Lease acquires or renews lease name for ttl: it succeeds if it is free, expired or already
	// held by holder.
	Lease(ctx context.Context, name, holder string, ttl time.Duration) (bool, error)
	Close() error
}

Registry is the executor registry.

Startup registration and claiming exclude each other: Claim succeeds only on an executor that sends no heartbeat, and Register waits until a running recovery of its workflows ends. So a node that comes back right while it is being recovered does not resume (through DBOS recovery) workflows another worker is already re-enqueuing.

type SQL

type SQL struct {
	// contains filtered or unexported fields
}

SQL is the Registry on Postgres or SQLite.

func Open

func Open(ctx context.Context, databaseURL string) (*SQL, error)

Open opens the registry on the DBOS database (database_url) and creates the tables if missing: on Postgres in the "manent" schema, on SQLite with the "manent_" prefix.

func (*SQL) ActivatePolicy

func (r *SQL) ActivatePolicy(ctx context.Context, revision, expected int64, reviewer string) error

ActivatePolicy implements PolicyStore: pointer and states change in one transaction, and the pointer is replaced only if it still holds the expected revision.

func (*SQL) ActivePolicy

func (r *SQL) ActivePolicy(ctx context.Context) (int64, error)

ActivePolicy implements PolicyStore.

func (*SQL) Claim

func (r *SQL) Claim(ctx context.Context, id, by string, staleBefore time.Time, force bool) (bool, error)

Claim implements Registry. An executor missing from the registry can be claimed only with force.

func (*SQL) ClaimDone

func (r *SQL) ClaimDone(ctx context.Context, id string) error

ClaimDone implements Registry.

func (*SQL) Close

func (r *SQL) Close() error

Close closes the connection.

func (*SQL) CreateFreeze

func (r *SQL) CreateFreeze(ctx context.Context, f FreezeRecord) error

CreateFreeze implements PolicyStore.

func (*SQL) CreatePolicy

func (r *SQL) CreatePolicy(ctx context.Context, p PolicyRecord) (int64, error)

CreatePolicy implements PolicyStore.

func (*SQL) DeleteFreeze

func (r *SQL) DeleteFreeze(ctx context.Context, name string) (bool, error)

DeleteFreeze implements PolicyStore.

func (*SQL) Heartbeat

func (r *SQL) Heartbeat(ctx context.Context, id string, startedAt time.Time) (string, error)

Heartbeat implements Registry.

func (*SQL) Lease

func (r *SQL) Lease(ctx context.Context, name, holder string, ttl time.Duration) (bool, error)

Lease implements Registry.

func (*SQL) List

func (r *SQL) List(ctx context.Context) ([]Executor, error)

List implements Registry.

func (*SQL) ListFreezes

func (r *SQL) ListFreezes(ctx context.Context) ([]FreezeRecord, error)

ListFreezes implements PolicyStore.

func (*SQL) ListTargets

func (r *SQL) ListTargets(ctx context.Context) ([]TargetsRecord, error)

ListTargets implements TargetStore.

func (*SQL) ListenPolicy

func (r *SQL) ListenPolicy(ctx context.Context, fn func()) error

ListenPolicy implements PolicyStore with a dedicated connection, opened again after an error.

func (*SQL) NotifyPolicy

func (r *SQL) NotifyPolicy(ctx context.Context) error

NotifyPolicy implements PolicyStore.

func (*SQL) Policies

func (r *SQL) Policies(ctx context.Context) ([]PolicyRecord, error)

Policies implements PolicyStore.

func (*SQL) Policy

func (r *SQL) Policy(ctx context.Context, revision int64) (*PolicyRecord, error)

Policy implements PolicyStore.

func (*SQL) Register

func (r *SQL) Register(ctx context.Context, e Executor, staleAfter time.Duration) (bool, error)

Register implements Registry. Registration replaces the existing row only if it belongs to the same startup read before (started_at): if another process registered meanwhile, read again and reevaluate.

func (*SQL) Report

func (r *SQL) Report(ctx context.Context, id string, startedAt time.Time, resources json.RawMessage) error

Report implements Registry.

func (*SQL) SaveTargets

func (r *SQL) SaveTargets(ctx context.Context, t TargetsRecord) error

SaveTargets implements TargetStore: it replaces the record of the same scope.

func (*SQL) Stopped

func (r *SQL) Stopped(ctx context.Context, id string, startedAt time.Time) error

Stopped implements Registry.

type TargetStore

type TargetStore interface {
	SaveTargets(ctx context.Context, r TargetsRecord) error
	// ListTargets returns all records, ordered by application, environment and scope.
	ListTargets(ctx context.Context) ([]TargetsRecord, error)
}

TargetStore stores the last state read from the targets, one record per application/environment and scope.

type TargetsRecord

type TargetsRecord struct {
	Application string
	Environment string
	// Tenant is the tenant of the deploy read ("" for untenanted deploys).
	Tenant string
	// Artifact is the artifact of the release read.
	Artifact  string
	Scope     string
	CheckedAt time.Time
	// Data is the state, in JSON (service.TargetsState).
	Data string
}

TargetsRecord is the state read from the targets of an application/environment by one reconciliation scope: "" for the workers without a region, the region otherwise.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL