Documentation
¶
Overview ¶
Package cluster keeps the registry of manent executors (workers): heartbeats, shutdown, claiming the workflows of a lost node and leases for the tasks only one worker at a time must perform (automatic recovery, retention, reconciliation). It also stores the state read from the targets by the reconciliation.
The registry lives in the DBOS database, in its own tables: without DBOS Conductor it is the only way to know which executors are still alive.
Index ¶
- Constants
- Variables
- type Executor
- type FreezeRecord
- type InUseError
- type PolicyRecord
- type PolicyStore
- type Registry
- type SQL
- func (r *SQL) ActivatePolicy(ctx context.Context, revision, expected int64, reviewer string) error
- func (r *SQL) ActivePolicy(ctx context.Context) (int64, error)
- func (r *SQL) Claim(ctx context.Context, id, by string, staleBefore time.Time, force bool) (bool, error)
- func (r *SQL) ClaimDone(ctx context.Context, id string) error
- func (r *SQL) Close() error
- func (r *SQL) CreateFreeze(ctx context.Context, f FreezeRecord) error
- func (r *SQL) CreatePolicy(ctx context.Context, p PolicyRecord) (int64, error)
- func (r *SQL) DeleteFreeze(ctx context.Context, name string) (bool, error)
- func (r *SQL) Heartbeat(ctx context.Context, id string, startedAt time.Time) (string, error)
- func (r *SQL) Lease(ctx context.Context, name, holder string, ttl time.Duration) (bool, error)
- func (r *SQL) List(ctx context.Context) ([]Executor, error)
- func (r *SQL) ListFreezes(ctx context.Context) ([]FreezeRecord, error)
- func (r *SQL) ListTargets(ctx context.Context) ([]TargetsRecord, error)
- func (r *SQL) ListenPolicy(ctx context.Context, fn func()) error
- func (r *SQL) NotifyPolicy(ctx context.Context) error
- func (r *SQL) Policies(ctx context.Context) ([]PolicyRecord, error)
- func (r *SQL) Policy(ctx context.Context, revision int64) (*PolicyRecord, error)
- func (r *SQL) Register(ctx context.Context, e Executor, staleAfter time.Duration) (bool, error)
- func (r *SQL) Report(ctx context.Context, id string, startedAt time.Time, resources json.RawMessage) error
- func (r *SQL) SaveTargets(ctx context.Context, t TargetsRecord) error
- func (r *SQL) Stopped(ctx context.Context, id string, startedAt time.Time) error
- type TargetStore
- type TargetsRecord
Constants ¶
const ClaimTimeout = 2 * time.Minute
ClaimTimeout is the time after which a recovery not marked as done (e.g. because whoever ran it crashed) no longer blocks the executor's registration.
Variables ¶
var ErrPolicyConflict = errors.New("another policy revision was activated meanwhile")
ErrPolicyConflict means another revision was activated meanwhile.
var ErrReplaced = errors.New("another process registered with the same executor_id")
ErrReplaced means another startup registered with the same executor_id.
Functions ¶
This section is empty.
Types ¶
type Executor ¶
type Executor struct {
ID string `json:"id"`
// Version is the application version of the workflows (app.version): a worker only resumes
// workflows of its own version.
Version string `json:"version"`
// Build is the manent binary version.
Build string `json:"build,omitempty"`
Hostname string `json:"hostname,omitempty"`
// Region is the worker's region (dbos.region), if configured.
Region string `json:"region,omitempty"`
StartedAt time.Time `json:"started_at"`
HeartbeatAt time.Time `json:"heartbeat_at"`
StoppedAt *time.Time `json:"stopped_at,omitempty"`
// RecoveredBy is who claimed the executor's workflows after it was lost.
RecoveredBy string `json:"recovered_by,omitempty"`
RecoveredAt *time.Time `json:"recovered_at,omitempty"`
// Queues are the DBOS queues the executor dequeues from.
Queues []string `json:"queues,omitempty"`
// Resources is the last resource sample sent by the executor, in JSON
// (service.WorkerResources): the registry stores it without interpreting it.
Resources json.RawMessage `json:"resources,omitempty"`
}
Executor is a registered worker.
type FreezeRecord ¶
FreezeRecord is a freeze window created at runtime; Data is the window in JSON.
type InUseError ¶
InUseError means an executor_id already registered by another live process.
func (*InUseError) Error ¶
func (e *InUseError) Error() string
type PolicyRecord ¶
type PolicyRecord struct {
Revision int64
SHA256 string
Status string
Author string
Comment string
ReviewedBy string
CreatedAt time.Time
ActivatedAt *time.Time
// Tests is the test report (JSON), Files the files of the revision (JSON object); Files
// is empty in listings.
Tests string
Files string
}
PolicyRecord is a governance policy revision stored in the database.
type PolicyStore ¶
type PolicyStore interface {
// CreatePolicy stores a draft revision and returns its number.
CreatePolicy(ctx context.Context, r PolicyRecord) (int64, error)
// Policies lists the revisions, newest first, without files.
Policies(ctx context.Context) ([]PolicyRecord, error)
// Policy returns a revision with its files, nil if missing.
Policy(ctx context.Context, revision int64) (*PolicyRecord, error)
// ActivePolicy returns the active revision, 0 if none.
ActivePolicy(ctx context.Context) (int64, error)
// ActivatePolicy makes revision active if the active one is still expected (0 = none):
// otherwise ErrPolicyConflict. reviewer, if set, is recorded as who reviewed a draft.
ActivatePolicy(ctx context.Context, revision, expected int64, reviewer string) error
CreateFreeze(ctx context.Context, f FreezeRecord) error
// DeleteFreeze returns false if the window does not exist.
DeleteFreeze(ctx context.Context, name string) (bool, error)
ListFreezes(ctx context.Context) ([]FreezeRecord, error)
// NotifyPolicy tells the other processes that policy or freezes changed (Postgres NOTIFY;
// nothing on SQLite, where they poll).
NotifyPolicy(ctx context.Context) error
// ListenPolicy calls fn at every notification until ctx ends; on SQLite it returns at once.
ListenPolicy(ctx context.Context, fn func()) error
}
PolicyStore stores the governance policy revisions and the runtime freeze windows.
type Registry ¶
type Registry interface {
// Register records the executor's startup and clears previous shutdown and claim.
// It returns false if a recovery of its workflows is running: retry later.
// It returns *InUseError if the same ID is registered by another startup that has not
// stopped and sent a heartbeat less than staleAfter ago: two processes with the same ID would
// resume each other's workflows. Call it before starting DBOS.
Register(ctx context.Context, e Executor, staleAfter time.Duration) (bool, error)
// Heartbeat updates the executor started at startedAt. It returns who claimed its workflows,
// if that happened while it was unreachable, or ErrReplaced if another startup registered
// with the same ID.
Heartbeat(ctx context.Context, id string, startedAt time.Time) (recoveredBy string, err error)
// Report stores the resource sample (JSON) of the executor started at startedAt.
Report(ctx context.Context, id string, startedAt time.Time, resources json.RawMessage) error
// Stopped records the orderly shutdown of the executor started at startedAt.
Stopped(ctx context.Context, id string, startedAt time.Time) error
// List returns the registered executors, ordered by ID.
List(ctx context.Context) ([]Executor, error)
// Claim claims the workflows of a lost executor: it succeeds only if the executor stopped or
// has sent no heartbeat since before staleBefore, and nobody claimed it already (or the claim
// stayed unfinished beyond ClaimTimeout). With force it always succeeds, even on an executor
// never registered.
Claim(ctx context.Context, id, by string, staleBefore time.Time, force bool) (bool, error)
// ClaimDone marks the recovery of the executor's workflows as done.
ClaimDone(ctx context.Context, id string) error
// Lease acquires or renews lease name for ttl: it succeeds if it is free, expired or already
// held by holder.
Lease(ctx context.Context, name, holder string, ttl time.Duration) (bool, error)
Close() error
}
Registry is the executor registry.
Startup registration and claiming exclude each other: Claim succeeds only on an executor that sends no heartbeat, and Register waits until a running recovery of its workflows ends. So a node that comes back right while it is being recovered does not resume (through DBOS recovery) workflows another worker is already re-enqueuing.
type SQL ¶
type SQL struct {
// contains filtered or unexported fields
}
SQL is the Registry on Postgres or SQLite.
func Open ¶
Open opens the registry on the DBOS database (database_url) and creates the tables if missing: on Postgres in the "manent" schema, on SQLite with the "manent_" prefix.
func (*SQL) ActivatePolicy ¶
ActivatePolicy implements PolicyStore: pointer and states change in one transaction, and the pointer is replaced only if it still holds the expected revision.
func (*SQL) ActivePolicy ¶
ActivePolicy implements PolicyStore.
func (*SQL) Claim ¶
func (r *SQL) Claim(ctx context.Context, id, by string, staleBefore time.Time, force bool) (bool, error)
Claim implements Registry. An executor missing from the registry can be claimed only with force.
func (*SQL) CreateFreeze ¶
func (r *SQL) CreateFreeze(ctx context.Context, f FreezeRecord) error
CreateFreeze implements PolicyStore.
func (*SQL) CreatePolicy ¶
CreatePolicy implements PolicyStore.
func (*SQL) DeleteFreeze ¶
DeleteFreeze implements PolicyStore.
func (*SQL) ListFreezes ¶
func (r *SQL) ListFreezes(ctx context.Context) ([]FreezeRecord, error)
ListFreezes implements PolicyStore.
func (*SQL) ListTargets ¶
func (r *SQL) ListTargets(ctx context.Context) ([]TargetsRecord, error)
ListTargets implements TargetStore.
func (*SQL) ListenPolicy ¶
ListenPolicy implements PolicyStore with a dedicated connection, opened again after an error.
func (*SQL) NotifyPolicy ¶
NotifyPolicy implements PolicyStore.
func (*SQL) Policies ¶
func (r *SQL) Policies(ctx context.Context) ([]PolicyRecord, error)
Policies implements PolicyStore.
func (*SQL) Register ¶
Register implements Registry. Registration replaces the existing row only if it belongs to the same startup read before (started_at): if another process registered meanwhile, read again and reevaluate.
func (*SQL) Report ¶
func (r *SQL) Report(ctx context.Context, id string, startedAt time.Time, resources json.RawMessage) error
Report implements Registry.
func (*SQL) SaveTargets ¶
func (r *SQL) SaveTargets(ctx context.Context, t TargetsRecord) error
SaveTargets implements TargetStore: it replaces the record of the same scope.
type TargetStore ¶
type TargetStore interface {
SaveTargets(ctx context.Context, r TargetsRecord) error
// ListTargets returns all records, ordered by application, environment and scope.
ListTargets(ctx context.Context) ([]TargetsRecord, error)
}
TargetStore stores the last state read from the targets, one record per application/environment and scope.
type TargetsRecord ¶
type TargetsRecord struct {
Application string
Environment string
// Tenant is the tenant of the deploy read ("" for untenanted deploys).
Tenant string
// Artifact is the artifact of the release read.
Artifact string
Scope string
CheckedAt time.Time
// Data is the state, in JSON (service.TargetsState).
Data string
}
TargetsRecord is the state read from the targets of an application/environment by one reconciliation scope: "" for the workers without a region, the region otherwise.