Documentation
¶
Overview ¶
Package cluster is the control plane: it decides what every node runs and learns what is actually running.
agent --GET /v1/agent/state (long-poll)--> control desired instances, mesh peers, proxy routes agent --POST /v1/agent/status ------------> control observed instance states, node metrics
The control node's own agent uses the same two calls in-process, so one code path runs every node. Desired state is recomputed from the store on every poll and identified by an ETag, so no change can be "missed": at worst an agent sees it at its next poll, a couple of seconds later, and Changed makes that immediate.
Index ¶
- Constants
- Variables
- func BackupStatus(b store.VolumeBackup, runs []store.BackupRun) types.VolumeBackups
- func CheckExposure(x *Exposure, allow []string, others map[string]Exposure) error
- func Env(rel *store.Release, in store.Instance) []string
- func Exposures(ctx context.Context, st *store.Store, apps []types.App) (map[string]Exposure, error)
- func HashToken(secret string) string
- func Hostname(in store.Instance) string
- func LoadIdentity(dataDir string) (tls.Certificate, string, error)
- func NextBackup(b store.VolumeBackup, last *store.BackupRun, now time.Time) time.Time
- func NodeSubnet(cluster netip.Prefix, index int) (netip.Prefix, netip.Addr)
- func ParseJoinToken(token string) (pin, secret string, err error)
- func ParseTarget(raw string, cluster netip.Prefix) (tls bool, target netip.AddrPort, err error)
- func PinnedTLS(pin string) *tls.Config
- func RouterLine(r types.Request) string
- func SplitList(v string) []string
- type BadSetting
- type Controller
- func (c *Controller) Activity(ctx context.Context, node string, act types.ProxyActivity) error
- func (c *Controller) AuthKey(ctx context.Context) (string, error)
- func (c *Controller) Authenticate(ctx context.Context, token string) (*store.Node, error)
- func (c *Controller) BackupCluster(ctx context.Context, actor string, log func(string)) (*types.BackupResult, error)
- func (c *Controller) BackupVolume(ctx context.Context, app, name, actor, keep string, log func(string)) (*types.BackupResult, error)
- func (c *Controller) CanSchedule(ctx context.Context) error
- func (c *Controller) Changed()
- func (c *Controller) ComputeState(ctx context.Context, name string) (*types.NodeState, error)
- func (c *Controller) CreateEdge(ctx context.Context, name, address string) (*types.Edge, error)
- func (c *Controller) CreateJoinToken(ctx context.Context, ttl time.Duration, reusable bool) (*types.JoinToken, error)
- func (c *Controller) DatabaseStats(ctx context.Context) ([]types.DatabaseStats, error)
- func (c *Controller) DatabaseStatsOf(ctx context.Context, app string) (*types.DatabaseStats, error)
- func (c *Controller) Exec(ctx context.Context, in store.Instance, argv, env []string, stdin string) ([]byte, error)
- func (c *Controller) Firewall(ctx context.Context) (*types.Firewall, error)
- func (c *Controller) FirewallReport(ctx context.Context, node string, rep types.FirewallReport) error
- func (c *Controller) FirewallSettings(ctx context.Context) (types.FirewallSettings, error)
- func (c *Controller) InstanceLogs(ctx context.Context, in store.Instance, n int) []string
- func (c *Controller) Join(ctx context.Context, req types.JoinRequest) (*types.JoinResponse, error)
- func (c *Controller) KeepAwake(ctx context.Context, app, why string) (done func(), err error)
- func (c *Controller) Latency(ctx context.Context) (*types.Latency, error)
- func (c *Controller) ListBackups(ctx context.Context, app, name string) (*types.VolumeBackups, error)
- func (c *Controller) ListClusterBackups(ctx context.Context) (*types.VolumeBackups, error)
- func (c *Controller) Logs(ctx context.Context, app string, o types.LogOptions, line func(string)) error
- func (c *Controller) MoveVolume(ctx context.Context, app, name, node string) error
- func (c *Controller) NetTests(ctx context.Context) (*types.NetTests, error)
- func (c *Controller) NodeInfo(n store.Node, now time.Time) types.Node
- func (c *Controller) OpenSession(ctx context.Context, in store.Instance) (io.ReadWriteCloser, error)
- func (c *Controller) Place(ctx context.Context, p Placement) (string, netip.Prefix, error)
- func (c *Controller) Rebalance(ctx context.Context, volumes, dryRun bool) (*types.Rebalance, error)
- func (c *Controller) Report(ctx context.Context, node string, st *types.NodeStatus) error
- func (c *Controller) Requests(ctx context.Context, app string, tail int, follow bool, fn func(types.Request)) error
- func (c *Controller) RestoreVolume(ctx context.Context, app, name, backupName, node, actor string) (string, string, error)
- func (c *Controller) RotateAuthKey(ctx context.Context) error
- func (c *Controller) Run(ctx context.Context)
- func (c *Controller) RunInDatabase(ctx context.Context, app string, e database.Engine, cmd database.Command) ([]byte, error)
- func (c *Controller) SetAddress(addr string)
- func (c *Controller) SetFirewall(ctx context.Context, app string, p types.FirewallPatch) error
- func (c *Controller) StartNetTest(ctx context.Context, req types.NetTestRequest) (*types.NetTestProgress, error)
- func (c *Controller) State(ctx context.Context, node, etag string) (*types.NodeState, error)
- func (c *Controller) Status(ctx context.Context) (*types.ClusterStatus, error)
- func (c *Controller) Unban(ctx context.Context, addr string) (bool, error)
- func (c *Controller) UpdateNodes(ctx context.Context) error
- func (c *Controller) Wake(ctx context.Context, app, why string) (bool, error)
- type EdgeBundle
- type ExecError
- type Exposure
- type External
- type LocalPlane
- type NetTestRefused
- type Placement
Constants ¶
const APIPort = 7443
APIPort is the control node's TCP port for joins and agents.
const AgentPort = 7444
AgentPort is each node's agent API (log streams), reachable only over the mesh.
const AuthPlugin = "http-auth"
Logins in front of apps (http-auth) are properties of plugin "http-auth": per app, mode (password or users; anything else is off), password-hash, users, allowed-ips and bypass-paths; globally, login-domain, session-days and session-key, which signs sessions. They aren't in props.Plugins: the hash and the key must never be shown by a report.
const DefaultClusterBackupPath = "jokku/cluster"
DefaultClusterBackupPath is where in its bucket the cluster is backed up unless told otherwise.
const DefaultSessionDays = 30
DefaultSessionDays is how long a login lasts unless set otherwise.
const ExposePlugin = "expose"
ExposePlugin holds what reaches an exposed database from outside (db:<engine>:expose): its domain, port, the client addresses let in and whether it stays on your network. The ingress nodes' and edges' proxies carry its connections as streams (types.ProxyStream), straight to its instance over the mesh.
const ExternalPlugin = "external"
External apps are services Jokku routes to but doesn't run: Home Assistant or a NAS on the LAN, say. Their settings are properties of the app (plugin "external"): url, via (the node that forwards edge traffic to the target; the control node by default) and insecure.
const FirewallPlugin = "firewall"
FirewallPlugin is the properties plugin of the firewall's settings.
const ScheduleActor = "schedule"
ScheduleActor is who scheduled backups are made by, in their records.
const UpdatePlugin = "cluster-update"
UpdatePlugin keeps the cluster's update in the global properties: the version the nodes are going to ("version"), and the node it stopped at and why ("failed", "error"). Kept in the store, it carries on after the control node restarts, and nodes that were down catch up when they return.
Variables ¶
var ( // NetTestSeconds is how long each link is loaded by default. NetTestSeconds = 3.0 // NetTestPings is how many echoes each link gets, 10ms apart. NetTestPings = 100 )
var ( MaxScheduledBackups = 2 RetryBackupAfter = 5 * time.Minute )
Scheduled backups: the controller loop starts each volume's backup when it is due, a few at a time, since each reads its whole disk. A failed one is tried again sooner than its schedule, but not more often than RetryBackupAfter.
var ( PollTimeout = 25 * time.Second PollRecheck = 2 * time.Second )
PollTimeout is how long a long-poll waits for a change before answering with the unchanged state, and PollRecheck how often it recomputes.
var CollectEvery = 6 * time.Hour
CollectEvery is how often a volume's unused blocks are deleted from its bucket, after a backup. Reading every backup file to find them costs more than pruning, which follows every backup.
var DBPollEvery = 10 * time.Second
DBPollEvery is how often the control node looks inside each running database, for "jokku top" (a variable so tests can shorten it).
var DrainGrace = 10 * time.Second
DrainGrace is how long a moved instance keeps running after its replacement takes traffic, to finish in-flight requests.
var EdgeRemovalGrace = 2 * time.Minute
EdgeRemovalGrace is how long a removed edge stays a peer, so its next poll (every PollTimeout at most) tells it to stop routing.
var ErrNotRunning = errors.New("not running")
ErrNotRunning is a database (or app) with no running instance to run a command in.
var FailoverAfter = 5 * time.Minute
FailoverAfter is how long a volume's node must have been silent before the volume is restored onto another node from its latest backup. Nodes stop the instances using such volumes after agent.FenceAfter without the control node, well before.
var LANRanges = []string{"10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "::1/128", "fc00::/7"}
LANRanges are the addresses a database exposed on your network (--lan) lets in unless told otherwise: the private ranges, Tailscale's and the nodes' own.
var RescheduleAfter = 60 * time.Second
RescheduleAfter is how long a node may be down before its instances are started elsewhere. Short blips (a reboot, an update) don't move anything. The timings here are variables so tests can shorten them.
var ReservedPorts = func() map[int]bool { ports := map[int]bool{22: true, 53: true, proxy.RelayPort: true, 443: true} for _, p := range strings.Split(mesh.ProtectedPorts, ",") { n, _ := strconv.Atoi(p) ports[n] = true } return ports }()
ReservedPorts are the TCP ports Jokku's nodes use themselves: ssh (for pushes), DNS, the proxies' HTTP and HTTPS, and the control node's API.
var RetryAfter = 30 * time.Second
RetryAfter spaces out attempts to replace the same instance.
var RetryFailoverAfter = time.Minute
RetryFailoverAfter spaces out attempts at a restore that can't happen yet (no node with room, say).
var UnexposeProperties = map[string]string{"domain": "", "port": "", "allow": "", "lan": ""}
UnexposeProperties clear an exposure.
var UpdateTimeout = 10 * time.Minute
UpdateTimeout is how long a node has to update and report back. (A variable so tests can shorten it.)
Functions ¶
func BackupStatus ¶
func BackupStatus(b store.VolumeBackup, runs []store.BackupRun) types.VolumeBackups
BackupStatus describes a volume's backup settings and its latest backups (newest first), without the list of backups in the bucket.
func CheckExposure ¶
CheckExposure completes x (the engine's own port when it has none) and checks it against the other exposures: a port carries one protocol, a plain stream has its port to itself, and a name on a shared port reaches one database. allow is the client addresses let in, as given.
func Env ¶
Env is the process environment: the image's, then config vars, then what Jokku sets. A compose service gets the compose file's environment instead of the config vars, as with docker compose (config vars fill in ${VAR} there).
func Exposures ¶
Exposures reads the exposed databases, by app. One that is misconfigured (which the API prevents) is left out rather than failing every node's state.
func LoadIdentity ¶
func LoadIdentity(dataDir string) (tls.Certificate, string, error)
LoadIdentity loads (or creates, on first start) the control node's TLS key and self-signed certificate, and returns the pin: the base64url SHA-256 of its public key. Nodes pin that key instead of trusting a CA, so the certificate's names and the address used don't matter.
func NextBackup ¶
NextBackup is when a volume's next scheduled backup is due, given its latest backup (nil if none): now if it has none; zero when it isn't scheduled.
func NodeSubnet ¶
NodeSubnet returns node index's /24 inside the /16 cluster network and the node's own address (.1) in it: index 3 in 10.210.0.0/16 is 10.210.3.0/24 and 10.210.3.1.
func ParseJoinToken ¶
ParseJoinToken splits JOKKU1.<pin>.<secret>.
func ParseTarget ¶
ParseTarget checks an external app's URL: http or https, to an IPv4 address and optional port. Edges reach the target through a node over the mesh, so it has to be an address, not a name only the LAN resolves.
func RouterLine ¶
RouterLine renders a request the way "jokku logs" shows it, alongside the app's own output.
Types ¶
type BadSetting ¶
type BadSetting struct {
// contains filtered or unexported fields
}
BadSetting is a firewall setting out of bounds.
func (*BadSetting) Error ¶
func (e *BadSetting) Error() string
type Controller ¶
type Controller struct {
Store *store.Store
Log *slog.Logger
Self string // the control node's name
ClusterCIDR netip.Prefix // e.g. 10.210.0.0/16
Version string
// Address is where joining nodes reach this node's API, host:port. Set
// it with SetAddress once the controller runs.
Address string
// Pin identifies this node's TLS key; it is embedded in join tokens.
Pin string
// TickEvery is how often the controller loop runs (default 3s).
TickEvery time.Duration
// AgentAddr is where a node's agent API listens, host:port. The default
// is its mesh address and AgentPort; tests override it.
AgentAddr func(store.Node) string
// DataDir is the control node's data directory, which cluster backups
// copy.
DataDir string
// Durations is how long the requests this node's proxy handled took
// (its agent's Latency); nil leaves this node out of Latency.
Durations *latency.Recorder
// contains filtered or unexported fields
}
func New ¶
func New(c *Controller) *Controller
func (*Controller) Activity ¶
func (c *Controller) Activity(ctx context.Context, node string, act types.ProxyActivity) error
Activity records what a node's proxy reports: apps it served, and apps a request is waiting to wake.
func (*Controller) AuthKey ¶
func (c *Controller) AuthKey(ctx context.Context) (string, error)
AuthKey returns the key that signs sessions, making it on first use.
func (*Controller) Authenticate ¶
Authenticate maps an agent's credential to its node.
func (*Controller) BackupCluster ¶
func (c *Controller) BackupCluster(ctx context.Context, actor string, log func(string)) (*types.BackupResult, error)
BackupCluster backs up the control node now, and records the backup.
func (*Controller) BackupVolume ¶
func (c *Controller) BackupVolume(ctx context.Context, app, name, actor, keep string, log func(string)) (*types.BackupResult, error)
BackupVolume backs up an app's volume now, on the node that holds its disk, and records the backup. Retention then keeps keep (when given), whatever its rules say. log is told the node's progress.
func (*Controller) CanSchedule ¶
func (c *Controller) CanSchedule(ctx context.Context) error
CanSchedule explains why no node can run microVMs right now, or returns nil, so a deploy can fail before building.
func (*Controller) Changed ¶
func (c *Controller) Changed()
Changed wakes every waiting agent so it recomputes its state now.
func (*Controller) ComputeState ¶
ComputeState builds a node's desired state from the store.
func (*Controller) CreateEdge ¶
CreateEdge adds an edge at address (its public IP or DNS name, with the WireGuard port if not 51820) and returns the command that installs it.
func (*Controller) CreateJoinToken ¶
func (c *Controller) CreateJoinToken(ctx context.Context, ttl time.Duration, reusable bool) (*types.JoinToken, error)
CreateJoinToken makes a token for adding a node, and the one-liner to run on it.
func (*Controller) DatabaseStats ¶
func (c *Controller) DatabaseStats(ctx context.Context) ([]types.DatabaseStats, error)
DatabaseStats is what the control node has seen inside every database.
func (*Controller) DatabaseStatsOf ¶
func (c *Controller) DatabaseStatsOf(ctx context.Context, app string) (*types.DatabaseStats, error)
DatabaseStatsOf is DatabaseStats for one database's app.
func (*Controller) Exec ¶
func (c *Controller) Exec(ctx context.Context, in store.Instance, argv, env []string, stdin string) ([]byte, error)
Exec runs argv in instance in, with env and fed stdin, and returns what it wrote to stdout. It is for the control node's own use (a database's statistics, its users), so unlike jokku enter it records no event. A command that fails is an *ExecError with what it wrote to stderr.
func (*Controller) Firewall ¶
Firewall is what the firewall is doing: its settings, the bans the proxies have, and their counts.
func (*Controller) FirewallReport ¶
func (c *Controller) FirewallReport(ctx context.Context, node string, rep types.FirewallReport) error
FirewallReport takes in what a node's proxy says it did.
func (*Controller) FirewallSettings ¶
func (c *Controller) FirewallSettings(ctx context.Context) (types.FirewallSettings, error)
FirewallSettings reads the firewall's settings.
func (*Controller) InstanceLogs ¶
InstanceLogs returns the last n lines an instance printed, wherever it runs, for deploy failure messages.
func (*Controller) Join ¶
func (c *Controller) Join(ctx context.Context, req types.JoinRequest) (*types.JoinResponse, error)
Join admits a node: it checks the token, gives the node a name, a subnet and credentials, and tells every node about its new peer.
func (*Controller) KeepAwake ¶
func (c *Controller) KeepAwake(ctx context.Context, app, why string) (done func(), err error)
KeepAwake wakes an app if it's asleep, and keeps it awake until done is called: a deploy or restart needs its instances running.
func (*Controller) Latency ¶
Latency is how long requests took over the last hour and day: every live node's histograms added up (each node's agent keeps its proxy's), so the percentiles are the cluster's, not an average of nodes'.
func (*Controller) ListBackups ¶
func (c *Controller) ListBackups(ctx context.Context, app, name string) (*types.VolumeBackups, error)
ListBackups lists the backups of an app's volume, newest first, from its bucket, with what this cluster recorded about them.
func (*Controller) ListClusterBackups ¶
func (c *Controller) ListClusterBackups(ctx context.Context) (*types.VolumeBackups, error)
ListClusterBackups lists the cluster's backups, newest first.
func (*Controller) Logs ¶
func (c *Controller) Logs(ctx context.Context, app string, o types.LogOptions, line func(string)) error
Logs streams an app's log lines from every node that is up, with the proxy's router lines (one per request) mixed in unless o.Process asks for a process type. Without Follow, lines are merged by timestamp and the last o.Tail returned; with Follow, lines are passed on as they arrive.
func (*Controller) MoveVolume ¶
func (c *Controller) MoveVolume(ctx context.Context, app, name, node string) error
MoveVolume moves an app's volume to node (storage:move), with the instance using it and that instance's other volumes.
func (*Controller) NetTests ¶ added in v0.8.19
NetTests is the running test, if any, and the latest finished ones.
func (*Controller) OpenSession ¶
func (c *Controller) OpenSession(ctx context.Context, in store.Instance) (io.ReadWriteCloser, error)
OpenSession opens a session with an instance's guest agent, through its node's agent (jokku enter, volume copies). The caller sends the request first.
func (*Controller) Place ¶
Place picks the node for an instance: among nodes that are up, schedulable, not draining, able to run microVMs and with the memory free, it prefers the one running the fewest instances of the same app and process type (spread), then the one with the most free memory. It returns the node and its instance subnet.
func (*Controller) Rebalance ¶ added in v0.8.21
Rebalance spreads instances over the nodes that take them (up, schedulable, not draining). Nothing moves back on its own once a drained node accepts instances again or a dead one returns; this is how it gets its share. Instances of one app and process type go to different nodes first, then memory is evened out. Each instance moves as a drain moves it: a copy starts on the new node and the original stops once that passes checks. One with volumes stays where their disks are unless volumes is set; then they move along, and it stops for a few seconds for the final copy. With dryRun, nothing moves: the result says what would.
func (*Controller) Report ¶
func (c *Controller) Report(ctx context.Context, node string, st *types.NodeStatus) error
Report records what a node's agent observed: it is alive, its metrics, and each instance's state. Retired instances the node no longer reports are deleted, since the node has removed them.
func (*Controller) Requests ¶
func (c *Controller) Requests(ctx context.Context, app string, tail int, follow bool, fn func(types.Request)) error
Requests streams the HTTP requests every node's proxy handles, or only one app's. Each is labelled with the node that received it and the instance (and its node) that answered. Without follow, the last tail requests are returned in time order.
func (*Controller) RestoreVolume ¶
func (c *Controller) RestoreVolume(ctx context.Context, app, name, backupName, node, actor string) (string, string, error)
RestoreVolume starts restoring an app's volume from one of its backups (the latest for name ""): on the disk's own node, or onto node when the disk's node is down or gone. It returns the backup and the node.
func (*Controller) RotateAuthKey ¶
func (c *Controller) RotateAuthKey(ctx context.Context) error
RotateAuthKey replaces the session key, if there is one, which logs everyone out: for when a machine that held it (an edge) leaves.
func (*Controller) Run ¶
func (c *Controller) Run(ctx context.Context)
Run is the controller loop: it notices nodes going down or coming back, moves instances off dead, removed and draining nodes, and cleans up.
func (*Controller) RunInDatabase ¶
func (c *Controller) RunInDatabase(ctx context.Context, app string, e database.Engine, cmd database.Command) ([]byte, error)
RunInDatabase runs cmd in app's database, e.
func (*Controller) SetAddress ¶
func (c *Controller) SetAddress(addr string)
SetAddress changes where joining nodes reach this node's API.
func (*Controller) SetFirewall ¶
func (c *Controller) SetFirewall(ctx context.Context, app string, p types.FirewallPatch) error
SetFirewall changes the firewall's settings, or with an app whether it is on for it.
func (*Controller) StartNetTest ¶ added in v0.8.19
func (c *Controller) StartNetTest(ctx context.Context, req types.NetTestRequest) (*types.NetTestProgress, error)
StartNetTest starts testing the network between the nodes, and returns at once; NetTests follows it.
func (*Controller) State ¶
State returns node's desired state once it differs from etag, or after pollTimeout. An empty etag returns right away.
func (*Controller) Status ¶
func (c *Controller) Status(ctx context.Context) (*types.ClusterStatus, error)
Status is the whole cluster at a glance: nodes, apps, instances and recent events.
func (*Controller) UpdateNodes ¶
func (c *Controller) UpdateNodes(ctx context.Context) error
UpdateNodes brings the other nodes to this control node's version, one at a time, starting again with the node a failed update stopped at.
type EdgeBundle ¶
type EdgeBundle struct {
Version string `json:"version"`
Control string `json:"control"` // the control node's API over the mesh, host:port
Pin string `json:"pin"`
NodeToken string `json:"node_token"`
AgentToken string `json:"agent_token"`
Node types.NodeIdentity `json:"node"`
PrivateKey string `json:"private_key"` // WireGuard
Peers []types.Peer `json:"peers"`
}
EdgeBundle is everything an edge needs to come up: who it is, its credentials, and the control node as its first peer, so the mesh is up before it has heard anything.
func ParseEdgeBundle ¶
func ParseEdgeBundle(s string) (*EdgeBundle, error)
ParseEdgeBundle reads what Encode wrote.
func (*EdgeBundle) Encode ¶
func (b *EdgeBundle) Encode() string
type Exposure ¶
type Exposure struct {
App string
Engine database.Engine
Host string
Port int
Allow []string // CIDRs; empty lets in any address
LAN bool // only the ingress nodes carry it, not the edges
}
Exposure is how a database is reached from outside.
func (Exposure) Properties ¶
Properties are what the store keeps of x.
type External ¶
type External struct {
App string
TLS bool // the target speaks HTTPS
Target netip.AddrPort
Via string // the node edges reach it through
Insecure bool // its certificate isn't checked
}
External is an external app's target, ready to route.
type LocalPlane ¶
type LocalPlane struct {
C *Controller
DataDir string
}
LocalPlane is the control plane as the control node's own agent sees it: the same calls workers make over HTTPS, made in-process.
func (*LocalPlane) Artifact ¶
Artifact is only needed when an artifact is missing locally, which on the control node (where they are built) means it was deleted.
func (*LocalPlane) Report ¶
func (p *LocalPlane) Report(ctx context.Context, st *types.NodeStatus) error
type NetTestRefused ¶ added in v0.8.19
type NetTestRefused struct {
// contains filtered or unexported fields
}
NetTestRefused says why a network test can't start.
func (*NetTestRefused) Error ¶ added in v0.8.19
func (e *NetTestRefused) Error() string
type Placement ¶
type Placement struct {
App string
ProcessType string
CPUs int
MemoryMB int
Exclude string // a node to avoid (the one being moved off)
// Pin is the only node the instance can run on: its volumes' disks are
// there. It may be draining or unschedulable (a drain moves it later),
// but it must be up. PinReason says why, for errors.
Pin, PinReason string
// Volumes asks for nodes whose agent can hold volumes, and DiskMB for
// free disk space to copy them to.
Volumes bool
DiskMB int
// OldCopies are nodes keeping an old copy of a volume that comes along
// (see storage:discard-old-copy): it can't be received there.
OldCopies []string
}
Placement asks for a node for one instance.