Documentation
¶
Index ¶
- Constants
- Variables
- type ActiveRunUpdate
- type Admission
- type AdmitInput
- type Backend
- type Command
- type CommandSubscription
- type CurrentRunPatch
- type CurrentRunView
- type Cursor
- type DecisionFenceActivator
- type DecisionResponse
- type DecisionResponseResult
- type DecisionStore
- type DecisionTarget
- type DistributedBackend
- type Event
- type Execution
- type FenceActivator
- type HistoryResetBackend
- type HistoryResetHandler
- type Key
- type LeaseCandidate
- type LivenessBackend
- type Manager
- func (m *Manager) Abort(ctx context.Context, botID, sessionID, runID string) (bool, error)
- func (m *Manager) AbortControl(ctx context.Context, botID, sessionID, runID, controlID string) (bool, error)
- func (m *Manager) AbortRun(ctx context.Context, handle RunHandle) (bool, error)
- func (m *Manager) Admit(ctx context.Context, in AdmitInput) (Admission, error)
- func (m *Manager) BeginBotHistoryReset(ctx context.Context, botID string) (context.Context, func(), error)
- func (m *Manager) BeginSessionHistoryReset(ctx context.Context, botID, sessionID string) (context.Context, func(), error)
- func (m *Manager) Close() error
- func (m *Manager) CloseContext(ctx context.Context) error
- func (m *Manager) DecisionContinuationContext(cmd Command) (context.Context, context.CancelFunc, error)
- func (m *Manager) DispatchActiveCommand(ctx context.Context, botID, sessionID, commandType, targetID string, ...) (bool, error)
- func (m *Manager) DispatchRunCommand(ctx context.Context, botID, sessionID, runID, commandType, targetID string, ...) (bool, error)
- func (m *Manager) FinishRun(ctx context.Context, handle RunHandle, status, message string) error
- func (m *Manager) FinishRunWithErrorCode(ctx context.Context, handle RunHandle, status, errorCode string) error
- func (m *Manager) HandleAgentEvent(ctx context.Context, handle RunHandle, event native.StreamEvent) ([]chatview.UIMessage, error)
- func (m *Manager) HandleAgentEventWithStatus(ctx context.Context, handle RunHandle, event native.StreamEvent) ([]chatview.UIMessage, string, error)
- func (m *Manager) IsDistributed() bool
- func (m *Manager) RouteDecisionResponse(ctx context.Context, response DecisionResponse) (DecisionResponseResult, error)
- func (m *Manager) RunRef(ctx context.Context, botID, sessionID, runID string) (RunRef, bool, error)
- func (m *Manager) SetCommandHandler(handler func(context.Context, Command) error)
- func (m *Manager) SetCommandReconciler(reconciler func(context.Context, Command) (bool, error))
- func (m *Manager) SetDecisionStore(store DecisionStore)
- func (m *Manager) SetHistoryResetHandler(handler HistoryResetHandler)
- func (m *Manager) SetTerminalObserver(observer func(context.Context, TerminalRun))
- func (m *Manager) SetTerminalReconciler(reconciler func(context.Context) error)
- func (m *Manager) Snapshot(ctx context.Context, botID, sessionID string) (Snapshot, error)
- func (m *Manager) Start(ctx context.Context) error
- func (m *Manager) StartRun(ctx context.Context, botID, sessionID, runID string, abortCh chan<- struct{}, ...) error
- func (m *Manager) StartRunHandle(ctx context.Context, botID, sessionID, runID string, abortCh chan<- struct{}, ...) (RunHandle, error)
- func (m *Manager) StartRunWithAdmissionBuilderAndOwnershipHandle(ctx context.Context, botID, sessionID, runID string, ...) (RunHandle, error)
- func (m *Manager) StartRunWithAdmissionBuilderHandle(ctx context.Context, botID, sessionID, runID string, ...) (RunHandle, error)
- func (m *Manager) Steer(ctx context.Context, botID, sessionID, runID, text string) (SteerState, error)
- func (m *Manager) SteerRun(ctx context.Context, handle RunHandle, text string) (SteerState, error)
- func (m *Manager) Subscribe(ctx context.Context, botID, sessionID string) (Subscription, error)
- func (m *Manager) ValidateRunOwnership(ctx context.Context, handle RunHandle) error
- func (m *Manager) WaitDecisionContinuationReady(ctx context.Context, cmd Command) error
- func (m *Manager) WaitForHistoryReset(ctx context.Context, botID, sessionID string) error
- type MemoryBackend
- func (b *MemoryBackend) AcquireHistoryReset(ctx context.Context, scope ResetScope, token string, ttl time.Duration) (ResetLease, bool, error)
- func (*MemoryBackend) AcquireLeaderLease(context.Context, string, time.Duration) (bool, error)
- func (b *MemoryBackend) Close() error
- func (b *MemoryBackend) EffectiveHistoryReset(ctx context.Context, scope ResetScope) (ResetLease, bool, error)
- func (*MemoryBackend) ExpiredLeaseCandidates(context.Context, int64) ([]LeaseCandidate, error)
- func (b *MemoryBackend) LivenessGeneration(context.Context) (string, error)
- func (b *MemoryBackend) Load(ctx context.Context, key Key) (Snapshot, bool, error)
- func (*MemoryBackend) Now(ctx context.Context) (time.Time, error)
- func (b *MemoryBackend) Publish(ctx context.Context, event Event) error
- func (b *MemoryBackend) ReleaseHistoryReset(ctx context.Context, lease ResetLease) (bool, error)
- func (*MemoryBackend) ReleaseLeaderLease(context.Context, string) error
- func (*MemoryBackend) ReleaseLeaseCandidate(context.Context, LeaseCandidate) (bool, error)
- func (b *MemoryBackend) RenewHistoryReset(ctx context.Context, lease ResetLease, ttl time.Duration) (ResetLease, bool, error)
- func (b *MemoryBackend) StartRunIfNoHistoryReset(ctx context.Context, key Key, update SnapshotUpdate) (Snapshot, bool, error)
- func (b *MemoryBackend) Subscribe(ctx context.Context, key Key) (Subscription, error)
- func (b *MemoryBackend) Update(ctx context.Context, key Key, update SnapshotUpdate) (Snapshot, bool, error)
- type Options
- type Reaper
- func (r *Reaper) Close(ctx context.Context) error
- func (r *Reaper) SetTerminalObserver(observer func(context.Context, TerminalRun))
- func (r *Reaper) SetTerminalReconciler(reconciler func(context.Context) error)
- func (r *Reaper) SetWaitingDecisionRecoverer(recoverer func(context.Context, LeaseCandidate) (bool, error))
- func (r *Reaper) Start(ctx context.Context) error
- type RedisBackend
- func (b *RedisBackend) AcquireHistoryReset(ctx context.Context, scope ResetScope, token string, ttl time.Duration) (ResetLease, bool, error)
- func (b *RedisBackend) AcquireLeaderLease(ctx context.Context, ownerID string, ttl time.Duration) (bool, error)
- func (b *RedisBackend) CheckHealth(ctx context.Context) error
- func (b *RedisBackend) Close() error
- func (b *RedisBackend) DeleteRunRef(ctx context.Context, ref RunRef) (bool, error)
- func (b *RedisBackend) EffectiveHistoryReset(ctx context.Context, scope ResetScope) (ResetLease, bool, error)
- func (b *RedisBackend) ExpiredLeaseCandidates(ctx context.Context, limit int64) ([]LeaseCandidate, error)
- func (b *RedisBackend) LivenessGeneration(ctx context.Context) (string, error)
- func (b *RedisBackend) Load(ctx context.Context, key Key) (Snapshot, bool, error)
- func (b *RedisBackend) LoadCommandResult(ctx context.Context, commandID string) (Command, bool, error)
- func (b *RedisBackend) LoadRunRef(ctx context.Context, key Key, runID string) (RunRef, bool, error)
- func (b *RedisBackend) Now(ctx context.Context) (time.Time, error)
- func (b *RedisBackend) Publish(ctx context.Context, event Event) error
- func (b *RedisBackend) PublishCommand(ctx context.Context, ownerID string, command Command) error
- func (b *RedisBackend) ReleaseHistoryReset(ctx context.Context, lease ResetLease) (bool, error)
- func (b *RedisBackend) ReleaseLeaderLease(ctx context.Context, ownerID string) error
- func (b *RedisBackend) ReleaseLeaseCandidate(ctx context.Context, candidate LeaseCandidate) (bool, error)
- func (b *RedisBackend) ReleaseRun(ctx context.Context, key Key, ref RunRef, update ActiveRunUpdate) (Snapshot, bool, error)
- func (b *RedisBackend) RenewHistoryReset(ctx context.Context, lease ResetLease, ttl time.Duration) (ResetLease, bool, error)
- func (b *RedisBackend) RenewLease(ctx context.Context, key Key, runID, ownerID, generation string, ...) error
- func (b *RedisBackend) StartRun(ctx context.Context, key Key, ref RunRef, update SnapshotUpdate) (Snapshot, bool, error)
- func (b *RedisBackend) StoreCommandResult(ctx context.Context, result Command, ttl time.Duration) error
- func (b *RedisBackend) Subscribe(ctx context.Context, key Key) (Subscription, error)
- func (b *RedisBackend) SubscribeCommands(ctx context.Context, ownerID string) (CommandSubscription, error)
- func (b *RedisBackend) Update(ctx context.Context, key Key, update SnapshotUpdate) (Snapshot, bool, error)
- func (b *RedisBackend) UpdateActiveRun(ctx context.Context, key Key, runID, generation string, update ActiveRunUpdate) (Snapshot, bool, error)
- func (b *RedisBackend) ValidateRunOwnership(ctx context.Context, key Key, ref RunRef) error
- type RedisOptions
- type ResetLease
- type ResetScope
- type RunAdmissionView
- type RunHandle
- type RunOperationView
- type RunRef
- type RuntimeDelta
- type RuntimeMessageAppend
- type RuntimeProgressAppend
- type Snapshot
- type SnapshotUpdate
- type SteerState
- type Subscription
- type TerminalRun
Constants ¶
const ( EventRuntimeSnapshot = "runtime_snapshot" EventRuntimeDelta = "runtime_delta" EventRuntimeDropped = "runtime_dropped" RunStatusRunning = "running" RunStatusAdmitting = "admitting" // RunStatusWaitingDecision keeps the admitted run active while its native // execution is parked on a durable approval or ask_user decision. RunStatusWaitingDecision = "waiting_decision" RunStatusAborting = "aborting" RunStatusCompleted = "completed" RunStatusAborted = "aborted" RunStatusErrored = "errored" RunStatusLost = "lost" SteerStatusPending = "pending" SteerStatusQueued = "queued" SteerStatusApplied = "applied" SteerStatusRejected = "rejected" RunOperationRetry = "retry" RunOperationEdit = "edit" CommandAbort = "abort" CommandSteer = "steer_current_run" CommandToolApprovalResponse = "tool_approval_response" CommandUserInputResponse = "user_input_response" CommandHistoryReset = "history_reset" CommandResult = "command_result" ResetScopeSession = "session" ResetScopeBot = "bot" )
Variables ¶
var ( // ErrSessionBusy means the session already has an active run and this is a // different invocation. SR-OWN-001 offers two ways to honour single-session // execution — durable queueing or a stable retryable busy answer — and this // runtime answers busy. It is an alias rather than a second sentinel so that // one errors.Is works whether the caller holds the ledger error or this one. // // The consequence is deliberate: the session runtime is not a general task // queue. Channel delivery and schedules already own a retry // mechanism, and reusing those is strictly better than teaching the runtime // to hold work for them, which would cost a queued state, a queue index, // promotion on every terminal transition, queued aborts, and a queue // projection on the client. ErrSessionBusy = ledger.ErrSessionBusy // ErrInvocationConflict means one invocation_id was submitted twice with // different canonical input. Per SR-ADM-001 that is a stable, identifiable // conflict rather than a new run: silently admitting it would give the // caller two runs for one retry. // // Both sentinels reach clients as apperror catalog codes — session_busy is // retryable, invocation_conflict is not. Domain code keeps the sentinel; // the HTTP boundary owns that mapping. ErrInvocationConflict = errors.New("runtime invocation conflicts with an earlier submission") // built without a ledger. Admission is durable-first by definition, so // there is no in-memory fallback to degrade to. ErrLedgerUnavailable = errors.New("session runtime ledger is not configured") // fence that claims depend on. Claiming without it would hand a run // ownership that PostgreSQL does not enforce, so the superseded owner would // keep writing history — the failure SR-OWN-002 exists to rule out. ErrFenceUnavailable = errors.New("session runtime persistence fence is not configured") // ErrSessionRuntimeSplit means the deployment declared more than one // instance but the live backend cannot arbitrate ownership between them. ErrSessionRuntimeSplit = errors.New("session runtime cluster mode requires a distributed backend") )
var ( ErrCommandTargetNotActive = errors.New("runtime command target is not active") ErrCommandTargetMismatch = errors.New("run does not belong to this session") ErrCommandExpired = errors.New("runtime command expired before acknowledgement") ErrCommandBusy = errors.New("runtime command executor is busy") ErrCommandPayloadConflict = errors.New("runtime command payload conflicts with an earlier request") ErrDecisionNotFound = errors.New("runtime decision was not found") ErrManagerClosed = errors.New("session runtime manager is closed") ErrRunOwnershipLost = errors.New("runtime run ownership was lost") ErrHistoryResetInProgress = errors.New("session history reset is in progress") ErrHistoryResetLeaseLost = runtimefence.ErrResetLeaseLost )
Functions ¶
This section is empty.
Types ¶
type ActiveRunUpdate ¶
type Admission ¶
type Admission struct {
RunID string
// TurnID and TurnPosition are decided at admission so the eventual terminal
// write targets a turn that already exists on paper.
TurnID string
TurnPosition int64
State ledger.State
// Replay is true when this invocation was already admitted. The rest of the
// struct then describes the original run, whatever state it has reached.
Replay bool
// Started is true when this call took ownership and began execution. It is
// false for a replay of a run someone else owns and for a replay of a
// finished run.
Started bool
// Cursor is the position at which this run became observable. A caller that
// both starts runs and subscribes to the session uses it to place the run in
// the stream it is already reading, rather than guessing whether a snapshot
// it holds predates its own submission (SR-OBS-002).
//
// It is set only when Started. A replay reserved nothing, so this call
// produced no position; where the session stands now is a different fact,
// and the subscription's snapshot is the authoritative answer to it.
Cursor Cursor
// Handle names the run this process now owns, including the fencing token
// its durable writes must carry. It is meaningful only when Started.
Handle RunHandle
}
Admission is the durable answer to one submission. It is safe to return to a caller as confirmation to stop retrying: everything in it is committed before the call returns.
type AdmitInput ¶
type AdmitInput struct {
BotID string
SessionID string
// InvocationID is the caller's retry identity. Callers that cannot mint one
// must derive it deterministically from their source (for example a channel
// message id), because a fresh id per attempt turns a retry into a second
// run.
InvocationID string
// Payload is the canonically encoded submission. Its fingerprint decides
// whether a repeated invocation_id is the same submit or a conflict, so the
// encoding must be stable for equal inputs — same key order, no timestamps,
// no request ids.
Payload []byte
// Execution is the owner-side plumbing used only if this call wins the
// claim. A replay that finds the run already owned never touches it.
Execution Execution
}
AdmitInput is one submission from a public entry point: HTTP, a channel adapter or a schedule. Every caller supplies the same three identities, which is what lets one admission path serve all of them.
type Backend ¶
type Backend interface {
Now(ctx context.Context) (time.Time, error)
// Load returns a snapshot the caller owns and may freely mutate.
Load(ctx context.Context, key Key) (Snapshot, bool, error)
Update(ctx context.Context, key Key, update SnapshotUpdate) (Snapshot, bool, error)
Publish(ctx context.Context, event Event) error
Subscribe(ctx context.Context, key Key) (Subscription, error)
Close() error
}
type Command ¶
type Command struct {
Type string `json:"type"`
ID string `json:"id,omitempty"`
ReplyOwnerID string `json:"reply_owner_id,omitempty"`
BotID string `json:"bot_id"`
SessionID string `json:"session_id"`
RunID string `json:"run_id"`
Generation string `json:"generation"`
FencingToken int64 `json:"fencing_token,omitempty"`
TargetID string `json:"target_id,omitempty"`
// DecisionResolved means the command target was resolved from PostgreSQL
// before routing. Owner-side execution must not consult the live UI
// projection again: it is derived state and may lag the durable decision.
DecisionResolved bool `json:"decision_resolved,omitempty"`
SteerID string `json:"steer_id,omitempty"`
Text string `json:"text,omitempty"`
Payload json.RawMessage `json:"payload,omitempty"`
PayloadHash string `json:"payload_hash,omitempty"`
ErrorCode string `json:"error_code,omitempty"`
Error string `json:"error,omitempty"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at,omitempty"`
}
type CommandSubscription ¶
type CommandSubscription struct {
C <-chan Command
Close func()
}
type CurrentRunPatch ¶
type CurrentRunPatch struct {
RunID string `json:"run_id"`
Status *string `json:"status,omitempty"`
ErrorCode *string `json:"error_code,omitempty"`
Error *string `json:"error,omitempty"`
Steer *SteerState `json:"steer,omitempty"`
UpdatedAt *time.Time `json:"updated_at,omitempty"`
OwnerLeaseExpiresAt *time.Time `json:"owner_lease_expires_at,omitempty"`
}
type CurrentRunView ¶
type CurrentRunView struct {
RunID string `json:"run_id" validate:"required" format:"uuid"`
// TurnID is the durable turn this run writes into, allocated at admission.
// It is part of the observable view because SR-OBS-003 requires every
// subscriber to agree on the run's turn, and a subscriber that only learns
// the run id cannot line the run up against persisted history.
TurnID string `json:"turn_id" validate:"required" format:"uuid"`
// InvocationID is the caller-supplied intent identity recorded at admission
// (session_runs.invocation_id). It rides the live view so the client that
// originated the send can match projection frames to its optimistic turn by
// reading the frame, instead of inferring the pairing from arrival timing
// while waiting for the acceptance that names the two. Subscribers that did
// not originate the run see an id unknown to them and treat the turn as
// foreign — which is the correct standalone rendering for cross-device runs.
InvocationID string `json:"invocation_id,omitempty"`
Generation string `json:"generation"`
Status string `json:"status"`
OwnerID string `json:"owner_id,omitempty"`
OwnerLeaseExpiresAt *time.Time `json:"owner_lease_expires_at,omitempty"`
StartedAt time.Time `json:"started_at"`
UpdatedAt time.Time `json:"updated_at"`
Messages []chatview.UIMessage `json:"messages"`
RequestUserTurn *chatview.UITurn `json:"request_user_turn,omitempty"`
ErrorCode string `json:"error_code,omitempty"`
Error string `json:"error,omitempty"`
Steer *SteerState `json:"steer,omitempty"`
Operation *RunOperationView `json:"operation,omitempty"`
}
type Cursor ¶
Cursor is a position in one session's observable event stream. Epoch and Seq travel as a pair because Seq restarts whenever the epoch does: a live backend that loses its state hands the session a new epoch, so comparing sequence numbers across epochs would order two unrelated streams against each other.
This is the position subscribers dedupe and recover on (SR-OBS-002). It is a different thing from ledger.Cursor, which is a keyset position in the reaper's sweep over durable rows.
type DecisionFenceActivator ¶
type DecisionFenceActivator interface {
ReclaimWaitingDecision(
ctx context.Context,
botID, sessionID, runID, ownerID, liveGeneration string,
previousToken, newToken int64,
decisionKind, decisionID string,
) error
}
DecisionFenceActivator advances a parked run's persistence fence while preserving the one pending decision that will resume it. Implementations must update the decision row to token in the same transaction that activates the session fence.
type DecisionResponse ¶
type DecisionResponse struct {
ControlID string
Type string
DecisionID string
BotID string
SessionID string
RunID string
Payload json.RawMessage
}
DecisionResponse is one transport-neutral answer. ControlID is minted by the caller and remains the command identity even after the addressed run leaves live state.
type DecisionResponseResult ¶
DecisionResponseResult separates "this is a runtime decision" from "the answer changed it". A resolved terminal decision is handled but not applied; an unfenced ACP/MCP request is not handled and follows its existing path.
type DecisionStore ¶
type DecisionStore interface {
ResolveRuntimeDecision(ctx context.Context, commandType, decisionID string) (DecisionTarget, error)
PendingRuntimeDecision(ctx context.Context, runID string) (DecisionTarget, bool, error)
}
DecisionStore is implemented by the application layer over the PostgreSQL decision tables. RouteDecisionResponse uses ResolveRuntimeDecision for every transport; recovery uses PendingRuntimeDecision to preserve exactly the decision that parked a run while advancing its fencing token.
type DecisionTarget ¶
type DecisionTarget struct {
Type string
ID string
BotID string
SessionID string
RunID string
TurnID string
Status string
FencingToken int64
ControlID string
PayloadHash string
}
DecisionTarget is the durable identity of one approval or user-input request. It is resolved from PostgreSQL and is the correctness boundary for decision routing; CurrentRunView.Messages is only a subscriber projection.
type DistributedBackend ¶
type DistributedBackend interface {
Backend
UpdateActiveRun(ctx context.Context, key Key, runID, generation string, update ActiveRunUpdate) (Snapshot, bool, error)
StartRun(ctx context.Context, key Key, ref RunRef, update SnapshotUpdate) (Snapshot, bool, error)
ReleaseRun(ctx context.Context, key Key, ref RunRef, update ActiveRunUpdate) (Snapshot, bool, error)
RenewLease(ctx context.Context, key Key, runID, ownerID, generation string, renewedAt, expiresAt time.Time) error
ValidateRunOwnership(ctx context.Context, key Key, ref RunRef) error
LoadRunRef(ctx context.Context, key Key, runID string) (RunRef, bool, error)
DeleteRunRef(ctx context.Context, ref RunRef) (bool, error)
PublishCommand(ctx context.Context, ownerID string, command Command) error
SubscribeCommands(ctx context.Context, ownerID string) (CommandSubscription, error)
StoreCommandResult(ctx context.Context, result Command, ttl time.Duration) error
LoadCommandResult(ctx context.Context, commandID string) (Command, bool, error)
}
DistributedBackend adds cross-process run ownership and command routing. MemoryBackend intentionally does not implement this interface.
type Event ¶
type Event struct {
Type string `json:"type"`
BotID string `json:"bot_id"`
SessionID string `json:"session_id"`
Epoch string `json:"epoch,omitempty"`
RunID string `json:"run_id,omitempty"`
Seq int64 `json:"seq"`
UpdatedAt *time.Time `json:"updated_at,omitempty"`
Snapshot *Snapshot `json:"snapshot,omitempty"`
Delta *RuntimeDelta `json:"delta,omitempty"`
Message string `json:"message,omitempty"`
}
type Execution ¶
type Execution struct {
// Admission persists the run's user turn and any replacement operation, and
// returns the view subscribers should see when the run leaves admitting. It
// runs after ownership is established, so its writes can be fenced with
// RunHandle.FencingToken.
Admission func(ctx context.Context, handle RunHandle) (RunAdmissionView, error)
// AbortCh and Cancel are how a routed abort reaches this run. A run without
// them is durably abortable but cannot be interrupted mid-turn.
AbortCh chan<- struct{}
Cancel context.CancelFunc
// InjectCh receives steering messages; nil means this caller does not
// support steering. The caller owns channel closure; the runtime only stops
// sending during teardown.
InjectCh chan<- turn.InjectMessage
// OwnershipCancel revokes the caller's execution context when the lease is
// lost, so a superseded owner stops producing output rather than racing the
// fence it can no longer pass.
OwnershipCancel context.CancelCauseFunc
}
Execution is what the winner of a claim needs in order to actually run the turn. It is part of the admit call rather than a second step because the two cannot be separated safely: live state reserved without an executor is a run holding a session's only slot with nothing driving it to a terminal state.
type FenceActivator ¶
type FenceActivator interface {
Activate(ctx context.Context, botID, sessionID string, token int64) error
}
FenceActivator hands durable persistence ownership to one run's fencing token. It is the second half of a claim: the ledger decides who owns the run, and this decides whose writes PostgreSQL will still accept (SR-OWN-002).
The token is the same one the claim wrote to session_runs, which is the whole point of drawing it from a shared monotonic sequence. Two independent orderings — one for run ownership, one for persistence — could disagree, and the disagreement would surface as a superseded owner still writing history.
It is expressed over (bot, session, token) rather than over a store handle so the runtime never learns that PostgreSQL exists; runtimefence.NewActivator satisfies it.
type HistoryResetBackend ¶
type HistoryResetBackend interface {
AcquireHistoryReset(ctx context.Context, scope ResetScope, token string, ttl time.Duration) (ResetLease, bool, error)
RenewHistoryReset(ctx context.Context, lease ResetLease, ttl time.Duration) (ResetLease, bool, error)
ReleaseHistoryReset(ctx context.Context, lease ResetLease) (bool, error)
EffectiveHistoryReset(ctx context.Context, scope ResetScope) (ResetLease, bool, error)
}
HistoryResetBackend provides a tokenized, expiring live gate. Redis uses key TTLs; MemoryBackend uses the same contract under its process mutex.
type HistoryResetHandler ¶
type HistoryResetHandler func(context.Context, ResetScope) error
HistoryResetHandler performs owner-local runtime teardown. Returning is the acknowledgement boundary: the ACP pool must not return until Session.Close and the owned process Close operation have completed.
type LeaseCandidate ¶
type LeaseCandidate struct {
Key Key
RunID string
FencingToken int64
// ExpiresAt is the deadline recorded in the index, sampled on the backend's
// clock rather than the reaper's.
ExpiresAt time.Time
}
LeaseCandidate is one entry of the live backend's lease index whose deadline has passed. It carries the fencing token that was current when the lease was written, which is what lets the reaper apply a fenced transition without trusting anything else about the vanished owner.
type LivenessBackend ¶
type LivenessBackend interface {
// LivenessGeneration identifies this incarnation of the live backend. It is
// created once and read by everyone, so a value different from the one
// stamped on a run means the backend that claimed that run is gone and its
// in-flight state cannot be recovered.
LivenessGeneration(ctx context.Context) (string, error)
// ExpiredLeaseCandidates returns up to limit entries whose lease deadline
// has passed. The backend samples its own clock: a reaper with a skewed
// clock must not be able to declare a healthy owner dead. Candidates are
// returned, not claimed — the leader is single, so a visibility timeout
// would add a second failure mode without removing one.
ExpiredLeaseCandidates(ctx context.Context, limit int64) ([]LeaseCandidate, error)
// ReleaseLeaseCandidate removes an index entry, but only while the entry
// still carries the token the candidate was read with. A run reclaimed
// between read and release keeps its new entry.
ReleaseLeaseCandidate(ctx context.Context, candidate LeaseCandidate) (bool, error)
// AcquireLeaderLease elects the single cluster-wide reaper. Renewal is the
// same call: an existing holder re-acquires its own lease. Failover is safe
// because every reaper duty is an idempotent fenced transition, so a new
// leader repeating work the old one had started changes nothing.
AcquireLeaderLease(ctx context.Context, ownerID string, ttl time.Duration) (bool, error)
// ReleaseLeaderLease hands leadership back on graceful shutdown so a peer
// takes over in one tick instead of one lease period.
ReleaseLeaderLease(ctx context.Context, ownerID string) error
}
LivenessBackend is the half of the runtime backend that answers "is an owner still alive". It is separate from Backend and DistributedBackend because liveness is the one thing PostgreSQL deliberately does not know: the ledger records ownership changes, never lease ticks, so only this interface can tell the reaper which runs to give up on.
Both backends implement it. On a memory backend there is a single process and therefore no lease to expire, but there is still an incarnation to compare against and orphans to repair, which is exactly how a single-instance install recovers after a crash.
type Manager ¶
type Manager struct {
// contains filtered or unexported fields
}
func NewManager ¶
func NewManagerFromConfig ¶
func NewManagerFromConfig(log *slog.Logger, cfg config.SessionRuntimeConfig, runs ledger.Store, fence FenceActivator) (*Manager, error)
NewManagerFromConfig is the single construction site where the two configured values turn into the full set of derived timings. runs is the durable ledger and fence is the persistence-ownership cutover that claims apply; they go together, and both may be nil only in tests that never admit a run.
func (*Manager) AbortControl ¶
func (m *Manager) AbortControl(ctx context.Context, botID, sessionID, runID, controlID string) (bool, error)
AbortControl aborts a run on behalf of a client that named the request with its own control id. Two things follow from that id and neither is available to plain Abort:
The abort becomes idempotent across instances. A client that retries — or that reconnects to a different server and retries there — gets the answer the first attempt produced rather than a second execution, because the control id keys the shared command result. Without it a retry after the run terminalized would report "not active" and contradict the ack the client already holds.
The intent is recorded durably before anything is routed, so a run that is aborted survives as aborted-on-purpose rather than as an unexplained cancellation, even if the owner never answers (SR-CTL-001).
func (*Manager) Admit ¶
Admit is the one durable entry point for starting a session run. It commits the admission before returning, so a caller that receives an Admission can stop retrying even if this server dies immediately afterwards (SR-ADM-001, SR-DUR-001).
The order is deliberate and not reorderable: persist, then claim, then execute. Reserving live state first would create a run that exists only in a process, which is the failure the ledger exists to prevent.
A caller that gets Started sees a run that is owned, fenced and reserved, and receives the handle its own durable writes must be fenced with. A caller that gets an Admission without Started is looking at a run someone else owns or one that already finished, and must not execute anything.
A session runs one turn at a time, so a different invocation arriving while one is active gets ErrSessionBusy and is expected to retry (SR-OWN-001). The runtime holds nothing on the caller's behalf.
func (*Manager) BeginBotHistoryReset ¶
func (*Manager) BeginSessionHistoryReset ¶
func (*Manager) DecisionContinuationContext ¶
func (m *Manager) DecisionContinuationContext(cmd Command) (context.Context, context.CancelFunc, error)
DecisionContinuationContext detaches the model continuation from the short command acknowledgement deadline while keeping it tied to the run owner's lifecycle and persistence fence.
func (*Manager) DispatchActiveCommand ¶
func (m *Manager) DispatchActiveCommand(ctx context.Context, botID, sessionID, commandType, targetID string, payload []byte) (bool, error)
DispatchActiveCommand is the legacy projection-based compatibility entry point used by older internal callers. New transports use RouteDecisionResponse and never use CurrentRunView.Messages for routing.
func (*Manager) DispatchRunCommand ¶
func (m *Manager) DispatchRunCommand(ctx context.Context, botID, sessionID, runID, commandType, targetID string, payload []byte) (bool, error)
DispatchRunCommand is the transport-facing decision route. In addition to the canonical decision id it checks the server-issued run id, preventing a stale UI response from being applied to a newer run in the same session.
func (*Manager) FinishRunWithErrorCode ¶
func (m *Manager) FinishRunWithErrorCode(ctx context.Context, handle RunHandle, status, errorCode string) error
FinishRunWithErrorCode records a stable public failure code without treating that code as a display message or persisting private provider diagnostics.
func (*Manager) HandleAgentEvent ¶
func (*Manager) HandleAgentEventWithStatus ¶
func (m *Manager) HandleAgentEventWithStatus(ctx context.Context, handle RunHandle, event native.StreamEvent) ([]chatview.UIMessage, string, error)
HandleAgentEventWithStatus applies an agent event and returns the run status from that same serialized mutation. Terminal callers use this instead of a second snapshot read so a routed control and terminal event have one winner even when a later backend read would fail.
func (*Manager) IsDistributed ¶
IsDistributed reports whether this manager coordinates owners through a cross-process backend. Memory managers intentionally return false.
func (*Manager) RouteDecisionResponse ¶
func (m *Manager) RouteDecisionResponse(ctx context.Context, response DecisionResponse) (DecisionResponseResult, error)
RouteDecisionResponse is the single decision entry point for WebSocket, HTTP, and gRPC callers. PostgreSQL resolves the decision before live ownership is consulted, while the client control id is checked first so a successful command remains replayable after the run terminalizes.
func (*Manager) SetCommandHandler ¶
SetCommandHandler installs the owner-local executor for routed runtime commands whose domain behavior lives outside the sessionruntime package.
func (*Manager) SetCommandReconciler ¶
SetCommandReconciler installs a read-only domain result checker. Unlike the owner-local command handler, it may run on any server after the owner or its local control disappears.
func (*Manager) SetDecisionStore ¶
func (m *Manager) SetDecisionStore(store DecisionStore)
SetDecisionStore installs the PostgreSQL-backed decision authority used by every response transport and by waiting-decision recovery.
func (*Manager) SetHistoryResetHandler ¶
func (m *Manager) SetHistoryResetHandler(handler HistoryResetHandler)
SetHistoryResetHandler installs the owner-local operation-boundary closer. SessionPool supplies it without sessionruntime depending on the ACP package.
func (*Manager) SetTerminalObserver ¶
func (m *Manager) SetTerminalObserver(observer func(context.Context, TerminalRun))
SetTerminalObserver installs the application-owned sink for authoritative durable run outcomes. The callback is invoked synchronously without holding the manager lock, and may be called more than once for the same run when a fenced terminal transition is replayed.
func (*Manager) SetTerminalReconciler ¶
SetTerminalReconciler installs a bounded application-owned repair pass for terminal runs whose observation was interrupted after the ledger commit. The elected reaper invokes it once per tick after its own terminal duties.
func (*Manager) Snapshot ¶
Snapshot returns the session's authoritative runtime view. The live backend answers first because it is the only place a run in flight exists; the ledger fallback below covers the case where it cannot answer at all.
func (*Manager) StartRun ¶
func (m *Manager) StartRun(ctx context.Context, botID, sessionID, runID string, abortCh chan<- struct{}, cancel context.CancelFunc, injectCh chan<- turn.InjectMessage) error
func (*Manager) StartRunHandle ¶
func (m *Manager) StartRunHandle(ctx context.Context, botID, sessionID, runID string, abortCh chan<- struct{}, cancel context.CancelFunc, injectCh chan<- turn.InjectMessage) (RunHandle, error)
func (*Manager) StartRunWithAdmissionBuilderAndOwnershipHandle ¶
func (m *Manager) StartRunWithAdmissionBuilderAndOwnershipHandle(ctx context.Context, botID, sessionID, runID string, builder func(context.Context, RunHandle) (RunAdmissionView, error), ownershipCancel context.CancelCauseFunc, abortCh chan<- struct{}, cancel context.CancelFunc, injectCh chan<- turn.InjectMessage) (RunHandle, error)
func (*Manager) StartRunWithAdmissionBuilderHandle ¶
func (m *Manager) StartRunWithAdmissionBuilderHandle(ctx context.Context, botID, sessionID, runID string, builder func(context.Context, RunHandle) (RunAdmissionView, error), abortCh chan<- struct{}, cancel context.CancelFunc, injectCh chan<- turn.InjectMessage) (RunHandle, error)
StartRunWithAdmissionBuilderHandle reserves the cross-server run before executing builder, then publishes the running view only after the canonical request turn and optional replacement operation are ready.
func (*Manager) ValidateRunOwnership ¶
ValidateRunOwnership fails closed before durable side effects when this process no longer owns the active runtime run.
func (*Manager) WaitDecisionContinuationReady ¶
WaitDecisionContinuationReady holds the resumed model call until the stream that produced the deferred decision has finished its terminal persistence. The decision itself is already committed and acknowledged; this barrier only prevents its tool result from racing the assistant tool-call write.
type MemoryBackend ¶
type MemoryBackend struct {
// contains filtered or unexported fields
}
func NewMemoryBackend ¶
func NewMemoryBackend() *MemoryBackend
func NewMemoryBackendWithTTL ¶
func NewMemoryBackendWithTTL(stateTTL time.Duration) *MemoryBackend
func (*MemoryBackend) AcquireHistoryReset ¶
func (b *MemoryBackend) AcquireHistoryReset(ctx context.Context, scope ResetScope, token string, ttl time.Duration) (ResetLease, bool, error)
func (*MemoryBackend) AcquireLeaderLease ¶
AcquireLeaderLease always succeeds: a single process is trivially the leader.
func (*MemoryBackend) Close ¶
func (b *MemoryBackend) Close() error
func (*MemoryBackend) EffectiveHistoryReset ¶
func (b *MemoryBackend) EffectiveHistoryReset(ctx context.Context, scope ResetScope) (ResetLease, bool, error)
func (*MemoryBackend) ExpiredLeaseCandidates ¶
func (*MemoryBackend) ExpiredLeaseCandidates(context.Context, int64) ([]LeaseCandidate, error)
ExpiredLeaseCandidates is always empty for a memory backend: one process owns every run it admitted, so a lease would only ever expire together with the process that was watching it.
func (*MemoryBackend) LivenessGeneration ¶
func (b *MemoryBackend) LivenessGeneration(context.Context) (string, error)
LivenessGeneration returns this process's incarnation. A memory backend keeps live state in the heap, so its incarnation ends with the process: after a restart every run the ledger still shows as active belongs to a generation that no longer exists, which is precisely the signal the recovery sweep needs.
func (*MemoryBackend) Publish ¶
func (b *MemoryBackend) Publish(ctx context.Context, event Event) error
func (*MemoryBackend) ReleaseHistoryReset ¶
func (b *MemoryBackend) ReleaseHistoryReset(ctx context.Context, lease ResetLease) (bool, error)
func (*MemoryBackend) ReleaseLeaderLease ¶
func (*MemoryBackend) ReleaseLeaderLease(context.Context, string) error
func (*MemoryBackend) ReleaseLeaseCandidate ¶
func (*MemoryBackend) ReleaseLeaseCandidate(context.Context, LeaseCandidate) (bool, error)
func (*MemoryBackend) RenewHistoryReset ¶
func (b *MemoryBackend) RenewHistoryReset(ctx context.Context, lease ResetLease, ttl time.Duration) (ResetLease, bool, error)
func (*MemoryBackend) StartRunIfNoHistoryReset ¶
func (b *MemoryBackend) StartRunIfNoHistoryReset(ctx context.Context, key Key, update SnapshotUpdate) (Snapshot, bool, error)
func (*MemoryBackend) Subscribe ¶
func (b *MemoryBackend) Subscribe(ctx context.Context, key Key) (Subscription, error)
func (*MemoryBackend) Update ¶
func (b *MemoryBackend) Update(ctx context.Context, key Key, update SnapshotUpdate) (Snapshot, bool, error)
type Options ¶
type Options struct {
OwnerID string
StateTTL time.Duration
OwnerLeaseTTL time.Duration
// BackendLossGrace comes from config because a Redis restart budget depends
// on the deployment. Everything else below OwnerLeaseTTL is derived.
BackendLossGrace time.Duration
// Cluster mirrors session_runtime.cluster.
Cluster bool
// Ledger is the durable session run store.
Ledger ledger.Store
// Fence applies the persistence-ownership cutover for a claimed run. It is
// required together with Ledger.
Fence FenceActivator
CommandAckTTL time.Duration
CommandWorkerLimit int
Logger *slog.Logger
EpochGenerator func() string
RunGenerationGenerator func() string
// ScanBatchSize and MaxScanBatchesPerTick exist so tests can watch recovery
// page rather than to be tuned in production; both default to package
// constants.
ScanBatchSize int32
MaxScanBatchesPerTick int
}
type Reaper ¶
type Reaper struct {
// contains filtered or unexported fields
}
Reaper is the single cluster-wide janitor for the session run ledger. Exactly one instance acts at a time, elected through a TTL lease in the live backend.
One leader rather than many workers is a deliberate simplification. The three duties below are all idempotent fenced transitions, so the only thing extra reapers would add is duplicate PostgreSQL traffic and a second way for a candidate to be consumed and then dropped by a crash. Failover costs at most one lease period and repeats at most one transition, which changes nothing.
The reaper is the only component that can decide a run is unrecoverable, because it is the only one that reads liveness and durable state together.
func NewReaper ¶
func NewReaper(runs ledger.Store, liveness LivenessBackend, tune tuning, ownerID string, logger *slog.Logger) *Reaper
NewReaper builds a reaper for one manager. It shares the manager's derived tuning so the leader lease and the tick that renews it cannot drift apart.
func (*Reaper) Close ¶
Close stops ticking and hands leadership back so a peer takes over in one tick instead of waiting out the lease.
func (*Reaper) SetTerminalObserver ¶
func (r *Reaper) SetTerminalObserver(observer func(context.Context, TerminalRun))
SetTerminalObserver installs the sink for authoritative durable outcomes.
func (*Reaper) SetTerminalReconciler ¶
SetTerminalReconciler installs the bounded repair pass run by the elected reaper after its ordinary ledger duties.
func (*Reaper) SetWaitingDecisionRecoverer ¶
func (r *Reaper) SetWaitingDecisionRecoverer(recoverer func(context.Context, LeaseCandidate) (bool, error))
SetWaitingDecisionRecoverer installs the owner-local half of parked-run recovery. It is set before Start, so the reaper never observes a partially configured callback.
type RedisBackend ¶
type RedisBackend struct {
// contains filtered or unexported fields
}
func NewRedisBackend ¶
func NewRedisBackend(ctx context.Context, opts RedisOptions) (*RedisBackend, error)
func (*RedisBackend) AcquireHistoryReset ¶
func (b *RedisBackend) AcquireHistoryReset(ctx context.Context, scope ResetScope, token string, ttl time.Duration) (ResetLease, bool, error)
func (*RedisBackend) AcquireLeaderLease ¶
func (*RedisBackend) CheckHealth ¶
func (b *RedisBackend) CheckHealth(ctx context.Context) error
func (*RedisBackend) Close ¶
func (b *RedisBackend) Close() error
func (*RedisBackend) DeleteRunRef ¶
func (*RedisBackend) EffectiveHistoryReset ¶
func (b *RedisBackend) EffectiveHistoryReset(ctx context.Context, scope ResetScope) (ResetLease, bool, error)
func (*RedisBackend) ExpiredLeaseCandidates ¶
func (b *RedisBackend) ExpiredLeaseCandidates(ctx context.Context, limit int64) ([]LeaseCandidate, error)
func (*RedisBackend) LivenessGeneration ¶
func (b *RedisBackend) LivenessGeneration(ctx context.Context) (string, error)
LivenessGeneration reads the shared incarnation marker, creating it with SET NX when absent so concurrently starting processes converge on one value instead of each minting its own. The marker lives and dies with the Redis dataset: a flushed or non-persistent Redis comes back without it, the next SET NX writes a new value, and every run stamped with the old one is unrecoverable by definition.
func (*RedisBackend) LoadCommandResult ¶
func (*RedisBackend) LoadRunRef ¶
func (*RedisBackend) Publish ¶
func (b *RedisBackend) Publish(ctx context.Context, event Event) error
func (*RedisBackend) PublishCommand ¶
func (*RedisBackend) ReleaseHistoryReset ¶
func (b *RedisBackend) ReleaseHistoryReset(ctx context.Context, lease ResetLease) (bool, error)
func (*RedisBackend) ReleaseLeaderLease ¶
func (b *RedisBackend) ReleaseLeaderLease(ctx context.Context, ownerID string) error
func (*RedisBackend) ReleaseLeaseCandidate ¶
func (b *RedisBackend) ReleaseLeaseCandidate(ctx context.Context, candidate LeaseCandidate) (bool, error)
ReleaseLeaseCandidate drops the index member only while it still carries the token it was read with. Because the token is part of the member string, the compare is free: a run reclaimed between read and release has a different member and this ZREM removes nothing.
func (*RedisBackend) ReleaseRun ¶
func (b *RedisBackend) ReleaseRun(ctx context.Context, key Key, ref RunRef, update ActiveRunUpdate) (Snapshot, bool, error)
func (*RedisBackend) RenewHistoryReset ¶
func (b *RedisBackend) RenewHistoryReset(ctx context.Context, lease ResetLease, ttl time.Duration) (ResetLease, bool, error)
func (*RedisBackend) RenewLease ¶
func (*RedisBackend) StartRun ¶
func (b *RedisBackend) StartRun(ctx context.Context, key Key, ref RunRef, update SnapshotUpdate) (Snapshot, bool, error)
func (*RedisBackend) StoreCommandResult ¶
func (*RedisBackend) Subscribe ¶
func (b *RedisBackend) Subscribe(ctx context.Context, key Key) (Subscription, error)
func (*RedisBackend) SubscribeCommands ¶
func (b *RedisBackend) SubscribeCommands(ctx context.Context, ownerID string) (CommandSubscription, error)
func (*RedisBackend) Update ¶
func (b *RedisBackend) Update(ctx context.Context, key Key, update SnapshotUpdate) (Snapshot, bool, error)
func (*RedisBackend) UpdateActiveRun ¶
func (b *RedisBackend) UpdateActiveRun(ctx context.Context, key Key, runID, generation string, update ActiveRunUpdate) (Snapshot, bool, error)
func (*RedisBackend) ValidateRunOwnership ¶
type ResetLease ¶
type ResetLease struct {
Scope ResetScope `json:"scope"`
Token string `json:"token"`
ExpiresAt time.Time `json:"expires_at"`
}
ResetLease is the live-backend half of the reset fence. The same token is used in PostgreSQL so renewal and release are successor-safe on both sides.
type ResetScope ¶
type ResetScope struct {
BotID string `json:"bot_id"`
SessionID string `json:"session_id,omitempty"`
}
ResetScope identifies the canonical history protected by a reset lease. SessionID is empty for a bot-wide reset.
type RunAdmissionView ¶
type RunAdmissionView struct {
RequestUserTurn *chatview.UITurn
Operation *RunOperationView
}
RunAdmissionView is the canonical state published when a reserved run becomes active. RequestUserTurn is intentionally runtime state rather than a durable history row; ordinary sends still persist user + assistant together when the run reaches a terminal result.
type RunHandle ¶
type RunHandle struct {
BotID string
SessionID string
RunID string
TurnID string
Generation string
// FencingToken is the ledger ownership token for this run. Callers need it
// to fence their own durable writes, which is why it travels with the
// handle rather than staying inside the runtime. It is zero for runs
// started through the pre-ledger entry points.
FencingToken int64
}
RunHandle identifies one admitted run. A run id can be reused by a client that replays an old reservation, so owner-side mutations also carry the generation.
type RunOperationView ¶
type RunRef ¶
type RunRef struct {
BotID string `json:"bot_id"`
SessionID string `json:"session_id"`
RunID string `json:"run_id"`
OwnerID string `json:"owner_id"`
Generation string `json:"generation"`
// FencingToken is the durable ownership token this reservation was made
// with. It is stored with the ref so the lease index entry can be written
// and removed from the backend's own copy, without a caller having to
// reconstruct a token it may no longer hold. Zero means the run has no
// ledger identity and therefore nothing for the reaper to transition.
FencingToken int64 `json:"fencing_token,omitempty"`
}
type RuntimeDelta ¶
type RuntimeDelta struct {
CurrentRunView *CurrentRunView `json:"current_run_view,omitempty"`
Run *CurrentRunPatch `json:"run,omitempty"`
MessageAppends []RuntimeMessageAppend `json:"message_appends,omitempty"`
ProgressAppends []RuntimeProgressAppend `json:"progress_appends,omitempty"`
MessageUpserts []chatview.UIMessage `json:"message_upserts,omitempty"`
ResetMessages bool `json:"reset_messages,omitempty"`
}
RuntimeDelta carries only the state changed by one committed runtime transition. Full snapshots are reserved for hydration and gap recovery.
type RuntimeMessageAppend ¶
type RuntimeMessageAppend struct {
ID int `json:"id"`
Type chatview.UIMessageType `json:"type"`
Content string `json:"content"`
}
type RuntimeProgressAppend ¶
type Snapshot ¶
type Snapshot struct {
BotID string `json:"bot_id"`
SessionID string `json:"session_id"`
Epoch string `json:"epoch"`
Seq int64 `json:"seq"`
CurrentRunView *CurrentRunView `json:"current_run_view,omitempty"`
UpdatedAt time.Time `json:"updated_at"`
}
Snapshot is the authoritative live view of one session. It holds at most one run: admission answers busy rather than queueing, so there is no pending list to project and a subscriber never has to reason about work it cannot see yet.
func EmptySnapshot ¶
EmptySnapshot returns the canonical empty runtime snapshot for a session.
type SnapshotUpdate ¶
type SteerState ¶
type Subscription ¶
type Subscription struct {
C <-chan Event
Close func()
}
type TerminalRun ¶
type TerminalRun struct {
RunID string
BotID string
SessionID string
FencingToken int64
State string
ErrorCode string
}
TerminalRun is the authoritative durable outcome of one admitted run. It is emitted only after the fenced session_runs transition has applied, or when a replay observes that the same run is already terminal. State uses the durable ledger vocabulary: completed, aborted, failed, or lost.
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
Package ledger is the durable half of the session runtime: the record of which runs were admitted, who owns them, and how they ended.
|
Package ledger is the durable half of the session runtime: the record of which runs were admitted, who owns them, and how they ended. |