Documentation
¶
Overview ¶
Package persis defines persistence contracts for Dagu's control plane.
Collection-backed records flow through Collection and Record. Domain-specific persistence contracts use dedicated store interfaces such as DAGDefinitionStore, DAGRunStore, and ProcStore.
Domain model changes for collection-backed records live inside Record.Data, leaving their physical schema unchanged.
Index ¶
- Constants
- Variables
- func DAGRunListReadBatchID(ctx context.Context) (uint64, bool)
- func Decode[T any](rec *Record, v *T) error
- func Encode(v any) ([]byte, error)
- func IsArtifactFileNameGlob(pattern string) bool
- func IsProcLockError(err error) bool
- func MatchArtifactFileName(path, pattern string) bool
- func NewProcLockError(err error) error
- func NewRunStateStore(repository *DAGRunRepository, prepared dagrun.Attempt) runstate.Store
- func WithDAGRunListReadBatch(ctx context.Context, batch *DAGRunListReadBatch) context.Context
- type ArtifactFile
- type ArtifactPage
- type ArtifactQuery
- type ArtifactRepository
- type ArtifactRun
- type ArtifactStore
- type Backend
- type Collection
- type DAGCatalog
- type DAGChangeEvent
- type DAGChangeType
- type DAGDefinition
- type DAGDefinitionStore
- type DAGEntry
- type DAGGrepResult
- type DAGListItem
- type DAGListOptions
- type DAGLoadOptions
- type DAGMatchSearchOptions
- type DAGRepository
- func (r *DAGRepository) Create(ctx context.Context, id string, source []byte) error
- func (r *DAGRepository) Delete(ctx context.Context, id string) error
- func (r *DAGRepository) GetDetails(ctx context.Context, id string, opts DAGLoadOptions) (*ir.DAG, error)
- func (r *DAGRepository) GetMetadata(ctx context.Context, id string) (*ir.DAG, error)
- func (r *DAGRepository) GetSpec(ctx context.Context, id string) (string, error)
- func (r *DAGRepository) Grep(ctx context.Context, pattern string) ([]*DAGGrepResult, []string, error)
- func (r *DAGRepository) IsSuspended(ctx context.Context, id string) (bool, error)
- func (r *DAGRepository) LabelList(ctx context.Context) ([]string, []string, error)
- func (r *DAGRepository) List(ctx context.Context, opts DAGListOptions) (pagination.PaginatedResult[DAGListItem], []string, error)
- func (r *DAGRepository) ListIncludingSearchPaths(ctx context.Context, opts DAGListOptions) (pagination.PaginatedResult[DAGListItem], []string, error)
- func (r *DAGRepository) LoadSpec(ctx context.Context, source []byte, name string, opts DAGLoadOptions) (*ir.DAG, error)
- func (r *DAGRepository) MigrateSuspensionState(ctx context.Context) error
- func (r *DAGRepository) Rename(ctx context.Context, oldID, newID string) error
- func (r *DAGRepository) SearchCursor(ctx context.Context, opts DAGSearchOptions) (*pagination.CursorResult[DAGSearchResult], []string, error)
- func (r *DAGRepository) SearchMatches(ctx context.Context, id string, opts DAGMatchSearchOptions) (*pagination.CursorResult[*textsearch.Match], error)
- func (r *DAGRepository) SetSuspended(ctx context.Context, id string, suspended bool) error
- func (r *DAGRepository) UpdateSpec(ctx context.Context, id string, source []byte) error
- type DAGRepositoryOptions
- type DAGRunCompareAndSwapOptions
- type DAGRunCompareAndSwapStatusRequest
- type DAGRunCreateAttemptOptions
- type DAGRunCreateAttemptRequest
- type DAGRunLatestAttemptOptions
- type DAGRunLatestAttemptQuery
- type DAGRunListOptions
- type DAGRunListReadBatch
- type DAGRunRemovalEnqueuer
- type DAGRunRemoveOptions
- type DAGRunRemoveRequest
- type DAGRunRepository
- func (r *DAGRunRepository) CancelFailedAutoRetryPendingRun(ctx context.Context, status *ir.DAGRunStatus) error
- func (r *DAGRunRepository) CompareAndSwapLatestAttemptStatus(ctx context.Context, dagRun ir.DAGRunRef, expectedAttemptID string, ...) (*ir.DAGRunStatus, bool, error)
- func (r *DAGRunRepository) CreateAttempt(ctx context.Context, dag *ir.DAG, timestamp time.Time, dagRunID string, ...) (dagrun.Attempt, error)
- func (r *DAGRunRepository) FindAttempt(ctx context.Context, ref ir.DAGRunRef) (dagrun.Attempt, error)
- func (r *DAGRunRepository) FindSubAttempt(ctx context.Context, root ir.DAGRunRef, childRunID string) (dagrun.Attempt, error)
- func (r *DAGRunRepository) LatestAttempt(ctx context.Context, name string, options DAGRunLatestAttemptOptions) (dagrun.Attempt, error)
- func (r *DAGRunRepository) ListRetryCandidates(ctx context.Context, from TimeInUTC) ([]*ir.DAGRunStatus, error)
- func (r *DAGRunRepository) ListStatuses(ctx context.Context, options DAGRunListOptions) ([]*ir.DAGRunStatus, error)
- func (r *DAGRunRepository) ListStatusesPage(ctx context.Context, options DAGRunListOptions) (DAGRunStatusPage, error)
- func (r *DAGRunRepository) MaterializeWorkDir(ctx context.Context, ref dagrun.WorkDirRef) (string, error)
- func (r *DAGRunRepository) OpenLog(ctx context.Context, path string) (io.ReadCloser, error)
- func (r *DAGRunRepository) RecentStatuses(ctx context.Context, name string, limit int) ([]ir.DAGRunStatus, error)
- func (r *DAGRunRepository) RemoveDAGRun(ctx context.Context, ref ir.DAGRunRef, options DAGRunRemoveOptions) error
- func (r *DAGRunRepository) RemoveOldDAGRuns(ctx context.Context, name string, retentionDays int, ...) ([]string, error)
- func (r *DAGRunRepository) ResolveRetryPath(ctx context.Context, root ir.DAGRunRef, targetRunID string, stepName string) (dagrun.RetryPath, *ir.DAGRunStatus, error)
- func (r *DAGRunRepository) SnapshotWorkDir(ctx context.Context, ref dagrun.WorkDirRef, localDir string) error
- type DAGRunRepositoryOptions
- type DAGRunRetentionOptions
- type DAGRunRetentionRequest
- type DAGRunStatusPage
- type DAGRunStatusQuery
- type DAGRunStore
- type DAGSearchOptions
- type DAGSearchResult
- type ListQuery
- type Page
- type ProcLockError
- type ProcRepository
- func (r *ProcRepository) Acquire(ctx context.Context, groupName string, meta proc.ProcMeta) (proc.ProcHandle, error)
- func (r *ProcRepository) CountAlive(ctx context.Context, groupName string) (int, error)
- func (r *ProcRepository) CountAliveByDAGName(ctx context.Context, groupName, dagName string) (int, error)
- func (r *ProcRepository) IsAttemptAlive(ctx context.Context, groupName string, dagRun ir.DAGRunRef, attemptID string) (bool, error)
- func (r *ProcRepository) IsRunAlive(ctx context.Context, groupName string, dagRun ir.DAGRunRef) (bool, error)
- func (r *ProcRepository) LatestFreshEntryByDAGName(ctx context.Context, groupName, dagName string) (*proc.ProcEntry, error)
- func (r *ProcRepository) LatestHeartbeat(ctx context.Context, groupName string, dagRun ir.DAGRunRef) (*proc.ProcHeartbeat, error)
- func (r *ProcRepository) ListAlive(ctx context.Context, groupName string) ([]ir.DAGRunRef, error)
- func (r *ProcRepository) ListAllAlive(ctx context.Context) (map[string][]ir.DAGRunRef, error)
- func (r *ProcRepository) ListAllEntries(ctx context.Context) ([]proc.ProcEntry, error)
- func (r *ProcRepository) ListEntries(ctx context.Context, groupName string) ([]proc.ProcEntry, error)
- func (r *ProcRepository) RemoveIfStale(ctx context.Context, entry proc.ProcEntry) error
- func (r *ProcRepository) Validate(ctx context.Context) error
- func (r *ProcRepository) WithLock(ctx context.Context, groupName string, fn func() error) error
- type ProcStore
- type Record
- type TimeInUTC
Constants ¶
const ( CollectionAPIKeys = "api_keys" CollectionActiveDistributedRuns = "active_distributed_runs" CollectionAgentSessionCleanups = "agent_session_cleanups" CollectionDAGRunLeases = "dag_run_leases" CollectionDAGSettings = "dag_settings" CollectionDAGState = "dag_state" CollectionDispatchTasks = "dispatch_tasks" CollectionIncidents = "incidents" CollectionLicense = "license" CollectionNotifications = "notifications" CollectionProfiles = "profiles" CollectionQueue = "queue" CollectionRemoteNodes = "remote_nodes" CollectionSchedulerState = "scheduler_state" CollectionSecrets = "secrets" CollectionUpgradeCheck = "upgrade_check" CollectionUsers = "users" CollectionViews = "views" CollectionWebhooks = "webhooks" CollectionWorkerHeartbeats = "worker_heartbeats" CollectionWorkspaces = "workspaces" )
Variables ¶
var ( // ErrInvalidArtifactCursor reports a cursor that does not belong to the // query it was presented with. ErrInvalidArtifactCursor = errors.New("invalid artifact cursor") // ErrInvalidArtifactFileName reports a file name pattern that is not a // valid glob. ErrInvalidArtifactFileName = errors.New("invalid artifact file name pattern") )
var ( ErrDAGAlreadyExists = errors.New("DAG already exists") ErrDAGNotFound = errors.New("DAG is not found") ErrDAGReadOnly = errors.New("DAG definition is read-only") )
var ( // ErrNotFound is returned when a requested record does not exist. ErrNotFound = errors.New("persis: record not found") // ErrConflict is returned when a conditional mutation cannot be applied. ErrConflict = errors.New("persis: compare-and-swap conflict") // ErrCorrupt is returned when a record exists but its stored representation // cannot be decoded by the backend. ErrCorrupt = errors.New("persis: corrupt record") )
Sentinel errors returned by Collection methods. Use errors.Is for matching; backends may wrap these with additional context.
var ErrInvalidDAGRunQueryCursor = errors.New("dagrun: invalid query cursor")
Functions ¶
func DAGRunListReadBatchID ¶ added in v2.14.0
DAGRunListReadBatchID returns the list-read batch ID associated with ctx.
func Decode ¶
Decode unmarshals rec.Data into v as JSON. Store adapters use this after receiving a Record from Collection.Get or Collection.List.
func Encode ¶
Encode marshals v to JSON. Store adapters use this before Collection.Put.
func IsArtifactFileNameGlob ¶ added in v2.17.0
IsArtifactFileNameGlob reports whether a file name pattern is a glob.
func IsProcLockError ¶ added in v2.14.0
IsProcLockError reports whether err represents process-group lock acquisition failure.
func MatchArtifactFileName ¶ added in v2.17.0
MatchArtifactFileName reports whether a run-relative artifact path matches a file name pattern. A pattern holding glob metacharacters is matched as a glob, the way artifact.list matches its own pattern; anything else is matched as a case-insensitive substring.
func NewProcLockError ¶ added in v2.14.0
NewProcLockError classifies a process-group lock-acquisition failure.
func NewRunStateStore ¶ added in v2.14.0
func NewRunStateStore(repository *DAGRunRepository, prepared dagrun.Attempt) runstate.Store
NewRunStateStore adapts a DAG-run repository to runtime execution state.
func WithDAGRunListReadBatch ¶ added in v2.14.0
func WithDAGRunListReadBatch(ctx context.Context, batch *DAGRunListReadBatch) context.Context
WithDAGRunListReadBatch associates a list-read batch with a context.
Types ¶
type ArtifactFile ¶ added in v2.17.0
type ArtifactFile struct {
// Path is relative to the run's artifact directory and uses forward
// slashes on every platform.
Path string
Size int64
}
ArtifactFile is one file within a run's artifact directory.
type ArtifactPage ¶ added in v2.17.0
type ArtifactPage struct {
Items []ArtifactRun
NextCursor string
}
ArtifactPage is one forward-only page of runs.
type ArtifactQuery ¶ added in v2.17.0
type ArtifactQuery struct {
// Name matches DAG names containing it, case-insensitively.
Name string
// FileName selects which of a run's files are returned. A pattern holding
// glob metacharacters is a glob; anything else is a case-insensitive
// substring. A run with no matching file is omitted.
FileName string
// From and To bound CreatedAt, not StartedAt.
From TimeInUTC
To TimeInUTC
WorkspaceFilter *workspace.WorkspaceFilter
Limit int
Cursor string
}
ArtifactQuery selects a page of runs.
type ArtifactRepository ¶ added in v2.17.0
type ArtifactRepository struct {
// contains filtered or unexported fields
}
ArtifactRepository provides application-level access to indexed DAG-run artifacts.
func NewArtifactRepository ¶ added in v2.17.0
func NewArtifactRepository(store ArtifactStore) *ArtifactRepository
NewArtifactRepository creates a repository backed by store.
func (*ArtifactRepository) List ¶ added in v2.17.0
func (r *ArtifactRepository) List(ctx context.Context, query ArtifactQuery) (ArtifactPage, error)
List returns one page of artifact files, newest run first.
type ArtifactRun ¶ added in v2.17.0
type ArtifactRun struct {
Name string
DAGRunID string
// CreatedAt is when the run's artifact directory was made, which is the
// value the date range filters and the listing orders on.
CreatedAt time.Time
// StartedAt is when the run began, which for a run that waited in a queue
// is later than CreatedAt.
StartedAt time.Time
// Files are sorted by path, and only those matching the query's FileName
// when one is set.
Files []ArtifactFile
// FilesTruncated reports that the run holds more files than were returned.
// Which of its files were returned is then not defined.
FilesTruncated bool
}
ArtifactRun is one DAG run together with the files it produced.
type ArtifactStore ¶ added in v2.17.0
type ArtifactStore interface {
QueryArtifacts(ctx context.Context, query ArtifactQuery) (ArtifactPage, error)
}
ArtifactStore lists root DAG runs that produced artifacts, newest first.
type Backend ¶
type Backend interface {
Collection(name string) Collection
}
Backend provides isolated, named control-plane collections. Collection names must be non-empty portable identifiers containing only ASCII letters, digits, hyphens, and underscores. Passing any other name is a programmer error and may panic.
type Collection ¶
type Collection interface {
// Get returns the record identified by id.
// Returns [ErrNotFound] if no record with that id exists.
Get(ctx context.Context, id string) (*Record, error)
// Put atomically creates or replaces a record.
Put(ctx context.Context, rec *Record) error
// Create atomically inserts rec. Returns [ErrConflict] when a record with
// rec.ID already exists. Implementations must guarantee the check-and-insert
// is atomic with respect to every concurrent mutation of the same record.
Create(ctx context.Context, rec *Record) error
// Delete removes the record with the given id.
// Returns nil if the record does not exist.
Delete(ctx context.Context, id string) error
// CompareAndDelete atomically removes expected.ID only when the current
// record still matches expected. Returns [ErrConflict] when it does not and
// [ErrNotFound] when the record does not exist.
CompareAndDelete(ctx context.Context, expected *Record) error
// List returns a page of records matching q, ordered by CreatedAt and then ID,
// both ascending.
List(ctx context.Context, q ListQuery) (*Page, error)
// CompareAndSwap atomically replaces record id only when its current Data
// bytes equal expected. Returns [ErrConflict] when they do not match and
// [ErrNotFound] when the record does not exist.
CompareAndSwap(ctx context.Context, id string, expected, next []byte) error
}
Collection is an isolated namespace of [Record]s. All methods must be safe for concurrent use. Mutations of the same record must be linearizable across clients sharing the physical namespace. Atomicity is per record; Collection does not define multi-record transactions. ErrConflict is reserved for failed conditional mutations, not transient backend write contention. Implementations map each collection to a distinct physical namespace.
type DAGCatalog ¶ added in v2.14.0
type DAGCatalog struct {
Items []DAGListItem
Issues []string
}
DAGCatalog is the unfiltered set of stored DAG definitions. Item order is unspecified.
type DAGChangeEvent ¶ added in v2.16.6
type DAGChangeEvent struct {
DAGEntry
Type DAGChangeType
}
DAGChangeEvent represents a DAG lifecycle event for a stored DAG definition.
type DAGChangeType ¶ added in v2.16.6
type DAGChangeType int
DAGChangeType identifies the kind of DAG lifecycle event.
const ( DAGChangeAdded DAGChangeType = iota DAGChangeUpdated DAGChangeDeleted )
type DAGDefinition ¶ added in v2.14.0
type DAGDefinition struct {
ID string
// SourcePath is the optional path where the definition was authored.
SourcePath string
Source []byte
}
DAGDefinition is a stored DAG specification.
type DAGDefinitionStore ¶ added in v2.14.0
type DAGDefinitionStore interface {
Create(ctx context.Context, id string, source []byte) error
Delete(ctx context.Context, id string) error
Get(ctx context.Context, id string) (DAGDefinition, error)
Update(ctx context.Context, id string, source []byte) error
Rename(ctx context.Context, oldID, newID string) error
GetMetadata(ctx context.Context, id string) (*ir.DAG, error)
Catalog(ctx context.Context) (DAGCatalog, error)
// CatalogIncludingSearchPaths is like Catalog but also includes DAG
// definitions found under the store's additional search paths (for example
// alt_dags_dir). Stores without additional search paths may return the same
// result as Catalog.
CatalogIncludingSearchPaths(ctx context.Context) (DAGCatalog, error)
MigrateSuspensionState(ctx context.Context) error
SetSuspended(ctx context.Context, id string, suspended bool) error
IsSuspended(ctx context.Context, id string) (bool, error)
}
DAGDefinitionStore persists definitions and exposes their listing projection.
type DAGEntry ¶ added in v2.16.6
DAGEntry pairs a DAG definition with its stable persistence identity.
type DAGGrepResult ¶ added in v2.14.0
type DAGGrepResult struct {
Name string
DAG *ir.DAG
Matches []*textsearch.Match
}
DAGGrepResult represents matches within one DAG definition.
type DAGListItem ¶ added in v2.14.0
DAGListItem contains a stable storage identity, metadata, and suspension state.
type DAGListOptions ¶ added in v2.14.0
type DAGListOptions struct {
Paginator *pagination.Paginator
Name string
Labels []string
ActiveOnly bool
Sort string
Order string
Time *time.Time
NextRunProjection func(*ir.DAG, time.Time) time.Time
WorkspaceFilter *workspace.WorkspaceFilter
}
DAGListOptions contains parameters for paginated DAG listing.
type DAGLoadOptions ¶ added in v2.14.0
type DAGLoadOptions struct {
// AllowBuildErrors returns a partially built DAG with its build errors attached.
AllowBuildErrors bool
}
DAGLoadOptions controls how a DAG specification is built.
type DAGMatchSearchOptions ¶ added in v2.14.0
type DAGMatchSearchOptions struct {
Cursor string
Limit int
Query string
Labels []string
WorkspaceFilter *workspace.WorkspaceFilter
}
DAGMatchSearchOptions contains parameters for loading snippets from one DAG.
type DAGRepository ¶ added in v2.14.0
type DAGRepository struct {
// contains filtered or unexported fields
}
DAGRepository provides application-level access to DAG definitions.
func NewDAGRepository ¶ added in v2.14.0
func NewDAGRepository(store DAGDefinitionStore, options DAGRepositoryOptions) *DAGRepository
NewDAGRepository creates a repository backed by the given definition store.
func (*DAGRepository) Delete ¶ added in v2.14.0
func (r *DAGRepository) Delete(ctx context.Context, id string) error
func (*DAGRepository) GetDetails ¶ added in v2.14.0
func (r *DAGRepository) GetDetails(ctx context.Context, id string, opts DAGLoadOptions) (*ir.DAG, error)
func (*DAGRepository) GetMetadata ¶ added in v2.14.0
func (*DAGRepository) Grep ¶ added in v2.14.0
func (r *DAGRepository) Grep(ctx context.Context, pattern string) ([]*DAGGrepResult, []string, error)
func (*DAGRepository) IsSuspended ¶ added in v2.14.0
func (*DAGRepository) List ¶ added in v2.14.0
func (r *DAGRepository) List(ctx context.Context, opts DAGListOptions) (pagination.PaginatedResult[DAGListItem], []string, error)
func (*DAGRepository) ListIncludingSearchPaths ¶ added in v2.15.0
func (r *DAGRepository) ListIncludingSearchPaths(ctx context.Context, opts DAGListOptions) (pagination.PaginatedResult[DAGListItem], []string, error)
ListIncludingSearchPaths is like List but also includes DAG definitions found under the store's additional search paths (for example alt_dags_dir). The combined collection is filtered, sorted, and paginated as a whole.
func (*DAGRepository) LoadSpec ¶ added in v2.14.0
func (r *DAGRepository) LoadSpec(ctx context.Context, source []byte, name string, opts DAGLoadOptions) (*ir.DAG, error)
func (*DAGRepository) MigrateSuspensionState ¶ added in v2.17.0
func (r *DAGRepository) MigrateSuspensionState(ctx context.Context) error
MigrateSuspensionState makes legacy suspension state available in primary storage without removing the legacy state.
func (*DAGRepository) Rename ¶ added in v2.14.0
func (r *DAGRepository) Rename(ctx context.Context, oldID, newID string) error
func (*DAGRepository) SearchCursor ¶ added in v2.14.0
func (r *DAGRepository) SearchCursor(ctx context.Context, opts DAGSearchOptions) (*pagination.CursorResult[DAGSearchResult], []string, error)
func (*DAGRepository) SearchMatches ¶ added in v2.14.0
func (r *DAGRepository) SearchMatches(ctx context.Context, id string, opts DAGMatchSearchOptions) (*pagination.CursorResult[*textsearch.Match], error)
func (*DAGRepository) SetSuspended ¶ added in v2.14.0
func (*DAGRepository) UpdateSpec ¶ added in v2.14.0
type DAGRepositoryOptions ¶ added in v2.14.0
DAGRepositoryOptions configures DAG specification loading.
type DAGRunCompareAndSwapOptions ¶ added in v2.14.0
type DAGRunCompareAndSwapOptions struct {
// RootDAGRun identifies the root containing a child attempt.
RootDAGRun ir.DAGRunRef
// ExpectedAttemptKey rejects an update when the persisted key differs.
ExpectedAttemptKey string
}
DAGRunCompareAndSwapOptions configures an atomic latest-attempt status update.
type DAGRunCompareAndSwapStatusRequest ¶ added in v2.14.0
type DAGRunCompareAndSwapStatusRequest struct {
DAGRun ir.DAGRunRef
RootDAGRun ir.DAGRunRef
ExpectedAttemptID string
ExpectedAttemptKey string
ExpectedStatus ir.Status
Mutate func(*ir.DAGRunStatus) error
}
DAGRunCompareAndSwapStatusRequest describes an atomic latest-attempt status update.
type DAGRunCreateAttemptOptions ¶ added in v2.14.0
type DAGRunCreateAttemptOptions struct {
// RootDAGRun identifies the root when creating a child attempt.
RootDAGRun ir.DAGRunRef
// Retry indicates that the attempt retries an existing DAG run.
Retry bool
// AttemptID uses a caller-assigned attempt identifier when non-empty.
AttemptID string
}
DAGRunCreateAttemptOptions configures creation of a run attempt.
type DAGRunCreateAttemptRequest ¶ added in v2.14.0
type DAGRunCreateAttemptRequest struct {
DAG *ir.DAG
RootDAGRun ir.DAGRunRef
Timestamp time.Time
DAGRunID string
AttemptID string
Retry bool
}
DAGRunCreateAttemptRequest identifies the run and attempt to create. A zero RootDAGRun creates a root run; a nonzero value creates its child.
type DAGRunLatestAttemptOptions ¶ added in v2.14.0
type DAGRunLatestAttemptOptions struct {
// AllHistory disables the repository's optional today-only lookup window.
AllHistory bool
}
DAGRunLatestAttemptOptions configures a latest-attempt lookup.
type DAGRunLatestAttemptQuery ¶ added in v2.14.0
DAGRunLatestAttemptQuery selects the newest visible attempt for a DAG.
type DAGRunListOptions ¶ added in v2.14.0
type DAGRunListOptions struct {
DAGRunID string
Name string
ExactName string
From TimeInUTC
To TimeInUTC
Statuses []ir.Status
Limit int
Cursor string
Labels []string
WorkspaceFilter *workspace.WorkspaceFilter
// AllHistory disables the implicit recent-time window when no range is set.
AllHistory bool
// Unbounded disables the repository's result limit.
Unbounded bool
}
DAGRunListOptions configures status listing.
type DAGRunListReadBatch ¶ added in v2.14.0
type DAGRunListReadBatch struct {
// contains filtered or unexported fields
}
DAGRunListReadBatch identifies list reads that may share the same storage work.
func NewDAGRunListReadBatch ¶ added in v2.14.0
func NewDAGRunListReadBatch() *DAGRunListReadBatch
NewDAGRunListReadBatch creates a new list-read batch.
type DAGRunRemovalEnqueuer ¶ added in v2.14.0
type DAGRunRemovalEnqueuer interface {
EnqueueDAGRunRemoval(ctx context.Context, root ir.DAGRunRef, resources []ir.AgentSessionResource) error
}
DAGRunRemovalEnqueuer durably records resources before their DAG-run history is removed.
type DAGRunRemoveOptions ¶ added in v2.14.0
type DAGRunRemoveOptions struct {
// RejectActive refuses to remove an active DAG run.
RejectActive bool
}
DAGRunRemoveOptions configures DAG-run removal.
type DAGRunRemoveRequest ¶ added in v2.14.0
DAGRunRemoveRequest identifies a DAG run to remove.
type DAGRunRepository ¶ added in v2.14.0
type DAGRunRepository struct {
// contains filtered or unexported fields
}
DAGRunRepository provides application-level access to persisted DAG runs.
func NewDAGRunRepository ¶ added in v2.14.0
func NewDAGRunRepository(store DAGRunStore, workDirs dagrun.WorkDirStore, options DAGRunRepositoryOptions) *DAGRunRepository
NewDAGRunRepository creates a repository backed by store.
func (*DAGRunRepository) CancelFailedAutoRetryPendingRun ¶ added in v2.14.0
func (r *DAGRunRepository) CancelFailedAutoRetryPendingRun( ctx context.Context, status *ir.DAGRunStatus, ) error
CancelFailedAutoRetryPendingRun marks the latest eligible failed attempt as aborted.
func (*DAGRunRepository) CompareAndSwapLatestAttemptStatus ¶ added in v2.14.0
func (r *DAGRunRepository) CompareAndSwapLatestAttemptStatus( ctx context.Context, dagRun ir.DAGRunRef, expectedAttemptID string, expectedStatus ir.Status, mutate func(*ir.DAGRunStatus) error, options DAGRunCompareAndSwapOptions, ) (*ir.DAGRunStatus, bool, error)
CompareAndSwapLatestAttemptStatus atomically updates a matching latest attempt.
func (*DAGRunRepository) CreateAttempt ¶ added in v2.14.0
func (r *DAGRunRepository) CreateAttempt( ctx context.Context, dag *ir.DAG, timestamp time.Time, dagRunID string, options DAGRunCreateAttemptOptions, ) (dagrun.Attempt, error)
CreateAttempt creates an attempt within a DAG run.
func (*DAGRunRepository) FindAttempt ¶ added in v2.14.0
func (r *DAGRunRepository) FindAttempt(ctx context.Context, ref ir.DAGRunRef) (dagrun.Attempt, error)
FindAttempt finds the latest visible attempt for a DAG run.
func (*DAGRunRepository) FindSubAttempt ¶ added in v2.14.0
func (r *DAGRunRepository) FindSubAttempt(ctx context.Context, root ir.DAGRunRef, childRunID string) (dagrun.Attempt, error)
FindSubAttempt finds the latest visible attempt for a child DAG run.
func (*DAGRunRepository) LatestAttempt ¶ added in v2.14.0
func (r *DAGRunRepository) LatestAttempt( ctx context.Context, name string, options DAGRunLatestAttemptOptions, ) (dagrun.Attempt, error)
LatestAttempt returns the newest visible attempt for a DAG.
func (*DAGRunRepository) ListRetryCandidates ¶ added in v2.14.0
func (r *DAGRunRepository) ListRetryCandidates(ctx context.Context, from TimeInUTC) ([]*ir.DAGRunStatus, error)
ListRetryCandidates returns failed latest attempts eligible for retry scanning.
func (*DAGRunRepository) ListStatuses ¶ added in v2.14.0
func (r *DAGRunRepository) ListStatuses(ctx context.Context, options DAGRunListOptions) ([]*ir.DAGRunStatus, error)
ListStatuses returns statuses in canonical list order.
func (*DAGRunRepository) ListStatusesPage ¶ added in v2.14.0
func (r *DAGRunRepository) ListStatusesPage(ctx context.Context, options DAGRunListOptions) (DAGRunStatusPage, error)
ListStatusesPage returns one forward-only page in canonical list order.
func (*DAGRunRepository) MaterializeWorkDir ¶ added in v2.14.0
func (r *DAGRunRepository) MaterializeWorkDir(ctx context.Context, ref dagrun.WorkDirRef) (string, error)
MaterializeWorkDir makes a DAG-run work directory available locally.
func (*DAGRunRepository) OpenLog ¶ added in v2.17.0
func (r *DAGRunRepository) OpenLog(ctx context.Context, path string) (io.ReadCloser, error)
OpenLog reads a regular log file up to its size when opened. The caller must close the reader. Missing files return an fs.ErrNotExist error.
func (*DAGRunRepository) RecentStatuses ¶ added in v2.14.0
func (r *DAGRunRepository) RecentStatuses(ctx context.Context, name string, limit int) ([]ir.DAGRunStatus, error)
RecentStatuses returns the newest readable status for recent DAG runs.
func (*DAGRunRepository) RemoveDAGRun ¶ added in v2.14.0
func (r *DAGRunRepository) RemoveDAGRun(ctx context.Context, ref ir.DAGRunRef, options DAGRunRemoveOptions) error
RemoveDAGRun removes a DAG run and all of its attempts.
func (*DAGRunRepository) RemoveOldDAGRuns ¶ added in v2.14.0
func (r *DAGRunRepository) RemoveOldDAGRuns( ctx context.Context, name string, retentionDays int, options DAGRunRetentionOptions, ) ([]string, error)
RemoveOldDAGRuns removes final DAG runs outside the configured retention policy.
func (*DAGRunRepository) ResolveRetryPath ¶ added in v2.14.0
func (r *DAGRunRepository) ResolveRetryPath( ctx context.Context, root ir.DAGRunRef, targetRunID string, stepName string, ) (dagrun.RetryPath, *ir.DAGRunStatus, error)
ResolveRetryPath resolves the ancestry of a persisted child DAG run.
func (*DAGRunRepository) SnapshotWorkDir ¶ added in v2.14.0
func (r *DAGRunRepository) SnapshotWorkDir(ctx context.Context, ref dagrun.WorkDirRef, localDir string) error
SnapshotWorkDir persists the current state of a DAG-run work directory.
type DAGRunRepositoryOptions ¶ added in v2.14.0
type DAGRunRepositoryOptions struct {
LatestStatusToday bool
Location *time.Location
Now func() time.Time
RemovalEnqueuer DAGRunRemovalEnqueuer
}
DAGRunRepositoryOptions configures application-level DAG-run behavior.
type DAGRunRetentionOptions ¶ added in v2.14.0
type DAGRunRetentionOptions struct {
DryRun bool
// RetentionRuns takes precedence over OlderThan and retention days when set.
RetentionRuns *int
// OlderThan takes precedence over retention days when RetentionRuns is unset.
OlderThan *time.Time
}
DAGRunRetentionOptions configures retention cleanup.
type DAGRunRetentionRequest ¶ added in v2.14.0
DAGRunRetentionRequest describes normalized DAG-run cleanup policy.
type DAGRunStatusPage ¶ added in v2.14.0
type DAGRunStatusPage struct {
Items []*ir.DAGRunStatus
NextCursor string
}
DAGRunStatusPage is one forward-only page of DAG-run statuses.
type DAGRunStatusQuery ¶ added in v2.14.0
type DAGRunStatusQuery struct {
DAGRunID string
Name string
ExactName string
From TimeInUTC
To TimeInUTC
Statuses []ir.Status
Limit int
Cursor string
Labels []string
WorkspaceFilter *workspace.WorkspaceFilter
}
DAGRunStatusQuery contains normalized backend filters for listing runs. Limit is zero for an unbounded query and positive otherwise.
type DAGRunStore ¶ added in v2.14.0
type DAGRunStore interface {
// OpenLog reads a regular log file up to its size when opened. The caller must close it.
OpenLog(ctx context.Context, path string) (io.ReadCloser, error)
CreateAttempt(ctx context.Context, req DAGRunCreateAttemptRequest) (dagrun.Attempt, error)
RecentStatuses(ctx context.Context, name string, limit int) ([]ir.DAGRunStatus, error)
LatestAttempt(ctx context.Context, query DAGRunLatestAttemptQuery) (dagrun.Attempt, error)
QueryStatuses(ctx context.Context, query DAGRunStatusQuery) (DAGRunStatusPage, error)
CompareAndSwapLatestAttemptStatus(ctx context.Context, req DAGRunCompareAndSwapStatusRequest) (*ir.DAGRunStatus, bool, error)
FindAttempt(ctx context.Context, ref ir.DAGRunRef) (dagrun.Attempt, error)
FindSubAttempt(ctx context.Context, root ir.DAGRunRef, childRunID string) (dagrun.Attempt, error)
RemoveOldDAGRuns(ctx context.Context, req DAGRunRetentionRequest) ([]ir.DAGRunRef, error)
RemoveDAGRun(ctx context.Context, req DAGRunRemoveRequest) error
}
DAGRunStore persists DAG runs and their attempts as one consistency boundary.
type DAGSearchOptions ¶ added in v2.14.0
type DAGSearchOptions struct {
Cursor string
Limit int
Query string
MatchLimit int
Labels []string
WorkspaceFilter *workspace.WorkspaceFilter
}
DAGSearchOptions contains parameters for cursor-based DAG search.
type DAGSearchResult ¶ added in v2.14.0
type DAGSearchResult struct {
Name string
FileName string
Workspace string
Matches []*textsearch.Match
HasMoreMatches bool
NextMatchesCursor string
}
DAGSearchResult is a lightweight DAG search hit.
type ListQuery ¶
type ListQuery struct {
// Prefix filters records whose ID starts with this string.
// An empty Prefix returns all records in the collection.
Prefix string
// Since is inclusive and Until is exclusive. Both bound results by
// Record.CreatedAt.
Since *time.Time
Until *time.Time
// Cursor resumes a previous [Page] iteration.
// Pass [Page.NextCursor] from the prior call; empty starts from the beginning.
Cursor string
// Limit caps the number of records returned. 0 uses the implementation default.
Limit int
}
ListQuery controls what Collection.List returns.
type Page ¶
Page is the result of a Collection.List call.
type ProcLockError ¶ added in v2.14.0
type ProcLockError struct {
// contains filtered or unexported fields
}
ProcLockError reports a failure to acquire a process-group lock.
func (*ProcLockError) Error ¶ added in v2.14.0
func (e *ProcLockError) Error() string
func (*ProcLockError) Unwrap ¶ added in v2.14.0
func (e *ProcLockError) Unwrap() error
type ProcRepository ¶ added in v2.14.0
type ProcRepository struct {
// contains filtered or unexported fields
}
ProcRepository provides application-level access to process liveness.
func NewProcRepository ¶ added in v2.14.0
func NewProcRepository(store ProcStore) *ProcRepository
NewProcRepository creates a process repository backed by store.
func (*ProcRepository) Acquire ¶ added in v2.14.0
func (r *ProcRepository) Acquire(ctx context.Context, groupName string, meta proc.ProcMeta) (proc.ProcHandle, error)
Acquire creates a process entry and starts its heartbeat.
func (*ProcRepository) CountAlive ¶ added in v2.14.0
CountAlive returns the number of distinct fresh DAG runs in a group.
func (*ProcRepository) CountAliveByDAGName ¶ added in v2.14.0
func (r *ProcRepository) CountAliveByDAGName(ctx context.Context, groupName, dagName string) (int, error)
CountAliveByDAGName returns the number of distinct fresh DAG runs for a DAG in a group.
func (*ProcRepository) IsAttemptAlive ¶ added in v2.14.0
func (r *ProcRepository) IsAttemptAlive(ctx context.Context, groupName string, dagRun ir.DAGRunRef, attemptID string) (bool, error)
IsAttemptAlive reports whether a DAG-run attempt has a fresh entry in the group.
func (*ProcRepository) IsRunAlive ¶ added in v2.14.0
func (r *ProcRepository) IsRunAlive(ctx context.Context, groupName string, dagRun ir.DAGRunRef) (bool, error)
IsRunAlive reports whether a DAG run has a fresh entry in the group.
func (*ProcRepository) LatestFreshEntryByDAGName ¶ added in v2.14.0
func (r *ProcRepository) LatestFreshEntryByDAGName(ctx context.Context, groupName, dagName string) (*proc.ProcEntry, error)
LatestFreshEntryByDAGName returns the newest fresh entry for a DAG in a group.
func (*ProcRepository) LatestHeartbeat ¶ added in v2.14.0
func (r *ProcRepository) LatestHeartbeat(ctx context.Context, groupName string, dagRun ir.DAGRunRef) (*proc.ProcHeartbeat, error)
LatestHeartbeat returns the latest heartbeat observation for a DAG run.
func (*ProcRepository) ListAlive ¶ added in v2.14.0
ListAlive returns distinct fresh DAG runs in canonical order.
func (*ProcRepository) ListAllAlive ¶ added in v2.14.0
ListAllAlive returns distinct fresh DAG runs grouped in canonical order.
func (*ProcRepository) ListAllEntries ¶ added in v2.14.0
ListAllEntries returns all entries, including stale entries.
func (*ProcRepository) ListEntries ¶ added in v2.14.0
func (r *ProcRepository) ListEntries(ctx context.Context, groupName string) ([]proc.ProcEntry, error)
ListEntries returns all entries in a group, including stale entries.
func (*ProcRepository) RemoveIfStale ¶ added in v2.14.0
RemoveIfStale removes the exact entry when it remains stale and unchanged.
type ProcStore ¶ added in v2.14.0
type ProcStore interface {
Validate(ctx context.Context) error
WithLock(ctx context.Context, groupName string, fn func() error) error
Acquire(ctx context.Context, groupName string, meta proc.ProcMeta) (proc.ProcHandle, error)
ListEntries(ctx context.Context, groupName string) ([]proc.ProcEntry, error)
LatestHeartbeat(ctx context.Context, groupName string, dagRun ir.DAGRunRef) (*proc.ProcHeartbeat, error)
ListAllEntries(ctx context.Context) ([]proc.ProcEntry, error)
RemoveIfStale(ctx context.Context, entry proc.ProcEntry) error
}
ProcStore persists process heartbeats for ProcRepository.
type Record ¶
Record is the storage primitive for collection-backed control-plane data.
ID uses "/" as a hierarchy separator so that a ListQuery.Prefix of "mydag/" returns all records whose IDs start with that prefix — enabling efficient tree traversal without storage-specific query syntax.
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
Package file provides filesystem persistence implementations.
|
Package file provides filesystem persistence implementations. |
|
artifact
Package artifact stores the per-run index of DAG-run artifact directories.
|
Package artifact stores the per-run index of DAG-run artifact directories. |
|
audit
Package audit provides a file-based implementation of the audit Store interface.
|
Package audit provides a file-based implementation of the audit Store interface. |
|
eventstore
Package eventstore provides a file-based implementation of the event store.
|
Package eventstore provides a file-based implementation of the event store. |
|
Package store consolidates small persistence stores that each wrap a persis.Collection.
|
Package store consolidates small persistence stores that each wrap a persis.Collection. |
|
Package testutil provides test helpers for the persistence layer.
|
Package testutil provides test helpers for the persistence layer. |