run

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 16, 2026 License: Apache-2.0 Imports: 21 Imported by: 0

Documentation

Index

Constants

View Source
const (
	WorkerObservationInterval   = 5 * time.Second
	WorkerObservationStaleAfter = 15 * time.Second
)
View Source
const DefaultIncrementalCheckpointNamespace = "normal"
View Source
const MaxEventInputBytes = 1024 * 1024

Variables

View Source
var (
	ErrNilPipeline                       = xerror.InvalidRequired(xerror.Pipeline)
	ErrContractMismatch                  = xerror.PipelineContractMismatch()
	ErrQueryOutputContractSchemaMismatch = xerror.QueryOutputContractSchemaMismatch()
)
View Source
var (
	ErrRefUnknownStep  = xerror.RefUnknownStep()
	ErrRefUnknownField = xerror.RefUnknownField()
	ErrRefStepNoResult = xerror.RefStepNoResult()
)
View Source
var (
	ErrTimelineNotFound              = xerror.NotFound(xerror.PipelineRun)
	ErrReplayCheckpointNotFound      = xerror.NotFound(xerror.PipelineCheckpoint)
	ErrIncrementalCheckpointNotFound = xerror.NotFound(xerror.PipelineCheckpoint)
)
View Source
var ErrSinkInputContractViolation = xerror.SinkInputContractViolation()

ErrSinkInputContractViolation is returned when the upstream step result does not satisfy the declared sink input contract.

Functions

func CheckStatusStrings

func CheckStatusStrings() []string

CheckStatusStrings returns a slice of all String values of the enum

func ContractData

func ContractData(source *StepResult) ([]string, []map[string]any, error)

func EffectiveRuntimePayloadContract

func EffectiveRuntimePayloadContract(
	name string,
	payload *pipepayload.Payload,
	declared *pipepayload.Contract,
) *pipepayload.Contract

func FailureReasonCodeStrings

func FailureReasonCodeStrings() []string

FailureReasonCodeStrings returns a slice of all String values of the enum

func FinalizeStructuralChecks

func FinalizeStructuralChecks(checks []StructuralCheck, at time.Time)

FinalizeStructuralChecks preserves completed checks after downstream failure.

func HTTPWatermarkTime

func HTTPWatermarkTime(value any) (time.Time, bool)

HTTPWatermarkTime parses HTTP progress without silently losing sub-nanosecond precision.

func HasContractFailure

func HasContractFailure(timeline RunTimeline) bool

HasContractFailure excludes unavailable evaluators and unrelated execution errors.

func IncrementalCheckpointStatusStrings

func IncrementalCheckpointStatusStrings() []string

IncrementalCheckpointStatusStrings returns a slice of all String values of the enum

func IncrementalCheckpointTransitionStatusStrings

func IncrementalCheckpointTransitionStatusStrings() []string

IncrementalCheckpointTransitionStatusStrings returns a slice of all String values of the enum

func ResolveParams

func ResolveParams(
	params map[string]any,
	completed map[slug.Slug]*StepResult,
) (map[string]any, error)

func RuntimeEdgeDiagnostics

func RuntimeEdgeDiagnostics(
	pipeline *pipelinespec.Pipeline,
	completedResults map[slug.Slug]*StepResult,
) []pipelinevalidate.EdgeCompatibilityResult

func SinkEffectStateStrings

func SinkEffectStateStrings() []string

SinkEffectStateStrings returns a slice of all String values of the enum

func StepOutputDetailKindStrings

func StepOutputDetailKindStrings() []string

StepOutputDetailKindStrings returns a slice of all String values of the enum

func ValidateQueryOutputContractAtRuntime

func ValidateQueryOutputContractAtRuntime(
	executionResult *result.ExecutionResult,
	contract *pipepayload.Contract,
) error

func ValidateSinkInputContract

func ValidateSinkInputContract(source *StepResult, contract *pipepayload.Contract) error

func WorkerLivenessStrings

func WorkerLivenessStrings() []string

WorkerLivenessStrings returns a slice of all String values of the enum

Types

type CheckEvidence

type CheckEvidence struct {
	Rows       int   `json:"rows"`
	DurationMs int64 `json:"duration_ms"`
	Empty      bool  `json:"empty"`
	// Completeness is unknown unless the connector explicitly reports truncation.
	BatchScope string `json:"batch_scope"`
}

type CheckStatus

type CheckStatus int
const (
	CheckStatusUnavailable CheckStatus = iota // unavailable
	CheckStatusPassed                         // passed
	CheckStatusFailed                         // failed
	CheckStatusSkipped                        // skipped
	CheckStatusError                          // error
)

func CheckStatusString

func CheckStatusString(s string) (CheckStatus, error)

CheckStatusString retrieves an enum value from the enum constants string name. Throws an error if the param is not part of the enum.

func CheckStatusValues

func CheckStatusValues() []CheckStatus

CheckStatusValues returns all values of the enum

func (CheckStatus) IsACheckStatus

func (i CheckStatus) IsACheckStatus() bool

IsACheckStatus returns "true" if the value is listed in the enum definition. "false" otherwise

func (CheckStatus) MarshalJSON

func (i CheckStatus) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for CheckStatus

func (CheckStatus) MarshalText

func (i CheckStatus) MarshalText() ([]byte, error)

MarshalText implements the encoding.TextMarshaler interface for CheckStatus

func (CheckStatus) String

func (i CheckStatus) String() string

func (*CheckStatus) UnmarshalJSON

func (i *CheckStatus) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for CheckStatus

func (*CheckStatus) UnmarshalText

func (i *CheckStatus) UnmarshalText(text []byte) error

UnmarshalText implements the encoding.TextUnmarshaler interface for CheckStatus

type CheckWindow

type CheckWindow = model.StructuralCheckWindow

type ContractFailureDiagnostic

type ContractFailureDiagnostic struct {
	ContractName   string                          `json:"contract_name"`
	FieldName      string                          `json:"field_name"`
	Rule           string                          `json:"rule"`
	ViolationCount int                             `json:"violation_count"`
	ReasonCode     pipelinevalidate.ContractReason `json:"reason_code"`
}

ContractFailureDiagnostic is a sanitized contract-validation failure summary.

type ContractFailureError

type ContractFailureError struct{ Diagnostic ContractFailureDiagnostic }

ContractFailureError carries only safe declaration diagnostics through wrapped errors.

func (*ContractFailureError) Error

func (e *ContractFailureError) Error() string

type EventInput

type EventInput struct {
	// contains filtered or unexported fields
}

EventInput is a validated, runtime-only JSON payload supplied after external verification.

func NewVerifiedJSONEventInput

func NewVerifiedJSONEventInput(key slug.Slug, document json.RawMessage) (*EventInput, error)

NewVerifiedJSONEventInput accepts bytes only after the caller verifies the external delivery.

func (*EventInput) Key

func (input *EventInput) Key() slug.Slug

func (*EventInput) Payload

func (input *EventInput) Payload() *pipepayload.Payload

func (*EventInput) Summary

func (input *EventInput) Summary() *EventInputSummary

type EventInputSummary

type EventInputSummary struct {
	Keys            []string                     `json:"keys"`
	Representation  model.ContractRepresentation `json:"representation"`
	SizeBytes       int64                        `json:"size_bytes"`
	DigestAlgorithm string                       `json:"digest_algorithm"`
	Digest          string                       `json:"digest"`
}

EventInputSummary is the only event-input data safe for run history.

type FailureContext

type FailureContext struct {
	OutcomeUncertain         bool                       `json:"outcome_uncertain,omitempty"`
	AutomaticRetryProhibited bool                       `json:"automatic_retry_prohibited,omitempty"`
	Step                     slug.Slug                  `json:"step"`
	Error                    string                     `json:"error"`
	At                       time.Time                  `json:"at"`
	Kind                     string                     `json:"kind,omitempty"`
	Side                     string                     `json:"side,omitempty"`
	ReasonCode               FailureReasonCode          `json:"reason_code,omitempty"`
	DataSourceSlug           string                     `json:"data_source_slug,omitempty"`
	ActionHint               string                     `json:"action_hint,omitempty"`
	Contract                 *ContractFailureDiagnostic `json:"contract,omitempty"`
}

FailureContext points to the failing step and a concise error message.

type FailureReasonCode

type FailureReasonCode int

FailureReasonCode identifies the bounded machine-readable cause of a failed run.

const (
	// FailureReasonCodeUnknown means the run failure does not have a stable reason code.
	FailureReasonCodeUnknown FailureReasonCode = iota // unknown
	// FailureReasonCodeDataSourceCredentialsFailed means connector authentication failed.
	FailureReasonCodeDataSourceCredentialsFailed // data_source_credentials_failed
	// FailureReasonCodeSourceQueryFailed means a source query failed during execution.
	FailureReasonCodeSourceQueryFailed // source_query_failed
	// FailureReasonCodeSourceResultShapeMismatch means source output violated its runtime contract.
	FailureReasonCodeSourceResultShapeMismatch // source_result_shape_mismatch
	// FailureReasonCodeTransformRuntimeFailed means sql_v1 transform execution failed.
	FailureReasonCodeTransformRuntimeFailed // transform_runtime_failed
	// FailureReasonCodeTransformContractMismatch means transform output violated its runtime contract.
	FailureReasonCodeTransformContractMismatch // transform_contract_mismatch
	// FailureReasonCodeSinkContractMismatch means sink input violated its runtime contract before writing.
	FailureReasonCodeSinkContractMismatch // sink_contract_mismatch
	// FailureReasonCodeS3ObjectWriteFailed means an S3/object sink write failed.
	FailureReasonCodeS3ObjectWriteFailed // s3_object_write_failed
	// FailureReasonCodeResourceScopeRejected means a selector or target violated the pinned resource scope.
	FailureReasonCodeResourceScopeRejected // resource_scope_rejected
	// FailureReasonCodeDataSourcePermissionDenied means the selected operation was denied.
	FailureReasonCodeDataSourcePermissionDenied // data_source_permission_denied
	// FailureReasonCodeDataSourceConnectionFailed means the endpoint could not be reached.
	FailureReasonCodeDataSourceConnectionFailed // data_source_connection_failed
	// FailureReasonCodeDataSourceTimeout means source execution exceeded its time limit.
	FailureReasonCodeDataSourceTimeout // data_source_timeout
	// FailureReasonCodeSourceQueryInvalid means the provider rejected the query.
	FailureReasonCodeSourceQueryInvalid // source_query_invalid
	// FailureReasonCodeSourceSchemaReferenceInvalid means a query schema reference could not resolve.
	FailureReasonCodeSourceSchemaReferenceInvalid // source_schema_reference_invalid
	// FailureReasonCodeWorkerOwnershipLost means an admitted worker attempt ended without completion.
	FailureReasonCodeWorkerOwnershipLost // worker_ownership_lost
	// FailureReasonCodeEventInputRejected means event input failed admission before execution.
	FailureReasonCodeEventInputRejected // event_input_rejected
	// FailureReasonCodeDestinationConstraintViolation means the destination rejected write data.
	FailureReasonCodeDestinationConstraintViolation // destination_constraint_violation
)

func FailureReasonCodeString

func FailureReasonCodeString(s string) (FailureReasonCode, error)

FailureReasonCodeString retrieves an enum value from the enum constants string name. Throws an error if the param is not part of the enum.

func FailureReasonCodeValues

func FailureReasonCodeValues() []FailureReasonCode

FailureReasonCodeValues returns all values of the enum

func (FailureReasonCode) IsAFailureReasonCode

func (i FailureReasonCode) IsAFailureReasonCode() bool

IsAFailureReasonCode returns "true" if the value is listed in the enum definition. "false" otherwise

func (FailureReasonCode) MarshalJSON

func (i FailureReasonCode) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for FailureReasonCode

func (FailureReasonCode) MarshalText

func (i FailureReasonCode) MarshalText() ([]byte, error)

MarshalText implements the encoding.TextMarshaler interface for FailureReasonCode

func (FailureReasonCode) String

func (i FailureReasonCode) String() string

func (*FailureReasonCode) UnmarshalJSON

func (i *FailureReasonCode) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for FailureReasonCode

func (*FailureReasonCode) UnmarshalText

func (i *FailureReasonCode) UnmarshalText(text []byte) error

UnmarshalText implements the encoding.TextUnmarshaler interface for FailureReasonCode

type IncrementalCheckpoint

type IncrementalCheckpoint struct {
	PipelineSlug        slug.Slug
	PipelineVersionID   int64
	SourceStepSlug      slug.Slug
	CheckpointNamespace string
	CheckpointKind      string
	DataSourceSlug      slug.Slug
	OperationType       string
	OperationRef        slug.Slug
	OperationInlineHash string
	OperationIdentity   string
	ObservedLowerBound  json.RawMessage
	ObservedUpperBound  json.RawMessage
	DisplayMetadata     json.RawMessage
	CommitStatus        IncrementalCheckpointStatus
	ObservedRunSlug     string
	CommittingRunSlug   string
	BlockedReason       string
	ObservedAt          time.Time
	CommittedAt         *time.Time
	SupersededAt        *time.Time
	BlockedAt           *time.Time
	ExpiredAt           *time.Time
	CreatedAt           time.Time
}

type IncrementalCheckpointCommitResult

type IncrementalCheckpointCommitResult struct {
	Status        string `json:"status"`
	BlockedByStep string `json:"blocked_by_step,omitempty"`
	BlockedReason string `json:"blocked_reason,omitempty"`
}

IncrementalCheckpointCommitResult describes checkpoint promotion for one source.

type IncrementalCheckpointInput

type IncrementalCheckpointInput struct {
	Lookup            IncrementalCheckpointLookup
	LowerBound        json.RawMessage
	UpperBound        json.RawMessage
	CurrentCheckpoint *IncrementalCheckpoint
	Window            *IncrementalSourceWindow
}

type IncrementalCheckpointLookup

type IncrementalCheckpointLookup struct {
	PipelineSlug         slug.Slug
	SourceStepSlug       slug.Slug
	CheckpointNamespace  string
	CheckpointKind       string
	DataSourceSlug       slug.Slug
	OperationType        string
	OperationRef         slug.Slug
	OperationInlineHash  string
	OperationIdentity    string
	PipelineVersionID    int64
	IncrementalField     string
	IncrementalCursor    string
	IncrementalOrdering  []string
	IncrementalDedupeKey string
	IncrementalLookback  string
}

type IncrementalCheckpointObservation

type IncrementalCheckpointObservation struct {
	SourceStepSlug      slug.Slug
	CheckpointNamespace string
	CheckpointKind      string
	ObservedLowerBound  json.RawMessage
	ObservedUpperBound  json.RawMessage
	DisplayMetadata     json.RawMessage
	ObservedAt          time.Time
}

type IncrementalCheckpointReader

type IncrementalCheckpointReader interface {
	GetCurrentIncrementalCheckpoint(
		ctx context.Context,
		orgSlug, projectSlug, environmentSlug slug.Slug,
		lookup IncrementalCheckpointLookup,
	) (*IncrementalCheckpoint, error)
}

type IncrementalCheckpointStatus

type IncrementalCheckpointStatus int
const (
	IncrementalCheckpointStatusUnknown    IncrementalCheckpointStatus = iota // unknown
	IncrementalCheckpointStatusCommitted                                     // committed
	IncrementalCheckpointStatusBlocked                                       // blocked
	IncrementalCheckpointStatusSuperseded                                    // superseded
)

func IncrementalCheckpointStatusString

func IncrementalCheckpointStatusString(s string) (IncrementalCheckpointStatus, error)

IncrementalCheckpointStatusString retrieves an enum value from the enum constants string name. Throws an error if the param is not part of the enum.

func IncrementalCheckpointStatusValues

func IncrementalCheckpointStatusValues() []IncrementalCheckpointStatus

IncrementalCheckpointStatusValues returns all values of the enum

func (IncrementalCheckpointStatus) IsAIncrementalCheckpointStatus

func (i IncrementalCheckpointStatus) IsAIncrementalCheckpointStatus() bool

IsAIncrementalCheckpointStatus returns "true" if the value is listed in the enum definition. "false" otherwise

func (IncrementalCheckpointStatus) MarshalJSON

func (i IncrementalCheckpointStatus) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for IncrementalCheckpointStatus

func (IncrementalCheckpointStatus) MarshalText

func (i IncrementalCheckpointStatus) MarshalText() ([]byte, error)

MarshalText implements the encoding.TextMarshaler interface for IncrementalCheckpointStatus

func (IncrementalCheckpointStatus) String

func (*IncrementalCheckpointStatus) UnmarshalJSON

func (i *IncrementalCheckpointStatus) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for IncrementalCheckpointStatus

func (*IncrementalCheckpointStatus) UnmarshalText

func (i *IncrementalCheckpointStatus) UnmarshalText(text []byte) error

UnmarshalText implements the encoding.TextUnmarshaler interface for IncrementalCheckpointStatus

type IncrementalCheckpointSummary

type IncrementalCheckpointSummary struct {
	Summary         string         `json:"summary"`
	DisplayMetadata map[string]any `json:"display_metadata,omitempty"`
	RawValueExposed bool           `json:"raw_value_exposed"`
}

IncrementalCheckpointSummary is a compact redacted checkpoint display value.

type IncrementalCheckpointTransition

type IncrementalCheckpointTransition struct {
	Status              IncrementalCheckpointTransitionStatus
	Lookup              IncrementalCheckpointLookup
	SourceStepSlug      slug.Slug
	CheckpointNamespace string
	CheckpointKind      string
	DataSourceSlug      slug.Slug
	OperationType       string
	OperationRef        slug.Slug
	OperationInlineHash string
	OperationIdentity   string
	ObservedLowerBound  json.RawMessage
	ObservedUpperBound  json.RawMessage
	DisplayMetadata     json.RawMessage
	ObservedRunSlug     string
	CommittingRunSlug   string
	BlockedByStep       slug.Slug
	BlockedReason       string
	ObservedAt          time.Time
}

type IncrementalCheckpointTransitionStatus

type IncrementalCheckpointTransitionStatus int
const (
	IncrementalCheckpointTransitionUnknown   IncrementalCheckpointTransitionStatus = iota // unknown
	IncrementalCheckpointTransitionCommitted                                              // committed
	IncrementalCheckpointTransitionBlocked                                                // blocked
)

func IncrementalCheckpointTransitionStatusString

func IncrementalCheckpointTransitionStatusString(s string) (IncrementalCheckpointTransitionStatus, error)

IncrementalCheckpointTransitionStatusString retrieves an enum value from the enum constants string name. Throws an error if the param is not part of the enum.

func IncrementalCheckpointTransitionStatusValues

func IncrementalCheckpointTransitionStatusValues() []IncrementalCheckpointTransitionStatus

IncrementalCheckpointTransitionStatusValues returns all values of the enum

func (IncrementalCheckpointTransitionStatus) IsAIncrementalCheckpointTransitionStatus

func (i IncrementalCheckpointTransitionStatus) IsAIncrementalCheckpointTransitionStatus() bool

IsAIncrementalCheckpointTransitionStatus returns "true" if the value is listed in the enum definition. "false" otherwise

func (IncrementalCheckpointTransitionStatus) MarshalJSON

func (i IncrementalCheckpointTransitionStatus) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for IncrementalCheckpointTransitionStatus

func (IncrementalCheckpointTransitionStatus) MarshalText

func (i IncrementalCheckpointTransitionStatus) MarshalText() ([]byte, error)

MarshalText implements the encoding.TextMarshaler interface for IncrementalCheckpointTransitionStatus

func (IncrementalCheckpointTransitionStatus) String

func (*IncrementalCheckpointTransitionStatus) UnmarshalJSON

func (i *IncrementalCheckpointTransitionStatus) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for IncrementalCheckpointTransitionStatus

func (*IncrementalCheckpointTransitionStatus) UnmarshalText

func (i *IncrementalCheckpointTransitionStatus) UnmarshalText(text []byte) error

UnmarshalText implements the encoding.TextUnmarshaler interface for IncrementalCheckpointTransitionStatus

type IncrementalCheckpointWindowSummary

type IncrementalCheckpointWindowSummary struct {
	Summary         string         `json:"summary"`
	LowerBound      string         `json:"lower_bound,omitempty"`
	UpperBound      string         `json:"upper_bound,omitempty"`
	DisplayMetadata map[string]any `json:"display_metadata,omitempty"`
	RawValueExposed bool           `json:"raw_value_exposed"`
}

IncrementalCheckpointWindowSummary summarizes the effective source read window.

type IncrementalReplaySafety

type IncrementalReplaySafety struct {
	Status        string `json:"status"`
	BlockedSink   string `json:"blocked_sink,omitempty"`
	BlockedReason string `json:"blocked_reason,omitempty"`
}

IncrementalReplaySafety exposes why an automatic replay is or is not blocked.

type IncrementalRunDiagnostics

type IncrementalRunDiagnostics struct {
	Sources      []IncrementalSourceDiagnostic `json:"sources"`
	ReplaySafety IncrementalReplaySafety       `json:"replay_safety"`
}

IncrementalRunDiagnostics contains sanitized per-source checkpoint diagnostics.

type IncrementalRunOptions

type IncrementalRunOptions struct {
	CheckpointNamespace  string
	SourceWindows        map[slug.Slug]IncrementalSourceWindow
	SkipCheckpointCommit bool
	PromoteCheckpoint    bool
	BackfillSeriesSlug   slug.Slug
	BackfillChunkIndex   int
	BackfillChunkCount   int
	TriggerSource        string
}

type IncrementalRunSummary

type IncrementalRunSummary struct {
	Mode                string `json:"mode"`
	CheckpointStatus    string `json:"checkpoint_status"`
	SourceStepSlug      string `json:"source_step_slug,omitempty"`
	BlockedReason       string `json:"blocked_reason,omitempty"`
	ReplaySafetyStatus  string `json:"replay_safety_status,omitempty"`
	ReplayBlockedReason string `json:"replay_blocked_reason,omitempty"`
	BackfillSeriesSlug  string `json:"backfill_series_slug,omitempty"`
	BackfillChunkIndex  int    `json:"backfill_chunk_index,omitempty"`
	BackfillChunkCount  int    `json:"backfill_chunk_count,omitempty"`
}

IncrementalRunSummary is the operator-facing run-level incremental state.

type IncrementalSourceDiagnostic

type IncrementalSourceDiagnostic struct {
	SourceStepSlug     string                             `json:"source_step_slug"`
	CheckpointKind     string                             `json:"checkpoint_kind"`
	PreviousCheckpoint IncrementalCheckpointSummary       `json:"previous_checkpoint"`
	Window             IncrementalCheckpointWindowSummary `json:"window"`
	ObservedCheckpoint IncrementalCheckpointSummary       `json:"observed_checkpoint"`
	CommitResult       IncrementalCheckpointCommitResult  `json:"commit_result"`
}

IncrementalSourceDiagnostic summarizes a single incremental source step.

type IncrementalSourceWindow

type IncrementalSourceWindow struct {
	LowerBound      json.RawMessage `json:"lower_bound,omitempty"`
	UpperBound      json.RawMessage `json:"upper_bound,omitempty"`
	DisplayMetadata json.RawMessage `json:"display_metadata,omitempty"`
}

type ReplayCheckpoint

type ReplayCheckpoint struct {
	DeploymentPlan             []byte `json:"deployment_plan,omitempty"`
	DeploymentResolutionDigest string `json:"deployment_resolution_digest,omitempty"`
	CompiledPlan               []byte `json:"compiled_plan,omitempty"`
	CompiledPlanDigest         string `json:"compiled_plan_digest,omitempty"`
	RunSlug                    string
	OrgSlug                    slug.Slug
	ProjectSlug                slug.Slug
	EnvironmentSlug            slug.Slug
	StepSlug                   slug.Slug
	Pipeline                   *pipelinespec.Pipeline
	Results                    map[slug.Slug]*StepResult
	IncrementalSourceWindows   map[slug.Slug]IncrementalSourceWindow
}

ReplayCheckpoint stores pinned input needed to reproduce a run from a step boundary.

type RunMetadata

type RunMetadata struct {
	DeploymentID               *int64                `json:"-"`
	DeploymentRevision         *int64                `json:"-"`
	DeploymentResolutionDigest string                `json:"deployment_resolution_digest,omitempty"`
	ProjectSlug                slug.Slug             `json:"project_slug"`
	EnvironmentSlug            slug.Slug             `json:"environment_slug"`
	RunSlug                    string                `json:"run_slug"`
	Source                     RunSourceSummary      `json:"source"`
	Status                     string                `json:"status"`
	DurationMs                 int64                 `json:"duration_ms"`
	StepCount                  int                   `json:"step_count"`
	PipelineVersionID          int64                 `json:"pipeline_version_id"`
	FailedStep                 *slug.Slug            `json:"failed_step,omitempty"`
	ReplayOfRunSlug            *string               `json:"replay_of_run_slug,omitempty"`
	ReplayFromStep             *string               `json:"replay_from_step,omitempty"`
	Incremental                IncrementalRunSummary `json:"incremental"`
}

RunMetadata captures stable run information for local execution output.

type RunPersister

type RunPersister interface {
	PersistRun(
		ctx context.Context,
		orgSlug, projectSlug, environmentSlug slug.Slug,
		pipeline *pipelinespec.Pipeline,
		runResult *RunResult,
	) error
}

RunPersister stores pipeline run metadata and timeline externally.

type RunResult

type RunResult struct {
	Metadata                         RunMetadata                                `json:"metadata"`
	Timeline                         RunTimeline                                `json:"timeline"`
	StepResults                      map[slug.Slug]*StepResult                  `json:"step_results,omitempty"`
	EdgeDiagnostics                  []pipelinevalidate.EdgeCompatibilityResult `json:"edge_diagnostics,omitempty"`
	IncrementalCheckpointTransitions []IncrementalCheckpointTransition          `json:"incremental_checkpoint_transitions,omitempty"`
	IncrementalDiagnostics           IncrementalRunDiagnostics                  `json:"incremental_diagnostics"`
}

RunResult contains run metadata, timeline, and per-step execution results.

type RunSourceScheduleSummary

type RunSourceScheduleSummary struct {
	TriggerSlug     slug.Slug                                  `json:"trigger_slug"      validate:"required"`
	TriggerRevision int64                                      `json:"trigger_revision"  validate:"required,min=1"`
	Cron            string                                     `json:"cron"              validate:"required"`
	Timezone        string                                     `json:"timezone"          validate:"required"`
	ScheduledFireAt time.Time                                  `json:"scheduled_fire_at" validate:"required"`
	ActualFireAt    time.Time                                  `json:"actual_fire_at"    validate:"required"`
	MisfireResult   model.PipelineTriggerScheduleMisfireResult `json:"misfire_result"    validate:"required,schedulemisfireresult"`
}

RunSourceScheduleSummary stores safe schedule trigger context for scheduled runs.

type RunSourceSummary

type RunSourceSummary struct {
	Type            model.PipelineRunSourceType       `json:"type"       validate:"required,runsourcetype"`
	Summary         string                            `json:"summary"    validate:"required"`
	EnvironmentSlug slug.Slug                         `json:"environment_slug,omitempty"`
	Entrypoint      model.PipelineRunSourceEntrypoint `json:"entrypoint" validate:"required,runsourceentrypoint"`
	ActorType       model.EventActorType              `json:"actor_type" validate:"required,eventactortype"`
	ActorSlug       *slug.Slug                        `json:"actor_slug,omitempty"`
	RequestID       *string                           `json:"request_id,omitempty"`
	TraceID         *string                           `json:"trace_id,omitempty"`
	ParameterKeys   []string                          `json:"parameter_keys"`
	SourceEventSlug *slug.Slug                        `json:"source_event_slug,omitempty"`
	AttemptNumber   int                               `json:"attempt_number"`
	RootRunSlug     slug.Slug                         `json:"root_run_slug,omitempty"`
	RetryOfRunSlug  *slug.Slug                        `json:"retry_of_run_slug,omitempty"`
	Schedule        *RunSourceScheduleSummary         `json:"schedule,omitempty"`
	Webhook         *RunSourceWebhookSummary          `json:"webhook,omitempty"`
}

RunSourceSummary stores safe immutable context explaining why a run started.

func NormalizeAttemptSource

func NormalizeAttemptSource(source RunSourceSummary, runSlug slug.Slug) RunSourceSummary

NormalizeAttemptSource fills the run-attempt linkage required for persisted source summaries.

type RunSourceWebhookSummary

type RunSourceWebhookSummary struct {
	DeliveryRecordID   int64     `json:"-"`
	TriggerSlug        slug.Slug `json:"trigger_slug"        validate:"required"`
	TriggerRevision    int64     `json:"trigger_revision"    validate:"required,min=1"`
	DeliveryID         string    `json:"delivery_id"         validate:"required,max=128"`
	PayloadDigest      string    `json:"payload_digest"      validate:"required,len=64,hexadecimal"`
	PayloadSizeBytes   int64     `json:"payload_size_bytes"  validate:"min=0"`
	ReceivedAt         time.Time `json:"received_at"         validate:"required"`
	VerificationStatus string    `json:"verification_status" validate:"required,eq=verified"`
}

RunSourceWebhookSummary stores safe webhook delivery context without payload values.

type RunStore

RunStore persists completed runs and provides replay state access.

type RunTimeline

type RunTimeline struct {
	SinkReplayIdentity         string                            `json:"sink_replay_identity,omitempty"`
	CompiledPlanDigest         string                            `json:"compiled_plan_digest,omitempty"`
	WorkerObservedAt           *time.Time                        `json:"worker_observed_at,omitempty"`
	Revision                   int64                             `json:"revision,omitempty"`
	ExecutionContext           *pipelineruntime.ExecutionContext `json:"execution_context,omitempty"`
	StructuralChecks           []StructuralCheck                 `json:"structural_checks,omitempty"`
	DeploymentResolutionDigest string                            `json:"deployment_resolution_digest,omitempty"`
	Admissions                 []deploymentplan.Admission        `json:"admissions,omitempty"`
	RunSlug                    string                            `json:"run_slug"`
	PipelineSlug               slug.Slug                         `json:"pipeline_slug"`
	ProjectSlug                slug.Slug                         `json:"project_slug"`
	EnvironmentSlug            slug.Slug                         `json:"environment_slug"`
	Status                     string                            `json:"status"`
	ReplayOfRunSlug            *string                           `json:"replay_of_run_slug,omitempty"`
	Events                     []TimelineEvent                   `json:"events"`
	Snapshots                  []StepSnapshot                    `json:"snapshots"`
	Failure                    *FailureContext                   `json:"failure,omitempty"`
}

RunTimeline contains a structured event stream and sanitized step snapshots.

type SinkEffect

type SinkEffect struct {
	Claimed   bool
	State     SinkEffectState
	StartedAt time.Time
}

SinkEffect stores the persisted state needed to decide a retry safely.

type SinkEffectAttempt

type SinkEffectAttempt struct {
	ReplayIdentity    string
	SinkStepSlug      slug.Slug
	IdempotencyKey    string
	IdempotencyClass  string
	ReplayMethod      string
	ReplayDecision    string
	DiagnosticCode    string
	DiagnosticSummary string
}

SinkEffectAttempt is the safe, durable identity for one sink delivery. It deliberately excludes payload values, credentials, and the idempotency key from run diagnostics; the key is used only as the repository identity.

type SinkEffectLedger

type SinkEffectLedger interface {
	// BeginOrGet claims an effect before dispatching it, or returns the previous
	// claim for the same replay identity. A returned succeeded effect is skipped;
	// an existing in-progress effect is replay-ambiguous and callers must apply
	// its replay policy instead of blindly dispatching the sink again.
	BeginOrGet(
		ctx context.Context,
		orgSlug, projectSlug, environmentSlug slug.Slug,
		attempt SinkEffectAttempt,
	) (*SinkEffect, error)
	// MarkSucceeded records that a claimed effect completed so later retries and
	// replays of the same identity can skip its already-applied side effect.
	MarkSucceeded(
		ctx context.Context,
		orgSlug, projectSlug, environmentSlug slug.Slug,
		attempt SinkEffectAttempt,
	) error
}

SinkEffectLedger creates a durable pre-effect attempt and records a returned success. It is intentionally transaction-neutral: a side effect can happen before the terminal pipeline-run row is persisted.

type SinkEffectState

type SinkEffectState int

SinkEffectState records the durable lifecycle of an outbound sink effect.

const (
	SinkEffectStateUnknown    SinkEffectState = iota // unknown
	SinkEffectStateInProgress                        // in_progress
	SinkEffectStateSucceeded                         // succeeded
)

func SinkEffectStateString

func SinkEffectStateString(s string) (SinkEffectState, error)

SinkEffectStateString retrieves an enum value from the enum constants string name. Throws an error if the param is not part of the enum.

func SinkEffectStateValues

func SinkEffectStateValues() []SinkEffectState

SinkEffectStateValues returns all values of the enum

func (SinkEffectState) IsASinkEffectState

func (i SinkEffectState) IsASinkEffectState() bool

IsASinkEffectState returns "true" if the value is listed in the enum definition. "false" otherwise

func (SinkEffectState) MarshalJSON

func (i SinkEffectState) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for SinkEffectState

func (SinkEffectState) MarshalText

func (i SinkEffectState) MarshalText() ([]byte, error)

MarshalText implements the encoding.TextMarshaler interface for SinkEffectState

func (SinkEffectState) String

func (i SinkEffectState) String() string

func (*SinkEffectState) UnmarshalJSON

func (i *SinkEffectState) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for SinkEffectState

func (*SinkEffectState) UnmarshalText

func (i *SinkEffectState) UnmarshalText(text []byte) error

UnmarshalText implements the encoding.TextUnmarshaler interface for SinkEffectState

type StepInputSnapshot

type StepInputSnapshot struct {
	OrganizationSlug    slug.Slug          `json:"organization_slug"`
	ProjectSlug         slug.Slug          `json:"project_slug"`
	EnvironmentSlug     slug.Slug          `json:"environment_slug"`
	DataSourceSlug      slug.Slug          `json:"data_source_slug"`
	OperationType       string             `json:"operation_type,omitempty"`
	OperationRef        slug.Slug          `json:"operation_ref,omitempty"`
	UsesInline          bool               `json:"uses_inline,omitempty"`
	ParameterKeys       []string           `json:"parameter_keys"`
	EventInput          *EventInputSummary `json:"event_input,omitempty"`
	SinkReplayStrategy  string             `json:"sink_replay_strategy,omitempty"`
	SinkReplayMethod    string             `json:"sink_replay_method,omitempty"`
	SinkReplayDecision  string             `json:"sink_replay_decision,omitempty"`
	SinkDeliveryOutcome string             `json:"sink_delivery_outcome,omitempty"`
	SinkEffectState     SinkEffectState    `json:"sink_effect_state,omitempty"`
}

StepInputSnapshot stores sanitized step inputs without parameter values.

type StepOutputDetailKind

type StepOutputDetailKind int

StepOutputDetailKind identifies the typed details payload attached to a run step output.

const (
	StepOutputDetailKindUnknown      StepOutputDetailKind = iota // unknown
	StepOutputDetailKindRows                                     // rows
	StepOutputDetailKindDocuments                                // documents
	StepOutputDetailKindHTTPResponse                             // http_response
	StepOutputDetailKindFileRead                                 // file_read
	StepOutputDetailKindFileWrite                                // file_write
)

func StepOutputDetailKindString

func StepOutputDetailKindString(s string) (StepOutputDetailKind, error)

StepOutputDetailKindString retrieves an enum value from the enum constants string name. Throws an error if the param is not part of the enum.

func StepOutputDetailKindValues

func StepOutputDetailKindValues() []StepOutputDetailKind

StepOutputDetailKindValues returns all values of the enum

func (StepOutputDetailKind) IsAStepOutputDetailKind

func (i StepOutputDetailKind) IsAStepOutputDetailKind() bool

IsAStepOutputDetailKind returns "true" if the value is listed in the enum definition. "false" otherwise

func (StepOutputDetailKind) MarshalJSON

func (i StepOutputDetailKind) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for StepOutputDetailKind

func (StepOutputDetailKind) MarshalText

func (i StepOutputDetailKind) MarshalText() ([]byte, error)

MarshalText implements the encoding.TextMarshaler interface for StepOutputDetailKind

func (StepOutputDetailKind) String

func (i StepOutputDetailKind) String() string

func (*StepOutputDetailKind) UnmarshalJSON

func (i *StepOutputDetailKind) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for StepOutputDetailKind

func (*StepOutputDetailKind) UnmarshalText

func (i *StepOutputDetailKind) UnmarshalText(text []byte) error

UnmarshalText implements the encoding.TextUnmarshaler interface for StepOutputDetailKind

type StepOutputDetails

type StepOutputDetails struct {
	Kind          StepOutputDetailKind             `json:"kind"`
	RowCount      *int                             `json:"row_count,omitempty"`
	ColumnNames   []string                         `json:"column_names,omitempty"`
	DocumentCount *int                             `json:"document_count,omitempty"`
	StatusCode    *int                             `json:"status_code,omitempty"`
	IsHeadlessCSV bool                             `json:"is_headless_csv,omitempty"`
	File          *filepayload.FilePayloadMetadata `json:"file,omitempty"`
	TargetPath    *string                          `json:"target_path,omitempty"`
	WrittenRows   *int                             `json:"written_rows,omitempty"`
	WrittenBytes  *int                             `json:"written_bytes,omitempty"`
}

StepOutputDetails stores result-type-specific run output metadata under a typed discriminator.

type StepOutputSnapshot

type StepOutputSnapshot struct {
	ResultType    string             `json:"result_type"`
	VendorType    string             `json:"vendor_type"`
	DurationMs    int64              `json:"duration_ms"`
	OperationType string             `json:"operation_type,omitempty"`
	Details       *StepOutputDetails `json:"details,omitempty"`
}

StepOutputSnapshot stores connector-agnostic execution metadata with typed result details.

type StepResult

type StepResult struct {
	// contains filtered or unexported fields
}

StepResult is a pipeline-level wrapper around step outputs. Query steps store connector ExecutionResult while transform steps store payload.Output directly to keep connector results free of transform payloads.

func NewRunInputStepResult

func NewRunInputStepResult(input *pipepayload.Payload) *StepResult

func NewStepResult

func NewStepResult(executionResult *result.ExecutionResult) *StepResult

func NewTransformStepResult

func NewTransformStepResult(
	transform *pipepayload.Output,
	durationMs int64,
	err error,
) *StepResult

func (*StepResult) Clone

func (sr *StepResult) Clone() *StepResult

func (*StepResult) DurationMs

func (sr *StepResult) DurationMs() int64

func (*StepResult) Error

func (sr *StepResult) Error() error

func (*StepResult) Execution

func (sr *StepResult) Execution() *result.ExecutionResult

func (*StepResult) FTP

func (sr *StepResult) FTP() (*result.FileResult, bool)

func (*StepResult) File

func (sr *StepResult) File() (*result.FileResult, bool)

func (*StepResult) Git

func (sr *StepResult) Git() (*result.FileResult, bool)

func (*StepResult) HTTP

func (sr *StepResult) HTTP() (*result.HTTPResult, bool)

func (*StepResult) IsRunInput

func (sr *StepResult) IsRunInput() bool

func (*StepResult) IsSuccess

func (sr *StepResult) IsSuccess() bool

func (*StepResult) MarshalJSON

func (sr *StepResult) MarshalJSON() ([]byte, error)

func (*StepResult) MongoDB

func (sr *StepResult) MongoDB() (*result.MongoDBResult, bool)

func (*StepResult) ObjectRows

func (sr *StepResult) ObjectRows() (*result.ObjectRowsResult, bool)

func (*StepResult) Payload

func (sr *StepResult) Payload() (*pipepayload.Payload, bool)

func (*StepResult) RedactedClone

func (sr *StepResult) RedactedClone(redact func(string) string) *StepResult

RedactedClone returns an independent copy with storage-visible strings redacted.

func (*StepResult) ResultTypeString

func (sr *StepResult) ResultTypeString() string

func (*StepResult) S3

func (sr *StepResult) S3() (*result.FileResult, bool)

S3 returns the S3 file result payload when this step came from an S3 execution.

func (*StepResult) SQL

func (sr *StepResult) SQL() (*result.SQLResult, bool)

func (*StepResult) Transform

func (sr *StepResult) Transform() (*pipepayload.Output, bool)

func (*StepResult) Type

func (*StepResult) UnmarshalJSON

func (sr *StepResult) UnmarshalJSON(data []byte) error

func (*StepResult) VendorType

func (sr *StepResult) VendorType() string

type StepSnapshot

type StepSnapshot struct {
	Step   slug.Slug           `json:"step"`
	Kind   string              `json:"kind,omitempty"`
	Input  StepInputSnapshot   `json:"input"`
	Output *StepOutputSnapshot `json:"output,omitempty"`
}

StepSnapshot stores sanitized input/output metadata for a step.

type StructuralCheck

type StructuralCheck struct {
	ContractFailure            *ContractFailureDiagnostic `json:"contract_failure,omitempty"`
	ID                         string                     `json:"id"`
	Step                       slug.Slug                  `json:"step"`
	Input                      string                     `json:"input,omitempty"`
	Origin                     string                     `json:"origin"`
	Phase                      string                     `json:"phase"`
	Declaration                string                     `json:"declaration"`
	ProjectionDigest           string                     `json:"projection_digest"`
	CompiledPlanDigest         string                     `json:"compiled_plan_digest,omitempty"`
	DeploymentResolutionDigest string                     `json:"deployment_resolution_digest,omitempty"`
	PipelineVersionID          int64                      `json:"-"`
	AttemptNumber              int                        `json:"attempt_number"`
	Evaluator                  string                     `json:"evaluator"`
	Policy                     string                     `json:"policy"`
	Target                     slug.Slug                  `json:"target,omitempty"`
	Status                     CheckStatus                `json:"status"`
	ReasonCode                 string                     `json:"reason_code"`
	StartedAt                  *time.Time                 `json:"started_at,omitempty"`
	CompletedAt                *time.Time                 `json:"completed_at,omitempty"`
	Evidence                   *CheckEvidence             `json:"evidence,omitempty"`
	EvidenceDigest             string                     `json:"evidence_digest,omitempty"`
	ReusedFromRun              string                     `json:"reused_from_run,omitempty"`
	Window                     CheckWindow                `json:"window"`
}

StructuralCheck records one executed obligation, never delivery or freshness. Only fixed codes, declaration identities and bounded aggregates belong here.

type TimelineEvent

type TimelineEvent struct {
	At     time.Time `json:"at"`
	Type   string    `json:"type"`
	Step   slug.Slug `json:"step,omitempty"`
	Status string    `json:"status"`
}

TimelineEvent captures important moments in a pipeline run lifecycle.

type TimelineStore

type TimelineStore interface {
	ObserveWorker(context.Context, slug.Slug, slug.Slug, slug.Slug, string, time.Time) error
	Get(ctx context.Context, orgSlug, projectSlug, environmentSlug slug.Slug, runSlug string) (*RunTimeline, error)
	SaveCheckpoint(
		ctx context.Context, orgSlug, projectSlug, environmentSlug slug.Slug, runSlug string, checkpoint ReplayCheckpoint,
	) error
	GetCheckpoint(
		ctx context.Context,
		orgSlug, projectSlug, environmentSlug slug.Slug,
		runSlug string,
	) (*ReplayCheckpoint, error)
}

TimelineStore retrieves persisted run timelines and replay checkpoints.

type WorkerLiveness

type WorkerLiveness int
const (
	WorkerLivenessUnavailable WorkerLiveness = iota // unavailable
	WorkerLivenessRecent                            // recent
	WorkerLivenessStale                             // stale
	WorkerLivenessTerminal                          // terminal
)

func WorkerLivenessString

func WorkerLivenessString(s string) (WorkerLiveness, error)

WorkerLivenessString retrieves an enum value from the enum constants string name. Throws an error if the param is not part of the enum.

func WorkerLivenessValues

func WorkerLivenessValues() []WorkerLiveness

WorkerLivenessValues returns all values of the enum

func (WorkerLiveness) IsAWorkerLiveness

func (i WorkerLiveness) IsAWorkerLiveness() bool

IsAWorkerLiveness returns "true" if the value is listed in the enum definition. "false" otherwise

func (WorkerLiveness) MarshalJSON

func (i WorkerLiveness) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for WorkerLiveness

func (WorkerLiveness) MarshalText

func (i WorkerLiveness) MarshalText() ([]byte, error)

MarshalText implements the encoding.TextMarshaler interface for WorkerLiveness

func (WorkerLiveness) String

func (i WorkerLiveness) String() string

func (*WorkerLiveness) UnmarshalJSON

func (i *WorkerLiveness) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for WorkerLiveness

func (*WorkerLiveness) UnmarshalText

func (i *WorkerLiveness) UnmarshalText(text []byte) error

UnmarshalText implements the encoding.TextUnmarshaler interface for WorkerLiveness

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL