pipelineruntime

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 16, 2026 License: Apache-2.0 Imports: 49 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	ErrNilPipeline              = xerror.InvalidRequired(xerror.Pipeline)
	ErrQueryExecutorNotProvided = xerror.UnexpectedRequired(xerror.QueryExecutor)
	ErrReplayRunSlugRequired    = xerror.InvalidRequired(xerror.ReplayRunSlug)
	ErrReplayRunNotFound        = xerror.ReplayRunNotFound()
	ErrReplayStepNotFound       = xerror.ReplayStepNotFound()
	ErrReplayUnsupportedMode    = xerror.ReplayUnsupportedMode()
	ErrReplayUnsafe             = xerror.ReplayUnsafe()
	ErrNilRunStore              = xerror.UnexpectedRequired(xerror.RunStore)
	ErrNilDataSourceService     = xerror.UnexpectedRequired(xerror.DataSourceService)
	ErrNilSinkRuntime           = xerror.UnexpectedRequired(xerror.SinkRuntime)

	ErrMissingOrgSlug         = xerror.InvalidRequired(xerror.OrganizationSlug)
	ErrMissingEnvironmentSlug = xerror.InvalidRequired(xerror.ProjectEnvironment).Detail("environment slug is required")

	ErrTransformUpstreamRequired         = xerror.TransformUpstreamRequired()
	ErrSinkUpstreamRequired              = xerror.SinkUpstreamRequired()
	ErrTransformBindingMissing           = xerror.TransformBindingMissing()
	ErrTransformBindingSourceNotFound    = xerror.TransformBindingSourceNotFound()
	ErrTransformBindingSourceAmbiguous   = xerror.TransformBindingSourceAmbiguous()
	ErrTransformBindingNodeUnsupported   = xerror.TransformBindingNodeUnsupported()
	ErrQueryOutputContractSchemaMismatch = xerror.QueryOutputContractSchemaMismatch()
	ErrSinkSourceResultNotFound          = xerror.SinkSourceResultNotFound()
)

Functions

func CheckpointValueSummary

func CheckpointValueSummary(checkpointKind string, raw json.RawMessage) string

CheckpointValueSummary renders a bounded display value and hides opaque cursor tokens.

func NewRunStore

NewRunStore creates the service-layer run store used by pipeline execution.

func ReplayInputModeStrings

func ReplayInputModeStrings() []string

ReplayInputModeStrings returns a slice of all String values of the enum

Types

type EnvironmentSecretResolver

type EnvironmentSecretResolver interface {
	ResolveRuntimeSecrets(ctx context.Context, orgSlug, environmentSlug slug.Slug) ([]model.RuntimeSecretValue, error)
}

EnvironmentSecretResolver supplies decrypted environment secret values inside the run boundary.

type ReplayInputMode

type ReplayInputMode int

ReplayInputMode controls where replay input values come from.

const (
	// ReplayInputModePinned reuses the exact stored checkpoint input.
	ReplayInputModePinned ReplayInputMode = iota // pinned
)

func ReplayInputModeString

func ReplayInputModeString(s string) (ReplayInputMode, error)

ReplayInputModeString retrieves an enum value from the enum constants string name. Throws an error if the param is not part of the enum.

func ReplayInputModeValues

func ReplayInputModeValues() []ReplayInputMode

ReplayInputModeValues returns all values of the enum

func (ReplayInputMode) IsAReplayInputMode

func (i ReplayInputMode) IsAReplayInputMode() bool

IsAReplayInputMode returns "true" if the value is listed in the enum definition. "false" otherwise

func (ReplayInputMode) MarshalJSON

func (i ReplayInputMode) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface for ReplayInputMode

func (ReplayInputMode) String

func (i ReplayInputMode) String() string

func (*ReplayInputMode) UnmarshalJSON

func (i *ReplayInputMode) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface for ReplayInputMode

type ReplayRequest

type ReplayRequest struct {
	Source            pipelinerun.RunSourceSummary
	OrgSlug           slug.Slug
	ProjectSlug       slug.Slug
	EnvironmentSlug   slug.Slug
	SourceRunSlug     string
	ReplayRunSlug     string
	FromStep          string
	InputMode         ReplayInputMode
	PipelineVersionID int64
}

ReplayRequest describes how to replay a previous run from a checkpoint.

type RunOutcome

type RunOutcome struct {
	Metadata               RunOutcomeMetadata
	Timeline               pipelinerun.RunTimeline
	IncrementalDiagnostics pipelinerun.IncrementalRunDiagnostics
}

RunOutcome is the service-facing result of a pipeline execution. It intentionally omits step results so service orchestration does not depend on runtime internals after submitting the run request.

type RunOutcomeMetadata

type RunOutcomeMetadata struct {
	PipelineVersionID int64
	PipelineSlug      slug.Slug
	ProjectSlug       slug.Slug
	EnvironmentSlug   slug.Slug
	RunSlug           string
	Source            pipelinerun.RunSourceSummary
	Status            string
	DurationMs        int64
	StepCount         int
	StepSlug          string
	ErrorSummary      *string
	FailureKind       *string
	StartedAt         time.Time
	EndedAt           *time.Time
	ReplayOfRunSlug   *string
	ReplayFromStep    *string
	Incremental       pipelinerun.IncrementalRunSummary
}

RunOutcomeMetadata contains run summary fields derived by the runtime.

type RunRequest

type RunRequest struct {
	DeploymentID               *int64
	DeploymentRevision         *int64
	DeploymentPlan             []byte
	DeploymentResolutionDigest string
	CompiledPlan               []byte
	CompiledPlanDigest         string
	OrgSlug                    slug.Slug
	ProjectSlug                slug.Slug
	EnvironmentSlug            slug.Slug
	RunSlug                    string
	Source                     pipelinerun.RunSourceSummary
	Pipeline                   *pipelinespec.Pipeline
	PipelineVersionID          int64
	RuntimeInput               map[string]any
	EventInput                 *pipelinerun.EventInput
	ReplayOfRunSlug            *string
	ReplayFromStep             *string
	Incremental                pipelinerun.IncrementalRunOptions
	// contains filtered or unexported fields
}

RunRequest contains inputs that affect run metadata generation.

type Runner

type Runner struct {
	// contains filtered or unexported fields
}

Runner executes a minimal pipeline by delegating query steps to the existing query service.

func NewRunner

func NewRunner(
	executor queryExecutor,
	clk clock.Clock,
	store pipelinerun.RunStore,
	dataSourceService datasourcessvc.Service,
	sinkRuntime SinkRuntime,
	options ...RunnerOption,
) (*Runner, error)

NewRunner creates a runner with explicit run-store, datasource service, and sink runtime dependencies.

func (*Runner) Execute

func (r *Runner) Execute(ctx context.Context, req RunRequest) (*RunOutcome, error)

Execute runs a pipeline and returns the service-facing execution outcome.

func (*Runner) PreValidateTransforms

func (r *Runner) PreValidateTransforms(
	ctx context.Context,
	orgSlug slug.Slug,
	pipeline *pipelinespec.Pipeline,
) error

PreValidateTransforms performs pre-flight validation of query and transform steps. Query output contracts are derived from cached schema when absent, and validated against cached schema when provided. Transform contracts are then validated against inferred upstream input types. This method returns an error if any query/transform prevalidation fails.

func (*Runner) PrepareCheckpointBootstrap

func (r *Runner) PrepareCheckpointBootstrap(ctx context.Context, req RunRequest) (lookups []pipelinerun.IncrementalCheckpointLookup, prepareErr error)

PrepareCheckpointBootstrap resolves the same source identities as execution without reading providers or running sinks.

func (*Runner) Recover

func (r *Runner) Recover(ctx context.Context, req ReplayRequest) (*RunOutcome, error)

Recover returns the service-facing outcome of a pinned sink recovery.

func (*Runner) Replay

func (r *Runner) Replay(
	ctx context.Context,
	req ReplayRequest,
) (runResult *pipelinerun.RunResult, runErr error)

Replay reruns a pipeline from a previously saved checkpoint with deterministic semantics.

func (*Runner) Run

func (r *Runner) Run(
	ctx context.Context,
	req RunRequest,
) (runResult *pipelinerun.RunResult, runErr error)

func (*Runner) TimelineByRunSlug

func (r *Runner) TimelineByRunSlug(
	ctx context.Context,
	orgSlug, projectSlug, environmentSlug slug.Slug,
	runSlug string,
) (*pipelinerun.RunTimeline, error)

TimelineByRunSlug returns a previously recorded run timeline by run slug.

type RunnerOption

type RunnerOption func(*Runner)

func WithEnvironmentSecretResolver

func WithEnvironmentSecretResolver(resolver EnvironmentSecretResolver) RunnerOption

WithEnvironmentSecretResolver enables runtime resolution for {{runtime.secrets.KEY}} placeholders.

func WithTransformExecutor

func WithTransformExecutor(executor TransformExecutor, limits transformsql.Limits) RunnerOption

WithTransformExecutor moves transform materialization to the isolated executor.

func WithTransformLimits

func WithTransformLimits(limits transformsql.Limits) RunnerOption

WithTransformLimits applies cumulative runtime budgets to every SQL transform.

type SinkRuntime

type SinkRuntime interface {
	Execute(ctx context.Context, input *SinkRuntimeInput) error
}

SinkRuntime executes sink-step delivery logic using resolved upstream output.

func NewSinkRuntime

func NewSinkRuntime(
	queryExec sinkQueryExecutor,
	effectLedgers ...pipelinerun.SinkEffectLedger,
) SinkRuntime

NewSinkRuntime creates a pipeline sink runtime backed by saved or inline sink-query execution. The optional durable ledger is required in production and remains optional for hermetic connector tests that do not exercise retry behavior.

type SinkRuntimeInput

type SinkRuntimeInput struct {
	OperationSnapshot      *contractplan.OperationContent
	OrgSlug                slug.Slug
	ProjectSlug            slug.Slug
	EnvironmentSlug        slug.Slug
	PipelineSlug           slug.Slug
	RunSlug                string
	ReplayIdentity         string
	SinkStepSlug           slug.Slug
	InputKind              model.TransformOutputKind
	SourceStep             slug.Slug
	SourceResult           *pipelinerun.StepResult
	SinkBackend            model.SinkBackend
	SinkDataSourceSlug     slug.Slug
	SinkDataSourceReadOnly bool
	SinkOperationType      string
	SinkOperationRef       slug.Slug
	SinkOperationInline    map[string]any
	SinkConfig             map[string]any
	IdempotencyClass       pipelinespec.SinkIdempotencyClass
	IdempotencyKey         string
	ReplayDecision         string
	EffectState            pipelinerun.SinkEffectState
	EffectStartedAt        time.Time
	EmptyInputPolicy       string
	DeliveryOutcome        string
	SinkResult             *result.ExecutionResult
}

SinkRuntimeInput contains the resolved output passed to sink runtime.

type TransformExecutionRequest

type TransformExecutionRequest struct {
	OrganizationSlug slug.Slug                   `json:"organization_slug"`
	Transform        transformsql.Transform      `json:"transform"`
	UpstreamPayloads map[string]*payload.Payload `json:"upstream_payloads"`
	Limits           transformsql.Limits         `json:"limits"`
}

TransformExecutionRequest is the bounded payload sent to an isolated executor.

type TransformExecutor

type TransformExecutor interface {
	ExecuteTransform(context.Context, TransformExecutionRequest) (*payload.Output, error)
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL