pipelineexec

package
v0.0.0-...-ae5b880 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 8, 2026 License: MIT Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source
const MaxLogLineSize = 64 << 10

MaxLogLineSize bounds a connector log line handed to a Handler.

View Source
const OwnerLabel = "synclet.io/owner"

OwnerLabel marks every container and Job with the id of the executor that created it.

View Source
const RedactedMask = "***"

RedactedMask replaces secret values in connector output.

Variables

This section is empty.

Functions

func GetWorkerID

func GetWorkerID() string

GetWorkerID returns a generated ID for worker identification.

func NilIfEmpty

func NilIfEmpty(s string) *string

NilIfEmpty returns nil if the string is empty, otherwise returns a pointer to the string.

func ResolveOwnerID

func ResolveOwnerID(id, prefix, setting, prefixSetting, sharedBy string, hostname func() (string, error)) (string, error)

ResolveOwnerID returns the executor's owner id: id when it is set, or "<prefix>-<n>" when prefix is set, n being the digits after the last "-" of the hostname, which for a StatefulSet pod is its ordinal. setting and prefixSetting name the variables that configure id and prefix, and sharedBy what executors that need distinct ids share, such as "Docker daemon", for the error; an executor without a prefix variable passes an empty prefix.

Types

type ClaimConnectorTaskResult

type ClaimConnectorTaskResult struct {
	TaskID      uuid.UUID
	TaskType    pipelineservice.ConnectorTaskType
	Image       string
	Config      []byte // Decrypted JSON config (nil for spec)
	WorkspaceID uuid.UUID
	// RuntimeConfig is the entity's raw runtime_config JSON; empty for spec
	// and inline-config checks.
	RuntimeConfig string
}

ClaimConnectorTaskResult bundles everything an executor needs to run a connector task.

type ClaimJobResult

type ClaimJobResult struct {
	Job                   *pipelineservice.Job
	ConnectionID          uuid.UUID
	WorkspaceID           uuid.UUID
	SourceID              uuid.UUID
	DestinationID         uuid.UUID
	SourceImage           string
	SourceConfig          []byte // Decrypted JSON
	DestImage             string
	DestConfig            []byte // Decrypted JSON
	ConfiguredCatalog     []byte // JSON-marshaled
	StateBlob             []byte // JSON, may be nil
	SourceRuntimeConfig   string
	DestRuntimeConfig     string
	NamespaceDefinition   pipelineservice.NamespaceDefinition
	CustomNamespaceFormat string
	StreamPrefix          string
	MapperSalt            []byte
	MaxAttempts           int
}

ClaimJobResult bundles everything an executor needs to run a sync.

type DefaultHandler

type DefaultHandler struct{}

DefaultHandler provides no-op implementations of all Handler methods. Embed it in concrete handlers to only override the methods you need.

func (DefaultHandler) OnDestControl

func (DefaultHandler) OnLog

func (DefaultHandler) OnSourceControl

func (DefaultHandler) OnSourceTrace

func (DefaultHandler) OnStateConfirmed

type ExecutorBackend

type ExecutorBackend interface {
	SyncReporter

	ClaimJob(ctx context.Context, workerID string) (*ClaimJobResult, error)
	UpdateJobStatus(ctx context.Context, params UpdateJobStatusParams) error
	// MarkJobRunning reports that the job's connectors have started.
	MarkJobRunning(ctx context.Context, jobID uuid.UUID) error
	Heartbeat(ctx context.Context, jobID uuid.UUID, recordsRead, bytesSynced int64) (*HeartbeatResult, error)
	ReportCompletion(ctx context.Context, params ReportCompletionParams) error
	// SetK8sJobName stores the Kubernetes Job name launched for a job.
	SetK8sJobName(ctx context.Context, jobID uuid.UUID, name string) error
	IsJobActive(ctx context.Context, jobID string) (bool, error)
	IsTaskActive(ctx context.Context, taskID string) (bool, error)
	FailJob(ctx context.Context, jobID string, reason string) error

	// ClaimConnectorTask claims the next pending connector task (D-15).
	ClaimConnectorTask(ctx context.Context, workerID string) (*ClaimConnectorTaskResult, error)

	// ReportConnectorTaskResult reports the result of a connector task (D-18/D-19).
	ReportConnectorTaskResult(ctx context.Context, params ReportConnectorTaskResultParams) error

	// ExecutorHeartbeat registers the executor in the executor registry.
	ExecutorHeartbeat(ctx context.Context, info ExecutorInfo) error
}

ExecutorBackend is the only way an executor reaches the control plane. Two implementations: the use-case adapter inside `server --standalone` and the ConnectRPC client of the executor commands.

type ExecutorGroup

type ExecutorGroup string

ExecutorGroup is the normalized group of the executor in this process.

type ExecutorInfo

type ExecutorInfo struct {
	// ID is the executor's owner id, unique per instance.
	ID      string
	Type    pipelineservice.ExecutorType
	Version string
	// Capacity is the number of concurrent syncs; 0 means unbounded.
	Capacity         int
	Capabilities     pipelineservice.ExecutorCapabilities
	ResourceDefaults pipelineservice.ExecutorResourceDefaults
	// Group is the executor group whose jobs and tasks the executor claims.
	Group string
}

ExecutorInfo describes an executor instance to the executor registry.

type Handler

type Handler interface {
	// OnStateConfirmed is called when the destination emits a STATE message,
	// confirming all preceding records are committed.
	OnStateConfirmed(ctx context.Context, msg *protocol.AirbyteStateMessage) error

	// OnSourceControl is called when the source emits a CONTROL message.
	OnSourceControl(ctx context.Context, msg *protocol.AirbyteControlMessage) error

	// OnDestControl is called when the destination emits a CONTROL message.
	OnDestControl(ctx context.Context, msg *protocol.AirbyteControlMessage) error

	// OnSourceTrace is called when the source emits a TRACE error message.
	OnSourceTrace(ctx context.Context, msg *protocol.AirbyteTraceMessage) error

	// OnLog is called when the source or destination emits a LOG or TRACE message.
	// The line is pre-formatted text (e.g., "[src] INFO: message").
	OnLog(ctx context.Context, line string) error
}

Handler defines callbacks for mode-specific side effects during message routing. The router owns RECORD forwarding, stats counting, LOG output, and STATE forwarding. Handler only receives callbacks for mode-specific side effects.

func RedactHandler

func RedactHandler(handler Handler, redactor *Redactor) Handler

RedactHandler wraps handler so log lines and trace errors pass through redactor before handler sees them, and log lines are cut to MaxLogLineSize once redacted. Secrets of a config a connector replaces through CONTROL are added to redactor. A nil redactor masks nothing.

type HeartbeatResult

type HeartbeatResult struct {
	Cancelled bool
}

HeartbeatResult carries cancellation signal per D-05.

type NamespaceRewriter

type NamespaceRewriter struct {
	// contains filtered or unexported fields
}

NamespaceRewriter rewrites namespace and stream name in RECORD and STATE messages to match the destination catalog's namespace/prefix settings, and maps the STATE the destination confirms back to the source catalog's names. It pre-computes both mappings from the source catalog so lookups are O(1).

func NewNamespaceRewriter

func NewNamespaceRewriter(
	catalog *protocol.ConfiguredAirbyteCatalog,
	nsDef pipelineservice.NamespaceDefinition,
	customFormat *string,
	prefix *string,
) *NamespaceRewriter

NewNamespaceRewriter builds a rewriter from the source catalog and connection namespace settings. The mapping is computed once and reused for all messages.

func (*NamespaceRewriter) RestoreState

func (r *NamespaceRewriter) RestoreState(state *protocol.AirbyteStateMessage)

RestoreState maps the stream descriptors of a STATE message confirmed by the destination back to the source catalog's names and namespaces, for STREAM and GLOBAL state. A descriptor that no single catalog stream maps to is kept.

func (*NamespaceRewriter) RewriteRecord

func (r *NamespaceRewriter) RewriteRecord(msg *protocol.AirbyteMessage)

RewriteRecord rewrites the stream name and namespace on a RECORD message. No-op if msg.Record is nil.

func (*NamespaceRewriter) RewriteState

func (r *NamespaceRewriter) RewriteState(msg *protocol.AirbyteMessage)

RewriteState rewrites stream descriptors in a STATE message. Handles stream, global, and legacy (no-op) state types. No-op if msg.State is nil.

func (*NamespaceRewriter) RewriteTrace

func (r *NamespaceRewriter) RewriteTrace(msg *protocol.AirbyteMessage)

RewriteTrace rewrites stream descriptors in TRACE messages (stream_status, error) so the destination can match them against its catalog. No-op if msg.Trace is nil.

type RecordTransformer

type RecordTransformer struct {
	// contains filtered or unexported fields
}

RecordTransformer applies each configured stream's mappers and field selection to the RECORDs the source emits, keyed by the source's own stream names. Streams with mappers run their compiled chain, which ends with the field selection; streams with only selected fields are filtered to them; other streams pass unchanged.

func NewRecordTransformer

func NewRecordTransformer(catalog *protocol.ConfiguredAirbyteCatalog, salt []byte, sync mapper.SyncInfo) (*RecordTransformer, error)

NewRecordTransformer compiles the mappers of every stream in catalog with salt as the hash salt; sync is what the mappers' expressions see as `sync`. A stream whose mappers do not compile is an error naming the stream.

func (*RecordTransformer) Apply

func (t *RecordTransformer) Apply(rec *protocol.AirbyteRecordMessage) (keep bool, err error)

Apply transforms rec.Data in place. keep is false when a filter or an on_error skip drops the record. A step failing under on_error fail returns an error naming the stream and the step.

type Redactor

type Redactor struct {
	// contains filtered or unexported fields
}

Redactor masks secret configuration values in connector output. The zero value and nil redact nothing. It is safe for concurrent use.

func NewRedactor

func NewRedactor(configs ...json.RawMessage) *Redactor

NewRedactor collects secret values from connector configs: string values of at least minSecretLen bytes under secret-looking keys, strings of at least minNestedSecretLen bytes nested beneath such a key, and the same inside string values that hold a JSON object. Configs that are not JSON objects are skipped.

func (*Redactor) Add

func (r *Redactor) Add(configs ...json.RawMessage)

Add masks the secrets of configs as well, such as a config a connector replaced during a sync. It does nothing on a nil Redactor.

func (*Redactor) Redact

func (r *Redactor) Redact(text string) string

Redact returns text with every secret value replaced by RedactedMask.

func (*Redactor) RedactHeadTruncated

func (r *Redactor) RedactHeadTruncated(text string) string

RedactHeadTruncated is Redact for text whose beginning was cut off, such as the newest part of a bounded buffer: it also masks a leading suffix, of at least minSecretLen bytes, of a secret value.

func (*Redactor) RedactTrace

RedactTrace returns a copy of msg with its error texts redacted.

func (*Redactor) RedactTruncated

func (r *Redactor) RedactTruncated(text string) string

RedactTruncated is Redact for text cut off at its end, such as output still being written: it also masks a trailing prefix, of at least minSecretLen bytes, of a secret value.

type Registrar

type Registrar struct {
	// contains filtered or unexported fields
}

Registrar keeps the executor listed in the control plane's executor registry: each Execute sends one registration heartbeat.

func NewRegistrar

func NewRegistrar(backend ExecutorBackend, info ExecutorInfo, logger *logging.Logger) *Registrar

NewRegistrar returns a Registrar that reports info through backend.

func (*Registrar) Execute

func (r *Registrar) Execute(ctx context.Context) error

Execute sends one registration heartbeat. A failure is logged and not returned: the next heartbeat retries it.

type ReportCompletionParams

type ReportCompletionParams struct {
	JobID           uuid.UUID
	ConnectionID    uuid.UUID
	Success         bool
	ErrorMessage    string
	RecordsRead     int64
	RecordsFiltered int64
	BytesSynced     int64
	DurationMs      int64
	// SourceExitCode and DestExitCode are the connectors' non-zero exit codes;
	// 0 means a clean exit or none known.
	SourceExitCode int
	DestExitCode   int
}

ReportCompletionParams holds parameters for reporting completion with stats.

type ReportConnectorTaskResultParams

type ReportConnectorTaskResultParams struct {
	TaskID       uuid.UUID
	Success      bool
	ErrorMessage string
	Result       []byte // JSON-encoded result
}

ReportConnectorTaskResultParams holds parameters for reporting connector task completion.

type RunConfig

type RunConfig struct {
	IdleTimeout time.Duration      // Kill if no RECORD for this duration. 0 = disabled.
	Rewriter    *NamespaceRewriter // Rewrite namespace/prefix on messages to the destination and restore source names on confirmed STATE. nil = no rewriting.
	Transformer *RecordTransformer // Run stream mappers and field selection on RECORDs before they are counted and forwarded. nil = records pass unchanged.
	// Redactor masks connector config secrets in every connector-provided
	// text the handler receives or the router logs. nil = nothing is masked.
	Redactor *Redactor

	// Catalog is the source's configured catalog. After the source succeeds,
	// every stream in it without a COMPLETE or INCOMPLETE status from the
	// source gets a COMPLETE status, as the Airbyte platform sends them.
	Catalog *protocol.ConfiguredAirbyteCatalog
	// SourceSucceeded is asked once the source output has ended cleanly and
	// reports whether the source exited 0. nil = a clean end of output means
	// the source succeeded.
	SourceSucceeded func() bool
}

RunConfig holds optional configuration for the routing session.

type Stats

type Stats struct {
	RecordsRead     int64
	RecordsFiltered int64
	BytesSynced     int64
	Duration        time.Duration
}

Stats holds the final statistics from a routing session. RecordsRead counts the RECORDs the source emitted, RecordsFiltered those a mapper dropped, and BytesSynced the data of the RECORDs forwarded to the destination.

func Run

func Run(ctx context.Context, sourceStdout io.Reader, destStdin io.WriteCloser, destStdout io.ReadCloser, handler Handler, cfg RunConfig, logger *logging.Logger) (*Stats, error)

Run reads messages from sourceStdout, routes them to destStdin, reads dest output from destStdout, and dispatches side effects to handler. It blocks until both source and dest readers reach EOF or ctx is cancelled.

type SyncBundle

type SyncBundle struct {
	Job                   *pipelineservice.Job
	ConnectionID          uuid.UUID
	WorkspaceID           uuid.UUID
	SourceID              uuid.UUID
	DestinationID         uuid.UUID
	SourceImage           string
	SourceConfig          json.RawMessage // Already decrypted
	DestImage             string
	DestConfig            json.RawMessage                    // Already decrypted
	ConfiguredCatalog     *protocol.ConfiguredAirbyteCatalog // Already unmarshaled
	StateBlob             json.RawMessage                    // May be nil
	SourceRuntimeConfig   string                             // JSON
	DestRuntimeConfig     string                             // JSON
	NamespaceDefinition   pipelineservice.NamespaceDefinition
	CustomNamespaceFormat *string
	StreamPrefix          *string
	MapperSalt            []byte
}

SyncBundle contains all pre-loaded data needed to execute a sync. Populated from the ClaimJobResult the ExecutorBackend returns.

type SyncReporter

type SyncReporter interface {
	ReportState(ctx context.Context, connectionID, jobID uuid.UUID, stateMsg *protocol.AirbyteStateMessage) error
	ReportConfigUpdate(ctx context.Context, connectorType pipelineservice.ConnectorType, connectorID uuid.UUID, config []byte) error
	ReportLog(ctx context.Context, jobID uuid.UUID, lines []string) error
}

SyncReporter is the part of the executor API a running sync reports through: confirmed state, connector config updates and log lines.

type UpdateJobStatusParams

type UpdateJobStatusParams struct {
	JobID           uuid.UUID
	Success         bool
	ErrorMessage    string
	RecordsRead     int64
	RecordsFiltered int64
	BytesSynced     int64
	DurationMs      int64
}

UpdateJobStatusParams holds parameters for reporting job completion/failure.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL