Documentation
¶
Index ¶
- Constants
- Variables
- func CreateLocalDir(rootDir string) (string, error)
- func CreateTempDir(rootDir string, files map[string][]byte) (string, error)
- func EnsureLocalRoot(root string) error
- func PrepareLocalRoot(ctx context.Context, root string, logger *logging.Logger) error
- type ConnectorClient
- func (c *ConnectorClient) Check(ctx context.Context, image string, config json.RawMessage, ...) (*protocol.AirbyteConnectionStatus, error)
- func (c *ConnectorClient) Discover(ctx context.Context, image string, config json.RawMessage, ...) (*protocol.AirbyteCatalog, error)
- func (c *ConnectorClient) Read(ctx context.Context, image string, config json.RawMessage, ...) (io.ReadCloser, func(), error)
- func (c *ConnectorClient) Spec(ctx context.Context, image string, limits ResourceLimits) (*protocol.ConnectorSpecification, error)
- func (c *ConnectorClient) Write(ctx context.Context, image string, config json.RawMessage, ...) (io.ReadCloser, func(), error)
- type ContainerRunner
- func (r *ContainerRunner) ListByLabel(ctx context.Context, labels ...string) ([]dockercontainer.Summary, error)
- func (r *ContainerRunner) OwnerID() string
- func (r *ContainerRunner) Pull(ctx context.Context, img string) (rerr error)
- func (r *ContainerRunner) Remove(ctx context.Context, containerID string) error
- func (r *ContainerRunner) ResolveDigest(ctx context.Context, img string) (string, error)
- func (r *ContainerRunner) Run(ctx context.Context, opts RunOptions) (*RunResult, error)
- func (r *ContainerRunner) Stop(ctx context.Context, containerID string) error
- func (r *ContainerRunner) StopWithTimeout(ctx context.Context, containerID string, timeoutSeconds int) error
- type DockerHandler
- func (h *DockerHandler) OnDestControl(ctx context.Context, msg *protocol.AirbyteControlMessage) error
- func (h *DockerHandler) OnLog(ctx context.Context, line string) error
- func (h *DockerHandler) OnSourceControl(ctx context.Context, msg *protocol.AirbyteControlMessage) error
- func (h *DockerHandler) OnSourceTrace(ctx context.Context, msg *protocol.AirbyteTraceMessage) error
- func (h *DockerHandler) OnStateConfirmed(ctx context.Context, msg *protocol.AirbyteStateMessage) error
- func (h *DockerHandler) StartLogFlush(ctx context.Context) (stop func())
- type DockerHandlerParams
- type DockerSyncExecutor
- type DockerSyncExecutorParams
- type DockerSyncWorker
- type DockerSyncWorkerParams
- type DockerTaskWorker
- type DockerTaskWorkerParams
- type OrphanCleaner
- type OrphanJobChecker
- type ResourceLimits
- type RunOptions
- type RunResult
Constants ¶
const OwnerLabel = pipelineexec.OwnerLabel
OwnerLabel marks every container with the id of the Synclet instance that created it.
Variables ¶
var ErrOutputDrainTimeout = errors.New("container exited but its output stream did not end")
ErrOutputDrainTimeout is the stdout read error of a container whose attach stream stayed open without data for the drain timeout after the container exited.
Functions ¶
func CreateLocalDir ¶
CreateLocalDir creates an empty directory under rootDir for a connector's /local mount, where Airbyte connectors write their files. It is world-writable because connector images run as users the host does not know. rootDir follows the rules of CreateTempDir.
func CreateTempDir ¶
CreateTempDir creates a temporary directory under rootDir and writes the provided files into it. The keys of the files map are filenames (e.g., "config.json"), and the values are file contents.
Pass rootDir == "" to use os.MkdirTemp's default (the OS temp dir). When synclet runs inside Docker and dispatches tasks to the host Docker daemon via /var/run/docker.sock, rootDir must point at a path that is bind-mounted to the same location on the host so the daemon can resolve the bind-mount source — see pipelineConfig.DockerTempDirRoot for the wiring.
func EnsureLocalRoot ¶
EnsureLocalRoot makes sure the persistent /local root exists. A missing root is created with mode 1777; an existing one is used as it is, its permissions and contents untouched.
func PrepareLocalRoot ¶
PrepareLocalRoot ensures the local root like EnsureLocalRoot and logs a warning when the root is not writable by others, so connector images that run as another user cannot write their files.
Types ¶
type ConnectorClient ¶
type ConnectorClient struct {
// contains filtered or unexported fields
}
ConnectorClient orchestrates Airbyte connector operations by composing the container runner with the protocol message reader/writer.
func NewConnectorClient ¶
func NewConnectorClient(runner pkgcontainer.Runner) *ConnectorClient
NewConnectorClient creates a new ConnectorClient backed by the given container runner.
func (*ConnectorClient) Check ¶
func (c *ConnectorClient) Check(ctx context.Context, image string, config json.RawMessage, limits ResourceLimits) (*protocol.AirbyteConnectionStatus, error)
Check validates a connector configuration by running the check command.
func (*ConnectorClient) Discover ¶
func (c *ConnectorClient) Discover(ctx context.Context, image string, config json.RawMessage, limits ResourceLimits) (*protocol.AirbyteCatalog, error)
Discover discovers the catalog (available streams) for a connector.
func (*ConnectorClient) Read ¶
func (c *ConnectorClient) Read(ctx context.Context, image string, config json.RawMessage, redactor *pipelineexec.Redactor, catalog *protocol.ConfiguredAirbyteCatalog, state json.RawMessage, labels map[string]string, limits ResourceLimits) (io.ReadCloser, func(), error)
Read starts a source connector and returns its stdout for the caller to read messages from. The caller must close the returned ReadCloser and call the cleanup function when done. When the connector exits with a non-zero code, the returned reader surfaces the error after all buffered messages are consumed (at the point where it would normally return EOF), with its stderr masked by redactor.
func (*ConnectorClient) Spec ¶
func (c *ConnectorClient) Spec(ctx context.Context, image string, limits ResourceLimits) (*protocol.ConnectorSpecification, error)
Spec runs the connector's spec command and returns its specification. The container runs with network mode "none" since no external access is needed.
func (*ConnectorClient) Write ¶
func (c *ConnectorClient) Write(ctx context.Context, image string, config json.RawMessage, redactor *pipelineexec.Redactor, catalog *protocol.ConfiguredAirbyteCatalog, stdin io.Reader, labels map[string]string, limits ResourceLimits) (io.ReadCloser, func(), error)
Write starts a destination connector with the given stdin reader and returns its stdout. The caller pipes source messages into stdin and reads dest output from the returned ReadCloser. The caller must close the returned ReadCloser and call the cleanup function when done. When the connector exits with a non-zero code, the returned reader surfaces the error after all buffered messages are consumed, with its stderr masked by redactor.
type ContainerRunner ¶
type ContainerRunner struct {
// contains filtered or unexported fields
}
ContainerRunner manages Docker containers for running Airbyte connectors.
func NewContainerRunner ¶
func NewContainerRunner(tempDirRoot, localRoot, ownerID string) (*ContainerRunner, error)
NewContainerRunner creates a new ContainerRunner using the default Docker client configuration (DOCKER_HOST env var, or the default Docker socket).
tempDirRoot is the directory under which per-task scratch directories are created. Pass "" for the OS default. In Docker-in-Docker deployments this must point at a path bind-mounted to the same location on the host so the host Docker daemon can resolve the bind-mount source — see pipelineConfig.DockerTempDirRoot for the wiring.
localRoot, when set, is the host directory mounted at /local in every container instead of a per-run directory under tempDirRoot; the runner never removes it. It follows the same host-path rule as tempDirRoot.
ownerID is stamped as the OwnerLabel on every container the runner creates.
func (*ContainerRunner) ListByLabel ¶
func (r *ContainerRunner) ListByLabel(ctx context.Context, labels ...string) ([]dockercontainer.Summary, error)
ListByLabel returns all containers matching every given label filter.
func (*ContainerRunner) OwnerID ¶
func (r *ContainerRunner) OwnerID() string
OwnerID returns the owner id stamped on the runner's containers.
func (*ContainerRunner) Pull ¶
func (r *ContainerRunner) Pull(ctx context.Context, img string) (rerr error)
Pull pulls a Docker image, discarding progress output.
func (*ContainerRunner) Remove ¶
func (r *ContainerRunner) Remove(ctx context.Context, containerID string) error
Remove removes a container. Use Force option if the container might still be running.
func (*ContainerRunner) ResolveDigest ¶
ResolveDigest returns the digest-pinned reference for a locally available image. It inspects the image metadata and extracts the first sha256 RepoDigest (SEC-13).
func (*ContainerRunner) Run ¶
func (r *ContainerRunner) Run(ctx context.Context, opts RunOptions) (*RunResult, error)
Run creates and starts a container with the given options. It returns immediately with I/O streams and a Done channel. The caller is responsible for reading stdout/stderr and calling Stop/Remove when finished.
func (*ContainerRunner) Stop ¶
func (r *ContainerRunner) Stop(ctx context.Context, containerID string) error
Stop stops a running container with a default timeout.
func (*ContainerRunner) StopWithTimeout ¶
func (r *ContainerRunner) StopWithTimeout(ctx context.Context, containerID string, timeoutSeconds int) error
StopWithTimeout stops a container with a specific grace period. Docker SDK handles SIGTERM -> wait timeout -> SIGKILL automatically.
type DockerHandler ¶
type DockerHandler struct {
pipelineexec.DefaultHandler
// contains filtered or unexported fields
}
DockerHandler implements Handler for Docker mode sync execution. It reports confirmed state and CONTROL config updates through the SyncReporter. Log lines are buffered and reported in batches, in the order they arrived.
func NewDockerHandler ¶
func NewDockerHandler(params DockerHandlerParams) *DockerHandler
NewDockerHandler creates a new DockerHandler.
func (*DockerHandler) OnDestControl ¶
func (h *DockerHandler) OnDestControl(ctx context.Context, msg *protocol.AirbyteControlMessage) error
OnDestControl reports CONTROL config updates from the destination connector; a failed report is logged and the sync goes on.
func (*DockerHandler) OnLog ¶
func (h *DockerHandler) OnLog(ctx context.Context, line string) error
OnLog buffers a pre-formatted log line and reports the buffer once it holds a full batch. Errors are non-fatal to avoid disrupting sync execution.
func (*DockerHandler) OnSourceControl ¶
func (h *DockerHandler) OnSourceControl(ctx context.Context, msg *protocol.AirbyteControlMessage) error
OnSourceControl reports CONTROL config updates from the source connector; the control plane encrypts and persists them. A failed report is logged and the sync goes on.
func (*DockerHandler) OnSourceTrace ¶
func (h *DockerHandler) OnSourceTrace(ctx context.Context, msg *protocol.AirbyteTraceMessage) error
OnSourceTrace logs error traces from the source connector.
func (*DockerHandler) OnStateConfirmed ¶
func (h *DockerHandler) OnStateConfirmed(ctx context.Context, msg *protocol.AirbyteStateMessage) error
OnStateConfirmed reports confirmed state to the control plane.
func (*DockerHandler) StartLogFlush ¶
func (h *DockerHandler) StartLogFlush(ctx context.Context) (stop func())
StartLogFlush reports buffered log lines every flush interval until the returned stop function is called. Stop reports whatever is still buffered and returns once it is stored, so it must run when the attempt ends, whatever the outcome. Reports outlive ctx cancellation.
type DockerHandlerParams ¶
type DockerHandlerParams struct {
ConnectionID uuid.UUID
SourceID uuid.UUID
DestinationID uuid.UUID
JobID uuid.UUID
Reporter pipelineexec.SyncReporter
LogBatchSize int
LogFlushInterval time.Duration
Logger *logging.Logger
}
DockerHandlerParams holds constructor parameters for DockerHandler.
type DockerSyncExecutor ¶
type DockerSyncExecutor struct {
// contains filtered or unexported fields
}
DockerSyncExecutor runs a sync job from the pre-loaded data of a SyncBundle and reports state, config updates and logs through the SyncReporter.
func NewDockerSyncExecutor ¶
func NewDockerSyncExecutor(params DockerSyncExecutorParams) *DockerSyncExecutor
NewDockerSyncExecutor creates a new sync executor.
func (*DockerSyncExecutor) Execute ¶
func (e *DockerSyncExecutor) Execute(ctx context.Context, bundle *pipelineexec.SyncBundle, onStarted func()) (*pipelineservice.SyncStats, error)
Execute runs a sync for the given bundle of pre-loaded data. onStarted is called once both the source and destination containers have started.
type DockerSyncExecutorParams ¶
type DockerSyncExecutorParams struct {
Client *ConnectorClient
Reporter pipelineexec.SyncReporter
RuntimeDefaults pipelineservice.RuntimeDefaults
IdleTimeout time.Duration
// TempDirRoot is where the job's /local directory is created; see
// NewContainerRunner.
TempDirRoot string
// LocalRoot, when set, is the host directory mounted at /local in every
// job instead of a per-job directory; it is never removed.
LocalRoot string
Logger *logging.Logger
}
DockerSyncExecutorParams holds all dependencies for DockerSyncExecutor.
type DockerSyncWorker ¶
type DockerSyncWorker struct {
// contains filtered or unexported fields
}
DockerSyncWorker is a jobber that polls for scheduled jobs, claims them, and runs sync execution using DockerSyncExecutor (which uses pipelineexec.Run with DockerHandler). It uses ExecutorBackend for all server communication per D-14. Runs at ~1s interval via the jobber framework.
func NewDockerSyncWorker ¶
func NewDockerSyncWorker(params DockerSyncWorkerParams) *DockerSyncWorker
NewDockerSyncWorker creates a new DockerSyncWorker with all dependencies. The workerID is derived from hostname for container identification per D-11.
func (*DockerSyncWorker) Execute ¶
func (w *DockerSyncWorker) Execute(ctx context.Context) error
Execute polls for and claims a scheduled job, then spawns a goroutine to execute it. Returns immediately so the jobber timer resets. Checks the concurrency semaphore BEFORE claiming to avoid claiming jobs that cannot be executed (job stays in queue for the next poll cycle).
type DockerSyncWorkerParams ¶
type DockerSyncWorkerParams struct {
Backend pipelineexec.ExecutorBackend
Executor *DockerSyncExecutor
Metrics *pipelinemetrics.MetricsCollector
Manager *jobutil.WorkerManager
MaxSyncDuration time.Duration
HeartbeatInterval time.Duration
MaxConcurrentJobs int
Logger *logging.Logger
}
DockerSyncWorkerParams holds all constructor dependencies for DockerSyncWorker.
type DockerTaskWorker ¶
type DockerTaskWorker struct {
// contains filtered or unexported fields
}
DockerTaskWorker polls for pending connector tasks and executes them using ConnectorClient (Docker). Follows the same semaphore pattern as DockerSyncWorker but without heartbeat (tasks are short-lived). Per D-16, uses a separate goroutine pool.
func NewDockerTaskWorker ¶
func NewDockerTaskWorker(params DockerTaskWorkerParams) *DockerTaskWorker
NewDockerTaskWorker creates a new DockerTaskWorker with all dependencies.
func (*DockerTaskWorker) Execute ¶
func (w *DockerTaskWorker) Execute(ctx context.Context) error
Execute polls for and claims a pending connector task, then spawns a goroutine to execute it. Returns immediately so the jobber timer resets. Checks the concurrency semaphore BEFORE claiming to avoid claiming tasks that cannot be executed.
type DockerTaskWorkerParams ¶
type DockerTaskWorkerParams struct {
Backend pipelineexec.ExecutorBackend
Client *ConnectorClient
RuntimeDefaults pipelineservice.RuntimeDefaults
Manager *jobutil.WorkerManager
MaxConcurrent int
Logger *logging.Logger
}
DockerTaskWorkerParams holds all constructor dependencies.
type OrphanCleaner ¶
type OrphanCleaner struct {
// contains filtered or unexported fields
}
OrphanCleaner finds and removes Docker containers that were created for sync jobs but are no longer associated with an active job. It touches only containers whose OwnerLabel matches its own owner id.
func NewOrphanCleaner ¶
func NewOrphanCleaner(runner *ContainerRunner, checker OrphanJobChecker, logger *logging.Logger) *OrphanCleaner
NewOrphanCleaner creates a new OrphanCleaner.
func (*OrphanCleaner) Cleanup ¶
func (oc *OrphanCleaner) Cleanup(ctx context.Context) error
Cleanup lists all Synclet-managed containers, checks if their associated job is still active, and stops+removes orphaned containers.
func (*OrphanCleaner) CleanupAll ¶
func (oc *OrphanCleaner) CleanupAll(ctx context.Context) error
CleanupAll lists all Synclet-managed containers and removes orphaned ones WITHOUT applying the grace period. Use this on startup when no containers from the current process exist yet (any managed container is from a previous run).
type OrphanJobChecker ¶
OrphanJobChecker checks if a sync job is still active in the database.
type ResourceLimits ¶
type ResourceLimits struct {
MemoryLimit int64
CPULimit float64
MemoryRequest int64
CPURequest float64
NetworkMode string
LocalDir string
}
ResourceLimits holds memory and CPU limits and requests for a container, plus Docker-specific runtime options: the network and the host directory mounted at /local.
type RunOptions ¶
type RunOptions = container.RunOptions
RunOptions is an alias for container.RunOptions for backwards compatibility.