airbyte

package

v0.0.0-...-8aeb8a1 Latest Latest Go to latest Published: Jun 29, 2023 License: MIT Imports: 26 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/jitsucom/jitsu

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func Init(ctx context.Context, containerizedRun bool, configDir, workspaceVolume string, ...) error
type Bridge
type Catalog
type CatalogRow
type LogRow
type RecordRow
type Row
type Runner
- func NewRunner(sourceID, dockerImage, imageVersion, identifier string) *Runner
type Schema
type StateRow
type StatusRow
type Stream
type WrappedStream

Constants ¶

View Source

const (
	BridgeType                  = "airbyte_bridge"
	DockerImageRepositoryPrefix = "airbyte/"

	VolumeAlias     = "/tmp/airbyte/"
	DockerCommand   = "docker"
	LatestVersion   = "latest"
	MountVolumeType = "volume"
)

View Source

const (
	LogType              = "LOG"
	ConnectionStatusType = "CONNECTION_STATUS"
	StateType            = "STATE"
	RecordType           = "RECORD"
	CatalogType          = "CATALOG"
	SpecType             = "SPEC"
)

Variables ¶

View Source

var (
	Instance      *Bridge
	InstanceError error
)

Functions ¶

func Init ¶

func Init(ctx context.Context, containerizedRun bool, configDir, workspaceVolume string, batchSize int, logWriter io.Writer) error

Init initializes airbyte Bridge

Types ¶

type Bridge ¶

type Bridge struct {
	LogWriter       io.Writer
	ConfigDir       string
	WorkspaceVolume string
	// contains filtered or unexported fields
}

func (*Bridge) AddAirbytePrefix ¶

func (b *Bridge) AddAirbytePrefix(dockerImage string) string

AddAirbytePrefix adds airbyte/ prefix to dockerImage if doesn't exist

func (*Bridge) BuildMsg ¶

func (b *Bridge) BuildMsg(prefix string, outWriter, errWriter *logging.StringWriter, err error) string

BuildMsg returns formatted error

func (*Bridge) IsImagePulled ¶

func (b *Bridge) IsImagePulled(dockerRepoImage, version string) (bool, error)

IsImagePulled returns true if the image is pulled or start pulling the image asynchronously and returns false

type Catalog ¶

type Catalog struct {
	Streams []*WrappedStream `json:"streams,omitempty"`
}

Catalog is a dto for formatted airbyte catalog serialization

type CatalogRow ¶

type CatalogRow struct {
	Streams []*Stream `json:"streams,omitempty"`
}

CatalogRow is a dto for Airbyte discover output serialization

type LogRow ¶

type LogRow struct {
	Level   string `json:"level,omitempty"`
	Message string `json:"message,omitempty"`
}

LogRow is a dto for airbyte logs serialization

type RecordRow ¶

type RecordRow struct {
	Stream string                 `json:"stream,omitempty"`
	Data   map[string]interface{} `json:"data,omitempty"`
}

RecordRow is a dto for airbyte record serialization

type Row ¶

type Row struct {
	Type             string                 `json:"type"`
	Log              *LogRow                `json:"log,omitempty"`
	ConnectionStatus *StatusRow             `json:"connectionStatus,omitempty"`
	State            *StateRow              `json:"state,omitempty"`
	Record           *RecordRow             `json:"record,omitempty"`
	Catalog          *CatalogRow            `json:"catalog,omitempty"`
	Spec             map[string]interface{} `json:"spec,omitempty"`
}

Row is a dto for airbyte output row representation

type Runner ¶

type Runner struct {

	//DockerImage without 'airbyte/' prefix
	DockerImage string
	Version     string
	// contains filtered or unexported fields
}

Runner is an Airbyte Docker runner Can only be used once Self-closed (see run() func)

func NewRunner ¶

func NewRunner(sourceID, dockerImage, imageVersion, identifier string) *Runner

NewRunner returns configured Airbyte Runner

func (*Runner) Check ¶

func (r *Runner) Check(airbyteSourceConfig interface{}) error

func (*Runner) Close ¶

func (r *Runner) Close() error

func (*Runner) Discover ¶

func (r *Runner) Discover(airbyteSourceConfig interface{}, timeout time.Duration) (*CatalogRow, error)

Discover returns discovered raw catalog

func (*Runner) IsReady ¶

func (r *Runner) IsReady() (bool, error)

func (*Runner) Read ¶

func (r *Runner) Read(dataConsumer base.CLIDataConsumer, streamsRepresentation map[string]*base.StreamRepresentation, taskLogger logging.TaskLogger, taskCloser base.CLITaskCloser, sourceID, statePath string) error

func (*Runner) Spec ¶

func (r *Runner) Spec() (interface{}, error)

Spec runs airbyte docker spec command and returns spec and err if occurred

func (*Runner) String ¶

func (r *Runner) String() string

String returns exec command string

type Schema ¶

type Schema struct {
	Properties map[string]*base.Property `json:"properties,omitempty"`
}

Schema is a dto for Airbyte catalog Schema object serialization

type StateRow ¶

type StateRow struct {
	Data map[string]interface{} `json:"data,omitempty"`
}

StateRow is a dto for airbyte state serialization

type StatusRow ¶

type StatusRow struct {
	Status  string `json:"status,omitempty"`
	Message string `json:"message,omitempty"`
}

StatusRow is a dto for airbyte result status serialization

type Stream ¶

type Stream struct {
	Name                    string     `json:"name,omitempty"`
	JsonSchema              *Schema    `json:"json_schema,omitempty"`
	SupportedSyncModes      []string   `json:"supported_sync_modes,omitempty"`
	SourceDefinedPrimaryKey [][]string `json:"source_defined_primary_key,omitempty"`
	SourceDefinedCursor     bool       `json:"source_defined_cursor"`
	DefaultCursorField      []string   `json:"default_cursor_field,omitempty"`

	Namespace string `json:"namespace,omitempty"`

	SyncMode            string   `json:"-" yaml:"-"` //without serialization
	SelectedCursorField []string `json:"-" yaml:"-"` //without serialization
}

Stream is a dto for Airbyte catalog Stream object serialization

type WrappedStream ¶

type WrappedStream struct {
	SyncMode            string   `json:"sync_mode,omitempty"`
	DestinationSyncMode string   `json:"destination_sync_mode,omitempty"`
	CursorField         []string `json:"cursor_field,omitempty"`
	Stream              *Stream  `json:"stream,omitempty"`
}

WrappedStream is a dto for formatted stream

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL