Documentation
¶
Index ¶
- Constants
- Variables
- func JSONSchemaValidator(jsonSchema string) (*jsonschema.Schema, error)
- func MatchesTable(name string, includeTablesPattern []string, skipTablesPattern []string) bool
- func RecordsDiff(sc *arrow.Schema, have, want []arrow.RecordBatch) string
- func TableDiff(have, want arrow.Table) string
- func TestWriterSuiteRunner(t *testing.T, p *Plugin, tests WriterTestSuiteTests, ...)
- func TotalRows(records []arrow.RecordBatch) int64
- func UnimplementedTestConnectionFn(context.Context, zerolog.Logger, []byte) error
- func ValidateNoEmptyColumns(t *testing.T, tables schema.Tables, messages message.SyncMessages)
- func ValidateSensitiveColumns(t *testing.T, tables schema.Tables)
- func WithHomogeneousTypes() func(o *WriterTestSuite)
- func WithRandomSeed(seed int64) func(o *WriterTestSuite)
- func WithTestDataOptions(opts schema.TestSourceOptions) func(o *WriterTestSuite)
- func WithTestIgnoreNullsInLists() func(o *WriterTestSuite)
- func WithTestSourceAllowNull(allowNull func(arrow.DataType) bool) func(o *WriterTestSuite)
- type AllowNullFunc
- type AssessCategory
- type AssessOptions
- type Assessor
- type BackendOptions
- type BuildTarget
- type Client
- type ColumnFinding
- type ConnectionTester
- type DestinationClient
- type Evidence
- type Kind
- type Meta
- type MigrateMode
- type Migrations
- type NewClientFunc
- type NewClientOptions
- type NewPluginFunc
- type NewSourceClientFunc
- type OnBeforeSender
- type OnSyncFinisher
- type Option
- type PackageType
- type Plugin
- func (p *Plugin) AssessTables(ctx context.Context, tables []TablePair, options AssessOptions) ([]TableFinding, error)
- func (p *Plugin) Close(ctx context.Context) error
- func (p *Plugin) Init(ctx context.Context, spec []byte, options NewClientOptions) error
- func (p *Plugin) InvocationID() string
- func (p *Plugin) JSONSchema() string
- func (p *Plugin) Kind() Kind
- func (p *Plugin) Meta() Meta
- func (p *Plugin) Name() string
- func (p *Plugin) OnBeforeSend(ctx context.Context, msg message.SyncMessage) (message.SyncMessage, error)
- func (p *Plugin) OnSyncFinish(ctx context.Context) error
- func (p *Plugin) PackageAndVersion() string
- func (p *Plugin) Read(ctx context.Context, table *schema.Table, res chan<- arrow.RecordBatch) error
- func (p *Plugin) SetLogger(logger zerolog.Logger)
- func (p *Plugin) SetSkipTableValidation(v bool)
- func (p *Plugin) SetSkipUsageClient(v bool)
- func (p *Plugin) Sync(ctx context.Context, options SyncOptions, res chan<- message.SyncMessage) error
- func (p *Plugin) SyncAll(ctx context.Context, options SyncOptions) (message.SyncMessages, error)
- func (p *Plugin) Tables(ctx context.Context, options TableOptions) (schema.Tables, error)
- func (p *Plugin) Targets() []BuildTarget
- func (p *Plugin) Team() string
- func (p *Plugin) TestConnection(ctx context.Context, logger zerolog.Logger, spec []byte) error
- func (p *Plugin) Transform(ctx context.Context, recvRecords <-chan arrow.RecordBatch, ...) error
- func (p *Plugin) TransformSchema(ctx context.Context, old *arrow.Schema) (*arrow.Schema, error)
- func (p *Plugin) Version() string
- func (p *Plugin) Write(ctx context.Context, res <-chan message.WriteMessage) error
- func (p *Plugin) WriteAll(ctx context.Context, resources []message.WriteMessage) error
- type SafeMigrations
- type Shard
- type SourceClient
- type SyncOptions
- type TableFinding
- type TableOptions
- type TablePair
- type TestConnError
- type TransformerClient
- type UnimplementedDestination
- type UnimplementedSource
- type UnimplementedTransformer
- type WriteTests
- type WriterTestSuite
- type WriterTestSuiteTests
Constants ¶
const ( GoOSLinux = "linux" GoOSWindows = "windows" GoOSDarwin = "darwin" GoArchAmd64 = "amd64" GoArchArm64 = "arm64" )
const AssessNotSupportedReason = "destination does not support assessment"
Variables ¶
var DefaultBuildTargets = []BuildTarget{ {OS: GoOSLinux, Arch: GoArchAmd64, Tags: []string{"grpcnotrace"}}, {OS: GoOSLinux, Arch: GoArchArm64, Tags: []string{"grpcnotrace"}}, {OS: GoOSWindows, Arch: GoArchAmd64, Tags: []string{"grpcnotrace"}}, {OS: GoOSDarwin, Arch: GoArchAmd64, Tags: []string{"grpcnotrace"}}, {OS: GoOSDarwin, Arch: GoArchArm64, Tags: []string{"grpcnotrace"}}, }
var ErrNotImplemented = errors.New("not implemented")
Functions ¶
func JSONSchemaValidator ¶ added in v4.10.0
func JSONSchemaValidator(jsonSchema string) (*jsonschema.Schema, error)
func MatchesTable ¶
func RecordsDiff ¶ added in v4.5.2
func RecordsDiff(sc *arrow.Schema, have, want []arrow.RecordBatch) string
func TestWriterSuiteRunner ¶
func TestWriterSuiteRunner(t *testing.T, p *Plugin, tests WriterTestSuiteTests, opts ...func(o *WriterTestSuite))
func TotalRows ¶
func TotalRows(records []arrow.RecordBatch) int64
func UnimplementedTestConnectionFn ¶ added in v4.43.0
func ValidateNoEmptyColumns ¶ added in v4.1.0
func ValidateSensitiveColumns ¶ added in v4.80.1
func WithHomogeneousTypes ¶ added in v4.31.0
func WithHomogeneousTypes() func(o *WriterTestSuite)
func WithRandomSeed ¶ added in v4.1.0
func WithRandomSeed(seed int64) func(o *WriterTestSuite)
func WithTestDataOptions ¶
func WithTestDataOptions(opts schema.TestSourceOptions) func(o *WriterTestSuite)
func WithTestIgnoreNullsInLists ¶
func WithTestIgnoreNullsInLists() func(o *WriterTestSuite)
func WithTestSourceAllowNull ¶
func WithTestSourceAllowNull(allowNull func(arrow.DataType) bool) func(o *WriterTestSuite)
Types ¶
type AllowNullFunc ¶
type AssessCategory ¶ added in v4.97.0
type AssessCategory int
const ( AssessCategoryUnknown AssessCategory = iota AssessCategoryNoChange AssessCategoryAutomaticallyMigratable AssessCategoryManualMigrationRequired AssessCategoryTableRemoved AssessCategoryFileSchemaChanged )
type AssessOptions ¶ added in v4.97.0
type AssessOptions struct {
MigrateForce bool
}
type Assessor ¶ added in v4.97.0
type Assessor interface {
AssessTables(ctx context.Context, tables []TablePair, options AssessOptions) ([]TableFinding, error)
}
Assessor is an optional DestinationClient interface that reports how schema changes would be applied, without writing anything. It is called after Init with NoConnection set, so implementations must not open database or cloud connections.
type BackendOptions ¶
type BuildTarget ¶ added in v4.5.0
type BuildTarget struct {
OS string `json:"os"`
Arch string `json:"arch"`
CGO bool `json:"cgo"`
Env []string `json:"env"`
Tags []string `json:"tags"`
IncludeSymbols bool `json:"include_symbols"`
}
func (BuildTarget) EnvVariables ¶ added in v4.27.2
func (t BuildTarget) EnvVariables() []string
type Client ¶
type Client interface {
SourceClient
DestinationClient
TransformerClient
}
type ColumnFinding ¶ added in v4.97.0
type ConnectionTester ¶ added in v4.43.0
type DestinationClient ¶
type Meta ¶ added in v4.24.0
type Meta struct {
Team cqapi.PluginTeam
Kind cqapi.PluginKind
Name cqapi.PluginName
SkipUsageClient bool
}
type MigrateMode ¶
type MigrateMode int
const ( MigrateModeSafe MigrateMode = iota MigrateModeForce )
func (MigrateMode) String ¶
func (m MigrateMode) String() string
type Migrations ¶ added in v4.34.0
Migrations defines which migrations should be skipped completely
type NewClientFunc ¶
type NewClientOptions ¶
type NewPluginFunc ¶
type NewPluginFunc func() *Plugin
type NewSourceClientFunc ¶
type OnBeforeSender ¶ added in v4.17.0
type OnBeforeSender interface {
OnBeforeSend(context.Context, message.SyncMessage) (message.SyncMessage, error)
}
type OnSyncFinisher ¶ added in v4.17.0
OnSyncFinisher is an interface that can be implemented by a plugin client to be notified when a sync finishes.
type Option ¶
type Option func(*Plugin)
func WithBuildTargets ¶ added in v4.8.0
func WithBuildTargets(targets []BuildTarget) Option
func WithConnectionTester ¶ added in v4.43.0
func WithConnectionTester(tester ConnectionTester) Option
WithConnectionTester can be specified by a plugin to enable explicit connection testing, given a spec.
func WithJSONSchema ¶ added in v4.9.0
type PackageType ¶ added in v4.5.0
type PackageType string
const (
PackageTypeNative PackageType = "native"
)
type Plugin ¶
type Plugin struct {
// contains filtered or unexported fields
}
Plugin is the base structure required to pass to sdk.serve We take a declarative approach to API here similar to Cobra
func NewPlugin ¶
func NewPlugin(name string, version string, newClient NewClientFunc, options ...Option) *Plugin
NewPlugin returns a new CloudQuery Plugin with the given name, version and implementation. Depending on the options, it can be a write-only plugin, read-only plugin, or both.
func NewSourcePlugin ¶
func NewSourcePlugin(name string, version string, newClient NewSourceClientFunc, options ...Option) *Plugin
NewSourcePlugin returns a new CloudQuery Plugin with the given name, version and implementation. Source plugins only support read operations. For Read & Write plugin use NewPlugin.
func (*Plugin) AssessTables ¶ added in v4.97.0
func (p *Plugin) AssessTables(ctx context.Context, tables []TablePair, options AssessOptions) ([]TableFinding, error)
AssessTables returns one Unknown finding per table when the client does not implement Assessor.
func (*Plugin) InvocationID ¶ added in v4.36.0
InvocationID returns the invocation ID for the current execution
func (*Plugin) JSONSchema ¶ added in v4.28.0
func (*Plugin) OnBeforeSend ¶ added in v4.17.0
func (p *Plugin) OnBeforeSend(ctx context.Context, msg message.SyncMessage) (message.SyncMessage, error)
OnBeforeSend gets called before every message is sent to the destination. A plugin client that implements the OnBeforeSender interface will have this method called.
func (*Plugin) OnSyncFinish ¶ added in v4.17.0
OnSyncFinish gets called after a sync finishes.
func (*Plugin) PackageAndVersion ¶ added in v4.79.0
func (*Plugin) Read ¶
Read is read data from the requested table to the given channel, returned in the same format as the table
func (*Plugin) SetSkipTableValidation ¶ added in v4.32.0
SetSkipTableValidation sets whether table validation should be skipped
func (*Plugin) SetSkipUsageClient ¶ added in v4.24.0
SetSkipUsageClient sets whether the usage client should be skipped
func (*Plugin) Sync ¶
func (p *Plugin) Sync(ctx context.Context, options SyncOptions, res chan<- message.SyncMessage) error
Sync is syncing data from the requested tables in spec to the given channel
func (*Plugin) SyncAll ¶
func (p *Plugin) SyncAll(ctx context.Context, options SyncOptions) (message.SyncMessages, error)
func (*Plugin) Targets ¶ added in v4.5.0
func (p *Plugin) Targets() []BuildTarget
func (*Plugin) TestConnection ¶ added in v4.43.0
func (*Plugin) Transform ¶ added in v4.50.0
func (p *Plugin) Transform(ctx context.Context, recvRecords <-chan arrow.RecordBatch, sendRecords chan<- arrow.RecordBatch) error
func (*Plugin) TransformSchema ¶ added in v4.56.0
type SafeMigrations ¶
type SafeMigrations struct {
AddColumn bool
AddColumnNotNull bool
RemoveColumn bool
RemoveColumnNotNull bool
ChangeColumn bool
RemoveUniqueConstraint bool
MovePKToCQOnly bool
}
SafeMigrations defines which migrations are supported by the plugin in safe migrate mode
type SourceClient ¶
type SourceClient interface {
Close(ctx context.Context) error
Tables(ctx context.Context, options TableOptions) (schema.Tables, error)
Sync(ctx context.Context, options SyncOptions, res chan<- message.SyncMessage) error
}
type SyncOptions ¶
type TableFinding ¶ added in v4.97.0
type TableFinding struct {
TableName string
Category AssessCategory
SafeModeBehavior string
ForcedModeBehavior string
Columns []ColumnFinding
Evidence []Evidence
IncompleteCoverageReason string
}
type TableOptions ¶
type TablePair ¶ added in v4.97.0
TablePair holds a table before and after a schema change. Old is nil for an added table, New is nil for a removed one.
type TestConnError ¶ added in v4.43.0
func NewTestConnError ¶ added in v4.43.0
func NewTestConnError(code string, err error) *TestConnError
func (*TestConnError) Error ¶ added in v4.43.0
func (e *TestConnError) Error() string
func (*TestConnError) Is ¶ added in v4.43.0
func (e *TestConnError) Is(err error) bool
func (*TestConnError) Unwrap ¶ added in v4.43.0
func (e *TestConnError) Unwrap() error
type TransformerClient ¶ added in v4.50.0
type UnimplementedDestination ¶
type UnimplementedDestination struct {
UnimplementedTransformer
}
func (UnimplementedDestination) Read ¶
func (UnimplementedDestination) Read(context.Context, *schema.Table, chan<- arrow.RecordBatch) error
func (UnimplementedDestination) Write ¶
func (UnimplementedDestination) Write(context.Context, <-chan message.WriteMessage) error
type UnimplementedSource ¶
type UnimplementedSource struct {
UnimplementedTransformer
}
func (UnimplementedSource) Sync ¶
func (UnimplementedSource) Sync(context.Context, SyncOptions, chan<- message.SyncMessage) error
func (UnimplementedSource) Tables ¶
func (UnimplementedSource) Tables(context.Context, TableOptions) (schema.Tables, error)
type UnimplementedTransformer ¶ added in v4.50.0
type UnimplementedTransformer struct{}
func (UnimplementedTransformer) Transform ¶ added in v4.50.0
func (UnimplementedTransformer) Transform(context.Context, <-chan arrow.RecordBatch, chan<- arrow.RecordBatch) error
func (UnimplementedTransformer) TransformSchema ¶ added in v4.56.0
type WriteTests ¶ added in v4.38.0
type WriteTests struct {
DuplicatePK bool
}
WriteTests defines which tests should be skipped in the write test suite
type WriterTestSuite ¶
type WriterTestSuite struct {
// contains filtered or unexported fields
}
type WriterTestSuiteTests ¶
type WriterTestSuiteTests struct {
// SkipUpsert skips testing with message.Insert and Upsert=true.
// Usually when a destination is not supporting primary keys
SkipUpsert bool
// SkipDeleteStale skips testing message.Delete events.
SkipDeleteStale bool
// SkipDeleteRecord skips testing message.DeleteRecord events.
SkipDeleteRecord bool
// SkipAppend skips testing message.Insert and Upsert=false.
SkipInsert bool
// SkipMigrate skips testing migration
SkipMigrate bool
// SafeMigrations defines which tests should work with force migration
// and which should pass with safe migration
SafeMigrations SafeMigrations
SkipSpecificMigrations Migrations
SkipSpecificWriteTests WriteTests
}
Source Files
¶
- append_from_string.go
- diff.go
- meta.go
- nulls.go
- options.go
- plugin.go
- plugin_assess.go
- plugin_destination.go
- plugin_package.go
- plugin_read.go
- plugin_source.go
- plugin_testconn.go
- plugin_transformer.go
- sort.go
- testing.go
- testing_validation.go
- testing_write.go
- testing_write_delete.go
- testing_write_insert.go
- testing_write_migrate.go
- testing_write_upsert.go
- validate.go