Documentation
¶
Overview ¶
Package specs includes specs for source and destination plugins including parsers and readers.
Index ¶
- Variables
- func ReplaceVariables(src string, variables Variables, shouldReplaceLocalhost bool) (string, error)
- func SpecUnmarshalYamlStrict(b []byte, spec *Spec) error
- func WarnOnOutdatedVersions(ctx context.Context, p *managedplugin.PluginVersionWarner, sources []*Source, ...)
- type BackendOptions
- type Destination
- type Kind
- type Metadata
- type MigrateMode
- type PKMode
- type PluginVariables
- type Registry
- type Source
- type Spec
- type SpecReader
- func (r *SpecReader) GetDestinationByName(name string) *Destination
- func (r *SpecReader) GetDestinationNamesForSource(name string) []string
- func (r *SpecReader) GetDestinationWarningsByName(name string) Warnings
- func (r *SpecReader) GetSourceByName(name string) *Source
- func (r *SpecReader) GetSourceWarningsByName(name string) Warnings
- func (r *SpecReader) GetTransformerWarningsByName(name string) Warnings
- type Transformer
- type Variables
- type Warnings
- type WriteMode
Constants ¶
This section is empty.
Variables ¶
var ( AllKinds = [...]string{ KindSource: "source", KindDestination: "destination", KindTransformer: "transformer", } )
var ( AllMigrateModes = [...]string{ MigrateModeSafe: "safe", MigrateModeForced: "forced", } )
var ( AllPKModes = [...]string{ PKModeDefaultKeys: "default", PKModeCQID: "cq-id-only", } )
var ( AllRegistries = [...]string{ RegistryUnset: "", RegistryGitHub: "github", RegistryLocal: "local", RegistryGRPC: "grpc", RegistryDocker: "docker", RegistryCloudQuery: "cloudquery", } )
var ( AllWriteModes = [...]string{ WriteModeOverwriteDeleteStale: "overwrite-delete-stale", WriteModeOverwrite: "overwrite", WriteModeAppend: "append", } )
Functions ¶
func ReplaceVariables ¶
ReplaceVariables replaces variables starting with @@ in the src string with the values from the values from variables by dot notation. Example: @@plugins.aws.connection will be replaced with the value of variables.Plugins["aws"].Connection
func SpecUnmarshalYamlStrict ¶
func WarnOnOutdatedVersions ¶
func WarnOnOutdatedVersions(ctx context.Context, p *managedplugin.PluginVersionWarner, sources []*Source, destinations []*Destination, transformers []*Transformer)
Types ¶
type BackendOptions ¶
type BackendOptions struct {
// The name of the table to store the key-value pairs for incremental progress.
TableName string `json:"table_name,omitempty" jsonschema:"required,minLength=1"`
// Connection string for the destination plugin.
// Can be either `@@plugin.name.connection` or a fully-qualified gRPC connection string.
Connection string `json:"connection,omitempty" jsonschema:"required,minLength=1"`
}
Backend options to be used in conjunction with incremental tables (stores the incremental progres)
func (*BackendOptions) PluginName ¶
func (b *BackendOptions) PluginName() string
PluginName returns the name of the plugin from the connection string variable.
Note that `Connection` gets string replaced with the actual connection value during the sync process, so calling this function will only work before the sync process starts.
type Destination ¶
type Destination struct {
Metadata
// Destination plugin write mode
WriteMode WriteMode `json:"write_mode,omitempty" jsonschema:"default=overwrite-delete-stale"`
// Destination plugin migrate mode
MigrateMode MigrateMode `json:"migrate_mode,omitempty" jsonschema:"default=safe"`
// Destination plugin PK mode
PKMode PKMode `json:"pk_mode,omitempty" jsonschema:"default=default"`
SyncGroupId string `json:"sync_group_id,omitempty"`
SyncSummary bool `json:"send_sync_summary,omitempty"`
// Transformers are the names of transformer plugins to send sync data through
Transformers []string `json:"transformers,omitempty"`
// Destination plugin own (nested) spec
Spec map[string]any `json:"spec,omitempty"`
}
Destination plugin spec
func (*Destination) GetWarnings ¶
func (*Destination) GetWarnings() Warnings
func (*Destination) RenderedSyncGroupId ¶
func (d *Destination) RenderedSyncGroupId(t time.Time, sync_id string) string
func (*Destination) SetDefaults ¶
func (d *Destination) SetDefaults()
func (*Destination) UnmarshalSpec ¶
func (d *Destination) UnmarshalSpec(out any) error
func (*Destination) Validate ¶
func (d *Destination) Validate() error
type Kind ¶
type Kind int
func KindFromString ¶
func (Kind) JSONSchemaExtend ¶
func (Kind) JSONSchemaExtend(sc *jsonschema.Schema)
func (Kind) MarshalJSON ¶
func (*Kind) UnmarshalJSON ¶
type Metadata ¶
type Metadata struct {
// Name of the plugin to use
Name string `json:"name" jsonschema:"required,minLength=1"`
// Version of the plugin to be used
Version string `json:"version"`
// Path is the canonical path to the plugin in a given registry
// For example:
// * for `registry: github` the `path` will be: `org/repo`
// * for `registry: local` the `path` will be the path to the binary: `./path/to/binary`
// * for `registry: grpc` the `path` will be the address of the gRPC server: `host:port`
// * for `registry: cloudquery` the `path` will be: `team/name`
Path string `json:"path" jsonschema:"required,minLength=1"`
// Registry can be "", "github", "local", "grpc", "docker", "cloudquery"
Registry Registry `json:"registry,omitempty" jsonschema:"default=cloudquery"`
// DockerRegistryAuthToken is the token to use to authenticate with the docker registry
DockerRegistryAuthToken string `json:"docker_registry_auth_token,omitempty"`
// contains filtered or unexported fields
}
Spec part to define exact plugin: name, version & location.
func (Metadata) JSONSchemaExtend ¶
func (Metadata) JSONSchemaExtend(sc *jsonschema.Schema)
JSONSchemaExtend has to be in sync with Registry.NeedVersion
func (*Metadata) RegistryInferred ¶
func (*Metadata) SetDefaults ¶
func (m *Metadata) SetDefaults()
func (Metadata) VersionString ¶
type MigrateMode ¶
type MigrateMode int
const ( MigrateModeSafe MigrateMode = iota MigrateModeForced )
func MigrateModeFromString ¶
func MigrateModeFromString(s string) (MigrateMode, error)
func (MigrateMode) JSONSchemaExtend ¶
func (MigrateMode) JSONSchemaExtend(sc *jsonschema.Schema)
func (MigrateMode) MarshalJSON ¶
func (m MigrateMode) MarshalJSON() ([]byte, error)
func (MigrateMode) String ¶
func (m MigrateMode) String() string
func (*MigrateMode) UnmarshalJSON ¶
func (m *MigrateMode) UnmarshalJSON(data []byte) (err error)
type PKMode ¶
type PKMode int
func PKModeFromString ¶
func (PKMode) JSONSchemaExtend ¶
func (PKMode) JSONSchemaExtend(sc *jsonschema.Schema)
func (PKMode) MarshalJSON ¶
func (*PKMode) UnmarshalJSON ¶
type PluginVariables ¶
type PluginVariables struct {
Connection string `json:"connection"`
}
type Registry ¶
type Registry int
func RegistryFromString ¶
func (Registry) JSONSchemaExtend ¶
func (Registry) JSONSchemaExtend(sc *jsonschema.Schema)
func (Registry) MarshalJSON ¶
func (Registry) NeedVersion ¶
NeedVersion has to be in sync with Metadata.JSONSchemaExtend
func (*Registry) UnmarshalJSON ¶
type Source ¶
type Source struct {
Metadata
// Tables to sync from the source plugin
Tables []string `json:"tables,omitempty" jsonschema:"required,minItems=1,minLength=1"`
// SkipTables defines tables to skip when syncing data. Useful if a glob pattern is used in Tables
SkipTables []string `json:"skip_tables,omitempty" jsonschema:"minLength=1"`
// SkipDependentTables changes the matching behavior with regard to dependent tables. If set to `false`, dependent tables will be included in the sync when their parents are matched, even if not explicitly included by the `tables` configuration.
SkipDependentTables *bool `json:"skip_dependent_tables,omitempty" jsonschema:"default=true"`
// Destinations are the names of destination plugins to send sync data to
Destinations []string `json:"destinations,omitempty" jsonschema:"required,minItems=1,minLength=1"`
// Optional Backend options for sync operation
BackendOptions *BackendOptions `json:"backend_options,omitempty"`
// Source plugin own (nested) spec
Spec map[string]any `json:"spec,omitempty"`
// DeterministicCQID is a flag that indicates whether the source plugin should generate a random UUID as the value of `_cq_id`
// or whether it should calculate a UUID that is a hash of the primary keys (if they exist) or the entire resource.
DeterministicCQID bool `json:"deterministic_cq_id,omitempty" jsonschema:"default=false"`
// If specified this will spawn the plugin with `--otel-endpoint`
OtelEndpoint string `json:"otel_endpoint,omitempty" jsonschema:"default="`
// If specified this will spawn the plugin with `--otel-endpoint-insecure`
OtelEndpointInsecure bool `json:"otel_endpoint_insecure,omitempty" jsonschema:"default=false"`
}
Source plugin spec
func (*Source) GetWarnings ¶
GetWarnings returns a list of deprecated options that were used in the source config. This should be called before SetDefaults.
func (Source) JSONSchemaExtend ¶
func (Source) JSONSchemaExtend(sc *jsonschema.Schema)
func (*Source) SetDefaults ¶
func (s *Source) SetDefaults()
func (*Source) UnmarshalSpec ¶
UnmarshalSpec unmarshals the internal spec into the given interface
type Spec ¶
type Spec struct {
// CloudQuery plugin kind
Kind Kind `json:"kind" jsonschema:"required"`
// CloudQuery plugin (top-level) spec
Spec any `json:"spec" jsonschema:"required"`
}
func (Spec) JSONSchemaExtend ¶
func (Spec) JSONSchemaExtend(sc *jsonschema.Schema)
func (*Spec) UnmarshalJSON ¶
type SpecReader ¶
type SpecReader struct {
Sources []*Source
Destinations []*Destination
Transformers []*Transformer
// contains filtered or unexported fields
}
func NewRelaxedSpecReader ¶
func NewRelaxedSpecReader(paths []string) (*SpecReader, error)
func NewSpecReader ¶
func NewSpecReader(paths []string) (*SpecReader, error)
func (*SpecReader) GetDestinationByName ¶
func (r *SpecReader) GetDestinationByName(name string) *Destination
func (*SpecReader) GetDestinationNamesForSource ¶
func (r *SpecReader) GetDestinationNamesForSource(name string) []string
func (*SpecReader) GetDestinationWarningsByName ¶
func (r *SpecReader) GetDestinationWarningsByName(name string) Warnings
func (*SpecReader) GetSourceByName ¶
func (r *SpecReader) GetSourceByName(name string) *Source
func (*SpecReader) GetSourceWarningsByName ¶
func (r *SpecReader) GetSourceWarningsByName(name string) Warnings
func (*SpecReader) GetTransformerWarningsByName ¶
func (r *SpecReader) GetTransformerWarningsByName(name string) Warnings
type Transformer ¶
type Transformer struct {
Metadata
// Transformer plugin own (nested) spec
Spec map[string]any `json:"spec,omitempty"`
}
Transformer plugin spec
func (*Transformer) GetWarnings ¶
func (*Transformer) GetWarnings() Warnings
func (*Transformer) UnmarshalSpec ¶
func (d *Transformer) UnmarshalSpec(out any) error
func (*Transformer) Validate ¶
func (d *Transformer) Validate() error
type Variables ¶
type Variables struct {
Plugins map[string]PluginVariables `json:"plugins"`
}
type WriteMode ¶
type WriteMode int
func WriteModeFromString ¶
func (WriteMode) JSONSchemaExtend ¶
func (WriteMode) JSONSchemaExtend(sc *jsonschema.Schema)