Help us shape the future of Go! Take the 2026 Go Developer Survey

specs

package
v6.41.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 10, 2026 License: MPL-2.0 Imports: 21 Imported by: 0

Documentation

Overview

Package specs includes specs for source and destination plugins including parsers and readers.

Index

Constants

This section is empty.

Variables

View Source
var (
	AllKinds = [...]string{
		KindSource:      "source",
		KindDestination: "destination",
		KindTransformer: "transformer",
	}
)
View Source
var (
	AllMigrateModes = [...]string{
		MigrateModeSafe:   "safe",
		MigrateModeForced: "forced",
	}
)
View Source
var (
	AllPKModes = [...]string{
		PKModeDefaultKeys: "default",
		PKModeCQID:        "cq-id-only",
	}
)
View Source
var (
	AllRegistries = [...]string{
		RegistryUnset:      "",
		RegistryGitHub:     "github",
		RegistryLocal:      "local",
		RegistryGRPC:       "grpc",
		RegistryDocker:     "docker",
		RegistryCloudQuery: "cloudquery",
	}
)
View Source
var (
	AllWriteModes = [...]string{
		WriteModeOverwriteDeleteStale: "overwrite-delete-stale",
		WriteModeOverwrite:            "overwrite",
		WriteModeAppend:               "append",
	}
)

Functions

func ReplaceVariables

func ReplaceVariables(src string, variables Variables, shouldReplaceLocalhost bool) (string, error)

ReplaceVariables replaces variables starting with @@ in the src string with the values from the values from variables by dot notation. Example: @@plugins.aws.connection will be replaced with the value of variables.Plugins["aws"].Connection

func SpecUnmarshalYamlStrict

func SpecUnmarshalYamlStrict(b []byte, spec *Spec) error

func WarnOnOutdatedVersions

func WarnOnOutdatedVersions(ctx context.Context, p *managedplugin.PluginVersionWarner, sources []*Source, destinations []*Destination, transformers []*Transformer)

Types

type BackendOptions

type BackendOptions struct {
	// The name of the table to store the key-value pairs for incremental progress.
	TableName string `json:"table_name,omitempty" jsonschema:"required,minLength=1"`

	// Connection string for the destination plugin.
	// Can be either `@@plugin.name.connection` or a fully-qualified gRPC connection string.
	Connection string `json:"connection,omitempty" jsonschema:"required,minLength=1"`
}

Backend options to be used in conjunction with incremental tables (stores the incremental progres)

func (*BackendOptions) PluginName

func (b *BackendOptions) PluginName() string

PluginName returns the name of the plugin from the connection string variable.

Note that `Connection` gets string replaced with the actual connection value during the sync process, so calling this function will only work before the sync process starts.

type Destination

type Destination struct {
	Metadata

	// Destination plugin write mode
	WriteMode WriteMode `json:"write_mode,omitempty" jsonschema:"default=overwrite-delete-stale"`

	// Destination plugin migrate mode
	MigrateMode MigrateMode `json:"migrate_mode,omitempty" jsonschema:"default=safe"`

	// Destination plugin PK mode
	PKMode PKMode `json:"pk_mode,omitempty" jsonschema:"default=default"`

	SyncGroupId string `json:"sync_group_id,omitempty"`

	SyncSummary bool `json:"send_sync_summary,omitempty"`

	// Transformers are the names of transformer plugins to send sync data through
	Transformers []string `json:"transformers,omitempty"`

	// Destination plugin own (nested) spec
	Spec map[string]any `json:"spec,omitempty"`
}

Destination plugin spec

func (*Destination) GetWarnings

func (*Destination) GetWarnings() Warnings

func (*Destination) RenderedSyncGroupId

func (d *Destination) RenderedSyncGroupId(t time.Time, sync_id string) string

func (*Destination) SetDefaults

func (d *Destination) SetDefaults()

func (*Destination) UnmarshalSpec

func (d *Destination) UnmarshalSpec(out any) error

func (*Destination) Validate

func (d *Destination) Validate() error

type Kind

type Kind int
const (
	KindSource Kind = iota
	KindDestination
	KindTransformer
)

func KindFromString

func KindFromString(s string) (Kind, error)

func (Kind) JSONSchemaExtend

func (Kind) JSONSchemaExtend(sc *jsonschema.Schema)

func (Kind) MarshalJSON

func (k Kind) MarshalJSON() ([]byte, error)

func (Kind) String

func (k Kind) String() string

func (*Kind) UnmarshalJSON

func (k *Kind) UnmarshalJSON(data []byte) (err error)

type Metadata

type Metadata struct {
	// Name of the plugin to use
	Name string `json:"name" jsonschema:"required,minLength=1"`

	// Version of the plugin to be used
	Version string `json:"version"`

	// Path is the canonical path to the plugin in a given registry
	// For example:
	// * for `registry: github` the `path` will be: `org/repo`
	// * for `registry: local` the `path` will be the path to the binary: `./path/to/binary`
	// * for `registry: grpc` the `path` will be the address of the gRPC server: `host:port`
	// * for `registry: cloudquery` the `path` will be: `team/name`
	Path string `json:"path" jsonschema:"required,minLength=1"`

	// Registry can be "", "github", "local", "grpc", "docker", "cloudquery"
	Registry Registry `json:"registry,omitempty" jsonschema:"default=cloudquery"`

	// DockerRegistryAuthToken is the token to use to authenticate with the docker registry
	DockerRegistryAuthToken string `json:"docker_registry_auth_token,omitempty"`
	// contains filtered or unexported fields
}

Spec part to define exact plugin: name, version & location.

func (Metadata) JSONSchemaExtend

func (Metadata) JSONSchemaExtend(sc *jsonschema.Schema)

JSONSchemaExtend has to be in sync with Registry.NeedVersion

func (*Metadata) RegistryInferred

func (m *Metadata) RegistryInferred() bool

func (*Metadata) SetDefaults

func (m *Metadata) SetDefaults()

func (*Metadata) Validate

func (m *Metadata) Validate() error

func (Metadata) VersionString

func (m Metadata) VersionString() string

type MigrateMode

type MigrateMode int
const (
	MigrateModeSafe MigrateMode = iota
	MigrateModeForced
)

func MigrateModeFromString

func MigrateModeFromString(s string) (MigrateMode, error)

func (MigrateMode) JSONSchemaExtend

func (MigrateMode) JSONSchemaExtend(sc *jsonschema.Schema)

func (MigrateMode) MarshalJSON

func (m MigrateMode) MarshalJSON() ([]byte, error)

func (MigrateMode) String

func (m MigrateMode) String() string

func (*MigrateMode) UnmarshalJSON

func (m *MigrateMode) UnmarshalJSON(data []byte) (err error)

type PKMode

type PKMode int
const (
	PKModeDefaultKeys PKMode = iota
	PKModeCQID
)

func PKModeFromString

func PKModeFromString(s string) (PKMode, error)

func (PKMode) JSONSchemaExtend

func (PKMode) JSONSchemaExtend(sc *jsonschema.Schema)

func (PKMode) MarshalJSON

func (m PKMode) MarshalJSON() ([]byte, error)

func (PKMode) String

func (m PKMode) String() string

func (*PKMode) UnmarshalJSON

func (m *PKMode) UnmarshalJSON(data []byte) (err error)

type PluginVariables

type PluginVariables struct {
	Connection string `json:"connection"`
}

type Registry

type Registry int
const (
	RegistryUnset  Registry = iota
	RegistryGitHub          // deprecated
	RegistryLocal
	RegistryGRPC
	RegistryDocker
	RegistryCloudQuery
)

func RegistryFromString

func RegistryFromString(s string) (Registry, error)

func (Registry) JSONSchemaExtend

func (Registry) JSONSchemaExtend(sc *jsonschema.Schema)

func (Registry) MarshalJSON

func (r Registry) MarshalJSON() ([]byte, error)

func (Registry) NeedVersion

func (r Registry) NeedVersion() bool

NeedVersion has to be in sync with Metadata.JSONSchemaExtend

func (Registry) String

func (r Registry) String() string

func (*Registry) UnmarshalJSON

func (r *Registry) UnmarshalJSON(data []byte) (err error)

type Source

type Source struct {
	Metadata

	// Tables to sync from the source plugin
	Tables []string `json:"tables,omitempty" jsonschema:"required,minItems=1,minLength=1"`
	// SkipTables defines tables to skip when syncing data. Useful if a glob pattern is used in Tables
	SkipTables []string `json:"skip_tables,omitempty" jsonschema:"minLength=1"`
	// SkipDependentTables changes the matching behavior with regard to dependent tables. If set to `false`, dependent tables will be included in the sync when their parents are matched, even if not explicitly included by the `tables` configuration.
	SkipDependentTables *bool `json:"skip_dependent_tables,omitempty" jsonschema:"default=true"`

	// Destinations are the names of destination plugins to send sync data to
	Destinations []string `json:"destinations,omitempty" jsonschema:"required,minItems=1,minLength=1"`

	// Optional Backend options for sync operation
	BackendOptions *BackendOptions `json:"backend_options,omitempty"`

	// Source plugin own (nested) spec
	Spec map[string]any `json:"spec,omitempty"`

	// DeterministicCQID is a flag that indicates whether the source plugin should generate a random UUID as the value of `_cq_id`
	// or whether it should calculate a UUID that is a hash of the primary keys (if they exist) or the entire resource.
	DeterministicCQID bool `json:"deterministic_cq_id,omitempty" jsonschema:"default=false"`

	// If specified this will spawn the plugin with `--otel-endpoint`
	OtelEndpoint string `json:"otel_endpoint,omitempty" jsonschema:"default="`
	// If specified this will spawn the plugin with `--otel-endpoint-insecure`
	OtelEndpointInsecure bool `json:"otel_endpoint_insecure,omitempty" jsonschema:"default=false"`
}

Source plugin spec

func (*Source) GetWarnings

func (s *Source) GetWarnings() Warnings

GetWarnings returns a list of deprecated options that were used in the source config. This should be called before SetDefaults.

func (Source) JSONSchemaExtend

func (Source) JSONSchemaExtend(sc *jsonschema.Schema)

func (*Source) SetDefaults

func (s *Source) SetDefaults()

func (*Source) UnmarshalSpec

func (s *Source) UnmarshalSpec(out any) error

UnmarshalSpec unmarshals the internal spec into the given interface

func (*Source) Validate

func (s *Source) Validate(relaxed bool) error

type Spec

type Spec struct {
	// CloudQuery plugin kind
	Kind Kind `json:"kind" jsonschema:"required"`

	// CloudQuery plugin (top-level) spec
	Spec any `json:"spec" jsonschema:"required"`
}

func (Spec) JSONSchemaExtend

func (Spec) JSONSchemaExtend(sc *jsonschema.Schema)

func (*Spec) UnmarshalJSON

func (s *Spec) UnmarshalJSON(data []byte) error

type SpecReader

type SpecReader struct {
	Sources      []*Source
	Destinations []*Destination
	Transformers []*Transformer
	// contains filtered or unexported fields
}

func NewRelaxedSpecReader

func NewRelaxedSpecReader(paths []string) (*SpecReader, error)

func NewSpecReader

func NewSpecReader(paths []string) (*SpecReader, error)

func NewSpecReaderWithoutValidation added in v6.37.0

func NewSpecReaderWithoutValidation(paths []string) (*SpecReader, error)

NewSpecReaderWithoutValidation reads and parses specs without enforcing that every source has a destination (or that any destination exists at all). The caller is responsible for calling Validate once destinations are finalized — e.g. after the platform destination has had a chance to be auto-injected.

func (*SpecReader) GetDestinationByName

func (r *SpecReader) GetDestinationByName(name string) *Destination

func (*SpecReader) GetDestinationNamesForSource

func (r *SpecReader) GetDestinationNamesForSource(name string) []string

func (*SpecReader) GetDestinationWarningsByName

func (r *SpecReader) GetDestinationWarningsByName(name string) Warnings

func (*SpecReader) GetSourceByName

func (r *SpecReader) GetSourceByName(name string) *Source

func (*SpecReader) GetSourceWarningsByName

func (r *SpecReader) GetSourceWarningsByName(name string) Warnings

func (*SpecReader) GetTransformerWarningsByName

func (r *SpecReader) GetTransformerWarningsByName(name string) Warnings

func (*SpecReader) SetDestinationsAndValidate added in v6.37.0

func (r *SpecReader) SetDestinationsAndValidate(destinations []*Destination) error

SetDestinationsAndValidate replaces the reader's destinations and runs the full (non-relaxed) validation, including the per-source requirement that at least one destination is configured. Setting and validating are coupled so destinations cannot be swapped in after reading without being validated.

type Transformer

type Transformer struct {
	Metadata

	// Transformer plugin own (nested) spec
	Spec map[string]any `json:"spec,omitempty"`
}

Transformer plugin spec

func (*Transformer) GetWarnings

func (*Transformer) GetWarnings() Warnings

func (*Transformer) UnmarshalSpec

func (d *Transformer) UnmarshalSpec(out any) error

func (*Transformer) Validate

func (d *Transformer) Validate() error

type Variables

type Variables struct {
	Plugins map[string]PluginVariables `json:"plugins"`
}

type Warnings

type Warnings map[string]string

Warnings is a map of field name to string, used mainly for deprecation notices.

type WriteMode

type WriteMode int
const (
	WriteModeOverwriteDeleteStale WriteMode = iota
	WriteModeOverwrite
	WriteModeAppend
)

func WriteModeFromString

func WriteModeFromString(s string) (WriteMode, error)

func (WriteMode) JSONSchemaExtend

func (WriteMode) JSONSchemaExtend(sc *jsonschema.Schema)

func (WriteMode) MarshalJSON

func (m WriteMode) MarshalJSON() ([]byte, error)

func (WriteMode) String

func (m WriteMode) String() string

func (*WriteMode) UnmarshalJSON

func (m *WriteMode) UnmarshalJSON(data []byte) (err error)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL