Documentation
¶
Overview ¶
Package config unmarshals the OpenMetadata ingestion workflow YAML. The struct layout intentionally mirrors the Python openmetadata-ingestion workflow config so existing YAML files can be reused unchanged. Unknown fields are ignored (KnownFields is left off) so the many Python options we do not model are tolerated rather than rejected.
Index ¶
- type AuthType
- type BaseConnection
- type ClickHouseConnection
- type ConnectionConfig
- type DatabaseMetadata
- type FilterPattern
- type MySQLConnection
- type OpenMetadataServerConfig
- type PostgresConnection
- type SecurityConfig
- type ServiceConnection
- type Sink
- type Source
- type SourceConfig
- type Workflow
- type WorkflowConfig
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type AuthType ¶
type AuthType struct {
Password string `yaml:"password"`
}
AuthType mirrors the Python authType block. Only password auth is supported.
type BaseConnection ¶
type BaseConnection struct {
Type string `yaml:"type"`
Username string `yaml:"username"`
AuthType AuthType `yaml:"authType"`
HostPort string `yaml:"hostPort"`
Database string `yaml:"database"`
IngestAllDatabases bool `yaml:"ingestAllDatabases"`
}
BaseConnection holds the connection fields common to all supported sources.
func (BaseConnection) Password ¶
func (b BaseConnection) Password() string
type ClickHouseConnection ¶
type ClickHouseConnection struct {
BaseConnection `yaml:",inline"`
Secure bool `yaml:"secure"`
HTTPS bool `yaml:"https"`
}
ClickHouseConnection is the ClickHouse serviceConnection.config.
func (*ClickHouseConnection) Base ¶
func (c *ClickHouseConnection) Base() BaseConnection
func (*ClickHouseConnection) ConnType ¶
func (c *ClickHouseConnection) ConnType() string
type ConnectionConfig ¶
type ConnectionConfig interface {
// ConnType returns the OpenMetadata connection type string (e.g. "Postgres").
ConnType() string
// Base returns the shared connection fields.
Base() BaseConnection
}
ConnectionConfig is implemented by every concrete source connection type. The concrete type is chosen from the YAML config.type discriminator.
type DatabaseMetadata ¶
type DatabaseMetadata struct {
Type string `yaml:"type"`
MarkDeletedTables bool `yaml:"markDeletedTables"`
IncludeTables *bool `yaml:"includeTables"`
IncludeViews *bool `yaml:"includeViews"`
SchemaFilterPattern FilterPattern `yaml:"schemaFilterPattern"`
TableFilterPattern FilterPattern `yaml:"tableFilterPattern"`
}
DatabaseMetadata is the "DatabaseMetadata" sourceConfig.config. IncludeTables and IncludeViews are pointers so an unset value can default to true (matching the Python default), while an explicit false is respected.
func (DatabaseMetadata) IncludeTablesOrDefault ¶
func (d DatabaseMetadata) IncludeTablesOrDefault() bool
IncludeTablesOrDefault returns IncludeTables, defaulting to true when unset.
func (DatabaseMetadata) IncludeViewsOrDefault ¶
func (d DatabaseMetadata) IncludeViewsOrDefault() bool
IncludeViewsOrDefault returns IncludeViews, defaulting to true when unset.
type FilterPattern ¶
type FilterPattern struct {
Includes []string `yaml:"includes"`
Excludes []string `yaml:"excludes"`
}
FilterPattern is an include/exclude regex pattern as expressed in YAML.
type MySQLConnection ¶
type MySQLConnection struct {
BaseConnection `yaml:",inline"`
DatabaseSchema string `yaml:"databaseSchema"`
}
MySQLConnection is the MySQL serviceConnection.config.
func (*MySQLConnection) Base ¶
func (c *MySQLConnection) Base() BaseConnection
func (*MySQLConnection) ConnType ¶
func (c *MySQLConnection) ConnType() string
type OpenMetadataServerConfig ¶
type OpenMetadataServerConfig struct {
HostPort string `yaml:"hostPort"`
AuthProvider string `yaml:"authProvider"`
SecurityConfig SecurityConfig `yaml:"securityConfig"`
}
OpenMetadataServerConfig is the connection to the OpenMetadata API.
type PostgresConnection ¶
type PostgresConnection struct {
BaseConnection `yaml:",inline"`
SSLMode string `yaml:"sslMode"`
}
PostgresConnection is the PostgreSQL serviceConnection.config.
func (*PostgresConnection) Base ¶
func (c *PostgresConnection) Base() BaseConnection
func (*PostgresConnection) ConnType ¶
func (c *PostgresConnection) ConnType() string
type SecurityConfig ¶
type SecurityConfig struct {
JWTToken string `yaml:"jwtToken"`
}
SecurityConfig carries the JWT bot token used to authenticate to OpenMetadata.
type ServiceConnection ¶
type ServiceConnection struct {
Config ConnectionConfig
}
ServiceConnection wraps the polymorphic serviceConnection.config block.
func (*ServiceConnection) UnmarshalYAML ¶
func (s *ServiceConnection) UnmarshalYAML(value *yaml.Node) error
UnmarshalYAML performs a two-pass decode: pass one reads config.type to pick the concrete connection struct, pass two decodes the config block into it.
type Sink ¶
Sink describes where extracted metadata is written. Only the "metadata-rest" sink (push to the OpenMetadata API) is supported.
type Source ¶
type Source struct {
Type string `yaml:"type"` // postgres | mysql | clickhouse
ServiceName string `yaml:"serviceName"`
ServiceConnection ServiceConnection `yaml:"serviceConnection"`
SourceConfig SourceConfig `yaml:"sourceConfig"`
}
Source describes what to ingest and how to connect to it.
type SourceConfig ¶
type SourceConfig struct {
Config DatabaseMetadata `yaml:"config"`
}
SourceConfig wraps the sourceConfig.config block.
type Workflow ¶
type Workflow struct {
Source Source `yaml:"source"`
Sink Sink `yaml:"sink"`
WorkflowConfig WorkflowConfig `yaml:"workflowConfig"`
}
Workflow is the root of a workflow YAML file.
type WorkflowConfig ¶
type WorkflowConfig struct {
LoggerLevel string `yaml:"loggerLevel"`
OpenMetadataServerConfig OpenMetadataServerConfig `yaml:"openMetadataServerConfig"`
}
WorkflowConfig holds run-level settings and the OpenMetadata server connection.