Documentation
¶
Overview ¶
Package yamlconfig provides YAML-based DWH connection config parsing and proto conversion.
Package yamlconfig provides YAML-based DWH connection config parsing and proto conversion.
Index ¶
- func ApplyDefaults(conns map[string]*Connection)
- func ConnectionsSchema() *jsonschema.Schema
- func FromProtoConnections(conns map[string]*agentdwhv1.Connection) map[string]*Connection
- func NewReflector(opts ...ReflectorOption) *jsonschema.Reflector
- func Parse(data []byte, target any, opts ParseOptions) error
- func ParseConnections(data []byte, opts ParseOptions) (map[string]*Connection, error)
- func ParseConnectionsOnly(data []byte, opts ParseOptions) (map[string]*Connection, error)
- func ResolveFiles(connections map[string]*Connection, opts ParseOptions) error
- func ToProtoConnection(id string, conn *Connection) (*agentdwhv1.Connection, error)
- func ToProtoConnections(conns map[string]*Connection) (map[string]*agentdwhv1.Connection, error)
- type BigQueryConf
- type ClickhouseConf
- type Connection
- type DatabricksConf
- type DuckDBConf
- type MSSQLConf
- type MySQLConf
- type OracleConf
- type ParseOptions
- type PostgresConf
- type RedshiftConf
- type ReflectorOption
- type SnowflakeConf
- type TrinoConf
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ApplyDefaults ¶
func ApplyDefaults(conns map[string]*Connection)
ApplyDefaults sets default values on parsed connections. Currently sets Parallelism to 8 when not explicitly configured.
func ConnectionsSchema ¶
func ConnectionsSchema() *jsonschema.Schema
ConnectionsSchema returns a JSON schema for a map of connections, suitable for embedding in a larger config schema.
func FromProtoConnections ¶
func FromProtoConnections(conns map[string]*agentdwhv1.Connection) map[string]*Connection
FromProtoConnections converts proto Connection messages back to YAML Connection structs.
func NewReflector ¶
func NewReflector(opts ...ReflectorOption) *jsonschema.Reflector
NewReflector returns a jsonschema.Reflector configured for YAML config structs. Use WithGoComments or WithYAMLConfigComments to add descriptions from Go doc comments.
func Parse ¶
func Parse(data []byte, target any, opts ParseOptions) error
Parse unmarshals YAML data into the target struct. If opts.ExpandEnv is true, ${VAR} references in scalar values are expanded.
func ParseConnections ¶
func ParseConnections(data []byte, opts ParseOptions) (map[string]*Connection, error)
ParseConnections parses a YAML document and extracts the connections map. The YAML must have a top-level "connections" key.
func ParseConnectionsOnly ¶
func ParseConnectionsOnly(data []byte, opts ParseOptions) (map[string]*Connection, error)
ParseConnectionsOnly parses a YAML document where the top-level keys are connection IDs directly (no wrapping "connections" key).
func ResolveFiles ¶
func ResolveFiles(connections map[string]*Connection, opts ParseOptions) error
ResolveFiles reads files referenced by *_file fields and populates the corresponding inline fields. Only acts when opts.ReadFile is non-nil.
func ToProtoConnection ¶
func ToProtoConnection(id string, conn *Connection) (*agentdwhv1.Connection, error)
ToProtoConnection converts a single YAML Connection to a proto Connection message.
func ToProtoConnections ¶
func ToProtoConnections(conns map[string]*Connection) (map[string]*agentdwhv1.Connection, error)
ToProtoConnections converts all YAML connections to proto Connection messages. Connection IDs are used as the map keys and as default names when name is empty.
Types ¶
type BigQueryConf ¶
type BigQueryConf struct {
// GCP project ID.
ProjectId string `yaml:"project_id" jsonschema:"required"`
// Region for BigQuery resources.
Region string `yaml:"region" jsonschema:"required"`
// Inline JSON content of the service account key.
ServiceAccountKey string `yaml:"service_account_key,omitempty"`
// Path to the service account key JSON file.
ServiceAccountKeyFile string `yaml:"service_account_key_file,omitempty"`
}
BigQueryConf contains BigQuery connection parameters. Exactly one of service_account_key or service_account_key_file should be set.
type ClickhouseConf ¶
type ClickhouseConf struct {
Host string `yaml:"host" jsonschema:"required"`
Port int `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
// Database to connect to. If empty, all databases are scraped.
Database string `yaml:"database,omitempty"`
Username string `yaml:"username" jsonschema:"required"`
Password string `yaml:"password" jsonschema:"required"`
// Disable SSL certificate verification.
AllowInsecure bool `yaml:"allow_insecure,omitempty"`
}
ClickhouseConf contains ClickHouse connection parameters.
type Connection ¶
type Connection struct {
// Display name for this connection. Defaults to the connection ID (map key).
Name string `yaml:"name,omitempty"`
// When true, the connection is skipped during execution.
Disabled bool `yaml:"disabled,omitempty"`
// Maximum number of parallel queries. Range: 1-256. Defaults to 8.
Parallelism int `yaml:"parallelism,omitempty" jsonschema:"minimum=1,maximum=256"`
Postgres *PostgresConf `yaml:"postgres,omitempty"`
Snowflake *SnowflakeConf `yaml:"snowflake,omitempty"`
BigQuery *BigQueryConf `yaml:"bigquery,omitempty"`
Redshift *RedshiftConf `yaml:"redshift,omitempty"`
MySQL *MySQLConf `yaml:"mysql,omitempty"`
Clickhouse *ClickhouseConf `yaml:"clickhouse,omitempty"`
Trino *TrinoConf `yaml:"trino,omitempty"`
Databricks *DatabricksConf `yaml:"databricks,omitempty"`
MSSQL *MSSQLConf `yaml:"mssql,omitempty"`
Oracle *OracleConf `yaml:"oracle,omitempty"`
DuckDB *DuckDBConf `yaml:"duckdb,omitempty"`
}
Connection is a single database connection entry in a connections map. The map key serves as the connection ID. Exactly one of the database-type fields must be set.
func FromProtoConnection ¶
func FromProtoConnection(proto *agentdwhv1.Connection) *Connection
FromProtoConnection converts a single proto Connection to a YAML Connection struct.
func (*Connection) DialectType ¶
func (c *Connection) DialectType() string
DialectType returns the warehouse type string for this connection, or empty if none is set.
type DatabricksConf ¶
type DatabricksConf struct {
// Databricks workspace URL.
WorkspaceUrl string `yaml:"workspace_url" jsonschema:"required"`
// Personal access token for authentication.
AuthToken string `yaml:"auth_token,omitempty"`
// OAuth client ID (M2M authentication).
AuthClient string `yaml:"auth_client,omitempty"`
// OAuth client secret (M2M authentication).
AuthSecret string `yaml:"auth_secret,omitempty"`
// SQL warehouse ID to use for queries.
Warehouse string `yaml:"warehouse,omitempty"`
RefreshTableMetrics bool `yaml:"refresh_table_metrics,omitempty"`
RefreshTableMetricsUseScan bool `yaml:"refresh_table_metrics_use_scan,omitempty"`
FetchTableTags bool `yaml:"fetch_table_tags,omitempty"`
UseShowCreateTable bool `yaml:"use_show_create_table,omitempty"`
}
DatabricksConf contains Databricks connection parameters. Authentication: set auth_token, or set both auth_client and auth_secret.
type DuckDBConf ¶
type DuckDBConf struct {
// File path, ':memory:' for in-memory, or MotherDuck database name.
Database string `yaml:"database,omitempty"`
// MotherDuck organization/account name (for cloud mode).
MotherduckAccount string `yaml:"motherduck_account,omitempty"`
// MotherDuck authentication token (required for cloud MotherDuck).
MotherduckToken string `yaml:"motherduck_token,omitempty"`
}
DuckDBConf contains DuckDB / MotherDuck connection parameters.
type MSSQLConf ¶
type MSSQLConf struct {
Host string `yaml:"host" jsonschema:"required"`
Port int `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
Database string `yaml:"database" jsonschema:"required"`
Username string `yaml:"username,omitempty"`
Password string `yaml:"password,omitempty"`
// Trust the server certificate without validation.
TrustCert bool `yaml:"trust_cert,omitempty"`
// Encryption mode (e.g. "true", "false", "strict").
Encrypt string `yaml:"encrypt,omitempty"`
// Federated authentication method (e.g. "ActiveDirectoryDefault").
FedAuth string `yaml:"fed_auth,omitempty"`
// Pre-acquired access token for Azure AD authentication.
AccessToken string `yaml:"access_token,omitempty"`
// Azure AD application client ID for service principal auth.
ApplicationClientId string `yaml:"application_client_id,omitempty"`
}
MSSQLConf contains Microsoft SQL Server / Azure SQL Database connection parameters.
type MySQLConf ¶
type MySQLConf struct {
Host string `yaml:"host" jsonschema:"required"`
Port int `yaml:"port" jsonschema:"required,minimum=1,maximum=65535"`
Database string `yaml:"database,omitempty"`
Username string `yaml:"username" jsonschema:"required"`
Password string `yaml:"password" jsonschema:"required"`
// Disable SSL certificate verification.
AllowInsecure bool `yaml:"allow_insecure,omitempty"`
// Additional DSN parameters passed to the driver.
Params map[string]string `yaml:"params,omitempty"`
}
MySQLConf contains MySQL connection parameters.
type OracleConf ¶
type OracleConf struct {
Host string `yaml:"host" jsonschema:"required"`
Port int `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
// Oracle service name.
ServiceName string `yaml:"service_name" jsonschema:"required"`
Username string `yaml:"username,omitempty"`
Password string `yaml:"password,omitempty"`
// Enable SSL/TLS for the connection.
SSL bool `yaml:"ssl,omitempty"`
// Verify the server's SSL certificate.
SSLVerify bool `yaml:"ssl_verify,omitempty"`
// Path to Oracle Wallet directory for authentication.
WalletPath string `yaml:"wallet_path,omitempty"`
// Enable Oracle Diagnostics Pack features (AWR, ASH).
UseDiagnosticsPack bool `yaml:"use_diagnostics_pack,omitempty"`
}
OracleConf contains Oracle Database connection parameters.
type ParseOptions ¶
type ParseOptions struct {
// ExpandEnv expands ${VAR} references in YAML scalar values.
// When true and LookupVar is nil, os.Getenv is used.
// When true and LookupVar is set, LookupVar is used instead.
ExpandEnv bool
// LookupVar resolves variable names to values during ${VAR} expansion.
// Called with the variable name (without ${}); returns the value and whether it was found.
// When nil and ExpandEnv is true, os.LookupEnv is used.
// Use this to read variables from a secrets manager instead of the process environment.
LookupVar func(name string) (string, bool)
// StrictEnv makes expansion fail when a referenced variable is not found.
// Only applies when ExpandEnv is true. Unset variables in ${VAR} syntax cause
// a parse error. Use ${VAR:-default} to provide a fallback, or ${VAR:-} for empty string.
StrictEnv bool
// BaseDir is the directory against which relative file paths are resolved.
// Typically filepath.Dir(configPath). Only used when ReadFile is set.
BaseDir string
// ReadFile reads a file at the given absolute path. Used to resolve *_file fields
// (e.g., private_key_file, service_account_key_file).
// When nil, file fields are left as-is (paths remain as strings, inline fields are not populated).
// When set, relative paths are resolved against BaseDir before calling ReadFile.
// Cloud deployments can pass a function that returns an error to prevent file access.
ReadFile func(path string) ([]byte, error)
}
ParseOptions controls how YAML config is parsed.
func CLIOptions ¶
func CLIOptions(configPath string) ParseOptions
CLIOptions returns ParseOptions suitable for CLI tools: env expansion enabled, file reading via os.ReadFile, paths resolved relative to the config file.
type PostgresConf ¶
type PostgresConf struct {
Host string `yaml:"host" jsonschema:"required"`
Port int `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
Database string `yaml:"database" jsonschema:"required"`
Username string `yaml:"username" jsonschema:"required"`
Password string `yaml:"password" jsonschema:"required"`
// Disable SSL certificate verification.
AllowInsecure bool `yaml:"allow_insecure,omitempty"`
}
PostgresConf contains PostgreSQL connection parameters.
type RedshiftConf ¶
type RedshiftConf struct {
Host string `yaml:"host" jsonschema:"required"`
Port int `yaml:"port" jsonschema:"required,minimum=1,maximum=65535"`
Database string `yaml:"database" jsonschema:"required"`
Username string `yaml:"username" jsonschema:"required"`
Password string `yaml:"password" jsonschema:"required"`
// Estimate table freshness from Redshift query logs instead of metadata.
FreshnessFromQueryLogs bool `yaml:"freshness_from_query_logs,omitempty"`
}
RedshiftConf contains Amazon Redshift connection parameters.
type ReflectorOption ¶
type ReflectorOption func(*jsonschema.Reflector)
ReflectorOption configures a jsonschema.Reflector.
func WithGoComments ¶
func WithGoComments(base, srcDir string) ReflectorOption
WithGoComments adds Go doc comments from a package as JSON schema descriptions. The base is the import path and srcDir is the local filesystem path to the source (can be relative to the working directory). Errors are silently ignored (source may not be available in CI or production).
type SnowflakeConf ¶
type SnowflakeConf struct {
// Snowflake account identifier.
Account string `yaml:"account" jsonschema:"required"`
// Virtual warehouse to use for queries.
Warehouse string `yaml:"warehouse" jsonschema:"required"`
// Role to assume after connecting.
Role string `yaml:"role" jsonschema:"required"`
Username string `yaml:"username" jsonschema:"required"`
Password string `yaml:"password,omitempty"`
// PEM-encoded private key content for key-pair authentication.
PrivateKey string `yaml:"private_key,omitempty"`
// Path to a PEM-encoded private key file.
PrivateKeyFile string `yaml:"private_key_file,omitempty"`
// Passphrase to decrypt the private key.
PrivateKeyPassphrase string `yaml:"private_key_passphrase,omitempty"`
// Databases to include. If empty, all accessible databases are scraped.
Databases []string `yaml:"databases,omitempty"`
// Use GET_DDL() to retrieve DDL for tables and views.
UseGetDdl bool `yaml:"use_get_ddl,omitempty"`
// Database containing the ACCOUNT_USAGE schema. Defaults to SNOWFLAKE.
AccountUsageDb string `yaml:"account_usage_db,omitempty"`
// Set to "externalbrowser" to use SSO browser-based authentication.
AuthType string `yaml:"auth_type,omitempty"`
}
SnowflakeConf contains Snowflake connection parameters. Authentication: provide password, private_key/private_key_file, or set auth_type to "externalbrowser".
type TrinoConf ¶
type TrinoConf struct {
Host string `yaml:"host" jsonschema:"required"`
Port int `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
// Use a plain HTTP connection instead of HTTPS.
UsePlaintext bool `yaml:"use_plaintext,omitempty"`
Username string `yaml:"username,omitempty"`
Password string `yaml:"password,omitempty"`
// Catalogs to include. Required for most Trino deployments.
Catalogs []string `yaml:"catalogs,omitempty"`
NoShowCreateView bool `yaml:"no_show_create_view,omitempty"`
NoShowCreateTable bool `yaml:"no_show_create_table,omitempty"`
NoMaterializedViews bool `yaml:"no_materialized_views,omitempty"`
FetchTableComments bool `yaml:"fetch_table_comments,omitempty"`
}
TrinoConf contains Trino / Starburst connection parameters.