yamlconfig

package
v0.12.8 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 1, 2026 License: Apache-2.0 Imports: 7 Imported by: 0

Documentation

Overview

Package yamlconfig provides YAML-based DWH connection config parsing and proto conversion.

Package yamlconfig provides YAML-based DWH connection config parsing and proto conversion.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ApplyDefaults

func ApplyDefaults(conns map[string]*Connection)

ApplyDefaults sets default values on parsed connections. Currently sets Parallelism to 8 when not explicitly configured.

func ConnectionsSchema

func ConnectionsSchema() *jsonschema.Schema

ConnectionsSchema returns a JSON schema for a map of connections, suitable for embedding in a larger config schema.

func FromProtoConnections

func FromProtoConnections(conns map[string]*agentdwhv1.Connection) map[string]*Connection

FromProtoConnections converts proto Connection messages back to YAML Connection structs.

func NewReflector

func NewReflector(opts ...ReflectorOption) *jsonschema.Reflector

NewReflector returns a jsonschema.Reflector configured for YAML config structs. Use WithGoComments or WithYAMLConfigComments to add descriptions from Go doc comments.

func Parse

func Parse(data []byte, target any, opts ParseOptions) error

Parse unmarshals YAML data into the target struct. If opts.ExpandEnv is true, ${VAR} references in scalar values are expanded.

func ParseConnections

func ParseConnections(data []byte, opts ParseOptions) (map[string]*Connection, error)

ParseConnections parses a YAML document and extracts the connections map. The YAML must have a top-level "connections" key.

func ParseConnectionsOnly

func ParseConnectionsOnly(data []byte, opts ParseOptions) (map[string]*Connection, error)

ParseConnectionsOnly parses a YAML document where the top-level keys are connection IDs directly (no wrapping "connections" key).

func ResolveFiles

func ResolveFiles(connections map[string]*Connection, opts ParseOptions) error

ResolveFiles reads files referenced by *_file fields and populates the corresponding inline fields. Only acts when opts.ReadFile is non-nil.

func ToProtoConnection

func ToProtoConnection(id string, conn *Connection) (*agentdwhv1.Connection, error)

ToProtoConnection converts a single YAML Connection to a proto Connection message.

func ToProtoConnections

func ToProtoConnections(conns map[string]*Connection) (map[string]*agentdwhv1.Connection, error)

ToProtoConnections converts all YAML connections to proto Connection messages. Connection IDs are used as the map keys and as default names when name is empty.

Types

type BigQueryConf

type BigQueryConf struct {
	// GCP project ID.
	ProjectId string `yaml:"project_id" jsonschema:"required"`
	// Region for BigQuery resources.
	Region string `yaml:"region" jsonschema:"required"`
	// Inline JSON content of the service account key.
	ServiceAccountKey string `yaml:"service_account_key,omitempty"`
	// Path to the service account key JSON file.
	ServiceAccountKeyFile string `yaml:"service_account_key_file,omitempty"`
}

BigQueryConf contains BigQuery connection parameters. Exactly one of service_account_key or service_account_key_file should be set.

type ClickhouseConf

type ClickhouseConf struct {
	Host string `yaml:"host"           jsonschema:"required"`
	Port int    `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
	// Database to connect to. If empty, all databases are scraped.
	Database string `yaml:"database,omitempty"`
	Username string `yaml:"username" jsonschema:"required"`
	Password string `yaml:"password" jsonschema:"required"`
	// Disable SSL certificate verification.
	AllowInsecure bool `yaml:"allow_insecure,omitempty"`
}

ClickhouseConf contains ClickHouse connection parameters.

type Connection

type Connection struct {
	// Display name for this connection. Defaults to the connection ID (map key).
	Name string `yaml:"name,omitempty"`
	// When true, the connection is skipped during execution.
	Disabled bool `yaml:"disabled,omitempty"`
	// Maximum number of parallel queries. Range: 1-256. Defaults to 8.
	Parallelism int `yaml:"parallelism,omitempty" jsonschema:"minimum=1,maximum=256"`

	Postgres   *PostgresConf   `yaml:"postgres,omitempty"`
	Snowflake  *SnowflakeConf  `yaml:"snowflake,omitempty"`
	BigQuery   *BigQueryConf   `yaml:"bigquery,omitempty"`
	Redshift   *RedshiftConf   `yaml:"redshift,omitempty"`
	MySQL      *MySQLConf      `yaml:"mysql,omitempty"`
	Clickhouse *ClickhouseConf `yaml:"clickhouse,omitempty"`
	Trino      *TrinoConf      `yaml:"trino,omitempty"`
	Databricks *DatabricksConf `yaml:"databricks,omitempty"`
	MSSQL      *MSSQLConf      `yaml:"mssql,omitempty"`
	Oracle     *OracleConf     `yaml:"oracle,omitempty"`
	DuckDB     *DuckDBConf     `yaml:"duckdb,omitempty"`
}

Connection is a single database connection entry in a connections map. The map key serves as the connection ID. Exactly one of the database-type fields must be set.

func FromProtoConnection

func FromProtoConnection(proto *agentdwhv1.Connection) *Connection

FromProtoConnection converts a single proto Connection to a YAML Connection struct.

func (*Connection) DialectType

func (c *Connection) DialectType() string

DialectType returns the warehouse type string for this connection, or empty if none is set.

type DatabricksConf

type DatabricksConf struct {
	// Databricks workspace URL.
	WorkspaceUrl string `yaml:"workspace_url" jsonschema:"required"`
	// Personal access token for authentication.
	AuthToken string `yaml:"auth_token,omitempty"`
	// OAuth client ID (M2M authentication).
	AuthClient string `yaml:"auth_client,omitempty"`
	// OAuth client secret (M2M authentication).
	AuthSecret string `yaml:"auth_secret,omitempty"`
	// SQL warehouse ID to use for queries.
	Warehouse                  string `yaml:"warehouse,omitempty"`
	RefreshTableMetrics        bool   `yaml:"refresh_table_metrics,omitempty"`
	RefreshTableMetricsUseScan bool   `yaml:"refresh_table_metrics_use_scan,omitempty"`
	FetchTableTags             bool   `yaml:"fetch_table_tags,omitempty"`
	UseShowCreateTable         bool   `yaml:"use_show_create_table,omitempty"`
}

DatabricksConf contains Databricks connection parameters. Authentication: set auth_token, or set both auth_client and auth_secret.

type DuckDBConf

type DuckDBConf struct {
	// File path, ':memory:' for in-memory, or MotherDuck database name.
	Database string `yaml:"database,omitempty"`
	// MotherDuck organization/account name (for cloud mode).
	MotherduckAccount string `yaml:"motherduck_account,omitempty"`
	// MotherDuck authentication token (required for cloud MotherDuck).
	MotherduckToken string `yaml:"motherduck_token,omitempty"`
}

DuckDBConf contains DuckDB / MotherDuck connection parameters.

type MSSQLConf

type MSSQLConf struct {
	Host     string `yaml:"host"           jsonschema:"required"`
	Port     int    `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
	Database string `yaml:"database"       jsonschema:"required"`
	Username string `yaml:"username,omitempty"`
	Password string `yaml:"password,omitempty"`
	// Trust the server certificate without validation.
	TrustCert bool `yaml:"trust_cert,omitempty"`
	// Encryption mode (e.g. "true", "false", "strict").
	Encrypt string `yaml:"encrypt,omitempty"`
	// Federated authentication method (e.g. "ActiveDirectoryDefault").
	FedAuth string `yaml:"fed_auth,omitempty"`
	// Pre-acquired access token for Azure AD authentication.
	AccessToken string `yaml:"access_token,omitempty"`
	// Azure AD application client ID for service principal auth.
	ApplicationClientId string `yaml:"application_client_id,omitempty"`
}

MSSQLConf contains Microsoft SQL Server / Azure SQL Database connection parameters.

type MySQLConf

type MySQLConf struct {
	Host     string `yaml:"host"           jsonschema:"required"`
	Port     int    `yaml:"port"           jsonschema:"required,minimum=1,maximum=65535"`
	Database string `yaml:"database,omitempty"`
	Username string `yaml:"username"       jsonschema:"required"`
	Password string `yaml:"password"       jsonschema:"required"`
	// Disable SSL certificate verification.
	AllowInsecure bool `yaml:"allow_insecure,omitempty"`
	// Additional DSN parameters passed to the driver.
	Params map[string]string `yaml:"params,omitempty"`
}

MySQLConf contains MySQL connection parameters.

type OracleConf

type OracleConf struct {
	Host string `yaml:"host" jsonschema:"required"`
	Port int    `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
	// Oracle service name.
	ServiceName string `yaml:"service_name" jsonschema:"required"`
	Username    string `yaml:"username,omitempty"`
	Password    string `yaml:"password,omitempty"`
	// Enable SSL/TLS for the connection.
	SSL bool `yaml:"ssl,omitempty"`
	// Verify the server's SSL certificate.
	SSLVerify bool `yaml:"ssl_verify,omitempty"`
	// Path to Oracle Wallet directory for authentication.
	WalletPath string `yaml:"wallet_path,omitempty"`
	// Enable Oracle Diagnostics Pack features (AWR, ASH).
	UseDiagnosticsPack bool `yaml:"use_diagnostics_pack,omitempty"`
}

OracleConf contains Oracle Database connection parameters.

type ParseOptions

type ParseOptions struct {
	// ExpandEnv expands ${VAR} references in YAML scalar values.
	// When true and LookupVar is nil, os.Getenv is used.
	// When true and LookupVar is set, LookupVar is used instead.
	ExpandEnv bool

	// LookupVar resolves variable names to values during ${VAR} expansion.
	// Called with the variable name (without ${}); returns the value and whether it was found.
	// When nil and ExpandEnv is true, os.LookupEnv is used.
	// Use this to read variables from a secrets manager instead of the process environment.
	LookupVar func(name string) (string, bool)

	// StrictEnv makes expansion fail when a referenced variable is not found.
	// Only applies when ExpandEnv is true. Unset variables in ${VAR} syntax cause
	// a parse error. Use ${VAR:-default} to provide a fallback, or ${VAR:-} for empty string.
	StrictEnv bool

	// BaseDir is the directory against which relative file paths are resolved.
	// Typically filepath.Dir(configPath). Only used when ReadFile is set.
	BaseDir string

	// ReadFile reads a file at the given absolute path. Used to resolve *_file fields
	// (e.g., private_key_file, service_account_key_file).
	// When nil, file fields are left as-is (paths remain as strings, inline fields are not populated).
	// When set, relative paths are resolved against BaseDir before calling ReadFile.
	// Cloud deployments can pass a function that returns an error to prevent file access.
	ReadFile func(path string) ([]byte, error)
}

ParseOptions controls how YAML config is parsed.

func CLIOptions

func CLIOptions(configPath string) ParseOptions

CLIOptions returns ParseOptions suitable for CLI tools: env expansion enabled, file reading via os.ReadFile, paths resolved relative to the config file.

type PostgresConf

type PostgresConf struct {
	Host     string `yaml:"host"           jsonschema:"required"`
	Port     int    `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
	Database string `yaml:"database"       jsonschema:"required"`
	Username string `yaml:"username"       jsonschema:"required"`
	Password string `yaml:"password"       jsonschema:"required"`
	// Disable SSL certificate verification.
	AllowInsecure bool `yaml:"allow_insecure,omitempty"`
}

PostgresConf contains PostgreSQL connection parameters.

type RedshiftConf

type RedshiftConf struct {
	Host     string `yaml:"host"     jsonschema:"required"`
	Port     int    `yaml:"port"     jsonschema:"required,minimum=1,maximum=65535"`
	Database string `yaml:"database" jsonschema:"required"`
	Username string `yaml:"username" jsonschema:"required"`
	Password string `yaml:"password" jsonschema:"required"`
	// Estimate table freshness from Redshift query logs instead of metadata.
	FreshnessFromQueryLogs bool `yaml:"freshness_from_query_logs,omitempty"`
}

RedshiftConf contains Amazon Redshift connection parameters.

type ReflectorOption

type ReflectorOption func(*jsonschema.Reflector)

ReflectorOption configures a jsonschema.Reflector.

func WithGoComments

func WithGoComments(base, srcDir string) ReflectorOption

WithGoComments adds Go doc comments from a package as JSON schema descriptions. The base is the import path and srcDir is the local filesystem path to the source (can be relative to the working directory). Errors are silently ignored (source may not be available in CI or production).

type SnowflakeConf

type SnowflakeConf struct {
	// Snowflake account identifier.
	Account string `yaml:"account" jsonschema:"required"`
	// Virtual warehouse to use for queries.
	Warehouse string `yaml:"warehouse" jsonschema:"required"`
	// Role to assume after connecting.
	Role     string `yaml:"role"     jsonschema:"required"`
	Username string `yaml:"username" jsonschema:"required"`
	Password string `yaml:"password,omitempty"`
	// PEM-encoded private key content for key-pair authentication.
	PrivateKey string `yaml:"private_key,omitempty"`
	// Path to a PEM-encoded private key file.
	PrivateKeyFile string `yaml:"private_key_file,omitempty"`
	// Passphrase to decrypt the private key.
	PrivateKeyPassphrase string `yaml:"private_key_passphrase,omitempty"`
	// Databases to include. If empty, all accessible databases are scraped.
	Databases []string `yaml:"databases,omitempty"`
	// Use GET_DDL() to retrieve DDL for tables and views.
	UseGetDdl bool `yaml:"use_get_ddl,omitempty"`
	// Database containing the ACCOUNT_USAGE schema. Defaults to SNOWFLAKE.
	AccountUsageDb string `yaml:"account_usage_db,omitempty"`
	// Set to "externalbrowser" to use SSO browser-based authentication.
	AuthType string `yaml:"auth_type,omitempty"`
}

SnowflakeConf contains Snowflake connection parameters. Authentication: provide password, private_key/private_key_file, or set auth_type to "externalbrowser".

type TrinoConf

type TrinoConf struct {
	Host string `yaml:"host" jsonschema:"required"`
	Port int    `yaml:"port,omitempty" jsonschema:"minimum=1,maximum=65535"`
	// Use a plain HTTP connection instead of HTTPS.
	UsePlaintext bool   `yaml:"use_plaintext,omitempty"`
	Username     string `yaml:"username,omitempty"`
	Password     string `yaml:"password,omitempty"`
	// Catalogs to include. Required for most Trino deployments.
	Catalogs            []string `yaml:"catalogs,omitempty"`
	NoShowCreateView    bool     `yaml:"no_show_create_view,omitempty"`
	NoShowCreateTable   bool     `yaml:"no_show_create_table,omitempty"`
	NoMaterializedViews bool     `yaml:"no_materialized_views,omitempty"`
	FetchTableComments  bool     `yaml:"fetch_table_comments,omitempty"`
}

TrinoConf contains Trino / Starburst connection parameters.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL