Documentation
¶
Overview ¶
+marmot:name=DBT +marmot:description=This plugin ingests metadata from DBT (Data Build Tool) projects, including models, tests, and lineage. +marmot:status=experimental +marmot:features=Assets, Lineage
Index ¶
- func RegisterAdapter(adapterType string, adapter Adapter)
- type Adapter
- type AlloyDBAdapter
- type AthenaAdapter
- type BaseAdapter
- type BigQueryAdapter
- type CatalogColumn
- type CatalogMetadata
- type CatalogNode
- type CatalogStat
- type ClickHouseAdapter
- type Config
- type DBTCatalog
- type DBTColumnFields
- type DBTConfigFields
- type DBTManifest
- type DBTModelFields
- type DBTRunResults
- type DBTSeedFields
- type DBTSourceFields
- type DBTStatsFields
- type DatabricksAdapter
- type DbtSchemaColumn
- type DremioAdapter
- type DuckDBAdapter
- type FabricAdapter
- type FabricSparkAdapter
- type GenericAdapter
- type GlueAdapter
- type LakebaseAdapter
- type ManifestMetadata
- type ManifestNode
- type MaterializeAdapter
- type MySQLAdapter
- type NetezzaAdapter
- type NodeColumn
- type NodeDependency
- type OracleAdapter
- type PostgresAdapter
- type RedshiftAdapter
- type RunResult
- type SQLServerAdapter
- type SalesforceAdapter
- type SingleStoreAdapter
- type SnowflakeAdapter
- type Source
- type SparkAdapter
- type SynapseAdapter
- type TeradataAdapter
- type TrinoAdapter
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func RegisterAdapter ¶
RegisterAdapter registers an adapter for a given dbt adapter type
Types ¶
type Adapter ¶
type Adapter interface {
// Name returns the canonical provider name for this adapter (e.g., "Snowflake", "BigQuery")
Name() string
// AssetTypeForMaterialization maps a dbt materialization type to the appropriate asset type
// For most adapters: table -> Table, view -> View, incremental -> Table
// Some adapters have special types (e.g., ClickHouse has Dictionary, Distributed Table)
AssetTypeForMaterialization(materialization string) string
// SupportsSchemas returns true if the adapter uses database.schema.table naming
// Some platforms like BigQuery use project.dataset.table instead
SupportsSchemas() bool
// DefaultMaterialization returns the default materialization if none is specified
DefaultMaterialization() string
}
Adapter defines the interface for dbt database adapters. Each adapter knows how to map dbt materializations to the correct asset types and provider names for its target platform.
func GetAdapter ¶
GetAdapter returns the adapter for the given dbt adapter type Falls back to a generic DBT adapter if not found
type AlloyDBAdapter ¶
type AlloyDBAdapter struct {
BaseAdapter
}
AlloyDBAdapter handles Google AlloyDB
func (*AlloyDBAdapter) Name ¶
func (a *AlloyDBAdapter) Name() string
type AthenaAdapter ¶
type AthenaAdapter struct {
BaseAdapter
}
AthenaAdapter handles AWS Athena
func (*AthenaAdapter) AssetTypeForMaterialization ¶
func (a *AthenaAdapter) AssetTypeForMaterialization(materialization string) string
func (*AthenaAdapter) Name ¶
func (a *AthenaAdapter) Name() string
type BaseAdapter ¶
type BaseAdapter struct {
ProviderName string
}
BaseAdapter provides default implementations for common adapter behavior
func (*BaseAdapter) AssetTypeForMaterialization ¶
func (a *BaseAdapter) AssetTypeForMaterialization(materialization string) string
func (*BaseAdapter) DefaultMaterialization ¶
func (a *BaseAdapter) DefaultMaterialization() string
func (*BaseAdapter) Name ¶
func (a *BaseAdapter) Name() string
func (*BaseAdapter) SupportsSchemas ¶
func (a *BaseAdapter) SupportsSchemas() bool
type BigQueryAdapter ¶
type BigQueryAdapter struct {
BaseAdapter
}
BigQueryAdapter handles Google BigQuery
func (*BigQueryAdapter) AssetTypeForMaterialization ¶
func (a *BigQueryAdapter) AssetTypeForMaterialization(materialization string) string
func (*BigQueryAdapter) Name ¶
func (a *BigQueryAdapter) Name() string
func (*BigQueryAdapter) SupportsSchemas ¶
func (a *BigQueryAdapter) SupportsSchemas() bool
type CatalogColumn ¶
type CatalogMetadata ¶
type CatalogNode ¶
type CatalogNode struct {
Metadata CatalogMetadata `json:"metadata"`
Columns map[string]CatalogColumn `json:"columns"`
Stats map[string]CatalogStat `json:"stats"`
UniqueID string `json:"unique_id"`
ResourceType string `json:"resource_type"`
}
type CatalogStat ¶
type ClickHouseAdapter ¶
type ClickHouseAdapter struct {
BaseAdapter
}
ClickHouseAdapter handles ClickHouse OLAP database
func (*ClickHouseAdapter) AssetTypeForMaterialization ¶
func (a *ClickHouseAdapter) AssetTypeForMaterialization(materialization string) string
func (*ClickHouseAdapter) Name ¶
func (a *ClickHouseAdapter) Name() string
type Config ¶
type Config struct {
plugin.BaseConfig `json:",inline"`
TargetPath string `` /* 126-byte string literal not displayed */
ProjectName string `json:"project_name" description:"DBT project name" validate:"required"`
Environment string `json:"environment,omitempty" description:"Environment name (e.g., production, staging)" default:"production"`
IncludeManifest bool `json:"include_manifest" description:"Include manifest.json for model definitions" default:"true"`
IncludeCatalog bool `json:"include_catalog" description:"Include catalog.json for table/column descriptions" default:"true"`
IncludeRunResults bool `json:"include_run_results" description:"Include run_results.json for test results" default:"false"`
IncludeSourcesJSON bool `json:"include_sources_json" description:"Include sources.json for source definitions" default:"false"`
DiscoverModels bool `json:"discover_models" description:"Discover DBT models" default:"true"`
DiscoverSources bool `json:"discover_sources" description:"Discover DBT sources" default:"true"`
DiscoverTests bool `json:"discover_tests" description:"Discover DBT tests" default:"false"`
ModelFilter *plugin.Filter `json:"model_filter,omitempty" description:"Filter configuration for models"`
}
Config for DBT plugin +marmot:config
type DBTCatalog ¶
type DBTCatalog struct {
Metadata ManifestMetadata `json:"metadata"`
Sources map[string]CatalogNode `json:"sources"`
Nodes map[string]CatalogNode `json:"nodes"`
Errors []map[string]interface{} `json:"errors"`
}
type DBTColumnFields ¶
type DBTColumnFields struct {
ColumnName string `json:"column_name" metadata:"column_name" description:"Column name"`
DataType string `json:"data_type" metadata:"data_type" description:"Column data type"`
ColumnDescription string `json:"column_description" metadata:"column_description" description:"Column description from DBT"`
ColumnTags []string `json:"column_tags" metadata:"column_tags" description:"Tags applied to this column"`
ColumnComment string `json:"column_comment" metadata:"column_comment" description:"Column comment from database catalog"`
}
DBTColumnFields represents DBT column-specific metadata fields +marmot:metadata
type DBTConfigFields ¶
type DBTConfigFields struct {
ConfigEnabled bool `json:"config_enabled" metadata:"config_enabled" description:"Whether model is enabled"`
ConfigMaterialized string `json:"config_materialized" metadata:"config_materialized" description:"Materialization strategy from config"`
ConfigTags string `json:"config_tags" metadata:"config_tags" description:"Tags from config"`
ConfigPersistDocs bool `json:"config_persist_docs" metadata:"config_persist_docs" description:"Whether to persist documentation to database"`
ConfigFullRefresh bool `json:"config_full_refresh" metadata:"config_full_refresh" description:"Whether to perform full refresh"`
ConfigOnSchemaChange string `` /* 143-byte string literal not displayed */
}
DBTConfigFields represents DBT config-specific metadata fields Fields with config_ prefix contain DBT model configuration +marmot:metadata
type DBTManifest ¶
type DBTManifest struct {
Metadata ManifestMetadata `json:"metadata"`
Nodes map[string]ManifestNode `json:"nodes"`
Sources map[string]ManifestNode `json:"sources"`
Macros map[string]interface{} `json:"macros"`
ChildMap map[string][]string `json:"child_map"`
ParentMap map[string][]string `json:"parent_map"`
Exposures map[string]interface{} `json:"exposures"`
Metrics map[string]interface{} `json:"metrics"`
Dependencies map[string]interface{} `json:"dependencies"`
}
DBT artifact structures
type DBTModelFields ¶
type DBTModelFields struct {
DBTUniqueID string `json:"dbt_unique_id" metadata:"dbt_unique_id" description:"DBT's unique identifier for this node"`
DBTPackage string `json:"dbt_package" metadata:"dbt_package" description:"DBT package name"`
DBTPath string `json:"dbt_path" metadata:"dbt_path" description:"Path to the model file"`
DBTOriginalPath string `json:"dbt_original_path" metadata:"dbt_original_path" description:"Original path in the DBT project"`
DBTMaterialized string `json:"dbt_materialized" metadata:"dbt_materialized" description:"Materialization type (table, view, incremental, ephemeral)"`
Database string `json:"database" metadata:"database" description:"Target database name"`
Schema string `json:"schema" metadata:"schema" description:"Target schema name"`
ModelName string `json:"model_name" metadata:"model_name" description:"DBT model name"`
TableName string `json:"table_name" metadata:"table_name" description:"Physical table/view name in database"`
FullyQualifiedName string `json:"fully_qualified_name" metadata:"fully_qualified_name" description:"Fully qualified name (database.schema.table)"`
ProjectName string `json:"project_name" metadata:"project_name" description:"DBT project name"`
Environment string `json:"environment" metadata:"environment" description:"Deployment environment (dev, prod, etc)"`
Alias string `json:"alias" metadata:"alias" description:"Table alias if different from model name"`
AdapterType string `json:"adapter_type" metadata:"adapter_type" description:"Database adapter type (postgres, snowflake, bigquery, etc)"`
DBTVersion string `json:"dbt_version" metadata:"dbt_version" description:"DBT version used to generate this model"`
LastRunStatus string `json:"last_run_status" metadata:"last_run_status" description:"Status of the last DBT run (success, error, skipped)"`
LastRunExecutionTime float64 `json:"last_run_execution_time" metadata:"last_run_execution_time" description:"Execution time of last run in seconds"`
LastRunMessage string `json:"last_run_message" metadata:"last_run_message" description:"Message from last DBT run"`
LastRunFailures int `json:"last_run_failures" metadata:"last_run_failures" description:"Number of failures in last run"`
RawSQL string `json:"raw_sql" metadata:"raw_sql" description:"Raw SQL before compilation"`
Owner string `json:"owner" metadata:"owner" description:"Table/view owner from database catalog"`
CatalogComment string `json:"catalog_comment" metadata:"catalog_comment" description:"Comment from database catalog"`
}
DBTModelFields represents DBT model-specific metadata fields +marmot:metadata
type DBTRunResults ¶
type DBTSeedFields ¶
type DBTSeedFields struct {
DBTUniqueID string `json:"dbt_unique_id" metadata:"dbt_unique_id" description:"DBT's unique identifier for this seed"`
DBTPackage string `json:"dbt_package" metadata:"dbt_package" description:"DBT package name"`
Database string `json:"database" metadata:"database" description:"Target database name"`
Schema string `json:"schema" metadata:"schema" description:"Target schema name"`
TableName string `json:"table_name" metadata:"table_name" description:"Seed table name"`
FullyQualifiedName string `json:"fully_qualified_name" metadata:"fully_qualified_name" description:"Fully qualified name (database.schema.table)"`
SeedPath string `json:"seed_path" metadata:"seed_path" description:"Path to seed CSV file"`
ProjectName string `json:"project_name" metadata:"project_name" description:"DBT project name"`
Environment string `json:"environment" metadata:"environment" description:"Deployment environment"`
}
DBTSeedFields represents DBT seed-specific metadata fields +marmot:metadata
type DBTSourceFields ¶
type DBTSourceFields struct {
DBTUniqueID string `json:"dbt_unique_id" metadata:"dbt_unique_id" description:"DBT's unique identifier for this source"`
DBTPackage string `json:"dbt_package" metadata:"dbt_package" description:"DBT package name"`
Database string `json:"database" metadata:"database" description:"Source database name"`
Schema string `json:"schema" metadata:"schema" description:"Source schema name"`
TableName string `json:"table_name" metadata:"table_name" description:"Source table name"`
FullyQualifiedName string `json:"fully_qualified_name" metadata:"fully_qualified_name" description:"Fully qualified name (database.schema.table)"`
SourceName string `json:"source_name" metadata:"source_name" description:"DBT source name"`
Identifier string `json:"identifier" metadata:"identifier" description:"Physical table identifier"`
Loaded bool `json:"loaded" metadata:"loaded" description:"Whether source was loaded at time of DBT execution"`
ProjectName string `json:"project_name" metadata:"project_name" description:"DBT project name"`
Environment string `json:"environment" metadata:"environment" description:"Deployment environment"`
FreshnessChecked bool `json:"freshness_checked" metadata:"freshness_checked" description:"Whether freshness checks are configured"`
}
DBTSourceFields represents DBT source-specific metadata fields +marmot:metadata
type DBTStatsFields ¶
type DBTStatsFields struct {
StatRowCount int64 `json:"stat_row_count" metadata:"stat_row_count" description:"Number of rows"`
StatBytes int64 `json:"stat_bytes" metadata:"stat_bytes" description:"Size in bytes"`
StatLastModified string `json:"stat_last_modified" metadata:"stat_last_modified" description:"Last modification timestamp"`
StatNumRows int64 `json:"stat_num_rows" metadata:"stat_num_rows" description:"Number of rows (alternative)"`
StatApproximateCount int64 `json:"stat_approximate_count" metadata:"stat_approximate_count" description:"Approximate row count"`
StatSize float64 `json:"stat_size" metadata:"stat_size" description:"Table size"`
}
DBTStatsFields represents DBT catalog statistics fields Fields with stat_ prefix contain statistics from database catalog +marmot:metadata
type DatabricksAdapter ¶
type DatabricksAdapter struct {
BaseAdapter
}
DatabricksAdapter handles Databricks
func (*DatabricksAdapter) AssetTypeForMaterialization ¶
func (a *DatabricksAdapter) AssetTypeForMaterialization(materialization string) string
func (*DatabricksAdapter) Name ¶
func (a *DatabricksAdapter) Name() string
type DbtSchemaColumn ¶
type DbtSchemaColumn struct {
Name string `json:"name"`
Type string `json:"type,omitempty"`
Description string `json:"description,omitempty"`
}
DbtSchemaColumn represents a column in the dbt schema format
type DremioAdapter ¶
type DremioAdapter struct {
BaseAdapter
}
DremioAdapter handles Dremio data lakehouse platform
func (*DremioAdapter) Name ¶
func (a *DremioAdapter) Name() string
type DuckDBAdapter ¶
type DuckDBAdapter struct {
BaseAdapter
}
DuckDBAdapter handles DuckDB
func (*DuckDBAdapter) Name ¶
func (a *DuckDBAdapter) Name() string
type FabricAdapter ¶
type FabricAdapter struct {
BaseAdapter
}
FabricAdapter handles Microsoft Fabric Data Warehouse
func (*FabricAdapter) Name ¶
func (a *FabricAdapter) Name() string
type FabricSparkAdapter ¶
type FabricSparkAdapter struct {
BaseAdapter
}
FabricSparkAdapter handles Microsoft Fabric Lakehouse
func (*FabricSparkAdapter) AssetTypeForMaterialization ¶
func (a *FabricSparkAdapter) AssetTypeForMaterialization(materialization string) string
func (*FabricSparkAdapter) Name ¶
func (a *FabricSparkAdapter) Name() string
type GenericAdapter ¶
type GenericAdapter struct {
BaseAdapter
}
GenericAdapter is the fallback adapter when no specific adapter is registered
func (*GenericAdapter) Name ¶
func (a *GenericAdapter) Name() string
type GlueAdapter ¶
type GlueAdapter struct {
BaseAdapter
}
GlueAdapter handles AWS Glue
func (*GlueAdapter) Name ¶
func (a *GlueAdapter) Name() string
type LakebaseAdapter ¶
type LakebaseAdapter struct {
BaseAdapter
}
LakebaseAdapter handles Databricks Lakebase
func (*LakebaseAdapter) Name ¶
func (a *LakebaseAdapter) Name() string
type ManifestMetadata ¶
type ManifestNode ¶
type ManifestNode struct {
UniqueID string `json:"unique_id"`
Name string `json:"name"`
ResourceType string `json:"resource_type"`
PackageName string `json:"package_name"`
Path string `json:"path"`
OriginalPath string `json:"original_file_path"`
Database string `json:"database"`
Schema string `json:"schema"`
Alias string `json:"alias"`
Description string `json:"description"`
Columns map[string]NodeColumn `json:"columns"`
Tags []string `json:"tags"`
Meta map[string]interface{} `json:"meta"`
DependsOn NodeDependency `json:"depends_on"`
Config map[string]interface{} `json:"config"`
CompiledSQL string `json:"compiled_sql"`
CompiledCode string `json:"compiled_code"`
RawSQL string `json:"raw_sql"`
RawCode string `json:"raw_code"`
Materialized string `json:"materialized"`
}
type MaterializeAdapter ¶
type MaterializeAdapter struct {
BaseAdapter
}
MaterializeAdapter handles Materialize streaming database
func (*MaterializeAdapter) AssetTypeForMaterialization ¶
func (a *MaterializeAdapter) AssetTypeForMaterialization(materialization string) string
func (*MaterializeAdapter) DefaultMaterialization ¶
func (a *MaterializeAdapter) DefaultMaterialization() string
func (*MaterializeAdapter) Name ¶
func (a *MaterializeAdapter) Name() string
type MySQLAdapter ¶
type MySQLAdapter struct {
BaseAdapter
}
MySQLAdapter handles MySQL
func (*MySQLAdapter) Name ¶
func (a *MySQLAdapter) Name() string
type NetezzaAdapter ¶
type NetezzaAdapter struct {
BaseAdapter
}
NetezzaAdapter handles IBM Netezza
func (*NetezzaAdapter) Name ¶
func (a *NetezzaAdapter) Name() string
type NodeColumn ¶
type NodeDependency ¶
type OracleAdapter ¶
type OracleAdapter struct {
BaseAdapter
}
OracleAdapter handles Oracle Database / Autonomous Database
func (*OracleAdapter) AssetTypeForMaterialization ¶
func (a *OracleAdapter) AssetTypeForMaterialization(materialization string) string
func (*OracleAdapter) Name ¶
func (a *OracleAdapter) Name() string
type PostgresAdapter ¶
type PostgresAdapter struct {
BaseAdapter
}
PostgresAdapter handles PostgreSQL
func (*PostgresAdapter) Name ¶
func (a *PostgresAdapter) Name() string
type RedshiftAdapter ¶
type RedshiftAdapter struct {
BaseAdapter
}
RedshiftAdapter handles Amazon Redshift
func (*RedshiftAdapter) AssetTypeForMaterialization ¶
func (a *RedshiftAdapter) AssetTypeForMaterialization(materialization string) string
func (*RedshiftAdapter) Name ¶
func (a *RedshiftAdapter) Name() string
type SQLServerAdapter ¶
type SQLServerAdapter struct {
BaseAdapter
}
SQLServerAdapter handles Microsoft SQL Server
func (*SQLServerAdapter) Name ¶
func (a *SQLServerAdapter) Name() string
type SalesforceAdapter ¶
type SalesforceAdapter struct {
BaseAdapter
}
SalesforceAdapter handles Salesforce Data Cloud
func (*SalesforceAdapter) AssetTypeForMaterialization ¶
func (a *SalesforceAdapter) AssetTypeForMaterialization(materialization string) string
func (*SalesforceAdapter) Name ¶
func (a *SalesforceAdapter) Name() string
type SingleStoreAdapter ¶
type SingleStoreAdapter struct {
BaseAdapter
}
SingleStoreAdapter handles SingleStore
func (*SingleStoreAdapter) Name ¶
func (a *SingleStoreAdapter) Name() string
type SnowflakeAdapter ¶
type SnowflakeAdapter struct {
BaseAdapter
}
SnowflakeAdapter handles Snowflake Data Cloud
func (*SnowflakeAdapter) AssetTypeForMaterialization ¶
func (a *SnowflakeAdapter) AssetTypeForMaterialization(materialization string) string
func (*SnowflakeAdapter) Name ¶
func (a *SnowflakeAdapter) Name() string
type Source ¶
type Source struct {
// contains filtered or unexported fields
}
func (*Source) Discover ¶
func (s *Source) Discover(ctx context.Context, pluginConfig plugin.RawPluginConfig) (*plugin.DiscoveryResult, error)
func (*Source) Validate ¶
func (s *Source) Validate(rawConfig plugin.RawPluginConfig) (plugin.RawPluginConfig, error)
type SparkAdapter ¶
type SparkAdapter struct {
BaseAdapter
}
SparkAdapter handles Apache Spark
func (*SparkAdapter) Name ¶
func (a *SparkAdapter) Name() string
type SynapseAdapter ¶
type SynapseAdapter struct {
BaseAdapter
}
SynapseAdapter handles Azure Synapse Analytics
func (*SynapseAdapter) Name ¶
func (a *SynapseAdapter) Name() string
type TeradataAdapter ¶
type TeradataAdapter struct {
BaseAdapter
}
TeradataAdapter handles Teradata data warehouse
func (*TeradataAdapter) AssetTypeForMaterialization ¶
func (a *TeradataAdapter) AssetTypeForMaterialization(materialization string) string
func (*TeradataAdapter) Name ¶
func (a *TeradataAdapter) Name() string
type TrinoAdapter ¶
type TrinoAdapter struct {
BaseAdapter
}
TrinoAdapter handles Trino/Starburst distributed SQL query engine
func (*TrinoAdapter) Name ¶
func (a *TrinoAdapter) Name() string