dbt

package
v0.5.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 12, 2026 License: MIT Imports: 12 Imported by: 0

Documentation

Overview

+marmot:name=DBT +marmot:description=This plugin ingests metadata from DBT (Data Build Tool) projects, including models, tests, and lineage. +marmot:status=experimental +marmot:features=Assets, Lineage

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func RegisterAdapter

func RegisterAdapter(adapterType string, adapter Adapter)

RegisterAdapter registers an adapter for a given dbt adapter type

Types

type Adapter

type Adapter interface {
	// Name returns the canonical provider name for this adapter (e.g., "Snowflake", "BigQuery")
	Name() string

	// AssetTypeForMaterialization maps a dbt materialization type to the appropriate asset type
	// For most adapters: table -> Table, view -> View, incremental -> Table
	// Some adapters have special types (e.g., ClickHouse has Dictionary, Distributed Table)
	AssetTypeForMaterialization(materialization string) string

	// SupportsSchemas returns true if the adapter uses database.schema.table naming
	// Some platforms like BigQuery use project.dataset.table instead
	SupportsSchemas() bool

	// DefaultMaterialization returns the default materialization if none is specified
	DefaultMaterialization() string
}

Adapter defines the interface for dbt database adapters. Each adapter knows how to map dbt materializations to the correct asset types and provider names for its target platform.

func GetAdapter

func GetAdapter(adapterType string) Adapter

GetAdapter returns the adapter for the given dbt adapter type Falls back to a generic DBT adapter if not found

type AlloyDBAdapter

type AlloyDBAdapter struct {
	BaseAdapter
}

AlloyDBAdapter handles Google AlloyDB

func (*AlloyDBAdapter) Name

func (a *AlloyDBAdapter) Name() string

type AthenaAdapter

type AthenaAdapter struct {
	BaseAdapter
}

AthenaAdapter handles AWS Athena

func (*AthenaAdapter) AssetTypeForMaterialization

func (a *AthenaAdapter) AssetTypeForMaterialization(materialization string) string

func (*AthenaAdapter) Name

func (a *AthenaAdapter) Name() string

type BaseAdapter

type BaseAdapter struct {
	ProviderName string
}

BaseAdapter provides default implementations for common adapter behavior

func (*BaseAdapter) AssetTypeForMaterialization

func (a *BaseAdapter) AssetTypeForMaterialization(materialization string) string

func (*BaseAdapter) DefaultMaterialization

func (a *BaseAdapter) DefaultMaterialization() string

func (*BaseAdapter) Name

func (a *BaseAdapter) Name() string

func (*BaseAdapter) SupportsSchemas

func (a *BaseAdapter) SupportsSchemas() bool

type BigQueryAdapter

type BigQueryAdapter struct {
	BaseAdapter
}

BigQueryAdapter handles Google BigQuery

func (*BigQueryAdapter) AssetTypeForMaterialization

func (a *BigQueryAdapter) AssetTypeForMaterialization(materialization string) string

func (*BigQueryAdapter) Name

func (a *BigQueryAdapter) Name() string

func (*BigQueryAdapter) SupportsSchemas

func (a *BigQueryAdapter) SupportsSchemas() bool

type CatalogColumn

type CatalogColumn struct {
	Type    string `json:"type"`
	Comment string `json:"comment"`
	Index   int    `json:"index"`
	Name    string `json:"name"`
}

type CatalogMetadata

type CatalogMetadata struct {
	Type     string `json:"type"`
	Database string `json:"database"`
	Schema   string `json:"schema"`
	Name     string `json:"name"`
	Owner    string `json:"owner"`
	Comment  string `json:"comment"`
}

type CatalogNode

type CatalogNode struct {
	Metadata     CatalogMetadata          `json:"metadata"`
	Columns      map[string]CatalogColumn `json:"columns"`
	Stats        map[string]CatalogStat   `json:"stats"`
	UniqueID     string                   `json:"unique_id"`
	ResourceType string                   `json:"resource_type"`
}

type CatalogStat

type CatalogStat struct {
	ID          string      `json:"id"`
	Label       string      `json:"label"`
	Value       interface{} `json:"value"`
	Description string      `json:"description"`
	Include     bool        `json:"include"`
}

type ClickHouseAdapter

type ClickHouseAdapter struct {
	BaseAdapter
}

ClickHouseAdapter handles ClickHouse OLAP database

func (*ClickHouseAdapter) AssetTypeForMaterialization

func (a *ClickHouseAdapter) AssetTypeForMaterialization(materialization string) string

func (*ClickHouseAdapter) Name

func (a *ClickHouseAdapter) Name() string

type Config

type Config struct {
	plugin.BaseConfig `json:",inline"`

	TargetPath string `` /* 126-byte string literal not displayed */

	ProjectName string `json:"project_name" description:"DBT project name" validate:"required"`
	Environment string `json:"environment,omitempty" description:"Environment name (e.g., production, staging)" default:"production"`

	IncludeManifest    bool `json:"include_manifest" description:"Include manifest.json for model definitions" default:"true"`
	IncludeCatalog     bool `json:"include_catalog" description:"Include catalog.json for table/column descriptions" default:"true"`
	IncludeRunResults  bool `json:"include_run_results" description:"Include run_results.json for test results" default:"false"`
	IncludeSourcesJSON bool `json:"include_sources_json" description:"Include sources.json for source definitions" default:"false"`

	DiscoverModels  bool           `json:"discover_models" description:"Discover DBT models" default:"true"`
	DiscoverSources bool           `json:"discover_sources" description:"Discover DBT sources" default:"true"`
	DiscoverTests   bool           `json:"discover_tests" description:"Discover DBT tests" default:"false"`
	ModelFilter     *plugin.Filter `json:"model_filter,omitempty" description:"Filter configuration for models"`
}

Config for DBT plugin +marmot:config

type DBTCatalog

type DBTCatalog struct {
	Metadata ManifestMetadata         `json:"metadata"`
	Sources  map[string]CatalogNode   `json:"sources"`
	Nodes    map[string]CatalogNode   `json:"nodes"`
	Errors   []map[string]interface{} `json:"errors"`
}

type DBTColumnFields

type DBTColumnFields struct {
	ColumnName        string   `json:"column_name" metadata:"column_name" description:"Column name"`
	DataType          string   `json:"data_type" metadata:"data_type" description:"Column data type"`
	ColumnDescription string   `json:"column_description" metadata:"column_description" description:"Column description from DBT"`
	ColumnTags        []string `json:"column_tags" metadata:"column_tags" description:"Tags applied to this column"`
	ColumnComment     string   `json:"column_comment" metadata:"column_comment" description:"Column comment from database catalog"`
}

DBTColumnFields represents DBT column-specific metadata fields +marmot:metadata

type DBTConfigFields

type DBTConfigFields struct {
	ConfigEnabled        bool   `json:"config_enabled" metadata:"config_enabled" description:"Whether model is enabled"`
	ConfigMaterialized   string `json:"config_materialized" metadata:"config_materialized" description:"Materialization strategy from config"`
	ConfigTags           string `json:"config_tags" metadata:"config_tags" description:"Tags from config"`
	ConfigPersistDocs    bool   `json:"config_persist_docs" metadata:"config_persist_docs" description:"Whether to persist documentation to database"`
	ConfigFullRefresh    bool   `json:"config_full_refresh" metadata:"config_full_refresh" description:"Whether to perform full refresh"`
	ConfigOnSchemaChange string `` /* 143-byte string literal not displayed */
}

DBTConfigFields represents DBT config-specific metadata fields Fields with config_ prefix contain DBT model configuration +marmot:metadata

type DBTManifest

type DBTManifest struct {
	Metadata     ManifestMetadata        `json:"metadata"`
	Nodes        map[string]ManifestNode `json:"nodes"`
	Sources      map[string]ManifestNode `json:"sources"`
	Macros       map[string]interface{}  `json:"macros"`
	ChildMap     map[string][]string     `json:"child_map"`
	ParentMap    map[string][]string     `json:"parent_map"`
	Exposures    map[string]interface{}  `json:"exposures"`
	Metrics      map[string]interface{}  `json:"metrics"`
	Dependencies map[string]interface{}  `json:"dependencies"`
}

DBT artifact structures

type DBTModelFields

type DBTModelFields struct {
	DBTUniqueID          string  `json:"dbt_unique_id" metadata:"dbt_unique_id" description:"DBT's unique identifier for this node"`
	DBTPackage           string  `json:"dbt_package" metadata:"dbt_package" description:"DBT package name"`
	DBTPath              string  `json:"dbt_path" metadata:"dbt_path" description:"Path to the model file"`
	DBTOriginalPath      string  `json:"dbt_original_path" metadata:"dbt_original_path" description:"Original path in the DBT project"`
	DBTMaterialized      string  `json:"dbt_materialized" metadata:"dbt_materialized" description:"Materialization type (table, view, incremental, ephemeral)"`
	Database             string  `json:"database" metadata:"database" description:"Target database name"`
	Schema               string  `json:"schema" metadata:"schema" description:"Target schema name"`
	ModelName            string  `json:"model_name" metadata:"model_name" description:"DBT model name"`
	TableName            string  `json:"table_name" metadata:"table_name" description:"Physical table/view name in database"`
	FullyQualifiedName   string  `json:"fully_qualified_name" metadata:"fully_qualified_name" description:"Fully qualified name (database.schema.table)"`
	ProjectName          string  `json:"project_name" metadata:"project_name" description:"DBT project name"`
	Environment          string  `json:"environment" metadata:"environment" description:"Deployment environment (dev, prod, etc)"`
	Alias                string  `json:"alias" metadata:"alias" description:"Table alias if different from model name"`
	AdapterType          string  `json:"adapter_type" metadata:"adapter_type" description:"Database adapter type (postgres, snowflake, bigquery, etc)"`
	DBTVersion           string  `json:"dbt_version" metadata:"dbt_version" description:"DBT version used to generate this model"`
	LastRunStatus        string  `json:"last_run_status" metadata:"last_run_status" description:"Status of the last DBT run (success, error, skipped)"`
	LastRunExecutionTime float64 `json:"last_run_execution_time" metadata:"last_run_execution_time" description:"Execution time of last run in seconds"`
	LastRunMessage       string  `json:"last_run_message" metadata:"last_run_message" description:"Message from last DBT run"`
	LastRunFailures      int     `json:"last_run_failures" metadata:"last_run_failures" description:"Number of failures in last run"`
	RawSQL               string  `json:"raw_sql" metadata:"raw_sql" description:"Raw SQL before compilation"`
	Owner                string  `json:"owner" metadata:"owner" description:"Table/view owner from database catalog"`
	CatalogComment       string  `json:"catalog_comment" metadata:"catalog_comment" description:"Comment from database catalog"`
}

DBTModelFields represents DBT model-specific metadata fields +marmot:metadata

type DBTRunResults

type DBTRunResults struct {
	Metadata      ManifestMetadata `json:"metadata"`
	Results       []RunResult      `json:"results"`
	ElapsedTime   float64          `json:"elapsed_time"`
	Args          interface{}      `json:"args"`
	GeneratedAt   time.Time        `json:"generated_at"`
	SuccessStatus string           `json:"success"`
}

type DBTSeedFields

type DBTSeedFields struct {
	DBTUniqueID        string `json:"dbt_unique_id" metadata:"dbt_unique_id" description:"DBT's unique identifier for this seed"`
	DBTPackage         string `json:"dbt_package" metadata:"dbt_package" description:"DBT package name"`
	Database           string `json:"database" metadata:"database" description:"Target database name"`
	Schema             string `json:"schema" metadata:"schema" description:"Target schema name"`
	TableName          string `json:"table_name" metadata:"table_name" description:"Seed table name"`
	FullyQualifiedName string `json:"fully_qualified_name" metadata:"fully_qualified_name" description:"Fully qualified name (database.schema.table)"`
	SeedPath           string `json:"seed_path" metadata:"seed_path" description:"Path to seed CSV file"`
	ProjectName        string `json:"project_name" metadata:"project_name" description:"DBT project name"`
	Environment        string `json:"environment" metadata:"environment" description:"Deployment environment"`
}

DBTSeedFields represents DBT seed-specific metadata fields +marmot:metadata

type DBTSourceFields

type DBTSourceFields struct {
	DBTUniqueID        string `json:"dbt_unique_id" metadata:"dbt_unique_id" description:"DBT's unique identifier for this source"`
	DBTPackage         string `json:"dbt_package" metadata:"dbt_package" description:"DBT package name"`
	Database           string `json:"database" metadata:"database" description:"Source database name"`
	Schema             string `json:"schema" metadata:"schema" description:"Source schema name"`
	TableName          string `json:"table_name" metadata:"table_name" description:"Source table name"`
	FullyQualifiedName string `json:"fully_qualified_name" metadata:"fully_qualified_name" description:"Fully qualified name (database.schema.table)"`
	SourceName         string `json:"source_name" metadata:"source_name" description:"DBT source name"`
	Identifier         string `json:"identifier" metadata:"identifier" description:"Physical table identifier"`
	Loaded             bool   `json:"loaded" metadata:"loaded" description:"Whether source was loaded at time of DBT execution"`
	ProjectName        string `json:"project_name" metadata:"project_name" description:"DBT project name"`
	Environment        string `json:"environment" metadata:"environment" description:"Deployment environment"`
	FreshnessChecked   bool   `json:"freshness_checked" metadata:"freshness_checked" description:"Whether freshness checks are configured"`
}

DBTSourceFields represents DBT source-specific metadata fields +marmot:metadata

type DBTStatsFields

type DBTStatsFields struct {
	StatRowCount         int64   `json:"stat_row_count" metadata:"stat_row_count" description:"Number of rows"`
	StatBytes            int64   `json:"stat_bytes" metadata:"stat_bytes" description:"Size in bytes"`
	StatLastModified     string  `json:"stat_last_modified" metadata:"stat_last_modified" description:"Last modification timestamp"`
	StatNumRows          int64   `json:"stat_num_rows" metadata:"stat_num_rows" description:"Number of rows (alternative)"`
	StatApproximateCount int64   `json:"stat_approximate_count" metadata:"stat_approximate_count" description:"Approximate row count"`
	StatSize             float64 `json:"stat_size" metadata:"stat_size" description:"Table size"`
}

DBTStatsFields represents DBT catalog statistics fields Fields with stat_ prefix contain statistics from database catalog +marmot:metadata

type DatabricksAdapter

type DatabricksAdapter struct {
	BaseAdapter
}

DatabricksAdapter handles Databricks

func (*DatabricksAdapter) AssetTypeForMaterialization

func (a *DatabricksAdapter) AssetTypeForMaterialization(materialization string) string

func (*DatabricksAdapter) Name

func (a *DatabricksAdapter) Name() string

type DbtSchemaColumn

type DbtSchemaColumn struct {
	Name        string `json:"name"`
	Type        string `json:"type,omitempty"`
	Description string `json:"description,omitempty"`
}

DbtSchemaColumn represents a column in the dbt schema format

type DremioAdapter

type DremioAdapter struct {
	BaseAdapter
}

DremioAdapter handles Dremio data lakehouse platform

func (*DremioAdapter) Name

func (a *DremioAdapter) Name() string

type DuckDBAdapter

type DuckDBAdapter struct {
	BaseAdapter
}

DuckDBAdapter handles DuckDB

func (*DuckDBAdapter) Name

func (a *DuckDBAdapter) Name() string

type FabricAdapter

type FabricAdapter struct {
	BaseAdapter
}

FabricAdapter handles Microsoft Fabric Data Warehouse

func (*FabricAdapter) Name

func (a *FabricAdapter) Name() string

type FabricSparkAdapter

type FabricSparkAdapter struct {
	BaseAdapter
}

FabricSparkAdapter handles Microsoft Fabric Lakehouse

func (*FabricSparkAdapter) AssetTypeForMaterialization

func (a *FabricSparkAdapter) AssetTypeForMaterialization(materialization string) string

func (*FabricSparkAdapter) Name

func (a *FabricSparkAdapter) Name() string

type GenericAdapter

type GenericAdapter struct {
	BaseAdapter
}

GenericAdapter is the fallback adapter when no specific adapter is registered

func (*GenericAdapter) Name

func (a *GenericAdapter) Name() string

type GlueAdapter

type GlueAdapter struct {
	BaseAdapter
}

GlueAdapter handles AWS Glue

func (*GlueAdapter) Name

func (a *GlueAdapter) Name() string

type LakebaseAdapter

type LakebaseAdapter struct {
	BaseAdapter
}

LakebaseAdapter handles Databricks Lakebase

func (*LakebaseAdapter) Name

func (a *LakebaseAdapter) Name() string

type ManifestMetadata

type ManifestMetadata struct {
	DBTVersion   string    `json:"dbt_version"`
	GeneratedAt  time.Time `json:"generated_at"`
	AdapterType  string    `json:"adapter_type"`
	ProjectName  string    `json:"project_name"`
	InvocationID string    `json:"invocation_id"`
}

type ManifestNode

type ManifestNode struct {
	UniqueID     string                 `json:"unique_id"`
	Name         string                 `json:"name"`
	ResourceType string                 `json:"resource_type"`
	PackageName  string                 `json:"package_name"`
	Path         string                 `json:"path"`
	OriginalPath string                 `json:"original_file_path"`
	Database     string                 `json:"database"`
	Schema       string                 `json:"schema"`
	Alias        string                 `json:"alias"`
	Description  string                 `json:"description"`
	Columns      map[string]NodeColumn  `json:"columns"`
	Tags         []string               `json:"tags"`
	Meta         map[string]interface{} `json:"meta"`
	DependsOn    NodeDependency         `json:"depends_on"`
	Config       map[string]interface{} `json:"config"`
	CompiledSQL  string                 `json:"compiled_sql"`
	CompiledCode string                 `json:"compiled_code"`
	RawSQL       string                 `json:"raw_sql"`
	RawCode      string                 `json:"raw_code"`
	Materialized string                 `json:"materialized"`
}

type MaterializeAdapter

type MaterializeAdapter struct {
	BaseAdapter
}

MaterializeAdapter handles Materialize streaming database

func (*MaterializeAdapter) AssetTypeForMaterialization

func (a *MaterializeAdapter) AssetTypeForMaterialization(materialization string) string

func (*MaterializeAdapter) DefaultMaterialization

func (a *MaterializeAdapter) DefaultMaterialization() string

func (*MaterializeAdapter) Name

func (a *MaterializeAdapter) Name() string

type MySQLAdapter

type MySQLAdapter struct {
	BaseAdapter
}

MySQLAdapter handles MySQL

func (*MySQLAdapter) Name

func (a *MySQLAdapter) Name() string

type NetezzaAdapter

type NetezzaAdapter struct {
	BaseAdapter
}

NetezzaAdapter handles IBM Netezza

func (*NetezzaAdapter) Name

func (a *NetezzaAdapter) Name() string

type NodeColumn

type NodeColumn struct {
	Name        string                 `json:"name"`
	Description string                 `json:"description"`
	Meta        map[string]interface{} `json:"meta"`
	Tags        []string               `json:"tags"`
	DataType    string                 `json:"data_type"`
}

type NodeDependency

type NodeDependency struct {
	Nodes  []string `json:"nodes"`
	Macros []string `json:"macros"`
}

type OracleAdapter

type OracleAdapter struct {
	BaseAdapter
}

OracleAdapter handles Oracle Database / Autonomous Database

func (*OracleAdapter) AssetTypeForMaterialization

func (a *OracleAdapter) AssetTypeForMaterialization(materialization string) string

func (*OracleAdapter) Name

func (a *OracleAdapter) Name() string

type PostgresAdapter

type PostgresAdapter struct {
	BaseAdapter
}

PostgresAdapter handles PostgreSQL

func (*PostgresAdapter) Name

func (a *PostgresAdapter) Name() string

type RedshiftAdapter

type RedshiftAdapter struct {
	BaseAdapter
}

RedshiftAdapter handles Amazon Redshift

func (*RedshiftAdapter) AssetTypeForMaterialization

func (a *RedshiftAdapter) AssetTypeForMaterialization(materialization string) string

func (*RedshiftAdapter) Name

func (a *RedshiftAdapter) Name() string

type RunResult

type RunResult struct {
	UniqueID        string                 `json:"unique_id"`
	Status          string                 `json:"status"`
	ExecutionTime   float64                `json:"execution_time"`
	Message         string                 `json:"message"`
	Failures        int                    `json:"failures"`
	AdapterResponse map[string]interface{} `json:"adapter_response"`
	Thread          string                 `json:"thread_id"`
}

type SQLServerAdapter

type SQLServerAdapter struct {
	BaseAdapter
}

SQLServerAdapter handles Microsoft SQL Server

func (*SQLServerAdapter) Name

func (a *SQLServerAdapter) Name() string

type SalesforceAdapter

type SalesforceAdapter struct {
	BaseAdapter
}

SalesforceAdapter handles Salesforce Data Cloud

func (*SalesforceAdapter) AssetTypeForMaterialization

func (a *SalesforceAdapter) AssetTypeForMaterialization(materialization string) string

func (*SalesforceAdapter) Name

func (a *SalesforceAdapter) Name() string

type SingleStoreAdapter

type SingleStoreAdapter struct {
	BaseAdapter
}

SingleStoreAdapter handles SingleStore

func (*SingleStoreAdapter) Name

func (a *SingleStoreAdapter) Name() string

type SnowflakeAdapter

type SnowflakeAdapter struct {
	BaseAdapter
}

SnowflakeAdapter handles Snowflake Data Cloud

func (*SnowflakeAdapter) AssetTypeForMaterialization

func (a *SnowflakeAdapter) AssetTypeForMaterialization(materialization string) string

func (*SnowflakeAdapter) Name

func (a *SnowflakeAdapter) Name() string

type Source

type Source struct {
	// contains filtered or unexported fields
}

func (*Source) Discover

func (s *Source) Discover(ctx context.Context, pluginConfig plugin.RawPluginConfig) (*plugin.DiscoveryResult, error)

func (*Source) Validate

func (s *Source) Validate(rawConfig plugin.RawPluginConfig) (plugin.RawPluginConfig, error)

type SparkAdapter

type SparkAdapter struct {
	BaseAdapter
}

SparkAdapter handles Apache Spark

func (*SparkAdapter) Name

func (a *SparkAdapter) Name() string

type SynapseAdapter

type SynapseAdapter struct {
	BaseAdapter
}

SynapseAdapter handles Azure Synapse Analytics

func (*SynapseAdapter) Name

func (a *SynapseAdapter) Name() string

type TeradataAdapter

type TeradataAdapter struct {
	BaseAdapter
}

TeradataAdapter handles Teradata data warehouse

func (*TeradataAdapter) AssetTypeForMaterialization

func (a *TeradataAdapter) AssetTypeForMaterialization(materialization string) string

func (*TeradataAdapter) Name

func (a *TeradataAdapter) Name() string

type TrinoAdapter

type TrinoAdapter struct {
	BaseAdapter
}

TrinoAdapter handles Trino/Starburst distributed SQL query engine

func (*TrinoAdapter) Name

func (a *TrinoAdapter) Name() string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL