pipeline

package
v0.11.310 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 7, 2025 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CommentTask TaskDefinitionType = "comment"
	YamlTask    TaskDefinitionType = "yaml"

	AssetTypePython                 = AssetType("python")
	AssetTypeSnowflakeQuery         = AssetType("sf.sql")
	AssetTypeSnowflakeSeed          = AssetType("sf.seed")
	AssetTypeSnowflakeQuerySensor   = AssetType("sf.sensor.query")
	AssetTypeSnowflakeTableSensor   = AssetType("sf.sensor.table")
	AssetTypeBigqueryQuery          = AssetType("bq.sql")
	AssetTypeBigqueryTableSensor    = AssetType("bq.sensor.table")
	AssetTypeBigqueryQuerySensor    = AssetType("bq.sensor.query")
	AssetTypeBigquerySource         = AssetType("bq.source")
	AssetTypeBigquerySeed           = AssetType("bq.seed")
	AssetTypeSnowflakeSource        = AssetType("sf.source")
	AssetTypePostgresSource         = AssetType("pg.source")
	AssetTypeRedshiftSource         = AssetType("rs.source")
	AssetTypeMsSQLSource            = AssetType("ms.source")
	AssetTypeMsSQLTableSensor       = AssetType("ms.sensor.table")
	AssetTypeDatabricksSource       = AssetType("databricks.source")
	AssetTypeSynapseSource          = AssetType("synapse.source")
	AssetTypeSynapseTableSensor     = AssetType("synapse.sensor.table")
	AssetTypeAthenaSource           = AssetType("athena.source")
	AssetTypeDuckDBSource           = AssetType("duckdb.source")
	AssetTypeDuckDBQuery            = AssetType("duckdb.sql")
	AssetTypeDuckDBSeed             = AssetType("duckdb.seed")
	AssetTypeDuckDBQuerySensor      = AssetType("duckdb.sensor.query")
	AssetTypeEmpty                  = AssetType("empty")
	AssetTypePostgresQuery          = AssetType("pg.sql")
	AssetTypePostgresSeed           = AssetType("pg.seed")
	AssetTypePostgresQuerySensor    = AssetType("pg.sensor.query")
	AssetTypePostgresTableSensor    = AssetType("pg.sensor.table")
	AssetTypeRedshiftQuery          = AssetType("rs.sql")
	AssetTypeRedshiftSeed           = AssetType("rs.seed")
	AssetTypeRedshiftQuerySensor    = AssetType("rs.sensor.query")
	AssetTypeRedshiftTableSensor    = AssetType("rs.sensor.table")
	AssetTypeAthenaQuery            = AssetType("athena.sql")
	AssetTypeAthenaSQLSensor        = AssetType("athena.sensor.query")
	AssetTypeAthenaTableSensor      = AssetType("athena.sensor.table")
	AssetTypeAthenaSeed             = AssetType("athena.seed")
	AssetTypeMsSQLQuery             = AssetType("ms.sql")
	AssetTypeMsSQLSeed              = AssetType("ms.seed")
	AssetTypeMsSQLQuerySensor       = AssetType("ms.sensor.query")
	AssetTypeDatabricksQuery        = AssetType("databricks.sql")
	AssetTypeDatabricksSeed         = AssetType("databricks.seed")
	AssetTypeDatabricksTableSensor  = AssetType("databricks.sensor.table")
	AssetTypeDatabricksQuerySensor  = AssetType("databricks.sensor.query")
	AssetTypeSynapseQuery           = AssetType("synapse.sql")
	AssetTypeSynapseSeed            = AssetType("synapse.seed")
	AssetTypeSynapseQuerySensor     = AssetType("synapse.sensor.query")
	AssetTypeIngestr                = AssetType("ingestr")
	AssetTypeTableau                = AssetType("tableau")
	AssetTypeTableauDatasource      = AssetType("tableau.datasource")
	AssetTypeTableauWorkbook        = AssetType("tableau.workbook")
	AssetTypeTableauWorksheet       = AssetType("tableau.worksheet")
	AssetTypeTableauDashboard       = AssetType("tableau.dashboard")
	AssetTypeClickHouse             = AssetType("clickhouse.sql")
	AssetTypeClickHouseSeed         = AssetType("clickhouse.seed")
	AssetTypeClickHouseQuerySensor  = AssetType("clickhouse.sensor.query")
	AssetTypeClickHouseTableSensor  = AssetType("clickhouse.sensor.table")
	AssetTypeClickHouseSource       = AssetType("clickhouse.source")
	AssetTypeEMRServerlessSpark     = AssetType("emr_serverless.spark")
	AssetTypeEMRServerlessPyspark   = AssetType("emr_serverless.pyspark")
	AssetTypeTrinoQuery             = AssetType("trino.sql")
	AssetTypeTrinoQuerySensor       = AssetType("trino.sensor.query")
	AssetTypeOracleQuery            = AssetType("oracle.sql")
	AssetTypeOracleSource           = AssetType("oracle.source")
	AssetTypeLooker                 = AssetType("looker")
	AssetTypeLookerStudio           = AssetType("looker_studio")
	AssetTypePowerBI                = AssetType("powerbi")
	AssetTypeQlikSense              = AssetType("qliksense")
	AssetTypeQlikView               = AssetType("qlikview")
	AssetTypeSisense                = AssetType("sisense")
	AssetTypeDomo                   = AssetType("domo")
	AssetTypeQuicksight             = AssetType("quicksight")
	AssetTypeMetabase               = AssetType("metabase")
	AssetTypeGrafana                = AssetType("grafana")
	AssetTypeSuperset               = AssetType("superset")
	AssetTypeModeBI                 = AssetType("modebi")
	AssetTypeRedash                 = AssetType("redash")
	AssetTypeGoodData               = AssetType("gooddata")
	AssetTypeS3KeySensor            = AssetType("s3.sensor.key_sensor")
	RunConfigFullRefresh            = RunConfig("full-refresh")
	RunConfigApplyIntervalModifiers = RunConfig("apply-interval-modifiers")
	RunConfigStartDate              = RunConfig("start-date")
	RunConfigEndDate                = RunConfig("end-date")
	RunConfigRunID                  = RunConfig("run-id")
	RunConfigQueryAnnotations       = RunConfig("query-annotations")
)
View Source 
const (
	MaterializationStrategyNone             MaterializationStrategy        = ""
	MaterializationStrategyCreateReplace    MaterializationStrategy        = "create+replace"
	MaterializationStrategyDeleteInsert     MaterializationStrategy        = "delete+insert"
	MaterializationStrategyTruncateInsert   MaterializationStrategy        = "truncate+insert"
	MaterializationStrategyAppend           MaterializationStrategy        = "append"
	MaterializationStrategyMerge            MaterializationStrategy        = "merge"
	MaterializationStrategyTimeInterval     MaterializationStrategy        = "time_interval"
	MaterializationStrategyDDL              MaterializationStrategy        = "ddl"
	MaterializationTimeGranularityDate      MaterializationTimeGranularity = "date"
	MaterializationTimeGranularityTimestamp MaterializationTimeGranularity = "timestamp"
	MaterializationStrategySCD2ByTime       MaterializationStrategy        = "scd2_by_time"
	MaterializationStrategySCD2ByColumn     MaterializationStrategy        = "scd2_by_column"
)

Variables

View Source 
var AllAvailableMaterializationStrategies = []MaterializationStrategy{
	MaterializationStrategyCreateReplace,
	MaterializationStrategyDeleteInsert,
	MaterializationStrategyTruncateInsert,
	MaterializationStrategyAppend,
	MaterializationStrategyMerge,
	MaterializationStrategyTimeInterval,
	MaterializationStrategyDDL,
	MaterializationStrategySCD2ByTime,
	MaterializationStrategySCD2ByColumn,
}
View Source 
var AssetTypeConnectionMapping = map[AssetType]string{
	AssetTypeBigqueryQuery:       "google_cloud_platform",
	AssetTypeBigqueryTableSensor: "google_cloud_platform",
	AssetTypeBigquerySeed:        "google_cloud_platform",
	AssetTypeBigquerySource:      "google_cloud_platform",
	AssetTypeBigqueryQuerySensor: "google_cloud_platform",

	AssetTypeSnowflakeQuery:        "snowflake",
	AssetTypeSnowflakeQuerySensor:  "snowflake",
	AssetTypeSnowflakeTableSensor:  "snowflake",
	AssetTypeSnowflakeSeed:         "snowflake",
	AssetTypeSnowflakeSource:       "snowflake",
	AssetTypePostgresQuery:         "postgres",
	AssetTypePostgresSeed:          "postgres",
	AssetTypePostgresQuerySensor:   "postgres",
	AssetTypePostgresTableSensor:   "postgres",
	AssetTypePostgresSource:        "postgres",
	AssetTypeRedshiftQuery:         "redshift",
	AssetTypeRedshiftSeed:          "redshift",
	AssetTypeRedshiftQuerySensor:   "redshift",
	AssetTypeRedshiftSource:        "redshift",
	AssetTypeRedshiftTableSensor:   "redshift",
	AssetTypeMsSQLQuery:            "mssql",
	AssetTypeMsSQLSeed:             "mssql",
	AssetTypeMsSQLQuerySensor:      "mssql",
	AssetTypeMsSQLTableSensor:      "mssql",
	AssetTypeMsSQLSource:           "mssql",
	AssetTypeDatabricksQuery:       "databricks",
	AssetTypeDatabricksSeed:        "databricks",
	AssetTypeDatabricksQuerySensor: "databricks",
	AssetTypeDatabricksSource:      "databricks",
	AssetTypeDatabricksTableSensor: "databricks",
	AssetTypeSynapseQuery:          "synapse",
	AssetTypeSynapseSeed:           "synapse",
	AssetTypeSynapseQuerySensor:    "synapse",
	AssetTypeSynapseSource:         "synapse",
	AssetTypeAthenaQuery:           "athena",
	AssetTypeAthenaSeed:            "athena",
	AssetTypeAthenaSQLSensor:       "athena",
	AssetTypeAthenaTableSensor:     "athena",
	AssetTypeAthenaSource:          "athena",
	AssetTypeDuckDBQuery:           "duckdb",
	AssetTypeDuckDBSeed:            "duckdb",
	AssetTypeDuckDBQuerySensor:     "duckdb",
	AssetTypeDuckDBSource:          "duckdb",
	AssetTypeClickHouse:            "clickhouse",
	AssetTypeClickHouseSeed:        "clickhouse",
	AssetTypeClickHouseQuerySensor: "clickhouse",
	AssetTypeClickHouseTableSensor: "clickhouse",
	AssetTypeClickHouseSource:      "clickhouse",
	AssetTypeEMRServerlessSpark:    "emr_serverless",
	AssetTypeEMRServerlessPyspark:  "emr_serverless",
	AssetTypeTrinoQuery:            "trino",
	AssetTypeTrinoQuerySensor:      "trino",
	AssetTypeOracleQuery:           "oracle",
	AssetTypeOracleSource:          "oracle",
	AssetTypeS3KeySensor:           "aws",
}
View Source 
var IngestrTypeConnectionMapping = map[string]AssetType{
	"athena":     AssetTypeAthenaQuery,
	"bigquery":   AssetTypeBigqueryQuery,
	"snowflake":  AssetTypeSnowflakeQuery,
	"postgres":   AssetTypePostgresQuery,
	"redshift":   AssetTypeRedshiftQuery,
	"mssql":      AssetTypeMsSQLQuery,
	"databricks": AssetTypeDatabricksQuery,
	"synapse":    AssetTypeSynapseQuery,
	"duckdb":     AssetTypeDuckDBQuery,
	"clickhouse": AssetTypeClickHouse,
	"oracle":     AssetTypeOracleQuery,
}
View Source 
var SupportedFileSuffixes = []string{"asset.yml", "asset.yaml", ".sql", ".py", "task.yml", "task.yaml"}
View Source 
var ValidQualityChecks = map[string]bool{
	"not_null":        true,
	"unique":          true,
	"positive":        true,
	"min":             true,
	"max":             true,
	"accepted_values": true,
	"negative":        true,
	"non_negative":    true,
	"pattern":         true,
}

Functions

func ClearSpacesAtLineEndings added in v0.11.27 

func ClearSpacesAtLineEndings(content string) string

func ModifyDate added in v0.11.179 

func ModifyDate(t time.Time, modifier TimeModifier) time.Time

Types

type Asset 

type Asset struct {
	ID                string             `json:"id" yaml:"-" mapstructure:"-"`
	URI               string             `json:"uri" yaml:"uri,omitempty" mapstructure:"uri"`
	Name              string             `json:"name" yaml:"name,omitempty" mapstructure:"name"`
	Type              AssetType          `json:"type" yaml:"type,omitempty" mapstructure:"type"`
	Description       string             `json:"description" yaml:"description,omitempty" mapstructure:"description"`
	StartDate         string             `json:"start_date" yaml:"start_date,omitempty" mapstructure:"start_date"`
	Connection        string             `json:"connection" yaml:"connection,omitempty" mapstructure:"connection"`
	Tags              EmptyStringArray   `json:"tags" yaml:"tags,omitempty" mapstructure:"tags"`
	Domains           EmptyStringArray   `json:"domains" yaml:"domains,omitempty" mapstructure:"domains"`
	Meta              EmptyStringMap     `json:"meta" yaml:"meta,omitempty" mapstructure:"meta"`
	Materialization   Materialization    `json:"materialization" yaml:"materialization,omitempty" mapstructure:"materialization"`
	Upstreams         []Upstream         `json:"upstreams" yaml:"depends,omitempty" mapstructure:"depends"`
	Image             string             `json:"image" yaml:"image,omitempty" mapstructure:"image"`
	Instance          string             `json:"instance" yaml:"instance,omitempty" mapstructure:"instance"`
	Owner             string             `json:"owner" yaml:"owner,omitempty" mapstructure:"owner"`
	Tier              int                `json:"tier,omitempty" yaml:"tier,omitempty" mapstructure:"tier"`
	ExecutableFile    ExecutableFile     `json:"executable_file" yaml:"-" mapstructure:"-"`
	DefinitionFile    TaskDefinitionFile `json:"definition_file" yaml:"-" mapstructure:"-"`
	Parameters        EmptyStringMap     `json:"parameters" yaml:"parameters,omitempty" mapstructure:"parameters"`
	Secrets           []SecretMapping    `json:"secrets" yaml:"secrets,omitempty" mapstructure:"secrets"`
	Extends           []string           `json:"extends" yaml:"extends,omitempty" mapstructure:"extends"`
	Columns           []Column           `json:"columns" yaml:"columns,omitempty" mapstructure:"columns"`
	CustomChecks      []CustomCheck      `json:"custom_checks" yaml:"custom_checks,omitempty" mapstructure:"custom_checks"`
	Metadata          EmptyStringMap     `json:"metadata" yaml:"metadata,omitempty" mapstructure:"metadata"`
	Snowflake         SnowflakeConfig    `json:"snowflake" yaml:"snowflake,omitempty" mapstructure:"snowflake"`
	Athena            AthenaConfig       `json:"athena" yaml:"athena,omitempty" mapstructure:"athena"`
	IntervalModifiers IntervalModifiers  `json:"interval_modifiers" yaml:"interval_modifiers,omitempty" mapstructure:"interval_modifiers"`
	// contains filtered or unexported fields
}

func ConvertYamlToTask 

func ConvertYamlToTask(content []byte) (*Asset, error)

func (*Asset) AddDownstream 

func (a *Asset) AddDownstream(asset *Asset)

func (*Asset) AddUpstream 

func (a *Asset) AddUpstream(asset *Asset)

func (*Asset) CheckCount added in v0.11.87 

func (a *Asset) CheckCount() int

func (*Asset) ColumnNames added in v0.5.0 

func (a *Asset) ColumnNames() []string

func (*Asset) ColumnNamesWithPrimaryKey added in v0.5.0 

func (a *Asset) ColumnNamesWithPrimaryKey() []string

func (*Asset) ColumnNamesWithUpdateOnMerge added in v0.5.0 

func (a *Asset) ColumnNamesWithUpdateOnMerge() []string

func (*Asset) EnrichFromEntityAttributes added in v0.11.9 

func (a *Asset) EnrichFromEntityAttributes(entities []*glossary.Entity) error

func (*Asset) FormatContent added in v0.11.126 

func (a *Asset) FormatContent() ([]byte, error)

func (*Asset) GetColumnWithName added in v0.11.11 

func (a *Asset) GetColumnWithName(name string) *Column

func (*Asset) GetDownstream 

func (a *Asset) GetDownstream() []*Asset

func (*Asset) GetFullDownstream 

func (a *Asset) GetFullDownstream() []*Asset

func (*Asset) GetFullUpstream 

func (a *Asset) GetFullUpstream() []*Asset

func (*Asset) GetNameIfItWasSetFromItsPath added in v0.11.216 

func (a *Asset) GetNameIfItWasSetFromItsPath(foundPipeline *Pipeline) (string, error)

func (*Asset) GetUpstream 

func (a *Asset) GetUpstream() []*Asset

func (*Asset) IsSQLAsset added in v0.11.143 

func (a *Asset) IsSQLAsset() bool

func (Asset) Persist added in v0.11.27 

func (a Asset) Persist(fs afero.Fs) error

func (*Asset) PrefixSchema added in v0.11.172 

func (a *Asset) PrefixSchema(prefix string)

func (*Asset) PrefixUpstreams added in v0.11.172 

func (a *Asset) PrefixUpstreams(prefix string)

type AssetMaterializationMap added in v0.5.0 

type AssetMaterializationMap map[MaterializationType]map[MaterializationStrategy]MaterializerFunc

type AssetMutator added in v0.11.212 

type AssetMutator func(ctx context.Context, asset *Asset, foundPipeline *Pipeline) (*Asset, error)

type AssetType 

type AssetType string

type AthenaConfig added in v0.11.20 

type AthenaConfig struct {
	Location string `json:"location"`
}

func (AthenaConfig) MarshalJSON added in v0.11.20 

func (s AthenaConfig) MarshalJSON() ([]byte, error)

type Builder added in v0.11.9 

type Builder struct {
	GlossaryReader glossaryReader
	// contains filtered or unexported fields
}

func NewBuilder 

func NewBuilder(config BuilderConfig, yamlTaskCreator TaskCreator, commentTaskCreator TaskCreator, fs afero.Fs, gr glossaryReader) *Builder

func (*Builder) AddAssetMutator added in v0.11.212 

func (b *Builder) AddAssetMutator(m AssetMutator)

func (*Builder) AddPipelineMutator added in v0.11.212 

func (b *Builder) AddPipelineMutator(m PipelineMutator)

func (*Builder) CreateAssetFromFile added in v0.11.9 

func (b *Builder) CreateAssetFromFile(filePath string, foundPipeline *Pipeline) (*Asset, error)

func (*Builder) CreatePipelineFromPath added in v0.11.9 

func (b *Builder) CreatePipelineFromPath(ctx context.Context, pathToPipeline string, opts ...CreatePipelineOption) (*Pipeline, error)

func (*Builder) MutateAsset added in v0.11.142 

func (b *Builder) MutateAsset(ctx context.Context, task *Asset, foundPipeline *Pipeline) (*Asset, error)

func (*Builder) MutatePipeline added in v0.11.212 

func (b *Builder) MutatePipeline(ctx context.Context, pipeline *Pipeline) (*Pipeline, error)

func (*Builder) SetAssetColumnFromGlossary added in v0.11.173 

func (b *Builder) SetAssetColumnFromGlossary(asset *Asset, pathToPipeline string) error

func (*Builder) SetGlossaryReader added in v0.11.9 

func (b *Builder) SetGlossaryReader(reader glossaryReader)

func (*Builder) SetNameFromPath added in v0.11.186 

func (b *Builder) SetNameFromPath(ctx context.Context, asset *Asset, foundPipeline *Pipeline) (*Asset, error)

func (*Builder) SetupDefaultsFromPipeline added in v0.11.193 

func (b *Builder) SetupDefaultsFromPipeline(ctx context.Context, asset *Asset, foundPipeline *Pipeline) (*Asset, error)

type BuilderConfig 

type BuilderConfig struct {
	PipelineFileName    []string
	TasksDirectoryName  string
	TasksDirectoryNames []string
	TasksFileSuffixes   []string
}

type Column 

type Column struct {
	EntityAttribute *EntityAttribute  `json:"entity_attribute" yaml:"-" mapstructure:"-"`
	Name            string            `json:"name" yaml:"name,omitempty" mapstructure:"name"`
	Type            string            `json:"type" yaml:"type,omitempty" mapstructure:"type"`
	Description     string            `json:"description" yaml:"description,omitempty" mapstructure:"description"`
	Tags            EmptyStringArray  `json:"tags" yaml:"tags,omitempty" mapstructure:"tags"`
	PrimaryKey      bool              `json:"primary_key" yaml:"primary_key,omitempty" mapstructure:"primary_key"`
	UpdateOnMerge   bool              `json:"update_on_merge" yaml:"update_on_merge,omitempty" mapstructure:"update_on_merge"`
	Nullable        DefaultTrueBool   `json:"nullable" yaml:"nullable,omitempty" mapstructure:"nullable"`
	Owner           string            `json:"owner" yaml:"owner,omitempty" mapstructure:"owner"`
	Domains         EmptyStringArray  `json:"domains" yaml:"domains,omitempty" mapstructure:"domains"`
	Meta            EmptyStringMap    `json:"meta" yaml:"meta,omitempty" mapstructure:"meta"`
	Extends         string            `json:"-" yaml:"extends,omitempty" mapstructure:"extends"`
	Checks          []ColumnCheck     `json:"checks" yaml:"checks,omitempty" mapstructure:"checks"`
	Upstreams       []*UpstreamColumn `json:"upstreams" yaml:"-" mapstructure:"-"`
}

func (*Column) HasCheck added in v0.9.0 

func (c *Column) HasCheck(check string) bool

type ColumnCheck 

type ColumnCheck struct {
	ID          string           `json:"id" yaml:"-" mapstructure:"-"`
	Name        string           `json:"name" yaml:"name,omitempty" mapstructure:"name"`
	Value       ColumnCheckValue `json:"value" yaml:"value,omitempty" mapstructure:"value"`
	Blocking    DefaultTrueBool  `json:"blocking" yaml:"blocking,omitempty" mapstructure:"blocking"`
	Description string           `json:"description" yaml:"description,omitempty" mapstructure:"description"`
}

func NewColumnCheck added in v0.3.0 

func NewColumnCheck(assetName, columnName, name string, value ColumnCheckValue, blocking *bool, description string) ColumnCheck

type ColumnCheckValue 

type ColumnCheckValue struct {
	IntArray    *[]int    `json:"int_array"`
	Int         *int      `json:"int"`
	Float       *float64  `json:"float"`
	StringArray *[]string `json:"string_array"`
	String      *string   `json:"string"`
	Bool        *bool     `json:"bool"`
}

func (*ColumnCheckValue) MarshalJSON 

func (ccv *ColumnCheckValue) MarshalJSON() ([]byte, error)

func (ColumnCheckValue) MarshalYAML added in v0.11.27 

func (ccv ColumnCheckValue) MarshalYAML() (interface{}, error)

func (*ColumnCheckValue) ToString added in v0.10.1 

func (ccv *ColumnCheckValue) ToString() string

func (*ColumnCheckValue) UnmarshalJSON added in v0.5.0 

func (ccv *ColumnCheckValue) UnmarshalJSON(data []byte) error

type CreatePipelineOption added in v0.11.156 

type CreatePipelineOption func(*createPipelineConfig)

func WithGitMetadata added in v0.11.156 

func WithGitMetadata() CreatePipelineOption

func WithMutate added in v0.11.156 

func WithMutate() CreatePipelineOption

func WithOnlyPipeline added in v0.11.226 

func WithOnlyPipeline() CreatePipelineOption

type CustomCheck 

type CustomCheck struct {
	ID          string          `json:"id" yaml:"-" mapstructure:"-"`
	Name        string          `json:"name" yaml:"name" mapstructure:"name"`
	Description string          `json:"description" yaml:"description,omitempty" mapstructure:"description"`
	Value       int64           `json:"value" yaml:"value" mapstructure:"value"`
	Count       *int64          `json:"count,omitempty" yaml:"count,omitempty" mapstructure:"count"`
	Blocking    DefaultTrueBool `json:"blocking" yaml:"blocking,omitempty" mapstructure:"blocking"`
	Query       string          `json:"query" yaml:"query" mapstructure:"query"`
}

type DefaultTrueBool added in v0.11.21 

type DefaultTrueBool struct {
	Value *bool
}

func (*DefaultTrueBool) Bool added in v0.11.27 

func (b *DefaultTrueBool) Bool() bool

func (DefaultTrueBool) IsZero added in v0.11.110 

func (b DefaultTrueBool) IsZero() bool

func (DefaultTrueBool) MarshalJSON added in v0.11.21 

func (b DefaultTrueBool) MarshalJSON() ([]byte, error)

func (DefaultTrueBool) MarshalYAML added in v0.11.27 

func (b DefaultTrueBool) MarshalYAML() (interface{}, error)

func (*DefaultTrueBool) UnmarshalJSON added in v0.11.21 

func (b *DefaultTrueBool) UnmarshalJSON(data []byte) error

func (*DefaultTrueBool) UnmarshalYAML added in v0.11.21 

func (b *DefaultTrueBool) UnmarshalYAML(value *yaml.Node) error

type DefaultValues added in v0.11.136 

type DefaultValues struct {
	Type              string            `json:"type" yaml:"type" mapstructure:"type"`
	Parameters        map[string]string `json:"parameters" yaml:"parameters" mapstructure:"parameters"`
	Secrets           []secretMapping   `json:"secrets" yaml:"secrets" mapstructure:"secrets"`
	IntervalModifiers IntervalModifiers `json:"interval_modifiers" yaml:"interval_modifiers" mapstructure:"interval_modifiers"`
}

type DefinitionFile 

type DefinitionFile struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

type DependsColumn added in v0.11.108 

type DependsColumn struct {
	Name  string `json:"name" yaml:"name" mapstructure:"name"`
	Usage string `json:"usage" yaml:"usage" mapstructure:"usage"`
}

type DiscordNotification added in v0.11.29 

type DiscordNotification struct {
	Connection         string `yaml:"connection" json:"connection" mapstructure:"connection"`
	NotificationCommon `yaml:",inline" json:",inline" mapstructure:",inline"`
}

type EmptyStringArray added in v0.11.0 

type EmptyStringArray []string

func (EmptyStringArray) MarshalJSON added in v0.11.0 

func (a EmptyStringArray) MarshalJSON() ([]byte, error)

func (*EmptyStringArray) UnmarshalJSON added in v0.11.0 

func (a *EmptyStringArray) UnmarshalJSON(data []byte) error

type EmptyStringMap 

type EmptyStringMap map[string]string

func (EmptyStringMap) MarshalJSON 

func (m EmptyStringMap) MarshalJSON() ([]byte, error)

func (*EmptyStringMap) UnmarshalJSON added in v0.5.0 

func (b *EmptyStringMap) UnmarshalJSON(data []byte) error

type EntityAttribute added in v0.11.9 

type EntityAttribute struct {
	Entity    string `json:"entity"`
	Attribute string `json:"attribute"`
}

type ExecutableFile 

type ExecutableFile struct {
	Name    string `json:"name"`
	Path    string `json:"path"`
	Content string `json:"content"`
}

type IntervalModifiers added in v0.11.179 

type IntervalModifiers struct {
	Start TimeModifier `json:"start" yaml:"start,omitempty" mapstructure:"start"`
	End   TimeModifier `json:"end" yaml:"end,omitempty" mapstructure:"end"`
}

func (IntervalModifiers) MarshalJSON added in v0.11.179 

func (im IntervalModifiers) MarshalJSON() ([]byte, error)

type MSTeamsNotification added in v0.11.20 

type MSTeamsNotification struct {
	Connection         string `yaml:"connection" json:"connection" mapstructure:"connection"`
	NotificationCommon `yaml:",inline" json:",inline" mapstructure:",inline"`
}

type Materialization 

type Materialization struct {
	Type            MaterializationType            `json:"type" yaml:"type,omitempty" mapstructure:"type"`
	Strategy        MaterializationStrategy        `json:"strategy" yaml:"strategy,omitempty" mapstructure:"strategy"`
	PartitionBy     string                         `json:"partition_by" yaml:"partition_by,omitempty" mapstructure:"partition_by"`
	ClusterBy       []string                       `json:"cluster_by" yaml:"cluster_by,omitempty" mapstructure:"cluster_by"`
	IncrementalKey  string                         `json:"incremental_key" yaml:"incremental_key,omitempty" mapstructure:"incremental_key"`
	TimeGranularity MaterializationTimeGranularity `json:"time_granularity" yaml:"time_granularity,omitempty" mapstructure:"time_granularity"`
}

func (Materialization) MarshalJSON 

func (m Materialization) MarshalJSON() ([]byte, error)

type MaterializationStrategy 

type MaterializationStrategy string

type MaterializationTimeGranularity added in v0.11.161 

type MaterializationTimeGranularity string

type MaterializationType 

type MaterializationType string
const (
	MaterializationTypeNone  MaterializationType = ""
	MaterializationTypeView  MaterializationType = "view"
	MaterializationTypeTable MaterializationType = "table"
)

type Materializer added in v0.5.0 

type Materializer struct {
	MaterializationMap AssetMaterializationMap
	FullRefresh        bool
}

func (*Materializer) IsFullRefresh added in v0.11.129 

func (m *Materializer) IsFullRefresh() bool

func (*Materializer) LogIfFullRefreshAndDDL added in v0.11.209 

func (m *Materializer) LogIfFullRefreshAndDDL(writer interface{}, asset *Asset) error

func (*Materializer) Render added in v0.5.0 

func (m *Materializer) Render(asset *Asset, query string) (string, error)

type MaterializerFunc added in v0.5.0 

type MaterializerFunc func(task *Asset, query string) (string, error)

type MetadataPush added in v0.11.29 

type MetadataPush struct {
	Global   bool `json:"-"`
	BigQuery bool `json:"bigquery" yaml:"bigquery" mapstructure:"bigquery"`
}

func (*MetadataPush) HasAnyEnabled added in v0.11.29 

func (mp *MetadataPush) HasAnyEnabled() bool

type NotificationCommon added in v0.11.21 

type NotificationCommon struct {
	Success DefaultTrueBool `yaml:"success" json:"success" mapstructure:"success"`
	Failure DefaultTrueBool `yaml:"failure" json:"failure" mapstructure:"failure"`
}

type Notifications 

type Notifications struct {
	Slack   []SlackNotification   `yaml:"slack" json:"slack" mapstructure:"slack"`
	MSTeams []MSTeamsNotification `yaml:"ms_teams" json:"ms_teams" mapstructure:"ms_teams"`
	Discord []DiscordNotification `yaml:"discord" json:"discord" mapstructure:"discord"`
	Webhook []WebhookNotification `yaml:"webhook" json:"webhook" mapstructure:"webhook"`
}

func (Notifications) MarshalJSON 

func (n Notifications) MarshalJSON() ([]byte, error)

type ParseError added in v0.10.1 

type ParseError struct {
	Msg string
}

func (ParseError) Error added in v0.10.1 

func (e ParseError) Error() string

type Pipeline 

type Pipeline struct {
	LegacyID           string                 `json:"legacy_id" yaml:"id" mapstructure:"id"`
	Name               string                 `json:"name" yaml:"name" mapstructure:"name"`
	Tags               EmptyStringArray       `json:"tags" yaml:"tags,omitempty" mapstructure:"tags"`
	Domains            EmptyStringArray       `json:"domains" yaml:"domains,omitempty" mapstructure:"domains"`
	Meta               EmptyStringMap         `json:"meta" yaml:"meta,omitempty" mapstructure:"meta"`
	Schedule           Schedule               `json:"schedule" yaml:"schedule" mapstructure:"schedule"`
	StartDate          string                 `json:"start_date" yaml:"start_date" mapstructure:"start_date"`
	DefinitionFile     DefinitionFile         `json:"definition_file"`
	DefaultConnections EmptyStringMap         `json:"default_connections" yaml:"default_connections" mapstructure:"default_connections"`
	Assets             []*Asset               `json:"assets"`
	Notifications      Notifications          `json:"notifications" yaml:"notifications" mapstructure:"notifications"`
	Catchup            bool                   `json:"catchup" yaml:"catchup" mapstructure:"catchup"`
	MetadataPush       MetadataPush           `json:"metadata_push" yaml:"metadata_push" mapstructure:"metadata_push"`
	Retries            int                    `json:"retries" yaml:"retries" mapstructure:"retries"`
	Concurrency        int                    `json:"concurrency" yaml:"concurrency" mapstructure:"concurrency"`
	DefaultValues      *DefaultValues         `json:"default,omitempty" yaml:"default,omitempty" mapstructure:"default,omitempty"`
	Commit             string                 `json:"commit"`
	Snapshot           string                 `json:"snapshot"`
	Agent              bool                   `json:"agent" yaml:"agent" mapstructure:"agent"`
	Variables          Variables              `json:"variables" yaml:"variables" mapstructure:"variables"`
	TasksByType        map[AssetType][]*Asset `json:"-"`
	// contains filtered or unexported fields
}

func PipelineFromPath added in v0.11.6 

func PipelineFromPath(filePath string, fs afero.Fs) (*Pipeline, error)

func (*Pipeline) GetAllConnectionNamesForAsset added in v0.11.102 

func (p *Pipeline) GetAllConnectionNamesForAsset(asset *Asset) ([]string, error)

func (*Pipeline) GetAssetByName added in v0.5.6 

func (p *Pipeline) GetAssetByName(assetName string) *Asset

func (*Pipeline) GetAssetByNameCaseInsensitive added in v0.11.138 

func (p *Pipeline) GetAssetByNameCaseInsensitive(assetName string) *Asset

func (*Pipeline) GetAssetByPath 

func (p *Pipeline) GetAssetByPath(assetPath string) *Asset

func (*Pipeline) GetAssetsByTag added in v0.11.0 

func (p *Pipeline) GetAssetsByTag(tag string) []*Asset

func (*Pipeline) GetCompatibilityHash added in v0.11.122 

func (p *Pipeline) GetCompatibilityHash() string

func (*Pipeline) GetConnectionNameForAsset 

func (p *Pipeline) GetConnectionNameForAsset(asset *Asset) (string, error)

func (*Pipeline) GetMajorityAssetTypesFromSQLAssets added in v0.9.1 

func (p *Pipeline) GetMajorityAssetTypesFromSQLAssets(defaultIfNone AssetType) AssetType

func (*Pipeline) HasAssetType 

func (p *Pipeline) HasAssetType(taskType AssetType) bool

func (*Pipeline) RelativeAssetPath 

func (p *Pipeline) RelativeAssetPath(t *Asset) string

func (*Pipeline) UnmarshalJSON added in v0.11.243 

func (p *Pipeline) UnmarshalJSON(data []byte) error

func (*Pipeline) UnmarshalYAML added in v0.11.243 

func (p *Pipeline) UnmarshalYAML(unmarshal func(interface{}) error) error

func (*Pipeline) WipeContentOfAssets added in v0.11.16 

func (p *Pipeline) WipeContentOfAssets()

WipeContentOfAssets removes the content of the executable files of all assets in the pipeline. This is useful when we want to serialize the pipeline to JSON and we don't want to include the content of the assets.

type PipelineMutator added in v0.11.212 

type PipelineMutator func(ctx context.Context, pipeline *Pipeline) (*Pipeline, error)

type RendererInterface added in v0.11.274 

type RendererInterface interface {
	Render(template string) (string, error)
}

RendererInterface is used to avoid circular dependencies.

type RunConfig added in v0.10.3 

type RunConfig string

type Schedule added in v0.11.123 

type Schedule string

type SecretMapping 

type SecretMapping struct {
	SecretKey   string `json:"secret_key"`
	InjectedKey string `json:"injected_key"`
}

func (SecretMapping) MarshalYAML added in v0.11.33 

func (s SecretMapping) MarshalYAML() (interface{}, error)

type SlackNotification 

type SlackNotification struct {
	Channel            string `json:"channel"`
	NotificationCommon `yaml:",inline" json:",inline" mapstructure:",inline"`
}

type SnowflakeConfig added in v0.11.14 

type SnowflakeConfig struct {
	Warehouse string `json:"warehouse"  yaml:"warehouse" `
}

func (SnowflakeConfig) MarshalJSON added in v0.11.14 

func (s SnowflakeConfig) MarshalJSON() ([]byte, error)

type TaskCreator 

type TaskCreator func(path string) (*Asset, error)

func CreateTaskFromFileComments 

func CreateTaskFromFileComments(fs afero.Fs) TaskCreator

func CreateTaskFromYamlDefinition 

func CreateTaskFromYamlDefinition(fs afero.Fs) TaskCreator

type TaskDefinitionFile 

type TaskDefinitionFile struct {
	Name string             `json:"name"`
	Path string             `json:"path"`
	Type TaskDefinitionType `json:"type"`
}

type TaskDefinitionType 

type TaskDefinitionType string

type TaskSchedule 

type TaskSchedule struct {
	Days []string `json:"days"`
}

type TimeModifier added in v0.11.179 

type TimeModifier struct {
	Months      int    `json:"months" yaml:"months,omitempty" mapstructure:"months"`
	Days        int    `json:"days" yaml:"days,omitempty" mapstructure:"days"`
	Hours       int    `json:"hours" yaml:"hours,omitempty" mapstructure:"hours"`
	Minutes     int    `json:"minutes" yaml:"minutes,omitempty" mapstructure:"minutes"`
	Seconds     int    `json:"seconds" yaml:"seconds,omitempty" mapstructure:"seconds"`
	CronPeriods int    `json:"cron_periods" yaml:"cron_periods,omitempty" mapstructure:"cron_periods"`
	Template    string `json:"template" yaml:"template,omitempty" mapstructure:"template"`
}

func (TimeModifier) MarshalJSON added in v0.11.179 

func (t TimeModifier) MarshalJSON() ([]byte, error)

func (TimeModifier) MarshalYAML added in v0.11.221 

func (t TimeModifier) MarshalYAML() (interface{}, error)

func (TimeModifier) ResolveTemplateToNew added in v0.11.274 

func (t TimeModifier) ResolveTemplateToNew(renderer RendererInterface) (TimeModifier, error)

func (*TimeModifier) UnmarshalYAML added in v0.11.179 

func (t *TimeModifier) UnmarshalYAML(value *yaml.Node) error

type Upstream added in v0.11.9 

type Upstream struct {
	Type     string          `json:"type" yaml:"type" mapstructure:"type"`
	Value    string          `json:"value" yaml:"value" mapstructure:"value"`
	Metadata EmptyStringMap  `json:"metadata,omitempty" yaml:"metadata,omitempty" mapstructure:"metadata"`
	Columns  []DependsColumn `json:"columns" yaml:"columns,omitempty" mapstructure:"columns"`
	Mode     UpstreamMode    `json:"mode" yaml:"mode,omitempty" mapstructure:"mode"`
}

func (Upstream) MarshalYAML added in v0.11.27 

func (u Upstream) MarshalYAML() (interface{}, error)

type UpstreamColumn added in v0.11.105 

type UpstreamColumn struct {
	Column string `json:"column" yaml:"column,omitempty" mapstructure:"column"`
	Table  string `json:"table" yaml:"table,omitempty" mapstructure:"table"`
}

type UpstreamMode added in v0.11.167 

type UpstreamMode int
const (
	UpstreamModeFull UpstreamMode = iota
	UpstreamModeSymbolic
)

func MarshalUpstreamMode added in v0.11.171 

func MarshalUpstreamMode(s string) UpstreamMode

func (*UpstreamMode) IsValid added in v0.11.167 

func (m *UpstreamMode) IsValid() bool

func (*UpstreamMode) MarshalJSON added in v0.11.171 

func (m *UpstreamMode) MarshalJSON() ([]byte, error)

func (*UpstreamMode) String added in v0.11.167 

func (m *UpstreamMode) String() string

func (*UpstreamMode) UnmarshalJSON added in v0.11.171 

func (m *UpstreamMode) UnmarshalJSON(data []byte) error

type Variables added in v0.11.209 

type Variables map[string]map[string]any

func (Variables) Merge added in v0.11.212 

func (v Variables) Merge(other map[string]any) error

func (Variables) Schema added in v0.11.209 

func (v Variables) Schema() any

func (Variables) Validate added in v0.11.209 

func (v Variables) Validate() error

func (Variables) Value added in v0.11.209 

func (v Variables) Value() map[string]any

type WebhookNotification added in v0.11.296 

type WebhookNotification struct {
	Connection         string `yaml:"connection" json:"connection" mapstructure:"connection"`
	NotificationCommon `yaml:",inline" json:",inline" mapstructure:",inline"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.