Documentation
¶
Index ¶
- Constants
- Variables
- func AI(isDebug *bool) *cli.Command
- func AddConnection() *cli.Command
- func ApplyAllFilters(ctx context.Context, f *Filter, s *scheduler.Scheduler, p *pipeline.Pipeline) error
- func AssetMetadata() *cli.Command
- func CheckLint(ctx context.Context, foundPipeline *pipeline.Pipeline, pipelinePath string, ...) error
- func Clean(str string) string
- func CleanCmd() *cli.Command
- func CloneEnvironment(isDebug *bool) *cli.Command
- func ConnectionSchemas() *cli.Command
- func Connections() *cli.Command
- func CreateEnvironment(isDebug *bool) *cli.Command
- func DBSummary() *cli.Command
- func DataDiffCmd() *cli.Command
- func DeleteConnection() *cli.Command
- func DeleteEnvironment(isDebug *bool) *cli.Command
- func DetermineStartDate(cliStartDate string, pipeline *pipeline.Pipeline, fullRefresh bool, ...) (time.Time, error)
- func Docs() *cli.Command
- func Environments(isDebug *bool) *cli.Command
- func FetchColumns() *cli.Command
- func FetchDatabases() *cli.Command
- func FetchTables() *cli.Command
- func FilterTaskTypes(ctx context.Context, f *Filter, s *scheduler.Scheduler, p *pipeline.Pipeline) error
- func Format(isDebug *bool) *cli.Command
- func GetPipelineAndAsset(ctx context.Context, inputPath string, fs afero.Fs, configFilePath string) (*ppInfo, error)
- func GetPipelinefromPath(ctx context.Context, inputPath string) (*pipeline.Pipeline, error)
- func HandleExcludeTags(ctx context.Context, f *Filter, s *scheduler.Scheduler, p *pipeline.Pipeline) error
- func HandleIncludeTags(ctx context.Context, f *Filter, s *scheduler.Scheduler, p *pipeline.Pipeline) error
- func HandleMultipleAssets(ctx context.Context, f *Filter, s *scheduler.Scheduler, p *pipeline.Pipeline) error
- func HandleSingleTask(ctx context.Context, f *Filter, s *scheduler.Scheduler, p *pipeline.Pipeline) error
- func Import() *cli.Command
- func ImportDatabase() *cli.Command
- func ImportScheduledQueries() *cli.Command
- func ImportTableauDashboards() *cli.Command
- func IngestrSources() *cli.Command
- func Init() *cli.Command
- func Internal() *cli.Command
- func Lineage() *cli.Command
- func Lint(isDebug *bool) *cli.Command
- func ListConnections() *cli.Command
- func ListEnvironments(isDebug *bool) *cli.Command
- func ListTemplates() *cli.Command
- func LockAssetDependencies() *cli.Command
- func NewRunID() string
- func ParseAsset() *cli.Command
- func ParseDate(startDateStr, endDateStr string, logger logger.Logger) (time.Time, time.Time, error)
- func ParseGlossary() *cli.Command
- func ParsePipeline() *cli.Command
- func Patch() *cli.Command
- func PatchAsset() *cli.Command
- func PatchPipeline() *cli.Command
- func PingConnection() *cli.Command
- func Query() *cli.Command
- func ReadState(fs afero.Fs, statePath string, filter *Filter) (*scheduler.PipelineState, error)
- func RecoverFromPanic()
- func Render() *cli.Command
- func RenderDDL() *cli.Command
- func Run(isDebug *bool) *cli.Command
- func SetupExecutors(s *scheduler.Scheduler, conn config.ConnectionAndDetailsGetter, ...) (map[pipeline.AssetType]executor.Config, error)
- func SkipAllTasksIfSingleCheck(ctx context.Context, f *Filter, s *scheduler.Scheduler, p *pipeline.Pipeline) error
- func UpdateEnvironment(isDebug *bool) *cli.Command
- func Upgrade() *cli.Command
- func Validate(shouldvalidate bool, assetCounter assetCounter, lintChecker lintChecker, ...) error
- func ValidateDateRange(startDate, endDate time.Time) error
- func ValidateRunConfig(runConfig *scheduler.RunConfig, inputPath string, logger logger.Logger) (time.Time, time.Time, string, error)
- func VersionCmd(commit string) *cli.Command
- type CleanCommand
- type ColumnSummary
- type ConnectionsCommand
- type EnvironmentCloneCommand
- type EnvironmentDeleteCommand
- type EnvironmentListCommand
- type EnvironmentUpdateCommand
- type ErrorResponse
- type ErrorResponses
- type ExecutionSummary
- type Filter
- type FilterMutator
- type JSONColumnDiff
- type JSONColumnStats
- type JSONCountDiff
- type JSONDiffOutput
- type JSONPropDiff
- type JSONStatistic
- type JSONSummary
- type Limiter
- type LineageCommand
- type LockDependenciesCommand
- type LockDependenciesResult
- type ModifierInfo
- type ParseCommand
- type PipelineInfo
- type QueryLog
- type QueryLogOptions
- type RenderCommand
- type ScheduledQuery
- type Selector
- type SuccessResponse
- type SummaryStats
- type TUIRenderer
- type TableComparer
- type TableSummary
- type TableauDashboard
- type TaskTypeStats
- type VersionInfo
- type WarningResponse
Constants ¶
const ( DefaultTemplate = "default" DefaultFolderName = "bruin-pipeline" )
const LogsFolder = "logs"
Variables ¶
var ( DefaultGlossaryReader = &glossary.GlossaryReader{ RepoFinder: &git.RepoFinder{}, FileNames: []string{"glossary.yml", "glossary.yaml"}, } DefaultPipelineBuilder = pipeline.NewBuilder(builderConfig, pipeline.CreateTaskFromYamlDefinition(fs), pipeline.CreateTaskFromFileComments(fs), fs, DefaultGlossaryReader) SeedAssetsValidator = &lint.SimpleRule{ Identifier: "assets-seed-validation", Fast: true, Severity: lint.ValidatorSeverityCritical, AssetValidator: lint.ValidateAssetSeedValidation, ApplicableLevels: []lint.Level{lint.LevelAsset}, } )
var ErrExcludeTagNotSupported = errors.New("exclude-tag flag is not supported for asset-only validation")
var PipelineDefinitionFiles = []string{"pipeline.yml", "pipeline.yaml"}
Functions ¶
func AddConnection ¶ added in v0.11.22
func ApplyAllFilters ¶ added in v0.11.169
func AssetMetadata ¶ added in v0.11.281
func CloneEnvironment ¶ added in v0.11.280
func ConnectionSchemas ¶ added in v0.11.55
func Connections ¶ added in v0.11.22
func CreateEnvironment ¶ added in v0.11.252
func DataDiffCmd ¶ added in v0.11.224
DataDiffCmd defines the 'data-diff' command.
func DeleteConnection ¶ added in v0.11.22
func DeleteEnvironment ¶ added in v0.11.254
func DetermineStartDate ¶ added in v0.11.296
func Environments ¶ added in v0.11.4
func FetchColumns ¶ added in v0.11.247
func FetchDatabases ¶ added in v0.11.247
func FetchTables ¶ added in v0.11.247
func FilterTaskTypes ¶ added in v0.11.169
func GetPipelineAndAsset ¶ added in v0.11.143
func GetPipelinefromPath ¶ added in v0.11.260
func HandleExcludeTags ¶ added in v0.11.169
func HandleIncludeTags ¶ added in v0.11.169
func HandleMultipleAssets ¶ added in v0.11.429
func HandleSingleTask ¶ added in v0.11.169
func ImportDatabase ¶ added in v0.11.260
func ImportScheduledQueries ¶ added in v0.11.280
func ImportTableauDashboards ¶ added in v0.11.304
func IngestrSources ¶ added in v0.11.421
IngestrSources returns a CLI command that lists available tables for ingestr sources.
func Init ¶ added in v0.6.6
Init creates and returns the CLI command for initializing new Bruin pipelines from templates.
func ListConnections ¶ added in v0.11.22
func ListEnvironments ¶ added in v0.11.4
func ListTemplates ¶ added in v0.11.278
func LockAssetDependencies ¶ added in v0.11.429
LockAssetDependencies returns a CLI command that locks Python dependencies for an asset. It finds the asset's dependency configuration and locks versions using either uv pip compile (for requirements.txt) or uv lock (for pyproject.toml).
func ParseAsset ¶ added in v0.10.1
func ParseGlossary ¶ added in v0.11.275
func ParsePipeline ¶ added in v0.11.16
func PatchAsset ¶ added in v0.11.27
func PatchPipeline ¶ added in v0.11.321
func PingConnection ¶ added in v0.11.106
func RecoverFromPanic ¶ added in v0.5.1
func RecoverFromPanic()
func SetupExecutors ¶ added in v0.11.216
func SetupExecutors( s *scheduler.Scheduler, conn config.ConnectionAndDetailsGetter, startDate, endDate, executionDate time.Time, pipelineName string, runID string, fullRefresh bool, usePipForPython bool, sensorMode string, renderer *jinja.Renderer, parser *sqlparser.SQLParser, ) (map[pipeline.AssetType]executor.Config, error)
func SkipAllTasksIfSingleCheck ¶ added in v0.11.192
func UpdateEnvironment ¶ added in v0.11.254
func ValidateDateRange ¶ added in v0.11.414
func ValidateRunConfig ¶ added in v0.11.123
func VersionCmd ¶ added in v0.11.66
Types ¶
type CleanCommand ¶ added in v0.1.3
type CleanCommand struct {
// contains filtered or unexported fields
}
type ColumnSummary ¶ added in v0.11.412
type ColumnSummary struct {
Name string `json:"name"`
Type string `json:"type"`
NormalizedType string `json:"normalized_type"`
Nullable bool `json:"nullable"`
PrimaryKey bool `json:"primary_key"`
Unique bool `json:"unique"`
Stats interface{} `json:"stats,omitempty"`
}
ColumnSummary represents statistics for a single column within a table summary.
type ConnectionsCommand ¶ added in v0.11.22
type ConnectionsCommand struct{}
func (*ConnectionsCommand) ListConnections ¶ added in v0.11.22
func (r *ConnectionsCommand) ListConnections(pathToProject, output, environment, configFilePath string) error
type EnvironmentCloneCommand ¶ added in v0.11.280
type EnvironmentCloneCommand struct{}
func (*EnvironmentCloneCommand) Run ¶ added in v0.11.280
func (r *EnvironmentCloneCommand) Run(sourceName, targetName, schemaPrefix, output, configFilePath string) error
type EnvironmentDeleteCommand ¶ added in v0.11.254
type EnvironmentDeleteCommand struct{}
type EnvironmentListCommand ¶ added in v0.11.4
type EnvironmentListCommand struct{}
func (*EnvironmentListCommand) Run ¶ added in v0.11.4
func (r *EnvironmentListCommand) Run(output, configFilePath string) error
type EnvironmentUpdateCommand ¶ added in v0.11.254
type EnvironmentUpdateCommand struct{}
func (*EnvironmentUpdateCommand) Run ¶ added in v0.11.254
func (r *EnvironmentUpdateCommand) Run(name, newName, schemaPrefix, output, configFilePath string) error
type ErrorResponse ¶ added in v0.11.47
type ErrorResponse struct {
Error string `json:"error"`
}
type ErrorResponses ¶ added in v0.11.47
type ErrorResponses struct {
Error []string `json:"error"`
}
type ExecutionSummary ¶ added in v0.11.260
type ExecutionSummary struct {
TotalTasks int
SuccessfulTasks int
FailedTasks int
SkippedTasks int
Assets TaskTypeStats
ColumnChecks TaskTypeStats
CustomChecks TaskTypeStats
MetadataPush TaskTypeStats
Duration time.Duration
}
type Filter ¶ added in v0.11.113
type Filter struct {
IncludeTag string // Tag to include assets (from `--tag`)
OnlyTaskTypes []string // Task types to include (from `--only`)
IncludeDownstream bool // Whether to include downstream tasks (from `--downstream`)
PushMetaData bool
SingleTask *pipeline.Asset // Single asset (from running asset file directly)
SelectedAssets []*pipeline.Asset // Multiple assets specified as positional arguments
ExcludeTag string
// contains filtered or unexported fields
}
type FilterMutator ¶ added in v0.11.169
type JSONColumnDiff ¶ added in v0.11.404
type JSONColumnDiff struct {
Name string `json:"name"`
Type JSONPropDiff `json:"type"`
Nullable JSONPropDiff `json:"nullable"`
Constraints JSONPropDiff `json:"constraints"`
Status string `json:"status"` // "added", "removed", "modified", "unchanged"
}
type JSONColumnStats ¶ added in v0.11.404
type JSONColumnStats struct {
ColumnName string `json:"columnName"`
DataType string `json:"dataType"`
Statistics []JSONStatistic `json:"statistics"`
}
type JSONCountDiff ¶ added in v0.11.404
type JSONDiffOutput ¶ added in v0.11.404
type JSONDiffOutput struct {
Summary JSONSummary `json:"summary"`
SchemaDiffs []JSONColumnDiff `json:"schemaDiffs"`
ColumnStatistics []JSONColumnStats `json:"columnStatistics"`
SourceTable string `json:"sourceTable"`
TargetTable string `json:"targetTable"`
HasDifferences bool `json:"hasDifferences"`
AlterStatements string `json:"alterStatements"`
}
JSON output types for --output json flag.
type JSONPropDiff ¶ added in v0.11.404
type JSONStatistic ¶ added in v0.11.404
type JSONSummary ¶ added in v0.11.404
type JSONSummary struct {
RowCount JSONCountDiff `json:"rowCount"`
ColumnCount JSONCountDiff `json:"columnCount"`
}
type LineageCommand ¶
type LineageCommand struct {
// contains filtered or unexported fields
}
type LockDependenciesCommand ¶ added in v0.11.433
type LockDependenciesCommand struct {
// contains filtered or unexported fields
}
LockDependenciesCommand handles locking Python dependencies for assets.
func (*LockDependenciesCommand) FindDependencyPath ¶ added in v0.11.465
func (l *LockDependenciesCommand) FindDependencyPath(inputPath string) (*LockDependenciesResult, error)
FindDependencyPath finds the dependency configuration for the given input. It handles direct requirements.txt files, pyproject.toml projects, and Python assets.
type LockDependenciesResult ¶ added in v0.11.433
type LockDependenciesResult struct {
RequirementsPath string
PythonVersion string
InputPath string
DependencyType python.DependencyType
ProjectRoot string
}
LockDependenciesResult contains the result of locking dependencies.
type ModifierInfo ¶ added in v0.11.197
type ParseCommand ¶ added in v0.10.1
type ParseCommand struct {
// contains filtered or unexported fields
}
func (*ParseCommand) ParsePipeline ¶ added in v0.11.16
type PipelineInfo ¶ added in v0.11.123
type PipelineInfo struct {
Pipeline *pipeline.Pipeline
RunningForAnAsset bool
RunDownstreamTasks bool
}
func GetPipeline ¶ added in v0.11.123
type QueryLog ¶ added in v0.11.401
type QueryLog struct {
Query string `json:"query"`
Timestamp time.Time `json:"timestamp"`
Connection string `json:"connection"`
Success bool `json:"success"`
Columns []string `json:"columns,omitempty"`
Rows [][]interface{} `json:"rows,omitempty"`
Error string `json:"error,omitempty"`
Asset string `json:"asset,omitempty"`
Environment string `json:"environment,omitempty"`
Limit int64 `json:"limit,omitempty"`
Timeout int `json:"timeout,omitempty"`
}
QueryLog represents the structure of a query log entry.
type QueryLogOptions ¶ added in v0.11.401
QueryLogOptions contains optional parameters for query logging.
type RenderCommand ¶
type RenderCommand struct {
// contains filtered or unexported fields
}
func (*RenderCommand) Run ¶
func (r *RenderCommand) Run(pl *pipeline.Pipeline, task *pipeline.Asset, modifierInfo ModifierInfo) error
type ScheduledQuery ¶ added in v0.11.280
type ScheduledQuery struct {
Name string
DisplayName string
Query string
Schedule string
Dataset string
Config *datatransferpb.TransferConfig
}
Scheduled query data structures.
type Selector ¶ added in v0.11.412
type Selector interface {
Select(ctx context.Context, query *query.Query) ([][]interface{}, error)
}
Selector interface for querying databases.
type SuccessResponse ¶ added in v0.11.237
type SummaryStats ¶ added in v0.11.242
type TUIRenderer ¶ added in v0.11.465
type TUIRenderer struct {
// contains filtered or unexported fields
}
TUIRenderer manages a live-updating terminal progress display for pipeline runs.
func NewTUIRenderer ¶ added in v0.11.465
NewTUIRenderer creates a TUI renderer. terminal should be the real terminal saved before logOutput().
func (*TUIRenderer) OnStatusChange ¶ added in v0.11.465
func (t *TUIRenderer) OnStatusChange(event scheduler.StatusChangeEvent)
OnStatusChange handles scheduler status change events.
func (*TUIRenderer) OnTaskEnded ¶ added in v0.11.465
func (t *TUIRenderer) OnTaskEnded(inst scheduler.TaskInstance, err error, dur time.Duration)
OnTaskEnded is called by the worker when it finishes executing a task.
func (*TUIRenderer) OnTaskStarted ¶ added in v0.11.465
func (t *TUIRenderer) OnTaskStarted(inst scheduler.TaskInstance)
OnTaskStarted is called by the worker when it begins executing a task.
func (*TUIRenderer) Start ¶ added in v0.11.465
func (t *TUIRenderer) Start()
Start begins the TUI rendering loop.
func (*TUIRenderer) Stop ¶ added in v0.11.465
func (t *TUIRenderer) Stop()
Stop stops the TUI rendering loop and restores cursor.
type TableComparer ¶ added in v0.11.224
type TableComparer interface {
CompareTables(ctx context.Context, table1, table2 string) (*diff.SchemaComparisonResult, error)
}
TableComparer defines an interface for connections that can compare tables.
type TableSummary ¶ added in v0.11.412
type TableSummary struct {
TableName string `json:"table_name"`
Connection string `json:"connection"`
RowCount int64 `json:"row_count"`
Columns []ColumnSummary `json:"columns"`
SampleValues map[string][]interface{} `json:"sample_values,omitempty"`
Error string `json:"error,omitempty"`
}
TableSummary represents comprehensive statistics for an entire table.
type TableauDashboard ¶ added in v0.11.304
type TableauDashboard struct {
ViewID string
ViewName string
WorkbookID string
WorkbookName string
WorkbookURL string // Added for workbook URL in meta
ProjectID string // Added for project hierarchy
ProjectName string
ProjectPath []string // Added for full project hierarchy path
OwnerName string
ContentURL string
ViewURL string
UpdatedAt string
Tags []string
DataSources []tableau.DataSourceInfo // Added for data source dependencies
Connections []tableau.WorkbookConnection // Added for connection tracking
}
TableauDashboard represents a dashboard/view with its workbook information.
type TaskTypeStats ¶ added in v0.11.260
type TaskTypeStats struct {
Total int
Succeeded int
Failed int // Failed in main execution
FailedDueToChecks int // Failed only due to checks (main execution succeeded)
Skipped int
}
func (TaskTypeStats) HasAny ¶ added in v0.11.260
func (s TaskTypeStats) HasAny() bool
func (TaskTypeStats) SuccessRate ¶ added in v0.11.260
func (s TaskTypeStats) SuccessRate() float64