lint

package
v0.11.310 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 7, 2025 License: Apache-2.0 Imports: 33 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	RuleTargetAsset    = RuleTarget("asset")
	RuleTargetPipeline = RuleTarget("pipeline")
)
View Source 
var TableSensorAllowedAssetTypes = map[pipeline.AssetType]bool{
	pipeline.AssetTypeBigqueryTableSensor:   true,
	pipeline.AssetTypeSnowflakeTableSensor:  true,
	pipeline.AssetTypeAthenaTableSensor:     true,
	pipeline.AssetTypeRedshiftTableSensor:   true,
	pipeline.AssetTypeDatabricksTableSensor: true,
	pipeline.AssetTypeClickHouseTableSensor: true,
	pipeline.AssetTypeMsSQLTableSensor:      true,
	pipeline.AssetTypePostgresTableSensor:   true,
	pipeline.AssetTypeSynapseTableSensor:    true,
}

Functions

func CallFuncForEveryAsset added in v0.6.6 

func CallFuncForEveryAsset(callable AssetValidator) func(ctx context.Context, pipeline *pipeline.Pipeline) ([]*Issue, error)

func ContainsTag added in v0.11.239 

func ContainsTag(tags []string, target string) bool

func EnsureNoNestedPipelines added in v0.4.1 

func EnsureNoNestedPipelines(pipelinePaths []string) error

func QueryColumnsMatchColumnsPolicy added in v0.11.244 

func QueryColumnsMatchColumnsPolicy(parser *sqlparser.SQLParser) func(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

Types

type AssetValidator added in v0.5.6 

type AssetValidator func(ctx context.Context, pipeline *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureExecutableFileIsValidForASingleAsset added in v0.5.6 

func EnsureExecutableFileIsValidForASingleAsset(fs afero.Fs) AssetValidator

func ValidateCustomCheckQueryDryRun added in v0.11.222 

func ValidateCustomCheckQueryDryRun(connections connectionManager, renderer jinja.RendererInterface) AssetValidator

ValidateCustomCheckQueryDryRun validates CustomCheck.Query using a dry-run against the DB.

type CrossPipelineValidator added in v0.11.280 

type CrossPipelineValidator func(ctx context.Context, pipelines []*pipeline.Pipeline) ([]*Issue, error)

type GlossaryChecker added in v0.11.9 

type GlossaryChecker struct {
	// contains filtered or unexported fields
}

func (*GlossaryChecker) EnsureAssetEntitiesExistInGlossary added in v0.11.9 

func (g *GlossaryChecker) EnsureAssetEntitiesExistInGlossary(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func (*GlossaryChecker) EnsureParentDomainsExistInGlossary added in v0.11.278 

func (g *GlossaryChecker) EnsureParentDomainsExistInGlossary(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

type Issue 

type Issue struct {
	Task        *pipeline.Asset
	Description string
	Context     []string
}

func EnsureAssetTierIsValidForASingleAsset added in v0.11.250 

func EnsureAssetTierIsValidForASingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureBigQueryQuerySensorHasQueryParameterForASingleAsset added in v0.11.296 

func EnsureBigQueryQuerySensorHasQueryParameterForASingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureDependencyExistsForASingleAsset added in v0.5.6 

func EnsureDependencyExistsForASingleAsset(ctx context.Context, p *pipeline.Pipeline, task *pipeline.Asset) ([]*Issue, error)

func EnsureIngestrAssetIsValidForASingleAsset added in v0.6.6 

func EnsureIngestrAssetIsValidForASingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureMSTeamsFieldInPipelineIsValid added in v0.11.20 

func EnsureMSTeamsFieldInPipelineIsValid(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func EnsureMaterializationValuesAreValidForSingleAsset added in v0.5.6 

func EnsureMaterializationValuesAreValidForSingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsurePipelineConcurrencyIsValid added in v0.11.243 

func EnsurePipelineConcurrencyIsValid(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func EnsurePipelineHasNoCycles 

func EnsurePipelineHasNoCycles(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

EnsurePipelineHasNoCycles ensures that the pipeline is a DAG, and contains no cycles. Since the pipelines are directed graphs, strongly connected components mean cycles, therefore they would be considered invalid for our pipelines. Strong connectivity wouldn't work for tasks that depend on themselves, therefore there's a specific check for that.

func EnsurePipelineNameIsValid 

func EnsurePipelineNameIsValid(ctx context.Context, pipeline *pipeline.Pipeline) ([]*Issue, error)

func EnsurePipelineScheduleIsValidCron 

func EnsurePipelineScheduleIsValidCron(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func EnsurePipelineStartDateIsValid added in v0.11.8 

func EnsurePipelineStartDateIsValid(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func EnsureSecretMappingsHaveKeyForASingleAsset added in v0.11.271 

func EnsureSecretMappingsHaveKeyForASingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureSlackFieldInPipelineIsValid 

func EnsureSlackFieldInPipelineIsValid(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func EnsureSnowflakeSensorHasQueryParameterForASingleAsset added in v0.5.6 

func EnsureSnowflakeSensorHasQueryParameterForASingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureTaskNameIsUnique 

func EnsureTaskNameIsUnique(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func EnsureTaskNameIsUniqueForASingleAsset added in v0.5.6 

func EnsureTaskNameIsUniqueForASingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureTaskNameIsValidForASingleAsset added in v0.5.6 

func EnsureTaskNameIsValidForASingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureTimeIntervalIsValidForAsset added in v0.11.300 

func EnsureTimeIntervalIsValidForAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func EnsureTypeIsCorrectForASingleAsset added in v0.5.6 

func EnsureTypeIsCorrectForASingleAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func ValidateAssetDirectoryExist added in v0.11.55 

func ValidateAssetDirectoryExist(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func ValidateAssetSeedValidation added in v0.11.128 

func ValidateAssetSeedValidation(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func ValidateCrossPipelineURIDependencies added in v0.11.280 

func ValidateCrossPipelineURIDependencies(ctx context.Context, pipelines []*pipeline.Pipeline) ([]*Issue, error)

ValidateCrossPipelineURIDependencies validates all URI dependencies across all pipelines and returns warnings for any URI dependencies that cannot be resolved or duplicate URIs.

func ValidateCustomCheckQueryExists added in v0.11.122 

func ValidateCustomCheckQueryExists(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

ValidateCustomCheckQueryExists checks for duplicate column checks within a single column. It returns a slice of Issues, each representing a duplicate column check found.

func ValidateDuplicateColumnNames added in v0.11.35 

func ValidateDuplicateColumnNames(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

ValidateDuplicateColumnNames checks for duplicate column names within a single asset. It returns a slice of Issues, each representing a duplicate column name found.

The function performs a case-insensitive comparison of column names.

Parameters:

  • ctx: The context for the validation operation
  • p: A pointer to the pipeline.Pipeline struct
  • asset: The pipeline.Asset to be validated for duplicate column names.

Returns:

  • A slice of *Issue, each describing a duplicate column name found.
  • An error, which is always nil in this implementation.

func ValidateDuplicateTags added in v0.11.280 

func ValidateDuplicateTags(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

ValidateDuplicateTags checks for duplicate tags within an asset and its columns. It performs case-insensitive comparisons to find duplicates and returns issues for any repeated tags found either on the asset itself or within individual columns.

func ValidateEMRServerlessAsset added in v0.11.171 

func ValidateEMRServerlessAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func ValidatePythonAssetMaterialization added in v0.11.143 

func ValidatePythonAssetMaterialization(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func ValidateTableSensorTableParameter added in v0.11.296 

func ValidateTableSensorTableParameter(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func ValidateVariables added in v0.11.209 

func ValidateVariables(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

type Level added in v0.5.6 

type Level int
const (
	LevelPipeline Level = iota
	LevelAsset
	LevelCrossPipeline
)

type Linter 

type Linter struct {
	// contains filtered or unexported fields
}

func NewLinter 

func NewLinter(findPipelines pipelineFinder, builder pipelineBuilder, rules []Rule, logger logger.Logger, sqlParser sqlParser) *Linter

func (*Linter) Lint 

func (l *Linter) Lint(ctx context.Context, rootPath string, pipelineDefinitionFileName []string, c *cli.Command) (*PipelineAnalysisResult, error)

func (*Linter) LintAsset added in v0.5.6 

func (l *Linter) LintAsset(ctx context.Context, rootPath string, pipelineDefinitionFileName []string, assetNameOrPath string, c *cli.Command) (*PipelineAnalysisResult, error)

func (*Linter) LintPipeline added in v0.4.1 

func (l *Linter) LintPipeline(ctx context.Context, p *pipeline.Pipeline) (*PipelineIssues, error)

func (*Linter) LintPipelines 

func (l *Linter) LintPipelines(ctx context.Context, pipelines []*pipeline.Pipeline) (*PipelineAnalysisResult, error)

type PipelineAnalysisResult 

type PipelineAnalysisResult struct {
	Pipelines                []*PipelineIssues `json:"pipelines"`
	AssetWithExcludeTagCount int
}

func (*PipelineAnalysisResult) ErrorCount 

func (p *PipelineAnalysisResult) ErrorCount() int

ErrorCount returns the number of errors found in an analysis result.

func (*PipelineAnalysisResult) MarshalJSON added in v0.6.8 

func (p *PipelineAnalysisResult) MarshalJSON() ([]byte, error)

func (*PipelineAnalysisResult) WarningCount added in v0.11.14 

func (p *PipelineAnalysisResult) WarningCount() int

WarningCount returns the number of warnings, a.k.a non-critical issues found in an analysis result.

type PipelineIssues 

type PipelineIssues struct {
	Pipeline *pipeline.Pipeline
	Issues   map[Rule][]*Issue
}

func RunLintRulesOnPipeline added in v0.4.1 

func RunLintRulesOnPipeline(ctx context.Context, p *pipeline.Pipeline, rules []Rule, sqlParser sqlParser) (*PipelineIssues, error)

func (*PipelineIssues) MarshalJSON added in v0.6.8 

func (p *PipelineIssues) MarshalJSON() ([]byte, error)

type PipelineValidator 

type PipelineValidator func(ctx context.Context, pipeline *pipeline.Pipeline) ([]*Issue, error)

type PolicySpecification added in v0.11.195 

type PolicySpecification struct {
	Definitions []*RuleDefinition `yaml:"custom_rules"`
	RuleSets    []RuleSet         `yaml:"rulesets"`
	// contains filtered or unexported fields
}

func (*PolicySpecification) Rules added in v0.11.195 

func (spec *PolicySpecification) Rules(sqlParser sqlParser) ([]Rule, error)

we need to pass in the sqlparser to the policy because of the query-matches-columns rule.

type Printer 

type Printer struct {
	RootCheckPath string
}

func (*Printer) PrintIssues 

func (l *Printer) PrintIssues(analysis *PipelineAnalysisResult)

func (*Printer) PrintJSON added in v0.6.8 

func (l *Printer) PrintJSON(analysis *PipelineAnalysisResult) error

type QueryValidatorRule 

type QueryValidatorRule struct {
	Identifier   string
	Fast         bool
	TaskType     pipeline.AssetType
	Connections  connectionManager
	Extractor    queryExtractor
	Materializer materializer
	WorkerCount  int
	Logger       logger.Logger
}

func (*QueryValidatorRule) GetApplicableLevels added in v0.5.6 

func (q *QueryValidatorRule) GetApplicableLevels() []Level

func (*QueryValidatorRule) GetSeverity added in v0.11.14 

func (q *QueryValidatorRule) GetSeverity() ValidatorSeverity

func (*QueryValidatorRule) IsFast added in v0.11.53 

func (q *QueryValidatorRule) IsFast() bool

func (*QueryValidatorRule) Name 

func (q *QueryValidatorRule) Name() string

func (*QueryValidatorRule) Validate 

func (q *QueryValidatorRule) Validate(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func (*QueryValidatorRule) ValidateAsset added in v0.5.6 

func (q *QueryValidatorRule) ValidateAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func (*QueryValidatorRule) ValidateCrossPipeline added in v0.11.280 

func (q *QueryValidatorRule) ValidateCrossPipeline(ctx context.Context, pipelines []*pipeline.Pipeline) ([]*Issue, error)

type Rule 

type Rule interface {
	Name() string
	IsFast() bool
	Validate(ctx context.Context, pipeline *pipeline.Pipeline) ([]*Issue, error)
	ValidateAsset(ctx context.Context, pipeline *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)
	ValidateCrossPipeline(ctx context.Context, pipelines []*pipeline.Pipeline) ([]*Issue, error)
	GetApplicableLevels() []Level
	GetSeverity() ValidatorSeverity
}

func FilterRulesByLevel added in v0.5.6 

func FilterRulesByLevel(rules []Rule, level Level) []Rule

func FilterRulesBySpeed added in v0.11.53 

func FilterRulesBySpeed(rules []Rule, fast bool) []Rule

func GetRules 

func GetRules(fs afero.Fs, finder repoFinder, excludeWarnings bool, parser *sqlparser.SQLParser, cacheFoundGlossary bool) ([]Rule, error)

type RuleDefinition added in v0.11.195 

type RuleDefinition struct {
	Name        string     `yaml:"name"`
	Description string     `yaml:"description"`
	Criteria    string     `yaml:"criteria"`
	RuleTarget  RuleTarget `yaml:"target"`
	// contains filtered or unexported fields
}

type RuleSet added in v0.11.195 

type RuleSet struct {
	Name     string           `yaml:"name"`
	Selector []map[string]any `yaml:"selector"`
	Rules    []string         `yaml:"rules"`
}

type RuleTarget added in v0.11.198 

type RuleTarget string

func (RuleTarget) Valid added in v0.11.198 

func (target RuleTarget) Valid() bool

type SimpleRule 

type SimpleRule struct {
	Identifier             string
	Fast                   bool
	Validator              PipelineValidator
	AssetValidator         AssetValidator
	CrossPipelineValidator CrossPipelineValidator
	ApplicableLevels       []Level
	Severity               ValidatorSeverity
}

func GetCustomCheckQueryDryRunRule added in v0.11.223 

func GetCustomCheckQueryDryRunRule(connectionManager connectionManager, renderer jinja.RendererInterface) *SimpleRule

func (*SimpleRule) GetApplicableLevels added in v0.5.6 

func (g *SimpleRule) GetApplicableLevels() []Level

func (*SimpleRule) GetSeverity added in v0.11.14 

func (g *SimpleRule) GetSeverity() ValidatorSeverity

func (*SimpleRule) IsFast added in v0.11.53 

func (g *SimpleRule) IsFast() bool

func (*SimpleRule) Name 

func (g *SimpleRule) Name() string

func (*SimpleRule) Validate 

func (g *SimpleRule) Validate(ctx context.Context, pipeline *pipeline.Pipeline) ([]*Issue, error)

func (*SimpleRule) ValidateAsset added in v0.5.6 

func (g *SimpleRule) ValidateAsset(ctx context.Context, pipeline *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func (*SimpleRule) ValidateCrossPipeline added in v0.11.280 

func (g *SimpleRule) ValidateCrossPipeline(ctx context.Context, pipelines []*pipeline.Pipeline) ([]*Issue, error)

type TableNameValidationRule added in v0.11.296 

type TableNameValidationRule struct {
	MinComponents int
	MaxComponents int
	FormatDesc    string
}

type UsedTableValidatorRule added in v0.11.14 

type UsedTableValidatorRule struct {
	// contains filtered or unexported fields
}

func (UsedTableValidatorRule) GetApplicableLevels added in v0.11.14 

func (u UsedTableValidatorRule) GetApplicableLevels() []Level

func (UsedTableValidatorRule) GetSeverity added in v0.11.14 

func (u UsedTableValidatorRule) GetSeverity() ValidatorSeverity

func (UsedTableValidatorRule) IsFast added in v0.11.53 

func (u UsedTableValidatorRule) IsFast() bool

func (UsedTableValidatorRule) Name added in v0.11.14 

func (u UsedTableValidatorRule) Name() string

func (UsedTableValidatorRule) Validate added in v0.11.14 

func (u UsedTableValidatorRule) Validate(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

func (UsedTableValidatorRule) ValidateAsset added in v0.11.14 

func (u UsedTableValidatorRule) ValidateAsset(ctx context.Context, p *pipeline.Pipeline, asset *pipeline.Asset) ([]*Issue, error)

func (UsedTableValidatorRule) ValidateCrossPipeline added in v0.11.280 

func (u UsedTableValidatorRule) ValidateCrossPipeline(ctx context.Context, pipelines []*pipeline.Pipeline) ([]*Issue, error)

type ValidatorSeverity added in v0.11.14 

type ValidatorSeverity int
const (
	ValidatorSeverityWarning ValidatorSeverity = iota
	ValidatorSeverityCritical
)

type WarnRegularYamlFiles added in v0.11.177 

type WarnRegularYamlFiles struct {
	// contains filtered or unexported fields
}

func (*WarnRegularYamlFiles) WarnRegularYamlFilesInRepo added in v0.11.177 

func (w *WarnRegularYamlFiles) WarnRegularYamlFilesInRepo(ctx context.Context, p *pipeline.Pipeline) ([]*Issue, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.