python

package
v0.11.689 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 20, 2026 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	LocalIngestr contextKey = "local_ingestr"
	// CtxIngestrVersion overrides the default ingestr PyPI version for the asset (e.g. "0.14.2").
	CtxIngestrVersion contextKey = "ingestr_version"
)
View Source 
const (
	Shell                   = "/bin/sh"
	ShellSubcommandFlag     = "-c"
	VirtualEnvBinaryFolder  = "bin"
	DefaultPythonExecutable = "python3"
)
View Source 
const (

	// IngestrVersionV0 is the legacy ingestr release pinned for parameters.version=v0.
	IngestrVersionV0 = "0.14.155"
	// IngestrVersionV1 is the current ingestr release used by default and for parameters.version=v1.
	IngestrVersionV1 = "1.1.4"
)
View Source 
const CtxUseWingetForUv = "use_winget_for_uv"

CtxUseWingetForUv is a context key for enabling winget-based uv installation on Windows.

View Source 
const PythonArrowTemplate = `` /* 6512-byte string literal not displayed */

Variables

View Source 
var AvailablePythonVersions = map[string]bool{
	"3.8":  true,
	"3.9":  true,
	"3.10": true,
	"3.11": true,
	"3.12": true,
	"3.13": true,
	"3.14": true,
}
View Source 
var BruinToIngestrStrategyMap = map[pipeline.MaterializationStrategy]string{
	pipeline.MaterializationStrategyCreateReplace: "replace",
	pipeline.MaterializationStrategyAppend:        "append",
	pipeline.MaterializationStrategyMerge:         "merge",
	pipeline.MaterializationStrategyDeleteInsert:  "delete+insert",
}
View Source 
var DatabasePrefixToSqlfluffDialect = map[string]string{
	"bq":         "bigquery",
	"sf":         "snowflake",
	"pg":         "postgres",
	"rs":         "redshift",
	"athena":     "athena",
	"ms":         "tsql",
	"fabric":     "tsql",
	"fw":         "tsql",
	"databricks": "sparksql",
	"synapse":    "tsql",
	"duckdb":     "duckdb",
	"clickhouse": "clickhouse",
}
View Source 
var SortedPythonVersions = []string{"3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"}

SortedPythonVersions is the list of available Python versions sorted ascending.

View Source 
var SupportedIngestrStrategies = []string{
	"replace",
	"append",
	"merge",
	"delete+insert",
	"truncate+insert",
}

SupportedIngestrStrategies lists all incremental strategies supported by ingestr.

View Source 
var SupportedPythonMaterializationStrategies = []pipeline.MaterializationStrategy{
	pipeline.MaterializationStrategyCreateReplace,
	pipeline.MaterializationStrategyAppend,
	pipeline.MaterializationStrategyMerge,
	pipeline.MaterializationStrategyDeleteInsert,
}

SupportedPythonMaterializationStrategies lists all materialization strategies supported by Python assets.

View Source 
var TypeHintMapping = map[string]string{}/* 138 elements not displayed */

TypeHintMapping maps portable / cross-platform column type aliases to ingestr (dlt) types. Destination-specific aliases live on the connection via IngestrTypeHintProvider. 'text' is the ingestr default when no hint is emitted.

Functions

func AddExtraPackages added in v0.11.161 

func AddExtraPackages(destURI, sourceURI string, extraPackages []string) []string

func ColumnHints added in v0.11.405 

func ColumnHints(cols []pipeline.Column, normaliseNames bool, overlay map[string]string, wrappers map[string]bool) string

ColumnHints returns an ingestr compatible type hint string that can be passed via the --column flag to the CLI. overlay may be nil; when set, its aliases are merged over TypeHintMapping. wrappers may be nil; when set, those parameterized types peel to their inner type.

func ConsolidatedParameters added in v0.11.161 

func ConsolidatedParameters(ctx context.Context, asset *pipeline.Asset, cmdArgs []string, columnOpts *ColumnHintOptions) ([]string, error)

func DetectDialectFromAssetType added in v0.11.243 

func DetectDialectFromAssetType(assetType string) string

DetectDialectFromAssetType extracts the dialect from an asset's type field by splitting on the first dot and mapping the prefix.

func GetSupportedIngestrMaterializationStrategiesString added in v0.11.666 

func GetSupportedIngestrMaterializationStrategiesString() string

func GetSupportedIngestrStrategiesString added in v0.11.359 

func GetSupportedIngestrStrategiesString() string

GetSupportedIngestrStrategiesString returns a comma-separated string of supported ingestr strategies.

func GetSupportedPythonStrategiesString added in v0.11.359 

func GetSupportedPythonStrategiesString() string

GetSupportedPythonStrategiesString returns a comma-separated string of supported Python materialization strategies.

func IsIngestrIncrementalKeyStrategy added in v0.11.666 

func IsIngestrIncrementalKeyStrategy(strategy string) bool

func IsIngestrMaterializationStrategySupported added in v0.11.666 

func IsIngestrMaterializationStrategySupported(strategy pipeline.MaterializationStrategy) bool

func IsIngestrStrategySupported added in v0.11.359 

func IsIngestrStrategySupported(strategy string) bool

IsIngestrStrategySupported checks if a given strategy string is supported by ingestr.

func IsPythonMaterializationStrategySupported added in v0.11.359 

func IsPythonMaterializationStrategySupported(strategy pipeline.MaterializationStrategy) bool

IsPythonMaterializationStrategySupported checks if a given strategy is supported for Python assets.

func MergeTypeHints added in v0.11.681 

func MergeTypeHints(base, overlay map[string]string) map[string]string

MergeTypeHints returns a copy of base with overlay entries applied. Overlay keys are normalised the same way as column types. Overlay wins on conflict. The returned map is always a clone so callers cannot mutate TypeHintMapping.

func NormaliseColumnType added in v0.11.405 

func NormaliseColumnType(typ string) string

func NormalizeColumnName added in v0.11.405 

func NormalizeColumnName(name string) string

func ResolvePythonVersion added in v0.11.475 

func ResolvePythonVersion(requiresPython string, defaultVersion string) (string, error)

ResolvePythonVersion resolves the minimum available Python version that satisfies the given requires-python specifier (PEP 440). Returns defaultVersion if the specifier is empty. Returns an error if no available version satisfies the constraint.

func TranslateBruinMaterializationStrategyToIngestr added in v0.11.666 

func TranslateBruinMaterializationStrategyToIngestr(strategy pipeline.MaterializationStrategy) (string, bool)

TranslateBruinMaterializationStrategyToIngestr converts Bruin materialization strategy names to ingestr incremental strategy names.

func TranslateBruinStrategyToIngestr added in v0.11.359 

func TranslateBruinStrategyToIngestr(strategy pipeline.MaterializationStrategy) (string, bool)

TranslateBruinStrategyToIngestr converts a Bruin materialization strategy to its ingestr equivalent.

func TypeHintOverlayForConnection added in v0.11.681 

func TypeHintOverlayForConnection(conn any) map[string]string

TypeHintOverlayForConnection returns DB-specific type aliases when the connection implements IngestrTypeHintProvider; otherwise nil.

func TypeWrappersForConnection added in v0.11.681 

func TypeWrappersForConnection(conn any) map[string]bool

TypeWrappersForConnection returns transparent type wrappers when the connection implements IngestrTypeWrapperProvider; otherwise nil.

func WithStreaming added in v0.11.681 

func WithStreaming(ctx context.Context) context.Context

WithStreaming marks a context as belonging to a long-running streaming ingestr run. RunIngestr uses it to launch the child as a managed process that is tied to the context and torn down gracefully on cancellation.

Types

type ColumnHintOptions added in v0.11.405 

type ColumnHintOptions struct {
	// NormalizeColumnNames converts column names to snake_case (used for CSV/seed assets).
	NormalizeColumnNames bool
	// EnforceSchemaByDefault determines behavior when enforce_schema parameter is not set.
	// If true, schema will be enforced by default (used for seed assets).
	// If false, schema will only be enforced when enforce_schema="true" is explicitly set.
	EnforceSchemaByDefault bool
	// TypeHintOverlay is an optional destination-specific alias map merged over
	// TypeHintMapping (e.g. from IngestrTypeHintProvider on the dest connection).
	TypeHintOverlay map[string]string
	// TypeWrappers is an optional set of transparent parameterized type names
	// (e.g. from IngestrTypeWrapperProvider) whose inner type should be resolved.
	TypeWrappers map[string]bool
}

ColumnHintOptions controls how column hints are added to ingestr commands.

type CommandInstance added in v0.11.104 

type CommandInstance struct {
	Name    string
	Args    []string
	EnvVars map[string]string
	// Managed ties the child process to the caller's context and enables graceful
	// process-group teardown (SIGTERM -> grace -> SIGKILL). It is used for
	// long-running streaming ingestr runs, which must be stoppable on
	// cancellation; one-shot commands keep the detached behaviour below.
	Managed bool
}

type CommandRunner added in v0.11.92 

type CommandRunner struct{}

func (*CommandRunner) Run added in v0.11.92 

func (l *CommandRunner) Run(ctx context.Context, repo *git.Repo, command *CommandInstance) error

func (*CommandRunner) RunAnyCommand added in v0.11.92 

func (l *CommandRunner) RunAnyCommand(ctx context.Context, cmd *exec.Cmd) error

type DependencyConfig added in v0.11.465 

type DependencyConfig struct {
	Type            DependencyType
	PyprojectPath   string // Path to pyproject.toml (empty if not found)
	RequirementsTxt string // Path to requirements.txt (empty if not found)
	ProjectRoot     string // Directory containing the dependency file
	RequiresPython  string // The requires-python specifier from pyproject.toml (e.g., ">=3.13")
}

DependencyConfig holds the discovered dependency configuration for a Python asset.

type DependencyType added in v0.11.465 

type DependencyType int

DependencyType represents the type of dependency configuration found. 

const (
	DependencyTypeNone DependencyType = iota
	DependencyTypeRequirementsTxt
	DependencyTypePyproject
)

type IngestrTypeHintProvider added in v0.11.681 

type IngestrTypeHintProvider interface {
	IngestrTypeHints() map[string]string
}

IngestrTypeHintProvider is an optional capability on destination connections. Packages like clickhouse implement this so DB-native type aliases can overlay the shared defaults without pkg/python needing to know about each platform.

type IngestrTypeWrapperProvider added in v0.11.681 

type IngestrTypeWrapperProvider interface {
	IngestrTypeWrappers() map[string]bool
}

IngestrTypeWrapperProvider is an optional capability for destinations that use transparent type wrappers (e.g. ClickHouse Nullable(T) / LowCardinality(T)).

type LocalOperator 

type LocalOperator struct {
	// contains filtered or unexported fields
}

func NewLocalOperator 

func NewLocalOperator(config config.ConnectionAndDetailsGetter, envVariables map[string]string) *LocalOperator

func (*LocalOperator) Run 

func (o *LocalOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (*LocalOperator) RunTask 

func (o *LocalOperator) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

type ModulePathFinder 

type ModulePathFinder struct {
	PathSeparatorOverride int32
}

func (*ModulePathFinder) FindDependencyConfig added in v0.11.465 

func (m *ModulePathFinder) FindDependencyConfig(path string, executable *pipeline.ExecutableFile) (*DependencyConfig, error)

FindDependencyConfig searches for dependency configuration files and returns the best option based on priority: requirements.txt > pyproject.toml. This ensures backward compatibility while supporting modern uv-based workflows. When pyproject.toml is found, UV will automatically handle lockfile detection.

func (*ModulePathFinder) FindModulePath 

func (m *ModulePathFinder) FindModulePath(repo *git.Repo, executable *pipeline.ExecutableFile) (string, error)

func (*ModulePathFinder) FindPyprojectTomlInPath added in v0.11.465 

func (m *ModulePathFinder) FindPyprojectTomlInPath(path string, executable *pipeline.ExecutableFile) (string, error)

FindPyprojectTomlInPath searches for pyproject.toml starting from the executable's directory and walking up the directory tree until reaching the repository root.

func (*ModulePathFinder) FindRequirementsTxtInPath 

func (m *ModulePathFinder) FindRequirementsTxtInPath(path string, executable *pipeline.ExecutableFile) (string, error)

type NoPyprojectFoundError added in v0.11.465 

type NoPyprojectFoundError struct{}

func (*NoPyprojectFoundError) Error added in v0.11.465 

func (m *NoPyprojectFoundError) Error() string

type NoRequirementsFoundError 

type NoRequirementsFoundError struct{}

func (*NoRequirementsFoundError) Error 

func (m *NoRequirementsFoundError) Error() string

type PyprojectToml added in v0.11.243 

type PyprojectToml struct {
	Tool struct {
		Sqlfluff map[string]any `toml:"sqlfluff"`
	} `toml:"tool"`
}

type SQLFileInfo added in v0.11.243 

type SQLFileInfo struct {
	FilePath string
	Dialect  string
}

type SqlfluffRunner added in v0.11.243 

type SqlfluffRunner struct {
	UvInstaller uvInstaller
	// contains filtered or unexported fields
}

func (*SqlfluffRunner) RunSqlfluffWithDialects added in v0.11.243 

func (s *SqlfluffRunner) RunSqlfluffWithDialects(ctx context.Context, sqlFiles []SQLFileInfo, repo *git.Repo) error

type UvChecker added in v0.11.87 

type UvChecker struct {
	uv.Checker
}

UvChecker handles checking and installing the uv package manager.

type UvPythonRunner added in v0.11.92 

type UvPythonRunner struct {
	Cmd         cmd
	UvInstaller uvInstaller
	// contains filtered or unexported fields
}

func (*UvPythonRunner) Run added in v0.11.92 

func (u *UvPythonRunner) Run(ctx context.Context, execCtx *executionContext) error

func (*UvPythonRunner) RunIngestr added in v0.11.92 

func (u *UvPythonRunner) RunIngestr(ctx context.Context, args, extraPackages []string, repo *git.Repo) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.