transformation

package
v0.0.14 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 18, 2025 License: GPL-3.0 Imports: 6 Imported by: 0

Documentation

Overview

Package transformation provides transformation model configuration and validation

Index

Constants

View Source
const TransformationTypeExec = "exec"

TransformationTypeExec identifies exec transformation models

View Source
const TransformationTypeSQL = "sql"

TransformationTypeSQL identifies SQL transformation models

Variables

View Source
var (
	// ErrDatabaseRequired is returned when database is not specified
	ErrDatabaseRequired = errors.New("database is required")
	// ErrTableRequired is returned when table is not specified
	ErrTableRequired = errors.New("table is required")
	// ErrNoSchedulesConfig is returned when no schedules are configured
	ErrNoSchedulesConfig = errors.New("at least one schedule must be configured")
	// ErrIntervalRequired is returned when interval is not specified
	ErrIntervalRequired = errors.New("interval configuration is required")
	// ErrIntervalMaxRequired is returned when interval.max is not specified
	ErrIntervalMaxRequired = errors.New("interval.max is required")
	// ErrInvalidInterval is returned when interval.min exceeds interval.max
	ErrInvalidInterval = errors.New("interval.min cannot exceed interval.max")
	// ErrDependenciesRequired is returned when dependencies are not specified
	ErrDependenciesRequired = errors.New("dependencies is required")
	// ErrInvalidLimits is returned when min limit is greater than max limit
	ErrInvalidLimits = errors.New("min limit cannot be greater than max limit")
	// ErrInvalidDependencyType is returned when dependency has invalid YAML type
	ErrInvalidDependencyType = errors.New("dependency must be a string or array of strings")
	// ErrInvalidDependencyArrayItem is returned when dependency array contains non-string
	ErrInvalidDependencyArrayItem = errors.New("expected string in dependency array")
	// ErrEmptyDependencyGroup is returned when dependency group is empty
	ErrEmptyDependencyGroup = errors.New("dependency group cannot be empty")
)
View Source
var (
	// ErrInvalidFrontmatter is returned when frontmatter is invalid
	ErrInvalidFrontmatter = errors.New("invalid frontmatter")
	// ErrSQLContentRequired is returned when SQL content is not specified
	ErrSQLContentRequired = errors.New("sql content is required")
)
View Source
var (
	// ErrExecRequired is returned when exec is not specified
	ErrExecRequired = errors.New("exec is required")
)

Functions

func ValidateScheduleFormat

func ValidateScheduleFormat(schedule string) error

ValidateScheduleFormat validates a cron schedule expression

Types

type Config

type Config struct {
	Database     string           `yaml:"database"` // Optional, can fall back to default
	Table        string           `yaml:"table"`
	Limits       *LimitsConfig    `yaml:"limits,omitempty"`
	Interval     *IntervalConfig  `yaml:"interval"`
	Schedules    *SchedulesConfig `yaml:"schedules"`
	Dependencies []Dependency     `yaml:"dependencies"`
	Tags         []string         `yaml:"tags"`

	// OriginalDependencies stores the dependencies before placeholder substitution
	// This is used by the template engine to create dual entries
	OriginalDependencies []Dependency `yaml:"-"`
}

Config defines the configuration for transformation models

func (*Config) AllowsPartialIntervals added in v0.0.7

func (c *Config) AllowsPartialIntervals() bool

AllowsPartialIntervals returns true if partial intervals are allowed

func (*Config) GetBackfillSchedule added in v0.0.7

func (c *Config) GetBackfillSchedule() string

GetBackfillSchedule returns the schedule for backfill operations

func (*Config) GetFlattenedDependencies added in v0.0.10

func (c *Config) GetFlattenedDependencies() []string

GetFlattenedDependencies returns all dependencies as a flat string array This includes all dependencies from both single and OR groups

func (*Config) GetForwardSchedule added in v0.0.2

func (c *Config) GetForwardSchedule() string

GetForwardSchedule returns the schedule for forward fill operations

func (*Config) GetID

func (c *Config) GetID() string

GetID returns the unique identifier for the transformation model

func (*Config) GetMaxInterval added in v0.0.7

func (c *Config) GetMaxInterval() uint64

GetMaxInterval returns the maximum interval size

func (*Config) GetMaxLimit added in v0.0.2

func (c *Config) GetMaxLimit() uint64

GetMaxLimit returns the maximum position limit (0 means no limit)

func (*Config) GetMinInterval added in v0.0.7

func (c *Config) GetMinInterval() uint64

GetMinInterval returns the minimum interval size

func (*Config) GetMinLimit added in v0.0.2

func (c *Config) GetMinLimit() uint64

GetMinLimit returns the minimum position limit

func (*Config) HasLimits added in v0.0.2

func (c *Config) HasLimits() bool

HasLimits returns true if limits are configured

func (*Config) IsBackfillEnabled added in v0.0.2

func (c *Config) IsBackfillEnabled() bool

IsBackfillEnabled returns true if backfill is configured

func (*Config) IsForwardFillEnabled added in v0.0.7

func (c *Config) IsForwardFillEnabled() bool

IsForwardFillEnabled returns true if forward fill is configured

func (*Config) SetDefaults added in v0.0.8

func (c *Config) SetDefaults(defaultDatabase string)

SetDefaults applies default values to the configuration

func (*Config) SubstituteDependencyPlaceholders added in v0.0.8

func (c *Config) SubstituteDependencyPlaceholders(externalDefaultDB, transformationDefaultDB string)

SubstituteDependencyPlaceholders replaces {{external}} and {{transformation}} placeholders with the actual default database names

func (*Config) Validate

func (c *Config) Validate() error

Validate checks if the transformation configuration is valid

type Dependency added in v0.0.10

type Dependency struct {
	// IsGroup indicates if this is an OR group (array) or a single dependency (string)
	IsGroup bool
	// SingleDep holds the dependency ID for single dependencies
	SingleDep string
	// GroupDeps holds multiple dependency IDs for OR groups
	GroupDeps []string
}

Dependency represents a dependency that can be either a string (AND) or an array of strings (OR)

func (*Dependency) GetAllDependencies added in v0.0.10

func (d *Dependency) GetAllDependencies() []string

GetAllDependencies returns all dependency IDs from this dependency (flattened)

func (Dependency) MarshalYAML added in v0.0.10

func (d Dependency) MarshalYAML() (interface{}, error)

MarshalYAML implements custom YAML marshaling for mixed dependency types

func (*Dependency) UnmarshalYAML added in v0.0.10

func (d *Dependency) UnmarshalYAML(node *yaml.Node) error

UnmarshalYAML implements custom YAML unmarshaling for mixed dependency types

type Exec

type Exec struct {
	Config `yaml:",inline"`
	Exec   string `yaml:"exec"`
}

Exec represents a transformation exec model with YAML config

func NewTransformationExec

func NewTransformationExec(content []byte) (*Exec, error)

NewTransformationExec creates a new transformation exec model from content

func (*Exec) GetConfig

func (c *Exec) GetConfig() *Config

GetConfig returns the transformation model configuration

func (*Exec) GetType

func (c *Exec) GetType() string

GetType returns the transformation model type

func (*Exec) GetValue

func (c *Exec) GetValue() string

GetValue returns the exec command

func (*Exec) SetDefaultDatabase added in v0.0.8

func (c *Exec) SetDefaultDatabase(defaultDB string)

SetDefaultDatabase applies the default database if not already set

func (*Exec) Validate

func (c *Exec) Validate() error

Validate checks if the transformation exec model is valid

type ExecParser

type ExecParser struct{}

ExecParser parses exec transformation models

type IntervalConfig added in v0.0.7

type IntervalConfig struct {
	Max uint64 `yaml:"max"` // Maximum interval size for processing
	Min uint64 `yaml:"min"` // Minimum interval size (0 = allow any partial size)
}

IntervalConfig defines interval configuration for transformations

func (*IntervalConfig) Validate added in v0.0.7

func (c *IntervalConfig) Validate() error

Validate checks if the interval configuration is valid

type LimitsConfig added in v0.0.2

type LimitsConfig struct {
	Min uint64 `yaml:"min,omitempty"`
	Max uint64 `yaml:"max,omitempty"`
}

LimitsConfig defines position limits for transformations

func (*LimitsConfig) Validate added in v0.0.2

func (c *LimitsConfig) Validate() error

Validate checks if the limits configuration is valid

type SQL

type SQL struct {
	Config  `yaml:",inline"`
	Content string `yaml:"-"`
}

SQL represents a transformation SQL model with YAML frontmatter

func NewTransformationSQL

func NewTransformationSQL(content []byte) (*SQL, error)

NewTransformationSQL creates a new transformation SQL model from content

func (*SQL) GetConfig

func (c *SQL) GetConfig() *Config

GetConfig returns the transformation model configuration

func (*SQL) GetType

func (c *SQL) GetType() string

GetType returns the transformation model type

func (*SQL) GetValue

func (c *SQL) GetValue() string

GetValue returns the SQL content

func (*SQL) SetDefaultDatabase added in v0.0.8

func (c *SQL) SetDefaultDatabase(defaultDB string)

SetDefaultDatabase applies the default database if not already set

func (*SQL) Validate

func (c *SQL) Validate() error

Validate checks if the transformation SQL model is valid

type SQLParser

type SQLParser struct{}

SQLParser parses SQL transformation models

type SchedulesConfig added in v0.0.7

type SchedulesConfig struct {
	ForwardFill string `yaml:"forwardfill,omitempty"` // Forward fill schedule (optional)
	Backfill    string `yaml:"backfill,omitempty"`    // Backfill schedule (optional)
}

SchedulesConfig defines scheduling configuration for transformations

func (*SchedulesConfig) Validate added in v0.0.7

func (c *SchedulesConfig) Validate() error

Validate checks if the schedules configuration is valid

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL