transformation

package
v0.1.3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 27, 2026 License: GPL-3.0 Imports: 11 Imported by: 0

Documentation

Overview

Package transformation provides transformation model configuration and validation

Index

Constants

View Source
const TransformationTypeExec = "exec"

TransformationTypeExec identifies exec transformation models

View Source
const TransformationTypeSQL = "sql"

TransformationTypeSQL identifies SQL transformation models

Variables

View Source
var (
	// ErrDatabaseRequired is returned when database is not specified
	ErrDatabaseRequired = errors.New("database is required")
	// ErrTableRequired is returned when table is not specified
	ErrTableRequired = errors.New("table is required")
	// ErrUnknownTransformationType is returned for unknown transformation types
	ErrUnknownTransformationType = errors.New("unknown transformation type")
)
View Source
var (
	// ErrInvalidDependencyType is returned when dependency has invalid YAML type
	ErrInvalidDependencyType = errors.New("dependency must be a string or array of strings")
	// ErrInvalidDependencyArrayItem is returned when dependency array contains non-string
	ErrInvalidDependencyArrayItem = errors.New("expected string in dependency array")
	// ErrEmptyDependencyGroup is returned when dependency group is empty
	ErrEmptyDependencyGroup = errors.New("dependency group cannot be empty")
)
View Source
var (
	// ErrInvalidFrontmatter is returned when frontmatter is invalid
	ErrInvalidFrontmatter = errors.New("invalid frontmatter")
	// ErrSQLContentRequired is returned when SQL content is not specified
	ErrSQLContentRequired = errors.New("sql content is required")
)
View Source
var (
	// ErrExecRequired is returned when exec is not specified
	ErrExecRequired = errors.New("exec is required")
)
View Source
var (
	// ErrHandlerNotRegistered is returned when no handler is registered for a type
	ErrHandlerNotRegistered = errors.New("no handler registered for type")
)

Functions

func ApplyMinMaxOverride added in v0.0.49

func ApplyMinMaxOverride(fieldName string, existingMin, existingMax uint64, v reflect.Value) (minVal, maxVal uint64, found bool)

ApplyMinMaxOverride extracts Min/Max uint64 overrides from a named nested struct field. Returns the updated min and max values, and whether any override was found. Used for Interval and Limits overrides in incremental transformations.

func ApplyScheduleOverride added in v0.0.49

func ApplyScheduleOverride(existingSchedule string, v reflect.Value) string

ApplyScheduleOverride extracts a schedule override from a reflect.Value and returns the new schedule if present, or the existing schedule if not. This is shared between transformation handlers that support schedule overrides.

func ApplySchedulesOverride added in v0.0.49

func ApplySchedulesOverride(existingForwardFill, existingBackfill string, v reflect.Value) (forwardFill, backfill string)

ApplySchedulesOverride extracts ForwardFill/Backfill schedule overrides from a reflect.Value. Returns the updated forwardFill and backfill values. Used for incremental transformation schedule overrides.

func ApplyTagsOverride added in v0.0.49

func ApplyTagsOverride(existingTags []string, v reflect.Value) []string

ApplyTagsOverride appends override tags to existing tags using reflection. This is shared between incremental and scheduled transformation handlers.

func RegisterHandler added in v0.0.25

func RegisterHandler(transformationType Type, factory HandlerFactory)

RegisterHandler registers a handler factory for a transformation type

func ValidateScheduleFormat

func ValidateScheduleFormat(schedule string) error

ValidateScheduleFormat validates a cron schedule expression.

Types

type AdminTable added in v0.0.25

type AdminTable struct {
	Database string
	Table    string
}

AdminTable holds the configuration for admin tables

type Config

type Config struct {
	Type     Type              `yaml:"type"`     // Required: "incremental" or "scheduled"
	Database string            `yaml:"database"` // Optional, can fall back to default
	Table    string            `yaml:"table"`    // Required
	Env      map[string]string `yaml:"env,omitempty"`
}

Config is the minimal configuration used to determine the transformation type Each type has its own complete configuration structure

func (*Config) GetID

func (c *Config) GetID() string

GetID returns the unique identifier for the transformation model

func (*Config) IsIncrementalType added in v0.0.25

func (c *Config) IsIncrementalType() bool

IsIncrementalType returns true if this is an incremental transformation

func (*Config) IsScheduledType added in v0.0.25

func (c *Config) IsScheduledType() bool

IsScheduledType returns true if this is a scheduled transformation

func (*Config) SetDefaults added in v0.0.8

func (c *Config) SetDefaults(defaultDatabase string)

SetDefaults applies default values to the configuration

func (*Config) Validate

func (c *Config) Validate() error

Validate checks if the base configuration is valid

type Dependency added in v0.0.10

type Dependency struct {
	// IsGroup indicates if this is an OR group (array) or a single dependency (string)
	IsGroup bool
	// SingleDep holds the dependency ID for single dependencies
	SingleDep string
	// GroupDeps holds multiple dependency IDs for OR groups
	GroupDeps []string
}

Dependency represents a dependency that can be either a string (AND) or an array of strings (OR)

func DeepCopyDependencies added in v0.0.49

func DeepCopyDependencies(deps []Dependency) []Dependency

DeepCopyDependencies creates a deep copy of a slice of dependencies.

func SubstituteDependencyPlaceholders added in v0.0.49

func SubstituteDependencyPlaceholders(deps []Dependency, externalDB, transformationDB string) []Dependency

SubstituteDependencyPlaceholders replaces {{external}} and {{transformation}} placeholders in all dependencies. It returns the original dependencies (deep copy) and modifies the input slice in place.

func (*Dependency) GetAllDependencies added in v0.0.10

func (d *Dependency) GetAllDependencies() []string

GetAllDependencies returns all dependency IDs from this dependency (flattened)

func (*Dependency) UnmarshalYAML added in v0.0.10

func (d *Dependency) UnmarshalYAML(node *yaml.Node) error

UnmarshalYAML implements custom YAML unmarshaling for mixed dependency types

type DependencyHandler added in v0.0.49

type DependencyHandler interface {
	GetFlattenedDependencies() []string
	GetDependencies() []Dependency
}

DependencyHandler provides dependency information for transformations.

type Exec

type Exec struct {
	Config  `yaml:",inline"`
	Exec    string  `yaml:"exec"`
	Handler Handler `yaml:"-"` // Type-specific handler
}

Exec represents a transformation exec model with YAML config

func NewTransformationExec

func NewTransformationExec(content []byte) (*Exec, error)

NewTransformationExec creates a new transformation exec model from content

func (*Exec) GetConfig

func (c *Exec) GetConfig() *Config

GetConfig returns the transformation model configuration

func (*Exec) GetHandler added in v0.0.25

func (c *Exec) GetHandler() Handler

GetHandler returns the type-specific handler

func (*Exec) GetID added in v0.0.25

func (c *Exec) GetID() string

GetID returns the unique identifier

func (*Exec) GetType

func (c *Exec) GetType() string

GetType returns the transformation model type

func (*Exec) GetValue

func (c *Exec) GetValue() string

GetValue returns the exec command

func (*Exec) SetDefaultDatabase added in v0.0.8

func (c *Exec) SetDefaultDatabase(defaultDB string)

SetDefaultDatabase applies the default database if not already set

func (*Exec) Validate

func (c *Exec) Validate() error

Validate checks if the transformation exec model is valid

type FillHandler added in v0.0.49

type FillHandler interface {
	GetFillDirection() string
	AllowGapSkipping() bool
	GetFillBuffer() uint64
}

FillHandler provides fill configuration for transformations.

type Handler added in v0.0.25

type Handler interface {
	// Type returns the transformation type
	Type() Type

	// Config returns the typed configuration
	Config() any

	// Validate validates the configuration
	Validate() error

	// ShouldTrackPosition indicates if this type tracks positions
	ShouldTrackPosition() bool

	// GetTemplateVariables returns template variables for this transformation
	GetTemplateVariables(ctx context.Context, taskInfo TaskInfo) map[string]any

	// GetAdminTable returns the admin table configuration for this type
	GetAdminTable() AdminTable

	// RecordCompletion records the completion of a transformation
	RecordCompletion(ctx context.Context, adminService any, modelID string, taskInfo TaskInfo) error
}

Handler defines the interface for transformation type handlers

func CreateHandler added in v0.0.25

func CreateHandler(transformationType Type, data []byte, adminTable AdminTable) (Handler, error)

CreateHandler creates a handler for the given type

type HandlerFactory added in v0.0.25

type HandlerFactory func(data []byte, adminTable AdminTable) (Handler, error)

HandlerFactory is a function that creates a handler from YAML data

type IntervalHandler added in v0.0.49

type IntervalHandler interface {
	GetMinInterval() uint64
	GetMaxInterval() uint64
	AllowsPartialIntervals() bool
	AllowGapSkipping() bool
}

IntervalHandler provides interval configuration for transformations. Handlers that process data in intervals should implement this interface.

type Limits added in v0.0.49

type Limits struct {
	Min uint64
	Max uint64
}

Limits defines position limits for transformations.

type LimitsHandler added in v0.0.49

type LimitsHandler interface {
	GetLimits() *Limits
}

LimitsHandler provides position limits for transformations. This is a subset of ScheduleHandler for cases where only limits are needed.

type Registry added in v0.0.25

type Registry struct {
	// contains filtered or unexported fields
}

Registry manages handler factories for different transformation types

type SQL

type SQL struct {
	BaseConfig Config  `yaml:",inline"` // Base configuration
	Handler    Handler // Type-specific handler
	Content    string  `yaml:"-"` // SQL content
}

SQL represents a transformation SQL model with YAML frontmatter

func NewTransformationSQL

func NewTransformationSQL(content []byte) (*SQL, error)

NewTransformationSQL creates a new transformation SQL model from content

func (*SQL) GetConfig

func (c *SQL) GetConfig() *Config

GetConfig returns the base transformation model configuration

func (*SQL) GetHandler added in v0.0.25

func (c *SQL) GetHandler() Handler

GetHandler returns the type-specific handler

func (*SQL) GetID added in v0.0.25

func (c *SQL) GetID() string

GetID returns the unique identifier

func (*SQL) GetType

func (c *SQL) GetType() string

GetType returns the transformation model type

func (*SQL) GetValue

func (c *SQL) GetValue() string

GetValue returns the SQL content

func (*SQL) SetDefaultDatabase added in v0.0.8

func (c *SQL) SetDefaultDatabase(defaultDB string)

SetDefaultDatabase applies the default database if not already set

func (*SQL) Validate

func (c *SQL) Validate() error

Validate checks if the transformation SQL model is valid

type ScheduleHandler added in v0.0.49

type ScheduleHandler interface {
	IsBackfillEnabled() bool
	IsForwardFillEnabled() bool
	GetLimits() *Limits
}

ScheduleHandler provides scheduling and limits configuration for transformations. Handlers that have directional processing should implement this interface.

type TaskInfo added in v0.0.25

type TaskInfo struct {
	Position  uint64
	Interval  uint64
	Timestamp time.Time
	Direction string
}

TaskInfo contains information about the current task execution

type Type added in v0.0.25

type Type string

Type represents the transformation execution pattern

const (
	// TypeIncremental represents incremental transformations that track position
	TypeIncremental Type = "incremental"
	// TypeScheduled represents scheduled transformations without position tracking
	TypeScheduled Type = "scheduled"
)

Directories

Path Synopsis
Package incremental provides the incremental transformation type handler
Package incremental provides the incremental transformation type handler
Package mock is a generated GoMock package.
Package mock is a generated GoMock package.
Package scheduled provides the scheduled transformation type handler
Package scheduled provides the scheduled transformation type handler

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL