models

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 19, 2025 License: GPL-3.0 Imports: 16 Imported by: 0

Documentation

Overview

Package models provides template rendering functionality for model SQL transformations

Index

Constants

View Source 
const (
	// TransformationTypeSQL represents SQL transformation models
	TransformationTypeSQL TransformationType = transformation.TransformationTypeSQL
	// TransformationTypeExec represents exec transformation models
	TransformationTypeExec TransformationType = transformation.TransformationTypeExec

	// ExtSQL is the SQL file extension
	ExtSQL = ".sql"
	// ExtYAML is the YAML file extension
	ExtYAML = ".yaml"
	// ExtYML is the alternative YAML file extension
	ExtYML = ".yml"
)

Variables

View Source 
var (
	// ErrNonExistentDependency is returned when a model depends on a non-existent model
	ErrNonExistentDependency = errors.New("model depends on non-existent model")
	// ErrInconsistentGraph is returned when the dependency graph is inconsistent
	ErrInconsistentGraph = errors.New("dependency graph is inconsistent: vertex count mismatch")
	// ErrNotExternalModel is returned when a model is not an external model
	ErrNotExternalModel = errors.New("model is not an external model")
	// ErrNotTransformationModel is returned when a model is not a transformation model
	ErrNotTransformationModel = errors.New("model is not a transformation model")
	// ErrInvalidNodeType is returned when a node has an invalid type
	ErrInvalidNodeType = errors.New("invalid node type")
	// ErrInvalidExternalModelType is returned when an external model has an invalid type
	ErrInvalidExternalModelType = errors.New("invalid external model type")
	// ErrInvalidTransformationModelType is returned when a transformation model has an invalid type
	ErrInvalidTransformationModelType = errors.New("invalid transformation model type")
)
View Source 
var (
	// ErrInvalidExternalType is returned when an invalid external type is specified
	ErrInvalidExternalType = errors.New("invalid external type")
)
View Source 
var (
	// ErrInvalidTransformationType is returned when an invalid transformation type is specified
	ErrInvalidTransformationType = errors.New("invalid transformation type")
)

Functions

This section is empty.

Types

type Config 

type Config struct {
	External       ExternalPathsConfig       `yaml:"external"`
	Transformation TransformationPathsConfig `yaml:"transformation"`
}

Config represents the complete coordinator configuration

func (*Config) Validate 

func (c *Config) Validate() error

Validate validates and sets defaults for the configuration

type DependencyGraph 

type DependencyGraph struct {
	// contains filtered or unexported fields
}

DependencyGraph manages the dependency graph for models

func NewDependencyGraph 

func NewDependencyGraph() *DependencyGraph

NewDependencyGraph creates a new dependency graph

func (*DependencyGraph) AddExternalModels 

func (d *DependencyGraph) AddExternalModels(models []External) error

AddExternalModels adds external models to the dependency graph

func (*DependencyGraph) AddTransformationEdges 

func (d *DependencyGraph) AddTransformationEdges(models []Transformation) error

AddTransformationEdges adds edges between transformation models based on dependencies

func (*DependencyGraph) AddTransformationModels 

func (d *DependencyGraph) AddTransformationModels(models []Transformation) error

AddTransformationModels adds transformation models to the dependency graph

func (*DependencyGraph) BuildGraph 

func (d *DependencyGraph) BuildGraph(transformationModels []Transformation, externalModels []External) error

BuildGraph builds the dependency graph from model configurations

func (*DependencyGraph) GetAllDependencies 

func (d *DependencyGraph) GetAllDependencies(modelID string) []string

GetAllDependencies returns all dependencies (recursive) of a model

func (*DependencyGraph) GetAllDependents 

func (d *DependencyGraph) GetAllDependents(modelID string) []string

GetAllDependents returns all dependents (recursive) of a model

func (*DependencyGraph) GetDependencies 

func (d *DependencyGraph) GetDependencies(modelID string) []string

GetDependencies returns the direct dependencies of a model

func (*DependencyGraph) GetDependents 

func (d *DependencyGraph) GetDependents(modelID string) []string

GetDependents returns the direct dependents of a model

func (*DependencyGraph) GetExternalNode 

func (d *DependencyGraph) GetExternalNode(modelID string) (External, error)

GetExternalNode retrieves an external model node from the dependency graph

func (*DependencyGraph) GetNode 

func (d *DependencyGraph) GetNode(modelID string) (Node, error)

GetNode retrieves a node from the dependency graph by model ID

func (*DependencyGraph) GetTransformationNode 

func (d *DependencyGraph) GetTransformationNode(modelID string) (Transformation, error)

GetTransformationNode retrieves a transformation model node from the dependency graph

func (*DependencyGraph) GetTransformationNodes 

func (d *DependencyGraph) GetTransformationNodes() []Transformation

GetTransformationNodes returns all transformation nodes from the dependency graph

func (*DependencyGraph) IsPathBetween 

func (d *DependencyGraph) IsPathBetween(fromModelID, toModelID string) bool

IsPathBetween checks if there's a path from one model to another

type External 

type External interface {
	GetType() string
	GetID() string
	GetConfig() external.Config
	GetValue() string
}

External defines the interface for external models

func NewExternal 

func NewExternal(content []byte, filePath string) (External, error)

NewExternal creates a new external model from file content

type ExternalPathsConfig 

type ExternalPathsConfig struct {
	Paths []string `yaml:"paths"`
}

ExternalPathsConfig defines paths for external model discovery

func (*ExternalPathsConfig) SetDefaults 

func (c *ExternalPathsConfig) SetDefaults()

SetDefaults sets default paths for external models

type ExternalType 

type ExternalType string

ExternalType represents the type of an external model 

const (
	// ExternalTypeSQL represents SQL external model type
	ExternalTypeSQL ExternalType = external.ExternalTypeSQL
)

type ModelFile 

type ModelFile struct {
	FilePath  string
	Extension string
	Content   []byte
}

ModelFile represents a discovered model file with its content

func DiscoverPaths 

func DiscoverPaths(directories []string) ([]*ModelFile, error)

DiscoverPaths walks directories to find model files (.sql, .yaml, .yml)

type Node 

type Node struct {
	NodeType NodeType
	Model    interface{}
}

Node represents a node in the dependency graph

type NodeType 

type NodeType string

NodeType represents the type of a node in the dependency graph 

const (
	// NodeTypeTransformation represents a transformation model node
	NodeTypeTransformation NodeType = "transformation"
	// NodeTypeExternal represents an external model node
	NodeTypeExternal NodeType = "external"
)

type Service 

type Service struct {
	// contains filtered or unexported fields
}

Service encapsulates the worker application logic

func NewService 

func NewService(log *logrus.Logger, cfg *Config, _ *redis.Client, clickhouseCfg *clickhouse.Config) (*Service, error)

NewService creates a new worker application

func (*Service) GetDAG 

func (s *Service) GetDAG() *DependencyGraph

GetDAG returns the dependency graph

func (*Service) GetTransformationEnvironmentVariables 

func (s *Service) GetTransformationEnvironmentVariables(model Transformation, position, interval uint64, startTime time.Time) (*[]string, error)

GetTransformationEnvironmentVariables returns environment variables for a transformation

func (*Service) RenderExternal 

func (s *Service) RenderExternal(model External) (string, error)

RenderExternal renders an external model template with variables

func (*Service) RenderTransformation 

func (s *Service) RenderTransformation(model Transformation, position, interval uint64, startTime time.Time) (string, error)

RenderTransformation renders a transformation model template with variables

func (*Service) Start 

func (s *Service) Start() error

Start initializes the models service and builds the dependency graph

func (*Service) Stop 

func (s *Service) Stop() error

Stop gracefully shuts down the models service

type TemplateEngine 

type TemplateEngine struct {
	// contains filtered or unexported fields
}

TemplateEngine provides template rendering with Sprig functions

func NewTemplateEngine 

func NewTemplateEngine(clickhouseCfg *clickhouse.Config, dag *DependencyGraph) *TemplateEngine

NewTemplateEngine creates a new template engine for rendering models

func (*TemplateEngine) GetTransformationEnvironmentVariables 

func (t *TemplateEngine) GetTransformationEnvironmentVariables(model Transformation, position, interval uint64, startTime time.Time) (*[]string, error)

GetTransformationEnvironmentVariables builds environment variables for transformation execution

func (*TemplateEngine) RenderExternal 

func (t *TemplateEngine) RenderExternal(model External) (string, error)

RenderExternal renders an external model template with variables

func (*TemplateEngine) RenderTransformation 

func (t *TemplateEngine) RenderTransformation(model Transformation, position, interval uint64, startTime time.Time) (string, error)

RenderTransformation renders a transformation model template with variables

type Transformation 

type Transformation interface {
	GetType() string
	GetID() string
	GetConfig() transformation.Config
	GetValue() string
}

Transformation defines the interface for transformation models

func NewTransformation 

func NewTransformation(content []byte, filePath string) (Transformation, error)

NewTransformation creates a new transformation model from file content

type TransformationPathsConfig 

type TransformationPathsConfig struct {
	Paths []string `yaml:"paths"`
}

TransformationPathsConfig defines paths for transformation model discovery

func (*TransformationPathsConfig) SetDefaults 

func (c *TransformationPathsConfig) SetDefaults()

SetDefaults sets default paths for transformation models

type TransformationType 

type TransformationType string

TransformationType defines the type of transformation model

Source Files

View all Source files

Directories

Path Synopsis
Package external provides external model configuration and validation
 Package external provides external model configuration and validation
Package transformation provides transformation model configuration and validation
 Package transformation provides transformation model configuration and validation

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.