models

package
v0.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 25, 2025 License: GPL-3.0 Imports: 16 Imported by: 0

Documentation

Overview

Package models provides template rendering functionality for model SQL transformations

Index

Constants

View Source
const (
	// TransformationTypeSQL represents SQL transformation models
	TransformationTypeSQL TransformationType = transformation.TransformationTypeSQL
	// TransformationTypeExec represents exec transformation models
	TransformationTypeExec TransformationType = transformation.TransformationTypeExec

	// ExtSQL is the SQL file extension
	ExtSQL = ".sql"
	// ExtYAML is the YAML file extension
	ExtYAML = ".yaml"
	// ExtYML is the alternative YAML file extension
	ExtYML = ".yml"
)

Variables

View Source
var (
	// ErrNonExistentDependency is returned when a model depends on a non-existent model
	ErrNonExistentDependency = errors.New("model depends on non-existent model")
	// ErrInconsistentGraph is returned when the dependency graph is inconsistent
	ErrInconsistentGraph = errors.New("dependency graph is inconsistent: vertex count mismatch")
	// ErrNotExternalModel is returned when a model is not an external model
	ErrNotExternalModel = errors.New("model is not an external model")
	// ErrNotTransformationModel is returned when a model is not a transformation model
	ErrNotTransformationModel = errors.New("model is not a transformation model")
	// ErrInvalidNodeType is returned when a node has an invalid type
	ErrInvalidNodeType = errors.New("invalid node type")
	// ErrInvalidExternalModelType is returned when an external model has an invalid type
	ErrInvalidExternalModelType = errors.New("invalid external model type")
	// ErrInvalidTransformationModelType is returned when a transformation model has an invalid type
	ErrInvalidTransformationModelType = errors.New("invalid transformation model type")
)
View Source
var (
	// ErrInvalidExternalType is returned when an invalid external type is specified
	ErrInvalidExternalType = errors.New("invalid external type")
)
View Source
var (
	// ErrInvalidTransformationType is returned when an invalid transformation type is specified
	ErrInvalidTransformationType = errors.New("invalid transformation type")
)

Functions

This section is empty.

Types

type Config

type Config struct {
	External       ExternalPathsConfig       `yaml:"external"`
	Transformation TransformationPathsConfig `yaml:"transformations"`
}

Config represents the complete coordinator configuration

func (*Config) Validate

func (c *Config) Validate() error

Validate validates and sets defaults for the configuration

type DAGReader added in v0.0.2

type DAGReader interface {
	// GetNode retrieves a node by its ID
	GetNode(id string) (Node, error)

	// GetTransformationNode retrieves a transformation node by its ID
	GetTransformationNode(id string) (Transformation, error)

	// GetExternalNode retrieves an external node by its ID
	GetExternalNode(id string) (External, error)

	// GetDependencies returns direct dependencies of a node
	GetDependencies(id string) []string

	// GetDependents returns nodes that depend on the given node
	GetDependents(id string) []string

	// GetAllDependencies returns all transitive dependencies
	GetAllDependencies(id string) []string

	// GetAllDependents returns all transitive dependents
	GetAllDependents(id string) []string

	// GetTransformationNodes returns all transformation nodes
	GetTransformationNodes() []Transformation

	// GetExternalNodes returns all external nodes
	GetExternalNodes() []Node

	// IsPathBetween checks if there's a path between two nodes
	IsPathBetween(from, to string) bool
}

DAGReader provides read-only access to the dependency graph (ethPandaOps pattern)

type DependencyGraph

type DependencyGraph struct {
	// contains filtered or unexported fields
}

DependencyGraph manages the dependency graph for models

func NewDependencyGraph

func NewDependencyGraph() *DependencyGraph

NewDependencyGraph creates a new dependency graph

func (*DependencyGraph) AddExternalModels

func (d *DependencyGraph) AddExternalModels(models []External) error

AddExternalModels adds external models to the dependency graph

func (*DependencyGraph) AddTransformationEdges

func (d *DependencyGraph) AddTransformationEdges(models []Transformation) error

AddTransformationEdges adds edges between transformation models based on dependencies

func (*DependencyGraph) AddTransformationModels

func (d *DependencyGraph) AddTransformationModels(models []Transformation) error

AddTransformationModels adds transformation models to the dependency graph

func (*DependencyGraph) BuildGraph

func (d *DependencyGraph) BuildGraph(transformationModels []Transformation, externalModels []External) error

BuildGraph builds the dependency graph from model configurations

func (*DependencyGraph) GetAllDependencies

func (d *DependencyGraph) GetAllDependencies(modelID string) []string

GetAllDependencies returns all dependencies (recursive) of a model

func (*DependencyGraph) GetAllDependents

func (d *DependencyGraph) GetAllDependents(modelID string) []string

GetAllDependents returns all dependents (recursive) of a model

func (*DependencyGraph) GetDependencies

func (d *DependencyGraph) GetDependencies(modelID string) []string

GetDependencies returns the direct dependencies of a model

func (*DependencyGraph) GetDependents

func (d *DependencyGraph) GetDependents(modelID string) []string

GetDependents returns the direct dependents of a model

func (*DependencyGraph) GetExternalNode

func (d *DependencyGraph) GetExternalNode(modelID string) (External, error)

GetExternalNode retrieves an external model node from the dependency graph

func (*DependencyGraph) GetExternalNodes added in v0.0.2

func (d *DependencyGraph) GetExternalNodes() []Node

GetExternalNodes returns all external nodes from the dependency graph

func (*DependencyGraph) GetNode

func (d *DependencyGraph) GetNode(modelID string) (Node, error)

GetNode retrieves a node from the dependency graph by model ID

func (*DependencyGraph) GetTransformationNode

func (d *DependencyGraph) GetTransformationNode(modelID string) (Transformation, error)

GetTransformationNode retrieves a transformation model node from the dependency graph

func (*DependencyGraph) GetTransformationNodes

func (d *DependencyGraph) GetTransformationNodes() []Transformation

GetTransformationNodes returns all transformation nodes from the dependency graph

func (*DependencyGraph) IsPathBetween

func (d *DependencyGraph) IsPathBetween(fromModelID, toModelID string) bool

IsPathBetween checks if there's a path from one model to another

type External

type External interface {
	GetType() string
	GetID() string
	GetConfig() external.Config
	GetValue() string
}

External defines the interface for external models

func NewExternal

func NewExternal(content []byte, filePath string) (External, error)

NewExternal creates a new external model from file content

type ExternalPathsConfig

type ExternalPathsConfig struct {
	Paths []string `yaml:"paths"`
}

ExternalPathsConfig defines paths for external model discovery

func (*ExternalPathsConfig) SetDefaults

func (c *ExternalPathsConfig) SetDefaults()

SetDefaults sets default paths for external models

type ExternalType

type ExternalType string

ExternalType represents the type of an external model

const (
	// ExternalTypeSQL represents SQL external model type
	ExternalTypeSQL ExternalType = external.ExternalTypeSQL
)

type ModelFile

type ModelFile struct {
	FilePath  string
	Extension string
	Content   []byte
}

ModelFile represents a discovered model file with its content

func DiscoverPaths

func DiscoverPaths(directories []string) ([]*ModelFile, error)

DiscoverPaths walks directories to find model files (.sql, .yaml, .yml)

type Node

type Node struct {
	NodeType NodeType
	Model    interface{}
}

Node represents a node in the dependency graph

type NodeType

type NodeType string

NodeType represents the type of a node in the dependency graph

const (
	// NodeTypeTransformation represents a transformation model node
	NodeTypeTransformation NodeType = "transformation"
	// NodeTypeExternal represents an external model node
	NodeTypeExternal NodeType = "external"
)

type Service

type Service interface {
	// Lifecycle methods
	Start() error
	Stop() error

	// DAG operations
	GetDAG() DAGReader

	// Rendering operations
	RenderTransformation(model Transformation, position, interval uint64, startTime time.Time) (string, error)
	RenderExternal(model External, cacheState map[string]interface{}) (string, error)
	GetTransformationEnvironmentVariables(model Transformation, position, interval uint64, startTime time.Time) (*[]string, error)
}

Service defines the interface for the models service (ethPandaOps pattern)

func NewService

func NewService(log logrus.FieldLogger, cfg *Config, _ *redis.Client, clickhouseCfg *clickhouse.Config) (Service, error)

NewService creates a new worker application

type TemplateEngine

type TemplateEngine struct {
	// contains filtered or unexported fields
}

TemplateEngine provides template rendering with Sprig functions

func NewTemplateEngine

func NewTemplateEngine(clickhouseCfg *clickhouse.Config, dag *DependencyGraph) *TemplateEngine

NewTemplateEngine creates a new template engine for rendering models

func (*TemplateEngine) GetTransformationEnvironmentVariables

func (t *TemplateEngine) GetTransformationEnvironmentVariables(model Transformation, position, interval uint64, startTime time.Time) (*[]string, error)

GetTransformationEnvironmentVariables builds environment variables for transformation execution

func (*TemplateEngine) RenderExternal

func (t *TemplateEngine) RenderExternal(model External, cacheState map[string]interface{}) (string, error)

RenderExternal renders an external model template with variables

func (*TemplateEngine) RenderTransformation

func (t *TemplateEngine) RenderTransformation(model Transformation, position, interval uint64, startTime time.Time) (string, error)

RenderTransformation renders a transformation model template with variables

type Transformation

type Transformation interface {
	GetType() string
	GetID() string
	GetConfig() *transformation.Config
	GetValue() string
}

Transformation defines the interface for transformation models

func NewTransformation

func NewTransformation(content []byte, filePath string) (Transformation, error)

NewTransformation creates a new transformation model from file content

type TransformationPathsConfig

type TransformationPathsConfig struct {
	Paths []string `yaml:"paths"`
}

TransformationPathsConfig defines paths for transformation model discovery

func (*TransformationPathsConfig) SetDefaults

func (c *TransformationPathsConfig) SetDefaults()

SetDefaults sets default paths for transformation models

type TransformationType

type TransformationType string

TransformationType defines the type of transformation model

Directories

Path Synopsis
Package external provides external model configuration and validation
Package external provides external model configuration and validation
Package transformation provides transformation model configuration and validation
Package transformation provides transformation model configuration and validation

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL