Documentation
¶
Overview ¶
Package models provides template rendering functionality for model SQL transformations
Index ¶
- Constants
- Variables
- type Config
- type DAGReader
- type DependencyGraph
- func (d *DependencyGraph) AddExternalModels(models []External) error
- func (d *DependencyGraph) AddTransformationEdges(models []Transformation) error
- func (d *DependencyGraph) AddTransformationModels(models []Transformation) error
- func (d *DependencyGraph) BuildGraph(transformationModels []Transformation, externalModels []External) error
- func (d *DependencyGraph) GetAllDependencies(modelID string) []string
- func (d *DependencyGraph) GetAllDependents(modelID string) []string
- func (d *DependencyGraph) GetDependencies(modelID string) []string
- func (d *DependencyGraph) GetDependents(modelID string) []string
- func (d *DependencyGraph) GetExternalNode(modelID string) (External, error)
- func (d *DependencyGraph) GetExternalNodes() []Node
- func (d *DependencyGraph) GetNode(modelID string) (Node, error)
- func (d *DependencyGraph) GetTransformationNode(modelID string) (Transformation, error)
- func (d *DependencyGraph) GetTransformationNodes() []Transformation
- func (d *DependencyGraph) IsPathBetween(fromModelID, toModelID string) bool
- type External
- type ExternalConfig
- type ExternalType
- type ModelFile
- type Node
- type NodeType
- type Service
- type TemplateEngine
- func (t *TemplateEngine) GetTransformationEnvironmentVariables(model Transformation, position, interval uint64, startTime time.Time) (*[]string, error)
- func (t *TemplateEngine) RenderExternal(model External, cacheState map[string]interface{}) (string, error)
- func (t *TemplateEngine) RenderTransformation(model Transformation, position, interval uint64, startTime time.Time) (string, error)
- type Transformation
- type TransformationConfig
- type TransformationType
Constants ¶
const ( // TransformationTypeSQL represents SQL transformation models TransformationTypeSQL TransformationType = transformation.TransformationTypeSQL // TransformationTypeExec represents exec transformation models TransformationTypeExec TransformationType = transformation.TransformationTypeExec // ExtSQL is the SQL file extension ExtSQL = ".sql" // ExtYAML is the YAML file extension ExtYAML = ".yaml" // ExtYML is the alternative YAML file extension ExtYML = ".yml" )
Variables ¶
var ( // ErrNonExistentDependency is returned when a model depends on a non-existent model ErrNonExistentDependency = errors.New("model depends on non-existent model") // ErrInconsistentGraph is returned when the dependency graph is inconsistent ErrInconsistentGraph = errors.New("dependency graph is inconsistent: vertex count mismatch") // ErrNotExternalModel is returned when a model is not an external model ErrNotExternalModel = errors.New("model is not an external model") // ErrNotTransformationModel is returned when a model is not a transformation model ErrNotTransformationModel = errors.New("model is not a transformation model") // ErrInvalidNodeType is returned when a node has an invalid type ErrInvalidNodeType = errors.New("invalid node type") // ErrInvalidExternalModelType is returned when an external model has an invalid type ErrInvalidExternalModelType = errors.New("invalid external model type") // ErrInvalidTransformationModelType is returned when a transformation model has an invalid type ErrInvalidTransformationModelType = errors.New("invalid transformation model type") )
var ( // ErrInvalidExternalType is returned when an invalid external type is specified ErrInvalidExternalType = errors.New("invalid external type") )
var ( // ErrInvalidTransformationType is returned when an invalid transformation type is specified ErrInvalidTransformationType = errors.New("invalid transformation type") )
Functions ¶
This section is empty.
Types ¶
type Config ¶
type Config struct {
External ExternalConfig `yaml:"external"`
Transformation TransformationConfig `yaml:"transformations"`
}
Config represents the complete coordinator configuration
type DAGReader ¶ added in v0.0.2
type DAGReader interface {
// GetNode retrieves a node by its ID
GetNode(id string) (Node, error)
// GetTransformationNode retrieves a transformation node by its ID
GetTransformationNode(id string) (Transformation, error)
// GetExternalNode retrieves an external node by its ID
GetExternalNode(id string) (External, error)
// GetDependencies returns direct dependencies of a node
GetDependencies(id string) []string
// GetDependents returns nodes that depend on the given node
GetDependents(id string) []string
// GetAllDependencies returns all transitive dependencies
GetAllDependencies(id string) []string
// GetAllDependents returns all transitive dependents
GetAllDependents(id string) []string
// GetTransformationNodes returns all transformation nodes
GetTransformationNodes() []Transformation
// GetExternalNodes returns all external nodes
GetExternalNodes() []Node
// IsPathBetween checks if there's a path between two nodes
IsPathBetween(from, to string) bool
}
DAGReader provides read-only access to the dependency graph (ethPandaOps pattern)
type DependencyGraph ¶
type DependencyGraph struct {
// contains filtered or unexported fields
}
DependencyGraph manages the dependency graph for models
func NewDependencyGraph ¶
func NewDependencyGraph() *DependencyGraph
NewDependencyGraph creates a new dependency graph
func (*DependencyGraph) AddExternalModels ¶
func (d *DependencyGraph) AddExternalModels(models []External) error
AddExternalModels adds external models to the dependency graph
func (*DependencyGraph) AddTransformationEdges ¶
func (d *DependencyGraph) AddTransformationEdges(models []Transformation) error
AddTransformationEdges adds edges between transformation models based on dependencies
func (*DependencyGraph) AddTransformationModels ¶
func (d *DependencyGraph) AddTransformationModels(models []Transformation) error
AddTransformationModels adds transformation models to the dependency graph
func (*DependencyGraph) BuildGraph ¶
func (d *DependencyGraph) BuildGraph(transformationModels []Transformation, externalModels []External) error
BuildGraph builds the dependency graph from model configurations
func (*DependencyGraph) GetAllDependencies ¶
func (d *DependencyGraph) GetAllDependencies(modelID string) []string
GetAllDependencies returns all dependencies (recursive) of a model
func (*DependencyGraph) GetAllDependents ¶
func (d *DependencyGraph) GetAllDependents(modelID string) []string
GetAllDependents returns all dependents (recursive) of a model
func (*DependencyGraph) GetDependencies ¶
func (d *DependencyGraph) GetDependencies(modelID string) []string
GetDependencies returns the direct dependencies of a model Note: This returns all dependencies flattened (including those in OR groups)
func (*DependencyGraph) GetDependents ¶
func (d *DependencyGraph) GetDependents(modelID string) []string
GetDependents returns the direct dependents of a model
func (*DependencyGraph) GetExternalNode ¶
func (d *DependencyGraph) GetExternalNode(modelID string) (External, error)
GetExternalNode retrieves an external model node from the dependency graph
func (*DependencyGraph) GetExternalNodes ¶ added in v0.0.2
func (d *DependencyGraph) GetExternalNodes() []Node
GetExternalNodes returns all external nodes from the dependency graph
func (*DependencyGraph) GetNode ¶
func (d *DependencyGraph) GetNode(modelID string) (Node, error)
GetNode retrieves a node from the dependency graph by model ID
func (*DependencyGraph) GetTransformationNode ¶
func (d *DependencyGraph) GetTransformationNode(modelID string) (Transformation, error)
GetTransformationNode retrieves a transformation model node from the dependency graph
func (*DependencyGraph) GetTransformationNodes ¶
func (d *DependencyGraph) GetTransformationNodes() []Transformation
GetTransformationNodes returns all transformation nodes from the dependency graph
func (*DependencyGraph) IsPathBetween ¶
func (d *DependencyGraph) IsPathBetween(fromModelID, toModelID string) bool
IsPathBetween checks if there's a path from one model to another
type External ¶
type External interface {
GetType() string
GetID() string
GetConfig() external.Config
GetValue() string
SetDefaultDatabase(defaultDB string)
}
External defines the interface for external models
type ExternalConfig ¶ added in v0.0.8
type ExternalConfig struct {
Paths []string `yaml:"paths"`
DefaultDatabase string `yaml:"defaultDatabase"`
}
ExternalConfig defines configuration for external models
func (*ExternalConfig) SetDefaults ¶ added in v0.0.8
func (c *ExternalConfig) SetDefaults()
SetDefaults sets default paths for external models
type ExternalType ¶
type ExternalType string
ExternalType represents the type of an external model
const ( // ExternalTypeSQL represents SQL external model type ExternalTypeSQL ExternalType = external.ExternalTypeSQL )
type ModelFile ¶
ModelFile represents a discovered model file with its content
func DiscoverPaths ¶
DiscoverPaths walks directories to find model files (.sql, .yaml, .yml)
type Node ¶
type Node struct {
NodeType NodeType
Model interface{}
}
Node represents a node in the dependency graph
type Service ¶
type Service interface {
// Lifecycle methods
Start() error
Stop() error
// DAG operations
GetDAG() DAGReader
// Rendering operations
RenderTransformation(model Transformation, position, interval uint64, startTime time.Time) (string, error)
RenderExternal(model External, cacheState map[string]interface{}) (string, error)
GetTransformationEnvironmentVariables(model Transformation, position, interval uint64, startTime time.Time) (*[]string, error)
}
Service defines the interface for the models service (ethPandaOps pattern)
func NewService ¶
func NewService(log logrus.FieldLogger, cfg *Config, _ *redis.Client, clickhouseCfg *clickhouse.Config) (Service, error)
NewService creates a new worker application
type TemplateEngine ¶
type TemplateEngine struct {
// contains filtered or unexported fields
}
TemplateEngine provides template rendering with Sprig functions
func NewTemplateEngine ¶
func NewTemplateEngine(clickhouseCfg *clickhouse.Config, dag *DependencyGraph) *TemplateEngine
NewTemplateEngine creates a new template engine for rendering models
func (*TemplateEngine) GetTransformationEnvironmentVariables ¶
func (t *TemplateEngine) GetTransformationEnvironmentVariables(model Transformation, position, interval uint64, startTime time.Time) (*[]string, error)
GetTransformationEnvironmentVariables builds environment variables for transformation execution
func (*TemplateEngine) RenderExternal ¶
func (t *TemplateEngine) RenderExternal(model External, cacheState map[string]interface{}) (string, error)
RenderExternal renders an external model template with variables
func (*TemplateEngine) RenderTransformation ¶
func (t *TemplateEngine) RenderTransformation(model Transformation, position, interval uint64, startTime time.Time) (string, error)
RenderTransformation renders a transformation model template with variables
type Transformation ¶
type Transformation interface {
GetType() string
GetID() string
GetConfig() *transformation.Config
GetValue() string
SetDefaultDatabase(defaultDB string)
}
Transformation defines the interface for transformation models
func NewTransformation ¶
func NewTransformation(content []byte, filePath string) (Transformation, error)
NewTransformation creates a new transformation model from file content
type TransformationConfig ¶ added in v0.0.8
type TransformationConfig struct {
Paths []string `yaml:"paths"`
DefaultDatabase string `yaml:"defaultDatabase"`
}
TransformationConfig defines configuration for transformation models
func (*TransformationConfig) SetDefaults ¶ added in v0.0.8
func (c *TransformationConfig) SetDefaults()
SetDefaults sets default paths for transformation models
type TransformationType ¶
type TransformationType string
TransformationType defines the type of transformation model
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
Package external provides external model configuration and validation
|
Package external provides external model configuration and validation |
|
Package transformation provides transformation model configuration and validation
|
Package transformation provides transformation model configuration and validation |