lineage

package
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 7, 2026 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	LineageEventsEmitted = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "caesium_lineage_events_emitted_total",
			Help: "Total number of OpenLineage events emitted by type and status.",
		},
		[]string{"event_type", "status"},
	)

	LineageEmitDuration = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "caesium_lineage_emit_duration_seconds",
			Help:    "Duration of OpenLineage event emission in seconds.",
			Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5},
		},
		[]string{"transport"},
	)
)

Functions

func RegisterMetrics

func RegisterMetrics()

Types

type BaseFacet

type BaseFacet struct {
	Producer  string `json:"_producer"`
	SchemaURL string `json:"_schemaURL"`
}

type CaesiumDAGFacet

type CaesiumDAGFacet struct {
	BaseFacet
	TotalTasks    int    `json:"totalTasks"`
	TriggerType   string `json:"triggerType,omitempty"`
	TriggerAlias  string `json:"triggerAlias,omitempty"`
	FailurePolicy string `json:"failurePolicy,omitempty"`
	ExecutionMode string `json:"executionMode,omitempty"`
}

type CaesiumDatasetFacet

type CaesiumDatasetFacet struct {
	BaseFacet
	// StepName is the task/step name that produced or consumed this dataset.
	StepName string `json:"stepName,omitempty"`
	// Direction is "input" or "output" from the step's perspective.
	Direction string `json:"direction"`
	// OutputKeys lists the structured-output keys (from ##caesium::output) whose
	// values contributed to this dataset's identity.  Empty when the dataset
	// was derived from a declared schema rather than structured output.
	OutputKeys []string `json:"outputKeys,omitempty"`
}

CaesiumDatasetFacet is a dataset-level facet carrying Caesium-specific lineage fields (step name, direction, and the structured output keys that produced or consumed this dataset). It is attached to every Dataset entry emitted in Inputs/Outputs of a task RunEvent.

type CaesiumExecutionFacet

type CaesiumExecutionFacet struct {
	BaseFacet
	Engine    string   `json:"engine"`
	Image     string   `json:"image"`
	Command   []string `json:"command,omitempty"`
	RuntimeID string   `json:"runtimeId,omitempty"`
	ClaimedBy string   `json:"claimedBy,omitempty"`
}

type CaesiumProvenanceFacet

type CaesiumProvenanceFacet struct {
	BaseFacet
	SourceID string `json:"sourceId,omitempty"`
	Repo     string `json:"repo,omitempty"`
	Ref      string `json:"ref,omitempty"`
	Commit   string `json:"commit,omitempty"`
	Path     string `json:"path,omitempty"`
}

type CaesiumSchemaFacet

type CaesiumSchemaFacet struct {
	BaseFacet
	// Schema is the raw JSON Schema object (map form) as declared in the job
	// manifest's outputSchema / inputSchema field for this step.
	Schema map[string]any `json:"schema,omitempty"`
}

CaesiumSchemaFacet carries the declared JSON Schema for a dataset so OpenLineage consumers can perform field-level compatibility checks.

type Config

type Config struct {
	Enabled       bool
	Transport     string
	URL           string
	Namespace     string
	Headers       string
	FilePath      string
	Timeout       time.Duration
	RetryAttempts uint
}

type Dataset

type Dataset struct {
	Namespace string         `json:"namespace"`
	Name      string         `json:"name"`
	Facets    map[string]any `json:"facets,omitempty"`
}

type ErrorMessageFacet

type ErrorMessageFacet struct {
	BaseFacet
	Message             string `json:"message"`
	ProgrammingLanguage string `json:"programmingLanguage,omitempty"`
	StackTrace          string `json:"stackTrace,omitempty"`
}

type EventType

type EventType string
const (
	EventTypeStart    EventType = "START"
	EventTypeRunning  EventType = "RUNNING"
	EventTypeComplete EventType = "COMPLETE"
	EventTypeFail     EventType = "FAIL"
	EventTypeAbort    EventType = "ABORT"
)

type HTTPTransportConfig

type HTTPTransportConfig struct {
	URL     string
	Headers map[string]string
	Timeout time.Duration
}

type ImpactNode

type ImpactNode struct {
	// DatasetNamespace and DatasetName are the OpenLineage identity of the
	// downstream dataset.
	DatasetNamespace string `json:"dataset_namespace"`
	DatasetName      string `json:"dataset_name"`
	// Direction is always "output" for nodes returned by Impact — each node
	// is a dataset produced by a downstream step.
	Direction string `json:"direction"`

	// ProducingStep is the task name that emits this dataset, sourced from
	// FacetSummary.caesium_dataset.step_name when available.
	ProducingStep string `json:"producing_step,omitempty"`

	// JobID and JobAlias identify the job that contains the producing step.
	// Populated via a join from task_runs → job_runs → jobs.
	JobID    uuid.UUID `json:"job_id"`
	JobAlias string    `json:"job_alias"`

	// ProvenanceCommit and ProvenanceRepo carry git provenance from the job
	// at the time the dataset row was last written.
	ProvenanceCommit string `json:"provenance_commit,omitempty"`
	ProvenanceRepo   string `json:"provenance_repo,omitempty"`

	// LastSeen is the created_at timestamp of the lineage_dataset row — a
	// proxy for "when was this dependency last observed."
	LastSeen time.Time `json:"last_seen"`

	// Depth is the transitive hop count from the root dataset (0 = direct
	// consumer, 1 = consumer of a consumer, …).
	Depth int `json:"depth"`
}

ImpactNode describes a dataset (or the step that produces it) that is transitively downstream of a changed dataset. Each node carries the producing step name, the job alias, and git provenance so callers can attribute "who wrote this and when."

type ImpactResult

type ImpactResult struct {
	// RootNamespace and RootName are the input dataset whose downstream
	// impact is being reported.
	RootNamespace string `json:"root_namespace"`
	RootName      string `json:"root_name"`

	// Downstream lists every transitively reachable output dataset, ordered
	// breadth-first (shallowest nodes first within each depth level).
	// When the same (namespace, name) is produced by multiple distinct
	// task runs / jobs, each (namespace+name+job_id) triple appears as a
	// separate node; BFS frontier expansion is keyed only on dataset identity
	// so the graph traversal still terminates.
	Downstream []ImpactNode `json:"downstream"`
}

ImpactResult is the response shape returned by QueryImpact.

func QueryImpact

func QueryImpact(ctx context.Context, db *gorm.DB, namespace, name string, maxDepth int) (*ImpactResult, error)

QueryImpact returns all datasets transitively downstream of the dataset identified by (namespace, name) — i.e. datasets whose producing steps consume namespace/name as an input, directly or transitively, across job boundaries.

maxDepth controls how many hops to traverse:

  • 0 (or unset): use the server default of 10.
  • 1–20: traverse that many hops.
  • >20: capped at 20.

The lineage_dataset table records (task_run_id, namespace, name, direction) rows for every observed input and output. Two rows form an edge when an output dataset in one task run shares its (namespace, name) with an input dataset in another task run: the consumer's task run is the next hop.

This is a pure read-side query over the existing dataset graph populated by C1. It does not touch mapper.go or any write path.

type Job

type Job struct {
	Namespace string         `json:"namespace"`
	Name      string         `json:"name"`
	Facets    map[string]any `json:"facets,omitempty"`
}

type JobTypeFacet

type JobTypeFacet struct {
	BaseFacet
	ProcessingType string `json:"processingType"`
	Integration    string `json:"integration"`
	JobType        string `json:"jobType"`
}

type NominalTimeFacet

type NominalTimeFacet struct {
	BaseFacet
	NominalStartTime string `json:"nominalStartTime"`
	NominalEndTime   string `json:"nominalEndTime,omitempty"`
}

type ParentJobRef

type ParentJobRef struct {
	Namespace string `json:"namespace"`
	Name      string `json:"name"`
}

type ParentRunFacet

type ParentRunFacet struct {
	BaseFacet
	Run ParentRunRef `json:"run"`
	Job ParentJobRef `json:"job"`
}

type ParentRunRef

type ParentRunRef struct {
	RunID uuid.UUID `json:"runId"`
}

type Run

type Run struct {
	RunID  uuid.UUID      `json:"runId"`
	Facets map[string]any `json:"facets,omitempty"`
}

type RunEvent

type RunEvent struct {
	EventTime time.Time `json:"eventTime"`
	EventType EventType `json:"eventType"`
	Producer  string    `json:"producer"`
	SchemaURL string    `json:"schemaURL"`
	Run       Run       `json:"run"`
	Job       Job       `json:"job"`
	Inputs    []Dataset `json:"inputs"`
	Outputs   []Dataset `json:"outputs"`
}

type SourceCodeLocationFacet

type SourceCodeLocationFacet struct {
	BaseFacet
	Type    string `json:"type"`
	URL     string `json:"url"`
	RepoURL string `json:"repoUrl,omitempty"`
	Path    string `json:"path,omitempty"`
	Version string `json:"version,omitempty"`
	Tag     string `json:"tag,omitempty"`
	Branch  string `json:"branch,omitempty"`
}

type Subscriber

type Subscriber struct {
	// contains filtered or unexported fields
}

func NewSubscriber

func NewSubscriber(bus event.Bus, transport Transport, namespace string, db *gorm.DB) *Subscriber

func (*Subscriber) SetTransportName

func (s *Subscriber) SetTransportName(name string)

func (*Subscriber) Start

func (s *Subscriber) Start(ctx context.Context) error

func (*Subscriber) StartWithReady

func (s *Subscriber) StartWithReady(ctx context.Context, ready chan<- struct{}) error

type Transport

type Transport interface {
	Emit(ctx context.Context, event RunEvent) error

	Close() error
}

func BuildTransport

func BuildTransport(cfg Config) (Transport, error)

func NewCompositeTransport

func NewCompositeTransport(transports ...Transport) Transport

func NewConsoleTransport

func NewConsoleTransport() Transport

func NewFileTransport

func NewFileTransport(path string) (Transport, error)

func NewHTTPTransport

func NewHTTPTransport(cfg HTTPTransportConfig) Transport

func NewRetryTransport

func NewRetryTransport(base Transport, attempts uint) Transport

NewRetryTransport wraps an existing transport with retry logic.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL