plugin

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 28, 2025 License: MIT Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source 
const SensitiveMask = "********"

Variables

View Source 
var ErrEndpointNotFound = fmt.Errorf("endpoint not found")

ErrEndpointNotFound is returned when an endpoint can't be resolved

Functions

func GetConfigType added in v0.3.1 

func GetConfigType(raw RawPluginConfig, source Source) interface{}

GetConfigType attempts to extract the config type from a source by unmarshaling into an empty interface and using reflection

func InterpolateTags 

func InterpolateTags(tags TagsConfig, metadata map[string]interface{}) []string

InterpolateTags processes tags and replaces variables with values from metadata

func MapToMetadata 

func MapToMetadata(source interface{}) map[string]interface{}

MapToMetadata converts a struct with metadata tags to a metadata map

func ProcessAWSTags 

func ProcessAWSTags(tagsToMetadata bool, includeTags []string, tags map[string]string) map[string]interface{}

func ShouldIncludeResource 

func ShouldIncludeResource(name string, filter Filter) bool

func UnmarshalPluginConfig 

func UnmarshalPluginConfig[T any](raw RawPluginConfig) (*T, error)

UnmarshalPluginConfig unmarshals raw config into a specific plugin config type

Types

type AWSConfig 

type AWSConfig struct {
	Credentials    AWSCredentials `json:"credentials" description:"AWS credentials configuration"`
	TagsToMetadata bool           `json:"tags_to_metadata,omitempty" description:"Convert AWS tags to Marmot metadata"`
	IncludeTags    []string       `json:"include_tags,omitempty" description:"List of AWS tags to include as metadata"`
	Filter         Filter         `json:"filter,omitempty" description:"Filter patterns for AWS resources"`
}

func ExtractAWSConfig added in v0.2.0 

func ExtractAWSConfig(rawConfig map[string]interface{}) (*AWSConfig, error)

func (*AWSConfig) NewAWSConfig added in v0.2.0 

func (a *AWSConfig) NewAWSConfig(ctx context.Context) (aws.Config, error)

func (*AWSConfig) Validate 

func (a *AWSConfig) Validate() error

type AWSCredentials 

type AWSCredentials struct {
	Profile        string `json:"profile,omitempty" description:"AWS profile to use from shared credentials file"`
	ID             string `json:"id,omitempty" description:"AWS access key ID"`
	Secret         string `json:"secret,omitempty" description:"AWS secret access key" sensitive:"true"`
	Token          string `json:"token,omitempty" description:"AWS session token"`
	Role           string `json:"role,omitempty" description:"AWS IAM role ARN to assume"`
	RoleExternalID string `json:"role_external_id,omitempty" description:"External ID for cross-account role assumption"`
	Region         string `json:"region,omitempty" description:"AWS region for services"`
	Endpoint       string `json:"endpoint,omitempty" description:"Custom endpoint URL for AWS services"`
}

type AWSPlugin 

type AWSPlugin struct {
	AWSConfig  `json:",inline"`
	BaseConfig `json:",inline"`
}

type BaseConfig 

type BaseConfig struct {
	GlobalDocumentation         []string       `json:"global_documentation,omitempty"`
	GlobalDocumentationPosition string         `json:"global_documentation_position,omitempty"`
	Metadata                    MetadataConfig `json:"metadata,omitempty"`
	Tags                        TagsConfig     `json:"tags,omitempty"`
	ExternalLinks               []ExternalLink `json:"external_links,omitempty"`
	AWSConfig                   *AWSConfig     `json:"aws,omitempty"`
}

type Config 

type Config struct {
	Name string      `json:"name" yaml:"name"`
	Runs []SourceRun `json:"runs" yaml:"runs"`
}

type DiscoveryResult 

type DiscoveryResult struct {
	Assets        []asset.Asset             `json:"assets"`
	Lineage       []lineage.LineageEdge     `json:"lineage"`
	Documentation []assetdocs.Documentation `json:"documentation"`
	Statistics    []Statistic               `json:"statistics"`
}

DiscoveryResult contains all discovered assets, lineage, and documentation 

type ExternalLink struct {
	Name string `json:"name"`
	Icon string `json:"icon"`
	URL  string `json:"url"`
}

ExternalLink defines an external resource link

type Filter 

type Filter struct {
	Include []string `json:"include,omitempty" description:"Include patterns for resource names (regex)"`
	Exclude []string `json:"exclude,omitempty" description:"Exclude patterns for resource names (regex)"`
}

type MetadataConfig 

type MetadataConfig struct {
	Allow []string `json:"allow,omitempty"`
}

type MetadataField 

type MetadataField struct {
	Path        string
	Description string
	Type        string
	Required    bool
	Sensitive   bool
}

MetadataField defines a metadata field with documentation and validation

func GetMetadataFields 

func GetMetadataFields(v interface{}) []MetadataField

GetMetadataFields extracts metadata field definitions from struct tags

type PluginConfig 

type PluginConfig struct {
	BaseConfig `json:",inline"`
	Source     string `json:"source,omitempty"`
}

PluginConfig combines base config with plugin-specific fields

type RawPluginConfig 

type RawPluginConfig map[string]interface{}

RawPluginConfig holds the raw JSON configuration for a plugin It uses a `map[string]interface{}` to unmarshal arbitrary JSON data for each plugin's specific config.

func (RawPluginConfig) MaskSensitiveFields added in v0.3.0 

func (r RawPluginConfig) MaskSensitiveFields(configStruct interface{}) RawPluginConfig

type Run added in v0.3.0 

type Run struct {
	ID           string          `json:"id"`
	PipelineName string          `json:"pipeline_name"`
	SourceName   string          `json:"source_name"`
	RunID        string          `json:"run_id"`
	Status       RunStatus       `json:"status"`
	StartedAt    time.Time       `json:"started_at"`
	CompletedAt  *time.Time      `json:"completed_at,omitempty"`
	ErrorMessage string          `json:"error_message,omitempty"`
	Config       RawPluginConfig `json:"config,omitempty"`
	Summary      *RunSummary     `json:"summary,omitempty"`
	CreatedBy    string          `json:"created_by"`
}

Run represents a single run

type RunCheckpoint added in v0.3.0 

type RunCheckpoint struct {
	ID           string    `json:"id"`
	RunID        string    `json:"run_id"`
	EntityType   string    `json:"entity_type"` // 'asset', 'lineage', 'documentation'
	EntityMRN    string    `json:"entity_mrn"`
	Operation    string    `json:"operation"`     // 'created', 'updated', 'deleted', 'skipped'
	SourceFields []string  `json:"source_fields"` // Which fields this source contributed
	CreatedAt    time.Time `json:"created_at"`
}

RunCheckpoint tracks what entities were processed in a run

type RunStatus added in v0.3.0 

type RunStatus string
const (
	StatusRunning   RunStatus = "running"
	StatusCompleted RunStatus = "completed"
	StatusFailed    RunStatus = "failed"
	StatusCancelled RunStatus = "cancelled"
)

type RunSummary added in v0.3.0 

type RunSummary struct {
	AssetsCreated      int `json:"assets_created"`
	AssetsUpdated      int `json:"assets_updated"`
	AssetsDeleted      int `json:"assets_deleted"`
	LineageCreated     int `json:"lineage_created"`
	LineageUpdated     int `json:"lineage_updated"`
	DocumentationAdded int `json:"documentation_added"`
	ErrorsCount        int `json:"errors_count"`
	TotalEntities      int `json:"total_entities"`
	DurationSeconds    int `json:"duration_seconds"`
}

RunSummary contains summary statistics for a run

type Source 

type Source interface {
	Validate(config RawPluginConfig) (RawPluginConfig, error)
	Discover(ctx context.Context, config RawPluginConfig) (*DiscoveryResult, error)
}

type SourceRun 

type SourceRun map[string]RawPluginConfig

SourceRun maps source names to their raw configurations

type StatefulRunContext added in v0.3.0 

type StatefulRunContext struct {
	PipelineName       string
	SourceName         string
	LastRunCheckpoints map[string]*RunCheckpoint // entity_mrn -> checkpoint
	CurrentRunID       string
}

StatefulRunContext provides context for stateful operations

type StatefulSource added in v0.3.0 

type StatefulSource interface {
	Source
	SupportsStatefulIngestion() bool
}

StatefulSource extends Source with stateful capabilities

type Statistic added in v0.3.1 

type Statistic struct {
	AssetMRN   string  `json:"asset_mrn"`
	MetricName string  `json:"metric_name"`
	Value      float64 `json:"value"`
}

type TagsConfig 

type TagsConfig []string

TagsConfig is just a string slice that supports interpolation

Source Files

View all Source files

Directories

Path Synopsis
providers
asyncapi
+marmot:name=AsyncAPI +marmot:description=This plugin enables fetching data from AsyncAPI specifications.
 +marmot:name=AsyncAPI +marmot:description=This plugin enables fetching data from AsyncAPI specifications.
bigquery
+marmot:name=BigQuery +marmot:description=This plugin discovers datasets and tables from Google BigQuery projects.
 +marmot:name=BigQuery +marmot:description=This plugin discovers datasets and tables from Google BigQuery projects.
kafka
+marmot:name=Kafka +marmot:description=This plugin discovers Kafka topics from Kafka clusters.
 +marmot:name=Kafka +marmot:description=This plugin discovers Kafka topics from Kafka clusters.
mongodb
+marmot:name=MongoDB +marmot:description=This plugin discovers databases and collections from MongoDB instances.
 +marmot:name=MongoDB +marmot:description=This plugin discovers databases and collections from MongoDB instances.
mysql
+marmot:name=MySQL +marmot:description=This plugin discovers databases and tables from MySQL instances.
 +marmot:name=MySQL +marmot:description=This plugin discovers databases and tables from MySQL instances.
openapi
+marmot:name=OpenAPI +marmot:description=This plugin discovers OpenAPI v3 specifications.
 +marmot:name=OpenAPI +marmot:description=This plugin discovers OpenAPI v3 specifications.
postgresql
+marmot:name=PostgreSQL +marmot:description=This plugin discovers databases and tables from PostgreSQL instances.
 +marmot:name=PostgreSQL +marmot:description=This plugin discovers databases and tables from PostgreSQL instances.
s3
+marmot:name=S3 +marmot:description=This plugin discovers S3 buckets from AWS accounts.
 +marmot:name=S3 +marmot:description=This plugin discovers S3 buckets from AWS accounts.
sns
+marmot:name=SNS +marmot:description=This plugin discovers SNS topics from AWS accounts.
 +marmot:name=SNS +marmot:description=This plugin discovers SNS topics from AWS accounts.
sqs
+marmot:name=SQS +marmot:description=This plugin discovers SQS queues from AWS accounts.
 +marmot:name=SQS +marmot:description=This plugin discovers SQS queues from AWS accounts.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.