plugin

package
v0.1.0-preview4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 2, 2025 License: MIT Imports: 14 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func InterpolateTags 

func InterpolateTags(tags TagsConfig, metadata map[string]interface{}) []string

InterpolateTags processes tags and replaces variables with values from metadata

func MapToMetadata 

func MapToMetadata(source interface{}) map[string]interface{}

MapToMetadata converts a struct with metadata tags to a metadata map

func NewAWSConfig 

func NewAWSConfig(ctx context.Context, rawConfig map[string]interface{}) (aws.Config, error)

NewAWSConfig loads AWS configuration with the provided credentials

func ProcessAWSTags 

func ProcessAWSTags(tagsToMetadata bool, includeTags []string, tags map[string]string) map[string]interface{}

ProcessAWSTags converts AWS tags to metadata based on configuration

func ShouldIncludeResource 

func ShouldIncludeResource(name string, filter Filter) bool

ShouldIncludeResource checks if a resource should be included based on filter patterns

func UnmarshalPluginConfig 

func UnmarshalPluginConfig[T any](raw RawPluginConfig) (*T, error)

UnmarshalPluginConfig unmarshals raw config into a specific plugin config type

Types

type AWSConfig 

type AWSConfig struct {
	Credentials    AWSCredentials `json:"credentials" description:"AWS credentials configuration"`
	TagsToMetadata bool           `json:"tags_to_metadata,omitempty" description:"Convert AWS tags to Marmot metadata"`
	IncludeTags    []string       `json:"include_tags,omitempty" description:"List of AWS tags to include as metadata"`
	Filter         Filter         `json:"filter,omitempty" description:"Filter patterns for AWS resources"`
}

AWSConfig represents common AWS configuration for plugins

func (*AWSConfig) Validate 

func (a *AWSConfig) Validate() error

Validate validates the AWSConfig

type AWSCredentials 

type AWSCredentials struct {
	Profile        string `json:"profile,omitempty" description:"AWS profile to use from shared credentials file"`
	ID             string `json:"id,omitempty" description:"AWS access key ID"`
	Secret         string `json:"secret,omitempty" description:"AWS secret access key"`
	Endpoint       string `json:"endpoint,omitempty" description:"AWS endpoint"`
	Token          string `json:"token,omitempty" description:"AWS session token"`
	Role           string `json:"role,omitempty" description:"AWS IAM role ARN to assume"`
	RoleExternalID string `json:"role_external_id,omitempty" description:"External ID for cross-account role assumption"`
	Region         string `json:"region" description:"AWS region for services" required:"true"`
}

AWSCredentials represents AWS authentication configuration

type AWSPlugin 

type AWSPlugin struct {
	AWSConfig  `json:",inline"`
	BaseConfig `json:",inline"`
}

type BaseConfig 

type BaseConfig struct {
	GlobalDocumentation         []string       `json:"global_documentation,omitempty"`
	GlobalDocumentationPosition string         `json:"global_documentation_position,omitempty"`
	Metadata                    MetadataConfig `json:"metadata,omitempty"`
	Tags                        TagsConfig     `json:"tags,omitempty"`
	Merge                       MergeConfig    `json:"merge,omitempty"`
	ExternalLinks               []ExternalLink `json:"external_links,omitempty"`
	AWSConfig                   *AWSConfig     `json:"aws,omitempty"`
}

BaseConfig contains configuration fields common to all plugins

type Config 

type Config struct {
	Runs []SourceRun `json:"runs"`
}

Config represents the top-level configuration

type DiscoveryResult 

type DiscoveryResult struct {
	Assets        []asset.Asset             `json:"assets"`
	Lineage       []lineage.LineageEdge     `json:"lineage"`
	Documentation []assetdocs.Documentation `json:"documentation"`
}

DiscoveryResult contains all discovered assets, lineage, and documentation 

type ExternalLink struct {
	Name string `json:"name"`
	Icon string `json:"icon"`
	URL  string `json:"url"`
}

type FieldMergeConfig 

type FieldMergeConfig struct {
	Strategy MergeStrategy  `json:"strategy"`
	Priority int            `json:"priority,omitempty"`
	Filter   MetadataFilter `json:"filter,omitempty"`
}

FieldMergeConfig defines merge behavior for a specific field

type Filter 

type Filter struct {
	Include []string `json:"include,omitempty" description:"Include patterns for resource names (regex)"`
	Exclude []string `json:"exclude,omitempty" description:"Exclude patterns for resource names (regex)"`
}

Filter represents include/exclude patterns for AWS resources

type MergeConfig 

type MergeConfig struct {
	Tags        string `json:"tags,omitempty"`        // append, append_only, overwrite, keep_first
	Metadata    string `json:"metadata,omitempty"`    // merge, overwrite, keep_first
	Name        string `json:"name,omitempty"`        // keep_first, overwrite
	Description string `json:"description,omitempty"` // keep_first, overwrite
}

MergeConfig defines how different asset fields should be merged

type MergePolicy 

type MergePolicy struct {
	Strategy        MergeStrategy               `json:"strategy"`
	Fields          map[string]FieldMergeConfig `json:"fields,omitempty"`
	MetadataMapping []MetadataMapping           `json:"metadata_mapping,omitempty"`
	MetadataRules   MetadataRules               `json:"metadata_rules,omitempty"`
}

MergePolicy defines how assets should be merged during sync

type MergeStrategy 

type MergeStrategy string

MergeStrategy defines different merge strategies 

const (
	MergeStrategyOverwrite MergeStrategy = "overwrite"  // Replace existing value
	MergeStrategyAppend    MergeStrategy = "append"     // Append to existing values
	MergeStrategyKeepFirst MergeStrategy = "keep_first" // Keep existing value if present
	MergeStrategyMerge     MergeStrategy = "merge"      // Deep merge for maps
)

type MetadataConfig 

type MetadataConfig struct {
	Allow []string `json:"allow,omitempty"`
}

MetadataConfig defines allowable metadata fields

type MetadataField 

type MetadataField struct {
	Path        string
	Description string
	Type        string
	Required    bool
}

MetadataField defines a metadata field with documentation and validation

func GetMetadataFields 

func GetMetadataFields(v interface{}) []MetadataField

GetMetadataFields extracts metadata field definitions from struct tags

type MetadataFilter 

type MetadataFilter struct {
	Mode     string   `json:"mode"`     // "allow_all", "allow_list", or "deny_list"
	Props    []string `json:"props"`    // Properties to allow or deny based on mode
	Prefixes []string `json:"prefixes"` // Property prefixes to allow or deny
}

MetadataFilter defines metadata filtering rules

type MetadataMapping 

type MetadataMapping struct {
	Source       string `json:"source"`            // JSON path in source object
	Target       string `json:"target"`            // Target metadata path
	Type         string `json:"type"`              // "asset" or "environment"
	DefaultValue any    `json:"default,omitempty"` // Optional default value
}

MetadataMapping defines how metadata should be mapped during merge

type MetadataRules 

type MetadataRules struct {
	Merge       MetadataFilter `json:"merge"`       // Controls which fields get merged between assets
	Environment MetadataFilter `json:"environment"` // Controls which fields get copied to environments
}

MetadataRules defines rules for metadata handling

type PluginConfig 

type PluginConfig struct {
	BaseConfig `json:",inline"`
	Source     string `json:"source,omitempty"`
}

PluginConfig combines base config with plugin-specific fields

type RawPluginConfig 

type RawPluginConfig map[string]interface{}

RawPluginConfig holds the raw JSON configuration for a plugin It uses a `map[string]interface{}` to unmarshal arbitrary JSON data for each plugin's specific config.

type Source 

type Source interface {
	// Validate checks if the provided configuration is valid
	Validate(config RawPluginConfig) error

	// Discover performs the discovery of assets, lineage, and documentation
	Discover(ctx context.Context, config RawPluginConfig) (*DiscoveryResult, error)
}

Source defines the interface that all plugins must implement

type SourceRun 

type SourceRun map[string]RawPluginConfig

SourceRun maps source names to their raw configurations

type TagsConfig 

type TagsConfig []string

TagsConfig is just a string slice that supports interpolation

Source Files

View all Source files

Directories

Path Synopsis
providers
asyncapi
metadata.go
 metadata.go
iceberg
+marmot:name=Iceberg +marmot:description=This plugin discovers Apache Iceberg tables from various catalog implementations.
 +marmot:name=Iceberg +marmot:description=This plugin discovers Apache Iceberg tables from various catalog implementations.
kafka
+marmot:name=Kafka +marmot:description=This plugin discovers Kafka topics from Kafka clusters.
 +marmot:name=Kafka +marmot:description=This plugin discovers Kafka topics from Kafka clusters.
postgresql
+marmot:name=PostgreSQL +marmot:description=This plugin discovers databases and tables from PostgreSQL instances.
 +marmot:name=PostgreSQL +marmot:description=This plugin discovers databases and tables from PostgreSQL instances.
sns
+marmot:name=SNS +marmot:description=This plugin discovers SNS topics from AWS accounts.
 +marmot:name=SNS +marmot:description=This plugin discovers SNS topics from AWS accounts.
sqs
+marmot:name=SQS +marmot:description=This plugin discovers SQS queues from AWS accounts.
 +marmot:name=SQS +marmot:description=This plugin discovers SQS queues from AWS accounts.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.