iceberg

package
v0.8.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 1, 2026 License: MIT Imports: 19 Imported by: 0

Documentation

Overview

+marmot:name=Iceberg +marmot:description=This plugin discovers namespaces, tables and views from Iceberg catalogs (REST and AWS Glue). +marmot:status=experimental +marmot:features=Assets,Lineage

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Config

type Config struct {
	plugin.BaseConfig `json:",inline"`
	*plugin.AWSConfig `json:",inline"`

	CatalogType string `json:"catalog_type" description:"Catalog backend type" default:"rest" validate:"omitempty,oneof=rest glue"`

	// REST catalog fields
	URI        string            `json:"uri" description:"REST catalog URI (required for catalog_type=rest)" show_when:"catalog_type:rest"`
	Warehouse  string            `json:"warehouse" description:"Warehouse identifier" show_when:"catalog_type:rest"`
	Credential string            `` /* 134-byte string literal not displayed */
	Token      string            `json:"token" description:"Bearer token for authentication" sensitive:"true" show_when:"catalog_type:rest"`
	Properties map[string]string `json:"properties" description:"Additional catalog properties" show_when:"catalog_type:rest"`
	Prefix     string            `json:"prefix" description:"Optional prefix for the REST catalog" show_when:"catalog_type:rest"`

	// Glue catalog fields
	GlueCatalogID string `json:"glue_catalog_id" description:"AWS Glue Data Catalog ID (defaults to caller's account)" show_when:"catalog_type:glue"`

	IncludeNamespaces bool `json:"include_namespaces" description:"Whether to discover namespaces as assets" default:"true"`
	IncludeViews      bool `json:"include_views" description:"Whether to discover views" default:"true"`
}

+marmot:config

type IcebergNamespaceFields added in v0.8.0

type IcebergNamespaceFields struct {
	Namespace string `json:"namespace" metadata:"namespace" description:"Namespace path"`
	Location  string `json:"location" metadata:"location" description:"Default location for tables"`
}

IcebergNamespaceFields represents Iceberg namespace metadata fields +marmot:metadata

type IcebergTableFields

type IcebergTableFields struct {
	TableUUID       string `json:"table_uuid" metadata:"table_uuid" description:"Table UUID"`
	Location        string `json:"location" metadata:"location" description:"Table data location"`
	FormatVersion   int    `json:"format_version" metadata:"format_version" description:"Iceberg format version (1, 2, or 3)"`
	CurrentSnapshot string `json:"current_snapshot_id" metadata:"current_snapshot_id" description:"Current snapshot ID"`
	SnapshotCount   int    `json:"snapshot_count" metadata:"snapshot_count" description:"Number of snapshots"`
	SchemaFields    int    `json:"schema_field_count" metadata:"schema_field_count" description:"Number of schema fields"`
	PartitionSpec   string `json:"partition_spec" metadata:"partition_spec" description:"Partition specification"`
	SortOrder       string `json:"sort_order" metadata:"sort_order" description:"Sort order specification"`
	LastUpdatedMs   int64  `json:"last_updated_ms" metadata:"last_updated_ms" description:"Last update timestamp in milliseconds"`
	TotalRecords    string `json:"total_records" metadata:"total_records" description:"Total record count"`
	TotalDataFiles  string `json:"total_data_files" metadata:"total_data_files" description:"Total data file count"`
	TotalSize       string `json:"total_file_size" metadata:"total_file_size" description:"Total file size in bytes"`
}

IcebergTableFields represents Iceberg table metadata fields +marmot:metadata

type IcebergViewFields added in v0.8.0

type IcebergViewFields struct {
	ViewUUID      string `json:"view_uuid" metadata:"view_uuid" description:"View UUID"`
	Location      string `json:"location" metadata:"location" description:"View metadata location"`
	FormatVersion int    `json:"format_version" metadata:"format_version" description:"View format version"`
	SchemaFields  int    `json:"schema_field_count" metadata:"schema_field_count" description:"Number of schema fields"`
	SQLDialect    string `json:"sql_dialect" metadata:"sql_dialect" description:"SQL dialect of the view definition"`
	SQL           string `json:"sql" metadata:"sql" description:"SQL definition of the view"`
}

IcebergViewFields represents Iceberg view metadata fields +marmot:metadata

type Source

type Source struct {
	// contains filtered or unexported fields
}

func (*Source) Discover

func (s *Source) Discover(ctx context.Context, pluginConfig plugin.RawPluginConfig) (*plugin.DiscoveryResult, error)

func (*Source) Validate

func (s *Source) Validate(rawConfig plugin.RawPluginConfig) (plugin.RawPluginConfig, error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL