Documentation
¶
Overview ¶
+marmot:name=Iceberg +marmot:description=This plugin discovers namespaces, tables and views from Iceberg catalogs (REST and AWS Glue). +marmot:status=experimental +marmot:features=Assets,Lineage
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Config ¶
type Config struct {
plugin.BaseConfig `json:",inline"`
*plugin.AWSConfig `json:",inline"`
CatalogType string `json:"catalog_type" description:"Catalog backend type" default:"rest" validate:"omitempty,oneof=rest glue"`
// REST catalog fields
URI string `json:"uri" description:"REST catalog URI (required for catalog_type=rest)" show_when:"catalog_type:rest"`
Warehouse string `json:"warehouse" description:"Warehouse identifier" show_when:"catalog_type:rest"`
Credential string `` /* 134-byte string literal not displayed */
Token string `json:"token" description:"Bearer token for authentication" sensitive:"true" show_when:"catalog_type:rest"`
Properties map[string]string `json:"properties" description:"Additional catalog properties" show_when:"catalog_type:rest"`
Prefix string `json:"prefix" description:"Optional prefix for the REST catalog" show_when:"catalog_type:rest"`
// Glue catalog fields
GlueCatalogID string `json:"glue_catalog_id" description:"AWS Glue Data Catalog ID (defaults to caller's account)" show_when:"catalog_type:glue"`
IncludeNamespaces bool `json:"include_namespaces" description:"Whether to discover namespaces as assets" default:"true"`
IncludeViews bool `json:"include_views" description:"Whether to discover views" default:"true"`
}
+marmot:config
type IcebergNamespaceFields ¶ added in v0.8.0
type IcebergNamespaceFields struct {
Namespace string `json:"namespace" metadata:"namespace" description:"Namespace path"`
Location string `json:"location" metadata:"location" description:"Default location for tables"`
}
IcebergNamespaceFields represents Iceberg namespace metadata fields +marmot:metadata
type IcebergTableFields ¶
type IcebergTableFields struct {
TableUUID string `json:"table_uuid" metadata:"table_uuid" description:"Table UUID"`
Location string `json:"location" metadata:"location" description:"Table data location"`
FormatVersion int `json:"format_version" metadata:"format_version" description:"Iceberg format version (1, 2, or 3)"`
CurrentSnapshot string `json:"current_snapshot_id" metadata:"current_snapshot_id" description:"Current snapshot ID"`
SnapshotCount int `json:"snapshot_count" metadata:"snapshot_count" description:"Number of snapshots"`
SchemaFields int `json:"schema_field_count" metadata:"schema_field_count" description:"Number of schema fields"`
PartitionSpec string `json:"partition_spec" metadata:"partition_spec" description:"Partition specification"`
SortOrder string `json:"sort_order" metadata:"sort_order" description:"Sort order specification"`
LastUpdatedMs int64 `json:"last_updated_ms" metadata:"last_updated_ms" description:"Last update timestamp in milliseconds"`
TotalRecords string `json:"total_records" metadata:"total_records" description:"Total record count"`
TotalDataFiles string `json:"total_data_files" metadata:"total_data_files" description:"Total data file count"`
TotalSize string `json:"total_file_size" metadata:"total_file_size" description:"Total file size in bytes"`
}
IcebergTableFields represents Iceberg table metadata fields +marmot:metadata
type IcebergViewFields ¶ added in v0.8.0
type IcebergViewFields struct {
ViewUUID string `json:"view_uuid" metadata:"view_uuid" description:"View UUID"`
Location string `json:"location" metadata:"location" description:"View metadata location"`
FormatVersion int `json:"format_version" metadata:"format_version" description:"View format version"`
SchemaFields int `json:"schema_field_count" metadata:"schema_field_count" description:"Number of schema fields"`
SQLDialect string `json:"sql_dialect" metadata:"sql_dialect" description:"SQL dialect of the view definition"`
SQL string `json:"sql" metadata:"sql" description:"SQL definition of the view"`
}
IcebergViewFields represents Iceberg view metadata fields +marmot:metadata
type Source ¶
type Source struct {
// contains filtered or unexported fields
}
func (*Source) Discover ¶
func (s *Source) Discover(ctx context.Context, pluginConfig plugin.RawPluginConfig) (*plugin.DiscoveryResult, error)
func (*Source) Validate ¶
func (s *Source) Validate(rawConfig plugin.RawPluginConfig) (plugin.RawPluginConfig, error)
Click to show internal directories.
Click to hide internal directories.