Documentation
¶
Overview ¶
+marmot:name=Delta Lake +marmot:description=This plugin discovers tables from Delta Lake transaction logs on local filesystems. +marmot:status=experimental +marmot:features=Assets
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Config ¶
type Config struct {
plugin.BaseConfig `json:",inline"`
*plugin.FileSourceConfig `json:",inline"`
TablePaths []string `` /* 142-byte string literal not displayed */
}
+marmot:config
type DeltaLakeTableFields ¶
type DeltaLakeTableFields struct {
TableID string `json:"table_id" metadata:"table_id" description:"Delta table unique identifier"`
Location string `json:"location" metadata:"location" description:"Table directory path"`
Format string `json:"format" metadata:"format" description:"Data format (e.g. parquet)"`
MinReaderVersion int `json:"min_reader_version" metadata:"min_reader_version" description:"Minimum reader protocol version"`
MinWriterVersion int `json:"min_writer_version" metadata:"min_writer_version" description:"Minimum writer protocol version"`
PartitionColumns string `json:"partition_columns" metadata:"partition_columns" description:"Comma-separated partition column names"`
SchemaFieldCount int `json:"schema_field_count" metadata:"schema_field_count" description:"Number of schema fields"`
CreatedTime int64 `json:"created_time" metadata:"created_time" description:"Table creation timestamp in milliseconds"`
NumFiles int `json:"num_files" metadata:"num_files" description:"Number of active data files"`
TotalSize int64 `json:"total_size" metadata:"total_size" description:"Total size of active data files in bytes"`
CurrentVersion int64 `json:"current_version" metadata:"current_version" description:"Current Delta log version"`
}
DeltaLakeTableFields represents Delta Lake table metadata fields +marmot:metadata
type Source ¶
type Source struct {
// contains filtered or unexported fields
}
func (*Source) Discover ¶
func (s *Source) Discover(ctx context.Context, pluginConfig plugin.RawPluginConfig) (*plugin.DiscoveryResult, error)
func (*Source) Validate ¶
func (s *Source) Validate(rawConfig plugin.RawPluginConfig) (plugin.RawPluginConfig, error)
Click to show internal directories.
Click to hide internal directories.