Documentation
¶
Overview ¶
Package iceberg provides read-only support for Apache Iceberg table metadata. It parses Iceberg metadata JSON files to resolve snapshots, manifests, and data file paths for querying existing Iceberg tables.
Index ¶
- type CatalogIntegration
- func (ci *CatalogIntegration) DiscoverAndRegister(ctx context.Context, name, tablePath string) (*TableInfo, error)
- func (ci *CatalogIntegration) RefreshTable(ctx context.Context, name, metadataPath string) (*TableInfo, error)
- func (ci *CatalogIntegration) RegisterTable(ctx context.Context, name, metadataPath string) (*TableInfo, error)
- type DataFileEntry
- type Field
- type ManifestEntry
- type ManifestFile
- type ManifestList
- type PartitionField
- type PartitionSpec
- type Schema
- type Snapshot
- type SortField
- type SortOrder
- type TableInfo
- type TableMetadata
- type TableReader
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CatalogIntegration ¶
type CatalogIntegration struct {
// contains filtered or unexported fields
}
CatalogIntegration bridges Iceberg table metadata into Wadjet's native catalog.
func NewCatalogIntegration ¶
func NewCatalogIntegration(cat *catalog.Catalog) *CatalogIntegration
NewCatalogIntegration creates a new integration bridge.
func NewCatalogIntegrationWithStore ¶
func NewCatalogIntegrationWithStore(cat *catalog.Catalog, store objstore.Store, bucket string) *CatalogIntegration
NewCatalogIntegrationWithStore creates an integration bridge with a custom store/bucket (e.g., when the Iceberg table is in a different bucket than the catalog).
func (*CatalogIntegration) DiscoverAndRegister ¶
func (ci *CatalogIntegration) DiscoverAndRegister(ctx context.Context, name, tablePath string) (*TableInfo, error)
DiscoverAndRegister discovers an Iceberg table at the given path and registers it in the catalog.
func (*CatalogIntegration) RefreshTable ¶
func (ci *CatalogIntegration) RefreshTable(ctx context.Context, name, metadataPath string) (*TableInfo, error)
RefreshTable updates an existing catalog table from the latest Iceberg metadata. It drops and re-creates the table to pick up schema changes.
func (*CatalogIntegration) RegisterTable ¶
func (ci *CatalogIntegration) RegisterTable(ctx context.Context, name, metadataPath string) (*TableInfo, error)
RegisterTable reads an Iceberg table from the given metadata path and registers it in the Wadjet catalog with the given name. Data files from the Iceberg manifest are registered as partition entries in the catalog.
type DataFileEntry ¶
type DataFileEntry struct {
FilePath string `json:"file_path"`
FileFormat string `json:"file_format"` // PARQUET, ORC, AVRO
RecordCount int64 `json:"record_count"`
FileSizeBytes int64 `json:"file_size_in_bytes"`
Partition map[string]any `json:"partition"`
ColumnSizes map[int]int64 `json:"column_sizes"`
ValueCounts map[int]int64 `json:"value_counts"`
NullCounts map[int]int64 `json:"null_value_counts"`
LowerBounds map[int]string `json:"lower_bounds"`
UpperBounds map[int]string `json:"upper_bounds"`
}
DataFileEntry describes a single data file in an Iceberg table.
type Field ¶
type Field struct {
ID int `json:"id"`
Name string `json:"name"`
Required bool `json:"required"`
Type string `json:"type"` // "boolean", "int", "long", "float", "double", "string", "date", "timestamp", "timestamptz", "binary", "uuid", "decimal(p,s)", "fixed[l]"
}
Field represents a column in an Iceberg schema.
type ManifestEntry ¶
type ManifestEntry struct {
ManifestPath string `json:"manifest_path"`
ManifestLength int64 `json:"manifest_length"`
PartitionSpecID int `json:"partition_spec_id"`
AddedSnapshotID int64 `json:"added_snapshot_id"`
Content int `json:"content"` // 0 = data, 1 = deletes
}
ManifestEntry points to a manifest file.
type ManifestFile ¶
type ManifestFile struct {
Entries []DataFileEntry
}
ManifestFile contains data file entries.
func ParseManifest ¶
func ParseManifest(data []byte) (*ManifestFile, error)
ParseManifest parses a manifest file JSON (simplified).
func (*ManifestFile) DataFiles ¶
func (mf *ManifestFile) DataFiles() []DataFileEntry
DataFiles returns all data file paths from a manifest, filtered to only include Parquet files.
type ManifestList ¶
type ManifestList struct {
Entries []ManifestEntry
}
ManifestList is a list of manifest file entries.
func ParseManifestList ¶
func ParseManifestList(data []byte) (*ManifestList, error)
ParseManifestList parses a manifest list JSON (simplified — real Iceberg uses Avro, but many implementations also support JSON manifest lists).
type PartitionField ¶
type PartitionField struct {
SourceID int `json:"source-id"`
FieldID int `json:"field-id"`
Name string `json:"name"`
Transform string `json:"transform"` // identity, bucket[N], truncate[N], year, month, day, hour, void
}
PartitionField describes how a source column is partitioned.
type PartitionSpec ¶
type PartitionSpec struct {
SpecID int `json:"spec-id"`
Fields []PartitionField `json:"fields"`
}
PartitionSpec groups partition fields under a spec ID.
type Schema ¶
type Schema struct {
SchemaID int `json:"schema-id"`
Type string `json:"type"`
Fields []Field `json:"fields"`
}
Schema represents an Iceberg table schema.
func (*Schema) ToParquetSchema ¶
ToParquetSchema converts an Iceberg schema to Wadjet's parquet schema.
type Snapshot ¶
type Snapshot struct {
SnapshotID int64 `json:"snapshot-id"`
ParentID *int64 `json:"parent-snapshot-id"`
TimestampMs int64 `json:"timestamp-ms"`
ManifestList string `json:"manifest-list"`
Summary map[string]string `json:"summary"`
SchemaID *int `json:"schema-id"`
}
Snapshot represents an Iceberg table snapshot.
type SortField ¶
type SortField struct {
Transform string `json:"transform"`
SourceID int `json:"source-id"`
Direction string `json:"direction"`
NullOrder string `json:"null-order"`
}
SortField defines a sort column and direction.
type TableInfo ¶
type TableInfo struct {
Metadata *TableMetadata
Schema parquet.Schema
DataFiles []DataFileEntry
PartitionKeys []string
Location string
}
TableInfo contains resolved information about an Iceberg table.
func (*TableInfo) SnapshotToJSON ¶
SnapshotToJSON serializes a table info summary to JSON for debugging.
type TableMetadata ¶
type TableMetadata struct {
FormatVersion int `json:"format-version"`
TableUUID string `json:"table-uuid"`
Location string `json:"location"`
Schema Schema `json:"schema"`
Schemas []Schema `json:"schemas"`
CurrentSchema int `json:"current-schema-id"`
PartitionSpec []PartitionField `json:"partition-spec"`
PartitionSpecs []PartitionSpec `json:"partition-specs"`
DefaultSpecID int `json:"default-spec-id"`
Properties map[string]string `json:"properties"`
Snapshots []Snapshot `json:"snapshots"`
CurrentSnapshotID *int64 `json:"current-snapshot-id"`
SortOrders []SortOrder `json:"sort-orders"`
}
TableMetadata represents an Iceberg table metadata file (v1 or v2).
func ParseTableMetadata ¶
func ParseTableMetadata(data []byte) (*TableMetadata, error)
ParseTableMetadata parses an Iceberg table metadata JSON file.
func (*TableMetadata) ActivePartitionSpec ¶
func (m *TableMetadata) ActivePartitionSpec() []PartitionField
ActivePartitionSpec returns the current partition spec.
func (*TableMetadata) ActiveSchema ¶
func (m *TableMetadata) ActiveSchema() *Schema
ActiveSchema returns the current schema for the table.
func (*TableMetadata) CurrentSnapshot ¶
func (m *TableMetadata) CurrentSnapshot() *Snapshot
CurrentSnapshot returns the current snapshot, or nil if none exists.
type TableReader ¶
type TableReader struct {
// contains filtered or unexported fields
}
TableReader reads Iceberg table metadata from an object store and resolves the current snapshot to a list of Parquet data files.
func NewTableReader ¶
func NewTableReader(store objstore.Store, bucket string) *TableReader
NewTableReader creates a reader for Iceberg tables stored in the given object store.
func (*TableReader) DiscoverTable ¶
DiscoverTable tries to find and read an Iceberg table by looking for the metadata directory in common locations.
Source Files