table

package
v0.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 28, 2026 License: Apache-2.0 Imports: 66 Imported by: 23

Documentation

Index

Constants

View Source 
const (
	ScanOptionArrowUseLargeTypes = "arrow.use_large_types"
	ScanOptionRowLineageEnabled  = "row_lineage.enabled"
)
View Source 
const (
	ArrowFieldDocKey = "doc"
	// Arrow schemas that are generated from the Parquet library will utilize
	// this key to identify the field id of the source Parquet field.
	// We use this when converting to Iceberg to provide field IDs
	ArrowParquetFieldIDKey = "PARQUET:field_id"
)

constants to look for as Keys in Arrow field metadata

View Source 
const (
	// WriteDeleteIsolationLevelKey controls isolation for DeleteFiles
	// and RowDelta eq-delete commits. Default: serializable (matches
	// Java trunk). Older Java releases and some pyiceberg codepaths
	// used "snapshot"; set this property explicitly to snapshot if
	// you are migrating from a pipeline that relied on that behavior.
	WriteDeleteIsolationLevelKey     = "write.delete.isolation-level"
	WriteDeleteIsolationLevelDefault = IsolationSerializable

	// WriteUpdateIsolationLevelKey controls isolation for overwrite /
	// merge commits. Default: serializable (matches Java trunk).
	// Older Java releases and some pyiceberg codepaths used
	// "snapshot"; set this property explicitly to snapshot if you are
	// migrating from a pipeline that relied on that behavior.
	// write.merge.isolation-level is not modeled separately here yet;
	// merge-style producers continue to use write.update.isolation-level.
	WriteUpdateIsolationLevelKey     = "write.update.isolation-level"
	WriteUpdateIsolationLevelDefault = IsolationSerializable
)

Property keys for configuring isolation per write operation. Names match the Java TableProperties constants so configs can be ported directly between engines.

View Source 
const (
	WriteDataPathKey                        = "write.data.path"
	WriteMetadataPathKey                    = "write.metadata.path"
	WriteMetadataLocationKey                = "write.metadata.location"
	WriteObjectStorePartitionedPathsKey     = "write.object-storage.partitioned-paths"
	WriteObjectStorePartitionedPathsDefault = true
	ObjectStoreEnabledKey                   = "write.object-storage.enabled"
	ObjectStoreEnabledDefault               = false

	// ReadSplitTargetSizeKey controls coalescing of safe row-group ranges during
	// local Parquet scan planning. Remote plans already contain catalog-owned
	// ranges and do not use this property.
	ReadSplitTargetSizeKey     = "read.split.target-size"
	ReadSplitTargetSizeDefault = 128 * 1024 * 1024 // 128 MiB

	// ReadManifestListCacheEnabledKey controls reuse of decoded snapshot
	// manifest lists across scans of a table handle. Disable it to avoid
	// retaining manifest descriptors between operations.
	ReadManifestListCacheEnabledKey     = "read.manifest-list-cache.enabled"
	ReadManifestListCacheEnabledDefault = true

	// These Java-compatible properties are reserved for future task-group
	// planning. They are exported so applications can share configuration
	// names across Iceberg implementations.
	ReadSplitPlanningLookbackKey     = "read.split.planning-lookback"
	ReadSplitPlanningLookbackDefault = 10
	ReadSplitOpenFileCostKey         = "read.split.open-file-cost"
	ReadSplitOpenFileCostDefault     = 4 * 1024 * 1024 // 4 MiB

	DefaultNameMappingKey = "schema.name-mapping.default"

	MetricsModeColumnConfPrefix    = "write.metadata.metrics.column"
	DefaultWriteMetricsModeKey     = "write.metadata.metrics.default"
	DefaultWriteMetricsModeDefault = "truncate(16)"

	ParquetRowGroupSizeBytesKey              = internal.ParquetRowGroupSizeBytesKey
	ParquetRowGroupSizeBytesDefault          = internal.ParquetRowGroupSizeBytesDefault
	ParquetRowGroupLimitKey                  = internal.ParquetRowGroupLimitKey
	ParquetRowGroupLimitDefault              = internal.ParquetRowGroupLimitDefault
	ParquetPageSizeBytesKey                  = internal.ParquetPageSizeBytesKey
	ParquetPageSizeBytesDefault              = internal.ParquetPageSizeBytesDefault
	ParquetPageRowLimitKey                   = internal.ParquetPageRowLimitKey
	ParquetPageRowLimitDefault               = internal.ParquetPageRowLimitDefault
	ParquetDictSizeBytesKey                  = internal.ParquetDictSizeBytesKey
	ParquetDictSizeBytesDefault              = internal.ParquetDictSizeBytesDefault
	ParquetPageVersionKey                    = internal.ParquetPageVersionKey
	ParquetPageVersionDefault                = internal.ParquetPageVersionDefault
	ParquetCompressionKey                    = internal.ParquetCompressionKey
	ParquetCompressionDefault                = internal.ParquetCompressionDefault
	ParquetCompressionLevelKey               = internal.ParquetCompressionLevelKey
	ParquetCompressionLevelDefault           = internal.ParquetCompressionLevelDefault
	ParquetBloomFilterMaxBytesKey            = internal.ParquetBloomFilterMaxBytesKey
	ParquetBloomFilterMaxBytesDefault        = internal.ParquetBloomFilterMaxBytesDefault
	ParquetBloomFilterColumnEnabledKeyPrefix = internal.ParquetBloomFilterColumnEnabledKeyPrefix

	ParquetBatchSizeKey     = internal.ParquetBatchSizeKey
	ParquetBatchSizeDefault = internal.ParquetBatchSizeDefault

	ManifestMergeEnabledKey     = "commit.manifest-merge.enabled"
	ManifestMergeEnabledDefault = false

	ManifestTargetSizeBytesKey     = "commit.manifest.target-size-bytes"
	ManifestTargetSizeBytesDefault = 8 * 1024 * 1024 // 8 MB

	ManifestMinMergeCountKey     = "commit.manifest.min-count-to-merge"
	ManifestMinMergeCountDefault = 100

	ManifestMergeMaxConcurrencyKey     = "commit.manifest-merge.max-concurrency"
	ManifestMergeMaxConcurrencyDefault = 0

	WritePartitionSummaryLimitKey     = "write.summary.partition-limit"
	WritePartitionSummaryLimitDefault = 0

	WriteDeleteModeKey     = "write.delete.mode"
	WriteDeleteModeDefault = WriteModeCopyOnWrite

	MetadataDeleteAfterCommitEnabledKey     = "write.metadata.delete-after-commit.enabled"
	MetadataDeleteAfterCommitEnabledDefault = false

	MetadataPreviousVersionsMaxKey     = "write.metadata.previous-versions-max"
	MetadataPreviousVersionsMaxDefault = 100

	MetadataCompressionKey     = "write.metadata.compression-codec"
	MetadataCompressionDefault = "none"

	WriteFormatDefaultKey     = "write.format.default"
	WriteFormatDefaultDefault = "parquet"

	WriteTargetFileSizeBytesKey     = "write.target-file-size-bytes"
	WriteTargetFileSizeBytesDefault = 512 * 1024 * 1024 // 512 MB

	ParquetShredVariantsKey         = internal.ParquetShredVariantsKey
	ParquetShredVariantsDefault     = internal.ParquetShredVariantsDefault
	ParquetVariantBufferSizeKey     = internal.ParquetVariantBufferSizeKey
	ParquetVariantBufferSizeDefault = internal.ParquetVariantBufferSizeDefault

	GCEnabledKey     = "gc.enabled"
	GCEnabledDefault = true

	MinSnapshotsToKeepKey     = "history.expire.min-snapshots-to-keep"
	MinSnapshotsToKeepDefault = 1

	MaxSnapshotAgeMsKey     = "history.expire.max-snapshot-age-ms"
	MaxSnapshotAgeMsDefault = int64(5 * 24 * 60 * 60 * 1000)

	MaxRefAgeMsKey     = "history.expire.max-ref-age-ms"
	MaxRefAgeMsDefault = int64(math.MaxInt64)

	// CommitNumRetriesKey is the number of commit retry attempts before
	// giving up on ErrCommitFailed from the catalog.
	//
	// The default is 0 (no retries). Each retry attempt reloads the
	// current catalog state and replays the update against it (see
	// doCommit's refresh-and-replay loop), so raising this can resolve
	// both transient catalog flakiness (dropped connections, brief 409
	// during leader election) and genuine OCC conflicts from concurrent
	// writers. It stays opt-in by default because a retry rebuilds the
	// snapshot's manifest list and adds latency.
	//
	// Commits that carry delete-file removals (e.g. the
	// deleteFilesToRemove argument of [Transaction.ReplaceFiles], or a
	// v3 merge-on-read delete superseding an existing deletion vector)
	// never replay regardless of this setting: removal identity is
	// snapshot-relative, so those commits fail with ErrCommitFailed on
	// the first conflict. The caller must reload the table and
	// re-resolve the removals against the current snapshot in a new
	// transaction; the failed transaction's staged removals cannot be
	// rebased.
	CommitNumRetriesKey     = "commit.retry.num-retries"
	CommitNumRetriesDefault = 0

	// CommitMinRetryWaitMsKey is the initial wait time in milliseconds
	// for exponential backoff between commit retry attempts. Default: 100ms.
	CommitMinRetryWaitMsKey     = "commit.retry.min-wait-ms"
	CommitMinRetryWaitMsDefault = 100

	// CommitMaxRetryWaitMsKey is the maximum wait time in milliseconds
	// between commit retry attempts. Default: 60s.
	CommitMaxRetryWaitMsKey     = "commit.retry.max-wait-ms"
	CommitMaxRetryWaitMsDefault = 60 * 1000

	// CommitTotalRetryTimeoutMsKey bounds the total time spent across all
	// retry attempts. Default: 30 minutes.
	CommitTotalRetryTimeoutMsKey     = "commit.retry.total-timeout-ms"
	CommitTotalRetryTimeoutMsDefault = 30 * 60 * 1000
)
View Source 
const (
	PropertyFormatVersion            = "format-version"
	PropertyUuid                     = "uuid"
	PropertySnapshotCount            = "snapshot-count"
	PropertyCurrentSnapshotId        = "current-snapshot-id"
	PropertyCurrentSnapshotSummary   = "current-snapshot-summary"
	PropertyCurrentSnapshotTimestamp = "current-snapshot-timestamp"
	PropertyCurrentSchema            = "current-schema"
	PropertyDefaultPartitionSpec     = "default-partition-spec"
	PropertyDefaultSortOrder         = "default-sort-order"
)

Reserved properties

View Source 
const (
	MetadataCompressionCodecNone = "none"
	MetadataCompressionCodecGzip = "gzip"
	MetadataCompressionCodecZstd = "zstd"
)

Metadata compression codecs

View Source 
const (
	WriteModeCopyOnWrite = "copy-on-write"
	WriteModeMergeOnRead = "merge-on-read"
)

Write modes

View Source 
const (
	InitialSortOrderID  = 1
	UnsortedSortOrderID = 0
)
View Source 
const (
	UpdateAddSpec      = "add-spec"
	UpdateAddSchema    = "add-schema"
	UpdateAddSnapshot  = "add-snapshot"
	UpdateAddSortOrder = "add-sort-order"

	UpdateAssignUUID = "assign-uuid"

	UpdateAddEncryptionKey          = "add-encryption-key"
	UpdateRemoveEncryptionKey       = "remove-encryption-key"
	UpdateRemovePartitionStatistics = "remove-partition-statistics"
	UpdateRemoveProperties          = "remove-properties"
	UpdateRemoveSchemas             = "remove-schemas"
	UpdateRemoveSnapshots           = "remove-snapshots"
	UpdateRemoveSnapshotRef         = "remove-snapshot-ref"
	UpdateRemoveSpec                = "remove-partition-specs"
	UpdateRemoveStatistics          = "remove-statistics"

	UpdateSetCurrentSchema       = "set-current-schema"
	UpdateSetDefaultSortOrder    = "set-default-sort-order"
	UpdateSetDefaultSpec         = "set-default-spec"
	UpdateSetLocation            = "set-location"
	UpdateSetPartitionStatistics = "set-partition-statistics"
	UpdateSetProperties          = "set-properties"
	UpdateSetSnapshotRef         = "set-snapshot-ref"
	UpdateSetStatistics          = "set-statistics"

	UpdateUpgradeFormatVersion = "upgrade-format-version"
)

These are the various update actions defined in the iceberg spec

View Source 
const DefaultFormatVersion = 2
View Source 
const MainBranch = "main"
View Source 
const ScanNoLimit = -1
View Source 
const TableRootID = -1

Variables

View Source 
var (
	// ErrConflictingDataFiles is returned when a concurrent commit
	// added data files that satisfy the committer's filter.
	ErrConflictingDataFiles = fmt.Errorf("%w: concurrent data files added", ErrCommitFailed)

	// ErrConflictingDeleteFiles is returned when a concurrent commit
	// added delete files that could mask rows the committer is
	// writing or replacing.
	ErrConflictingDeleteFiles = fmt.Errorf("%w: concurrent delete files added", ErrCommitFailed)

	// ErrDataFilesMissing is returned when a concurrent commit deleted
	// data files the committer explicitly references (e.g. a position
	// delete pointing at a file that is no longer reachable).
	ErrDataFilesMissing = fmt.Errorf("%w: referenced data files missing", ErrCommitFailed)
)

Retryable conflict sentinels. All wrap ErrCommitFailed so the retry loop in doCommit treats them as retryable. Callers that need to distinguish which kind of conflict occurred can match on the specific sentinel via errors.Is.

View Source 
var (
	ErrInvalidMetadataFormatVersion = errors.New("invalid or missing format-version in table metadata")
	ErrInvalidMetadata              = errors.New("invalid metadata")
	ErrPartitionSpecNotFound        = errors.New("partition spec not found")
)
View Source 
var (
	ErrInvalidOperation  = errors.New("invalid operation value")
	ErrMissingOperation  = errors.New("missing operation key")
	ErrInvalidRowLineage = errors.New("invalid row lineage")
)
View Source 
var (
	ErrInvalidSortOrderID   = errors.New("invalid sort order ID")
	ErrInvalidSortSourceID  = errors.New("invalid sort source ID")
	ErrInvalidTransform     = errors.New("invalid transform, must be a valid transform string or a transform object")
	ErrInvalidSortDirection = errors.New("invalid sort direction, must be 'asc' or 'desc'")
	ErrInvalidNullOrder     = errors.New("invalid null order, must be 'nulls-first' or 'nulls-last'")
)
View Source 
var ErrAmbiguousEqualityColumn = errors.New("equality delete column is ambiguous")
View Source 
var ErrCommitDiverged = errors.New("commit diverged: base snapshot is no longer on the branch")

ErrCommitDiverged is returned when the committer's base snapshot is no longer on the branch, so conflict validation cannot enumerate concurrent snapshots. The committer must refresh and rebuild their commit from the new base — naive retry would fail identically.

Unlike the other conflict sentinels in this file, ErrCommitDiverged does NOT wrap ErrCommitFailed: it is terminal for the current attempt. This mirrors Java's ValidationException, which the SnapshotProducer retry machinery does not catch.

View Source 
var ErrCommitFailed = errors.New("commit failed, refresh and try again")

ErrCommitFailed is the sentinel error returned by catalogs when a commit fails due to a concurrent modification (e.g. HTTP 409 Conflict from the REST catalog). Catalog implementations should wrap this error so that callers using errors.Is(err, table.ErrCommitFailed) can detect retryable commit conflicts.

Currently only catalog/rest wraps this sentinel; Glue, SQL, and Hive catalogs return their conflict errors raw and will not trigger retries until follow-up work wires them through (tracked under issue #830).

View Source 
var ErrEmptyEqualityFieldIDs = errors.New("equality field IDs must not be empty")
View Source 
var ErrInvalidIsolationLevel = errors.New("invalid isolation level")

ErrInvalidIsolationLevel is returned when a configured isolation level value is invalid. It is terminal for the current commit attempt and does not wrap ErrCommitFailed, so config errors are not retried as conflicts.

View Source 
var ErrInvalidRefType = errors.New("invalid snapshot ref type, should be 'branch' or 'tag'")
View Source 
var ErrInvalidRequirement = errors.New("invalid requirement")
View Source 
var ErrSnapshotNotFound = errors.New("snapshot not found")

ErrSnapshotNotFound is returned (wrapped) by metadata lookups when a snapshot ID does not exist in the table's snapshot list. Tests pin meaning via errors.Is(err, ErrSnapshotNotFound).

View Source 
var ErrWriteIORequired = fmt.Errorf("%w: file system does not implement WriteFileIO", iceberg.ErrNotImplemented)

ErrWriteIORequired is returned by write paths when the table's file system does not implement io.WriteFileIO. Commit retries also fail fast on this condition because manifest-list rebuilds need write access; skipping that rebuild can reintroduce stale-parent data loss. Callers should use errors.Is(err, ErrWriteIORequired) to detect the precise condition, or errors.Is(err, iceberg.ErrNotImplemented) for compatibility with older WriteRecords behavior.

View Source 
var ErrWriterClosed = errors.New("writer is closed")

ErrWriterClosed is returned when records are added to a writer after its streaming goroutine has already stopped.

View Source 
var PositionalDeleteArrowSchema, _ = SchemaToArrowSchema(iceberg.PositionalDeleteSchema, nil, true, false)
View Source 
var ReservedProperties = [9]string{
	PropertyFormatVersion,
	PropertyUuid,
	PropertySnapshotCount,
	PropertyCurrentSnapshotId,
	PropertyCurrentSnapshotSummary,
	PropertyCurrentSnapshotTimestamp,
	PropertyCurrentSchema,
	PropertyDefaultPartitionSpec,
	PropertyDefaultSortOrder,
}
View Source 
var UnsortedSortOrder = SortOrder{/* contains filtered or unexported fields */}

A default Sort Order indicating no sort order at all

Functions

func AllDataFilesSchema added in v0.7.0 

func AllDataFilesSchema(partitionType *iceberg.StructType) *iceberg.Schema

AllDataFilesSchema returns the schema of the all_data_files metadata table.

func AllDeleteFilesSchema added in v0.7.0 

func AllDeleteFilesSchema(partitionType *iceberg.StructType) *iceberg.Schema

AllDeleteFilesSchema returns the schema of the all_delete_files metadata table.

func AllEntriesSchema added in v0.7.0 

func AllEntriesSchema(partitionType *iceberg.StructType) *iceberg.Schema

AllEntriesSchema returns the schema of the all_entries metadata table.

func AllFilesSchema added in v0.7.0 

func AllFilesSchema(partitionType *iceberg.StructType) *iceberg.Schema

AllFilesSchema returns the schema of the all_files metadata table.

func AllManifestsSchema added in v0.7.0 

func AllManifestsSchema() *iceberg.Schema

AllManifestsSchema returns the schema of the all_manifests metadata table.

func ArrowSchemaToIceberg 

func ArrowSchemaToIceberg(sc *arrow.Schema, downcastNsTimestamp bool, nameMapping iceberg.NameMapping) (*iceberg.Schema, error)

func ArrowSchemaToIcebergWithFreshIDs added in v0.3.0 

func ArrowSchemaToIcebergWithFreshIDs(sc *arrow.Schema, downcastNsTimestamp bool) (*iceberg.Schema, error)

func ArrowSchemaToIcebergWithOptions added in v0.7.0 

func ArrowSchemaToIcebergWithOptions(sc *arrow.Schema, opts ArrowToIcebergOptions) (*iceberg.Schema, error)

ArrowSchemaToIcebergWithOptions converts an Arrow schema to an Iceberg schema. Options carry conversion inputs that are not encoded in the Arrow schema itself, such as table metadata needed for exact CRS recovery. When TableSchema and TableProperties are provided, GeoArrow PROJJSON metadata can be matched back to the table schema's projjson:<property> CRS references.

func ArrowTypeToIceberg 

func ArrowTypeToIceberg(dt arrow.DataType, downcastNsTimestamp bool) (iceberg.Type, error)

func CollectDeadPositionDeletes added in v0.7.0 

func CollectDeadPositionDeletes(
	ctx context.Context,
	fs iceio.IO,
	snap *Snapshot,
	rewrittenPaths map[string]struct{},
) ([]iceberg.DataFile, error)

CollectDeadPositionDeletes walks the given snapshot's manifests and returns the classic position-delete files made dead by a rewrite that removes rewrittenPaths. A returned delete references only data files the rewrite is removing, so it is safe to expunge in the same commit.

A file-scoped delete is resolved through referenced_data_file or equal file_path bounds. A partition-scoped delete is retained when a surviving data file in the same (specID, partition) has a sequence number to which the delete still applies.

Deletion vectors are intentionally excluded: they are one-to-one with their referenced data file and are removed from a rewrite using the task result that identified the rewritten file.

func CollectSafeDeletionVectors added in v0.7.0 

func CollectSafeDeletionVectors(tasks []FileScanTask) []iceberg.DataFile

CollectSafeDeletionVectors returns the tasks' deletion vectors, deduplicated by referenced data file.

Scan planning attaches to a task only the DV referencing its own data file, so every returned DV references a file in the rewrite set. A hand-built FileScanTask carrying a DV for some other live data file would have that DV expunged here — populate DeletionVectorFiles only from scan planning.

func CollectSafePositionDeletes added in v0.6.0 

func CollectSafePositionDeletes(tasks []FileScanTask) []iceberg.DataFile

CollectSafePositionDeletes returns position delete files from the given tasks that are safe to remove during compaction.

A position delete file is safe to remove when it was matched to a data file (via scan planning) and that data file is being rewritten in this compaction group. Since ReadTasks applies the deletes during reading, the new output files will not contain the deleted rows.

Only position deletes (EntryContentPosDeletes) are considered. Equality deletes are decided by [compaction.DecideDeadEqualityDeletesWithSpecs] (which needs partition-wide visibility, not just the task scope). Deletion vectors will be handled when DV read support lands.

Caller contract: every data file referenced by a returned pos-delete must be in the caller's rewrite set across the entire commit. This function only sees one group's tasks, but a pos-delete file can reference data files across multiple groups (the planner bin-packs within a partition via [compaction.Config.PlanCompaction] and skips files via MinInputFiles). If a pos-delete is reported safe by one group but references a still-live data file in another group — or a file the planner skipped — committing only this group's rewrite would orphan the still-live data file's deletes. Coordinators that aggregate multiple groups into one rewrite snapshot are responsible for re-checking against the full set of rewritten paths, or for moving this computation leader-side once worker outputs have aggregated.

ExecuteCompactionGroup calls this internally to populate CompactionGroupResult.SafePosDeletes. It is kept exported for custom workers that want the spec-shaped predicate without taking the rest of ExecuteCompactionGroup's read+write pipeline.

func DataFileFromMetadata added in v0.7.0 

func DataFileFromMetadata(args DataFileArgs) (iceberg.DataFile, error)

DataFileFromMetadata builds a fully populated iceberg.DataFile from a file's in-memory, format-specific metadata object, extracting per-column statistics (column_sizes, value_counts, null_value_counts, lower_bounds, upper_bounds), split offsets, and the manifest-entry shape required for the requested content type. The format-specific extraction is dispatched through the internal FileFormat interface; no filesystem reads are performed.

The returned DataFile is intended to be handed to Transaction.AddDataFiles (data files) or Transaction.NewRowDelta (delete files); it is NOT committed automatically. The file bytes described by args.Metadata must already have been uploaded to args.FilePath.

Scope and limitations:

  • Only Parquet is supported today (args.Format must be iceberg.ParquetFile and args.Metadata a *metadata.FileMetaData).
  • All three manifest-entry contents are supported end to end: data, equality deletes, and positional deletes.
  • Row lineage is supported: a data file's first_row_id can be set via DataFileArgs.FirstRowID for format-version-3+ tables.
  • Iceberg v3 deletion vectors (Puffin DVs) are not supported in this helper function; it emits only Parquet position-delete files. A file-scoped position delete may still record its target via DataFileArgs.ReferencedDataFile; partition-scoped deletes leave it nil, in which case engines may infer scoping from the file_path column bounds.

func DataFilesSchema added in v0.7.0 

func DataFilesSchema(partitionType *iceberg.StructType) *iceberg.Schema

DataFilesSchema returns the common content-file schema used by the files metadata tables. The partition field is omitted for an unpartitioned table, as required by the Iceberg metadata-table spec.

func DeleteFilesSchema added in v0.7.0 

func DeleteFilesSchema(partitionType *iceberg.StructType) *iceberg.Schema

DeleteFilesSchema returns the content-file schema used by the delete_files metadata table. It is shared with DataFilesSchema so both tables expose the same partition-evolution and nested metric semantics.

func EntriesSchema added in v0.7.0 

func EntriesSchema(partitionType *iceberg.StructType) *iceberg.Schema

EntriesSchema returns the schema of the entries metadata table.

func FileToDataFile added in v0.7.0 

func FileToDataFile(ctx context.Context, fileIO iceio.IO, filePath string, currentSchema *iceberg.Schema, currentSpec iceberg.PartitionSpec, sortOrderID int, props iceberg.Properties) (df iceberg.DataFile, err error)

FileToDataFile builds a DataFile for an existing parquet file at filePath, reading its footer to populate record count, file size, column sizes, value/null counts and lower/upper bounds, and to infer partition values for order-preserving transforms. sortOrderID is stamped onto the DataFile so callers converting foreign parquet footers can convey the file's sort layout (spec data-file field sort_order_id); pass 0 to make no claim. Panics from the unexported conversion are recovered into an error.

func FilesSchema added in v0.7.0 

func FilesSchema(partitionType *iceberg.StructType) *iceberg.Schema

FilesSchema returns the schema shared by the files metadata tables.

func GetPartitionRecord added in v0.6.0 

func GetPartitionRecord(dataFile iceberg.DataFile, partitionType *iceberg.StructType) iceberg.StructLike

GetPartitionRecord converts a DataFile's partition map into a positional record ordered by the fields of the given partition struct type.

func HistorySchema added in v0.7.0 

func HistorySchema() *iceberg.Schema

HistorySchema returns the Iceberg schema of the history metadata table. The field IDs are fixed by the Iceberg metadata-tables spec and match the Java, PyIceberg, and Rust clients for cross-client parity. A fresh schema value is returned on each call; callers must not mutate it or rely on pointer identity.

func IsAncestorOf added in v0.6.0 

func IsAncestorOf(snapshotID, ancestorID int64, lookup SnapshotLookup) bool

IsAncestorOf returns true if ancestorID is in the parent chain of snapshotID (or equal to snapshotID, provided snapshotID resolves).

Returns false if snapshotID cannot be resolved, if ancestorID is not reachable from snapshotID via ParentSnapshotID, or if a cycle prevents reaching ancestorID.

Note: every snapshot on the walked chain must be resolvable via lookup. This diverges from Java's SnapshotUtil.isAncestorOf which walks parent *ids* regardless of snapshot resolvability. The Go semantics are stricter (conservative) — broken chains return false rather than matching an unreachable ancestor id.

func IsDeletionVector added in v0.7.0 

func IsDeletionVector(df iceberg.DataFile) bool

IsDeletionVector reports whether df is a deletion vector: a Puffin file with position-delete content. The content-type guard matters because df is an arbitrary DataFile, so a non-pos-delete Puffin from an external writer must not be misclassified. Keying on format rather than referenced_data_file avoids misclassifying a Parquet pos-delete that legally sets it.

func ManifestsSchema added in v0.7.0 

func ManifestsSchema() *iceberg.Schema

ManifestsSchema returns the Iceberg schema of the manifests metadata table.

func MetadataLogEntriesSchema added in v0.7.0 

func MetadataLogEntriesSchema() *iceberg.Schema

MetadataLogEntriesSchema returns a fresh Iceberg schema for the metadata-log-entries metadata table. The field IDs and names match Java's implementation; callers should not rely on pointer identity.

func NewAddEncryptionKeyUpdate added in v0.6.0 

func NewAddEncryptionKeyUpdate(key EncryptionKey) *addEncryptionKeyUpdate

NewAddEncryptionKeyUpdate creates a new Update that adds or replaces an encryption key (indexed by its key-id) in the table metadata.

func NewAddPartitionSpecUpdate added in v0.2.0 

func NewAddPartitionSpecUpdate(spec *iceberg.PartitionSpec, initial bool) *addPartitionSpecUpdate

NewAddPartitionSpecUpdate creates a new update that adds the given partition spec to the table metadata. If the initial flag is set to true, the spec is considered the initial spec of the table, and all other previously added specs in the metadata builder are removed.

func NewAddSchemaUpdate added in v0.2.0 

func NewAddSchemaUpdate(schema *iceberg.Schema) *addSchemaUpdate

NewAddSchemaUpdate creates a new update that adds the given schema and updates the lastColumnID based on the schema.

func NewAddSnapshotUpdate added in v0.2.0 

func NewAddSnapshotUpdate(snapshot *Snapshot) *addSnapshotUpdate

NewAddSnapshotUpdate creates a new update that adds the given snapshot to the table metadata.

func NewAddSortOrderUpdate added in v0.2.0 

func NewAddSortOrderUpdate(sortOrder *SortOrder) *addSortOrderUpdate

NewAddSortOrderUpdate creates a new update that adds the given sort order to the table metadata. If the initial flag is set to true, the sort order is considered the initial sort order of the table, and all previously added sort orders in the metadata builder are removed.

func NewAssignUUIDUpdate added in v0.2.0 

func NewAssignUUIDUpdate(uuid uuid.UUID) *assignUUIDUpdate

NewAssignUUIDUpdate creates a new update to assign a UUID to the table metadata.

func NewRemoveEncryptionKeyUpdate added in v0.6.0 

func NewRemoveEncryptionKeyUpdate(keyID string) *removeEncryptionKeyUpdate

NewRemoveEncryptionKeyUpdate creates a new Update that removes the encryption key with the given key-id from the table metadata.

func NewRemovePartitionStatisticsUpdate added in v0.6.0 

func NewRemovePartitionStatisticsUpdate(snapshotID int64) *removePartitionStatisticsUpdate

NewRemovePartitionStatisticsUpdate creates a new Update that removes the partition statistics file for the given snapshot ID from the table metadata.

func NewRemovePropertiesUpdate added in v0.2.0 

func NewRemovePropertiesUpdate(removals []string) *removePropertiesUpdate

NewRemovePropertiesUpdate creates a new update that removes properties from the table metadata. The properties are identified by their names, and if a property with the given name does not exist, it is ignored.

func NewRemoveSchemasUpdate added in v0.4.0 

func NewRemoveSchemasUpdate(schemaIds []int) *removeSchemasUpdate

NewRemoveSchemasUpdate creates a new Update that removes a list of schemas from the table metadata.

func NewRemoveSnapshotRefUpdate added in v0.2.0 

func NewRemoveSnapshotRefUpdate(ref string) *removeSnapshotRefUpdate

NewRemoveSnapshotRefUpdate creates a new update that removes a snapshot reference from the table metadata.

func NewRemoveSnapshotsUpdate added in v0.2.0 

func NewRemoveSnapshotsUpdate(ids []int64, postCommit bool) *removeSnapshotsUpdate

NewRemoveSnapshotsUpdate creates a new update that removes all snapshots from the table metadata with the given snapshot IDs.

func NewRemoveSpecUpdate added in v0.4.0 

func NewRemoveSpecUpdate(specIds []int) *removeSpecUpdate

NewRemoveSpecUpdate creates a new Update that removes a list of partition specs from the table metadata.

func NewRemoveStatisticsUpdate added in v0.6.0 

func NewRemoveStatisticsUpdate(snapshotID int64) *removeStatisticsUpdate

NewRemoveStatisticsUpdate creates a new Update that removes the statistics file for the given snapshot ID from the table metadata.

func NewSetCurrentSchemaUpdate added in v0.2.0 

func NewSetCurrentSchemaUpdate(id int) *setCurrentSchemaUpdate

NewSetCurrentSchemaUpdate creates a new update that sets the current schema of the table metadata to the given schema ID.

func NewSetDefaultSortOrderUpdate added in v0.2.0 

func NewSetDefaultSortOrderUpdate(id int) *setDefaultSortOrderUpdate

NewSetDefaultSortOrderUpdate creates a new update that sets the default sort order of the table metadata to the given sort order ID.

func NewSetDefaultSpecUpdate added in v0.2.0 

func NewSetDefaultSpecUpdate(id int) *setDefaultSpecUpdate

NewSetDefaultSpecUpdate creates a new update that sets the default partition spec of the table metadata to the given spec ID.

func NewSetPartitionStatisticsUpdate added in v0.6.0 

func NewSetPartitionStatisticsUpdate(stats PartitionStatisticsFile) *setPartitionStatisticsUpdate

NewSetPartitionStatisticsUpdate creates a new Update that adds or replaces the partition statistics file for the given snapshot ID in the table metadata.

func NewSetPropertiesUpdate added in v0.2.0 

func NewSetPropertiesUpdate(updates iceberg.Properties) *setPropertiesUpdate

NewSetPropertiesUpdate creates a new update that sets the given properties in the table metadata.

func NewSetSnapshotRefUpdate added in v0.2.0 

func NewSetSnapshotRefUpdate(
	name string,
	snapshotID int64,
	refType RefType,
	maxRefAgeMs, maxSnapshotAgeMs int64,
	minSnapshotsToKeep int,
) *setSnapshotRefUpdate

NewSetSnapshotRefUpdate creates a new update that sets the given snapshot reference as the current snapshot of the table metadata. MaxRefAgeMs, MaxSnapshotAgeMs, and MinSnapshotsToKeep are optional, and any non-positive values are ignored.

func NewSetStatisticsUpdate added in v0.6.0 

func NewSetStatisticsUpdate(stats StatisticsFile) *setStatisticsUpdate

NewSetStatisticsUpdate creates a new Update that adds or replaces the statistics file for the given snapshot ID in the table metadata.

func NewUpgradeFormatVersionUpdate added in v0.2.0 

func NewUpgradeFormatVersionUpdate(formatVersion int) *upgradeFormatVersionUpdate

NewUpgradeFormatVersionUpdate creates a new update that upgrades the format version of the table metadata to the given formatVersion.

func PartitionsSchema added in v0.7.0 

func PartitionsSchema(partitionType *iceberg.StructType) *iceberg.Schema

PartitionsSchema returns the partition metadata-table schema. For an unpartitioned table the partition and spec_id columns are omitted, matching Java's PartitionsTable behavior.

func PositionDeletesSchema added in v0.7.0 

func PositionDeletesSchema(
	tableSchema *iceberg.Schema,
	partitionType *iceberg.StructType,
	formatVersion int,
) *iceberg.Schema

PositionDeletesSchema returns the schema of the position_deletes metadata table.

func RefsSchema added in v0.7.0 

func RefsSchema() *iceberg.Schema

RefsSchema returns a fresh Iceberg schema for the refs metadata table. The field IDs and names match Java's RefsTable for cross-client parity; callers should not rely on pointer identity.

func SchemaToArrowSchema 

func SchemaToArrowSchema(sc *iceberg.Schema, metadata map[string]string, includeFieldIDs, useLargeTypes bool) (*arrow.Schema, error)

SchemaToArrowSchema converts an Iceberg schema to an Arrow schema. If the metadata parameter is non-nil, it will be included as the top-level metadata in the schema. If includeFieldIDs is true, then each field of the schema will contain a metadata key PARQUET:field_id set to the field id from the iceberg schema.

func SchemaToArrowSchemaWithOptions added in v0.7.0 

func SchemaToArrowSchemaWithOptions(sc *iceberg.Schema, opts ArrowSchemaOptions) (*arrow.Schema, error)

SchemaToArrowSchemaWithOptions converts an Iceberg schema to an Arrow schema. TableProperties are used to resolve references like projjson:<property> CRS which point to metadata in the table properties.

func SnapshotsSchema added in v0.7.0 

func SnapshotsSchema() *iceberg.Schema

SnapshotsSchema returns the Iceberg schema of the snapshots metadata table. The field IDs are fixed by the Iceberg metadata-tables spec and match the Java, PyIceberg, and Rust clients for cross-client parity. A fresh schema value is returned on each call; callers must not mutate it or rely on pointer identity.

The summary map's values are optional, matching PyIceberg and Rust. Java declares them required; the choice is not observable in practice because snapshot-summary values are always non-null strings.

func ToRequestedSchema 

func ToRequestedSchema(ctx context.Context, requested, fileSchema *iceberg.Schema, batch arrow.RecordBatch, opts SchemaOptions) (arrow.RecordBatch, error)

ToRequestedSchema will construct a new record batch matching the requested iceberg schema casting columns if necessary as appropriate.

func TypeToArrowType 

func TypeToArrowType(t iceberg.Type, includeFieldIDs bool, useLargeTypes bool) (arrow.DataType, error)

TypeToArrowType converts a given iceberg type, into the equivalent Arrow data type. For dealing with nested fields (List, Struct, Map) if includeFieldIDs is true, then the child fields will contain a metadata key PARQUET:field_id set to the field id.

func TypeToArrowTypeWithOptions added in v0.7.0 

func TypeToArrowTypeWithOptions(t iceberg.Type, opts ArrowSchemaOptions) (arrow.DataType, error)

TypeToArrowTypeWithOptions converts a given Iceberg type into the equivalent Arrow data type. Options control representation details and provide table properties used to resolve metadata references such as CRS projjson:<property> which are stored as table properties.

func VisitArrowSchema 

func VisitArrowSchema[T any](sc *arrow.Schema, visitor ArrowSchemaVisitor[T]) (res T, err error)

func WithMaxRefAgeMs added in v0.2.0 

func WithMaxRefAgeMs(maxRefAgeMs int64) setSnapshotRefOption

func WithMaxSnapshotAgeMs added in v0.2.0 

func WithMaxSnapshotAgeMs(maxSnapshotAgeMs int64) setSnapshotRefOption

func WithMinSnapshotsToKeep added in v0.2.0 

func WithMinSnapshotsToKeep(minSnapshotsToKeep int) setSnapshotRefOption

func WriteRecords added in v0.6.0 

func WriteRecords(ctx context.Context, tbl *Table,
	schema *arrow.Schema,
	records iter.Seq2[arrow.RecordBatch, error],
	opts ...WriteRecordOption,
) iter.Seq2[iceberg.DataFile, error]

WriteRecords writes Arrow record batches to Parquet data files for the given table, returning an iterator of the resulting DataFile objects.

The provided Arrow schema must be compatible with the table's current Iceberg schema: each field in the Arrow schema is matched to the table schema by field ID (or by name via the table's name mapping if field IDs are absent). The Arrow schema may be a subset of the table schema (projection), but every field present must have a type that is promotable to the corresponding table field type. When the table uses microsecond timestamps, Arrow nanosecond timestamps are also accepted and are downcast during the write path.

WriteRecords releases each RecordBatch it consumes. If the caller needs a batch to remain valid after it has been yielded, it must call Retain before yielding and is then responsible for the corresponding Release.

Types

type AddFilesOption added in v0.6.0 

type AddFilesOption func(addFilesOp *addFilesOperation)

func WithAddFilesConcurrency added in v0.6.0 

func WithAddFilesConcurrency(concurrency int) AddFilesOption

WithAddFilesConcurrency overwrites the default concurrency for add files operation. Default: runtime.GOMAXPROCS(0)

type AddedRowsScanTask added in v0.7.0 

type AddedRowsScanTask struct {
	FileScanTask
	// contains filtered or unexported fields
}

AddedRowsScanTask is a changelog insert produced by adding a data file. Matching delete files committed in the same snapshot, or from squashed snapshots, are applied while reading so deleted rows are not emitted as inserts.

func NewAddedRowsScanTask added in v0.7.0 

func NewAddedRowsScanTask(dataFile iceberg.DataFile, deletes []iceberg.DataFile, changeOrdinal int, commitSnapshotID int64) (AddedRowsScanTask, error)

NewAddedRowsScanTask constructs an insert task for dataFile. deletes are delete files that apply while reading the added file. Position deletes, equality deletes, and deletion vectors are stored on the matching FileScanTask fields.

func (AddedRowsScanTask) ChangeOrdinal added in v0.7.0 

func (t AddedRowsScanTask) ChangeOrdinal() int

func (AddedRowsScanTask) CommitSnapshotID added in v0.7.0 

func (t AddedRowsScanTask) CommitSnapshotID() int64

func (AddedRowsScanTask) Deletes added in v0.7.0 

func (t AddedRowsScanTask) Deletes() []iceberg.DataFile

Deletes returns every delete file applied while reading the added data file: position deletes, then equality deletes, then deletion vectors.

func (AddedRowsScanTask) Operation added in v0.7.0 

func (t AddedRowsScanTask) Operation() ChangelogOperation

func (AddedRowsScanTask) ScanTask added in v0.7.0 

func (t AddedRowsScanTask) ScanTask() FileScanTask

type ArrowSchemaOptions added in v0.7.0 

type ArrowSchemaOptions struct {
	Metadata        map[string]string
	IncludeFieldIDs bool
	UseLargeTypes   bool
	TableProperties iceberg.Properties
}

ArrowSchemaOptions controls conversion from an Iceberg schema or type to Arrow.

type ArrowSchemaVisitor 

type ArrowSchemaVisitor[T any] interface {
	Schema(*arrow.Schema, T) T
	Struct(*arrow.StructType, []T) T
	Field(arrow.Field, T) T
	List(arrow.ListLikeType, T) T
	Map(mt *arrow.MapType, keyResult T, valueResult T) T
	Primitive(arrow.DataType) T
}

ArrowSchemaVisitor is an interface that can be implemented and used to call VisitArrowSchema for iterating

type ArrowToIcebergOptions added in v0.7.0 

type ArrowToIcebergOptions struct {
	DowncastNsTimestamp bool
	NameMapping         iceberg.NameMapping
	TableSchema         *iceberg.Schema
	TableProperties     iceberg.Properties
}

ArrowToIcebergOptions controls conversion from an Arrow schema to an Iceberg schema.

type BlobMetadata added in v0.4.0 

type BlobMetadata struct {
	Type           BlobType          `json:"type"`
	SnapshotID     int64             `json:"snapshot-id"`
	SequenceNumber int64             `json:"sequence-number"`
	Fields         []int32           `json:"fields"`
	Properties     map[string]string `json:"properties"`
}

BlobMetadata is the metadata of a statistics or indices blob.

type BlobType added in v0.4.0 

type BlobType string

BlobType is the type of blob in a Puffin file. 

const (
	BlobTypeApacheDatasketchesThetaV1 BlobType = "apache-datasketches-theta-v1"
	BlobTypeDeletionVectorV1          BlobType = "deletion-vector-v1"
)

func (BlobType) IsKnown added in v0.7.0 

func (bt BlobType) IsKnown() bool

IsKnown reports whether bt is one of the standard blob types understood by this package. Unknown non-empty wire values may still be preserved for forward compatibility.

func (*BlobType) IsValid added in v0.4.0 

func (bt *BlobType) IsValid() bool

IsValid reports whether bt is non-nil and names a standard blob type. Unknown non-empty values can be decoded and preserved, but are not valid for operations that require a known blob type.

func (*BlobType) UnmarshalJSON added in v0.4.0 

func (bt *BlobType) UnmarshalJSON(data []byte) error

type CatalogIO added in v0.2.0 

type CatalogIO interface {
	LoadTable(context.Context, Identifier) (*Table, error)
	CommitTable(context.Context, Identifier, []Requirement, []Update) (Metadata, string, error)
}

type ChangelogOperation added in v0.7.0 

type ChangelogOperation string

ChangelogOperation is the kind of change a changelog scan task produces. 

const (
	ChangelogOpInsert       ChangelogOperation = "INSERT"
	ChangelogOpDelete       ChangelogOperation = "DELETE"
	ChangelogOpUpdateBefore ChangelogOperation = "UPDATE_BEFORE"
	ChangelogOpUpdateAfter  ChangelogOperation = "UPDATE_AFTER"
)

type ChangelogScanTask added in v0.7.0 

type ChangelogScanTask interface {

	// ScanTask returns the underlying file scan task for reading the data file.
	ScanTask() FileScanTask
	Operation() ChangelogOperation
	ChangeOrdinal() int
	CommitSnapshotID() int64
	// contains filtered or unexported methods
}

ChangelogScanTask is a unit of work that produces changelog rows.

type ColumnUpdate added in v0.5.0 

type ColumnUpdate struct {
	Name         iceberg.Optional[string]
	FieldType    iceberg.Optional[iceberg.Type]
	Required     iceberg.Optional[bool]
	WriteDefault iceberg.Optional[iceberg.Literal]
	Doc          iceberg.Optional[string]
}

type CompactionGroupFailure added in v0.7.0 

type CompactionGroupFailure struct {
	PartitionKey string
	Err          error
}

CompactionGroupFailure records a group that could not be committed during a partial-progress rewrite.

type CompactionGroupOption added in v0.6.0 

type CompactionGroupOption func(*compactionGroupConfig)

CompactionGroupOption configures a single ExecuteCompactionGroup call. Use the With* helpers to construct values.

func WithCompactionArrowBatchSize added in v0.7.0 

func WithCompactionArrowBatchSize(n int) CompactionGroupOption

WithCompactionArrowBatchSize caps the number of rows decoded per Arrow record batch while reading the group's tasks, forwarded to the scan as WithArrowBatchSize. The record pipeline holds at most 

workers x (rows in the largest task + n) + (recordBatchBufferSize + 2) x n

rows, where workers is the scan's worker count (see WithCompactionScanConcurrency) and recordBatchBufferSize is the WithCompactionRecordBatchBufferSize value. Each scan worker holds the batches of at most one task at a time, in its hands, in the records channel, in the reorder heap or in the sequenced channel, and does not start its next task until the last batch of the previous one has been consumed; a task's batches total at most its row count rounded up to a multiple of n. The clustered writer holds one batch while routing it, recordBatchBufferSize batches in its input channel and one batch while encoding. Not covered: the Parquet reader's own buffers, including the raw batch it last returned, and delete-side memory, since positional deletes and deletion-vector bitmaps for the group's tasks are materialized up front and sized by delete volume, not by these knobs. A non-positive value keeps the table's read.parquet.batch-size property.

func WithCompactionParquetRowGroupLimit added in v0.7.0 

func WithCompactionParquetRowGroupLimit(n int) CompactionGroupOption

WithCompactionParquetRowGroupLimit caps the rows per Parquet row group in the compacted output files, forwarded to WriteRecords as WithParquetRowGroupLimit. A non-positive value keeps the table's write.parquet.row-group-limit property.

func WithCompactionRecordBatchBufferSize added in v0.7.0 

func WithCompactionRecordBatchBufferSize(n int) CompactionGroupOption

WithCompactionRecordBatchBufferSize sets the capacity, in record batches, of the write pipeline's per-writer input buffer, forwarded to WriteRecords as WithRecordBatchBufferSize. The default is 64 batches. A non-positive value is ignored.

func WithCompactionScanConcurrency added in v0.6.0 

func WithCompactionScanConcurrency(n int) CompactionGroupOption

WithCompactionScanConcurrency sets the scan concurrency used when reading the group's tasks. Forwarded to Table.Scan as WithMaxConcurrency. Zero (the default) means runtime.GOMAXPROCS. The scan runs min(n, number of tasks) workers, and each worker holds the decoded batches of at most one task until the writer has taken them, so the worker count multiplies the read-side term of the memory bound stated on WithCompactionArrowBatchSize.

func WithCompactionTargetFileSize added in v0.6.0 

func WithCompactionTargetFileSize(size int64) CompactionGroupOption

WithCompactionTargetFileSize sets the size target for output files written by ExecuteCompactionGroup. Forwarded to WriteRecords as WithTargetFileSize. A non-positive value (including the zero default) means inherit the table's `write.target-file-size-bytes` property.

type CompactionGroupResult added in v0.6.0 

type CompactionGroupResult struct {
	// PartitionKey mirrors [CompactionTaskGroup.PartitionKey] for
	// display/logging on the coordinator.
	PartitionKey string

	// OldDataFiles are the data files this group replaces.
	OldDataFiles []iceberg.DataFile

	// NewDataFiles are the consolidated outputs the worker wrote.
	NewDataFiles []iceberg.DataFile

	// SafePosDeletes are position-delete files referenced by tasks in
	// this group whose target data file is being rewritten, computed
	// via [CollectSafePositionDeletes]. Partial-progress callers must
	// re-check them against the complete commit batch before expunging
	// them because one file can reference data files in multiple groups.
	SafePosDeletes []iceberg.DataFile

	// SafeDeletionVectors are deletion vectors attached to tasks in this
	// group, computed via [CollectSafeDeletionVectors]. Each is bound to
	// a data file being rewritten, so all are safe to expunge.
	SafeDeletionVectors []iceberg.DataFile

	// BytesBefore is [CompactionTaskGroup.TotalSizeBytes] passed
	// through, recorded so the coordinator can roll up metrics
	// without re-reading the plan.
	BytesBefore int64

	// BytesAfter is the sum of [iceberg.DataFile.FileSizeBytes] across
	// NewDataFiles.
	BytesAfter int64
}

CompactionGroupResult is the per-group output of a compaction worker: the new files written, the old files being replaced, and the position delete files safe to expunge in the rewrite snapshot.

A distributed coordinator aggregates results from N workers and applies them to a RewriteFiles builder via RewriteFiles.ApplyResult to commit a single atomic snapshot. Each field is plain data ([]iceberg.DataFile values plus scalars) — callers serialize the contained DataFiles across process boundaries themselves; the typical pattern is to have the worker write a manifest containing the new files and ship the manifest path to the coordinator, which re-reads it.

func ExecuteCompactionGroup added in v0.6.0 

func ExecuteCompactionGroup(ctx context.Context, tbl *Table, group CompactionTaskGroup, opts ...CompactionGroupOption) (CompactionGroupResult, error)

ExecuteCompactionGroup reads a compaction group's tasks (with deletes applied), writes consolidated output files via WriteRecords, and computes the position-delete files safe to expunge in the rewrite snapshot. It does not commit — the caller hands the result to a coordinator that uses Transaction.NewRewrite + RewriteFiles.ApplyResult + RewriteFiles.Commit to stage the atomic commit.

Empty groups return a zero CompactionGroupResult without doing any I/O.

In-process callers should prefer Transaction.RewriteDataFiles, which drives this and the commit step in one call.

Tunables are exposed via CompactionGroupOption. The clustered write path is always used (a compaction group is single-partition by construction so its read stream is trivially clustered).

type CompactionTaskGroup added in v0.6.0 

type CompactionTaskGroup struct {
	// PartitionKey is an opaque grouping key for display/logging.
	PartitionKey string

	// Tasks are the FileScanTasks to compact.
	Tasks []FileScanTask

	// TotalSizeBytes is the sum of data file sizes in this group.
	TotalSizeBytes int64
}

CompactionTaskGroup is a set of scan tasks in the same partition that should be compacted together. This bridges the compaction planner (table/compaction package) and the executor, avoiding a circular import between table and table/compaction.

Use [compaction.Config.PlanCompaction] to produce groups, then convert [compaction.Group] → CompactionTaskGroup to call Transaction.RewriteDataFiles or ExecuteCompactionGroup.

type DataFileArgs added in v0.7.0 

type DataFileArgs struct {
	// Schema is the Iceberg schema the file's contents conform to.
	// Per-column statistics are matched by leaf-column path: each file
	// column's dotted path MUST equal the path derived from this schema's
	// field names. Embedded field-id metadata in the file (e.g. the Parquet
	// "PARQUET:field_id" key) is not consulted, so a column renamed relative
	// to the file will not resolve.
	Schema *iceberg.Schema

	// Spec is the partition spec the resulting DataFile is bound to —
	// typically tbl.Spec(). The zero value [iceberg.PartitionSpec]{} is
	// equivalent to *iceberg.UnpartitionedSpec and is accepted for
	// unpartitioned tables.
	Spec iceberg.PartitionSpec

	// Format identifies the on-disk file format and selects the internal
	// FileFormat implementation used to extract per-column statistics from
	// Metadata. It must match the concrete type of Metadata: the only supported
	// combination today is [iceberg.ParquetFile] with a *metadata.FileMetaData.
	Format iceberg.FileFormat

	// Metadata is the format-specific, in-memory file metadata object produced
	// by the external writer.
	//
	// Today only Format == [iceberg.ParquetFile] with *metadata.FileMetaData is supported
	Metadata any

	// FilePath is the fully qualified location the file bytes were
	// written to (the same path that will be recorded in the manifest entry).
	FilePath string

	// FileSize is the byte length of the object at FilePath. Must be
	// greater than zero.
	FileSize int64

	// Content selects which manifest-entry kind to produce — one of
	// [iceberg.EntryContentData], [iceberg.EntryContentEqDeletes] or
	// [iceberg.EntryContentPosDeletes].
	Content iceberg.ManifestEntryContent

	// PartitionValues maps spec field ID → partition value. Leave the map
	// nil/empty for unpartitioned tables, where any supplied value is rejected.
	//
	// Values must already be the transform results in the expected Go types.
	// A field is resolved as follows:
	//   - key present with a non-nil value: that value is recorded verbatim;
	//   - key present with a nil value, or key absent: treated as "not supplied"
	//     and, for order-preserving transforms (identity, truncate, the date/time
	//     transforms), inferred from the file's column statistics; for
	//     non-order-preserving transforms (bucket, void) it stays nil.
	//
	// Void-transform fields may be omitted (they always resolve to nil).
	//
	// This helper validates only that any supplied field IDs belong to the spec
	// (no stray IDs); it does not coerce or type-check the values themselves.
	// Callers are responsible for supplying values of the correct type.
	PartitionValues map[int]any

	// SortOrderID is the sort-order ID recorded on the manifest entry,
	// typically tbl.SortOrder().OrderID(). Zero means "unsorted".
	SortOrderID int

	// Properties are consulted only for write-side metrics-mode keys
	// (write.metadata.metrics.default and per-column
	// write.metadata.metrics.column.<name> overrides) that decide
	// which columns get full vs truncated vs no statistics. Pass
	// tbl.Properties() for table-default behavior; nil is safe.
	Properties iceberg.Properties

	// EqualityFieldIDs lists the schema field IDs that an equality
	// delete file matches on. Required when
	// Content == [iceberg.EntryContentEqDeletes] and MUST be empty
	// for every other Content value. Every listed ID must be present in Schema.
	EqualityFieldIDs []int

	// FormatVersion is the table's format version
	FormatVersion int

	// FirstRowID sets the data file's first_row_id (requires FormatVersion >= 3).
	// It must always be nil for delete files, and for v1/v2 data files (the
	// field is absent from those schemas).
	//
	// For v3 data files, whether it is required depends on which consuming API
	// the resulting DataFile is handed to:
	//   - [Transaction.AddDataFiles]: REQUIRED. These are externally-written
	//     files, so the library cannot fabricate row IDs for them; a nil
	//     first_row_id is rejected at commit time.
	//   - [Transaction.ReplaceDataFilesWithDataFiles]: REQUIRED for the added
	//     files, except when the caller opts into rewrite/compaction semantics
	//     (WithRewriteSemantics), where the manifest-list writer assigns it by
	//     inheritance and it may be left nil.
	//   - [Transaction.NewRowDelta] (via RowDelta.AddRows): NOT required — the
	//     value is assigned by inheritance from the manifest at commit time, so
	//     leaving it nil is correct.
	//
	// In short: set it explicitly whenever the file goes through AddDataFiles
	// (or a non-rewrite replace); otherwise leave it nil and let inheritance
	// assign it.
	FirstRowID *int64

	// ReferencedDataFile, when non-nil, records the single data file that a
	// file-scoped position-delete file applies to.
	//
	// It is only valid for position delete files. Leave it nil for data files, equality-delete files,
	// and partition-scoped position deletes.
	//
	// For partition-scoped position deletes whose file_path column references more
	// than one data file (cardinality > 1) — such a file has no single referenced data file.
	ReferencedDataFile *string
}

DataFileArgs collects everything needed to compute a fully populated iceberg.DataFile (data, equality-delete, or positional-delete) from a file's in-memory, format-specific metadata object without performing any filesystem reads.

type DeleteFileAddition added in v0.7.0 

type DeleteFileAddition struct {
	File               iceberg.DataFile
	DataSequenceNumber int64
}

DeleteFileAddition is a delete file added by a rewrite. The data sequence number must be copied from the delete files being replaced so the new file has the same applicability boundary as the old files. When one rewrite contains multiple replacement groups, callers are responsible for assigning each output the maximum sequence number of the exact source files it replaces; the transaction can validate membership in the overall source sequence set, but the grouping is not represented by this API.

type DeleteOption added in v0.5.0 

type DeleteOption func(deleteOp *deleteOperation)

func WithDeleteCaseInsensitive added in v0.5.0 

func WithDeleteCaseInsensitive() DeleteOption

WithDeleteCaseInsensitive changes the binding of the filter to be case insensitive instead of the Default: case sensitive Note that the sensitivity only applies to the field name and not the evaluation of the literals on string fields.

func WithDeleteConcurrency added in v0.5.0 

func WithDeleteConcurrency(concurrency int) DeleteOption

WithDeleteConcurrency overwrites the default concurrency for delete operations. Default: runtime.GOMAXPROCS(0)

type DeletedDataFileScanTask added in v0.7.0 

type DeletedDataFileScanTask struct {
	FileScanTask
	// contains filtered or unexported fields
}

DeletedDataFileScanTask is a changelog delete produced by removing a data file. ExistingDeletes are delete files that were already present and must be applied so only rows that were live when the file was removed appear as deletes.

func NewDeletedDataFileScanTask added in v0.7.0 

func NewDeletedDataFileScanTask(dataFile iceberg.DataFile, existingDeletes []iceberg.DataFile, changeOrdinal int, commitSnapshotID int64) (DeletedDataFileScanTask, error)

NewDeletedDataFileScanTask constructs a delete task for a removed data file.

func (DeletedDataFileScanTask) ChangeOrdinal added in v0.7.0 

func (t DeletedDataFileScanTask) ChangeOrdinal() int

func (DeletedDataFileScanTask) CommitSnapshotID added in v0.7.0 

func (t DeletedDataFileScanTask) CommitSnapshotID() int64

func (DeletedDataFileScanTask) ExistingDeletes added in v0.7.0 

func (t DeletedDataFileScanTask) ExistingDeletes() []iceberg.DataFile

ExistingDeletes returns delete files that applied before the data file was removed.

func (DeletedDataFileScanTask) Operation added in v0.7.0 

func (t DeletedDataFileScanTask) Operation() ChangelogOperation

func (DeletedDataFileScanTask) ScanTask added in v0.7.0 

func (t DeletedDataFileScanTask) ScanTask() FileScanTask

type DeletedRowsScanTask added in v0.7.0 

type DeletedRowsScanTask struct {
	FileScanTask
	// contains filtered or unexported fields
}

DeletedRowsScanTask is a changelog delete produced by adding delete files against a data file that remains in the table. AddedDeletes remove rows that should appear in the changelog. ExistingDeletes already applied and those rows must not be emitted again.

func NewDeletedRowsScanTask added in v0.7.0 

func NewDeletedRowsScanTask(dataFile iceberg.DataFile, addedDeletes, existingDeletes []iceberg.DataFile, changeOrdinal int, commitSnapshotID int64) (DeletedRowsScanTask, error)

NewDeletedRowsScanTask constructs a row-level delete task. existingDeletes are stored on the embedded FileScanTask so later readers can reuse the normal scan delete path for the live-row baseline.

func (DeletedRowsScanTask) AddedDeletes added in v0.7.0 

func (t DeletedRowsScanTask) AddedDeletes() []iceberg.DataFile

AddedDeletes returns delete files whose removals should appear in the changelog.

func (DeletedRowsScanTask) ChangeOrdinal added in v0.7.0 

func (t DeletedRowsScanTask) ChangeOrdinal() int

func (DeletedRowsScanTask) CommitSnapshotID added in v0.7.0 

func (t DeletedRowsScanTask) CommitSnapshotID() int64

func (DeletedRowsScanTask) ExistingDeletes added in v0.7.0 

func (t DeletedRowsScanTask) ExistingDeletes() []iceberg.DataFile

ExistingDeletes returns delete files that already applied before this snapshot's added deletes.

func (DeletedRowsScanTask) Operation added in v0.7.0 

func (t DeletedRowsScanTask) Operation() ChangelogOperation

func (DeletedRowsScanTask) ScanTask added in v0.7.0 

func (t DeletedRowsScanTask) ScanTask() FileScanTask

type EncryptionKey added in v0.6.0 

type EncryptionKey struct {
	KeyID                string            `json:"key-id"`
	EncryptedKeyMetadata string            `json:"encrypted-key-metadata"`
	EncryptedByID        *string           `json:"encrypted-by-id,omitempty"`
	Properties           map[string]string `json:"properties,omitempty"`
}

EncryptionKey represents an encryption key stored in table metadata (V3+).

func (EncryptionKey) Equals added in v0.6.0 

func (e EncryptionKey) Equals(other EncryptionKey) bool

type ErrIncompatibleSchema added in v0.5.0 

type ErrIncompatibleSchema struct {
	// contains filtered or unexported fields
}

func (ErrIncompatibleSchema) Error added in v0.5.0 

func (e ErrIncompatibleSchema) Error() string

func (ErrIncompatibleSchema) Unwrap added in v0.5.0 

func (e ErrIncompatibleSchema) Unwrap() error

type ExpireSnapshotsOpt added in v0.4.0 

type ExpireSnapshotsOpt func(*expireSnapshotsCfg)

func WithOlderThan added in v0.4.0 

func WithOlderThan(t time.Duration) ExpireSnapshotsOpt

WithOlderThan expires snapshots older than the given duration, measured from the current time. It overrides the MaxSnapshotAgeMsKey table property for this call. The most recent snapshots are still retained up to the count set by WithRetainLast. This option does not expire branches or tags based on their age; configure MaxRefAgeMsKey on the ref or table for that.

func WithPostCommit added in v0.5.0 

func WithPostCommit(postCommit bool) ExpireSnapshotsOpt

WithPostCommit controls whether orphaned files (manifests, manifest lists, data files) are deleted immediately after expiring snapshots. Defaults to true. Set to false to defer file deletion to a separate maintenance job, avoiding conflicts with in-flight queries that may still reference those files.

func WithRetainLast added in v0.4.0 

func WithRetainLast(n int) ExpireSnapshotsOpt

WithRetainLast sets the minimum number of snapshots to keep per branch, regardless of age. It overrides the MinSnapshotsToKeepKey table property for this call. Snapshots beyond this count become eligible for expiry only once they are also older than the configured age (see WithOlderThan).

type FSysF added in v0.4.0 

type FSysF func(ctx context.Context) (icebergio.IO, error)

type FileScanTask 

type FileScanTask struct {
	File                iceberg.DataFile
	DeleteFiles         []iceberg.DataFile // positional delete files
	EqualityDeleteFiles []iceberg.DataFile // equality delete files
	DeletionVectorFiles []iceberg.DataFile // deletion vectors (puffin files)
	Start, Length       int64
	// Residual is the portion of the scan filter that must still be evaluated
	// for this task. Local and remote planners may simplify the original filter using
	// file metadata; nil means the caller did not provide a task residual.
	// ReadTasks applies the scan's original row filter and each task residual.
	Residual iceberg.BooleanExpression

	// Row lineage (v3): constants used when reading to synthesize _row_id and _last_updated_sequence_number.
	// FirstRowID is the effective first_row_id for this file (from manifest entry, after inheritance).
	// DataSequenceNumber is the data sequence number of the file's manifest entry.
	FirstRowID         *int64
	DataSequenceNumber *int64
}

type FullRemoteScanPlanner added in v0.7.0 

type FullRemoteScanPlanner interface {
	SupportsFullRemoteScanPlanning() bool
}

FullRemoteScanPlanner is an optional capability extension for ScanPlanner. A planner that exposes separate submission and continuation capabilities can implement SupportsFullRemoteScanPlanning so auto mode only selects it when the complete remote scan flow is available. Planners that do not implement this extension retain the original ScanPlanner behavior.

type Identifier 

type Identifier = []string

type IncompatibleField added in v0.5.0 

type IncompatibleField struct {
	Field           iceberg.NestedField
	ColName         string
	UnsupportedType *UnsupportedType
	InvalidDefault  *InvalidDefault
}

type IncrementalAppendScan added in v0.7.0 

type IncrementalAppendScan struct {
	// contains filtered or unexported fields
}

IncrementalAppendScan plans data files added by append snapshots between a starting snapshot and an ending snapshot. It follows one snapshot ancestry chain and never returns files inherited from an earlier snapshot. Every selected snapshot must identify a recognized operation; planning fails with ErrMissingOperation or ErrInvalidOperation rather than silently omitting snapshots whose operation cannot be determined.

func (*IncrementalAppendScan) FromSnapshotExclusive added in v0.7.0 

func (s *IncrementalAppendScan) FromSnapshotExclusive(snapshotID int64) *IncrementalAppendScan

FromSnapshotExclusive starts after the given snapshot. The starting snapshot must be a parent ancestor of the ending snapshot when planning.

func (*IncrementalAppendScan) FromSnapshotInclusive added in v0.7.0 

func (s *IncrementalAppendScan) FromSnapshotInclusive(snapshotID int64) *IncrementalAppendScan

FromSnapshotInclusive includes files added by the starting snapshot. The snapshot is validated when files are planned.

func (*IncrementalAppendScan) PlanFiles added in v0.7.0 

func (s *IncrementalAppendScan) PlanFiles(ctx context.Context) ([]FileScanTask, error)

PlanFiles returns one task per newly added data file and emits a ScanReport through the configured reporter on successful planning. Delete files are not applied because appended files are not present before the append snapshot.

func (*IncrementalAppendScan) ToSnapshot added in v0.7.0 

func (s *IncrementalAppendScan) ToSnapshot(snapshotID int64) *IncrementalAppendScan

ToSnapshot sets the inclusive ending snapshot. The snapshot is validated when files are planned.

type IncrementalChangelogScan added in v0.7.0 

type IncrementalChangelogScan struct {
	// contains filtered or unexported fields
}

IncrementalChangelogScan plans data-file changes between snapshots. It emits insert and delete tasks for data-manifest entries and skips replace snapshots. PlanFiles returns an error if any in-range snapshot's manifest list references a delete manifest, including ones carried forward from earlier snapshots.

func (*IncrementalChangelogScan) FromSnapshotExclusive added in v0.7.0 

func (s *IncrementalChangelogScan) FromSnapshotExclusive(snapshotID int64) *IncrementalChangelogScan

FromSnapshotExclusive starts after the given snapshot. The starting snapshot must be a parent ancestor of the ending snapshot when planning. The snapshot is validated when files are planned.

func (*IncrementalChangelogScan) FromSnapshotInclusive added in v0.7.0 

func (s *IncrementalChangelogScan) FromSnapshotInclusive(snapshotID int64) *IncrementalChangelogScan

FromSnapshotInclusive includes changes committed by the starting snapshot. The snapshot is validated when files are planned.

func (*IncrementalChangelogScan) PlanFiles added in v0.7.0 

func (s *IncrementalChangelogScan) PlanFiles(ctx context.Context) ([]ChangelogScanTask, error)

PlanFiles returns one task for each added or deleted data-file entry. A cancelled context returns its cancellation error before planning starts. Tasks are ordered by change ordinal, then by DELETE before INSERT within an ordinal, and finally by data-file path. When an ending snapshot is available, it emits a ScanReport through the configured reporter on successful planning. Changelog reports count every returned task in ResultDataFiles and TotalFileSizeInBytes, so a file inserted and deleted within the range is counted twice.

func (*IncrementalChangelogScan) ToSnapshot added in v0.7.0 

func (s *IncrementalChangelogScan) ToSnapshot(snapshotID int64) *IncrementalChangelogScan

ToSnapshot sets the inclusive ending snapshot. The snapshot is validated when files are planned.

type InspectOption added in v0.7.0 

type InspectOption func(*InspectTable)

InspectOption configures an InspectTable.

func WithInspectAllocator added in v0.7.0 

func WithInspectAllocator(alloc memory.Allocator) InspectOption

WithInspectAllocator sets the Arrow memory allocator used to build metadata-table records. Tests can pass a memory.CheckedAllocator to detect leaks; callers with memory accounting can inject their own pool.

type InspectTable added in v0.7.0 

type InspectTable struct {
	// contains filtered or unexported fields
}

InspectTable exposes a table's metadata (snapshots, history, manifests, and so on) as queryable Arrow tables. It mirrors the metadata tables offered by the Java, PyIceberg, and Rust clients.

Obtain one via Table.Inspect. Each method returns an array.RecordReader that the caller is responsible for releasing.

func (InspectTable) AllDataFiles added in v0.7.0 

func (i InspectTable) AllDataFiles(ctx context.Context) (array.RecordReader, error)

AllDataFiles returns the live data files reachable from every snapshot currently tracked by the table.

func (InspectTable) AllDeleteFiles added in v0.7.0 

func (i InspectTable) AllDeleteFiles(ctx context.Context) (array.RecordReader, error)

AllDeleteFiles returns the live delete files reachable from every snapshot currently tracked by the table.

func (InspectTable) AllEntries added in v0.7.0 

func (i InspectTable) AllEntries(ctx context.Context) (array.RecordReader, error)

AllEntries returns every entry from each manifest reachable from any snapshot currently tracked by the table. Shared manifests are scanned once, and deleted entries remain visible.

func (InspectTable) AllFiles added in v0.7.0 

func (i InspectTable) AllFiles(ctx context.Context) (array.RecordReader, error)

AllFiles returns the live data and delete files reachable from every snapshot currently tracked by the table. Shared manifests are scanned once, while duplicate file rows from different manifests are preserved.

func (InspectTable) AllManifests added in v0.7.0 

func (i InspectTable) AllManifests(ctx context.Context) (array.RecordReader, error)

AllManifests returns every manifest referenced by every snapshot currently tracked by the table. A manifest referenced by multiple snapshots produces one row per reference, identified by reference_snapshot_id. Results are streamed in bounded record batches while each snapshot's manifest list is traversed.

func (InspectTable) DataFiles added in v0.7.0 

func (i InspectTable) DataFiles(ctx context.Context) (array.RecordReader, error)

DataFiles returns the live data files in the current snapshot. Deleted manifest entries are omitted, matching the data_files metadata table.

func (InspectTable) DeleteFiles added in v0.7.0 

func (i InspectTable) DeleteFiles(ctx context.Context) (array.RecordReader, error)

DeleteFiles returns the live equality-delete, position-delete, and deletion-vector files in the current snapshot. It reports file metadata and does not open or scan the delete-file contents.

func (InspectTable) Entries added in v0.7.0 

func (i InspectTable) Entries(ctx context.Context) (array.RecordReader, error)

Entries returns every manifest entry in the current snapshot, including entries marked deleted. This exposes commit history that data_files and delete_files intentionally hide.

func (InspectTable) Files added in v0.7.0 

func (i InspectTable) Files(ctx context.Context) (array.RecordReader, error)

Files returns all live data and delete files in the current snapshot. Deleted manifest entries are omitted.

func (InspectTable) History added in v0.7.0 

func (i InspectTable) History(ctx context.Context) (array.RecordReader, error)

History returns the chronological log of every snapshot that was ever the table's current snapshot, one row per snapshot-log entry. Rolled-back snapshots remain visible but are flagged via is_current_ancestor.

Columns:

  • made_current_at (timestamptz, required): when the snapshot became current
  • snapshot_id (long, required): the snapshot that became current
  • parent_id (long, optional): the snapshot's parent, null when the snapshot has no parent or has since been expired
  • is_current_ancestor (boolean, required): whether the snapshot is an ancestor of the current snapshot; false for rolled-back snapshots

is_current_ancestor is derived by walking the current snapshot's parent chain. If an intermediate ancestor has been expired (removed from the snapshot list), that walk is truncated and snapshots below the gap are reported as non-ancestors. Well-formed tables never hit this: ExpireSnapshots keeps the current snapshot and its full ancestry intact. This matches the silent-truncation behavior of the Java, PyIceberg, and Rust clients.

The returned reader holds a single record batch. The caller must Release it.

func (InspectTable) Manifests added in v0.7.0 

func (i InspectTable) Manifests(ctx context.Context) (array.RecordReader, error)

Manifests returns one row for each manifest in the current snapshot. Partition summaries are exposed as the human-readable values used by the other Iceberg clients.

func (InspectTable) MetadataLogEntries added in v0.7.0 

func (i InspectTable) MetadataLogEntries(ctx context.Context) (array.RecordReader, error)

MetadataLogEntries returns one row for every metadata file in the table's metadata log, plus the current metadata file when its location is set. Snapshot information is resolved from the snapshot log at each metadata file's timestamp.

Columns:

  • timestamp (timestamptz, required): when the metadata file was written
  • file (string, required): metadata file location
  • latest_snapshot_id (long, optional): latest snapshot visible then
  • latest_schema_id (int, optional): schema used by that snapshot
  • latest_sequence_number (long, optional): sequence number of that snapshot

The returned reader holds a single record batch. The caller must Release it.

func (InspectTable) Partitions added in v0.7.0 

func (i InspectTable) Partitions(ctx context.Context) (array.RecordReader, error)

Partitions returns one row per live partition, aggregating data and delete files from the current snapshot. Files from evolved specs are coerced into the table-wide partition type before grouping.

func (InspectTable) PositionDeletes added in v0.7.0 

func (i InspectTable) PositionDeletes(ctx context.Context) (array.RecordReader, error)

PositionDeletes returns the individual position-delete records referenced by the current snapshot. Parquet position-delete files and V3 deletion vectors are exposed through the same schema.

func (InspectTable) Refs added in v0.7.0 

func (i InspectTable) Refs(ctx context.Context) (array.RecordReader, error)

Refs returns one row per snapshot reference known to the table. Reference names are sorted to make the result deterministic even though table metadata stores refs in a map.

Columns:

  • name (string, required): the branch or tag name
  • type (string, required): BRANCH or TAG
  • snapshot_id (long, required): the referenced snapshot
  • max_reference_age_in_ms (long, optional): tag/branch reference retention
  • min_snapshots_to_keep (int, optional): branch snapshot retention
  • max_snapshot_age_in_ms (long, optional): branch snapshot retention

The returned reader holds a single record batch. The caller must Release it.

func (InspectTable) Snapshots added in v0.7.0 

func (i InspectTable) Snapshots(ctx context.Context) (array.RecordReader, error)

Snapshots returns one row per snapshot known to the table, in the order they are stored in metadata.

Columns:

  • committed_at (timestamptz, required): when the snapshot was committed
  • snapshot_id (long, required): the snapshot id
  • parent_id (long, optional): the parent snapshot id, null for a root
  • operation (string, optional): the snapshot summary operation, null when the snapshot carries no summary
  • manifest_list (string, optional): path to the snapshot's manifest list
  • summary (map<string,string>, optional): the stored snapshot summary, including the "operation" key alongside its additional properties; null when the snapshot carries no summary

The returned reader holds a single record batch. The caller must Release it.

type InvalidDefault added in v0.5.0 

type InvalidDefault struct {
	MinFormatVersion  int
	MustBeNullForType bool
}

type IsolationLevel added in v0.6.0 

type IsolationLevel string

IsolationLevel controls how strictly a commit rejects concurrent writes on the same branch. It mirrors org.apache.iceberg.IsolationLevel from the Java reference. 

const (
	// IsolationSerializable rejects any concurrent commit that added
	// data files matching the committer's filter since the base
	// snapshot. This is the strongest guarantee: a serializable commit
	// behaves as if it ran in isolation against the base snapshot.
	IsolationSerializable IsolationLevel = "serializable"

	// IsolationSnapshot only rejects concurrent commits that touched
	// files the committer explicitly references (e.g. a position
	// delete whose referenced data file was removed). Concurrent
	// appends into the committer's filter region are allowed.
	IsolationSnapshot IsolationLevel = "snapshot"
)

type LocationProvider added in v0.2.0 

type LocationProvider interface {
	NewDataLocation(dataFileName string) string
	NewTableMetadataFileLocation(newVersion int) (string, error)
	NewMetadataLocation(metadataFileName string) string
}

func LoadLocationProvider added in v0.2.0 

func LoadLocationProvider(tableLocation string, tableProps iceberg.Properties) (LocationProvider, error)

type Metadata 

type Metadata interface {
	// Version indicates the version of this metadata, 1 for V1, 2 for V2, etc.
	Version() int
	// TableUUID returns a UUID that identifies the table, generated when the
	// table is created. Implementations must throw an exception if a table's
	// UUID does not match the expected UUID after refreshing metadata.
	TableUUID() uuid.UUID
	// Location is the table's base location. This is used by writerFactory to determine
	// where to store data files, manifest files, and table metadata files.
	Location() string
	// LastUpdatedMillis is the timestamp in milliseconds from the unix epoch when
	// the table was last updated. Each table metadata file should update this
	// field just before writing.
	LastUpdatedMillis() int64
	// LastColumnID returns the highest assigned column ID for the table.
	// This is used to ensure fields are always assigned an unused ID when
	// evolving schemas.
	LastColumnID() int
	// Schemas returns the list of schemas, stored as objects with their
	// schema-id.
	Schemas() []*iceberg.Schema
	// CurrentSchema returns the table's current schema.
	CurrentSchema() *iceberg.Schema
	// PartitionSpecs returns the list of all partition specs in the table.
	PartitionSpecs() []iceberg.PartitionSpec
	// PartitionSpec returns the current partition spec that the table is using.
	PartitionSpec() iceberg.PartitionSpec
	// PartitionSpecByID returns the partition spec with the given ID. Returns
	// nil if the ID is not found in the list of partition specs.
	PartitionSpecByID(int) *iceberg.PartitionSpec
	// DefaultPartitionSpec is the ID of the current spec that writerFactory should
	// use by default.
	DefaultPartitionSpec() int
	// LastPartitionSpecID returns the persisted last assigned partition field ID,
	// which may be stale relative to partition spec history.
	LastPartitionSpecID() *int
	// Snapshots returns the list of valid snapshots. Valid snapshots are
	// snapshots for which all data files exist in the file system. A data
	// file must not be deleted from the file system until the last snapshot
	// in which it was listed is garbage collected.
	Snapshots() []Snapshot
	// SnapshotByID find and return a specific snapshot by its ID. Returns
	// nil if the ID is not found in the list of snapshots.
	SnapshotByID(int64) *Snapshot
	// SnapshotByName searches the list of snapshots for a snapshot with a given
	// ref name. Returns nil if there's no ref with this name for a snapshot.
	SnapshotByName(name string) *Snapshot
	// CurrentSnapshot returns the table's current snapshot.
	CurrentSnapshot() *Snapshot
	// Ref returns the snapshot ref for the main branch.
	Ref() SnapshotRef
	// Refs returns a list of snapshot name/reference pairs.
	Refs() iter.Seq2[string, SnapshotRef]
	// SnapshotLogs returns the list of snapshot logs for the table.
	SnapshotLogs() iter.Seq[SnapshotLogEntry]
	// SortOrder returns the table's current sort order, ie: the one with the
	// ID that matches the default-sort-order-id.
	SortOrder() SortOrder
	// SortOrders returns the list of sort orders in the table.
	SortOrders() []SortOrder
	// DefaultSortOrder returns the ID of the current sort order that writerFactory
	// should use by default.
	DefaultSortOrder() int
	// Properties is a string to string map of table properties. This is used
	// to control settings that affect reading and writing and is not intended
	// to be used for arbitrary metadata. For example, commit.retry.num-retries
	// is used to control the number of commit retries.
	Properties() iceberg.Properties
	// PreviousFiles returns the list of metadata log entries for the table.
	PreviousFiles() iter.Seq[MetadataLogEntry]
	Equals(Metadata) bool

	NameMapping() iceberg.NameMapping

	LastSequenceNumber() int64
	// NextRowID returns the next available row ID for v3 tables.
	// Returns 0 for v1/v2 tables or if not set.
	NextRowID() int64
	// Statistics returns an optional list of table statistics.
	// Table statistics files are valid Puffin files.
	// StatisticsFile are informational. A reader can choose to ignore statistics information.
	// StatisticsFile support is not required to read the table correctly.
	// A table can contain many statistics files associated with different table snapshots.
	Statistics() iter.Seq[StatisticsFile]
	// PartitionStatistics returns an optional list of partition statistics files.
	// Partition statistics are not required for reading or planning
	// and readers may ignore them. Each table snapshot may be associated
	// with at most one partition statistics file. A writer can optionally
	// write the partition statistics file during each write operation,
	// or it can also be computed on demand.
	PartitionStatistics() iter.Seq[PartitionStatisticsFile]
	// EncryptionKeys returns the list of encryption keys stored in table metadata (V3+).
	// Returns an empty sequence for V1/V2 tables.
	EncryptionKeys() iter.Seq[EncryptionKey]
}

Metadata for an iceberg table as specified in the Iceberg spec

https://iceberg.apache.org/spec/#iceberg-table-spec

func NewMetadata added in v0.2.0 

func NewMetadata(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrder SortOrder, location string, props iceberg.Properties) (Metadata, error)

NewMetadata creates a new table metadata object using the provided schema, information, generating a fresh UUID for the new table metadata. By default, this will generate a V2 table metadata, but this can be modified by adding a "format-version" property to the props map. An error will be returned if the "format-version" property exists and is not a valid version number.

location should be non-empty: ParseMetadata rejects persisted metadata without one, so metadata built with an empty location cannot be read back. The constructor stays permissive because catalog create-via-commit paths seed a location-less base and supply the location as a set-location update.

func NewMetadataWithUUID added in v0.2.0 

func NewMetadataWithUUID(sc *iceberg.Schema, partitions *iceberg.PartitionSpec, sortOrder SortOrder, location string, props iceberg.Properties, tableUuid uuid.UUID) (Metadata, error)

NewMetadataWithUUID is like NewMetadata, but allows the caller to specify the UUID of the table rather than creating a new one.

func ParseMetadata 

func ParseMetadata(r io.Reader) (Metadata, error)

ParseMetadata parses json metadata provided by the passed in reader, returning an error if one is encountered.

func ParseMetadataBytes 

func ParseMetadataBytes(b []byte) (Metadata, error)

ParseMetadataBytes is like ParseMetadataString but for a byte slice.

func ParseMetadataBytesDeferredSnapshots added in v0.7.0 

func ParseMetadataBytesDeferredSnapshots(b []byte) (Metadata, error)

ParseMetadataBytesDeferredSnapshots parses metadata while retaining the full snapshot array as raw JSON. Snapshots targeted by refs are decoded eagerly; unreferenced history is materialized on first access. Operations requiring the complete snapshot collection, including MetadataBuilderFromBase, decode the full history and therefore may cost more than eager parsing overall.

func ParseMetadataString 

func ParseMetadataString(s string) (Metadata, error)

ParseMetadataString is like ParseMetadata, but for a string rather than an io.Reader.

func UpdateTableMetadata added in v0.4.0 

func UpdateTableMetadata(base Metadata, updates []Update, metadataLoc string) (Metadata, error)

type MetadataBuilder added in v0.2.0 

type MetadataBuilder struct {
	// contains filtered or unexported fields
}

MetadataBuilder is a struct used for building and updating Iceberg table metadata.

It keeps track of applied changes in the `updates` field. This can be used to commit changes made to a table to the catalog.

func MetadataBuilderFromBase added in v0.2.0 

func MetadataBuilderFromBase(metadata Metadata, currentFileLocation string) (*MetadataBuilder, error)

MetadataBuilderFromBase creates a MetadataBuilder from an existing Metadata object. currentFileLocation is the location where the current version of the metadata file is stored. This is used to update the metadata log. If currentFileLocation is empty, the metadata log will not be updated. This should only be used to stage-create tables.

func NewMetadataBuilder added in v0.2.0 

func NewMetadataBuilder(formatVersion int) (*MetadataBuilder, error)

func (*MetadataBuilder) AddEncryptionKey added in v0.6.0 

func (b *MetadataBuilder) AddEncryptionKey(key EncryptionKey) error

AddEncryptionKey adds or replaces an encryption key indexed by its key-id. Encryption keys are only supported for format version 3 and above.

func (*MetadataBuilder) AddPartitionSpec added in v0.2.0 

func (b *MetadataBuilder) AddPartitionSpec(spec *iceberg.PartitionSpec, initial bool) error

func (*MetadataBuilder) AddSchema added in v0.2.0 

func (b *MetadataBuilder) AddSchema(schema *iceberg.Schema) error

func (*MetadataBuilder) AddSnapshot added in v0.2.0 

func (b *MetadataBuilder) AddSnapshot(snapshot *Snapshot) error

func (*MetadataBuilder) AddSnapshotUpdate added in v0.6.0 

func (b *MetadataBuilder) AddSnapshotUpdate(u *addSnapshotUpdate) error

AddSnapshotUpdate adds a snapshot to the builder and stores the supplied *addSnapshotUpdate as the corresponding entry in builder.updates, preserving runtime-only fields (such as the manifest-list rebuild closure used by the OCC retry path) that would be lost if a fresh update object were constructed.

Callers without runtime-only fields should keep using AddSnapshot, which constructs a default *addSnapshotUpdate internally.

func (*MetadataBuilder) AddSortOrder added in v0.2.0 

func (b *MetadataBuilder) AddSortOrder(sortOrder *SortOrder) error

func (*MetadataBuilder) AppendMetadataLog added in v0.2.0 

func (b *MetadataBuilder) AppendMetadataLog(entry MetadataLogEntry) *MetadataBuilder

func (*MetadataBuilder) Build added in v0.2.0 

func (b *MetadataBuilder) Build() (Metadata, error)

func (*MetadataBuilder) CurrentSchema added in v0.2.0 

func (b *MetadataBuilder) CurrentSchema() *iceberg.Schema

func (*MetadataBuilder) CurrentSpec added in v0.2.0 

func (b *MetadataBuilder) CurrentSpec() (*iceberg.PartitionSpec, error)

func (*MetadataBuilder) GetSchemaByID added in v0.2.0 

func (b *MetadataBuilder) GetSchemaByID(id int) (*iceberg.Schema, error)

func (*MetadataBuilder) GetSortOrderByID added in v0.2.0 

func (b *MetadataBuilder) GetSortOrderByID(id int) (*SortOrder, error)

func (*MetadataBuilder) GetSpecByID added in v0.2.0 

func (b *MetadataBuilder) GetSpecByID(id int) (*iceberg.PartitionSpec, error)

func (*MetadataBuilder) HasChanges added in v0.2.0 

func (b *MetadataBuilder) HasChanges() bool

func (*MetadataBuilder) LastColumnID added in v0.6.0 

func (b *MetadataBuilder) LastColumnID() int

LastColumnID returns the highest field id ever assigned in this table's lifetime, as tracked by the Iceberg spec's last-column-id counter.

func (*MetadataBuilder) LastUpdatedMS added in v0.2.0 

func (b *MetadataBuilder) LastUpdatedMS() int64

func (*MetadataBuilder) NameMapping added in v0.2.0 

func (b *MetadataBuilder) NameMapping() iceberg.NameMapping

func (*MetadataBuilder) NewRetainingSnapshotRefUpdate added in v0.7.0 

func (b *MetadataBuilder) NewRetainingSnapshotRefUpdate(name string, snapshotID int64, refType RefType) *setSnapshotRefUpdate

func (*MetadataBuilder) NextRowID added in v0.5.0 

func (b *MetadataBuilder) NextRowID() int64

NextRowID returns the next available row ID (for v3 row lineage). For v1/v2 returns 0.

func (*MetadataBuilder) RemoveEncryptionKey added in v0.6.0 

func (b *MetadataBuilder) RemoveEncryptionKey(keyID string) error

RemoveEncryptionKey removes the encryption key with the given key-id. It is not an error if no such key exists.

func (*MetadataBuilder) RemovePartitionSpecs added in v0.4.0 

func (b *MetadataBuilder) RemovePartitionSpecs(ints []int) error

func (*MetadataBuilder) RemovePartitionStatistics added in v0.6.0 

func (b *MetadataBuilder) RemovePartitionStatistics(snapshotID int64) error

RemovePartitionStatistics removes the partition statistics file associated with the given snapshot ID. It is not an error if no such file exists.

func (*MetadataBuilder) RemoveProperties added in v0.2.0 

func (b *MetadataBuilder) RemoveProperties(keys []string) error

func (*MetadataBuilder) RemoveSchemas added in v0.4.0 

func (b *MetadataBuilder) RemoveSchemas(ints []int) error

func (*MetadataBuilder) RemoveSnapshotRef added in v0.4.0 

func (b *MetadataBuilder) RemoveSnapshotRef(name string) error

func (*MetadataBuilder) RemoveSnapshots added in v0.4.0 

func (b *MetadataBuilder) RemoveSnapshots(snapshotIds []int64, postCommit bool) error

func (*MetadataBuilder) RemoveStatistics added in v0.6.0 

func (b *MetadataBuilder) RemoveStatistics(snapshotID int64) error

RemoveStatistics removes the statistics file associated with the given snapshot ID. It is not an error if no such file exists.

func (*MetadataBuilder) SetCurrentSchemaID added in v0.2.0 

func (b *MetadataBuilder) SetCurrentSchemaID(currentSchemaID int) error

func (*MetadataBuilder) SetDefaultSortOrderID added in v0.2.0 

func (b *MetadataBuilder) SetDefaultSortOrderID(defaultSortOrderID int) error

func (*MetadataBuilder) SetDefaultSpecID added in v0.2.0 

func (b *MetadataBuilder) SetDefaultSpecID(defaultSpecID int) error

func (*MetadataBuilder) SetFormatVersion added in v0.2.0 

func (b *MetadataBuilder) SetFormatVersion(formatVersion int) error

func (*MetadataBuilder) SetLastUpdatedMS added in v0.2.0 

func (b *MetadataBuilder) SetLastUpdatedMS() *MetadataBuilder

func (*MetadataBuilder) SetLoc added in v0.2.0 

func (b *MetadataBuilder) SetLoc(loc string) error

func (*MetadataBuilder) SetPartitionStatistics added in v0.6.0 

func (b *MetadataBuilder) SetPartitionStatistics(stats PartitionStatisticsFile) error

SetPartitionStatistics adds or replaces a partition statistics file for the given snapshot. If a partition statistics file with the same snapshot ID already exists it is replaced, otherwise the file is appended.

func (*MetadataBuilder) SetProperties added in v0.2.0 

func (b *MetadataBuilder) SetProperties(props iceberg.Properties) error

func (*MetadataBuilder) SetSnapshotRef added in v0.2.0 

func (b *MetadataBuilder) SetSnapshotRef(
	name string,
	snapshotID int64,
	refType RefType,
	options ...setSnapshotRefOption,
) error

func (*MetadataBuilder) SetStatistics added in v0.6.0 

func (b *MetadataBuilder) SetStatistics(stats StatisticsFile) error

SetStatistics adds or replaces a statistics file for the given snapshot. If a statistics file with the same snapshot ID already exists it is replaced, otherwise the file is appended.

func (*MetadataBuilder) SetUUID added in v0.2.0 

func (b *MetadataBuilder) SetUUID(newUUID uuid.UUID) error

func (*MetadataBuilder) SnapshotByID added in v0.2.0 

func (b *MetadataBuilder) SnapshotByID(id int64) (*Snapshot, error)

func (*MetadataBuilder) TrimMetadataLogs added in v0.2.0 

func (b *MetadataBuilder) TrimMetadataLogs(maxEntries int) *MetadataBuilder

type MetadataLogEntry 

type MetadataLogEntry struct {
	MetadataFile string `json:"metadata-file"`
	TimestampMs  int64  `json:"timestamp-ms"`
}

type MoveOp added in v0.5.0 

type MoveOp string
const (
	MoveOpFirst  MoveOp = "first"
	MoveOpBefore MoveOp = "before"
	MoveOpAfter  MoveOp = "after"
)

type NoOpReason added in v0.7.0 

type NoOpReason int

NoOpReason explains why a rewrite changed nothing. It lets callers tell a table with no current snapshot apart from one whose manifests are already optimal, which would otherwise both surface as an empty result. 

const (
	// NoOpNone means the rewrite produced changes (not a no-op).
	NoOpNone NoOpReason = iota
	// NoOpNoSnapshot means the table had no current snapshot to rewrite.
	NoOpNoSnapshot
	// NoOpAlreadyOptimal means the eligible manifests were already optimal.
	NoOpAlreadyOptimal
)

func (NoOpReason) String added in v0.7.0 

func (r NoOpReason) String() string

type NullOrder 

type NullOrder string
const (
	NullsFirst NullOrder = "nulls-first"
	NullsLast  NullOrder = "nulls-last"
)

type Operation 

type Operation string
const (
	OpAppend    Operation = "append"
	OpReplace   Operation = "replace"
	OpOverwrite Operation = "overwrite"
	OpDelete    Operation = "delete"
)

func ValidOperation 

func ValidOperation(s string) (Operation, error)

ValidOperation ensures that a given string is one of the valid operation types: append,replace,overwrite,delete

type Option added in v0.7.0 

type Option func(*Table)

Option configures a Table at construction. Options are applied in order after the core fields are set.

func WithMetricsReporter added in v0.7.0 

func WithMetricsReporter(r metrics.Reporter) Option

WithMetricsReporter sets the metrics reporter for the table; scans created from the table inherit it. A nil reporter is ignored (the table keeps its default no-op reporter).

func WithScanPlanningIOProperties added in v0.7.0 

func WithScanPlanningIOProperties(props iceberg.Properties) Option

WithScanPlanningIOProperties supplies the table-scoped FileIO properties that a remote scan planner needs when it builds a plan-scoped FileIO. This is intended for catalog implementations whose table-load response carries FileIO configuration separately from table metadata properties.

type OrphanCleanupOption added in v0.4.0 

type OrphanCleanupOption func(*orphanCleanupConfig)

func WithCleanupMaxConcurrency added in v0.7.0 

func WithCleanupMaxConcurrency(maxWorkers int) OrphanCleanupOption

WithCleanupMaxConcurrency sets the maximum number of goroutines for parallel deletion. Defaults to a reasonable number based on the system. Only used when deleteFunc is nil or when the FileIO doesn't support bulk operations.

func WithDeleteFunc added in v0.4.0 

func WithDeleteFunc(deleteFunc func(string) error) OrphanCleanupOption

WithDeleteFunc sets a custom delete function. If not provided, the table's FileIO delete method will be used.

func WithDryRun added in v0.4.0 

func WithDryRun(enabled bool) OrphanCleanupOption

func WithEqualAuthorities added in v0.4.0 

func WithEqualAuthorities(authorities map[string]string) OrphanCleanupOption

WithEqualAuthorities specifies authorities that should be considered equivalent. For example, map["endpoint1.s3.amazonaws.com,endpoint2.s3.amazonaws.com"] = "s3.amazonaws.com" treats different S3 endpoints as equivalent. The key can be a comma-separated list.

func WithEqualSchemes added in v0.4.0 

func WithEqualSchemes(schemes map[string]string) OrphanCleanupOption

WithEqualSchemes specifies schemes that should be considered equivalent. For example, map["s3,s3a,s3n"] = "s3" treats all S3 scheme variants as equivalent. The key can be a comma-separated list of schemes that map to the value scheme.

func WithFilesOlderThan added in v0.4.0 

func WithFilesOlderThan(duration time.Duration) OrphanCleanupOption

func WithLocation added in v0.4.0 

func WithLocation(location string) OrphanCleanupOption

func WithPrefixMismatchMode added in v0.4.0 

func WithPrefixMismatchMode(mode PrefixMismatchMode) OrphanCleanupOption

WithPrefixMismatchMode sets how to handle situations when metadata references files that match listed files except for authority/scheme differences.

type OrphanCleanupPlan added in v0.7.0 

type OrphanCleanupPlan struct {
	// contains filtered or unexported fields
}

OrphanCleanupPlan contains the exact orphan files identified during one scan. The file list is private so callers can only obtain copies, keeping the plan stable between confirmation and execution.

func (OrphanCleanupPlan) Cutoff added in v0.7.0 

func (p OrphanCleanupPlan) Cutoff() time.Time

Cutoff returns the age cutoff used to create the cleanup plan. It is informational only; executing a plan does not re-evaluate file ages.

func (OrphanCleanupPlan) Files added in v0.7.0 

func (p OrphanCleanupPlan) Files() []string

Files returns a copy of the files in the cleanup plan.

func (OrphanCleanupPlan) OrphanFiles added in v0.7.0 

func (p OrphanCleanupPlan) OrphanFiles() []OrphanFile

OrphanFiles returns copies of the path and size entries in the cleanup plan.

func (OrphanCleanupPlan) TotalSizeBytes added in v0.7.0 

func (p OrphanCleanupPlan) TotalSizeBytes() int64

TotalSizeBytes returns the combined size of files in the cleanup plan.

type OrphanCleanupResult added in v0.4.0 

type OrphanCleanupResult struct {
	// OrphanFileLocations is retained for backward compatibility with callers
	// that consume only orphan paths. Prefer OrphanFiles for canonical path+size data.
	OrphanFileLocations []string
	// OrphanFiles is the canonical richer orphan result, carrying both path and size.
	OrphanFiles  []OrphanFile
	DeletedFiles []string
	// TotalSizeBytes is the combined size of orphan files only, not all scanned files.
	TotalSizeBytes int64
}

type OrphanFile added in v0.7.0 

type OrphanFile struct {
	Path      string
	SizeBytes int64
}

type OverwriteOption added in v0.5.0 

type OverwriteOption func(op *overwriteOperation)

OverwriteOption applies options to overwrite operations

func WithOverwriteCaseInsensitive added in v0.5.0 

func WithOverwriteCaseInsensitive() OverwriteOption

WithOverwriteCaseInsensitive overwrites the default case sensitivity that applies on the binding of the filter. Default: case sensitive Note that the sensitivity only applies to the field name and not the evaluation of the literals on string fields.

func WithOverwriteConcurrency added in v0.5.0 

func WithOverwriteConcurrency(concurrency int) OverwriteOption

WithOverwriteConcurrency overwrites the default concurrency for overwrite operations. Default: runtime.GOMAXPROCS(0)

func WithOverwriteFilter added in v0.5.0 

func WithOverwriteFilter(filter iceberg.BooleanExpression) OverwriteOption

WithOverwriteFilter overwrites the default deletion filter on overwrite operations. Default: iceberg.AlwaysTrue

type PartitionStatisticsFile added in v0.4.0 

type PartitionStatisticsFile struct {
	SnapshotID      int64  `json:"snapshot-id"`
	StatisticsPath  string `json:"statistics-path"`
	FileSizeInBytes int64  `json:"file-size-in-bytes"`
}

PartitionStatisticsFile represents a partition statistics file that can be used to read table data more efficiently.

Statistics are informational. A reader can choose to ignore statistics information. Statistics support is not required to read the table correctly.

type PlanIO added in v0.7.0 

type PlanIO interface {
	Load(context.Context) (icebergio.IO, error)
	Close() error
}

PlanIO lazily loads the FileIO used to read a planned scan and closes any resources it holds (e.g. plan-scoped credentials) once the plan is replaced. Nil means the scan keeps using the table's normal FileIO. Remote planners may return a PlanIO backed by plan-scoped storage credentials. Implementations must use a comparable dynamic type with stable identity, normally a pointer.

Delivery contract (OQ1): a returned ScanPlanningResult.IO is stored on the Scan that planned it; every ReadTasks call loads from it instead of the table's FileIO. Replacing the plan releases that Scan's ownership; the old IO closes after its remaining scan owners and active record iterators finish. Call Scan.Close when the scan is no longer needed, including when planning succeeds but its tasks are not read. Close is idempotent and waits for active ReadTasks iterators before the plan IO is closed. This ties a plan-scoped scan to PlanFiles -> ReadTasks: tasks from a remote plan must be read by the Scan that produced them or one of its derived scans. A Scan carrying plan-scoped IO is not safe for concurrent PlanFiles or ReadTasks calls, but consuming an existing iterator while replanning is safe.

type PositionDeltaWriter added in v0.7.0 

type PositionDeltaWriter struct {
	// contains filtered or unexported fields
}

PositionDeltaWriter writes data files for the position-delta MoR pattern, distinguishing reinserted rows (survivors of a position-delta rewrite that preserve their original _row_id) from fresh inserts (rows that get a new _row_id synthesized at read time).

Scope: this writer produces *data files only*. The position-delete entries that pair with reinserts (and that turn an UPDATE into delete-old + reinsert) are not emitted here — the engine driver is responsible for composing this writer with a position-delete writer and committing both via a RowDelta-style snapshot. This mirrors the data-file half of Java's SparkPositionDeltaWrite.reinsert(meta, row) vs insert(row) split.

_last_updated_sequence_number is intentionally not written: the row-id-only file schema lets the reader synthesize it from the manifest entry's data_sequence_number, which after the rewrite is the new snapshot's sequence number — the value the spec requires.

Memory model: Reinsert and Insert retain each batch and accumulate them in memory until PositionDeltaWriter.Close flushes everything to Parquet in a single WriteRecords pass. Peak memory therefore scales with the total Arrow payload of all staged batches, which is appropriate for engine-driver UPDATE flows but not for full-partition compaction. A streaming variant that takes the iterator at construction time is left for the position-delete writer PR.

Usage: 

w, err := table.NewPositionDeltaWriter(tbl)
w.Reinsert(survivorBatch)  // batch must include _row_id column with non-null values
w.Insert(freshBatch)       // batch without _row_id (writer appends nulls)
dataFiles, err := w.Close(ctx)

func NewPositionDeltaWriter added in v0.7.0 

func NewPositionDeltaWriter(tbl *Table, opts ...WriteRecordOption) (*PositionDeltaWriter, error)

NewPositionDeltaWriter creates a writer for the position-delta MoR update pattern on the given table. The table must be format version 3 or higher.

Caller-supplied opts are forwarded to the underlying WriteRecords call at PositionDeltaWriter.Close. WithWriteUUID and WithPreserveRowLineage are reserved by this writer and will override any caller-supplied values for those two options; everything else (target file size, worker count, clustered write) flows through.

func (*PositionDeltaWriter) Close added in v0.7.0 

func (w *PositionDeltaWriter) Close(ctx context.Context) ([]iceberg.DataFile, error)

Close finalizes the writer and returns the data files produced. The returned files contain both reinserted and fresh rows, with the _row_id column written explicitly for reinserted rows (non-null) and left null for fresh inserts.

The caller is responsible for adding these files to a snapshot (typically via a Transaction's snapshot producer) along with any position-delete entries that pair with the reinserts.

func (*PositionDeltaWriter) Insert added in v0.7.0 

func (w *PositionDeltaWriter) Insert(batch arrow.RecordBatch) error

Insert adds fresh rows that get a new _row_id at read time. The batch should NOT contain a _row_id column; if it does, all values must be null. These rows represent genuinely new data (not survivors of a rewrite).

func (*PositionDeltaWriter) Reinsert added in v0.7.0 

func (w *PositionDeltaWriter) Reinsert(batch arrow.RecordBatch) error

Reinsert adds survivor rows that preserve their original _row_id. The batch MUST contain a _row_id column (field name "_row_id") with non-null int64 values representing the preserved row identities.

Per the Iceberg spec, only position-delete rewrites and CoW can preserve lineage. Equality deletes cannot preserve lineage because the engine writes without reading old identity.

type PrefixMismatchMode added in v0.4.0 

type PrefixMismatchMode int

PrefixMismatchMode defines how to handle cases where candidate files have different URI schemes or authorities compared to table location during orphan cleanup. This is useful when files may be referenced using different but equivalent schemes (e.g., s3:// vs s3a:// vs s3n://) or when cleaning up files across different locations. 

const (
	// PrefixMismatchError causes cleanup to fail with an error when candidate files
	// have URI schemes/authorities that don't match the table location and are not
	// covered by configured equivalences. This is the safest default behavior.
	PrefixMismatchError PrefixMismatchMode = iota // default

	// PrefixMismatchIgnore skips candidate files that have mismatched URI schemes/authorities
	// without treating it as an error. Files are silently ignored and not considered for deletion.
	PrefixMismatchIgnore

	// PrefixMismatchDelete treats candidate files with mismatched URI schemes/authorities
	// as orphans and includes them for deletion. Use with caution as this may delete
	// files from unexpected locations.
	PrefixMismatchDelete
)

func (PrefixMismatchMode) String added in v0.4.0 

func (p PrefixMismatchMode) String() string

type ReassignedIds added in v0.4.0 

type ReassignedIds struct {
	// contains filtered or unexported fields
}

type RefType 

type RefType string

RefType will be either a BranchRef or a TagRef 

const (
	BranchRef RefType = "branch"
	TagRef    RefType = "tag"
)

type Requirement added in v0.2.0 

type Requirement interface {
	// Validate checks that the current table metadata satisfies the requirement.
	Validate(Metadata) error
	GetType() string
}

A Requirement is a validation rule that must be satisfied before attempting to make and commit changes to a table. Requirements are used to ensure that the table is in a valid state before making changes.

func AssertCreate added in v0.2.0 

func AssertCreate() Requirement

AssertCreate creates a requirement that the table does not already exist.

func AssertCurrentSchemaID added in v0.2.0 

func AssertCurrentSchemaID(id int) Requirement

AssertCurrentSchemaId creates a requirement that the table's current schema ID matches the given id.

func AssertDefaultSortOrderID added in v0.2.0 

func AssertDefaultSortOrderID(id int) Requirement

AssertDefaultSortOrderID creates a requirement that the table's default sort order ID matches the given id.

func AssertDefaultSpecID added in v0.2.0 

func AssertDefaultSpecID(id int) Requirement

AssertDefaultSpecID creates a requirement that the table's default partition spec ID matches the given id.

func AssertLastAssignedFieldID added in v0.2.0 

func AssertLastAssignedFieldID(id int) Requirement

AssertLastAssignedFieldID validates that the table's last assigned column ID matches the given id.

func AssertLastAssignedPartitionID added in v0.2.0 

func AssertLastAssignedPartitionID(id int) Requirement

AssertLastAssignedPartitionID creates a requriement that the table's last assigned partition ID matches the given id.

func AssertRefSnapshotID added in v0.2.0 

func AssertRefSnapshotID(ref string, id *int64) Requirement

AssertRefSnapshotID creates a requirement which ensures that the table branch or tag identified by the given ref must reference the given snapshot id. If the id is nil, the ref must not already exist.

func AssertTableUUID added in v0.2.0 

func AssertTableUUID(uuid uuid.UUID) Requirement

AssertTableUUID creates a requirement that the table UUID matches the given UUID.

func ParseRequirement added in v0.3.0 

func ParseRequirement(r io.Reader) (Requirement, error)

ParseRequirement parses json data provided by the reader into a Requirement

func ParseRequirementBytes added in v0.3.0 

func ParseRequirementBytes(b []byte) (Requirement, error)

ParseRequirementBytes parses json bytes into a Requirement

func ParseRequirementString added in v0.3.0 

func ParseRequirementString(s string) (Requirement, error)

ParseRequirementString parses json string into a Requirement

type Requirements added in v0.4.0 

type Requirements []Requirement

func (*Requirements) UnmarshalJSON added in v0.4.0 

func (r *Requirements) UnmarshalJSON(data []byte) error

type RewriteDataFilesOptions added in v0.6.0 

type RewriteDataFilesOptions struct {
	// PartialProgress, when true, commits non-empty groups in durable catalog
	// batches. When false (the default), every group is staged in a single
	// atomic rewrite snapshot.
	PartialProgress bool

	// MaxCommits bounds the number of catalog snapshots produced in
	// partial-progress mode. Groups are batched so all groups are processed
	// while staying within this bound. Zero uses the default of 10. It has no
	// effect for atomic rewrites.
	MaxCommits int

	// MaxFailedCommits bounds catalog commit failures in partial-progress mode.
	// Zero (the default) and negative values allow unlimited failures. Failures
	// within the bound are returned in RewriteResult.FailedGroups without making
	// the operation return an error.
	MaxFailedCommits int

	// SnapshotProps are added to the rewrite snapshot's summary.
	// In partial-progress mode the same properties land on every batch
	// snapshot rather than being summed or split.
	SnapshotProps iceberg.Properties

	// ExtraDeleteFilesToRemove are delete files that are dead after
	// the rewrite and that the caller wants expunged in the same
	// snapshot. Honored only when PartialProgress is false.
	//
	// Use [compaction.CollectDeadEqualityDeletesWithSpecs] and
	// [compaction.CollectDeadPositionDeletes] to compute this list
	// from the current snapshot. Position deletes attached to
	// rewritten tasks are already removed by the per-group staging;
	// listing them again here is harmless (each file is removed and
	// counted once).
	ExtraDeleteFilesToRemove []iceberg.DataFile

	// GroupOptions are forwarded to every [ExecuteCompactionGroup]
	// call to tune the per-group read+write pipeline (target file
	// size, scan concurrency). See the With* helpers returning
	// [CompactionGroupOption].
	GroupOptions []CompactionGroupOption
}

RewriteDataFilesOptions bundles the per-rewrite knobs for Transaction.RewriteDataFiles.

type RewriteFiles added in v0.6.0 

type RewriteFiles struct {
	// contains filtered or unexported fields
}

RewriteFiles is the snapshot-operation builder for rewrite (compaction) commits. It is the snapshot-level sibling of RowDelta and mirrors Java's org.apache.iceberg.RewriteFiles interface (returned by Table.newRewrite() in Java).

Compared to a raw Transaction.ReplaceFiles call, the builder owns the rewrite-specific isolation contract internally:

  • The overwrite producer's default isolation validator is suppressed (concurrent appends into rewritten partitions are allowed; this is the defining behavior of a rewrite).
  • A rewrite-specific conflict validator is registered so concurrent pos/eq-delete files targeting any rewritten data file are rejected pre-flight at Transaction.Commit time. A pos-delete is matched to a rewritten file via its referenced_data_file column, via equal file_path lower/upper bounds when that column is unset, or — when neither resolves a single path — via partition overlap with a rewritten file. Eq-deletes are matched conservatively: any concurrent eq-delete during the rewrite conflicts.

Distributed compaction coordinators construct one RewriteFiles on the leader transaction, feed worker outputs in via RewriteFiles.ApplyResult, and commit one snapshot. In-process callers can use Transaction.RewriteDataFiles which drives this builder internally.

The builder follows the same fail-fast pattern as [view.MetadataBuilder]: a method that hits an invalid input stages the error and short-circuits all subsequent calls until RewriteFiles.Commit drains it. The builder is single-use; once Commit has been called, a second call returns an error regardless of whether the first call succeeded.

func (*RewriteFiles) AddDataFile added in v0.6.0 

func (r *RewriteFiles) AddDataFile(df iceberg.DataFile) *RewriteFiles

AddDataFile queues a new data file. Use RewriteFiles.AddDeleteFile for positional deletes, equality deletes, or deletion vectors.

func (*RewriteFiles) AddDeleteFile added in v0.7.0 

func (r *RewriteFiles) AddDeleteFile(df iceberg.DataFile) *RewriteFiles

AddDeleteFile queues a new positional delete, equality delete, or deletion vector for this rewrite. When existing delete files are replaced, the new file inherits the maximum data sequence number of those files.

func (*RewriteFiles) AddDeleteFileWithDataSequenceNumber added in v0.7.0 

func (r *RewriteFiles) AddDeleteFileWithDataSequenceNumber(df iceberg.DataFile, seq int64) *RewriteFiles

AddDeleteFileWithDataSequenceNumber is a convenience form of RewriteFiles.AddDeleteFile. The caller must assign the maximum data sequence number of the exact source delete files represented by df.

func (*RewriteFiles) AddFile added in v0.7.0 

func (r *RewriteFiles) AddFile(df iceberg.DataFile) *RewriteFiles

AddFile queues a file according to its content type. It is useful for coordinators that carry data and delete files in one result slice.

func (*RewriteFiles) Apply deprecated added in v0.6.0 

func (r *RewriteFiles) Apply(deletes, adds, safeDeletes []iceberg.DataFile) *RewriteFiles

Apply is a bulk shortcut that routes three slices onto this builder: every entry in deletes is queued via RewriteFiles.DeleteFile (which routes data vs. delete files by content type), every entry in adds via RewriteFiles.AddFile, and every entry in safeDeletes as an automatic delete-file removal.

Deprecated: use RewriteFiles.ApplyResult, which also carries SafeDeletionVectors. Apply has no slot for them, so a coordinator wiring worker output through Apply leaves deletion vectors for the rewritten files orphaned.

func (*RewriteFiles) ApplyResult added in v0.6.0 

func (r *RewriteFiles) ApplyResult(gr CompactionGroupResult) *RewriteFiles

ApplyResult queues a worker's CompactionGroupResult onto this builder, routing OldDataFiles (DeleteFile), NewDataFiles (AddFile), SafePosDeletes and SafeDeletionVectors as automatic delete-file removals. Prefer it over RewriteFiles.Apply, which cannot carry SafeDeletionVectors.

Typical distributed-coordinator pattern: 

rewrite := leaderTxn.NewRewrite(snapshotProps)
for _, gr := range workerResults {
    rewrite.ApplyResult(gr)
}
if err := rewrite.Commit(ctx); err != nil { ... }

func (*RewriteFiles) Commit added in v0.6.0 

func (r *RewriteFiles) Commit(ctx context.Context) error

Commit stages the rewrite snapshot on the underlying transaction. The catalog commit happens once, later, at Transaction.Commit time.

Commit is single-shot: any second call returns an error regardless of whether the first call succeeded, and neither re-stages the rewrite nor re-registers the conflict validator. Returns an error if any file passed to RewriteFiles.AddFile, RewriteFiles.AddDataFile, or RewriteFiles.DeleteFile had an unsupported content type, if the builder has no file changes, or if the underlying Transaction.ReplaceFiles call fails.

func (*RewriteFiles) DataSequenceNumber added in v0.7.0 

func (r *RewriteFiles) DataSequenceNumber(seq int64) *RewriteFiles

DataSequenceNumber configures the data sequence number used for every data file added by this rewrite. This is useful when replacement data files must retain the applicability of delete files committed before the rewrite snapshot.

func (*RewriteFiles) DeleteFile added in v0.6.0 

func (r *RewriteFiles) DeleteFile(df iceberg.DataFile) *RewriteFiles

DeleteFile marks a file for removal in this rewrite. Routes by content type: data files are queued as data-file replacements; pos/eq-delete files are queued for delete-file removal alongside the data rewrite (typical when a delete is fully applied to data files being rewritten and is therefore safe to expunge).

Any other content type stages an error that is returned from the next RewriteFiles.Commit call.

type RewriteManifestsOpt added in v0.7.0 

type RewriteManifestsOpt func(*rewriteManifestsCfg)

RewriteManifestsOpt configures Transaction.RewriteManifests.

func WithManifestTargetSize added in v0.7.0 

func WithManifestTargetSize(size int64) RewriteManifestsOpt

WithManifestTargetSize overrides the target manifest size in bytes. The clustered writer observes the size after Avro blocks flush, so an output can exceed the target by nearly one block. A non-positive size is ignored, leaving the default from the commit.manifest.target-size-bytes property.

func WithRewriteManifestClusterBy added in v0.7.0 

func WithRewriteManifestClusterBy(clusterBy func(iceberg.DataFile) any) RewriteManifestsOpt

WithRewriteManifestClusterBy groups rewritten data files by the key returned by clusterBy. Files with the same key and partition spec are written to the same manifest until the target size is reached, after which another manifest is started for that key. Clustering replaces normal size-only bin-packing, including minCountToMerge. One writer remains open for each distinct (partition spec, key) pair, so clusterBy should have reasonable cardinality to avoid exhausting object-store handles. The key must be non-nil, comparable, and equal to itself. The callback should be deterministic because an OCC retry may need to re-cluster manifests displaced by a concurrent writer. When all rewritten inputs remain present, the existing clustered output is reused instead.

func WithRewriteManifestPredicate added in v0.7.0 

func WithRewriteManifestPredicate(pred func(iceberg.ManifestFile) bool) RewriteManifestsOpt

WithRewriteManifestPredicate only rewrites manifests for which pred is true. Manifests that don't match are left untouched.

func WithRewriteSpecID added in v0.7.0 

func WithRewriteSpecID(id int) RewriteManifestsOpt

WithRewriteSpecID restricts the rewrite to manifests of one partition spec.

type RewriteManifestsResult added in v0.7.0 

type RewriteManifestsResult struct {
	// RewrittenManifests are the old manifests that were replaced.
	RewrittenManifests []iceberg.ManifestFile
	// AddedManifests are the new manifests written in their place.
	AddedManifests []iceberg.ManifestFile
	// NoOpReason is set when the rewrite changed nothing, distinguishing a
	// missing snapshot from an already-optimal layout. NoOpNone otherwise.
	NoOpReason NoOpReason
}

RewriteManifestsResult reports the manifests changed by a rewrite.

func (*RewriteManifestsResult) IsNoOp added in v0.7.0 

func (r *RewriteManifestsResult) IsNoOp() bool

IsNoOp reports whether the rewrite changed nothing. Callers should skip the commit in that case rather than staging an empty REPLACE snapshot. NoOpReason is the single source of truth.

type RewriteResult added in v0.6.0 

type RewriteResult struct {
	// Table is the latest committed table when PartialProgress is enabled.
	// It is nil for an atomic rewrite until the caller commits the transaction.
	Table *Table

	// RewrittenGroups is the number of compaction groups committed.
	RewrittenGroups int

	// AddedDataFiles is the total number of new data files written.
	AddedDataFiles int

	// RemovedDataFiles is the total number of old data files replaced.
	RemovedDataFiles int

	// RemovedPositionDeleteFiles is the count of position delete files
	// removed because their referenced data file was rewritten or
	// because they were passed via
	// [RewriteDataFilesOptions.ExtraDeleteFilesToRemove].
	RemovedPositionDeleteFiles int

	// RemovedEqualityDeleteFiles is the count of equality delete files
	// removed via [RewriteDataFilesOptions.ExtraDeleteFilesToRemove].
	// The caller computes which eq-deletes are dead — typically via
	// [compaction.CollectDeadEqualityDeletesWithSpecs] — and passes the list in.
	RemovedEqualityDeleteFiles int

	// RemovedDeletionVectorFiles is the count of deletion vectors removed
	// because their referenced data file was rewritten.
	RemovedDeletionVectorFiles int

	// BytesBefore is the total size of input data files (from the compaction plan).
	BytesBefore int64

	// BytesAfter is the total size of output data files (measured from written files).
	BytesAfter int64

	// CompletedGroups contains the groups whose snapshots were committed in
	// partial-progress mode.
	CompletedGroups []CompactionGroupResult

	// FailedGroups contains groups whose catalog commits failed. A partial
	// rewrite can return both completed and failed groups.
	FailedGroups []CompactionGroupFailure
}

RewriteResult summarizes a completed compaction.

type RollingDataWriter added in v0.4.0 

type RollingDataWriter struct {
	// contains filtered or unexported fields
}

RollingDataWriter writes Arrow records for a specific partition, rolling to new data files when the actual compressed file size reaches the target.

func (*RollingDataWriter) Add added in v0.4.0 

func (r *RollingDataWriter) Add(record arrow.RecordBatch) error

Add appends a record to the writer's buffer.

type RowDelta added in v0.6.0 

type RowDelta struct {
	// contains filtered or unexported fields
}

RowDelta encodes a set of row-level changes to a table: new data files (inserts) and delete files (equality or position deletes). A delta that replaces a data file's deletion vector can remove the superseded DV via RowDelta.RemoveDeletes. All changes are committed atomically in a single snapshot.

The operation type of the produced snapshot is determined automatically:

  • Data files only → OpAppend
  • Delete files only → OpDelete
  • Both data and delete files → OpOverwrite

This matches the semantics of Java's BaseRowDelta. It is the primary API for CDC/streaming workloads where INSERTs, UPDATEs, and DELETEs must be committed together.

Client-side conflict validation runs before the commit is sent to the catalog:

  • Position deletes: referenced data files must still be reachable from the current branch head (validateDataFilesExist).
  • Equality deletes under write.delete.isolation-level=serializable (the default): concurrent data files in the same partition(s) as the equality deletes are rejected. For partitioned tables an OR-of-equalities filter is built from the eq-delete files' partition tuples and routed through validateAddedDataFilesMatchingFilter (spec-evolution safe, manifest-summary pruning, type-aware evaluation). For unpartitioned tables the check is conservative (AlwaysTrue — any concurrent append is a conflict). Opt out by setting write.delete.isolation-level=snapshot.

The pre-flight runs before cat.CommitTable on every commit attempt. On the first attempt the writer's view and the catalog state are the same, so there are no concurrent snapshots to inspect and the checks short-circuit; on retries doCommit refreshes the catalog state and the checks run against the freshly loaded branch head.

Usage: 

rd := tx.NewRowDelta(snapshotProps)
rd.AddRows(dataFile1, dataFile2)
rd.AddDeletes(equalityDeleteFile1)
err := rd.Commit(ctx)

func (*RowDelta) AddDeletes added in v0.6.0 

func (rd *RowDelta) AddDeletes(files ...iceberg.DataFile) *RowDelta

AddDeletes adds delete files (equality or position) to this RowDelta. Equality delete files must have ContentType == EntryContentEqDeletes and non-empty EqualityFieldIDs referencing valid schema columns. Position delete files must have ContentType == EntryContentPosDeletes. DVs are rejected by Commit unless the table is v3 or later.

func (*RowDelta) AddRows added in v0.6.0 

func (rd *RowDelta) AddRows(files ...iceberg.DataFile) *RowDelta

AddRows adds data files containing new rows (inserts) to this RowDelta.

func (*RowDelta) Commit added in v0.6.0 

func (rd *RowDelta) Commit(ctx context.Context) error

Commit validates and commits all accumulated row-level changes as a single atomic snapshot. Returns an error if there are no files to commit, if any file has an unexpected content type, if a removed deletion vector lacks a replacement or is not referenced by the current snapshot, if the delta would leave two live deletion vectors on one data file (a replacement added while the superseded live DV is not removed, or two added replacements for one data file), or if the table format version does not support delete files. Deletes require v2 or later, DVs require v3 or later.

With removals present the staged commit is non-replayable: an optimistic-concurrency conflict fails the transaction's Commit with ErrCommitFailed rather than refreshing and retrying (see RemoveDeletes).

func (*RowDelta) Operation added in v0.6.0 

func (rd *RowDelta) Operation() Operation

Operation returns the snapshot operation type that will be used when this RowDelta is committed:

  • data only → OpAppend
  • deletes only → OpDelete
  • both → OpOverwrite

func (*RowDelta) RemoveDeletes added in v0.7.0 

func (rd *RowDelta) RemoveDeletes(files ...iceberg.DataFile) *RowDelta

RemoveDeletes marks superseded deletion vectors as removed in the snapshot produced by this RowDelta, mirroring Java's RowDelta#removeDeletes.

The v3 spec allows at most one live deletion vector per data file: a writer adding a DV for a data file that already carries one must write a DV containing all of the previous DV's positions and remove the previous DV in the same snapshot. RemoveDeletes is the removal half of that contract; Commit validates that every removed DV's referenced data file gets a replacement DV among the files passed to AddDeletes, so the delta cannot silently resurrect deleted rows.

Only deletion vectors (Puffin position deletes with a referenced data file) can be removed here. Expunging fully-applied position or equality delete files belongs to the rewrite path (Transaction.ReplaceFiles / RewriteFiles), which owns the data-file replacement validation that makes such removals safe.

Removal identity is (file path, referenced data file): a multi-blob Puffin file may carry deletion vectors for several data files under one path, and removing one of its entries leaves the others live. When exactly one live entry carries the removed file's path, it is selected regardless of the caller copy's (possibly stale) reference; when several live entries share the path, the removed file's ReferencedDataFile selects which entry to remove, and Commit rejects the removal as ambiguous if it does not identify exactly one.

Removal resolution is snapshot-relative, so a commit carrying removals is never replayed on an optimistic-concurrency conflict: if a concurrent writer advances the branch first, Commit's transaction fails with ErrCommitFailed instead of retrying, and the caller must reload the table and rebuild the delta against the fresh snapshot.

type Scan 

type Scan struct {
	// contains filtered or unexported fields
}

Scan represents a table scan. It implements io.Closer; callers should close it when they are done, including early exits after remote planning succeeds but before all records are consumed.

func (*Scan) Close added in v0.7.0 

func (scan *Scan) Close() error

Close releases the plan-scoped resources owned by this scan. It is safe to call more than once. Active ReadTasks iterators retain their reader lease and can finish; the plan IO closes after the last lease is released. A scan must not be used after Close.

func (*Scan) PlanFiles 

func (scan *Scan) PlanFiles(ctx context.Context) ([]FileScanTask, error)

PlanFiles orchestrates the fetching and filtering of manifests, building a list of FileScanTasks that match the current Scan criteria. When planning happens locally it times the whole operation and emits a ScanReport to the scan's reporter on success; remote (server-side) planning reports its own metrics and does not emit here.

func (*Scan) Projection 

func (scan *Scan) Projection() (*iceberg.Schema, error)

func (*Scan) ReadTasks added in v0.6.0 

func (scan *Scan) ReadTasks(ctx context.Context, tasks []FileScanTask) (*arrow.Schema, iter.Seq2[arrow.RecordBatch, error], error)

ReadTasks reads Arrow records from a specific set of FileScanTasks, applying the scan's projection, per-task residual filters, and delete handling. This is useful when the caller has already planned or selected specific tasks to read. Positional- and equality-delete read errors are delivered through the iterator only if iteration reaches the task that encounters the error; a row limit or early termination may finish the scan before the error is observed. Deletion-vector manifest references are validated during setup. Deletion-vector format and blob-level validation happen when the Puffin file is read. Deletion-vector read errors are delivered through the iterator when a task referencing that file is reached; if no such task is processed, the file is not read and its error is not returned. The returned iterator is single-use.

With WithMaxConcurrency above one, tasks are decoded in parallel and the batches are returned in task order. Each worker holds the decoded batches of at most one task until the iterator has returned them, so while an early task is slow to open or decode, the batches waiting behind it total at most the worker count times the rows in the largest task, rounded up to a multiple of the batch size (see WithArrowBatchSize). One large file among small ones can therefore still hold its whole decoded contents while it waits. The bound for a compaction pipeline is stated on WithCompactionArrowBatchSize.

func (*Scan) Reporter added in v0.7.0 

func (scan *Scan) Reporter() metrics.Reporter

Reporter returns the metrics reporter for this scan, never nil. The scan-planning instrumentation emits its ScanReport through it.

func (*Scan) ResolveSnapshot added in v0.7.0 

func (scan *Scan) ResolveSnapshot() (*Snapshot, error)

ResolveSnapshot resolves the snapshot selected by this scan. Live scans use the table's current snapshot; explicit snapshot IDs and as-of timestamps must resolve to an existing snapshot.

func (*Scan) Snapshot 

func (scan *Scan) Snapshot() *Snapshot

Snapshot returns the snapshot selected by this scan. It returns nil when an explicit snapshot cannot be resolved; use ResolveSnapshot when the reason for that result must be distinguished from a table with no current snapshot.

func (*Scan) ToArrowRecords 

func (scan *Scan) ToArrowRecords(ctx context.Context) (*arrow.Schema, iter.Seq2[arrow.RecordBatch, error], error)

ToArrowRecords returns the arrow schema of the expected records and an interator that can be used with a range expression to read the records as they are available. If an error is encountered, during the planning and setup then this will return the error directly. If the error occurs while iterating the records, it will be returned by the iterator. Deletion-vector manifest references are validated during setup. Deletion-vector format and blob-level validation happen when the Puffin file is read. Deletion-vector Puffin files are loaded lazily, so a Puffin read error is returned by the iterator when a task referencing that file is reached. If no such task is processed, the file is not read and its error is not returned.

The purpose for returning the schema up front is to handle the case where there are no rows returned. The resulting Arrow Schema of the projection will still be known. The memory held while tasks are decoded in parallel is described on Scan.ReadTasks.

func (*Scan) ToArrowTable 

func (scan *Scan) ToArrowTable(ctx context.Context) (arrow.Table, error)

ToArrowTable calls ToArrowRecords and then gathers all of the records together and returns an arrow.Table make from those records.

func (*Scan) UseRef 

func (scan *Scan) UseRef(name string) (*Scan, error)

UseRef selects a named snapshot reference. UseRef(MainBranch) is the one intentional exception to selector exclusivity: it returns a clone without changing an existing snapshot or as-of selector. Any conflicting selectors recorded by scan options are still surfaced by scan execution.

func (*Scan) UseRowLimit 

func (scan *Scan) UseRowLimit(n int64) *Scan

type ScanOption 

type ScanOption func(*Scan)

func WitMaxConcurrency deprecated 

func WitMaxConcurrency(n int) ScanOption

WitMaxConcurrency is a deprecated alias for WithMaxConcurrency, kept for backward compatibility with the pre-existing typo'd name.

Deprecated: use WithMaxConcurrency.

func WithArrowBatchSize added in v0.7.0 

func WithArrowBatchSize(n int) ScanOption

WithArrowBatchSize caps the number of rows decoded per Arrow record batch when reading data files, overriding the table's read.parquet.batch-size property for this scan. Smaller batches bound the memory a scan holds per decoded batch, which matters when the consumer buffers batches (e.g. a compaction's read+write pipeline). The cap is stored on the scan itself rather than in the options map, so it applies regardless of ordering relative to WithOptions. A non-positive value is ignored.

func WithCaseSensitive 

func WithCaseSensitive(b bool) ScanOption

func WithLimit 

func WithLimit(n int64) ScanOption

func WithMaxConcurrency added in v0.4.0 

func WithMaxConcurrency(n int) ScanOption

WithMaxConcurrency sets the maximum concurrency for table scan and plan operations. When unset it defaults to runtime.GOMAXPROCS. Above one, a scan holds the decoded batches of up to n tasks at once while it returns them in task order; Scan.ReadTasks states the bound.

func WithOptions 

func WithOptions(opts iceberg.Properties) ScanOption

func WithReporter added in v0.7.0 

func WithReporter(r metrics.Reporter) ScanOption

WithReporter overrides the metrics reporter for a single scan, taking precedence over the reporter inherited from the table. A nil reporter is ignored.

func WithRowFilter 

func WithRowFilter(e iceberg.BooleanExpression) ScanOption

func WithRowLineage added in v0.7.0 

func WithRowLineage() ScanOption

WithRowLineage projects the row-lineage metadata columns (_row_id and _last_updated_sequence_number) so that row identity and per-row update sequence are preserved through rewrites and compactions. Requires a v3 table — calling Scan.Projection on a v1/v2 table after applying this option returns an error.

func WithScanPlanningMode added in v0.7.0 

func WithScanPlanningMode(mode ScanPlanningMode) ScanOption

WithScanPlanningMode sets the scan-planning mode for a scan. The default is ScanPlanningLocal.

func WithSelectedFields 

func WithSelectedFields(fields ...string) ScanOption

func WithSnapshotAsOf added in v0.4.0 

func WithSnapshotAsOf(timeStampMs int64) ScanOption

func WithSnapshotID 

func WithSnapshotID(n int64) ScanOption

type ScanPlanner added in v0.7.0 

type ScanPlanner interface {
	SupportsRemoteScanPlanning() bool
	PlanFiles(context.Context, ScanPlanningRequest) (ScanPlanningResult, error)
}

ScanPlanner plans scans for a table. rest.Catalog implements it; non-REST catalogs leave it nil and planning stays local.

SupportsRemoteScanPlanning reports whether the planner can submit a remote plan for the requested scan. A planner with separate continuation capabilities can implement FullRemoteScanPlanner so auto mode can require the complete remote scan flow.

type ScanPlanningMetadata added in v0.7.0 

type ScanPlanningMetadata interface {
	CurrentSchema() *iceberg.Schema
	Schemas() []*iceberg.Schema
	PartitionSpec() iceberg.PartitionSpec
	PartitionSpecByID(int) *iceberg.PartitionSpec
	CurrentSnapshot() *Snapshot
	SnapshotByID(int64) *Snapshot
	Properties() iceberg.Properties
}

ScanPlanningMetadata is the subset of table.Metadata a ScanPlanner needs: schema binding, snapshot resolution, partition decode, and property-based mode resolution. Narrowing from the full Metadata interface keeps the seam contract honest — planners do not depend on metadata logs, sort orders, or file lists. table.Metadata satisfies it.

type ScanPlanningMode added in v0.7.0 

type ScanPlanningMode string

ScanPlanningMode is the user-facing scan option: three values (local/remote/auto) selecting how (*Scan).PlanFiles plans a scan. Local planning remains the default; remote is opt-in via WithScanPlanningMode.

This is deliberately distinct from the REST table-config key `scan-planning-mode` (values `client`/`server`), which is a server directive resolved separately (OQ4): a `client` table forces local planning, a `server` table forces remote planning, and explicit conflicting scan options fail fast. There is intentionally no fourth `server` value here; the directive lives in the table config, not the user option. 

const (
	// ScanPlanningLocal always plans locally by reading manifests through the
	// table's FileIO. This is the default and current behavior.
	ScanPlanningLocal ScanPlanningMode = "local"
	// ScanPlanningRemote requires a planner that advertises remote capability
	// and fails loudly if remote planning is unavailable.
	ScanPlanningRemote ScanPlanningMode = "remote"
	// ScanPlanningAuto uses remote planning when available, otherwise it falls
	// back to local.
	ScanPlanningAuto ScanPlanningMode = "auto"
)

type ScanPlanningRequest added in v0.7.0 

type ScanPlanningRequest struct {
	Identifier Identifier
	// Metadata is the narrowed planner view of table metadata (see
	// ScanPlanningMetadata); MetadataLocation is kept separate.
	Metadata         ScanPlanningMetadata
	Schema           *iceberg.Schema
	MetadataLocation string
	// FileIOProperties are the table-scoped properties used to build the
	// table's normal FileIO. A remote planner can overlay plan-scoped
	// credentials on these properties without serializing them into the plan
	// request.
	FileIOProperties iceberg.Properties
	SnapshotID       *int64
	SelectedFields   []string
	RowFilter        iceberg.BooleanExpression
	MinRowsRequested *int64
	StatsFields      []string
	// MaxConcurrency is the maximum number of concurrent operations a planner
	// may use for this scan. Zero means use the planner's default.
	MaxConcurrency int
	// CaseSensitive must carry the Scan's value (which defaults to true), not
	// Go's false zero value, or the wire request would flip the spec default.
	// Nil means use the scan default.
	CaseSensitive *bool
	// UseSnapshotSchema is a pointer to distinguish the spec default from an
	// explicit false when the scanner-delegation phase binds it to table config.
	UseSnapshotSchema *bool
}

ScanPlanningRequest is the input a Scan hands to a ScanPlanner. It carries the resolved scan state a planner needs without depending on catalog/rest.

Schema is the schema resolved for this scan. It is kept separate from Metadata.CurrentSchema because a historical/tag scan may use a snapshot schema while a branch scan uses the table schema. Planners should use this same schema for filter binding and returned residual decoding.

type ScanPlanningResult added in v0.7.0 

type ScanPlanningResult struct {
	Tasks []FileScanTask
	IO    PlanIO
}

ScanPlanningResult is what a ScanPlanner returns.

type SchemaOptions added in v0.6.0 

type SchemaOptions struct {
	DowncastTimestamp bool
	// FormatVersion enables format-version-specific read promotions.
	FormatVersion        int
	IncludeFieldIDs      bool
	UseLargeTypes        bool
	UseWriteDefault      bool
	AllowMissingRequired bool
	TableProperties      iceberg.Properties
}

SchemaOptions controls the behaviour of ToRequestedSchema.

type SequenceNumberValidator added in v0.5.0 

type SequenceNumberValidator interface {
	LastSequenceNumber() int64
}

SequenceNumberValidator defines an interface for types that can validate sequence numbers

type Snapshot 

type Snapshot struct {
	SnapshotID        int64    `json:"snapshot-id"`
	ParentSnapshotID  *int64   `json:"parent-snapshot-id,omitempty"`
	SequenceNumber    int64    `json:"sequence-number"`
	TimestampMs       int64    `json:"timestamp-ms"`
	ManifestList      string   `json:"manifest-list,omitempty"`
	ManifestLocations []string `json:"manifests,omitempty"` // V1: Embedded manifest locations
	Summary           *Summary `json:"summary,omitempty"`
	SchemaID          *int     `json:"schema-id,omitempty"`
	FirstRowID        *int64   `json:"first-row-id,omitempty"` // V3: Starting row ID for this snapshot
	AddedRows         *int64   `json:"added-rows,omitempty"`   // V3: Number of rows added by this snapshot
}

func AncestorsBetween added in v0.6.0 

func AncestorsBetween(latestID, baseID int64, lookup SnapshotLookup) ([]Snapshot, bool)

AncestorsBetween returns the snapshots from latestID (inclusive) down to but not including baseID, by walking the parent chain from latestID backward. The second return value (baseFound) is true when baseID was actually reached during the walk.

These snapshots are the "concurrent" snapshots that a writer based on baseID needs to examine for conflict detection.

When baseFound is false, the walk terminated without reaching baseID — either because latestID was unknown, baseID is not in latestID's ancestry (diverged branch or expired base), the chain was broken by a missing intermediate snapshot, or a cycle was detected in malformed metadata. In all of these cases the returned slice is diagnostic context only — it is NOT an enumerable "concurrent snapshots" list. Callers performing conflict detection MUST treat baseFound=false as divergent and refuse the commit.

Returns (nil, true) when latestID == baseID (no concurrent snapshots).

Snapshots are returned by value in reverse-chronological order.

func AncestorsOf added in v0.6.0 

func AncestorsOf(snapshotID int64, lookup SnapshotLookup) []Snapshot

AncestorsOf walks the parent chain starting from snapshotID and yields every snapshot reachable via ParentSnapshotID, including the starting snapshot itself. Iteration stops when a snapshot has no parent, the parent cannot be resolved via lookup, or a cycle is detected in malformed metadata.

Returns snapshots by value in reverse-chronological order (newest first). Returns an empty slice if snapshotID cannot be resolved.

The returned slice may be truncated if an intermediate snapshot is missing from the lookup (e.g. expired) or if a cycle is encountered. Callers that need to distinguish a complete walk from a truncated one should use AncestorsOfChecked instead.

func AncestorsOfChecked added in v0.6.0 

func AncestorsOfChecked(snapshotID int64, lookup SnapshotLookup) ([]Snapshot, bool)

AncestorsOfChecked is AncestorsOf with completeness tracking. The second return value is true when the walk terminated at a snapshot with no parent (a clean root). It is false when the walk was truncated by an unresolvable starting snapshot, a missing intermediate snapshot, or a cycle in malformed metadata.

Callers performing conflict detection (where a truncated ancestry equates to under-counting concurrent snapshots) MUST treat complete=false as divergent and refuse the commit, mirroring AncestorsBetween's baseFound=false contract. When complete is false the returned slice is the partial walk before truncation — diagnostic context only, NOT an enumerable ancestry.

Snapshots are returned by value in reverse-chronological order. Returns an empty slice and false when snapshotID cannot be resolved.

func (Snapshot) Equals 

func (s Snapshot) Equals(other Snapshot) bool

func (Snapshot) Manifests 

func (s Snapshot) Manifests(fio iceio.IO) (_ []iceberg.ManifestFile, err error)

func (Snapshot) MarshalJSON added in v0.7.0 

func (s Snapshot) MarshalJSON() ([]byte, error)

func (Snapshot) String 

func (s Snapshot) String() string

func (*Snapshot) UnmarshalJSON added in v0.7.0 

func (s *Snapshot) UnmarshalJSON(data []byte) error

func (Snapshot) ValidateRowLineage added in v0.5.0 

func (s Snapshot) ValidateRowLineage() error

type SnapshotLogEntry 

type SnapshotLogEntry struct {
	SnapshotID  int64 `json:"snapshot-id"`
	TimestampMs int64 `json:"timestamp-ms"`
}

type SnapshotLookup added in v0.6.0 

type SnapshotLookup func(id int64) *Snapshot

SnapshotLookup returns the snapshot for the given id, or nil if not found. It matches the signature of Metadata.SnapshotByID.

type SnapshotRef 

type SnapshotRef struct {
	SnapshotID         int64   `json:"snapshot-id"`
	SnapshotRefType    RefType `json:"type"`
	MinSnapshotsToKeep *int    `json:"min-snapshots-to-keep,omitempty"`
	MaxSnapshotAgeMs   *int64  `json:"max-snapshot-age-ms,omitempty"`
	MaxRefAgeMs        *int64  `json:"max-ref-age-ms,omitempty"`
}

SnapshotRef represents the reference information for a specific snapshot

func (*SnapshotRef) Equals 

func (s *SnapshotRef) Equals(rhs SnapshotRef) bool

func (*SnapshotRef) UnmarshalJSON 

func (s *SnapshotRef) UnmarshalJSON(b []byte) error

type SnapshotSummaryCollector added in v0.2.0 

type SnapshotSummaryCollector struct {
	// contains filtered or unexported fields
}

type SortDirection 

type SortDirection string
const (
	SortASC  SortDirection = "asc"
	SortDESC SortDirection = "desc"
)

type SortField 

type SortField struct {
	// SourceIDs contains the source column ids from the table's schema.
	// For single-argument transforms this will have exactly one element.
	// For multi-argument transforms this will have multiple elements.
	SourceIDs []int `json:"-"`
	// Transform is the tranformation used to produce values to be
	// sorted on from the source column.
	Transform iceberg.Transform `json:"transform"`
	// Direction is an enum indicating ascending or descending direction.
	Direction SortDirection `json:"direction"`
	// NullOrder describes the order of null values when sorting
	// should be only either nulls-first or nulls-last enum values.
	NullOrder NullOrder `json:"null-order"`
}

SortField describes a field used in a sort order definition.

func (SortField) Equals added in v0.6.0 

func (s SortField) Equals(other SortField) bool

func (SortField) MarshalJSON 

func (s SortField) MarshalJSON() ([]byte, error)

func (SortField) SourceID 

func (s SortField) SourceID() int

SourceID returns the first source column id.

func (*SortField) String 

func (s *SortField) String() string

func (*SortField) UnmarshalJSON 

func (s *SortField) UnmarshalJSON(b []byte) error

type SortOrder 

type SortOrder struct {
	// contains filtered or unexported fields
}

SortOrder describes how the data is sorted within the table.

Data can be sorted within partitions by columns to gain performance. The order of the sort fields within the list defines the order in which the sort is applied to the data.

func AssignFreshSortOrderIDs added in v0.2.0 

func AssignFreshSortOrderIDs(sortOrder SortOrder, old, fresh *iceberg.Schema) (SortOrder, error)

AssignFreshSortOrderIDs updates and reassigns the field source IDs from the old schema to the corresponding fields in the fresh schema, while also giving the Sort Order a fresh ID of 0 (the initial Sort Order ID).

func AssignFreshSortOrderIDsWithID added in v0.2.0 

func AssignFreshSortOrderIDsWithID(sortOrder SortOrder, old, fresh *iceberg.Schema, sortOrderID int) (SortOrder, error)

AssignFreshSortOrderIDsWithID is like AssignFreshSortOrderIDs but allows specifying the id of the returned SortOrder.

func NewSortOrder added in v0.4.0 

func NewSortOrder(orderID int, fields []SortField) (SortOrder, error)

NewSortOrder creates a new SortOrder.

The orderID must be greater than or equal to 0. If orderID is 0, no fields can be passed, this is equal to UnsortedSortOrder. If fields is empty, orderID must be 0. Fields need to have non-nil Transform, valid Direction and NullOrder values, and non-empty source IDs.

func (*SortOrder) CheckCompatibility added in v0.4.0 

func (s *SortOrder) CheckCompatibility(schema *iceberg.Schema) error

func (SortOrder) Equals 

func (s SortOrder) Equals(rhs SortOrder) bool

func (SortOrder) Field added in v0.7.0 

func (s SortOrder) Field(i int) SortField

Field returns a copy of the sort field at index i, like Fields does, so a caller can't reach the sort order's internals through SortField.SourceIDs. It panics if i is out of range.

func (SortOrder) Fields 

func (s SortOrder) Fields() iter.Seq2[int, SortField]

func (SortOrder) IsUnsorted added in v0.4.0 

func (s SortOrder) IsUnsorted() bool

func (SortOrder) Len added in v0.4.0 

func (s SortOrder) Len() int

func (SortOrder) MarshalJSON added in v0.4.0 

func (s SortOrder) MarshalJSON() ([]byte, error)

func (SortOrder) OrderID 

func (s SortOrder) OrderID() int

func (SortOrder) String 

func (s SortOrder) String() string

func (*SortOrder) UnmarshalJSON 

func (s *SortOrder) UnmarshalJSON(b []byte) error

type StagedTable added in v0.2.0 

type StagedTable struct {
	*Table
}

func (*StagedTable) Refresh added in v0.2.0 

func (s *StagedTable) Refresh(ctx context.Context) (*Table, error)

func (*StagedTable) Scan added in v0.2.0 

func (s *StagedTable) Scan(opts ...ScanOption) *Scan

type StatisticsFile added in v0.4.0 

type StatisticsFile struct {
	SnapshotID            int64          `json:"snapshot-id"`
	StatisticsPath        string         `json:"statistics-path"`
	FileSizeInBytes       int64          `json:"file-size-in-bytes"`
	FileFooterSizeInBytes int64          `json:"file-footer-size-in-bytes"`
	KeyMetadata           *string        `json:"key-metadata,omitempty"`
	BlobMetadata          []BlobMetadata `json:"blob-metadata"`
}

StatisticsFile represents a statistics file in the Puffin format, that can be used to read table data more efficiently.

Statistics are informational. A reader can choose to ignore statistics information. Statistics support is not required to read the table correctly.

type Summary 

type Summary struct {
	Operation  Operation
	Properties iceberg.Properties
}

Summary stores the summary information for a snapshot indicating the operation that created the snapshot, and various properties which might exist in the summary.

func (*Summary) Equals 

func (s *Summary) Equals(other *Summary) bool

func (*Summary) MarshalJSON 

func (s *Summary) MarshalJSON() ([]byte, error)

func (*Summary) String 

func (s *Summary) String() string

func (*Summary) UnmarshalJSON 

func (s *Summary) UnmarshalJSON(b []byte) (err error)

type Table 

type Table struct {
	// contains filtered or unexported fields
}

func New 

func New(ident Identifier, meta Metadata, metadataLocation string, fsF FSysF, cat CatalogIO, opts ...Option) *Table

New constructs a Table. If cat implements ScanPlanner — as rest.Catalog does for servers that support remote scan planning — it is wired as the table's planner so (*Scan).PlanFiles can delegate to it; catalogs that do not implement ScanPlanner leave planner nil and planning stays local. This is the concrete Catalog -> Table -> Scan wiring for #1178: the catalog already satisfies ScanPlanner, so no catalog accessor is needed.

func NewFromLocation 

func NewFromLocation(
	ctx context.Context,
	ident Identifier,
	metalocation string,
	fsysF FSysF,
	cat CatalogIO,
	opts ...Option,
) (_ *Table, err error)

func (Table) AllManifests added in v0.2.0 

func (t Table) AllManifests(ctx context.Context) iter.Seq2[iceberg.ManifestFile, error]

AllManifests returns the manifests referenced by the table's snapshots. Tables without snapshots return an empty sequence without resolving file IO.

func (Table) Append added in v0.3.0 

func (t Table) Append(ctx context.Context, rdr array.RecordReader, snapshotProps iceberg.Properties) (*Table, error)

Append is a shortcut for NewTransaction().Append() and then committing the transaction

func (Table) AppendTable added in v0.3.0 

func (t Table) AppendTable(ctx context.Context, tbl arrow.Table, batchSize int64, snapshotProps iceberg.Properties) (*Table, error)

AppendTable is a shortcut for NewTransaction().AppendTable() and then committing the transaction

func (Table) CurrentSnapshot 

func (t Table) CurrentSnapshot() *Snapshot

func (Table) Delete added in v0.5.0 

func (t Table) Delete(ctx context.Context, filter iceberg.BooleanExpression, snapshotProps iceberg.Properties, opts ...DeleteOption) (*Table, error)

Delete is a shortcut for NewTransaction().Delete() and then committing the transaction.

The provided filter acts as a row-level predicate on existing data:

  • Files where all rows match the filter (strict match) are completely deleted
  • Files where some rows match and others don't (partial match) are rewritten to keep only non-matching rows
  • Files where no rows match the filter are kept unchanged

The filter uses both inclusive and strict metrics evaluators on file statistics to classify files:

  • Inclusive evaluator identifies candidate files that may contain matching rows
  • Strict evaluator determines if all rows in a file must match the filter
  • Files that pass inclusive but not strict evaluation are rewritten with filtered data

The concurrency parameter controls the level of parallelism for manifest processing and file rewriting and can be overridden using the WithOverwriteConcurrency option. Defaults to runtime.GOMAXPROCS(0).

func (Table) DeleteOrphanFiles added in v0.4.0 

func (t Table) DeleteOrphanFiles(ctx context.Context, opts ...OrphanCleanupOption) (OrphanCleanupResult, error)

DeleteOrphanFiles identifies files under a table location that are no longer referenced by table metadata and deletes them unless dry-run is enabled.

The table filesystem must implement iceio.ListableIO so orphan cleanup can fully enumerate candidate files before deciding what is safe to delete.

func (Table) Equals 

func (t Table) Equals(other Table) bool

func (Table) ExecuteOrphanCleanup added in v0.7.0 

func (t Table) ExecuteOrphanCleanup(ctx context.Context, plan OrphanCleanupPlan, opts ...OrphanCleanupOption) (OrphanCleanupResult, error)

ExecuteOrphanCleanup deletes exactly the files in plan. It does not perform another orphan scan, so files appearing after planning are not included.

func (Table) FS 

func (t Table) FS(ctx context.Context) (icebergio.IO, error)

func (Table) Identifier 

func (t Table) Identifier() Identifier

func (Table) Inspect added in v0.7.0 

func (t Table) Inspect(opts ...InspectOption) InspectTable

Inspect returns an InspectTable for reading this table's metadata tables, allocating Arrow buffers from opts, if provided, otherwise the default allocator.

func (Table) Location 

func (t Table) Location() string

func (Table) LocationProvider added in v0.2.0 

func (t Table) LocationProvider() (LocationProvider, error)

func (Table) Metadata 

func (t Table) Metadata() Metadata

func (Table) MetadataLocation 

func (t Table) MetadataLocation() string

func (Table) MetricsReporter added in v0.7.0 

func (t Table) MetricsReporter() metrics.Reporter

MetricsReporter returns the table's metrics reporter, never nil.

func (Table) NameMapping added in v0.2.0 

func (t Table) NameMapping() iceberg.NameMapping

func (Table) NewIncrementalAppendScan added in v0.7.0 

func (t Table) NewIncrementalAppendScan(opts ...ScanOption) *IncrementalAppendScan

NewIncrementalAppendScan creates an incremental append file planner. Planning-related ScanOptions configure snapshot selection, filtering, case sensitivity, concurrency, planning mode, and reporting. Projection and row limits are not applied to the returned file tasks; callers can read planned tasks with a separately configured Scan.ReadTasks. Auto planning falls back to local planning. Remote planning returns ErrInvalidOperation until incremental remote planning is implemented.

func (Table) NewIncrementalChangelogScan added in v0.7.0 

func (t Table) NewIncrementalChangelogScan(opts ...ScanOption) *IncrementalChangelogScan

NewIncrementalChangelogScan creates an incremental changelog planner. Projection and row limits are not applied to returned tasks. Auto planning falls back to local planning, while remote planning is not supported. Use ChangelogScanTask.ScanTask with Scan.ReadTasks to read the returned files. Row filters are attached to each task as residuals without partition-specific simplification, matching the existing incremental append scan behavior.

func (Table) NewTransaction added in v0.2.0 

func (t Table) NewTransaction() *Transaction

func (Table) NewTransactionOnBranch added in v0.6.0 

func (t Table) NewTransactionOnBranch(branch string) *Transaction

NewTransactionOnBranch creates a new transaction that commits to the named branch. Use [NewTransaction] to commit to the default "main" branch.

func (Table) NewTransactionOnBranchWithError added in v0.7.0 

func (t Table) NewTransactionOnBranchWithError(branch string) (*Transaction, error)

NewTransactionOnBranchWithError creates a new transaction and returns any metadata initialization error that prevents builder construction.

This preserves the old non-failing constructor contract while allowing callers to receive the precise initialization error instead of hitting panic/undefined behavior later.

func (Table) Overwrite added in v0.5.0 

func (t Table) Overwrite(ctx context.Context, rdr array.RecordReader, snapshotProps iceberg.Properties, opts ...OverwriteOption) (*Table, error)

Overwrite is a shortcut for NewTransaction().Overwrite() and then committing the transaction.

An optional filter (see WithOverwriteFilter) determines which existing data to delete or rewrite:

  • If filter is nil or AlwaysTrue, all existing data files are deleted and replaced with new data.
  • If a filter is provided, it acts as a row-level predicate on existing data:
  • Files where all rows match the filter (strict match) are completely deleted
  • Files where some rows match and others don't (partial match) are rewritten to keep only non-matching rows
  • Files where no rows match the filter are kept unchanged

The filter uses both inclusive and strict metrics evaluators on file statistics to classify files:

  • Inclusive evaluator identifies candidate files that may contain matching rows
  • Strict evaluator determines if all rows in a file must match the filter
  • Files that pass inclusive but not strict evaluation are rewritten with filtered data

New data from the provided RecordReader is written to the table regardless of the filter.

The concurrency parameter controls the level of parallelism for manifest processing and file rewriting and can be overridden using the WithOverwriteConcurrency option. Defaults to runtime.GOMAXPROCS(0).

func (Table) OverwriteTable added in v0.5.0 

func (t Table) OverwriteTable(ctx context.Context, tbl arrow.Table, batchSize int64, snapshotProps iceberg.Properties, opts ...OverwriteOption) (*Table, error)

OverwriteTable is a shortcut for NewTransaction().OverwriteTable() and then committing the transaction.

An optional filter (see WithOverwriteFilter) determines which existing data to delete or rewrite:

  • If filter is nil or AlwaysTrue, all existing data files are deleted and replaced with new data.
  • If a filter is provided, it acts as a row-level predicate on existing data:
  • Files where all rows match the filter (strict match) are completely deleted
  • Files where some rows match and others don't (partial match) are rewritten to keep only non-matching rows
  • Files where no rows match the filter are kept unchanged

The filter uses both inclusive and strict metrics evaluators on file statistics to classify files:

  • Inclusive evaluator identifies candidate files that may contain matching rows
  • Strict evaluator determines if all rows in a file must match the filter
  • Files that pass inclusive but not strict evaluation are rewritten with filtered data

New data from the provided table is written to the table regardless of the filter.

The batchSize parameter refers to the batch size for reading the input data, not the batch size for writes. The concurrency parameter controls the level of parallelism for manifest processing and file rewriting and can be overridden using the WithOverwriteConcurrency option. Defaults to runtime.GOMAXPROCS(0).

func (Table) PlanOrphanFiles added in v0.7.0 

func (t Table) PlanOrphanFiles(ctx context.Context, opts ...OrphanCleanupOption) (OrphanCleanupPlan, error)

PlanOrphanFiles identifies orphan files without deleting anything. The returned plan can be shown to a user and passed to ExecuteOrphanCleanup to delete exactly that set.

func (Table) Properties 

func (t Table) Properties() iceberg.Properties

func (Table) PurgeFiles added in v0.7.0 

func (t Table) PurgeFiles(ctx context.Context) error

PurgeFiles physically deletes all files under the table's warehouse location and any referenced files written outside the location root (e.g., via write.data.path or write.metadata.path properties).

It operates on a best-effort basis. Errors from individual file deletions are collected and returned together as a joined error. If files cannot be deleted (e.g. due to permission errors or missing paths), the errors are logged but the overall catalog drop operation should typically proceed so the catalog does not get out of sync with storage. Non-bulk deletion invokes Remove concurrently and is bounded to defaultPurgeMaxConcurrency operations because it is typically I/O-bound.

func (*Table) Refresh added in v0.4.0 

func (t *Table) Refresh(ctx context.Context) error

func (Table) RewriteDataFiles added in v0.7.0 

func (t Table) RewriteDataFiles(ctx context.Context, groups []CompactionTaskGroup, opts RewriteDataFilesOptions) (*RewriteResult, error)

RewriteDataFiles runs a compaction rewrite as a table-level action. Atomic rewrites are committed before this method returns. Partial-progress rewrites commit each group as they run and return the latest table through RewriteResult.Table.

Use Transaction.RewriteDataFiles when the rewrite must be staged alongside other updates in one transaction. Partial progress is terminal and should normally be invoked through this action so callers do not accidentally try to commit the parent transaction again.

func (Table) Scan 

func (t Table) Scan(opts ...ScanOption) *Scan

func (Table) Schema 

func (t Table) Schema() *iceberg.Schema

func (Table) Schemas 

func (t Table) Schemas() map[int]*iceberg.Schema

func (Table) SnapshotAsOf added in v0.4.0 

func (t Table) SnapshotAsOf(timestampMs int64, inclusive bool) *Snapshot

SnapshotAsOf finds the snapshot that was current as of or right before the given timestamp.

func (Table) SnapshotByID 

func (t Table) SnapshotByID(id int64) *Snapshot

func (Table) SnapshotByName 

func (t Table) SnapshotByName(name string) *Snapshot

func (Table) SortOrder 

func (t Table) SortOrder() SortOrder

func (Table) Spec 

func (t Table) Spec() iceberg.PartitionSpec

type TableCommit added in v0.6.0 

type TableCommit struct {
	Identifier   Identifier
	Requirements []Requirement
	Updates      []Update
}

TableCommit holds the identifier, requirements, and updates for a single table within a multi-table transaction. It is used with [catalog.TransactionalCatalog.CommitTransaction] to atomically commit changes across multiple tables.

type Transaction added in v0.2.0 

type Transaction struct {
	// contains filtered or unexported fields
}

func (*Transaction) AddDataFiles added in v0.5.0 

func (t *Transaction) AddDataFiles(ctx context.Context, dataFiles []iceberg.DataFile, snapshotProps iceberg.Properties, opts ...WriteOption) error

AddDataFiles adds pre-built DataFiles to the table without scanning them from storage. This is useful for clients who have already constructed DataFile objects with metadata, avoiding the need to read files to extract schema and statistics.

Unlike AddFiles, this method does not read files from storage. It validates only metadata that can be checked without opening files (for example spec-id and partition field IDs).

Each DataFile may target any partition spec registered in the table metadata, identified by its SpecID; files are grouped into one manifest per spec. This permits writers to continue producing data under a previous spec after a partition evolution, and rewrites that migrate files between specs. Unregistered spec ids are rejected.

By default this method automatically sets the schema name mapping in table properties if one does not already exist. Pass WithoutAutoNameMapping to disable this behavior, for example when working with catalogs that reject the name mapping property.

Callers are responsible for ensuring each DataFile is valid and consistent with the table. Supplying incorrect DataFile metadata can produce an invalid snapshot and break reads.

func (*Transaction) AddFiles added in v0.2.0 

func (t *Transaction) AddFiles(ctx context.Context, filePaths []string, snapshotProps iceberg.Properties, ignoreDuplicates bool, opts ...AddFilesOption) error

func (*Transaction) Append added in v0.2.0 

func (t *Transaction) Append(ctx context.Context, rdr array.RecordReader, snapshotProps iceberg.Properties) error

func (*Transaction) AppendTable added in v0.3.0 

func (t *Transaction) AppendTable(ctx context.Context, tbl arrow.Table, batchSize int64, snapshotProps iceberg.Properties) error

func (*Transaction) AssertDefaultShape added in v0.7.0 

func (t *Transaction) AssertDefaultShape() error

AssertDefaultShape registers requirements pinning the table's default partition spec ID and default sort order ID at their values in the transaction's base metadata, so a concurrent change to either fails this commit instead of interleaving silently — the shape analogue of AssertRefSnapshotID's explicit branch fence. It gives commits that stage no shape change of their own (for example a properties-only revision marker) the same serialization against concurrent spec or sort-order evolutions that Java UpdateRequirements.forUpdateTable registers for SetDefaultPartitionSpec / SetDefaultSortOrder, and that UpdateSpec and ReplaceSortOrder already register for shape-changing commits; when both are present they pin the same base values and deduplicate.

Register the fence BEFORE staging spec or sort-order changes: requirements are validated against the staged metadata at registration time, so a fence registered after the transaction moved a default fails immediately. Unlike ref assertions the fence is never rewritten by the retry loop's refresh-and-replay, and a transaction with no updates never contacts the catalog, so the fence alone does not force a commit.

func (*Transaction) AssertRefSnapshotID added in v0.7.0 

func (t *Transaction) AssertRefSnapshotID(branch string) error

AssertRefSnapshotID records a requirement that the given branch is unchanged from the transaction's base table metadata — the catalog state this transaction read: the branch must still reference the same snapshot id, or still not exist when the base has none. An empty branch means the transaction's target branch (main unless the transaction was created with NewTransactionOnBranch).

It provides optimistic concurrency for metadata-only commits, such as SetProperties used for exactly-once bookkeeping. Commit normally retries against the fresh branch head; with this requirement it instead fails with ErrCommitFailed if any snapshot was committed to the branch after the transaction's read. Snapshot producers targeting the branch in the same transaction fail the same way rather than retry.

The requirement is submitted together with the transaction's updates; a transaction with no updates never contacts the catalog, so the requirement alone does not force a commit.

func (*Transaction) Commit added in v0.2.0 

func (t *Transaction) Commit(ctx context.Context) (*Table, error)

Commit persists the transaction's staged changes. Metadata validation may be deferred when applying an operation adds no new requirement to the transaction. The staged metadata is validated when a later transaction operation needs an immutable metadata view, or while the catalog applies the commit.

func (*Transaction) CommitEqualityDeletes added in v0.7.0 

func (t *Transaction) CommitEqualityDeletes(ctx context.Context, equalityFieldIDs []int, records iter.Seq2[arrow.RecordBatch, error], snapshotProps iceberg.Properties) error

CommitEqualityDeletes is a convenience wrapper that writes the given records as equality delete files and stages a delete-only row delta in this transaction. It composes Transaction.WriteEqualityDeletes with Transaction.NewRowDelta + RowDelta.AddDeletes + RowDelta.Commit, so it is the delete-only shortcut for CDC/upsert workloads whose caller already holds the equality-key values.

Like the other row-level operations, the change is staged within the transaction; call Transaction.Commit to make it durable in the catalog. Conflict detection runs at that point.

It adds no behavior of its own beyond that composition. In particular:

  • the format-version >= 2 requirement is enforced by WriteEqualityDeletes;
  • equality field IDs are validated by WriteEqualityDeletes and again by RowDelta.Commit;
  • concurrent-writer conflict detection is inherited from RowDelta, which validates against write.delete.isolation-level (serializable by default) when the transaction is committed.

snapshotProps are recorded in the resulting snapshot summary. For update CDC events that must commit deletes and appends in one snapshot, use the lower-level RowDelta flow (AddRows + AddDeletes) directly.

When records yields no rows, no delete files are produced and this is a no-op that returns nil rather than staging an empty snapshot.

Usage: 

if err := tx.CommitEqualityDeletes(ctx, []int{1, 2}, records, nil); err != nil {
	return err
}
_, err := tx.Commit(ctx)

func (*Transaction) Delete added in v0.5.0 

func (t *Transaction) Delete(ctx context.Context, filter iceberg.BooleanExpression, snapshotProps iceberg.Properties, opts ...DeleteOption) (err error)

Delete deletes records matching the provided filter.

The provided filter acts as a row-level predicate on existing data:

  • Files where all rows match the filter (strict match) are completely deleted
  • Files where some rows match and others don't (partial match) are rewritten to keep only non-matching rows
  • Files where no rows match the filter are kept unchanged

The filter uses both inclusive and strict metrics evaluators on file statistics to classify files:

  • Inclusive evaluator identifies candidate files that may contain matching rows
  • Strict evaluator determines if all rows in a file must match the filter
  • Files that pass inclusive but not strict evaluation are rewritten with filtered data

The concurrency parameter controls the level of parallelism for manifest processing and file rewriting and can be overridden using the WithOverwriteConcurrency option. Defaults to runtime.GOMAXPROCS(0).

func (*Transaction) ExpireSnapshots added in v0.4.0 

func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error

ExpireSnapshots removes expired snapshots from the table metadata, staging the changes on the transaction. Call Transaction.Commit to persist them.

A snapshot is retained when it is referenced by a branch or tag, or when it is needed to satisfy the retention rules. Retention is resolved per ref, with reference age falling back through the ref's max-ref-age-ms, the table's MaxRefAgeMsKey property, and the specification default. Snapshot age falls back through the ref's own settings, the options passed here, and finally the table's MinSnapshotsToKeepKey and MaxSnapshotAgeMsKey properties. The snapshot a retained ref points at is always kept, for every branch and tag, so expiry can never leave a ref referencing a snapshot that no longer exists.

By default the now-unreferenced manifests, manifest lists, and data files are deleted once the commit lands; pass WithPostCommit(false) to defer that cleanup to a separate maintenance job and avoid racing in-flight readers. 

txn := tbl.NewTransaction()
err := txn.ExpireSnapshots(
	table.WithOlderThan(7*24*time.Hour),
	table.WithRetainLast(10),
)
if err != nil {
	// ...
}
newTbl, err := txn.Commit(ctx)

The "iceberg expire-snapshots" CLI command wraps the same operation.

func (*Transaction) MarkCommitted added in v0.6.0 

func (t *Transaction) MarkCommitted()

MarkCommitted marks the transaction as committed, preventing further use. This should be called after a successful multi-table commit via [catalog.TransactionalCatalog.CommitTransaction].

func (*Transaction) NewRewrite added in v0.6.0 

func (t *Transaction) NewRewrite(snapshotProps iceberg.Properties) *RewriteFiles

NewRewrite returns a RewriteFiles builder bound to this transaction. Mirrors Java's org.apache.iceberg.Table#newRewrite. snapshotProps is cloned and the clone is added to the rewrite snapshot's summary; pass nil for none.

Usage: 

rewrite := tx.NewRewrite(nil)
rewrite.DeleteFile(oldDataFile)
rewrite.AddDataFile(newDataFile)
if err := rewrite.Commit(ctx); err != nil { ... }
committed, err := tx.Commit(ctx)

func (*Transaction) NewRowDelta added in v0.6.0 

func (t *Transaction) NewRowDelta(snapshotProps iceberg.Properties) *RowDelta

NewRowDelta creates a new RowDelta for committing row-level changes within this transaction. The provided properties are included in the snapshot summary.

func (*Transaction) Overwrite added in v0.5.0 

func (t *Transaction) Overwrite(ctx context.Context, rdr array.RecordReader, snapshotProps iceberg.Properties, opts ...OverwriteOption) error

Overwrite overwrites the table data using a RecordReader.

An optional filter (see WithOverwriteFilter) determines which existing data to delete or rewrite:

  • If filter is nil or AlwaysTrue, all existing data files are deleted and replaced with new data.
  • If a filter is provided, it acts as a row-level predicate on existing data:
  • Files where all rows match the filter (strict match) are completely deleted
  • Files where some rows match and others don't (partial match) are rewritten to keep only non-matching rows
  • Files where no rows match the filter are kept unchanged

The filter uses both inclusive and strict metrics evaluators on file statistics to classify files:

  • Inclusive evaluator identifies candidate files that may contain matching rows
  • Strict evaluator determines if all rows in a file must match the filter
  • Files that pass inclusive but not strict evaluation are rewritten with filtered data

New data from the provided RecordReader is written to the table regardless of the filter.

The concurrency parameter controls the level of parallelism for manifest processing and file rewriting and can be overridden using the WithOverwriteConcurrency option. If concurrency <= 0, defaults to runtime.GOMAXPROCS(0).

func (*Transaction) OverwriteTable added in v0.5.0 

func (t *Transaction) OverwriteTable(ctx context.Context, tbl arrow.Table, batchSize int64, snapshotProps iceberg.Properties, opts ...OverwriteOption) error

OverwriteTable overwrites the table data using an Arrow Table.

An optional filter (see WithOverwriteFilter) determines which existing data to delete or rewrite:

  • If filter is nil or AlwaysTrue, all existing data files are deleted and replaced with new data.
  • If a filter is provided, it acts as a row-level predicate on existing data:
  • Files where all rows match the filter (strict match) are completely deleted
  • Files where some rows match and others don't (partial match) are rewritten to keep only non-matching rows
  • Files where no rows match the filter are kept unchanged

The filter uses both inclusive and strict metrics evaluators on file statistics to classify files:

  • Inclusive evaluator identifies candidate files that may contain matching rows
  • Strict evaluator determines if all rows in a file must match the filter
  • Files that pass inclusive but not strict evaluation are rewritten with filtered data

New data from the provided table is written to the table regardless of the filter.

The batchSize parameter refers to the batch size for reading the input data, not the batch size for writes. The concurrency parameter controls the level of parallelism for manifest processing and file rewriting and can be overridden using the WithOverwriteConcurrency option. If concurrency <= 0, defaults to runtime.GOMAXPROCS(0).

func (*Transaction) RemoveProperties added in v0.7.0 

func (t *Transaction) RemoveProperties(keys []string) error

RemoveProperties removes the given property keys from the table. Keys that are not currently set are ignored.

func (*Transaction) ReplaceDataFiles added in v0.3.0 

func (t *Transaction) ReplaceDataFiles(ctx context.Context, filesToDelete, filesToAdd []string, snapshotProps iceberg.Properties) error

ReplaceFiles is actually just an overwrite operation with multiple files deleted and added.

TODO: technically, this could be a REPLACE operation but we aren't performing any validation here that there are no changes to the underlying data. A REPLACE operation is only valid if the data is exactly the same as the previous snapshot.

For now, we'll keep using an overwrite operation.

func (*Transaction) ReplaceDataFilesWithDataFiles added in v0.5.0 

func (t *Transaction) ReplaceDataFilesWithDataFiles(ctx context.Context, filesToDelete, filesToAdd []iceberg.DataFile, snapshotProps iceberg.Properties, opts ...WriteOption) error

ReplaceDataFilesWithDataFiles replaces files using pre-built DataFile objects. This avoids scanning files to extract schema and statistics - the caller provides DataFile objects directly with all required metadata.

For the files to add, use iceberg.NewDataFileBuilder to construct DataFile objects with the appropriate metadata (path, record count, file size, partition values).

This method does not open files. It validates only metadata that can be checked without reading file contents.

By default this method automatically sets the schema name mapping in table properties if one does not already exist. Pass WithoutAutoNameMapping to disable this behavior, for example when working with catalogs that reject the name mapping property.

Callers are responsible for ensuring each DataFile is valid and consistent with the table. Supplying incorrect DataFile metadata can produce an invalid snapshot and break reads.

This is useful when:

  • Files are written via a separate I/O path and metadata is already known
  • Avoiding file scanning improves performance or reliability
  • Working with storage systems where immediate file reads may be unreliable

func (*Transaction) ReplaceFiles added in v0.6.0 

func (t *Transaction) ReplaceFiles(ctx context.Context, dataFilesToDelete, dataFilesToAdd, deleteFilesToRemove []iceberg.DataFile, snapshotProps iceberg.Properties, opts ...WriteOption) error

ReplaceFiles atomically replaces data files and removes associated delete files in a single snapshot. This is the commit primitive for compaction: old data files are replaced with new (compacted) data files, and delete files that are fully applied are removed.

func (*Transaction) ReplaceFilesWithDeleteFiles added in v0.7.0 

func (t *Transaction) ReplaceFilesWithDeleteFiles(ctx context.Context, dataFilesToDelete, dataFilesToAdd, deleteFilesToRemove []iceberg.DataFile, deleteFilesToAdd []DeleteFileAddition, snapshotProps iceberg.Properties, opts ...WriteOption) error

ReplaceFilesWithDeleteFiles atomically replaces data files, adds new delete files, and removes superseded delete files in one snapshot. Every added delete file must carry the data sequence number of the delete files it replaces. This is intended for rewrites that change delete-file representation, such as rewriting position deletes into deletion vectors.

func (*Transaction) ReplaceSortOrder added in v0.7.0 

func (t *Transaction) ReplaceSortOrder(order SortOrder) error

ReplaceSortOrder stages order as the table's new default sort order, the analogue of Java's Table.replaceSortOrder() / BaseReplaceSortOrder. The metadata builder reuses the ID of an equivalent existing order or assigns the next available one, so order's own ID is ignored. The commit is fenced by AssertDefaultSortOrderID, matching UpdateRequirements.forUpdateTable on SetDefaultSortOrder. Replacing the default with a field-identical order is a no-op.

func (*Transaction) RewriteDataFiles added in v0.6.0 

func (t *Transaction) RewriteDataFiles(ctx context.Context, groups []CompactionTaskGroup, opts RewriteDataFilesOptions) (*RewriteResult, error)

RewriteDataFiles compacts the given groups by reading data with deletes applied, writing new consolidated files, and atomically replacing the old files. Position delete files that are fully applied (every referenced data file is in the rewrite set) are removed automatically.

Cleanup beyond that per-group staging is the caller's responsibility: compute the dead sets with [compaction.CollectDeadEqualityDeletesWithSpecs] and [compaction.CollectDeadPositionDeletes] (against the same snapshot the rewrite is staged on) and pass them via RewriteDataFilesOptions.ExtraDeleteFilesToRemove. The executor only orchestrates the commit; it does not impose a cleanup policy. This split keeps the pure spec predicate in table/compaction and the unexported snapshot machinery in table.

Use [compaction.Config.PlanCompaction] to produce the groups, then convert [compaction.Group] → CompactionTaskGroup and pass them here. Distributed coordinators stage worker results via ExecuteCompactionGroup and commit them via Transaction.NewRewrite + RewriteFiles.ApplyResult + RewriteFiles.Commit instead.

func (*Transaction) RewriteManifests added in v0.7.0 

func (t *Transaction) RewriteManifests(ctx context.Context, opts ...RewriteManifestsOpt) (*RewriteManifestsResult, error)

RewriteManifests merges small data manifests in the current snapshot into fewer, target-sized ones and stages the result as a REPLACE snapshot. It rewrites metadata only; no data files are read or written. Delete manifests are left untouched.

By default, manifests are clustered by size only (bin-packed toward the target size). With WithRewriteManifestClusterBy, entries are grouped by the returned key and partition spec before the same target-size rolling is applied. This can reduce planning time when the key matches common query filters, because each manifest contains fewer unrelated partition values.

On a V3 table each rewrite advances next-row-id by the eligible manifests' row count even though no rows are written, so running it on a cadence steadily consumes row-ID space.

A no-op result (IsNoOp) means there was nothing to do — either the target branch has no head (NoOpNoSnapshot) or the eligible manifests are already optimal (NoOpAlreadyOptimal). A no-op writes no manifest list and stages nothing on the transaction, so a following Commit is a true no-op; callers can skip it either way.

func (*Transaction) RollbackToSnapshot added in v0.6.0 

func (t *Transaction) RollbackToSnapshot(snapshotID int64) error

func (*Transaction) Scan added in v0.2.0 

func (t *Transaction) Scan(opts ...ScanOption) (*Scan, error)

func (*Transaction) SetProperties added in v0.2.0 

func (t *Transaction) SetProperties(props iceberg.Properties) error

func (*Transaction) StagedTable added in v0.2.0 

func (t *Transaction) StagedTable() (*StagedTable, error)

func (*Transaction) TableCommit added in v0.6.0 

func (t *Transaction) TableCommit() (TableCommit, error)

TableCommit returns a TableCommit representing the pending changes in this transaction without actually committing them. This is intended for multi-table transactions where several TableCommit values are collected and submitted together via [catalog.TransactionalCatalog.CommitTransaction].

Most callers should use [catalog.MultiTableTransaction] instead of calling this method directly — it handles extraction, commit, and lifecycle management automatically.

The returned requirements are the ones Transaction.Commit would send, so the multi-table path is fenced by the target branch's base head. A transaction with no updates yields an empty payload and enforces none.

TableCommit does not mark the transaction as committed — the caller is responsible for either calling Commit (single-table) or submitting the returned TableCommit via CommitTransaction (multi-table). After a successful multi-table commit the caller should call MarkCommitted to prevent accidental reuse.

PostCommit hooks are NOT executed by this method. Because the multi-table commit endpoint returns 204 No Content (no metadata), callers must LoadTable after a successful CommitTransaction if they need updated state.

func (*Transaction) UpdateSchema added in v0.5.0 

func (t *Transaction) UpdateSchema(caseSensitive bool, allowIncompatibleChanges bool, opts ...UpdateSchemaOption) *UpdateSchema

UpdateSchema creates a new UpdateSchema instance for managing schema changes within this transaction.

Parameters:

  • caseSensitive: If true, field name lookups are case-sensitive; if false, field names are matched case-insensitively.
  • allowIncompatibleChanges: If true, allows schema changes that would normally be rejected for being incompatible (e.g., adding required fields without default values, changing field types in non-promotable ways, or changing column nullability from optional to required).
  • opts: Optional configuration functions to customize the UpdateSchema behavior.

Returns an UpdateSchema instance that can be used to build and apply schema changes.

func (*Transaction) UpdateSpec added in v0.4.0 

func (t *Transaction) UpdateSpec(caseSensitive bool) *UpdateSpec

func (*Transaction) UpgradeFormatVersion added in v0.6.0 

func (t *Transaction) UpgradeFormatVersion(version int) error

UpgradeFormatVersion upgrades the table to the given format version. Downgrading is not allowed. If the table is already at the given version, this is a no-op.

func (*Transaction) WriteEqualityDeletes added in v0.6.0 

func (t *Transaction) WriteEqualityDeletes(ctx context.Context, equalityFieldIDs []int, records iter.Seq2[arrow.RecordBatch, error]) ([]iceberg.DataFile, error)

WriteEqualityDeletes writes Arrow record batches as equality delete Parquet files and returns the resulting DataFiles. The returned files have ContentType == EntryContentEqDeletes and EqualityFieldIDs set, ready to be passed to RowDelta.AddDeletes.

The equalityFieldIDs identify which columns in the table schema form the delete key. The provided records must contain exactly those columns.

The table must use format version 2 or higher.

For partitioned tables, the provided records must include the partition source columns in addition to the equality key columns so that records can be routed to the correct partition directories. If the partition source columns overlap with the equality key columns, no extra columns are needed.

Usage: 

deleteFiles, err := tx.WriteEqualityDeletes(ctx, []int{1, 2}, records)
rd := tx.NewRowDelta(nil)
rd.AddDeletes(deleteFiles...)
err = rd.Commit(ctx)

type UnboundSortOrder added in v0.7.0 

type UnboundSortOrder struct {
	SortOrder
}

UnboundSortOrder decodes a sort order whose source IDs have not yet been resolved against the schema that decides them, so unlike bound field IDs they need not be positive. Two wire forms arrive this way:

  • A create-table request's write order, which carries the client's placeholder source IDs rather than table field IDs. A client numbers those placeholders however it likes: Spark numbers the columns of a new table from zero, so sorting by the first column arrives as source-id 0. Binding the embedded order to the schema the client sent resolves them.
  • An add-sort-order commit payload, which applying the update checks against the table's current schema.

Decoding unbound defers the check rather than dropping it: every source ID must still resolve in the schema it is checked against, and no transform is exempt, so an order the schema cannot satisfy is rejected when the update is applied.

Use SortOrder for orders read from table metadata, where source IDs are bound field IDs and must be positive. Catalog implementations should decode both the REST create-table request's write order and an add-sort-order commit payload into this type.

func (*UnboundSortOrder) UnmarshalJSON added in v0.7.0 

func (u *UnboundSortOrder) UnmarshalJSON(b []byte) error

type UnsupportedType added in v0.5.0 

type UnsupportedType struct {
	MinFormatVersion int
}

type Update added in v0.2.0 

type Update interface {
	// Action returns the name of the action that the update represents.
	Action() string
	// Apply applies the update to the given metadata builder.
	Apply(*MetadataBuilder) error
	// PostCommit is called after successful commit of the update
	PostCommit(context.Context, *Table, *Table) error
}

Update represents a change to a table's metadata.

func NewSetLocationUpdate added in v0.2.0 

func NewSetLocationUpdate(loc string) Update

NewSetLocationUpdate creates a new update that sets the location of the table metadata.

type UpdateSchema added in v0.5.0 

type UpdateSchema struct {
	// contains filtered or unexported fields
}

UpdateSchema manages schema evolution operations within a transaction. It supports adding, deleting, renaming, updating, and reordering columns, and ensures all changes are validated before being committed.

Operations can be chained together and are applied in the order they are called. Changes are not persisted until Commit() is called.

Basic Usage: 

txn := table.NewTransaction()
updateSchema := txn.UpdateSchema(true, false)

// Add a new column
updateSchema.AddColumn([]string{"email"}, iceberg.PrimitiveTypes.String, "Email address", false, nil)

// Commit changes
if err := updateSchema.Commit(); err != nil {
    return err
}
if _, err := txn.Commit(ctx); err != nil {
    return err
}

Chaining Operations: 

updateSchema.
    AddColumn([]string{"age"}, iceberg.PrimitiveTypes.Int, "User age", false, nil).
    RenameColumn([]string{"name"}, "full_name").
    MoveFirst([]string{"id"}).
    Commit()

Adding Nested Columns: 

// Add a column to a struct field
updateSchema.AddColumn([]string{"address", "country"}, iceberg.PrimitiveTypes.String, "Country code", false, iceberg.StringLiteral("US"))

// Commit the schema update
if err := updateSchema.Commit(); err != nil {
    return err
}
if _, err := txn.Commit(ctx); err != nil {
    return err
}

func NewUpdateSchema added in v0.5.0 

func NewUpdateSchema(txn *Transaction, caseSensitive bool, allowIncompatibleChanges bool, opts ...UpdateSchemaOption) *UpdateSchema

NewUpdateSchema creates a new UpdateSchema instance for managing schema changes within a transaction.

Parameters:

  • txn: The transaction that this schema update will be applied to.
  • caseSensitive: If true, field name lookups are case-sensitive; if false, field names are matched case-insensitively.
  • allowIncompatibleChanges: If true, allows schema changes that would normally be rejected for being incompatible (e.g., adding required fields without default values, changing field types in non-promotable ways, or changing column nullability from optional to required).
  • opts: Optional configuration functions to customize the UpdateSchema behavior.

Returns an UpdateSchema instance that can be used to build and apply schema changes.

func (*UpdateSchema) AddColumn added in v0.5.0 

func (u *UpdateSchema) AddColumn(path []string, fieldType iceberg.Type, doc string, required bool, defaultValue iceberg.Literal) *UpdateSchema

func (*UpdateSchema) Apply added in v0.5.0 

func (u *UpdateSchema) Apply() (*iceberg.Schema, error)

func (*UpdateSchema) BuildUpdates added in v0.5.0 

func (u *UpdateSchema) BuildUpdates() ([]Update, []Requirement, error)

func (*UpdateSchema) Commit added in v0.5.0 

func (u *UpdateSchema) Commit() error

func (*UpdateSchema) DeleteColumn added in v0.5.0 

func (u *UpdateSchema) DeleteColumn(path []string) *UpdateSchema

func (*UpdateSchema) MoveAfter added in v0.5.0 

func (u *UpdateSchema) MoveAfter(path, relativeTo []string) *UpdateSchema

func (*UpdateSchema) MoveBefore added in v0.5.0 

func (u *UpdateSchema) MoveBefore(path, relativeTo []string) *UpdateSchema

func (*UpdateSchema) MoveColumn added in v0.5.0 

func (u *UpdateSchema) MoveColumn(op MoveOp, path, relativeTo []string) *UpdateSchema

func (*UpdateSchema) MoveFirst added in v0.5.0 

func (u *UpdateSchema) MoveFirst(path []string) *UpdateSchema

func (*UpdateSchema) RenameColumn added in v0.5.0 

func (u *UpdateSchema) RenameColumn(path []string, newName string) *UpdateSchema

func (*UpdateSchema) SetIdentifierField added in v0.5.0 

func (u *UpdateSchema) SetIdentifierField(paths [][]string) *UpdateSchema

func (*UpdateSchema) UnionByNameWith added in v0.7.0 

func (u *UpdateSchema) UnionByNameWith(newSchema *iceberg.Schema) *UpdateSchema

UnionByNameWith stages changes to evolve the current schema into the union of itself and newSchema, matching fields by name (honoring caseSensitive):

  • New fields are added as optional (regardless of their incoming required flag), keeping their Doc/InitialDefault/WriteDefault and getting fresh ids.
  • Existing fields may be evolved: required->optional is applied, optional->required is skipped. A primitive type change is applied only if it is a valid promotion (int->long, float->double, decimal same-scale wider-precision, or exact match); narrowing is ignored and any other change errors.
  • A Doc is updated only when the incoming Doc is non-empty and differs (an empty Doc never clears an existing one).
  • A WriteDefault is updated when the incoming value differs; an existing column's InitialDefault is never modified.
  • Map keys are immutable and cross-kind changes (list->map, struct->list, etc.) are rejected rather than silently grafted onto the existing column.

Matching consults only the current schema, not columns staged by an earlier op in the same UpdateSchema; pre-merge such schemas before a single call.

func (*UpdateSchema) UpdateColumn added in v0.5.0 

func (u *UpdateSchema) UpdateColumn(path []string, update ColumnUpdate) *UpdateSchema

type UpdateSchemaOption added in v0.5.0 

type UpdateSchemaOption func(*UpdateSchema)

UpdateSchemaOption is a functional option for configuring UpdateSchema.

func WithNameMapping added in v0.5.0 

func WithNameMapping(nameMapping iceberg.NameMapping) UpdateSchemaOption

WithNameMapping configures the UpdateSchema to use the provided name mapping for tracking field name changes and ensuring consistency during schema evolution.

type UpdateSpec added in v0.4.0 

type UpdateSpec struct {
	// contains filtered or unexported fields
}

UpdateSpec implements a builder for evolving a table's partition specification.

It accumulates a sequence of partition spec update operations (e.g., AddField, RemoveField, RenameField) which are applied during BuildUpdates.

Use the builder methods to chain operations, and call BuildUpdates to apply them and produce the final set of partition fields and update requirements, or call Commit to apply the updates in the transaction.

func NewUpdateSpec added in v0.4.0 

func NewUpdateSpec(t *Transaction, caseSensitive bool) *UpdateSpec

NewUpdateSpec starts a partition spec evolution.

If the table's current spec contains an unknown transform, the returned UpdateSpec carries an error and no evolution is possible -- including renaming or dropping an unrelated field. That matches Java's BaseUpdatePartitionSpec, which rejects at construction rather than at commit. Loading such a table is still fine; only evolving its spec is blocked.

func (*UpdateSpec) AddField added in v0.4.0 

func (us *UpdateSpec) AddField(sourceColName string, transform iceberg.Transform, partitionFieldName string) *UpdateSpec

func (*UpdateSpec) AddIdentity added in v0.4.0 

func (us *UpdateSpec) AddIdentity(sourceColName string) *UpdateSpec

func (*UpdateSpec) Apply added in v0.4.0 

func (us *UpdateSpec) Apply() (iceberg.PartitionSpec, error)

func (*UpdateSpec) BuildUpdates added in v0.4.0 

func (us *UpdateSpec) BuildUpdates() ([]Update, []Requirement, error)

func (*UpdateSpec) Commit added in v0.4.0 

func (us *UpdateSpec) Commit() error

func (*UpdateSpec) RemoveField added in v0.4.0 

func (us *UpdateSpec) RemoveField(name string) *UpdateSpec

func (*UpdateSpec) RenameField added in v0.4.0 

func (us *UpdateSpec) RenameField(name string, newName string) *UpdateSpec

type Updates added in v0.4.0 

type Updates []Update

func (*Updates) UnmarshalJSON added in v0.4.0 

func (u *Updates) UnmarshalJSON(data []byte) error

type WriteOption added in v0.6.0 

type WriteOption func(*dataFileCfg)

WriteOption is an option for methods that operate on pre-built DataFile objects.

func WithoutAutoNameMapping added in v0.6.0 

func WithoutAutoNameMapping() WriteOption

WithoutAutoNameMapping disables the automatic setting of the schema name mapping in table properties. By default, methods like Transaction.AddDataFiles and Transaction.ReplaceDataFilesWithDataFiles will set the name mapping if one does not already exist. This option is useful when working with catalogs (such as Databricks Unity Catalog) that reject the name mapping property.

func WithoutDuplicateCheck added in v0.6.0 

func WithoutDuplicateCheck() WriteOption

WithoutDuplicateCheck disables the duplicate file path check against existing data files in the current snapshot. By default, Transaction.AddDataFiles scans all manifests to ensure no file being added already exists in the table. For tables with many manifests this scan can be expensive because each manifest must be read from storage. Use this option when the caller can guarantee that the files being added are not already in the table.

type WriteRecordOption added in v0.6.0 

type WriteRecordOption func(*writeRecordConfig)

WriteRecordOption configures the behavior of WriteRecords.

func WithClusteredWrite added in v0.6.0 

func WithClusteredWrite() WriteRecordOption

WithClusteredWrite enables the memory-efficient clustered write path for partitioned tables. It keeps at most one partition writer open at a time: when a record arrives for a new partition, the current writer is flushed and closed before a new one is opened.

The input must be clustered by partition across batches: once a partition's writer has been closed, encountering further records for that partition returns an error. Within a single batch the writer reclusters rows by partition, so interleaved values like [a,b,a,b] are accepted; the strict check fires only across batch boundaries. This is the natural order for compaction, where each source data file typically belongs to a single partition. If the input is not clustered across batches, use the fanout writer (the default) instead.

Combining this option with WithMaxWriteWorkers is rejected by WriteRecords: the clustered path is single-threaded by design.

func WithMaxWriteWorkers added in v0.6.0 

func WithMaxWriteWorkers(n int) WriteRecordOption

WithMaxWriteWorkers overrides the default number of fanout workers used for partitioned writes. Each worker processes record batches, partitions them, and writes to the appropriate partition files. Fewer workers means fewer concurrent parquet writers compressing pages simultaneously, which reduces peak memory. A value of 0 (the default) uses config.EnvConfig.MaxWorkers.

Combining this option with WithClusteredWrite is rejected by WriteRecords: the clustered write path is single-threaded by design, so the two options have no meaningful interaction.

func WithParquetRowGroupLimit added in v0.7.0 

func WithParquetRowGroupLimit(n int) WriteRecordOption

WithParquetRowGroupLimit overrides the table's write.parquet.row-group-limit property for this write, capping the number of rows per Parquet row group in the output files. Smaller row groups bound the writer's buffered memory before each flush. A non-positive value is ignored.

func WithPreserveRowLineage added in v0.7.0 

func WithPreserveRowLineage(schema *iceberg.Schema) WriteRecordOption

WithPreserveRowLineage sets the output file schema to include the v3 row- lineage metadata columns (_row_id, _last_updated_sequence_number) so that row identity is preserved through rewrites and compactions. The input records must already carry _row_id (e.g. from a scan that projected the lineage columns).

WriteRecords validates this option against the table state and the input Arrow schema: it errors when applied to a v1/v2 table or when the input records don't include the _row_id column. The schema parameter is the projected Iceberg schema (typically iceberg.SchemaWithRowLineage) and is used to write the output Parquet's field IDs.

func WithRecordBatchBufferSize added in v0.7.0 

func WithRecordBatchBufferSize(n int) WriteRecordOption

WithRecordBatchBufferSize sets the capacity, in record batches, of each rolling data writer's input channel. Every buffered batch is retained in memory until its writer consumes it, so this bound times the batch row count caps the memory a stalled writer can hold. The default is 64 batches. A non-positive value is ignored.

func WithTargetFileSize added in v0.6.0 

func WithTargetFileSize(size int64) WriteRecordOption

WithTargetFileSize overrides the table's default target file size.

func WithWriteUUID added in v0.6.0 

func WithWriteUUID(id uuid.UUID) WriteRecordOption

WithWriteUUID sets a specific UUID for file naming.

type WriteTask added in v0.3.0 

type WriteTask struct {
	Uuid        uuid.UUID
	ID          int
	PartitionID int // PartitionID is the partition identifier used in data file naming.
	FileCount   int // FileCount is a sequential counter for files written by this task.
	Schema      *iceberg.Schema
	Batches     []arrow.RecordBatch
	// SortOrderID claims Batches are globally sorted by that order; writeFile
	// records it unverified. Must stay zero for position deletes.
	SortOrderID int
}

WriteTask writes all its Batches into one file. The rolling writer bypasses it, naming files via dataFileName directly.

func (WriteTask) GenerateDataFileName added in v0.3.0 

func (w WriteTask) GenerateDataFileName(extension string) string

Source Files

View all Source files

Directories

Path Synopsis
Package compaction provides bin-pack compaction planning for Iceberg tables.
 Package compaction provides bin-pack compaction planning for Iceberg tables.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.