parquetio

type Codec string

type ReadOptions struct {
	// Columns projects the file to a subset of top-level columns by
	// name. nil or empty = read all columns.
	//
	// Names not present in the file's schema return ErrColumnNotFound.
	// Column projection is applied at the parquet reader layer: the
	// excluded columns are never fetched, decompressed, or materialized
	// into arrow arrays. The savings scale with how large those columns
	// are relative to the file — narrow analytical files where the caller
	// wants a few columns out of many benefit most.
	Columns []string

	// ChunkRows is the arrow record-batch size used by
	// ReadFileChunksFunc. Each RecordReader.Next() call produces at
	// most ChunkRows rows. 0 = DefaultChunkRows. Ignored by ReadFile.
	//
	// Sub-partitioning a row group into fixed-size batches is what
	// bounds streaming memory to ~one batch at a time regardless of
	// the file's row-group sizes.
	ChunkRows int

	// Allocator overrides the Arrow allocator. nil = memory.DefaultAllocator.
	Allocator memory.Allocator

	// Predicate is a hint from the optimizer for row-group skipping.
	// When set, ReadFile / ReadFileChunksFunc walk each row-group's
	// footer statistics and skip whole groups whose (min, max) bounds
	// prove no row could satisfy the predicate. The Filter operation
	// above the read still runs — this is a coarse fast-path that
	// avoids fetching irrelevant row-groups off disk.
	//
	// Predicates only prune when they reference columns present in
	// the file's schema. Unrecognized columns silently prevent
	// pruning (conservative — a "maybe" survives). Uses the same
	// Expr type as Frame.FilterExpr.
	Predicate gobi.Expr

	// RowGroups optionally restricts the read to a specific set of
	// row-group indices. When set, ReadFile / ReadFileChunksFunc /
	// ScanFile process only those row-groups; when nil or empty,
	// all row-groups are read.
	//
	// Primarily used internally to partition scans across parallel
	// workers (see ScanWorkers) — each worker gets a disjoint
	// RowGroups slice. Callers can set it directly for very
	// targeted reads (e.g. "just the last row-group of this file"),
	// though the more common way to restrict is via Columns or
	// Predicate.
	RowGroups []int

	// ScanWorkers controls row-group-level parallelism for ScanFile.
	// 0 (default) = runtime.GOMAXPROCS(0), capped at NumRowGroups.
	// 1 = single-threaded (the previous behavior). n > 1 = n
	// workers, capped at NumRowGroups.
	//
	// Ignored by ReadFile (always single-threaded — reads one whole
	// file) and by ReadFileChunksFunc (also single-threaded — the
	// callback API is fundamentally serial). Applies only when the
	// scan flows through the Layer 6 executor via ScanFile +
	// LazyFrame.Collect.
	ScanWorkers int
}

type WriteOptions struct {
	// Codec selects the Parquet page compression codec. Empty string
	// defaults to CodecSnappy — matches parquet-arrow's own default
	// and is the common choice for good balance between size and
	// decode speed.
	Codec Codec

	// RowGroupRows caps the maximum number of rows per row group. 0
	// uses parquet-arrow's default (~1M rows).
	//
	// Smaller row groups → more granular predicate pushdown (readers
	// can skip whole groups via rowgroup statistics) and lower peak
	// memory when streaming one group at a time. Larger row groups →
	// better compression ratios and less per-group metadata overhead.
	// 64k–256k is a reasonable range for analytical workloads that
	// filter on min/max stats; leave at 0 for archive/bulk-load files
	// where read patterns are full-scan.
	RowGroupRows int64

	// BloomFilterColumns names columns that should have a bloom
	// filter attached to each row group. High-cardinality equality-
	// filtered columns (user IDs, hashes, categorical keys) benefit
	// most; skew-free min/max distributions do not — parquet's row-
	// group statistics already handle those.
	//
	// gobi's own reader does not yet consume bloom filters for row-
	// group skipping (that lands with the query optimizer). Files
	// produced here are still consumed correctly by DuckDB, Spark,
	// Polars, and pyarrow, which do use bloom filters for predicate
	// pushdown on equality filters.
	BloomFilterColumns []string

	// BloomFilterFPP is the target false-positive probability for
	// the bloom filters written above. 0 uses arrow-go's default
	// (0.05). Lower FPP → larger filter on disk; reasonable range
	// 0.01–0.1. Ignored when BloomFilterColumns is empty.
	BloomFilterFPP float64
}