dataset

type AndPredicate struct{ Left, Right Predicate }

type BinaryValueSet struct {
	// contains filtered or unexported fields
}

type BuilderOptions struct {
	// PageSizeHint is the soft limit for the size of the page. Builders try to
	// fill pages as close to this size as possible, but the actual size may be
	// slightly larger or smaller.
	PageSizeHint int

	// PageMaxRowCount is the limit for the number of rows of the page.
	// When 0 or a negative number, then builders use the [BuilderOptions.PageSizeHint]
	// option to determine when a page needs to be flushed.
	PageMaxRowCount int

	// Type is the type of data in the column. Type.Physical is used for
	// encoding; Type.Logical is used as a hint to readers.
	Type ColumnType

	// Encoding is the encoding algorithm to use for values.
	Encoding datasetmd.EncodingType

	// Compression is the compression algorithm to use for values.
	Compression datasetmd.CompressionType

	// CompressionOptions holds optional configuration for compression.
	CompressionOptions CompressionOptions

	// StatisticsOptions holds optional configuration for statistics.
	Statistics StatisticsOptions
}

type Column interface {
	// ColumnDesc returns the metadata for the Column.
	ColumnDesc() *ColumnDesc

	// ListPages returns the set of ordered pages in the column.
	ListPages(ctx context.Context) result.Seq[Page]
}

type ColumnBuilder struct {
	// contains filtered or unexported fields
}

type ColumnDesc struct {
	Type        ColumnType                // Type of values in the column.
	Tag         string                    // Optional string to distinguish columns with the same type.
	Compression datasetmd.CompressionType // Compression used for the column.

	PagesCount       int // Total number of pages in the column.
	RowsCount        int // Total number of rows in the column.
	ValuesCount      int // Total number of non-NULL values in the column.
	CompressedSize   int // Total size of all pages in the column after compression.
	UncompressedSize int // Total size of all pages in the column before compression.

	Statistics *datasetmd.Statistics // Optional statistics for the column.
}

type ColumnType struct {
	// Physical is the type of values physically stored within the column.
	Physical datasetmd.PhysicalType

	// Logical is a custom string indicating how dataset-derived sections
	// should interpret the physical type.
	Logical string
}

type CompressionOptions struct {
	// Zstd holds encoding options for Zstd compression. Only used for
	// [datasetmd.COMPRESSION_TYPE_ZSTD].
	Zstd []zstd.EOption
}

type Dataset interface {
	// ListColumns returns the set of [Column]s in the Dataset. The order of
	// Columns in the returned sequence must be consistent across calls.
	ListColumns(ctx context.Context) result.Seq[Column]

	// ListPages retrieves a set of [Pages] given a list of [Column]s.
	// Implementations of Dataset may use ListPages to optimize for batch reads.
	// The order of [Pages] in the returned sequence must match the order of the
	// columns argument.
	ListPages(ctx context.Context, columns []Column) result.Seq[Pages]

	// ReadPages returns the set of [PageData] for the specified slice of pages.
	// Implementations of Dataset may use ReadPages to optimize for batch reads.
	// The order of [PageData] in the returned sequence must match the order of
	// the pages argument.
	ReadPages(ctx context.Context, pages []Page) result.Seq[PageData]
}

type DownloadStats struct {
	// Use the following three stats together.
	// 1. PagesScanned is the total number of ReadPage calls made to the downloader.
	// 2. PagesFoundInCache is the number of pages that were found in cache and
	//    did not require a download.
	// 3. BatchDownloadRequests is the number of batch download requests made by
	//    the downloader when a page is not found in cache.
	PagesScanned          uint64
	PagesFoundInCache     uint64
	BatchDownloadRequests uint64

	PageDownloadTime time.Duration // Total time taken for downloading paga data or metadata.

	PrimaryColumnPages               uint64 // Number of pages downloaded for primary columns
	SecondaryColumnPages             uint64 // Number of pages downloaded for secondary columns
	PrimaryColumnBytes               uint64 // Total bytes downloaded for primary columns
	SecondaryColumnBytes             uint64 // Total bytes downloaded for secondary columns
	PrimaryColumnUncompressedBytes   uint64 // Total uncompressed bytes for primary columns
	SecondaryColumnUncompressedBytes uint64 // Total uncompressed bytes for secondary columns
}

type EqualPredicate struct {
	Column Column // Column to check.
	Value  Value  // Value to check equality for.
}

type FalsePredicate struct{}

type FuncPredicate struct {
	Column Column // Column to check.

	// Keep is invoked with the column and value pair to check. Keep is given
	// the Column instance to allow for reusing the same function across
	// multiple columns, if necessary.
	//
	// If Keep returns true, the row is kept.
	Keep func(column Column, value Value) bool
}

type GreaterThanPredicate struct {
	Column Column // Column to check.
	Value  Value  // Value for which rows in Column must be greater than.
}

type InPredicate struct {
	Column Column   // Column to check.
	Values ValueSet // Set of values to check.
}

type Int64Set struct {
	// contains filtered or unexported fields
}

type InvalidTypeError struct {
	Expected datasetmd.PhysicalType
	Actual   datasetmd.PhysicalType
}

type LessThanPredicate struct {
	Column Column // Column to check.
	Value  Value  // Value for which rows in Column must be less than.
}

type MemColumn struct {
	Desc  ColumnDesc // Description of the column.
	Pages []*MemPage // The set of pages in the column.
}

type MemPage struct {
	Desc PageDesc // Description of the page.
	Data PageData // Data for the page.
}

type NotPredicate struct{ Inner Predicate }

type OrPredicate struct{ Left, Right Predicate }

type Page interface {
	// PageDesc returns the metadata for the Page.
	PageDesc() *PageDesc

	// ReadPage returns the [PageData] for the Page.
	ReadPage(ctx context.Context) (PageData, error)
}

type PageData []byte

type PageDesc struct {
	UncompressedSize int    // UncompressedSize is the size of a page before compression.
	CompressedSize   int    // CompressedSize is the size of a page after compression.
	CRC32            uint32 // CRC32 checksum of the page after encoding and compression.
	RowCount         int    // RowCount is the number of rows in the page, including NULLs.
	ValuesCount      int    // ValuesCount is the number of non-NULL values in the page.

	Encoding datasetmd.EncodingType // Encoding used for values in the page.
	Stats    *datasetmd.Statistics  // Optional statistics for the page.
}

type Pages []Page

type Predicate interface {
	// contains filtered or unexported methods
}

type Reader struct {
	// contains filtered or unexported fields
}

type ReaderOptions struct {
	Dataset Dataset // Dataset to read from.

	// Columns to read from the Dataset. It is invalid to provide a Column that
	// is not in Dataset.
	//
	// The set of Columns can include columns not used in Predicate; such columns
	// are considered non-predicate columns.
	Columns []Column

	// Predicates filter the data returned by a Reader. Predicates are optional; if
	// nil, all rows from Columns are returned.
	//
	// Expressions in Predicate may only reference columns in Columns.
	// Holds a list of predicates that can be sequentially applied to the dataset.
	Predicates []Predicate

	// Prefetch enables bulk retrieving pages from the dataset when reading
	// starts. To reduce read latency, this option should only be disabled when
	// the entire Dataset is already held in memory.
	Prefetch bool
}

type ReaderStats struct {

	// Column statistics
	PrimaryColumns   uint64 // Number of primary columns to read from the dataset
	SecondaryColumns uint64 // Number of secondary columns to read from the dataset

	// Page statistics
	PrimaryColumnPages   uint64        // Total pages in primary columns
	SecondaryColumnPages uint64        // Total pages in secondary columns
	DownloadStats        DownloadStats // Download statistics for primary and secondary columns

	ReadCalls int64 // Total number of read calls made to the reader

	// Row statistics
	MaxRows                uint64 // Maximum number of rows across all columns
	RowsToReadAfterPruning uint64 // Total number of primary rows to read after page pruning

	PrimaryRowsRead   uint64 // Actual number of primary rows read.
	SecondaryRowsRead uint64 // Actual number of secondary rows read.

	PrimaryRowBytes   uint64 // Total bytes read for primary rows
	SecondaryRowBytes uint64 // Total bytes read for secondary rows
	// contains filtered or unexported fields
}

type Row struct {
	Index  int     // Index of the row in the dataset.
	Values []Value // Values for the row, one per [Column].
}

type StatisticsOptions struct {
	// StoreRangeStats indicates whether to store value range statistics for the
	// column and pages.
	StoreRangeStats bool

	// StoreCardinalityStats indicates whether to store cardinality estimations,
	// facilitated by hyperloglog
	StoreCardinalityStats bool
}

type TruePredicate struct{}

type Uint64ValueSet struct {
	// contains filtered or unexported fields
}

type UnsupportedTypeError struct {
	Got datasetmd.PhysicalType
}

type Value struct {
	// contains filtered or unexported fields
}

type ValueSet interface {
	Contains(value Value) bool
	Iter() iter.Seq[Value]
	Size() int
}