core

package
v0.0.0-...-edd961b Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 3, 2026 License: MIT Imports: 37 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// AsOfBackward is an alias for AsofBackward.
	AsOfBackward = AsofBackward
	// AsOfForward is an alias for AsofForward.
	AsOfForward = AsofForward
	// AsOfNearest is an alias for AsofNearest.
	AsOfNearest = AsofNearest
)
View Source 
const DefaultMemoryLimit int64 = 1 << 30

DefaultMemoryLimit is the default per-engine memory quota in bytes.

Variables

View Source 
var (
	// ErrClosed reports that a resource is closed.
	ErrClosed = errors.New("golars: resource is closed")
	// ErrColumnNotFound reports that a column is missing.
	ErrColumnNotFound = errors.New("golars: column not found")
	// ErrTypeMismatch reports an incompatible type operation.
	ErrTypeMismatch = errors.New("golars: type mismatch")
	// ErrInvalidPlan reports an invalid lazy or eager operation.
	ErrInvalidPlan = errors.New("golars: invalid lazy plan")
)
View Source 
var (
	// Null is the Arrow null type.
	Null = DataType{arrow.Null}
	// Boolean is the Arrow boolean type.
	Boolean = DataType{arrow.FixedWidthTypes.Boolean}
	// Int8 is the signed 8-bit integer type.
	Int8 = DataType{arrow.PrimitiveTypes.Int8}
	// Int16 is the signed 16-bit integer type.
	Int16 = DataType{arrow.PrimitiveTypes.Int16}
	// Int32 is the signed 32-bit integer type.
	Int32 = DataType{arrow.PrimitiveTypes.Int32}
	// Int64 is the signed 64-bit integer type.
	Int64 = DataType{arrow.PrimitiveTypes.Int64}
	// Uint8 is the unsigned 8-bit integer type.
	Uint8 = DataType{arrow.PrimitiveTypes.Uint8}
	// Uint16 is the unsigned 16-bit integer type.
	Uint16 = DataType{arrow.PrimitiveTypes.Uint16}
	// Uint32 is the unsigned 32-bit integer type.
	Uint32 = DataType{arrow.PrimitiveTypes.Uint32}
	// Uint64 is the unsigned 64-bit integer type.
	Uint64 = DataType{arrow.PrimitiveTypes.Uint64}
	// Float16 is the 16-bit floating-point type.
	Float16 = DataType{arrow.FixedWidthTypes.Float16}
	// Float32 is the 32-bit floating-point type.
	Float32 = DataType{arrow.PrimitiveTypes.Float32}
	// Float64 is the 64-bit floating-point type.
	Float64 = DataType{arrow.PrimitiveTypes.Float64}
	// String is the Arrow UTF-8 string type.
	String = DataType{arrow.BinaryTypes.String}
	// LargeString is the 64-bit-offset UTF-8 string type.
	LargeString = DataType{arrow.BinaryTypes.LargeString}
	// Binary is the variable-width binary type.
	Binary = DataType{arrow.BinaryTypes.Binary}
	// LargeBinary is the 64-bit-offset binary type.
	LargeBinary = DataType{arrow.BinaryTypes.LargeBinary}
	// StringView is the Arrow string-view type.
	StringView = DataType{arrow.BinaryTypes.StringView}
	// BinaryView is the Arrow binary-view type.
	BinaryView = DataType{arrow.BinaryTypes.BinaryView}
	// Date32 is a date stored as days since the epoch.
	Date32 = DataType{arrow.FixedWidthTypes.Date32}
	// Date64 is a date stored as milliseconds since the epoch.
	Date64 = DataType{arrow.FixedWidthTypes.Date64}
	// UInt8 is an alias for Uint8.
	UInt8 = DataType{arrow.PrimitiveTypes.Uint8}
	// UInt16 is an alias for Uint16.
	UInt16 = DataType{arrow.PrimitiveTypes.Uint16}
	// UInt32 is an alias for Uint32.
	UInt32 = DataType{arrow.PrimitiveTypes.Uint32}
	// UInt64 is an alias for Uint64.
	UInt64 = DataType{arrow.PrimitiveTypes.Uint64}
	// Utf8 is an alias for String.
	Utf8 = DataType{arrow.BinaryTypes.String}
	// Date is an alias for Date32.
	Date = DataType{arrow.FixedWidthTypes.Date32}
	// Categorical is an unordered dictionary type.
	Categorical = DataType{&arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Uint32, ValueType: arrow.BinaryTypes.String}}
	// Enum is an ordered dictionary type.
	Enum = DataType{&arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Uint32, ValueType: arrow.BinaryTypes.String, Ordered: true}}
	// IntervalMonth is a month interval type.
	IntervalMonth = DataType{arrow.FixedWidthTypes.MonthInterval}
	// IntervalDayTime is a day-time interval type.
	IntervalDayTime = DataType{arrow.FixedWidthTypes.DayTimeInterval}
	// IntervalMonthDayNano is a month-day-nanosecond interval type.
	IntervalMonthDayNano = DataType{arrow.FixedWidthTypes.MonthDayNanoInterval}
)

Functions

func RegisterObjectStore 

func RegisterObjectStore(scheme string, store ObjectStore) error

RegisterObjectStore adds a store to the package default engine. This is an integration hook, not a required session lifecycle.

func WriteArrow 

func WriteArrow(ctx context.Context, df *DataFrame, destination string, options ...WriteOption) error

WriteArrow writes a DataFrame as an Arrow IPC file.

func WriteCSV 

func WriteCSV(ctx context.Context, df *DataFrame, destination string, options ...WriteOption) error

WriteCSV writes a DataFrame as CSV.

func WriteIPCFile 

func WriteIPCFile(ctx context.Context, df *DataFrame, destination string, options ...WriteOption) error

WriteIPCFile writes a DataFrame as an Arrow IPC file.

func WriteIPCStream 

func WriteIPCStream(ctx context.Context, df *DataFrame, destination string, options ...WriteOption) error

WriteIPCStream writes a DataFrame as an Arrow IPC stream.

func WriteJSON 

func WriteJSON(ctx context.Context, df *DataFrame, destination string, options ...WriteOption) error

WriteJSON writes a DataFrame as a JSON array.

func WriteNDJSON 

func WriteNDJSON(ctx context.Context, df *DataFrame, destination string, options ...WriteOption) error

WriteNDJSON writes a DataFrame as newline-delimited JSON.

func WriteParquet 

func WriteParquet(ctx context.Context, df *DataFrame, destination string, options ...WriteOption) error

WriteParquet writes a DataFrame as Parquet.

func WriteParquetDataset 

func WriteParquetDataset(ctx context.Context, df *DataFrame, destination string, options ...DatasetWriteOption) error

WriteParquetDataset writes a Hive-style local Parquet dataset. Create mode is non-destructive and writes _SUCCESS plus a small schema manifest; Append adds new part files without replacing existing objects.

Types

type AsOfJoinOption 

type AsOfJoinOption func(*asofJoinConfig)

AsOfJoinOption configures an as-of join.

func WithAsOfAllowExact 

func WithAsOfAllowExact(enabled bool) AsOfJoinOption

WithAsOfAllowExact is an alias for WithAsofAllowExact.

func WithAsOfCheckSortedness 

func WithAsOfCheckSortedness(enabled bool) AsOfJoinOption

WithAsOfCheckSortedness is an alias for WithAsofCheckSortedness.

func WithAsOfStrategy 

func WithAsOfStrategy(strategy AsOfStrategy) AsOfJoinOption

WithAsOfStrategy is an alias for WithAsofStrategy.

func WithAsofAllowExact 

func WithAsofAllowExact(enabled bool) AsOfJoinOption

WithAsofAllowExact controls whether equal keys may match.

func WithAsofCheckSortedness 

func WithAsofCheckSortedness(enabled bool) AsOfJoinOption

WithAsofCheckSortedness asks the native engine to validate key ordering.

func WithAsofStrategy 

func WithAsofStrategy(strategy AsofStrategy) AsOfJoinOption

WithAsofStrategy selects backward, forward, or nearest matching.

type AsOfStrategy 

type AsOfStrategy = AsofStrategy

AsOfStrategy is the conventional initialism spelling of AsofStrategy.

type AsofStrategy 

type AsofStrategy uint8

AsofStrategy selects the nearest-key direction for an as-of join. 

const (
	// AsofBackward matches the last right key less than or equal to the left key.
	AsofBackward AsofStrategy = iota
	// AsofForward matches the first right key greater than or equal to the left key.
	AsofForward
	// AsofNearest matches the closest right key.
	AsofNearest
)

func (AsofStrategy) String 

func (strategy AsofStrategy) String() string

type ColumnNotFoundError 

type ColumnNotFoundError struct {
	// Name is the missing column name.
	Name string
	// Available lists the columns present in the frame.
	Available []string
}

ColumnNotFoundError identifies the missing column and the available schema.

func (*ColumnNotFoundError) Error 

func (e *ColumnNotFoundError) Error() string

Error returns a description of the missing column.

func (*ColumnNotFoundError) Unwrap 

func (e *ColumnNotFoundError) Unwrap() error

Unwrap returns ErrColumnNotFound for errors.Is checks.

type DataFrame 

type DataFrame struct {
	// contains filtered or unexported fields
}

DataFrame is an eager, immutable Arrow-backed table. Operations return new frames and do not mutate their receiver. Close releases its Arrow reference deterministically; the finalizer is only a safety net.

func DataFrameFromArrow 

func DataFrameFromArrow(table arrow.Table) (*DataFrame, error)

DataFrameFromArrow wraps an Arrow table as a DataFrame.

func DataFrameFromMaps 

func DataFrameFromMaps(rows []map[string]any) (*DataFrame, error)

DataFrameFromMaps creates an eager frame from row-oriented maps. Columns are sorted by name, and a missing key is represented as null.

func DataFrameFromRecordBatch 

func DataFrameFromRecordBatch(batch arrow.RecordBatch) (*DataFrame, error)

DataFrameFromRecordBatch creates a DataFrame from one record batch.

func DataFrameFromRecordBatches 

func DataFrameFromRecordBatches(batches []arrow.RecordBatch) (*DataFrame, error)

DataFrameFromRecordBatches creates a DataFrame from schema-compatible batches.

func DataFrameFromStructs 

func DataFrameFromStructs[T any](rows []T) (*DataFrame, error)

DataFrameFromStructs creates an eager frame from exported fields in rows. Field names use the json tag when present, falling back to the Go field name. A json:"-" field is omitted.

func NewDataFrame 

func NewDataFrame(columns ...*Series) (*DataFrame, error)

NewDataFrame creates an eager frame from columns.

func NewDataFrameFromArrow 

func NewDataFrameFromArrow(table arrow.Table) (*DataFrame, error)

NewDataFrameFromArrow creates a DataFrame from an Arrow table.

func NewDataFrameFromMaps 

func NewDataFrameFromMaps(rows []map[string]any) (*DataFrame, error)

NewDataFrameFromMaps is an alias for DataFrameFromMaps.

func NewDataFrameFromRecordBatch 

func NewDataFrameFromRecordBatch(batch arrow.RecordBatch) (*DataFrame, error)

NewDataFrameFromRecordBatch creates a DataFrame from one record batch.

func NewDataFrameFromRecordBatches 

func NewDataFrameFromRecordBatches(batches []arrow.RecordBatch) (*DataFrame, error)

NewDataFrameFromRecordBatches creates a DataFrame from schema-compatible batches.

func NewDataFrameFromStructs 

func NewDataFrameFromStructs[T any](rows []T) (*DataFrame, error)

NewDataFrameFromStructs is an alias for DataFrameFromStructs.

func ReadArrow 

func ReadArrow(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadArrow reads an Arrow IPC file into an eager DataFrame.

func ReadCSV 

func ReadCSV(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadCSV reads a CSV path into an eager DataFrame.

func ReadCSVReader 

func ReadCSVReader(ctx context.Context, reader io.Reader, options ...ScanOption) (*DataFrame, error)

ReadCSVReader reads CSV data from a reader into an eager DataFrame.

func ReadIPCFile 

func ReadIPCFile(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadIPCFile reads an Arrow IPC file into an eager DataFrame.

func ReadIPCStreamReader 

func ReadIPCStreamReader(ctx context.Context, reader io.Reader, options ...ScanOption) (*DataFrame, error)

ReadIPCStreamReader reads an Arrow IPC stream from a reader into a DataFrame.

func ReadJSON 

func ReadJSON(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadJSON reads a JSON array into an eager DataFrame.

func ReadJSONReader 

func ReadJSONReader(ctx context.Context, reader io.Reader, options ...ScanOption) (*DataFrame, error)

ReadJSONReader reads a JSON array from a reader into a DataFrame.

func ReadNDJSON 

func ReadNDJSON(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadNDJSON reads newline-delimited JSON into an eager DataFrame.

func ReadNDJSONReader 

func ReadNDJSONReader(ctx context.Context, reader io.Reader, options ...ScanOption) (*DataFrame, error)

ReadNDJSONReader reads newline-delimited JSON from a reader into a DataFrame.

func ReadParquet 

func ReadParquet(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadParquet reads a Parquet path into an eager DataFrame.

func (*DataFrame) ArrowTable 

func (df *DataFrame) ArrowTable() (arrow.Table, error)

ArrowTable returns a retained Arrow table. The caller must Release it.

func (*DataFrame) Clear 

func (df *DataFrame) Clear() (*DataFrame, error)

Clear returns an empty frame with the same schema.

func (*DataFrame) Clone 

func (df *DataFrame) Clone() (*DataFrame, error)

Clone returns an independent frame retaining the same Arrow data.

func (*DataFrame) Close 

func (df *DataFrame) Close()

Close releases the frame's Arrow and native resources.

func (*DataFrame) Column 

func (df *DataFrame) Column(name string) (*Series, error)

Column returns a named column.

func (*DataFrame) Columns 

func (df *DataFrame) Columns() []string

Columns returns the frame's column names in order.

func (*DataFrame) Drop 

func (df *DataFrame) Drop(names ...string) (*DataFrame, error)

Drop removes named columns.

func (*DataFrame) DropNaNs 

func (df *DataFrame) DropNaNs(names ...string) (*DataFrame, error)

DropNaNs removes rows containing NaN values.

func (*DataFrame) DropNulls 

func (df *DataFrame) DropNulls(names ...string) (*DataFrame, error)

DropNulls removes rows with nulls in the named columns.

func (*DataFrame) Explode 

func (df *DataFrame) Explode(columns ...string) (*DataFrame, error)

Explode expands list values into separate rows.

func (*DataFrame) Filter 

func (df *DataFrame) Filter(predicate Expr) (*DataFrame, error)

Filter keeps rows matching predicate.

func (*DataFrame) ForEachRow 

func (df *DataFrame) ForEachRow(callback func(Row) error) error

ForEachRow calls callback once per row in an eager DataFrame.

func (*DataFrame) GroupBy 

func (df *DataFrame) GroupBy(exprs ...Expr) *GroupBy

GroupBy starts an eager grouped aggregation.

func (*DataFrame) Head 

func (df *DataFrame) Head(n int64) (*DataFrame, error)

Head returns the first n rows.

func (*DataFrame) Height 

func (df *DataFrame) Height() int64

Height returns the number of rows.

func (*DataFrame) IsEmpty 

func (df *DataFrame) IsEmpty() bool

IsEmpty reports whether the frame has no rows.

func (*DataFrame) Join 

func (df *DataFrame) Join(other *DataFrame, leftOn, rightOn []Expr, options ...JoinOption) (*DataFrame, error)

Join evaluates a join between two eager frames.

func (*DataFrame) JoinAsOf 

func (df *DataFrame) JoinAsOf(other *DataFrame, leftOn, rightOn []Expr, options ...AsOfJoinOption) (*DataFrame, error)

JoinAsOf evaluates a nearest-key join between two eager frames.

func (*DataFrame) Lazy 

func (df *DataFrame) Lazy() *LazyFrame

Lazy converts the eager frame to a lazy plan.

func (*DataFrame) Limit 

func (df *DataFrame) Limit(n int64) (*DataFrame, error)

Limit returns at most n rows from the start of the frame.

func (*DataFrame) Melt 

func (df *DataFrame) Melt(on, index []string, options ...UnpivotOption) (*DataFrame, error)

Melt is an alias for Unpivot.

func (*DataFrame) Release 

func (df *DataFrame) Release()

Release is an alias for Close for consistency with Arrow's reference-counted objects. Close remains the preferred name in ordinary Go code.

func (*DataFrame) Rename 

func (df *DataFrame) Rename(mapping map[string]string) (*DataFrame, error)

Rename changes column names using mapping.

func (*DataFrame) Reverse 

func (df *DataFrame) Reverse() (*DataFrame, error)

Reverse reverses the row order.

func (*DataFrame) Rows 

func (df *DataFrame) Rows() ([]Row, error)

Rows materializes an eager DataFrame as detached row maps.

func (*DataFrame) Schema 

func (df *DataFrame) Schema() Schema

Schema returns the frame schema.

func (*DataFrame) Select 

func (df *DataFrame) Select(exprs ...Expr) (*DataFrame, error)

Select evaluates a projection and returns a new frame.

func (*DataFrame) SelectSeq 

func (df *DataFrame) SelectSeq(exprs ...Expr) (*DataFrame, error)

SelectSeq evaluates a projection sequentially and returns a new frame.

func (*DataFrame) Shape 

func (df *DataFrame) Shape() (int64, int)

Shape returns row and column counts.

func (*DataFrame) Slice 

func (df *DataFrame) Slice(offset, length int64) (*DataFrame, error)

Slice returns a row slice. Negative offsets count from the end.

func (*DataFrame) Sort 

func (df *DataFrame) Sort(exprs ...Expr) (*DataFrame, error)

Sort orders rows by the supplied expressions.

func (*DataFrame) SortByColumns 

func (df *DataFrame) SortByColumns(names ...string) (*DataFrame, error)

SortByColumns is a convenience for common column-name sorting.

func (*DataFrame) Tail 

func (df *DataFrame) Tail(n int64) (*DataFrame, error)

Tail returns the last n rows.

func (*DataFrame) ToDicts 

func (df *DataFrame) ToDicts() ([]Row, error)

ToDicts is a Polars-compatible alias for Rows. Go callers should prefer Rows.

func (*DataFrame) Unique 

func (df *DataFrame) Unique(columns ...string) (*DataFrame, error)

Unique keeps one row for each distinct key.

func (*DataFrame) Unnest 

func (df *DataFrame) Unnest(columns ...string) (*DataFrame, error)

Unnest expands struct fields into columns.

func (*DataFrame) Unpivot 

func (df *DataFrame) Unpivot(on, index []string, options ...UnpivotOption) (*DataFrame, error)

Unpivot converts selected wide columns into variable/value rows.

func (*DataFrame) Width 

func (df *DataFrame) Width() int

Width returns the number of columns.

func (*DataFrame) WithColumns 

func (df *DataFrame) WithColumns(exprs ...Expr) (*DataFrame, error)

WithColumns adds or replaces columns.

func (*DataFrame) WithColumnsSeq 

func (df *DataFrame) WithColumnsSeq(exprs ...Expr) (*DataFrame, error)

WithColumnsSeq evaluates column expressions sequentially.

func (*DataFrame) WriteArrow 

func (df *DataFrame) WriteArrow(ctx context.Context, destination string, options ...WriteOption) error

WriteArrow writes the frame as an Arrow IPC file.

func (*DataFrame) WriteCSV 

func (df *DataFrame) WriteCSV(ctx context.Context, destination string, options ...WriteOption) error

WriteCSV writes the frame as CSV.

func (*DataFrame) WriteIPCFile 

func (df *DataFrame) WriteIPCFile(ctx context.Context, destination string, options ...WriteOption) error

WriteIPCFile writes the frame as an Arrow IPC file.

func (*DataFrame) WriteJSON 

func (df *DataFrame) WriteJSON(ctx context.Context, destination string, options ...WriteOption) error

WriteJSON writes the frame as a JSON array.

func (*DataFrame) WriteNDJSON 

func (df *DataFrame) WriteNDJSON(ctx context.Context, destination string, options ...WriteOption) error

WriteNDJSON writes the frame as newline-delimited JSON.

func (*DataFrame) WriteParquet 

func (df *DataFrame) WriteParquet(ctx context.Context, destination string, options ...WriteOption) error

WriteParquet writes the frame as Parquet.

type DataType 

type DataType struct {
	// contains filtered or unexported fields
}

DataType is Golars' stable type descriptor. ArrowType enables zero-copy Arrow interop.

func DataTypeFromArrow 

func DataTypeFromArrow(t arrow.DataType) DataType

DataTypeFromArrow wraps an Arrow data type.

func Datetime 

func Datetime(unit arrow.TimeUnit, timezone string) DataType

Datetime creates a timestamp type; it is an alias for Timestamp.

func Decimal32 

func Decimal32(precision, scale int32) DataType

Decimal32 creates a 32-bit decimal type.

func Decimal64 

func Decimal64(precision, scale int32) DataType

Decimal64 creates a 64-bit decimal type.

func Decimal128 

func Decimal128(precision, scale int32) DataType

Decimal128 creates a 128-bit decimal type.

func Decimal256 

func Decimal256(precision, scale int32) DataType

Decimal256 creates a 256-bit decimal type.

func DictionaryOf 

func DictionaryOf(index, value DataType, ordered bool) DataType

DictionaryOf creates a dictionary type.

func Duration 

func Duration(unit arrow.TimeUnit) DataType

Duration creates a duration type with the given unit.

func FixedSizeBinary 

func FixedSizeBinary(width int) DataType

FixedSizeBinary creates a fixed-width binary type.

func FixedSizeListOf 

func FixedSizeListOf(length int32, elem DataType) DataType

FixedSizeListOf creates a fixed-length list type.

func LargeListOf 

func LargeListOf(elem DataType) DataType

LargeListOf creates a 64-bit-offset list type containing elem.

func LargeListViewOf 

func LargeListViewOf(elem DataType) DataType

LargeListViewOf creates a 64-bit-offset list-view type containing elem.

func ListOf 

func ListOf(elem DataType) DataType

ListOf creates a list type containing elem.

func ListViewOf 

func ListViewOf(elem DataType) DataType

ListViewOf creates a list-view type containing elem.

func MapOf 

func MapOf(key, item DataType) DataType

MapOf creates a map type with key and item types.

func RunEndEncodedOf 

func RunEndEncodedOf(runEnds, values DataType) DataType

RunEndEncodedOf creates a run-end encoded type.

func StructOf 

func StructOf(fields ...Field) DataType

StructOf creates a struct type from fields.

func Time 

func Time(unit arrow.TimeUnit) DataType

Time creates a time type using the appropriate Arrow width.

func Time32 

func Time32(unit arrow.TimeUnit) DataType

Time32 creates a 32-bit time type with the given unit.

func Time64 

func Time64(unit arrow.TimeUnit) DataType

Time64 creates a 64-bit time type with the given unit.

func Timestamp 

func Timestamp(unit arrow.TimeUnit, timezone string) DataType

Timestamp creates a timestamp type with the given unit and timezone.

func (DataType) ArrowType 

func (t DataType) ArrowType() arrow.DataType

ArrowType returns the underlying Arrow type.

func (DataType) Elem 

func (t DataType) Elem() (DataType, bool)

Elem returns the element type for a nested type.

func (DataType) Equal 

func (t DataType) Equal(other DataType) bool

Equal reports whether two data types are equal.

func (DataType) Fields 

func (t DataType) Fields() []Field

Fields returns nested struct fields, if any.

func (DataType) ID 

func (t DataType) ID() arrow.Type

ID returns the Arrow type identifier.

func (DataType) IndexType 

func (t DataType) IndexType() (DataType, bool)

IndexType returns a dictionary index type.

func (DataType) IsCategorical 

func (t DataType) IsCategorical() bool

IsCategorical reports whether the type is an unordered dictionary.

func (DataType) IsEnum 

func (t DataType) IsEnum() bool

IsEnum reports whether the type is an ordered dictionary.

func (DataType) IsFloat 

func (t DataType) IsFloat() bool

IsFloat reports whether the type is floating point.

func (DataType) IsInteger 

func (t DataType) IsInteger() bool

IsInteger reports whether the type is an integer.

func (DataType) IsNested 

func (t DataType) IsNested() bool

IsNested reports whether the type contains nested values.

func (DataType) IsNumeric 

func (t DataType) IsNumeric() bool

IsNumeric reports whether the type is numeric.

func (DataType) IsTemporal 

func (t DataType) IsTemporal() bool

IsTemporal reports whether the type represents time.

func (DataType) PrecisionScale 

func (t DataType) PrecisionScale() (precision, scale int32, ok bool)

PrecisionScale returns decimal precision and scale.

func (DataType) String 

func (t DataType) String() string

String returns the Arrow type name.

func (DataType) TimeUnit 

func (t DataType) TimeUnit() (arrow.TimeUnit, bool)

TimeUnit returns the temporal unit.

func (DataType) TimeZone 

func (t DataType) TimeZone() (string, bool)

TimeZone returns a timestamp timezone.

func (DataType) ValueType 

func (t DataType) ValueType() (DataType, bool)

ValueType returns a dictionary or map value type.

type DatasetMode 

type DatasetMode uint8

DatasetMode controls whether a dataset is created or appended to. 

const (
	// DatasetCreate creates a new dataset.
	DatasetCreate DatasetMode = iota
	// DatasetAppend appends files to an existing dataset.
	DatasetAppend
)

type DatasetWriteOption 

type DatasetWriteOption func(*datasetWriteConfig)

DatasetWriteOption configures a dataset sink.

func WithDatasetMode 

func WithDatasetMode(mode DatasetMode) DatasetWriteOption

WithDatasetMode sets the dataset creation mode.

func WithPartitionBy 

func WithPartitionBy(columns ...string) DatasetWriteOption

WithPartitionBy sets dataset partition columns.

func WithRowsPerFile 

func WithRowsPerFile(rows int64) DatasetWriteOption

WithRowsPerFile sets the target rows per dataset file.

type DtExpr 

type DtExpr struct {
	// contains filtered or unexported fields
}

DtExpr provides calendar extraction from Arrow date/timestamp columns.

func (DtExpr) Date 

func (d DtExpr) Date() Expr

Date extracts the date component.

func (DtExpr) Day 

func (d DtExpr) Day() Expr

Day extracts the day component.

func (DtExpr) Hour 

func (d DtExpr) Hour() Expr

Hour extracts the hour component.

func (DtExpr) Minute 

func (d DtExpr) Minute() Expr

Minute extracts the minute component.

func (DtExpr) Month 

func (d DtExpr) Month() Expr

Month extracts the month component.

func (DtExpr) Quarter 

func (d DtExpr) Quarter() Expr

Quarter extracts the quarter number.

func (DtExpr) Second 

func (d DtExpr) Second() Expr

Second extracts the second component.

func (DtExpr) Strftime 

func (d DtExpr) Strftime(format string) Expr

Strftime formats temporal values with a Polars strftime pattern.

func (DtExpr) Time 

func (d DtExpr) Time() Expr

Time extracts the time component.

func (DtExpr) Week 

func (d DtExpr) Week() Expr

Week extracts the ISO week number.

func (DtExpr) Weekday 

func (d DtExpr) Weekday() Expr

Weekday extracts the ISO weekday number.

func (DtExpr) Year 

func (d DtExpr) Year() Expr

Year extracts the year component.

type Engine 

type Engine struct {
	// contains filtered or unexported fields
}

Engine owns execution configuration and memory accounting. The package API uses a lazily-created default Engine; applications only need Engine for isolation or tuning.

func NewEngine 

func NewEngine(options ...EngineOption) (*Engine, error)

NewEngine creates an execution engine with the supplied options.

func (*Engine) BatchSize 

func (e *Engine) BatchSize() int64

BatchSize returns the preferred Arrow batch size.

func (*Engine) Close 

func (e *Engine) Close()

Close releases engine-owned stores and prevents new work.

func (*Engine) MemoryLimit 

func (e *Engine) MemoryLimit() int64

MemoryLimit returns the engine memory quota in bytes.

func (*Engine) MemoryUsed 

func (e *Engine) MemoryUsed() int64

MemoryUsed returns the currently accounted bytes.

func (*Engine) ObjectStore 

func (e *Engine) ObjectStore(scheme string) (ObjectStore, bool)

ObjectStore returns the store registered for scheme.

func (*Engine) Parallelism 

func (e *Engine) Parallelism() int

Parallelism returns the configured parallelism hint.

func (*Engine) ReadArrow 

func (e *Engine) ReadArrow(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadArrow reads an Arrow IPC file into an eager DataFrame on an Engine.

func (*Engine) ReadCSV 

func (e *Engine) ReadCSV(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadCSV reads a CSV path into an eager DataFrame on an Engine.

func (*Engine) ReadIPCFile 

func (e *Engine) ReadIPCFile(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadIPCFile reads an Arrow IPC file into an eager DataFrame on an Engine.

func (*Engine) ReadJSON 

func (e *Engine) ReadJSON(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadJSON reads a JSON array into an eager DataFrame on an Engine.

func (*Engine) ReadNDJSON 

func (e *Engine) ReadNDJSON(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadNDJSON reads newline-delimited JSON into an eager DataFrame on an Engine.

func (*Engine) ReadParquet 

func (e *Engine) ReadParquet(ctx context.Context, path string, options ...ScanOption) (*DataFrame, error)

ReadParquet reads a Parquet path into an eager DataFrame on an Engine.

func (*Engine) RegisterObjectStore 

func (e *Engine) RegisterObjectStore(scheme string, store ObjectStore) error

RegisterObjectStore adds a store to this engine.

func (*Engine) Scan 

func (e *Engine) Scan(path string, options ...ScanOption) *LazyFrame

Scan creates a lazy scan on an Engine whose format is inferred from path.

func (*Engine) ScanArrow 

func (e *Engine) ScanArrow(path string, options ...ScanOption) *LazyFrame

ScanArrow creates a lazy Arrow IPC file scan on an Engine.

func (*Engine) ScanCSV 

func (e *Engine) ScanCSV(path string, options ...ScanOption) *LazyFrame

ScanCSV creates a lazy CSV scan on an Engine.

func (*Engine) ScanIPCFile 

func (e *Engine) ScanIPCFile(path string, options ...ScanOption) *LazyFrame

ScanIPCFile creates a lazy Arrow IPC file scan on an Engine.

func (*Engine) ScanIPCStream 

func (e *Engine) ScanIPCStream(path string, options ...ScanOption) *LazyFrame

ScanIPCStream creates a lazy Arrow IPC stream scan on an Engine.

func (*Engine) ScanJSON 

func (e *Engine) ScanJSON(path string, options ...ScanOption) *LazyFrame

ScanJSON creates a lazy JSON array scan on an Engine.

func (*Engine) ScanNDJSON 

func (e *Engine) ScanNDJSON(path string, options ...ScanOption) *LazyFrame

ScanNDJSON creates a lazy newline-delimited JSON scan on an Engine.

func (*Engine) ScanParquet 

func (e *Engine) ScanParquet(path string, options ...ScanOption) *LazyFrame

ScanParquet creates a lazy Parquet scan on an Engine.

type EngineOption 

type EngineOption func(*engineConfig) error

EngineOption configures an Engine.

func WithBatchSize 

func WithBatchSize(rows int64) EngineOption

WithBatchSize sets the preferred Arrow batch size.

func WithMemoryLimit 

func WithMemoryLimit(bytes int64) EngineOption

WithMemoryLimit sets the engine memory quota in bytes.

func WithObjectStore 

func WithObjectStore(scheme string, store ObjectStore) EngineOption

WithObjectStore registers a store for a URI scheme on an Engine.

func WithParallelism 

func WithParallelism(n int) EngineOption

WithParallelism sets the engine's parallelism hint.

type Expr 

type Expr struct {
	// contains filtered or unexported fields
}

Expr is an immutable expression tree used by both eager and lazy operations.

func All 

func All() Expr

All selects all columns in a projection.

func Col 

func Col(name string) Expr

Col creates a column expression.

func Count 

func Count() Expr

Count creates a row-count aggregation expression.

func DecodeJSON 

func DecodeJSON(expr Expr, target any, options ...JSONDecodeOption) Expr

DecodeJSON parses a JSON string expression into a DataType or a Go struct value. Struct targets use exported fields and standard json tags; pointers mark nullable fields. Invalid rows become null by default.

func DecodeJSONAs 

func DecodeJSONAs[T any](expr Expr, options ...JSONDecodeOption) Expr

DecodeJSONAs parses JSON into the schema derived from T. It is the type-only form of DecodeJSON for callers that do not want to create a zero value.

func Lit 

func Lit(value any) Expr

Lit creates a literal expression.

func (Expr) Abs 

func (e Expr) Abs() Expr

Abs returns absolute values.

func (Expr) Add 

func (e Expr) Add(rhs Expr) Expr

Add adds two expressions.

func (Expr) Alias 

func (e Expr) Alias(name string) Expr

Alias assigns an output name to an expression.

func (Expr) All 

func (e Expr) All() Expr

All reports whether all values in an aggregation are true.

func (Expr) And 

func (e Expr) And(rhs Expr) Expr

And combines boolean expressions with Kleene logic.

func (Expr) Any 

func (e Expr) Any() Expr

Any reports whether any value in an aggregation is true.

func (Expr) ArgMax 

func (e Expr) ArgMax() Expr

ArgMax returns the index of the maximum value in an aggregation.

func (Expr) ArgMin 

func (e Expr) ArgMin() Expr

ArgMin returns the index of the minimum value in an aggregation.

func (Expr) Cast 

func (e Expr) Cast(dtype DataType) Expr

Cast converts an expression to dtype.

func (Expr) Ceil 

func (e Expr) Ceil() Expr

Ceil rounds values up to the nearest integer.

func (Expr) Clip 

func (e Expr) Clip(min, max Expr) Expr

Clip limits values to the inclusive range [min, max].

func (Expr) Cos 

func (e Expr) Cos() Expr

Cos returns cosine values.

func (Expr) Count 

func (e Expr) Count() Expr

Count counts non-null values in an expression.

func (Expr) CumSum 

func (e Expr) CumSum() Expr

CumSum computes a cumulative sum.

func (Expr) DenseRank 

func (e Expr) DenseRank() Expr

DenseRank returns dense rank within each window partition.

func (Expr) Desc 

func (e Expr) Desc() Expr

Desc sorts an expression in descending order.

func (Expr) Div 

func (e Expr) Div(rhs Expr) Expr

Div divides e by rhs.

func (Expr) DropNaNs 

func (e Expr) DropNaNs() Expr

DropNaNs removes NaN values from an expression result.

func (Expr) DropNulls 

func (e Expr) DropNulls() Expr

DropNulls removes null values from an expression result.

func (Expr) Dt 

func (e Expr) Dt() DtExpr

Dt returns date and timestamp operations for an expression.

func (Expr) Eq 

func (e Expr) Eq(rhs Expr) Expr

Eq compares two expressions for equality.

func (Expr) Exp 

func (e Expr) Exp() Expr

Exp returns e raised to the natural exponential.

func (Expr) Explode 

func (e Expr) Explode() Expr

Explode expands list or string values into rows.

func (Expr) FillNaN 

func (e Expr) FillNaN(value Expr) Expr

FillNaN replaces NaN values with value.

func (Expr) FillNull 

func (e Expr) FillNull(value Expr) Expr

FillNull replaces null values with value.

func (Expr) First 

func (e Expr) First() Expr

First returns the first value in an aggregation.

func (Expr) Floor 

func (e Expr) Floor() Expr

Floor rounds values down to the nearest integer.

func (Expr) FloorDiv 

func (e Expr) FloorDiv(rhs Expr) Expr

FloorDiv divides e by rhs and rounds the result toward negative infinity.

func (Expr) Ge 

func (e Expr) Ge(rhs Expr) Expr

Ge compares whether e is greater than or equal to rhs.

func (Expr) Gt 

func (e Expr) Gt(rhs Expr) Expr

Gt compares whether e is greater than rhs.

func (Expr) Implode 

func (e Expr) Implode() Expr

Implode collects an aggregation into a list.

func (Expr) IsBetween 

func (e Expr) IsBetween(lower, upper Expr) Expr

IsBetween tests whether e is between lower and upper.

func (Expr) IsFinite 

func (e Expr) IsFinite() Expr

IsFinite tests for finite numeric values.

func (Expr) IsIn 

func (e Expr) IsIn(values Expr) Expr

IsIn tests membership in values.

func (Expr) IsInfinite 

func (e Expr) IsInfinite() Expr

IsInfinite tests for infinite numeric values.

func (Expr) IsNaN 

func (e Expr) IsNaN() Expr

IsNaN tests for NaN values.

func (Expr) IsNotNaN 

func (e Expr) IsNotNaN() Expr

IsNotNaN tests for values that are not NaN.

func (Expr) IsNotNull 

func (e Expr) IsNotNull() Expr

IsNotNull tests for non-null values.

func (Expr) IsNull 

func (e Expr) IsNull() Expr

IsNull tests for null values.

func (Expr) IsValid 

func (e Expr) IsValid() bool

IsValid reports whether the expression contains a node.

func (Expr) Last 

func (e Expr) Last() Expr

Last returns the last value in an aggregation.

func (Expr) Le 

func (e Expr) Le(rhs Expr) Expr

Le compares whether e is less than or equal to rhs.

func (Expr) Len 

func (e Expr) Len() Expr

Len returns the number of values in an aggregation.

func (Expr) List 

func (e Expr) List() ListExpr

List returns list operations for an expression.

func (Expr) Log 

func (e Expr) Log() Expr

Log returns natural logarithms.

func (Expr) Log10 

func (e Expr) Log10() Expr

Log10 returns base-10 logarithms.

func (Expr) Lt 

func (e Expr) Lt(rhs Expr) Expr

Lt compares whether e is less than rhs.

func (Expr) Max 

func (e Expr) Max() Expr

Max computes a maximum aggregation.

func (Expr) Mean 

func (e Expr) Mean() Expr

Mean computes a mean aggregation.

func (Expr) Median 

func (e Expr) Median() Expr

Median computes the median aggregation.

func (Expr) Min 

func (e Expr) Min() Expr

Min computes a minimum aggregation.

func (Expr) Mod 

func (e Expr) Mod(rhs Expr) Expr

Mod computes the remainder of e divided by rhs.

func (Expr) Mul 

func (e Expr) Mul(rhs Expr) Expr

Mul multiplies two expressions.

func (Expr) NUnique 

func (e Expr) NUnique() Expr

NUnique counts distinct values in an aggregation.

func (Expr) Ne 

func (e Expr) Ne(rhs Expr) Expr

Ne compares two expressions for inequality.

func (Expr) Neg 

func (e Expr) Neg() Expr

Neg negates numeric values.

func (Expr) Not 

func (e Expr) Not() Expr

Not inverts a boolean expression.

func (Expr) Or 

func (e Expr) Or(rhs Expr) Expr

Or combines boolean expressions with Kleene logic.

func (Expr) Over 

func (e Expr) Over(partitionBy ...Expr) Expr

Over evaluates an expression independently for each partition. Aggregate expressions (for example Col("value").Sum().Over(Col("group"))) return one value per input row, matching the shape of the input frame.

func (Expr) Pow 

func (e Expr) Pow(rhs Expr) Expr

Pow raises e to the power rhs.

func (Expr) Product 

func (e Expr) Product() Expr

Product computes the product aggregation.

func (Expr) Rank 

func (e Expr) Rank() Expr

Rank returns the rank within each window partition.

func (Expr) Reverse 

func (e Expr) Reverse() Expr

Reverse reverses each expression result.

func (Expr) RollingMax 

func (e Expr) RollingMax(windowSize int64, options ...RollingOption) Expr

RollingMax computes a trailing fixed-size rolling maximum.

func (Expr) RollingMean 

func (e Expr) RollingMean(windowSize int64, options ...RollingOption) Expr

RollingMean computes a trailing fixed-size rolling mean. The default minimum period count is the window size; use WithRollingMinPeriods to override it.

func (Expr) RollingMin 

func (e Expr) RollingMin(windowSize int64, options ...RollingOption) Expr

RollingMin computes a trailing fixed-size rolling minimum.

func (Expr) RollingStd 

func (e Expr) RollingStd(windowSize int64, options ...RollingOption) Expr

RollingStd computes a trailing fixed-size rolling standard deviation.

func (Expr) RollingSum 

func (e Expr) RollingSum(windowSize int64, options ...RollingOption) Expr

RollingSum computes a trailing fixed-size rolling sum.

func (Expr) RollingVar 

func (e Expr) RollingVar(windowSize int64, options ...RollingOption) Expr

RollingVar computes a trailing fixed-size rolling variance.

func (Expr) Round 

func (e Expr) Round(decimals int) Expr

Round rounds numeric values to decimals places.

func (Expr) RowNumber 

func (e Expr) RowNumber() Expr

RowNumber returns the row number within each window partition.

func (Expr) Shift 

func (e Expr) Shift(periods Expr) Expr

Shift shifts values by periods, inserting nulls.

func (Expr) ShiftAndFill 

func (e Expr) ShiftAndFill(periods, fill Expr) Expr

ShiftAndFill shifts values and fills the introduced nulls.

func (Expr) Sin 

func (e Expr) Sin() Expr

Sin returns sine values.

func (Expr) Slice 

func (e Expr) Slice(offset, length Expr) Expr

Slice selects a contiguous range from each expression result.

func (Expr) Sqrt 

func (e Expr) Sqrt() Expr

Sqrt returns square roots.

func (Expr) Str 

func (e Expr) Str() StrExpr

Str returns string operations for an expression.

func (Expr) String 

func (e Expr) String() string

String formats the expression for display and plan explanations.

func (Expr) Struct 

func (e Expr) Struct() StructExpr

Struct returns struct field operations for an expression.

func (Expr) Sub 

func (e Expr) Sub(rhs Expr) Expr

Sub subtracts rhs from e.

func (Expr) Sum 

func (e Expr) Sum() Expr

Sum computes a sum aggregation.

func (Expr) Tan 

func (e Expr) Tan() Expr

Tan returns tangent values.

func (Expr) Unique 

func (e Expr) Unique() Expr

Unique keeps distinct values in an expression result.

type Field 

type Field struct {
	// Name is the field name.
	Name string
	// Type is the field data type.
	Type DataType
	// Nullable reports whether the field may contain nulls.
	Nullable bool
	// Metadata contains Arrow field metadata.
	Metadata arrow.Metadata
}

Field describes one schema field.

type GCSStore 

type GCSStore = internalstorage.GCSStore

GCSStore adapts Google Cloud Storage to the scan/sink API.

func NewGCSStore 

func NewGCSStore(ctx context.Context, options ...option.ClientOption) (*GCSStore, error)

NewGCSStore creates a Google Cloud Storage-backed ObjectStore.

func NewGCSStoreWithClient 

func NewGCSStoreWithClient(client *storage.Client) (*GCSStore, error)

NewGCSStoreWithClient wraps an existing Cloud Storage client as an ObjectStore.

type GroupBy 

type GroupBy struct {
	// contains filtered or unexported fields
}

GroupBy is an eager grouped aggregation builder.

func (*GroupBy) Agg 

func (g *GroupBy) Agg(exprs ...Expr) (*DataFrame, error)

Agg evaluates grouped aggregate expressions.

func (*GroupBy) Count 

func (g *GroupBy) Count() (*DataFrame, error)

Count counts non-null values for every non-key column per group.

func (*GroupBy) First 

func (g *GroupBy) First() (*DataFrame, error)

First returns the first value of every non-key column per group.

func (*GroupBy) Last 

func (g *GroupBy) Last() (*DataFrame, error)

Last returns the last value of every non-key column per group.

func (*GroupBy) Len 

func (g *GroupBy) Len() (*DataFrame, error)

Len returns one row count per group.

func (*GroupBy) Max 

func (g *GroupBy) Max() (*DataFrame, error)

Max computes the maximum of every non-key column per group.

func (*GroupBy) Mean 

func (g *GroupBy) Mean() (*DataFrame, error)

Mean computes the mean of every non-key column per group.

func (*GroupBy) Median 

func (g *GroupBy) Median() (*DataFrame, error)

Median computes the median of every non-key column per group.

func (*GroupBy) Min 

func (g *GroupBy) Min() (*DataFrame, error)

Min computes the minimum of every non-key column per group.

func (*GroupBy) NUnique 

func (g *GroupBy) NUnique() (*DataFrame, error)

NUnique counts distinct values of every non-key column per group.

func (*GroupBy) Sum 

func (g *GroupBy) Sum() (*DataFrame, error)

Sum sums every non-key column per group.

type JSONDecodeOption 

type JSONDecodeOption func(*jsonDecodeOptions)

JSONDecodeOption configures DecodeJSON.

func WithJSONCoerce 

func WithJSONCoerce(enabled bool) JSONDecodeOption

WithJSONCoerce enables or disables compatible JSON-to-type coercions.

func WithJSONErrorMode 

func WithJSONErrorMode(mode JSONErrorMode) JSONDecodeOption

WithJSONErrorMode sets the error policy for DecodeJSON.

func WithJSONStrictRequiredFields 

func WithJSONStrictRequiredFields(enabled bool) JSONDecodeOption

WithJSONStrictRequiredFields makes non-nullable struct fields required.

type JSONErrorMode 

type JSONErrorMode uint8

JSONErrorMode controls how invalid JSON rows are handled. 

const (
	// JSONNullOnError replaces malformed or incompatible rows with null.
	JSONNullOnError JSONErrorMode = iota
	// JSONErrorOnError returns the first row-level decoding error.
	JSONErrorOnError
)

type JoinOption 

type JoinOption func(*joinConfig)

JoinOption configures a join.

func WithJoinSuffix 

func WithJoinSuffix(suffix string) JoinOption

WithJoinSuffix sets the suffix for overlapping right-side columns.

func WithJoinType 

func WithJoinType(how JoinType) JoinOption

WithJoinType sets the join type.

type JoinType 

type JoinType uint8

JoinType selects the relational join semantics. 

const (
	// JoinInner keeps rows with matching keys from both inputs.
	JoinInner JoinType = iota
	// JoinLeft keeps every row from the left input.
	JoinLeft
	// JoinFull keeps every row from both inputs.
	JoinFull
	// JoinRight keeps every row from the right input.
	JoinRight
	// JoinSemi keeps left rows with a matching right key.
	JoinSemi
	// JoinAnti keeps left rows without a matching right key.
	JoinAnti
	// JoinCross computes the Cartesian product.
	JoinCross
)

func (JoinType) String 

func (t JoinType) String() string

String returns the join type name.

type LazyFrame 

type LazyFrame struct {
	// contains filtered or unexported fields
}

LazyFrame is a deferred logical plan. Constructing transformations does not touch data; errors are returned by terminal methods. Close releases retained Arrow references deterministically; the finalizer is only a safety net.

func Scan 

func Scan(path string, options ...ScanOption) *LazyFrame

Scan creates a lazy scan whose format is inferred from path.

func ScanArrow 

func ScanArrow(path string, options ...ScanOption) *LazyFrame

ScanArrow creates a lazy Arrow IPC file scan.

func ScanCSV 

func ScanCSV(path string, options ...ScanOption) *LazyFrame

ScanCSV creates a lazy CSV scan.

func ScanCSVReader 

func ScanCSVReader(reader io.Reader, options ...ScanOption) *LazyFrame

ScanCSVReader creates a lazy CSV scan from a reader.

func ScanIPCFile 

func ScanIPCFile(path string, options ...ScanOption) *LazyFrame

ScanIPCFile creates a lazy Arrow IPC file scan.

func ScanIPCStream 

func ScanIPCStream(path string, options ...ScanOption) *LazyFrame

ScanIPCStream creates a lazy Arrow IPC stream scan.

func ScanIPCStreamReader 

func ScanIPCStreamReader(reader io.Reader, options ...ScanOption) *LazyFrame

ScanIPCStreamReader creates a lazy Arrow IPC stream scan from a reader.

func ScanJSON 

func ScanJSON(path string, options ...ScanOption) *LazyFrame

ScanJSON creates a lazy JSON array scan.

func ScanJSONReader 

func ScanJSONReader(reader io.Reader, options ...ScanOption) *LazyFrame

ScanJSONReader creates a lazy JSON array scan from a reader.

func ScanNDJSON 

func ScanNDJSON(path string, options ...ScanOption) *LazyFrame

ScanNDJSON creates a lazy newline-delimited JSON scan.

func ScanNDJSONReader 

func ScanNDJSONReader(reader io.Reader, options ...ScanOption) *LazyFrame

ScanNDJSONReader creates a lazy newline-delimited JSON scan from a reader.

func ScanParquet 

func ScanParquet(path string, options ...ScanOption) *LazyFrame

ScanParquet creates a lazy Parquet scan.

func (*LazyFrame) Clear 

func (lf *LazyFrame) Clear() *LazyFrame

Clear produces an empty lazy frame with the current schema.

func (*LazyFrame) Close 

func (lf *LazyFrame) Close()

Close releases retained plan resources.

func (*LazyFrame) Collect 

func (lf *LazyFrame) Collect(ctx context.Context) (_ *DataFrame, err error)

Collect executes the lazy plan and returns an eager DataFrame.

func (*LazyFrame) CollectBatches 

func (lf *LazyFrame) CollectBatches(ctx context.Context) ([]arrow.RecordBatch, error)

CollectBatches executes the plan and returns retained Arrow record batches.

func (*LazyFrame) DropNaNs 

func (lf *LazyFrame) DropNaNs(columns ...string) *LazyFrame

DropNaNs removes rows containing NaN values. If columns are provided, only those columns are checked.

func (*LazyFrame) Explain 

func (lf *LazyFrame) Explain() string

Explain returns the optimized-facing plan description.

func (*LazyFrame) ExplainOptimized 

func (lf *LazyFrame) ExplainOptimized() string

ExplainOptimized returns the Go-facing plan sent to the native optimizer.

func (*LazyFrame) ExplainUnoptimized 

func (lf *LazyFrame) ExplainUnoptimized() string

ExplainUnoptimized returns the Go-facing plan before native collection.

func (*LazyFrame) Explode 

func (lf *LazyFrame) Explode(columns ...string) *LazyFrame

Explode expands list values into separate rows.

func (*LazyFrame) Filter 

func (lf *LazyFrame) Filter(predicate Expr) *LazyFrame

Filter adds a row predicate to the lazy plan.

func (*LazyFrame) ForEachBatch 

func (lf *LazyFrame) ForEachBatch(ctx context.Context, callback func(arrow.RecordBatch) error) (err error)

ForEachBatch executes the plan in Polars and iterates its Arrow result in batches. The callback receives a borrowed record batch that remains valid for the duration of the callback; retain it if it must outlive the callback.

func (*LazyFrame) ForEachRow 

func (lf *LazyFrame) ForEachRow(ctx context.Context, callback func(Row) error) error

ForEachRow calls callback once per row without materializing the complete result. Rows passed to callback are detached from the borrowed Arrow batch.

func (*LazyFrame) GroupBy 

func (lf *LazyFrame) GroupBy(keys ...Expr) *LazyGroupBy

GroupBy starts a lazy grouped aggregation.

func (*LazyFrame) Head 

func (lf *LazyFrame) Head(n int64) *LazyFrame

Head adds a limit for the first n rows.

func (*LazyFrame) Height 

func (lf *LazyFrame) Height(ctx context.Context) (int64, error)

Height executes a lazy plan as a stream and counts its rows without retaining the result. For an eager DataFrame, use Height directly.

func (*LazyFrame) Join 

func (lf *LazyFrame) Join(other *LazyFrame, leftOn, rightOn []Expr, options ...JoinOption) *LazyFrame

Join adds a deferred relational join. Keys are evaluated as expressions on both inputs, so computed keys and temporal/numeric coercion use the same expression engine as Select and Filter.

func (*LazyFrame) JoinAsOf 

func (lf *LazyFrame) JoinAsOf(other *LazyFrame, leftOn, rightOn []Expr, options ...AsOfJoinOption) *LazyFrame

JoinAsOf adds a deferred nearest-key join. Both inputs must be sorted by their single join expression unless sortedness checks are disabled.

func (*LazyFrame) Limit 

func (lf *LazyFrame) Limit(n int64) *LazyFrame

Limit adds a row limit to the lazy plan.

func (*LazyFrame) Melt 

func (lf *LazyFrame) Melt(on, index []string, options ...UnpivotOption) *LazyFrame

Melt is an alias for Unpivot.

func (*LazyFrame) Release 

func (lf *LazyFrame) Release()

Release is an alias for Close for consistency with Arrow's reference-counted objects. Close remains the preferred name in ordinary Go code.

func (*LazyFrame) Reverse 

func (lf *LazyFrame) Reverse() *LazyFrame

Reverse reverses the row order of the lazy frame.

func (*LazyFrame) Rows 

func (lf *LazyFrame) Rows(ctx context.Context) ([]Row, error)

Rows executes a lazy plan and materializes its result as detached row maps.

func (*LazyFrame) Schema 

func (lf *LazyFrame) Schema() (Schema, error)

Schema executes the plan and returns its result schema.

func (*LazyFrame) Select 

func (lf *LazyFrame) Select(exprs ...Expr) *LazyFrame

Select adds a projection to the lazy plan.

func (*LazyFrame) SelectSeq 

func (lf *LazyFrame) SelectSeq(exprs ...Expr) *LazyFrame

SelectSeq adds a sequential projection to the lazy plan.

func (*LazyFrame) Sink 

func (lf *LazyFrame) Sink(ctx context.Context, destination string, options ...WriteOption) error

Sink executes the plan and chooses a writer from destination's extension.

func (*LazyFrame) SinkArrow 

func (lf *LazyFrame) SinkArrow(ctx context.Context, destination string, options ...WriteOption) error

SinkArrow executes the plan and writes an Arrow IPC file.

func (*LazyFrame) SinkCSV 

func (lf *LazyFrame) SinkCSV(ctx context.Context, destination string, options ...WriteOption) error

SinkCSV executes the plan and writes CSV.

func (*LazyFrame) SinkIPCFile 

func (lf *LazyFrame) SinkIPCFile(ctx context.Context, destination string, options ...WriteOption) error

SinkIPCFile executes the plan and writes an Arrow IPC file.

func (*LazyFrame) SinkIPCStream 

func (lf *LazyFrame) SinkIPCStream(ctx context.Context, destination string, options ...WriteOption) error

SinkIPCStream executes the plan and writes an Arrow IPC stream.

func (*LazyFrame) SinkJSON 

func (lf *LazyFrame) SinkJSON(ctx context.Context, destination string, options ...WriteOption) error

SinkJSON executes the plan and writes a JSON array.

func (*LazyFrame) SinkNDJSON 

func (lf *LazyFrame) SinkNDJSON(ctx context.Context, destination string, options ...WriteOption) error

SinkNDJSON executes the plan and writes newline-delimited JSON.

func (*LazyFrame) SinkParquet 

func (lf *LazyFrame) SinkParquet(ctx context.Context, destination string, options ...WriteOption) error

SinkParquet executes the plan and writes Parquet.

func (*LazyFrame) Slice 

func (lf *LazyFrame) Slice(offset, length int64) *LazyFrame

Slice adds a row slice to the lazy plan. Negative offsets count from the end.

func (*LazyFrame) Sort 

func (lf *LazyFrame) Sort(exprs ...Expr) *LazyFrame

Sort adds ordering expressions to the lazy plan.

func (*LazyFrame) Tail 

func (lf *LazyFrame) Tail(n int64) *LazyFrame

Tail adds a limit for the last n rows.

func (*LazyFrame) ToDicts 

func (lf *LazyFrame) ToDicts(ctx context.Context) ([]Row, error)

ToDicts is a Polars-compatible alias for Rows. Go callers should prefer Rows.

func (*LazyFrame) Unique 

func (lf *LazyFrame) Unique(columns ...string) *LazyFrame

Unique keeps one row for each distinct key.

func (*LazyFrame) Unnest 

func (lf *LazyFrame) Unnest(columns ...string) *LazyFrame

Unnest expands struct fields into columns.

func (*LazyFrame) Unpivot 

func (lf *LazyFrame) Unpivot(on, index []string, options ...UnpivotOption) *LazyFrame

Unpivot converts selected wide columns into variable/value rows. Columns in index remain as identifiers; when on is empty, every non-index column is unpivoted by the native engine.

func (*LazyFrame) WithColumns 

func (lf *LazyFrame) WithColumns(exprs ...Expr) *LazyFrame

WithColumns adds or replaces columns in the lazy plan.

func (*LazyFrame) WithColumnsSeq 

func (lf *LazyFrame) WithColumnsSeq(exprs ...Expr) *LazyFrame

WithColumnsSeq adds or replaces columns sequentially in the lazy plan.

type LazyGroupBy 

type LazyGroupBy struct {
	// contains filtered or unexported fields
}

LazyGroupBy builds a deferred grouped aggregation.

func (*LazyGroupBy) Agg 

func (g *LazyGroupBy) Agg(exprs ...Expr) *LazyFrame

Agg adds grouped aggregate expressions to the lazy plan.

func (*LazyGroupBy) Count 

func (g *LazyGroupBy) Count() *LazyFrame

Count counts non-null values for every non-key column per group.

func (*LazyGroupBy) First 

func (g *LazyGroupBy) First() *LazyFrame

First returns the first value of every non-key column per group.

func (*LazyGroupBy) Last 

func (g *LazyGroupBy) Last() *LazyFrame

Last returns the last value of every non-key column per group.

func (*LazyGroupBy) Len 

func (g *LazyGroupBy) Len() *LazyFrame

Len returns one row count per group.

func (*LazyGroupBy) Max 

func (g *LazyGroupBy) Max() *LazyFrame

Max computes the maximum of every non-key column per group.

func (*LazyGroupBy) Mean 

func (g *LazyGroupBy) Mean() *LazyFrame

Mean computes the mean of every non-key column per group.

func (*LazyGroupBy) Median 

func (g *LazyGroupBy) Median() *LazyFrame

Median computes the median of every non-key column per group.

func (*LazyGroupBy) Min 

func (g *LazyGroupBy) Min() *LazyFrame

Min computes the minimum of every non-key column per group.

func (*LazyGroupBy) NUnique 

func (g *LazyGroupBy) NUnique() *LazyFrame

NUnique counts distinct values of every non-key column per group.

func (*LazyGroupBy) Sum 

func (g *LazyGroupBy) Sum() *LazyFrame

Sum sums every non-key column per group.

type ListExpr 

type ListExpr struct {
	// contains filtered or unexported fields
}

ListExpr provides operations on list-valued expressions.

func (ListExpr) Contains 

func (l ListExpr) Contains(value Expr) Expr

Contains tests whether each list contains value.

func (ListExpr) Explode 

func (l ListExpr) Explode() Expr

Explode expands each list item into a row.

func (ListExpr) First 

func (l ListExpr) First() Expr

First returns the first value in each list.

func (ListExpr) Get 

func (l ListExpr) Get(index Expr) Expr

Get selects an item from each list. Out-of-bounds values become null.

func (ListExpr) Head 

func (l ListExpr) Head(n Expr) Expr

Head returns the first n values from each list.

func (ListExpr) Last 

func (l ListExpr) Last() Expr

Last returns the last value in each list.

func (ListExpr) Len 

func (l ListExpr) Len() Expr

Len returns the length of each list.

func (ListExpr) Max 

func (l ListExpr) Max() Expr

Max returns the maximum value in each list.

func (ListExpr) Mean 

func (l ListExpr) Mean() Expr

Mean returns the mean of each list.

func (ListExpr) Min 

func (l ListExpr) Min() Expr

Min returns the minimum value in each list.

func (ListExpr) NUnique 

func (l ListExpr) NUnique() Expr

NUnique counts distinct values in each list.

func (ListExpr) Reverse 

func (l ListExpr) Reverse() Expr

Reverse reverses each list.

func (ListExpr) Slice 

func (l ListExpr) Slice(offset, length Expr) Expr

Slice selects a range from each list.

func (ListExpr) Sum 

func (l ListExpr) Sum() Expr

Sum returns the sum of each list.

func (ListExpr) Tail 

func (l ListExpr) Tail(n Expr) Expr

Tail returns the last n values from each list.

func (ListExpr) Unique 

func (l ListExpr) Unique() Expr

Unique keeps distinct values in each list.

type MemoryLimitError 

type MemoryLimitError struct {
	// Limit is the configured memory quota in bytes.
	Limit int64
	// Allocated is the accounted usage before the rejected allocation.
	Allocated int64
	// Requested is the rejected allocation size in bytes.
	Requested int64
}

MemoryLimitError reports an allocation rejected by an Engine memory limit.

func (*MemoryLimitError) Error 

func (e *MemoryLimitError) Error() string

Error returns a description of the rejected allocation.

type ObjectInfo 

type ObjectInfo = internalstorage.ObjectInfo

ObjectInfo contains the metadata needed for random-access scans.

type ObjectStore 

type ObjectStore = internalstorage.ObjectStore

ObjectStore is the minimal storage contract used by scans and sinks. The built-in s3:// and gs:// adapters are created lazily; custom stores can be registered by URI scheme without introducing a session lifecycle.

type RangeObjectStore 

type RangeObjectStore = internalstorage.RangeObjectStore

RangeObjectStore is an optional extension used by Parquet scans. Stores that implement it allow metadata and column chunks to be read with range requests.

type RollingOption 

type RollingOption func(*rollingConfig)

RollingOption configures a fixed-size rolling expression.

func WithRollingCenter 

func WithRollingCenter(enabled bool) RollingOption

WithRollingCenter centers the rolling window around each row.

func WithRollingMinPeriods 

func WithRollingMinPeriods(periods int64) RollingOption

WithRollingMinPeriods sets the minimum number of non-null values required to produce a rolling result.

type Row 

type Row = map[string]any

Row is one DataFrame row keyed by column name. Values are Go values suitable for JSON encoding; nulls are represented by nil.

type S3Store 

type S3Store = internalstorage.S3Store

S3Store adapts an AWS S3-compatible object store to the scan/sink API.

func NewS3Store 

func NewS3Store(ctx context.Context, options ...func(*s3.Options)) (*S3Store, error)

NewS3Store creates an S3-backed ObjectStore.

func NewS3StoreWithClient 

func NewS3StoreWithClient(client *s3.Client) (*S3Store, error)

NewS3StoreWithClient wraps an existing S3 client as an ObjectStore.

type ScanOption 

type ScanOption func(*scanConfig)

ScanOption configures a scan.

func WithColumns 

func WithColumns(columns ...string) ScanOption

WithColumns limits a scan to the named columns before executing the plan.

func WithComment 

func WithComment(comment rune) ScanOption

WithComment ignores CSV records beginning with comment.

func WithDelimiter 

func WithDelimiter(delimiter rune) ScanOption

WithDelimiter sets the delimiter for delimited input.

func WithHasHeader 

func WithHasHeader(enabled bool) ScanOption

WithHasHeader controls whether delimited input has a header row.

func WithInferSchemaLength 

func WithInferSchemaLength(rows int64) ScanOption

WithInferSchemaLength sets the number of CSV rows used for type inference. A negative value requests a full-file inference scan.

func WithNRows 

func WithNRows(rows int64) ScanOption

WithNRows limits the number of rows read from the source. A negative value disables the limit.

func WithNullValues 

func WithNullValues(values ...string) ScanOption

WithNullValues treats the supplied CSV values as null.

func WithQuoteChar 

func WithQuoteChar(quote rune) ScanOption

WithQuoteChar sets the CSV quote character. Passing zero disables quoting.

func WithScanBatchSize 

func WithScanBatchSize(rows int64) ScanOption

WithScanBatchSize sets the preferred scan batch size.

func WithSkipRows 

func WithSkipRows(rows int64) ScanOption

WithSkipRows skips rows before the scan's header/data rows.

type Schema 

type Schema struct {
	// contains filtered or unexported fields
}

Schema describes a table's fields and metadata.

func NewSchema 

func NewSchema(fields ...Field) Schema

NewSchema creates a schema from fields.

func NewSchemaWithMetadata 

func NewSchemaWithMetadata(fields []Field, metadata arrow.Metadata) Schema

NewSchemaWithMetadata creates a schema with fields and metadata.

func SchemaFromArrow 

func SchemaFromArrow(schema *arrow.Schema) Schema

SchemaFromArrow wraps an Arrow schema.

func (Schema) ArrowSchema 

func (s Schema) ArrowSchema() *arrow.Schema

ArrowSchema returns the underlying Arrow schema.

func (Schema) Field 

func (s Schema) Field(name string) (Field, bool)

Field returns a named field.

func (Schema) Fields 

func (s Schema) Fields() []Field

Fields returns the schema fields.

func (Schema) Metadata 

func (s Schema) Metadata() arrow.Metadata

Metadata returns schema metadata.

func (Schema) Names 

func (s Schema) Names() []string

Names returns the schema field names.

func (Schema) String 

func (s Schema) String() string

String returns the Arrow schema representation.

type Series 

type Series struct {
	// contains filtered or unexported fields
}

Series is an immutable named Arrow chunked array. Close releases its Arrow reference deterministically; the finalizer is only a safety net.

func NewSeries 

func NewSeries(name string, values any) (*Series, error)

NewSeries creates a named series from Go or Arrow-compatible values.

func NewSeriesFromArrow 

func NewSeriesFromArrow(name string, values arrow.Array) (*Series, error)

NewSeriesFromArrow creates a series from an Arrow array.

func SeriesFromArrow 

func SeriesFromArrow(name string, values arrow.Array) *Series

SeriesFromArrow wraps an Arrow array as a series.

func SeriesFromArrowChunked 

func SeriesFromArrowChunked(name string, values *arrow.Chunked) *Series

SeriesFromArrowChunked wraps an Arrow chunked array as a series.

func (*Series) ArrowChunked 

func (s *Series) ArrowChunked() *arrow.Chunked

ArrowChunked returns a retained Arrow chunked array.

func (*Series) Clone 

func (s *Series) Clone() (*Series, error)

Clone returns an independent series retaining the same Arrow data.

func (*Series) Close 

func (s *Series) Close()

Close releases the series' Arrow resources.

func (*Series) DataType 

func (s *Series) DataType() DataType

DataType returns the series data type.

func (*Series) IsEmpty 

func (s *Series) IsEmpty() bool

IsEmpty reports whether the series has no values.

func (*Series) Len 

func (s *Series) Len() int64

Len returns the number of values.

func (*Series) NChunks 

func (s *Series) NChunks() int

NChunks returns the number of Arrow chunks backing the series.

func (*Series) Name 

func (s *Series) Name() string

Name returns the series name.

func (*Series) NullCount 

func (s *Series) NullCount() int64

NullCount returns the number of null values.

func (*Series) Release 

func (s *Series) Release()

Release is an alias for Close for consistency with Arrow's reference-counted objects. Close remains the preferred name in ordinary Go code.

func (*Series) Slice 

func (s *Series) Slice(offset, length int64) (*Series, error)

Slice returns a zero-copy row slice. Negative offsets count from the end.

func (*Series) ToSlice 

func (s *Series) ToSlice() ([]any, error)

ToSlice is an alias for Values.

func (*Series) Values 

func (s *Series) Values() ([]any, error)

Values returns detached Go values in row order. Nested values are copied so the returned slice remains valid after the series is closed.

type StrExpr 

type StrExpr struct {
	// contains filtered or unexported fields
}

StrExpr provides string kernels while keeping the main Expr namespace small.

func (StrExpr) Contains 

func (s StrExpr) Contains(pattern Expr) Expr

Contains tests whether a string contains pattern.

func (StrExpr) EndsWith 

func (s StrExpr) EndsWith(pattern Expr) Expr

EndsWith tests whether a string ends with pattern.

func (StrExpr) LenBytes 

func (s StrExpr) LenBytes() Expr

LenBytes returns the byte length of each string.

func (StrExpr) Lengths 

func (s StrExpr) Lengths() Expr

Lengths returns string lengths.

func (StrExpr) Replace 

func (s StrExpr) Replace(pattern, value Expr) Expr

Replace replaces the first matching substring.

func (StrExpr) ReplaceAll 

func (s StrExpr) ReplaceAll(pattern, value Expr) Expr

ReplaceAll replaces all matching substrings.

func (StrExpr) Slice 

func (s StrExpr) Slice(offset, length Expr) Expr

Slice extracts a substring using byte offsets represented by expressions.

func (StrExpr) Split 

func (s StrExpr) Split(delimiter Expr) Expr

Split splits each string by a delimiter.

func (StrExpr) StartsWith 

func (s StrExpr) StartsWith(pattern Expr) Expr

StartsWith tests whether a string starts with pattern.

func (StrExpr) StripChars 

func (s StrExpr) StripChars(characters Expr) Expr

StripChars removes leading and trailing characters. An empty expression uses the default Unicode whitespace set.

func (StrExpr) ToLowerCase 

func (s StrExpr) ToLowerCase() Expr

ToLowerCase converts strings to lower case.

func (StrExpr) ToUpperCase 

func (s StrExpr) ToUpperCase() Expr

ToUpperCase converts strings to upper case.

type StructExpr 

type StructExpr struct {
	// contains filtered or unexported fields
}

StructExpr provides access to fields in struct-valued expressions.

func (StructExpr) Element 

func (s StructExpr) Element(name string) Expr

Element selects a named field from a struct-valued expression.

func (StructExpr) JSONEncode 

func (s StructExpr) JSONEncode() Expr

JSONEncode serializes struct values as JSON strings.

func (StructExpr) Unnest 

func (s StructExpr) Unnest() Expr

Unnest expands struct fields into separate columns.

type UnpivotOption 

type UnpivotOption func(*unpivotConfig)

UnpivotOption configures the names of generated columns.

func WithUnpivotValueName 

func WithUnpivotValueName(name string) UnpivotOption

WithUnpivotValueName sets the generated value column name.

func WithUnpivotVariableName 

func WithUnpivotVariableName(name string) UnpivotOption

WithUnpivotVariableName sets the generated variable column name.

type WhenThen 

type WhenThen struct {
	// contains filtered or unexported fields
}

WhenThen builds a conditional expression.

func When 

func When(predicate Expr) WhenThen

When starts a conditional expression.

func (WhenThen) Otherwise 

func (w WhenThen) Otherwise(value Expr) Expr

Otherwise supplies the value for a false predicate.

func (WhenThen) Then 

func (w WhenThen) Then(value Expr) WhenThen

Then supplies the value for a true predicate.

type WriteOption 

type WriteOption func(*writeConfig)

WriteOption configures a file sink.

func WithOverwrite 

func WithOverwrite(enabled bool) WriteOption

WithOverwrite controls whether an existing output is replaced.

func WithWriteBatchSize 

func WithWriteBatchSize(rows int64) WriteOption

WithWriteBatchSize sets the preferred sink batch size.

func WithWriteDelimiter 

func WithWriteDelimiter(delimiter rune) WriteOption

WithWriteDelimiter sets the delimiter for delimited output.

func WithWriteHeader 

func WithWriteHeader(enabled bool) WriteOption

WithWriteHeader controls whether delimited output includes a header row.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.