arreflect

package
v18.6.0-rc0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 22, 2026 License: Apache-2.0, BSD-3-Clause Imports: 15 Imported by: 0

Documentation

Overview

Package arreflect provides utilities for converting between Apache Arrow arrays and Go structs using reflection.

The primary entry points are the generic functions At, ToSlice, FromSlice, RecordToSlice, and RecordFromSlice, which convert between Arrow arrays/records and Go slices of structs.

Schema inference is available via InferSchema and InferType.

Arrow struct tags control field mapping: 

type MyRow struct {
    Name  string  `arrow:"name"`
    Score float64 `arrow:"score"`
    Skip  string  `arrow:"-"`
    Enc   string  `arrow:"enc,dict"`
    T32   time.Time `arrow:"t32,time32"`
}

Temporal type overrides for time.Time fields: 

arrow:"field,date32"   — use Date32 instead of Timestamp
arrow:"field,date64"   — use Date64 instead of Timestamp
arrow:"field,time32"   — use Time32(ms) instead of Timestamp
arrow:"field,time64"   — use Time64(ns) instead of Timestamp

Additional tag options: 

arrow:"field,view"                  — use STRING_VIEW/BINARY_VIEW for string/bytes fields, or LIST_VIEW for slice fields
arrow:"field,ree"                   — run-end encoding at top-level only (struct fields not supported)
arrow:"field,decimal(precision,scale)" — override decimal precision and scale (e.g., arrow:",decimal(18,2)")

Index

Examples

Constants

This section is empty.

Variables

View Source 
var (
	ErrUnsupportedType = errors.New("arreflect: unsupported type")
	ErrTypeMismatch    = errors.New("arreflect: type mismatch")
)

Functions

func At 

func At[T any](arr arrow.Array, i int) (T, error)
Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	b := array.NewStringBuilder(mem)
	defer b.Release()
	b.Append("alpha")
	b.Append("beta")
	b.Append("gamma")
	arr := b.NewArray()
	defer arr.Release()

	val, err := arreflect.At[string](arr, 1)
	if err != nil {
		panic(err)
	}
	fmt.Println(val)
}

Output:
beta

func AtAny 

func AtAny(arr arrow.Array, i int) (any, error)

AtAny converts a single element at index i of an Arrow array to a Go value, inferring the Go type from the Arrow DataType at runtime via InferGoType. Useful when the column type is not known at compile time. Null elements are returned as the Go zero value of the inferred type; use arr.IsNull(i) to distinguish a null element from a genuine zero. For typed access when T is known, prefer At.

func FromSlice 

func FromSlice[T any](vals []T, mem memory.Allocator, opts ...Option) (arrow.Array, error)
Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	arr, err := arreflect.FromSlice([]int32{10, 20, 30}, mem)
	if err != nil {
		panic(err)
	}
	defer arr.Release()

	fmt.Println("Type:", arr.DataType())
	fmt.Println("Len:", arr.Len())
	for i := 0; i < arr.Len(); i++ {
		fmt.Println(arr.(*array.Int32).Value(i))
	}
}

Output:
Type: int32
Len: 3
10
20
30
Example (LargeStruct) 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	type Event struct {
		Name string `arrow:"name,large"`
		Code int32  `arrow:"code"`
	}

	schema, err := arreflect.InferSchema[Event]()
	if err != nil {
		panic(err)
	}
	fmt.Println("Schema:", schema)

	mem := memory.NewGoAllocator()
	arr, err := arreflect.FromSlice([]Event{{"click", 1}, {"view", 2}}, mem)
	if err != nil {
		panic(err)
	}
	defer arr.Release()

	sa := arr.(*array.Struct)
	fmt.Println("Name type:", sa.Field(0).DataType())
	fmt.Println("Code type:", sa.Field(1).DataType())
}

Output:
Schema: schema:
  fields: 2
    - name: type=large_utf8
    - code: type=int32
Name type: large_utf8
Code type: int32
Example (StructSlice) 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	type Row struct {
		Name  string  `arrow:"name"`
		Score float64 `arrow:"score"`
	}

	arr, err := arreflect.FromSlice([]Row{
		{"alice", 9.5},
		{"bob", 7.0},
	}, mem)
	if err != nil {
		panic(err)
	}
	defer arr.Release()

	sa := arr.(*array.Struct)
	fmt.Println("Type:", sa.DataType())
	fmt.Println("Names:", sa.Field(0))
	fmt.Println("Scores:", sa.Field(1))
}

Output:
Type: struct<name: utf8, score: float64>
Names: ["alice" "bob"]
Scores: [9.5 7]
Example (ViewStruct) 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
)

func main() {
	type Event struct {
		Name string `arrow:"name,view"`
		Code int32  `arrow:"code"`
	}

	schema, err := arreflect.InferSchema[Event]()
	if err != nil {
		panic(err)
	}
	fmt.Println("Schema:", schema)
}

Output:
Schema: schema:
  fields: 2
    - name: type=string_view
    - code: type=int32
Example (WithDecimal) 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/decimal128"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	vals := []decimal128.Num{
		decimal128.FromI64(12345),
		decimal128.FromI64(-67890),
	}
	arr, err := arreflect.FromSlice(vals, mem, arreflect.WithDecimal(10, 2))
	if err != nil {
		panic(err)
	}
	defer arr.Release()

	fmt.Println("Type:", arr.DataType())
	fmt.Println("Len:", arr.Len())
}

Output:
Type: decimal(10, 2)
Len: 2
Example (WithDict) 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	arr, err := arreflect.FromSlice(
		[]string{"red", "green", "red", "blue", "green"},
		mem,
		arreflect.WithDict(),
	)
	if err != nil {
		panic(err)
	}
	defer arr.Release()

	fmt.Println("Type:", arr.DataType())
	dict := arr.(*array.Dictionary)
	fmt.Println("Indices:", dict.Indices())
	fmt.Println("Dictionary:", dict.Dictionary())
}

Output:
Type: dictionary<values=utf8, indices=int32, ordered=false>
Indices: [0 1 0 2 1]
Dictionary: ["red" "green" "blue"]

func InferGoType 

func InferGoType(dt arrow.DataType) (reflect.Type, error)

InferGoType returns the Go reflect.Type corresponding to the given Arrow DataType. For STRUCT types it constructs an anonymous struct type at runtime using reflect.StructOf; field names are exported (capitalised) with the original Arrow field name preserved in an arrow struct tag. Nullable Arrow fields (field.Nullable == true) become pointer types (*T). For DICTIONARY and RUN_END_ENCODED types it returns the Go type of the value/encoded type respectively (dictionaries are resolved transparently).

func InferSchema 

func InferSchema[T any]() (*arrow.Schema, error)

InferSchema infers an *arrow.Schema from a Go struct type T. T must be a struct type; returns an error otherwise. For column-level Arrow type inspection, use InferType. Field names come from arrow struct tags or Go field names. Pointer fields are marked Nullable=true.

Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
)

func main() {
	type Event struct {
		ID      int64   `arrow:"id"`
		Name    string  `arrow:"name"`
		Score   float64 `arrow:"score"`
		Comment *string `arrow:"comment"`
	}

	schema, err := arreflect.InferSchema[Event]()
	if err != nil {
		panic(err)
	}
	fmt.Println(schema)
}

Output:
schema:
  fields: 4
    - id: type=int64
    - name: type=utf8
    - score: type=float64
    - comment: type=utf8, nullable

func InferType 

func InferType[T any]() (arrow.DataType, error)

InferType infers the Arrow DataType for a Go type T. For struct types, InferSchema is preferred when the result will be used with arrow.Record or array.NewRecord; InferType returns an arrow.DataType that would require an additional cast to *arrow.StructType.

func RecordAt 

func RecordAt[T any](rec arrow.RecordBatch, i int) (T, error)

RecordAt converts the row at index i of an Arrow Record to a Go value of type T. T must be a struct type whose fields correspond to the record's columns.

func RecordAtAny 

func RecordAtAny(rec arrow.RecordBatch, i int) (any, error)

RecordAtAny converts the row at index i of an Arrow Record to a Go value, inferring the Go type from the record's schema at runtime via InferGoType. Equivalent to AtAny on the struct array underlying the record.

func RecordFromSlice 

func RecordFromSlice[T any](vals []T, mem memory.Allocator, opts ...Option) (arrow.RecordBatch, error)
Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

type Measurement struct {
	Sensor string  `arrow:"sensor"`
	Value  float64 `arrow:"value"`
}

func main() {
	mem := memory.NewGoAllocator()

	rows := []Measurement{
		{"temp-1", 23.5},
		{"temp-2", 19.8},
	}
	rec, err := arreflect.RecordFromSlice(rows, mem)
	if err != nil {
		panic(err)
	}
	defer rec.Release()

	fmt.Println("Schema:", rec.Schema())
	fmt.Println("Rows:", rec.NumRows())
	fmt.Println("Col 0:", rec.Column(0))
	fmt.Println("Col 1:", rec.Column(1))
}

Output:
Schema: schema:
  fields: 2
    - sensor: type=utf8
    - value: type=float64
Rows: 2
Col 0: ["temp-1" "temp-2"]
Col 1: [23.5 19.8]

func RecordToAnySlice 

func RecordToAnySlice(rec arrow.RecordBatch) ([]any, error)

RecordToAnySlice converts all rows of an Arrow Record to Go values, inferring the Go type at runtime via InferGoType. Equivalent to ToAnySlice on the struct array underlying the record.

func RecordToSlice 

func RecordToSlice[T any](rec arrow.RecordBatch) ([]T, error)
Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

type Measurement struct {
	Sensor string  `arrow:"sensor"`
	Value  float64 `arrow:"value"`
}

func main() {
	mem := memory.NewGoAllocator()

	rows := []Measurement{
		{"temp-1", 23.5},
		{"temp-2", 19.8},
	}
	rec, err := arreflect.RecordFromSlice(rows, mem)
	if err != nil {
		panic(err)
	}
	defer rec.Release()

	got, err := arreflect.RecordToSlice[Measurement](rec)
	if err != nil {
		panic(err)
	}
	for _, m := range got {
		fmt.Printf("%s: %.1f\n", m.Sensor, m.Value)
	}
}

Output:
temp-1: 23.5
temp-2: 19.8

func ToAnySlice 

func ToAnySlice(arr arrow.Array) ([]any, error)

ToAnySlice converts all elements of an Arrow array to Go values, inferring the Go type from the Arrow DataType at runtime via InferGoType. All elements share the same inferred Go type. Null elements are returned as the Go zero value of the inferred type; use arr.IsNull(i) to distinguish a null element from a genuine zero value. For typed access when T is known, prefer ToSlice.

Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow"
	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	st := arrow.StructOf(
		arrow.Field{Name: "city", Type: arrow.BinaryTypes.String},
		arrow.Field{Name: "pop", Type: arrow.PrimitiveTypes.Int64},
	)
	sb := array.NewStructBuilder(mem, st)
	defer sb.Release()

	sb.Append(true)
	sb.FieldBuilder(0).(*array.StringBuilder).Append("Tokyo")
	sb.FieldBuilder(1).(*array.Int64Builder).Append(14000000)

	sb.Append(true)
	sb.FieldBuilder(0).(*array.StringBuilder).Append("Paris")
	sb.FieldBuilder(1).(*array.Int64Builder).Append(2200000)

	arr := sb.NewArray()
	defer arr.Release()

	rows, err := arreflect.ToAnySlice(arr)
	if err != nil {
		panic(err)
	}
	for _, row := range rows {
		fmt.Println(row)
	}
}

Output:
{Tokyo 14000000}
{Paris 2200000}
Example (NullableFields) 
package main

import (
	"fmt"
	"reflect"

	"github.com/apache/arrow-go/v18/arrow"
	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	st := arrow.StructOf(
		arrow.Field{Name: "name", Type: arrow.BinaryTypes.String, Nullable: false},
		arrow.Field{Name: "score", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
	)
	sb := array.NewStructBuilder(mem, st)
	defer sb.Release()

	sb.Append(true)
	sb.FieldBuilder(0).(*array.StringBuilder).Append("alice")
	sb.FieldBuilder(1).(*array.Float64Builder).Append(9.5)

	sb.Append(true)
	sb.FieldBuilder(0).(*array.StringBuilder).Append("bob")
	sb.FieldBuilder(1).(*array.Float64Builder).AppendNull()

	arr := sb.NewArray()
	defer arr.Release()

	rows, err := arreflect.ToAnySlice(arr)
	if err != nil {
		panic(err)
	}
	for _, row := range rows {
		v := reflect.ValueOf(row)
		var name string
		var scoreField reflect.Value
		for i := 0; i < v.NumField(); i++ {
			switch v.Type().Field(i).Tag.Get("arrow") {
			case "name":
				name = v.Field(i).String()
			case "score":
				scoreField = v.Field(i)
			}
		}
		if scoreField.IsNil() {
			fmt.Printf("%s: <null>\n", name)
		} else {
			fmt.Printf("%s: %.1f\n", name, scoreField.Elem().Float())
		}
	}
}

Output:
alice: 9.5
bob: <null>

func ToSlice 

func ToSlice[T any](arr arrow.Array) ([]T, error)
Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	b := array.NewFloat64Builder(mem)
	defer b.Release()
	b.Append(1.1)
	b.Append(2.2)
	b.Append(3.3)
	arr := b.NewArray()
	defer arr.Release()

	vals, err := arreflect.ToSlice[float64](arr)
	if err != nil {
		panic(err)
	}
	fmt.Println(vals)
}

Output:
[1.1 2.2 3.3]

Types

type Option 

type Option func(*tagOpts)

Option configures encoding behavior for FromSlice and RecordFromSlice.

func WithDecimal 

func WithDecimal(precision, scale int32) Option

WithDecimal sets the precision and scale for decimal types.

func WithDict 

func WithDict() Option

WithDict requests dictionary encoding for the top-level array.

func WithLarge 

func WithLarge() Option

WithLarge requests Large type variants (LARGE_STRING, LARGE_BINARY, LARGE_LIST, LARGE_LIST_VIEW) for the top-level array and recursively for nested types.

Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	arr, err := arreflect.FromSlice([]string{"hello", "world"}, mem, arreflect.WithLarge())
	if err != nil {
		panic(err)
	}
	defer arr.Release()

	fmt.Println("Type:", arr.DataType())
	fmt.Println("Len:", arr.Len())
}

Output:
Type: large_utf8
Len: 2

func WithREE 

func WithREE() Option

WithREE requests run-end encoding for the top-level array.

func WithTemporal 

func WithTemporal(temporal string) Option

WithTemporal overrides the Arrow temporal encoding for time.Time slices. Valid values: "date32", "date64", "time32", "time64", "timestamp" (default). Equivalent to tagging a struct field with arrow:",date32" etc. Invalid values cause FromSlice to return an error.

func WithView 

func WithView() Option

WithView requests view-type encoding (STRING_VIEW, BINARY_VIEW, LIST_VIEW) for the top-level array and recursively for nested types.

Example 
package main

import (
	"fmt"

	"github.com/apache/arrow-go/v18/arrow/array/arreflect"
	"github.com/apache/arrow-go/v18/arrow/memory"
)

func main() {
	mem := memory.NewGoAllocator()

	arr, err := arreflect.FromSlice([]string{"hello", "world"}, mem, arreflect.WithView())
	if err != nil {
		panic(err)
	}
	defer arr.Release()

	fmt.Println("Type:", arr.DataType())
	fmt.Println("Len:", arr.Len())
}

Output:
Type: string_view
Len: 2

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.