encoding

Oct 1, 2026

Changes in this version

+ const DefaultDedupRatioFloor

+ const FormatVersionV1

+ const FormatVersionV2

+ const GroupIndexWidth

+ const GroupVerdictAdmitted

+ const GroupVerdictDroppedTooNarrow

+ const GroupVerdictLowRatio

+ const MaxFormatVersion

+ const MinElisionRows

+ var MaxGroupEntries uint64 = math.MaxUint32 + 1

+ var ZstdMagic = [4]byte

+ func CompressedCohortError() *errors.CodedError

+ func ForEachRecord(r io.Reader, stride int, fn func(rec []byte) error) (int64, error)

+ func IsSupportedFormatVersion(v byte) bool

+ func IsZstdMagic(b []byte) bool

+ func RefuseGroups(s *Schema, operation string, code errors.Code) error

+ func RewriteShardCategoricalsKeepNulls(shardBytes []byte, targetSchema *Schema, remap map[int]DictRemap) ([]byte, error)

+ func SupportedFormatVersions() []byte

+ func ValidateGroupCohesion(canonical, incoming *Schema) error

+ func WritePreamble(w io.Writer, s *Schema) error

+ type ConstantDetector struct

+ func NewConstantDetector(flat *Schema) (*ConstantDetector, error)

+ func (d *ConstantDetector) ConstantFields() []int

+ func (d *ConstantDetector) FirstRow() []byte

+ func (d *ConstantDetector) IsNullConstant(fi int) bool

+ func (d *ConstantDetector) Observe(lrow []byte) error

+ func (d *ConstantDetector) Rows() int64

+ func (d *ConstantDetector) Stride() int

+ type ConstantPlan struct

+ BytesSaved int64

+ Fields []string

+ Retained string

+ Skipped string

+ Spec *GroupSpec

+ func PlanConstantElision(d *ConstantDetector, reserved []string) (*ConstantPlan, error)

+ func PlanConstantElisionFor(flat *Schema, constant []int, rows int64, reserved []string) (*ConstantPlan, error)

type DecodeFields

+ Indices []int

+ type DedupGate struct

+ RatioFloor float64

+ Strict bool

+ func (gt DedupGate) AssessRatios(grouped *Schema, specs []GroupSpec, rows int64) ([]GroupViability, []*errors.CodedError, error)

+ func (gt DedupGate) Floor() float64

+ func (gt DedupGate) ScreenWidths(flat *Schema, specs []GroupSpec) ([]GroupSpec, []GroupViability, []*errors.CodedError, error)

+ type Group struct

+ Entries []byte

+ Kind GroupKind

+ Members []GroupMember

+ func CloneGroups(groups []Group) []Group

+ type GroupEncoder struct

+ func NewGroupEncoder(flat *Schema, specs []GroupSpec) (*GroupEncoder, error)

+ func (e *GroupEncoder) EncodeRow(dst, lrow []byte) ([]byte, error)

+ func (e *GroupEncoder) EncodeStream(spool io.Writer, lr io.Reader) (int64, error)

+ func (e *GroupEncoder) LogicalStride() int

+ func (e *GroupEncoder) PhysicalStride() int

+ func (e *GroupEncoder) Rows() int64

+ func (e *GroupEncoder) Schema() *Schema

+ func (e *GroupEncoder) Seed(seed *Schema) error

+ type GroupEntryDecoder struct

+ func (d *GroupEntryDecoder) Decode(e int, sink IndexedReusableRecord) error

+ type GroupIndexHeadroom struct

+ Capacity uint64

+ Entries int

+ Group int

+ Headroom uint64

+ Kind string

+ Label string

+ func GroupIndexHeadroomFor(s *Schema) []GroupIndexHeadroom

+ type GroupIndexRecord interface

+ SetGroupIndices func(schema *Schema, idx []uint32)

+ type GroupKind uint8

+ const GroupKindConstant

+ const GroupKindIndexed

+ func (k GroupKind) String() string

+ type GroupMember struct

+ Field int

+ Key bool

+ type GroupSpec struct

+ Key []string

+ Kind GroupKind

+ Members []string

+ Ordinal int

+ func GroupSpecsOf(s *Schema) []GroupSpec

+ func (sp GroupSpec) Label(g int) string

+ type GroupViability struct

+ BreakEvenRatio float64

+ ByteDelta int64

+ DedupedBytes int64

+ DescriptorBytes int

+ DictionaryBytes int64

+ EntryCount int

+ EntryWidth int

+ Group int

+ IndexWidth int

+ Label string

+ MemberRowBytes int

+ Ratio float64

+ RatioFloor float64

+ Reason string

+ Rows int64

+ UndedupedBytes int64

+ Verdict string

+ func AssessGroup(grouped *Schema, g int, rows int64, floor float64) GroupViability

+ func (v GroupViability) Finding() (low, grows bool)

+ func (v GroupViability) Viable() bool

+ type IndexedReusableRecord interface

+ ClearForRow func()

+ SetNullFieldAt func(idx int)

+ SetNumericAt func(idx int, value float64)

+ SetWideFieldAt func(idx int, value any)

type RecordReader

+ func (rr *RecordReader) GroupIndex(g int) (uint32, bool)

+ type RowExpander struct

+ func NewRowExpander(s *Schema) (*RowExpander, error)

+ func (x *RowExpander) Expand(dst, phys []byte) ([]byte, error)

+ func (x *RowExpander) GroupIndex(g int) uint32

+ func (x *RowExpander) LogicalStride() int

+ func (x *RowExpander) PhysicalStride() int

+ type RunSkipRecord interface

+ BeginRunRow func(token uint64, keep bool) bool

+ ClearNullAt func(idx int)

type Schema

+ Groups []Group

+ func DedupCohort(dst io.Writer, src io.Reader, specs []GroupSpec) (*Schema, int64, error)

+ func FlattenCohortBytes(cohort []byte) ([]byte, *Schema, error)

+ func NewLogicalStream(r io.Reader, s *Schema) (io.Reader, *Schema, error)

+ func ReadPreamble(r io.Reader) (*Schema, byte, error)

+ func (s *Schema) DecodeGroupEntry(g, e int, sink IndexedReusableRecord) error

+ func (s *Schema) FieldGroup(fi int) (g, k int, ok bool)

+ func (s *Schema) GroupEntry(g, e int) []byte

+ func (s *Schema) GroupEntryCount(g int) int

+ func (s *Schema) GroupEntryWidth(g int) int

+ func (s *Schema) GroupIndexOffset(g int) int

+ func (s *Schema) GroupMemberBytes(g, k int, entry []byte) []byte

+ func (s *Schema) GroupMemberIsNull(g, k int, entry []byte) bool

+ func (s *Schema) GroupMemberRowBytes(g int) int

+ func (s *Schema) GroupSpecOf(g int) GroupSpec

+ func (s *Schema) HasGroups() bool

+ func (s *Schema) Logical() *Schema

+ func (s *Schema) NewGroupEntryDecoder(g int, fields []int) (*GroupEntryDecoder, error)

+ func (s *Schema) RequiredFormatVersion() byte

+ func (s *Schema) ValidateGroups() error

+ type TypedSetRecord interface

+ SetNarrowSetAt func(idx int, mask uint64)

+ SetWideSetAt func(idx int, m SetMask)

Sep 24, 2026

Changes in this version

+ const SetMaskBits

+ const SetMaskWords

+ func CheckSetWiden(from, to FieldType) error

+ func PutSetMask(dst []byte, ft FieldType, m SetMask) error

+ func WriteSetMask(w io.Writer, ft FieldType, m SetMask) error

type FieldType

+ const FieldTypeSetU128

+ const FieldTypeSetU256

+ func SetLadder() []FieldType

+ func SetTypeFor(elements int) (FieldType, bool)

+ func WidestSetType() FieldType

+ func (ft FieldType) IsWideSet() bool

type Schema

+ func WidenSchemaSetField(s *Schema, field string, target FieldType) (*Schema, error)

+ type SetMask struct

+ func ReadSetMask(r io.Reader, ft FieldType) (SetMask, error)

+ func SetMaskFromBytes(ft FieldType, src []byte) (SetMask, error)

+ func SetMaskFromUint64(low uint64) SetMask

+ func SetMaskFromWords(words [SetMaskWords]uint64) SetMask

+ func (m SetMask) Bits() iter.Seq[int]

+ func (m SetMask) Equal(o SetMask) bool

+ func (m SetMask) FitsFieldType(ft FieldType) bool

+ func (m SetMask) Has(i int) bool

+ func (m SetMask) HighestBit() int

+ func (m SetMask) Intersect(o SetMask) SetMask

+ func (m SetMask) IsEmpty() bool

+ func (m SetMask) Labels(dict *Dictionary) []string

+ func (m SetMask) NextBit(from int) (int, bool)

+ func (m SetMask) PopCount() int

+ func (m SetMask) Uint64() (uint64, bool)

+ func (m SetMask) Union(o SetMask) SetMask

+ func (m SetMask) WithBit(i int) SetMask

+ func (m SetMask) WithoutBit(i int) SetMask

+ func (m SetMask) Words() [SetMaskWords]uint64

+ type SetWidenPlan struct

+ Field string

+ FieldIndex int

+ From FieldType

+ IncomingFrom FieldType

+ To FieldType

+ UnionEntries int

+ func PlanSetWidening(canonical, incoming *Schema) ([]SetWidenPlan, error)

+ type SetWidthHeadroom struct

+ Capacity int

+ Field string

+ Headroom int

+ NextType string

+ Type string

+ Used int

+ func SetWidthHeadroomFor(s *Schema) []SetWidthHeadroom

+ type WidenReport struct

+ Field string

+ From FieldType

+ Records int64

+ StrideAfter int

+ StrideBefore int

+ To FieldType

+ func WidenSetFieldBytes(cohort []byte, field string, target FieldType) ([]byte, *WidenReport, error)

+ func WidenSetFieldFile(fsys afero.Fs, path, field string, target FieldType) (*WidenReport, error)

Sep 12, 2026

Changes in this version

+ const IndexManifestFormatVersion

+ const IndexManifestKind

+ const IndexManifestSuffix

+ func IndexManifestPath(cohortPath string) string

+ func WriteIndexManifest(fsys afero.Fs, cohortPath string, m *IndexManifest) error

+ type IndexManifest struct

+ Cohort string

+ FormatVersion string

+ Indexes []IndexManifestEntry

+ Kind string

+ func NewIndexManifest(cohortPath string) *IndexManifest

+ func ReadIndexManifest(fsys afero.Fs, cohortPath string) (*IndexManifest, bool, error)

+ func (m *IndexManifest) Remove(indexBase string) bool

+ func (m *IndexManifest) Upsert(entry IndexManifestEntry)

+ type IndexManifestEntry struct

+ DistinctKeys int

+ IndexPath string

+ IndexedRecords int

+ KeyHash string

+ Keys []IndexManifestKey

+ func IndexManifestEntryFor(indexPath string, idx *Index) IndexManifestEntry

+ func (e IndexManifestEntry) KeyNames() []string

+ type IndexManifestKey struct

+ Name string

+ Type string

Sep 12, 2026

Changes in this version

type Schema

+ func (s *Schema) RecordCountForPayload(payloadBytes int64) (count, trailing int64, ok bool)

Aug 27, 2026

Changes in this version

+ const CanonicalDateTimeLayout

+ const SecondsPerDay

+ var DateTimeFormats = []string

+ func DateTimeToDay(sec int64) int64

+ func FormatDateTime(raw uint64) string

+ func ParseDateTime(raw string) (uint64, error)

type FieldType

+ const FieldTypeDateTime

Jul 27, 2026

Changes in this version

+ const FingerprintSize

+ const IndexFormatVersion

+ const IndexHeaderSize

+ var DateFormats = []string

+ var IndexMagicBytes = [8]byte

+ func BucketIndex(key []byte, bucketCount uint32) uint32

+ func HashKey(key []byte) uint64

+ func ParseDate(raw string) (uint32, error)

+ func ReadIndexHeader(r io.Reader) error

+ func SidecarIndexPath(cohortPath string, keyFields []string) string

+ func WriteIndex(w io.Writer, idx *Index) error

+ func WriteIndexFile(fsys afero.Fs, path string, idx *Index) error

+ func WriteIndexHeader(w io.Writer) error

+ type Fingerprint [FingerprintSize]byte

+ func ComputeFingerprint(r io.Reader) (Fingerprint, error)

+ type Index struct

+ Buckets []IndexBucket

+ Fingerprint Fingerprint

+ Keys []IndexKeySpec

+ SourceModTime int64

+ SourceSize uint64

+ func ReadIndex(r io.Reader) (*Index, error)

+ func ReadIndexFile(fsys afero.Fs, path string) (*Index, error)

+ type IndexBucket struct

+ Entries []IndexEntry

+ func ReadBucketByKey(rs io.ReadSeeker, meta *IndexMeta, key []byte) (*IndexBucket, error)

+ type IndexEntry struct

+ Key []byte

+ RowIDs []uint64

+ type IndexKeySpec struct

+ Name string

+ Type FieldType

+ type IndexMeta struct

+ BucketCount uint32

+ Fingerprint Fingerprint

+ Keys []IndexKeySpec

+ OffsetTableStart int64

+ SourceModTime int64

+ SourceSize uint64

+ func ReadIndexMeta(r io.Reader) (*IndexMeta, error)

+ func ReadIndexMetaFile(fsys afero.Fs, path string) (*IndexMeta, error)

+ func (m *IndexMeta) BucketDataStart() int64

+ type RecordLocator struct

+ RecordRegionStart int64

+ Schema *Schema

+ Stride int64

+ TotalRecords uint64

+ func NewRecordLocator(r *bytes.Reader, schema *Schema) (*RecordLocator, error)

+ func (loc *RecordLocator) Offset(i uint64) int64

+ func (loc *RecordLocator) ReadRecordAt(r io.ReadSeeker, i uint64, values map[string]float64, nulls map[string]bool, ...) error

Jul 14, 2026

Changes in this version

type RecordReader

+ func (rr *RecordReader) ReadRecordReusedWithPlan(rec ReusableRecord, keep FieldFilter, plan *DecodePlan) error

Jun 11, 2026

Changes in this version

+ type DecodeFields struct

+ Fields []*Field

+ type DecodePlan struct

+ Segments []Segment

type RecordReader

+ func (rr *RecordReader) ReadRecordWithWidePlan(values map[string]float64, nulls map[string]bool, wide map[string]any, ...) error

type Schema

+ func (s *Schema) BuildDecodePlan(retained []string) (*DecodePlan, error)

+ type Segment interface

+ type SkipBytes struct

+ N int

Jun 10, 2026

Changes in this version

type FieldType

+ const FieldTypeSetU16

+ const FieldTypeSetU32

+ const FieldTypeSetU64

+ const FieldTypeSetU8

+ func ParseFieldType(name string) (FieldType, bool)

+ func (ft FieldType) HasDictionary() bool

+ func (ft FieldType) IsSet() bool

+ func (ft FieldType) MaxDictEntries() uint32

+ func (ft FieldType) MaxSetEntries() uint32

type Schema

+ func (s *Schema) SetField(name string) (*Dictionary, bool)

May 20, 2026

Changes in this version

+ func BitmapIsNull(bitmap []byte, i int) bool

+ func BitmapSetNull(bitmap []byte, i int)

+ func ReadBitmap(r io.Reader, byteLen int) ([]byte, error)

+ func WriteBitmap(w io.Writer, bitmap []byte) error

type Field

+ Nullable bool

type FieldType

+ const FieldTypeU4

+ func (ft FieldType) IsBitPacked() bool

type Schema

+ func (s *Schema) BitmapByteSize() int

+ func (s *Schema) HasBitmap() bool

May 19, 2026

Changes in this version

type Schema

+ func (s *Schema) RecordByteSize() int

May 18, 2026

Changes in this version

+ const ReservedSchemaName

+ var SchemaDocMagic = [4]byte

+ var ZipMagic = [4]byte

+ func IsArchive(r io.ReaderAt, size int64) (bool, error)

+ func RewriteShardCategoricals(shardBytes []byte, targetSchema *Schema, remap map[int]DictRemap) ([]byte, error)

+ func WriteSchemaDoc(w io.Writer, schema *Schema, agg uint64, shardCount uint16) error

+ type Archive struct

+ func OpenArchive(r io.ReaderAt, size int64) (*Archive, error)

+ func (a *Archive) Entries() []ArchiveEntry

+ func (a *Archive) IsStored(name string) bool

+ func (a *Archive) Open(name string) (io.ReadCloser, error)

+ func (a *Archive) OpenAt(name string) (io.SectionReader, error)

+ func (a *Archive) PeekShardHeader(name string) error

+ func (a *Archive) PeekShardRecordCount(name string) (int64, error)

+ type ArchiveEntry struct

+ Name string

+ Offset int64

+ Size int64

+ type CohesionWarning struct

+ Code string

+ Details map[string]any

+ Message string

+ func ValidateStructuralCohesion(canonical, incoming *Schema) ([]CohesionWarning, error)

+ type DictRemap = map[uint32]uint32

+ type FieldFilter func(name string) bool

type RecordReader

+ func (rr *RecordReader) ReadRecordWithWideProjected(values map[string]float64, nulls map[string]bool, wide map[string]any, ...) error

type Schema

+ func MergeDictUnion(canonical, incoming *Schema) (*Schema, map[int]DictRemap, error)

+ func ValidateDictPrefixRule(canonical, incoming *Schema) (*Schema, error)

+ type SchemaDoc struct

+ AggregateRecordCount uint64

+ Schema *Schema

+ ShardCount uint16

+ func ReadSchemaDoc(r io.Reader) (*SchemaDoc, error)

May 15, 2026

Changes in this version

type FieldType

+ func (ft FieldType) IsNumeric() bool

+ func (ft FieldType) IsNumericForAnalytics() bool

May 11, 2026

Changes in this version

+ const MaxDecimalPrecision

+ const MinDecimalScale

+ func CrossesAntimeridian(pts []PointF64) bool

+ func EncodeDecimal128(d Decimal128) [16]byte

+ func EncodeH3Cell(c H3Cell) [8]byte

+ func EncodePointF64(p PointF64) [16]byte

+ func FormatH3CellHex(c H3Cell) string

+ func FormatWKTPoint(p PointF64) string

+ func HaversineMeters(a, b PointF64) float64

+ func NullDecimalSentinel() [16]byte

+ func PromoteAdd(p1, s1, p2, s2 uint8) (uint8, uint8)

+ func PromoteDiv(p1, s1, p2, s2 uint8) (uint8, uint8)

+ func PromoteMul(p1, s1, p2, s2 uint8) (uint8, uint8)

+ func ValidatePrecisionScale(precision, scale uint8) error

+ func WriteDecimal128(w io.Writer, d Decimal128) error

+ func WriteDecimal128Null(w io.Writer) error

+ func WriteH3Cell(w io.Writer, c H3Cell) error

+ func WritePointF64(w io.Writer, p PointF64) error

+ type Decimal128 struct

+ func DecodeDecimal128(buf [16]byte) (Decimal128, bool)

+ func NewDecimal128FromBigInt(m *big.Int) (Decimal128, error)

+ func NewDecimal128FromInt(i int64) Decimal128

+ func ParseDecimal128(s string) (Decimal128, uint8, error)

+ func ReadDecimal128(r io.Reader) (Decimal128, bool, error)

+ func ZeroDecimal128() Decimal128

+ func (d Decimal128) Add(o Decimal128) (Decimal128, error)

+ func (d Decimal128) Cmp(o Decimal128) int

+ func (d Decimal128) Div(o Decimal128, s1, s2, resultScale uint8) (Decimal128, error)

+ func (d Decimal128) FitsPrecision(precision uint8) bool

+ func (d Decimal128) Float64(scale uint8) float64

+ func (d Decimal128) Mantissa() *big.Int

+ func (d Decimal128) Mul(o Decimal128) (Decimal128, error)

+ func (d Decimal128) Rescale(sourceScale, targetScale uint8) (Decimal128, error)

+ func (d Decimal128) Sign() int

+ func (d Decimal128) Sqrt(sourceScale, targetScale uint8) (Decimal128, error)

+ func (d Decimal128) String(scale uint8) string

+ func (d Decimal128) Sub(o Decimal128) (Decimal128, error)

type Field

+ H3Resolution uint8

+ Precision uint8

+ Scale uint8

type FieldType

+ const FieldTypeDecimal128

+ const FieldTypeH3Cell

+ const FieldTypeNullableDecimal128

+ const FieldTypePointF64

+ func (ft FieldType) IsDecimal() bool

+ func (ft FieldType) IsGeo() bool

+ func (ft FieldType) IsKnown() bool

+ type H3Cell uint64

+ func DecodeH3Cell(buf [8]byte) H3Cell

+ func ParseH3CellHex(s string) (H3Cell, error)

+ func ReadH3Cell(r io.Reader) (H3Cell, error)

+ type PointF64 struct

+ Lat float64

+ Lon float64

+ func CentroidUnitSphere(pts []PointF64) (PointF64, bool)

+ func DecodePointF64(buf [16]byte) PointF64

+ func ParseWKTPoint(s string) (PointF64, error)

+ func ReadPointF64(r io.Reader) (PointF64, error)

+ func (p PointF64) Validate() error

+ type Polygon struct

+ Ring []PointF64

+ func ParseWKTPolygon(s string) (*Polygon, error)

+ func (poly *Polygon) Contains(p PointF64) bool

type RecordReader

+ func (rr *RecordReader) ReadRecordReused(rec ReusableRecord) error

+ func (rr *RecordReader) ReadRecordWithWide(values map[string]float64, nulls map[string]bool, wide map[string]any) error

+ type ReusableRecord interface

+ ClearForRow func()

+ SetNullField func(name string)

+ SetNumeric func(name string, value float64)

+ SetWideField func(name string, value any)

Apr 27, 2026

Changes in this version

+ const FormatVersion

+ const HeaderSize

+ const MaxDescriptionBytes

+ var MagicBytes = [8]byte

+ func ReadBit(r io.Reader, bitPos uint) (bool, error)

+ func ReadDescription(r io.Reader) (string, error)

+ func ReadFieldValue(r io.Reader, ft FieldType) (uint64, error)

+ func ReadHeader(r io.Reader) error

+ func ReadNibble(r io.Reader, high bool) (uint8, error)

+ func WriteBit(w io.Writer, bitPos uint, val bool) error

+ func WriteDescription(w io.Writer, desc string) error

+ func WriteFieldValue(w io.Writer, ft FieldType, val uint64) error

+ func WriteHeader(w io.Writer) error

+ func WriteNibble(w io.Writer, high bool, val uint8) error

+ func WriteSchema(w io.Writer, s *Schema) error

+ type Dictionary struct

+ func NewDictionary() *Dictionary

+ func (d *Dictionary) Add(s string) (uint32, error)

+ func (d *Dictionary) AddWithLimit(s string, maxEntries uint32) (uint32, error)

+ func (d *Dictionary) Count() int

+ func (d *Dictionary) IDFor(s string) (uint32, bool)

+ func (d *Dictionary) ReadFrom(r io.Reader) error

+ func (d *Dictionary) Resolve(id uint32) string

+ func (d *Dictionary) Values() []string

+ func (d *Dictionary) WriteTo(w io.Writer) error

+ type Field struct

+ BitPosition int

+ ByteOffset int

+ CsvColumnIdx int

+ Description string

+ Dictionary *Dictionary

+ Name string

+ Type FieldType

+ type FieldType byte

+ const FieldTypeCategoricalU16

+ const FieldTypeCategoricalU32

+ const FieldTypeCategoricalU8

+ const FieldTypeDate

+ const FieldTypeF32

+ const FieldTypeF64

+ const FieldTypeNullableBool

+ const FieldTypeNullableU16

+ const FieldTypeNullableU4

+ const FieldTypeNullableU8

+ const FieldTypePackedBool

+ const FieldTypeU16

+ const FieldTypeU32

+ const FieldTypeU64

+ const FieldTypeU8

+ func (ft FieldType) ByteSize() int

+ func (ft FieldType) IsCategorical() bool

+ func (ft FieldType) MaxCategoricalEntries() uint32

+ func (ft FieldType) String() string

+ type RecordReader struct

+ func NewRecordReader(r io.Reader, schema *Schema) *RecordReader

+ func (rr *RecordReader) ReadRecord(values map[string]float64, nulls map[string]bool) error

+ type Schema struct

+ Fields []Field

+ func ReadSchema(r io.Reader) (*Schema, error)

+ func (s *Schema) Categorical(name string) (*Dictionary, bool)

+ func (s *Schema) Field(name string) *Field