Help us shape the future of Go! Take the 2026 Go Developer Survey

data

package
v0.1.0-rc.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 26, 2026 License: MIT Imports: 5 Imported by: 0

Documentation

Overview

Package data defines the run-scoped Arrow ownership and result contracts.

Index

Constants

View Source 
const (
	ReportVersion        = 2
	ScopeData            = "data"
	ScopePlan            = "plan"
	StatusCompleted      = "completed"
	StatusSchemaRejected = "schema_rejected"

	ScopeRow     = "row"
	ScopeSchema  = "schema"
	ScopeDataset = "dataset"

	OriginRule    = "rule"
	OriginIngest  = "ingest"
	OriginDataset = "dataset"

	ValidationPass = "pass"
	ValidationFail = "fail"
)

Report vocabulary shared by producers and the sink. The JSON spelling is the constant value.

Variables

This section is empty.

Functions

func CheckedAdd 

func CheckedAdd(a, b uint64) (uint64, error)

CheckedAdd rejects counter/provenance overflow instead of wrapping.

Types

type Batch 

type Batch struct {
	Record    arrow.RecordBatch
	States    map[string][]State
	Locations []Location
	Raw       []json.RawMessage
	Ingest    []Issue
	Evidence  *GroupEvidence
	Cleanup   func()
	// contains filtered or unexported fields
}

Batch owns Record and bounded metadata. Release must be called on every path. States omitted for a column means use the Arrow validity bitmap (valid/null).

func (*Batch) BindState 

func (b *Batch) BindState(column string, index int) ColumnState

BindState binds a previously resolved schema index without resolving a name for each row. A negative index represents an absent column.

func (*Batch) Release 

func (b *Batch) Release()

func (*Batch) State 

func (b *Batch) State(column string, row int) State

type Bucket 

type Bucket uint8

Bucket is the output partition chosen for one evaluated row. 

const (
	BucketPass Bucket = iota
	BucketWarn
	BucketError
)

func (Bucket) MarshalText 

func (b Bucket) MarshalText() ([]byte, error)

MarshalText encodes b with its report spelling and rejects unknown buckets.

func (Bucket) String 

func (b Bucket) String() string

String returns the report spelling of b, or "bucket(N)" when b is unknown.

func (*Bucket) UnmarshalText 

func (b *Bucket) UnmarshalText(text []byte) error

UnmarshalText decodes the report spelling written by MarshalText.

func (Bucket) Valid 

func (b Bucket) Valid() bool

Valid reports whether b is one of the defined buckets.

type CheckEvidence 

type CheckEvidence struct {
	Code                   string   `json:"code"`
	CheckIndex             int      `json:"check_index"`
	Columns                []string `json:"columns"`
	Claim                  string   `json:"claim"`
	Fields                 []string `json:"fields"`
	Reason                 string   `json:"reason"`
	ImplicitConstraintKind string   `json:"implicit_constraint_kind,omitempty"`
}

type ColumnState 

type ColumnState struct {
	// contains filtered or unexported fields
}

ColumnState binds ingestion overrides and Arrow validity once per batch. It borrows the batch's arrays and is valid only until Batch.Release.

func (ColumnState) State 

func (c ColumnState) State(row int) State

State returns the ingestion state of the bound row.

type ColumnStatistics 

type ColumnStatistics struct {
	Name               string
	Type               arrow.DataType
	PhysicalType       string
	PhysicalLength     int
	NullCount          *int64
	NaNCount           *int64
	MinValue, MaxValue []byte
	MinExact, MaxExact *bool
	TypeDefinedOrder   bool
	LegacyOnly         bool
}

ColumnStatistics borrows modern bounds from the guarded dataset footer. Optional fields preserve absence; no generated getter defaults are evidence.

type Coverage 

type Coverage struct {
	ValueRows               uint64  `json:"value_rows"`
	MetadataClaimedRows     uint64  `json:"metadata_claimed_rows"`
	AccountedRows           uint64  `json:"accounted_rows"`
	RowGroupsScanned        *uint64 `json:"row_groups_scanned"`
	RowGroupsStatisticsOnly *uint64 `json:"row_groups_statistics_only"`
	RowGroupsTotal          *uint64 `json:"row_groups_total"`
}

Coverage separates decoded outcomes from producer-declared metadata claims.

type DatasetResult 

type DatasetResult struct {
	Code          string `json:"code"`
	Metric        string `json:"metric"`
	Value         any    `json:"value"`
	Status        string `json:"status"`
	Reason        string `json:"reason,omitempty"`
	Numerator     uint64 `json:"numerator,omitempty"`
	Denominator   uint64 `json:"denominator,omitempty"`
	Contributing  uint64 `json:"contributing,omitempty"`
	Excluded      uint64 `json:"excluded,omitempty"`
	NotApplicable uint64 `json:"not_applicable"`
	Unevaluable   uint64 `json:"unevaluable"`
}

type GroupEvidence 

type GroupEvidence struct {
	SourceOrdinal         uint64            `json:"source_ordinal"`
	RowGroup              uint64            `json:"row_group"`
	DeclaredRows          uint64            `json:"declared_rows"`
	Source                string            `json:"source"`
	FooterSHA256          string            `json:"footer_sha256"`
	FooterOffset          int64             `json:"footer_offset"`
	FooterLength          int64             `json:"footer_length"`
	IdentityMethod        string            `json:"identity_method"`
	Identity              map[string]string `json:"identity"`
	Action                string            `json:"action"`
	Reason                string            `json:"reason"`
	Checks                []CheckEvidence   `json:"checks"`
	IngestionVerification string            `json:"ingestion_verification"`
	PayloadVerification   string            `json:"payload_verification"`
}

type Issue 

type Issue struct {
	Location
	Code     string `json:"code"`
	Severity string `json:"severity"`
	Origin   string `json:"origin"`
	Scope    string `json:"scope"`
	Column   string `json:"column,omitempty"`
	Message  string `json:"message,omitempty"`
	Reason   string `json:"reason,omitempty"`
	Count    uint64 `json:"count,omitempty"`
}

type Location 

type Location struct {
	Source        string  `json:"source"`
	SourceOrdinal uint64  `json:"source_ordinal"`
	Row           uint64  `json:"row"`
	GlobalRow     uint64  `json:"global_row"`
	Line          uint64  `json:"line,omitempty"`
	RowGroup      *uint64 `json:"row_group,omitempty"`
	RowInGroup    uint64  `json:"row_in_group,omitempty"`
	PositionBasis string  `json:"position_basis,omitempty"`
}

type ParquetGroup 

type ParquetGroup struct {
	SourceOrdinal  uint64
	RowGroup       uint64
	DeclaredRows   uint64
	Source         string
	FooterSHA256   string
	FooterOffset   int64
	FooterLength   int64
	IdentityMethod string
	Identity       map[string]string
	Columns        []ColumnStatistics
}

type Result 

type Result struct {
	Buckets []Bucket
	Issues  []Issue
	Cleanup func()
	// contains filtered or unexported fields
}

Result owns evaluation storage independently of its Program and Batch. Call Release after its final consumer, including on unsuccessful writes.

func (*Result) Release 

func (r *Result) Release()

Release returns this result's storage exactly once. It is safe on nil and may run concurrently with evaluation of a different batch.

type RowMaterialization 

type RowMaterialization struct {
	Policy  string `json:"policy"`
	Emitted uint64 `json:"emitted"`
	Omitted uint64 `json:"omitted"`
}

type RuleCoverage 

type RuleCoverage struct {
	ValueRows           uint64 `json:"value_rows"`
	MetadataClaimedRows uint64 `json:"metadata_claimed_rows"`
}

type State 

type State uint8

State is the ingestion state of one cell. 

const (
	Valid State = iota
	Missing
	Null
	Invalid
)

func (State) MarshalText 

func (s State) MarshalText() ([]byte, error)

MarshalText encodes s with its report spelling and rejects unknown states.

func (State) String 

func (s State) String() string

String returns the report spelling of s, or "state(N)" when s is unknown.

func (*State) UnmarshalText 

func (s *State) UnmarshalText(text []byte) error

UnmarshalText decodes the report spelling written by MarshalText.

type StatisticsPlan 

type StatisticsPlan struct {
	Groups []GroupEvidence
	// contains filtered or unexported fields
}

StatisticsPlan owns its immutable decisions and their managed reservation. Retain gives a consumer a distinct release handle sharing the same storage. Dataset.ApplyStatisticsPlan retains its own handle; callers still Release.

func NewStatisticsPlan 

func NewStatisticsPlan(groups []GroupEvidence, cleanup func()) *StatisticsPlan

func (*StatisticsPlan) Release 

func (p *StatisticsPlan) Release()

func (*StatisticsPlan) Retain 

func (p *StatisticsPlan) Retain() *StatisticsPlan

type Summary 

type Summary struct {
	ReportVersion           int                           `json:"report_version"`
	RunID                   string                        `json:"run_id"`
	Status                  string                        `json:"status"`
	Validation              string                        `json:"validation"`
	Scope                   string                        `json:"scope"`
	OutputMode              string                        `json:"output_mode"`
	Verification            string                        `json:"verification"`
	TrustPolicy             string                        `json:"trust_policy"`
	IngestionVerification   string                        `json:"ingestion_verification"`
	ProvenancePositionBasis string                        `json:"provenance_position_basis"`
	Coverage                Coverage                      `json:"coverage"`
	RuleCoverage            map[string]RuleCoverage       `json:"rule_coverage"`
	RowOutput               map[string]RowMaterialization `json:"row_output"`
	Total                   uint64                        `json:"total"`
	Passed                  uint64                        `json:"pass"`
	Warned                  uint64                        `json:"warn"`
	Errored                 uint64                        `json:"error"`
	RowsScanned             uint64                        `json:"rows_scanned"`
	RuleCounts              map[string]uint64             `json:"rule_counts"`
	IngestCounts            map[string]uint64             `json:"ingest_counts"`
	Unevaluable             map[string]uint64             `json:"unevaluable"`
	NotApplicable           map[string]uint64             `json:"not_applicable"`
	Stats                   map[string]any                `json:"stats"`
	DatasetResults          []DatasetResult               `json:"dataset_results"`
	DatasetIssues           []Issue                       `json:"dataset_issues"`
	SchemaIssues            []Issue                       `json:"schema_issues"`
	DurationSeconds         float64                       `json:"duration_seconds"`
	Passes                  int                           `json:"passes"`
	ManagedPeak             int64                         `json:"managed_peak_bytes"`
	ManifestURI             string                        `json:"manifest_uri,omitempty"`
}

func NewSummary 

func NewSummary() *Summary

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.