source

type BatchSource interface {
	Schema() *dtype.Schema

	// Next returns the next batch, or (nil, io.EOF) when the stream is exhausted.
	// An empty non-nil batch is allowed and is not end-of-stream.
	Next(ctx context.Context) (*data.Batch, error)

	Close() error
}

type Openable interface {
	Open(ctx context.Context, spec ScanSpec) (BatchSource, error)
}

type ScanSpec struct {
	// Projection is the exact column list to EMIT, in this order. nil means every
	// column.
	//
	// Honouring it is optional: a source that ignores it gets a trim operator
	// inserted above the scan. Honouring it is where the win is, though — the
	// point of projection pushdown is that the reader never touches the bytes.
	Projection []string

	// Predicate holds conjuncts the source said it could use, via
	// plan.PredicatePushdown.ClassifyPredicates. A source that never claimed a
	// capability never sees this non-empty.
	//
	// # The read set is Projection PLUS these columns
	//
	// A conjunct classified Exact is removed from the plan, so nothing above the
	// scan references its columns and they are absent from Projection. The source
	// must still read them to evaluate the predicate, and must NOT emit them.
	//
	//	Projection: [a]   Predicate: [c > 5]
	//	  =>  decode {a, c}, filter on c, emit {a}
	//
	// A reader that decodes only Projection and then evaluates Predicate is
	// reading a column it never loaded. This is the one place where ignoring a
	// field is not the safe option: having claimed Exact, the source is now the
	// only thing applying that filter.
	//
	// Dropping rows the predicate rejects is required only for a conjunct the
	// source classified Exact. For Inexact, returning a superset is the definition
	// — skip what you can prove cannot match and let the Filter above finish.
	Predicate []expr.Node

	// MaxRows is an upper bound on the rows that will be consumed; 0 means
	// unlimited. Stopping early is the entire value of Head(n) on a large file.
	//
	// Overshooting is fine — a Parquet reader stopping at a row-group boundary,
	// say. A Limit operator above the scan always enforces the real bound.
	MaxRows int

	// Threads is how many workers the query is running on, so a source that can
	// use more than one core knows whether it may. 0 or 1 means stay serial.
	//
	// A source is free to ignore it — most have nothing to parallelise, because
	// reading is I/O and the work above them is where the CPU goes. The CSV and
	// Parquet readers are the exceptions: turning text into typed columns, and
	// decompressing and decoding pages, is real compute, and each used to do all of
	// it on one core while eight sat idle. The Parquet reader decodes row groups in
	// parallel (step 102), and a profile had found it at nearly half the CPU of the
	// queries it measured.
	//
	// It must be honoured in the direction that matters: WithThreads(1) is
	// documented as reproducing the serial operator tree exactly, so a source that
	// parallelised regardless would make that knob a lie.
	Threads int

	// BatchSize is the requested rows per batch. A source may return fewer.
	BatchSize int

	// ReadAhead is how many bytes of decoded batches a source may hold ahead of its
	// consumer; 0 is the source's own default. The Parquet reader decodes several
	// row groups at once and returns them in order, and before step 214 a row group
	// not at the head stopped three batches in, so most of each decoded only once it
	// was the head, one at a time.
	//
	// It is not charged to the memory budget: it is transient and bounded, and
	// charging it would make sinks spill for bytes about to be consumed. The engine
	// derives the bound from the budget instead.
	ReadAhead int64

	// Eval evaluates an expression over a batch, as the engine does. A source that
	// evaluates a pushed conjunct itself uses it; the engine sets it, and a source
	// handed none evaluates nothing. The Parquet reader evaluates a string
	// conjunct once per dictionary value, to drop the rows that fail it before
	// copying them (step 193).
	Eval func(ctx context.Context, n expr.Node, b *data.Batch) (*data.Column, error)
}