options

type CSVConfig struct {
	// Delimiter separates fields on a row. It defaults to ',' for CSV and '\t' for TSV.
	Delimiter rune
	// HasHeader says whether the first row names the columns rather than holding data.
	HasHeader bool
}

type Config struct {
	// Logger receives ProcessFile diagnostics, when set.
	Logger Logger
	// Transformer changes each decoded record before it is encoded, when set.
	Transformer Transformer
	// OnRowErrorSkip, if set, is called once per row MaxRowErrors allows to be skipped, with the
	// decode error that row raised. It must be safe for concurrent use: one decode worker per
	// goroutine may call it.
	OnRowErrorSkip func(err error)
	// OptionErr is a deferred validation-error slot: an option func that fails validation stores
	// its error here instead of panicking, and New's caller checks it once after every option has
	// run.
	OptionErr error
	// InputFormat is the source file's format, or empty to detect it from the file extension.
	InputFormat OutputFormat
	// OutputFormat is the format documents are delivered in.
	OutputFormat OutputFormat
	// Run holds settings for the decode/dispatch pool, shared by every format.
	Run RunConfig
	// NDJSON holds settings only the NDJSON format reads.
	NDJSON NDJSONConfig
	// Parquet holds settings only the Parquet format reads.
	Parquet ParquetConfig
	// CSV holds settings the CSV and TSV formats read; the two differ only in CSV.Delimiter.
	CSV CSVConfig
	// MaxRowErrors caps how many rows may fail to decode and be skipped before the run fails,
	// mirroring BigQuery's load-job max_bad_records: its zero value means fail immediately on the
	// first row error, matching every version of streamio before this option existed. Only a
	// decoder error that identifies itself as safely skippable (see formatio.RowError) counts
	// against this cap — currently NDJSON and CSV/TSV field errors, not Parquet or a CSV/TSV syntax
	// error, which always fail the run regardless of MaxRowErrors.
	MaxRowErrors int
}

type DocumentHandler func(ctx context.Context, doc []byte) error

type Logger interface {
	Printf(format string, args ...any)
}

type NDJSONConfig struct {
	// ChunkSize is the reader's fixed work-queue chunk size, in bytes.
	ChunkSize int
}

type Option func(*Config)

type OutputFormat string

type ParquetConfig struct {
	// MaxOpenReaders caps simultaneously-open row-group readers.
	MaxOpenReaders int
}

type PathTransformRule struct {
	// contains filtered or unexported fields
}

type PathTransformer struct {
	// contains filtered or unexported fields
}

type Result struct {
	Stats Stats
}

type RunConfig struct {
	// ReadBufferSize is the read buffer size, in bytes, used while scanning the input file.
	ReadBufferSize int
	// BatchSize is how many decoded documents a decode worker batches into one send to a dispatch
	// worker.
	BatchSize int
	// Workers is the number of concurrent decode workers.
	Workers int
	// SingleFileOutput requests one continuous output document spanning the whole run, for formats
	// that support it, instead of one document per batch.
	//
	// It forces Workers down to a single decode worker: see formatio.FinalizableRecordEncoder.
	SingleFileOutput bool
}

type Source struct {
	Reader io.ReaderAt
	Name   string
	Size   int64
}

type Stats struct {
	// RowsRead is how many input rows were decoded, regardless of how many documents they became.
	RowsRead int64
	// RowsSkipped is how many rows WithMaxRowErrors allowed to be skipped. Always zero under the
	// default MaxRowErrors of 0.
	RowsSkipped int64
	// DocumentsDispatched is how many documents were handed to the caller's handler. It can differ
	// from RowsRead when an encoder renders a whole batch as one document (Parquet) rather than one
	// document per row (NDJSON).
	DocumentsDispatched int64
	// ReadDuration sums every worker's decode+encode time.
	ReadDuration time.Duration
	// DispatchDuration sums every worker's time spent in the caller's handler.
	DispatchDuration time.Duration
}

type TooManyRowErrorsError struct {
	// Errors holds every row error collected before the run stopped, in the order encountered
	// across every decode worker sharing the run's limit.
	Errors []error
	// Limit is the MaxRowErrors value the run was configured with.
	Limit int
}

type Transformer interface {
	Transform(record.Record) (record.Record, error)
}