options

type CSVConfig struct {
	// Delimiter separates fields on a row. It defaults to ',' for CSV and '\t' for TSV.
	Delimiter rune
	// HasHeader says whether the first row names the columns rather than holding data.
	HasHeader bool
}

type Compression string

type Config struct {
	Transformer         Transformer
	OptionErr           error
	Logger              Logger
	OnRowErrorSkip      func(err error)
	OutputFormat        OutputFormat
	InputCompression    Compression
	InputFormat         OutputFormat
	Profile             ProfileConfig
	Run                 RunConfig
	MaxDecompressedSize int64
	NDJSON              NDJSONConfig
	Parquet             ParquetConfig
	MaxRowErrors        int
	CSV                 CSVConfig
}

type DocumentHandler func(ctx context.Context, doc []byte) error

type Logger interface {
	Printf(format string, args ...any)
}

type NDJSONConfig struct {
	// ChunkSize is the reader's fixed work-queue chunk size, in bytes.
	ChunkSize int
}

type Option func(*Config)

type OutputFormat string

type ParquetConfig struct {
	// MaxOpenReaders caps simultaneously-open row-group readers.
	MaxOpenReaders int
}

type PathTransformRule struct {
	// contains filtered or unexported fields
}

type PathTransformer struct {
	// contains filtered or unexported fields
}

type ProfileConfig struct {
	// Columns, when non-empty, restricts the profile to these attribute paths.
	Columns []string
	// TopN is how many most-frequent values are kept per attribute.
	TopN int
	// MaxDistinct caps how many distinct values are counted per attribute.
	MaxDistinct int
}

type Result struct {
	Stats Stats
}

type RunConfig struct {
	// ReadBufferSize is the read buffer size, in bytes, used while scanning the input file.
	ReadBufferSize int
	// BatchSize is how many decoded documents a decode worker batches into one send to a dispatch
	// worker.
	BatchSize int
	// Workers is the number of concurrent decode workers.
	Workers int
	// SingleFileOutput requests one continuous output document spanning the whole run, for formats
	// that support it, instead of one document per batch.
	//
	// It forces Workers down to a single decode worker: see formatio.FinalizableRecordEncoder.
	SingleFileOutput bool
}

type Source struct {
	Reader io.ReaderAt
	Name   string
	Size   int64
}

type Stats struct {
	// RowsRead is how many input rows were decoded, regardless of how many documents they became.
	RowsRead int64
	// RowsSkipped is how many rows WithMaxRowErrors allowed to be skipped. Always zero under the
	// default MaxRowErrors of 0.
	RowsSkipped int64
	// DocumentsDispatched is how many documents were handed to the caller's handler. It can differ
	// from RowsRead when an encoder renders a whole batch as one document (Parquet) rather than one
	// document per row (NDJSON).
	DocumentsDispatched int64
	// ReadDuration sums every worker's decode+encode time.
	ReadDuration time.Duration
	// DispatchDuration sums every worker's time spent in the caller's handler.
	DispatchDuration time.Duration
}

type TooManyRowErrorsError struct {
	// Errors holds every row error collected before the run stopped, in the order encountered
	// across every decode worker sharing the run's limit.
	Errors []error
	// Limit is the MaxRowErrors value the run was configured with.
	Limit int
}

type Transformer interface {
	Transform(record.Record) (record.Record, error)
}