formatters

package
v1.7.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 24, 2025 License: MIT Imports: 11 Imported by: 0

Documentation

Index

Constants

View Source
const (
	FormatJSONL   = "jsonl"
	FormatCSV     = "csv"
	FormatParquet = "parquet"
)

Format type constants

Variables

View Source
var ErrNoColumns = errors.New("table schema has no columns")

ErrNoColumns is returned when a table schema has no columns

Functions

func UsesInternalCompression

func UsesInternalCompression(format string) bool

UsesInternalCompression returns true if the format handles compression internally

Types

type CSVFormatter

type CSVFormatter struct{}

CSVFormatter handles CSV format output

func NewCSVFormatter

func NewCSVFormatter() *CSVFormatter

NewCSVFormatter creates a new CSV formatter

func (*CSVFormatter) Extension

func (f *CSVFormatter) Extension() string

Extension returns the file extension for CSV files

func (*CSVFormatter) Format

func (f *CSVFormatter) Format(rows []map[string]interface{}) ([]byte, error)

Format converts rows to CSV format

func (*CSVFormatter) MIMEType

func (f *CSVFormatter) MIMEType() string

MIMEType returns the MIME type for CSV

type CSVReader added in v1.5.3

type CSVReader struct {
	// contains filtered or unexported fields
}

CSVReader reads CSV format with header detection

func NewCSVReader added in v1.5.3

func NewCSVReader(r io.Reader) (*CSVReader, error)

NewCSVReader creates a new CSV reader

func NewCSVReaderWithCloser added in v1.5.3

func NewCSVReaderWithCloser(r io.ReadCloser) (*CSVReader, error)

NewCSVReaderWithCloser creates a new CSV reader with a closable reader

func (*CSVReader) Close added in v1.5.3

func (r *CSVReader) Close() error

Close closes the underlying reader if it's closable

func (*CSVReader) ReadAll added in v1.5.3

func (r *CSVReader) ReadAll() ([]map[string]interface{}, error)

ReadAll reads all remaining rows from the CSV stream

func (*CSVReader) ReadChunk added in v1.5.3

func (r *CSVReader) ReadChunk(chunkSize int) ([]map[string]interface{}, error)

ReadChunk reads a chunk of rows from the CSV stream

type CSVStreamingFormatter added in v1.4.0

type CSVStreamingFormatter struct{}

CSVStreamingFormatter handles CSV format output in streaming mode

func NewCSVStreamingFormatter added in v1.4.0

func NewCSVStreamingFormatter() *CSVStreamingFormatter

NewCSVStreamingFormatter creates a new CSV streaming formatter

func (*CSVStreamingFormatter) Extension added in v1.4.0

func (f *CSVStreamingFormatter) Extension() string

Extension returns the file extension for CSV files

func (*CSVStreamingFormatter) MIMEType added in v1.4.0

func (f *CSVStreamingFormatter) MIMEType() string

MIMEType returns the MIME type for CSV

func (*CSVStreamingFormatter) NewWriter added in v1.4.0

func (f *CSVStreamingFormatter) NewWriter(w io.Writer, schema TableSchema) (StreamWriter, error)

NewWriter creates a new CSV stream writer

type ColumnSchema added in v1.4.0

type ColumnSchema interface {
	GetName() string
	GetType() string
}

ColumnSchema represents metadata about a column

type Formatter

type Formatter interface {
	// Format converts database rows to the target format
	Format(rows []map[string]interface{}) ([]byte, error)

	// Extension returns the file extension for this format (e.g., ".jsonl", ".csv", ".parquet")
	Extension() string

	// MIMEType returns the MIME type for this format
	MIMEType() string
}

Formatter defines the interface for output format handlers

func GetFormatter

func GetFormatter(format string) Formatter

GetFormatter returns the appropriate formatter based on the format string

func GetFormatterWithCompression

func GetFormatterWithCompression(format string, compression string) Formatter

GetFormatterWithCompression returns the appropriate formatter with compression settings For Parquet, this enables internal compression. For other formats, compression parameter is ignored.

type JSONLFormatter

type JSONLFormatter struct{}

JSONLFormatter handles JSONL (JSON Lines) format output

func NewJSONLFormatter

func NewJSONLFormatter() *JSONLFormatter

NewJSONLFormatter creates a new JSONL formatter

func (*JSONLFormatter) Extension

func (f *JSONLFormatter) Extension() string

Extension returns the file extension for JSONL files

func (*JSONLFormatter) Format

func (f *JSONLFormatter) Format(rows []map[string]interface{}) ([]byte, error)

Format converts rows to JSONL format (one JSON object per line)

func (*JSONLFormatter) MIMEType

func (f *JSONLFormatter) MIMEType() string

MIMEType returns the MIME type for JSONL

type JSONLReader added in v1.5.3

type JSONLReader struct {
	// contains filtered or unexported fields
}

JSONLReader reads JSONL format (one JSON object per line)

func NewJSONLReader added in v1.5.3

func NewJSONLReader(r io.Reader) *JSONLReader

NewJSONLReader creates a new JSONL reader

func NewJSONLReaderWithCloser added in v1.5.3

func NewJSONLReaderWithCloser(r io.ReadCloser) *JSONLReader

NewJSONLReaderWithCloser creates a new JSONL reader with a closable reader

func (*JSONLReader) Close added in v1.5.3

func (r *JSONLReader) Close() error

Close closes the underlying reader if it's closable

func (*JSONLReader) ReadAll added in v1.5.3

func (r *JSONLReader) ReadAll() ([]map[string]interface{}, error)

ReadAll reads all remaining rows from the JSONL stream

func (*JSONLReader) ReadChunk added in v1.5.3

func (r *JSONLReader) ReadChunk(chunkSize int) ([]map[string]interface{}, error)

ReadChunk reads a chunk of rows from the JSONL stream

type JSONLStreamingFormatter added in v1.4.0

type JSONLStreamingFormatter struct{}

JSONLStreamingFormatter handles JSONL format output in streaming mode

func NewJSONLStreamingFormatter added in v1.4.0

func NewJSONLStreamingFormatter() *JSONLStreamingFormatter

NewJSONLStreamingFormatter creates a new JSONL streaming formatter

func (*JSONLStreamingFormatter) Extension added in v1.4.0

func (f *JSONLStreamingFormatter) Extension() string

Extension returns the file extension for JSONL files

func (*JSONLStreamingFormatter) MIMEType added in v1.4.0

func (f *JSONLStreamingFormatter) MIMEType() string

MIMEType returns the MIME type for JSONL

func (*JSONLStreamingFormatter) NewWriter added in v1.4.0

NewWriter creates a new JSONL stream writer

type ParquetFormatter

type ParquetFormatter struct {
	// contains filtered or unexported fields
}

ParquetFormatter handles Parquet format output

func NewParquetFormatter

func NewParquetFormatter() *ParquetFormatter

NewParquetFormatter creates a new Parquet formatter

func NewParquetFormatterWithCompression

func NewParquetFormatterWithCompression(compression string) *ParquetFormatter

NewParquetFormatterWithCompression creates a Parquet formatter with specified compression

func (*ParquetFormatter) Extension

func (f *ParquetFormatter) Extension() string

Extension returns the file extension for Parquet files

func (*ParquetFormatter) Format

func (f *ParquetFormatter) Format(rows []map[string]interface{}) ([]byte, error)

Format converts rows to Parquet format

func (*ParquetFormatter) MIMEType

func (f *ParquetFormatter) MIMEType() string

MIMEType returns the MIME type for Parquet

type ParquetReader added in v1.5.3

type ParquetReader struct {
	// contains filtered or unexported fields
}

ParquetReader reads Parquet format

func NewParquetReader added in v1.5.3

func NewParquetReader(r io.Reader) (*ParquetReader, error)

NewParquetReader creates a new Parquet reader Note: Parquet requires io.ReaderAt, so we read the entire file into memory

func NewParquetReaderWithCloser added in v1.5.3

func NewParquetReaderWithCloser(r io.ReadCloser) (*ParquetReader, error)

NewParquetReaderWithCloser creates a new Parquet reader with a closable reader Note: Parquet requires io.ReaderAt, so we read the entire file into memory

func (*ParquetReader) Close added in v1.5.3

func (r *ParquetReader) Close() error

Close closes the underlying reader

func (*ParquetReader) ReadAll added in v1.5.3

func (r *ParquetReader) ReadAll() ([]map[string]interface{}, error)

ReadAll reads all remaining rows from the Parquet stream

func (*ParquetReader) ReadChunk added in v1.5.3

func (r *ParquetReader) ReadChunk(chunkSize int) ([]map[string]interface{}, error)

ReadChunk reads a chunk of rows from the Parquet stream

type ParquetStreamingFormatter added in v1.4.0

type ParquetStreamingFormatter struct {
	// contains filtered or unexported fields
}

ParquetStreamingFormatter handles Parquet format output in streaming mode

func NewParquetStreamingFormatter added in v1.4.0

func NewParquetStreamingFormatter() *ParquetStreamingFormatter

NewParquetStreamingFormatter creates a new Parquet streaming formatter with default compression

func NewParquetStreamingFormatterWithCompression added in v1.4.0

func NewParquetStreamingFormatterWithCompression(compression string) *ParquetStreamingFormatter

NewParquetStreamingFormatterWithCompression creates a Parquet streaming formatter with specified compression

func (*ParquetStreamingFormatter) Extension added in v1.4.0

func (f *ParquetStreamingFormatter) Extension() string

Extension returns the file extension for Parquet files

func (*ParquetStreamingFormatter) MIMEType added in v1.4.0

func (f *ParquetStreamingFormatter) MIMEType() string

MIMEType returns the MIME type for Parquet

func (*ParquetStreamingFormatter) NewWriter added in v1.4.0

func (f *ParquetStreamingFormatter) NewWriter(w io.Writer, tableSchema TableSchema) (StreamWriter, error)

NewWriter creates a new Parquet stream writer For Parquet, we need to build the schema from the TableSchema

type StreamWriter added in v1.4.0

type StreamWriter interface {
	// WriteChunk writes a chunk of rows to the output
	WriteChunk(rows []map[string]interface{}) error

	// Close finalizes the output (writes footers, flushes buffers, etc.)
	Close() error
}

StreamWriter handles writing rows in a streaming fashion

type StreamingFormatter added in v1.4.0

type StreamingFormatter interface {
	// NewWriter creates a new StreamWriter that writes to the given io.Writer
	// schema provides column information needed for formats like CSV (headers) and Parquet (schema)
	NewWriter(w io.Writer, schema TableSchema) (StreamWriter, error)

	// Extension returns the file extension for this format (e.g., ".jsonl", ".csv", ".parquet")
	Extension() string

	// MIMEType returns the MIME type for this format
	MIMEType() string
}

StreamingFormatter defines the interface for streaming output format handlers

func GetStreamingFormatter added in v1.4.0

func GetStreamingFormatter(format string) StreamingFormatter

GetStreamingFormatter returns the appropriate streaming formatter based on the format string

type TableSchema added in v1.4.0

type TableSchema interface {
	GetColumns() []ColumnSchema
}

TableSchema represents the structure needed for streaming formatters It's a simplified interface to avoid circular dependencies with cmd package

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL