Documentation
¶
Index ¶
- Constants
- Variables
- func UsesInternalCompression(format string) bool
- type CSVFormatter
- type CSVReader
- type CSVStreamingFormatter
- type ColumnSchema
- type Formatter
- type JSONLFormatter
- type JSONLReader
- type JSONLStreamingFormatter
- type ParquetFormatter
- type ParquetReader
- type ParquetStreamingFormatter
- type StreamWriter
- type StreamingFormatter
- type TableSchema
Constants ¶
const ( FormatJSONL = "jsonl" FormatCSV = "csv" FormatParquet = "parquet" )
Format type constants
Variables ¶
var ErrNoColumns = errors.New("table schema has no columns")
ErrNoColumns is returned when a table schema has no columns
Functions ¶
func UsesInternalCompression ¶
UsesInternalCompression returns true if the format handles compression internally
Types ¶
type CSVFormatter ¶
type CSVFormatter struct{}
CSVFormatter handles CSV format output
func NewCSVFormatter ¶
func NewCSVFormatter() *CSVFormatter
NewCSVFormatter creates a new CSV formatter
func (*CSVFormatter) Extension ¶
func (f *CSVFormatter) Extension() string
Extension returns the file extension for CSV files
func (*CSVFormatter) Format ¶
func (f *CSVFormatter) Format(rows []map[string]interface{}) ([]byte, error)
Format converts rows to CSV format
func (*CSVFormatter) MIMEType ¶
func (f *CSVFormatter) MIMEType() string
MIMEType returns the MIME type for CSV
type CSVReader ¶ added in v1.5.3
type CSVReader struct {
// contains filtered or unexported fields
}
CSVReader reads CSV format with header detection
func NewCSVReader ¶ added in v1.5.3
NewCSVReader creates a new CSV reader
func NewCSVReaderWithCloser ¶ added in v1.5.3
func NewCSVReaderWithCloser(r io.ReadCloser) (*CSVReader, error)
NewCSVReaderWithCloser creates a new CSV reader with a closable reader
type CSVStreamingFormatter ¶ added in v1.4.0
type CSVStreamingFormatter struct{}
CSVStreamingFormatter handles CSV format output in streaming mode
func NewCSVStreamingFormatter ¶ added in v1.4.0
func NewCSVStreamingFormatter() *CSVStreamingFormatter
NewCSVStreamingFormatter creates a new CSV streaming formatter
func (*CSVStreamingFormatter) Extension ¶ added in v1.4.0
func (f *CSVStreamingFormatter) Extension() string
Extension returns the file extension for CSV files
func (*CSVStreamingFormatter) MIMEType ¶ added in v1.4.0
func (f *CSVStreamingFormatter) MIMEType() string
MIMEType returns the MIME type for CSV
func (*CSVStreamingFormatter) NewWriter ¶ added in v1.4.0
func (f *CSVStreamingFormatter) NewWriter(w io.Writer, schema TableSchema) (StreamWriter, error)
NewWriter creates a new CSV stream writer
type ColumnSchema ¶ added in v1.4.0
ColumnSchema represents metadata about a column
type Formatter ¶
type Formatter interface {
// Format converts database rows to the target format
Format(rows []map[string]interface{}) ([]byte, error)
// Extension returns the file extension for this format (e.g., ".jsonl", ".csv", ".parquet")
Extension() string
// MIMEType returns the MIME type for this format
MIMEType() string
}
Formatter defines the interface for output format handlers
func GetFormatter ¶
GetFormatter returns the appropriate formatter based on the format string
func GetFormatterWithCompression ¶
GetFormatterWithCompression returns the appropriate formatter with compression settings For Parquet, this enables internal compression. For other formats, compression parameter is ignored.
type JSONLFormatter ¶
type JSONLFormatter struct{}
JSONLFormatter handles JSONL (JSON Lines) format output
func NewJSONLFormatter ¶
func NewJSONLFormatter() *JSONLFormatter
NewJSONLFormatter creates a new JSONL formatter
func (*JSONLFormatter) Extension ¶
func (f *JSONLFormatter) Extension() string
Extension returns the file extension for JSONL files
func (*JSONLFormatter) Format ¶
func (f *JSONLFormatter) Format(rows []map[string]interface{}) ([]byte, error)
Format converts rows to JSONL format (one JSON object per line)
func (*JSONLFormatter) MIMEType ¶
func (f *JSONLFormatter) MIMEType() string
MIMEType returns the MIME type for JSONL
type JSONLReader ¶ added in v1.5.3
type JSONLReader struct {
// contains filtered or unexported fields
}
JSONLReader reads JSONL format (one JSON object per line)
func NewJSONLReader ¶ added in v1.5.3
func NewJSONLReader(r io.Reader) *JSONLReader
NewJSONLReader creates a new JSONL reader
func NewJSONLReaderWithCloser ¶ added in v1.5.3
func NewJSONLReaderWithCloser(r io.ReadCloser) *JSONLReader
NewJSONLReaderWithCloser creates a new JSONL reader with a closable reader
func (*JSONLReader) Close ¶ added in v1.5.3
func (r *JSONLReader) Close() error
Close closes the underlying reader if it's closable
func (*JSONLReader) ReadAll ¶ added in v1.5.3
func (r *JSONLReader) ReadAll() ([]map[string]interface{}, error)
ReadAll reads all remaining rows from the JSONL stream
type JSONLStreamingFormatter ¶ added in v1.4.0
type JSONLStreamingFormatter struct{}
JSONLStreamingFormatter handles JSONL format output in streaming mode
func NewJSONLStreamingFormatter ¶ added in v1.4.0
func NewJSONLStreamingFormatter() *JSONLStreamingFormatter
NewJSONLStreamingFormatter creates a new JSONL streaming formatter
func (*JSONLStreamingFormatter) Extension ¶ added in v1.4.0
func (f *JSONLStreamingFormatter) Extension() string
Extension returns the file extension for JSONL files
func (*JSONLStreamingFormatter) MIMEType ¶ added in v1.4.0
func (f *JSONLStreamingFormatter) MIMEType() string
MIMEType returns the MIME type for JSONL
func (*JSONLStreamingFormatter) NewWriter ¶ added in v1.4.0
func (f *JSONLStreamingFormatter) NewWriter(w io.Writer, _ TableSchema) (StreamWriter, error)
NewWriter creates a new JSONL stream writer
type ParquetFormatter ¶
type ParquetFormatter struct {
// contains filtered or unexported fields
}
ParquetFormatter handles Parquet format output
func NewParquetFormatter ¶
func NewParquetFormatter() *ParquetFormatter
NewParquetFormatter creates a new Parquet formatter
func NewParquetFormatterWithCompression ¶
func NewParquetFormatterWithCompression(compression string) *ParquetFormatter
NewParquetFormatterWithCompression creates a Parquet formatter with specified compression
func (*ParquetFormatter) Extension ¶
func (f *ParquetFormatter) Extension() string
Extension returns the file extension for Parquet files
func (*ParquetFormatter) Format ¶
func (f *ParquetFormatter) Format(rows []map[string]interface{}) ([]byte, error)
Format converts rows to Parquet format
func (*ParquetFormatter) MIMEType ¶
func (f *ParquetFormatter) MIMEType() string
MIMEType returns the MIME type for Parquet
type ParquetReader ¶ added in v1.5.3
type ParquetReader struct {
// contains filtered or unexported fields
}
ParquetReader reads Parquet format
func NewParquetReader ¶ added in v1.5.3
func NewParquetReader(r io.Reader) (*ParquetReader, error)
NewParquetReader creates a new Parquet reader Note: Parquet requires io.ReaderAt, so we read the entire file into memory
func NewParquetReaderWithCloser ¶ added in v1.5.3
func NewParquetReaderWithCloser(r io.ReadCloser) (*ParquetReader, error)
NewParquetReaderWithCloser creates a new Parquet reader with a closable reader Note: Parquet requires io.ReaderAt, so we read the entire file into memory
func (*ParquetReader) Close ¶ added in v1.5.3
func (r *ParquetReader) Close() error
Close closes the underlying reader
func (*ParquetReader) ReadAll ¶ added in v1.5.3
func (r *ParquetReader) ReadAll() ([]map[string]interface{}, error)
ReadAll reads all remaining rows from the Parquet stream
type ParquetStreamingFormatter ¶ added in v1.4.0
type ParquetStreamingFormatter struct {
// contains filtered or unexported fields
}
ParquetStreamingFormatter handles Parquet format output in streaming mode
func NewParquetStreamingFormatter ¶ added in v1.4.0
func NewParquetStreamingFormatter() *ParquetStreamingFormatter
NewParquetStreamingFormatter creates a new Parquet streaming formatter with default compression
func NewParquetStreamingFormatterWithCompression ¶ added in v1.4.0
func NewParquetStreamingFormatterWithCompression(compression string) *ParquetStreamingFormatter
NewParquetStreamingFormatterWithCompression creates a Parquet streaming formatter with specified compression
func (*ParquetStreamingFormatter) Extension ¶ added in v1.4.0
func (f *ParquetStreamingFormatter) Extension() string
Extension returns the file extension for Parquet files
func (*ParquetStreamingFormatter) MIMEType ¶ added in v1.4.0
func (f *ParquetStreamingFormatter) MIMEType() string
MIMEType returns the MIME type for Parquet
func (*ParquetStreamingFormatter) NewWriter ¶ added in v1.4.0
func (f *ParquetStreamingFormatter) NewWriter(w io.Writer, tableSchema TableSchema) (StreamWriter, error)
NewWriter creates a new Parquet stream writer For Parquet, we need to build the schema from the TableSchema
type StreamWriter ¶ added in v1.4.0
type StreamWriter interface {
// WriteChunk writes a chunk of rows to the output
WriteChunk(rows []map[string]interface{}) error
// Close finalizes the output (writes footers, flushes buffers, etc.)
Close() error
}
StreamWriter handles writing rows in a streaming fashion
type StreamingFormatter ¶ added in v1.4.0
type StreamingFormatter interface {
// NewWriter creates a new StreamWriter that writes to the given io.Writer
// schema provides column information needed for formats like CSV (headers) and Parquet (schema)
NewWriter(w io.Writer, schema TableSchema) (StreamWriter, error)
// Extension returns the file extension for this format (e.g., ".jsonl", ".csv", ".parquet")
Extension() string
// MIMEType returns the MIME type for this format
MIMEType() string
}
StreamingFormatter defines the interface for streaming output format handlers
func GetStreamingFormatter ¶ added in v1.4.0
func GetStreamingFormatter(format string) StreamingFormatter
GetStreamingFormatter returns the appropriate streaming formatter based on the format string
type TableSchema ¶ added in v1.4.0
type TableSchema interface {
GetColumns() []ColumnSchema
}
TableSchema represents the structure needed for streaming formatters It's a simplified interface to avoid circular dependencies with cmd package