Documentation
¶
Index ¶
- func DecodeFromString(s string, v interface{}) error
- func EncodeToString(v interface{}) (string, error)
- func Marshal(v interface{}) ([]byte, error)
- func NewDecoder(r io.Reader) *json.Decoder
- func NewDefaultParquetWriteOptions() pqarrow.ArrowWriterProperties
- func NewDefaultParquetWriterProperties() *parquet.WriterProperties
- func NewEncoder(w io.Writer) *json.Encoder
- func PrettyPrint(v interface{}) (string, error)
- func Unmarshal(data []byte, v interface{}) error
- func ValidateJSON(data []byte) error
- func ValidateJSONString(s string) error
- type AvroReadOptions
- type AvroReader
- type CSVReadOptions
- type CSVReader
- type CSVWriteOptions
- type CSVWriter
- type DuckDBReadOptions
- type DuckDBRecordReader
- type IPCRecordReader
- type IPCRecordWriter
- type JSONReadOptions
- type JSONReader
- type JSONWriter
- type ParquetReadOptions
- type ParquetReader
- type ParquetWriter
- type SchemaReader
- type SchemaWriter
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func DecodeFromString ¶
DecodeFromString decodes JSON data from a string into the provided value.
func EncodeToString ¶
EncodeToString marshals and encodes the provided value directly into a string.
func NewDecoder ¶
NewDecoder initializes and returns a new JSON Decoder.
func NewDefaultParquetWriteOptions ¶
func NewDefaultParquetWriteOptions() pqarrow.ArrowWriterProperties
NewDefaultParquetWriteOptions returns default write options for Parquet files.
func NewDefaultParquetWriterProperties ¶
func NewDefaultParquetWriterProperties() *parquet.WriterProperties
NewDefaultParquetWriterProperties returns default writer properties.
func NewEncoder ¶
NewEncoder initializes and returns a new JSON Encoder.
func PrettyPrint ¶
PrettyPrint marshals the provided value into a pretty-printed JSON string.
func ValidateJSON ¶
ValidateJSON checks if the provided byte slice is valid JSON.
func ValidateJSONString ¶
ValidateJSONString checks if the provided string is valid JSON.
Types ¶
type AvroReadOptions ¶
type AvroReadOptions struct {
ChunkSize int64
}
AvroReadOptions defines options for reading Avro files.
type AvroReader ¶
type AvroReader struct {
// contains filtered or unexported fields
}
AvroReader reads records from Avro files and implements the Reader interface.
func NewAvroReader ¶
func NewAvroReader(ctx context.Context, filePath string, opts *AvroReadOptions) (*AvroReader, error)
NewAvroReader creates a new reader for reading records from an Avro file.
func (*AvroReader) Close ¶
func (r *AvroReader) Close() error
Close releases resources associated with the Avro reader.
func (*AvroReader) Read ¶
func (r *AvroReader) Read() (arrow.Record, error)
Read reads the next record from the Avro file.
func (*AvroReader) Schema ¶
func (r *AvroReader) Schema() *arrow.Schema
Schema returns the schema of the records being read from the Avro file.
type CSVReadOptions ¶
type CSVReadOptions struct {
ChunkSize int64
Delimiter rune
HasHeader bool
NullValues []string
StringsCanBeNull bool
}
CSVReadOptions defines options for reading CSV files.
type CSVReader ¶
type CSVReader struct {
// contains filtered or unexported fields
}
CSVReader reads records from a CSV file and implements the Reader interface.
func NewCSVReader ¶
func NewCSVReader(ctx context.Context, filePath string, schema *arrow.Schema, opts *CSVReadOptions) (*CSVReader, error)
NewCSVReader creates a new CSV reader for reading records from a CSV file.
type CSVWriteOptions ¶
type CSVWriteOptions struct {
Delimiter rune
IncludeHeader bool
NullValue string
StringsReplacer *strings.Replacer
BoolFormatter func(bool) string
}
CSVWriteOptions defines options for writing CSV files.
type CSVWriter ¶
type CSVWriter struct {
// contains filtered or unexported fields
}
CSVWriter writes records to a CSV file and implements the Writer interface.
func NewCSVWriter ¶
func NewCSVWriter(ctx context.Context, filePath string, schema *arrow.Schema, opts *CSVWriteOptions) (*CSVWriter, error)
NewCSVWriter creates a new CSV writer for writing records to a CSV file.
type DuckDBReadOptions ¶
type DuckDBReadOptions struct {
FileType string
FilePath string
Extensions []duckdb.DuckDBExtension
}
ReadOptions defines options for reading various file types using DuckDB.
type DuckDBRecordReader ¶
type DuckDBRecordReader struct {
// contains filtered or unexported fields
}
DuckDBRecordReader reads records from DuckDB queries and implements the SchemaReader interface.
func NewDuckDBRecordReader ¶
func NewDuckDBRecordReader(ctx context.Context, runner *duckdb.DuckDBReader, query string) (*DuckDBRecordReader, error)
NewDuckDBRecordReader creates a new reader for reading records from a DuckDB query.
func (*DuckDBRecordReader) Close ¶
func (r *DuckDBRecordReader) Close() error
Close releases resources associated with the DuckDB reader.
func (*DuckDBRecordReader) Read ¶
func (r *DuckDBRecordReader) Read() (arrow.Record, error)
Read reads the next record from the DuckDB query result.
func (*DuckDBRecordReader) Schema ¶
func (r *DuckDBRecordReader) Schema() *arrow.Schema
Schema returns the schema of the records being read from the DuckDB query.
type IPCRecordReader ¶
type IPCRecordReader struct {
// contains filtered or unexported fields
}
IPCRecordReader implements SchemaReader for reading records from IPC files.
func (*IPCRecordReader) Close ¶
func (r *IPCRecordReader) Close() error
Close releases resources associated with the IPC reader.
func (*IPCRecordReader) Read ¶
func (r *IPCRecordReader) Read() (arrow.Record, error)
Read reads the next record from the IPC file.
func (*IPCRecordReader) Schema ¶
func (r *IPCRecordReader) Schema() *arrow.Schema
Schema returns the schema of the records being read from the IPC file.
type IPCRecordWriter ¶
type IPCRecordWriter struct {
// contains filtered or unexported fields
}
IPCRecordWriter implements SchemaWriter for writing records to IPC files.
func (*IPCRecordWriter) Close ¶
func (w *IPCRecordWriter) Close() error
Close closes the IPC writer.
func (*IPCRecordWriter) Schema ¶
func (w *IPCRecordWriter) Schema() *arrow.Schema
Schema returns the schema of the records being written to the IPC file.
type JSONReadOptions ¶
type JSONReadOptions struct {
ChunkSize int
}
JSONReadOptions defines options for reading JSON files.
type JSONReader ¶
type JSONReader struct {
// contains filtered or unexported fields
}
JSONReader reads records from a JSON file and implements the Reader interface.
func NewJSONReader ¶
func NewJSONReader(ctx context.Context, filePath string, schema *arrow.Schema, opts *JSONReadOptions) (*JSONReader, error)
NewJSONReader creates a new reader for reading records from a JSON file.
func (*JSONReader) Close ¶
func (r *JSONReader) Close() error
Close releases resources associated with the JSON reader.
func (*JSONReader) Read ¶
func (r *JSONReader) Read() (arrow.Record, error)
Read reads the next record from the JSON file.
func (*JSONReader) Schema ¶
func (r *JSONReader) Schema() *arrow.Schema
Schema returns the schema of the records being read from the JSON file.
type JSONWriter ¶
type JSONWriter struct {
// contains filtered or unexported fields
}
JSONWriter writes records to a JSON file and implements the Writer interface.
func NewJSONWriter ¶
func NewJSONWriter(ctx context.Context, filePath string) (*JSONWriter, error)
NewJSONWriter creates a new writer for writing records to a JSON file.
type ParquetReadOptions ¶
type ParquetReadOptions struct {
MemoryMap bool
ColumnIndices []int
RowGroups []int
Parallel bool
ChunkSize int64
}
ReadOptions defines options for reading Parquet files.
type ParquetReader ¶
type ParquetReader struct {
// contains filtered or unexported fields
}
ParquetReader reads Parquet files and implements the Reader interface.
func NewParquetReader ¶
func NewParquetReader(ctx context.Context, filePath string, opts *ParquetReadOptions) (*ParquetReader, error)
NewParquetReader creates a new Parquet file reader.
func (*ParquetReader) Close ¶
func (p *ParquetReader) Close() error
func (*ParquetReader) Schema ¶
func (p *ParquetReader) Schema() *arrow.Schema
type ParquetWriter ¶
type ParquetWriter struct {
// contains filtered or unexported fields
}
ParquetWriter writes records to Parquet files.
func NewParquetWriter ¶
func NewParquetWriter( filePath string, schema *arrow.Schema, parquetWriterProps *parquet.WriterProperties, ) (*ParquetWriter, error)
NewParquetWriter creates a new Parquet file writer.
func (*ParquetWriter) Close ¶
func (p *ParquetWriter) Close() error
type SchemaReader ¶
SchemaReader is an interface that extends arrio.Reader to include a Schema method.
func NewIPCRecordReader ¶
func NewIPCRecordReader(ctx context.Context, filePath string) (SchemaReader, error)
NewIPCRecordReader creates a new reader for reading records from an IPC file.
func ReadFileStream ¶
func ReadFileStream(ctx context.Context, opts *DuckDBReadOptions) (SchemaReader, error)
ReadFileStream reads data from a file using DuckDB based on the specified file type and returns a SchemaReader.
type SchemaWriter ¶
SchemaWriter is an interface that extends arrio.Writer to include a Schema method.
func NewIPCRecordWriter ¶
func NewIPCRecordWriter(ctx context.Context, filePath string, schema *arrow.Schema) (SchemaWriter, error)
NewIPCRecordWriter creates a new writer for writing records to an IPC file.
Source Files