file

package
v0.22.5 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 12, 2026 License: Apache-2.0 Imports: 37 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func AddFileWriter added in v0.11.0

func AddFileWriter(format string, ds *data.DataStream, writer io.WriteCloser) (data.DataWriter, error)

func BuildFullPath added in v0.10.1

func BuildFullPath(parsed *url.URL, filename string) (*url.URL, error)

func CreateCompressedWriter added in v0.11.0

func CreateCompressedWriter(bufWriter io.WriteCloser, compressionType string, format string) (io.WriteCloser, error)

func GetIo

func GetIo(ctx context.Context, counting io.WriteCloser, filePath *url.URL) (io.WriteCloser, error)

func GetPathAndIO added in v0.11.0

func GetPathAndIO(ctx context.Context, path *url.URL, counter io.WriteCloser, compression, format string) (io.WriteCloser, error)

func NewProgressBar added in v0.11.1

func NewProgressBar() *progressbar.ProgressBar

func ValueToString added in v0.15.0

func ValueToString(value any, col data.Column) (string, error)

func WriteEmptyFile added in v0.22.0

func WriteEmptyFile(format string, writer io.Writer) error

Types

type ArrowBatchWriter added in v0.21.0

type ArrowBatchWriter struct {
	// contains filtered or unexported fields
}

func (*ArrowBatchWriter) WriteBatch added in v0.21.0

func (ab *ArrowBatchWriter) WriteBatch(batch data.Batch) error

type ArrowDataWriter added in v0.21.0

type ArrowDataWriter struct {
	// contains filtered or unexported fields
}

func NewArrowDataWriter added in v0.21.0

func NewArrowDataWriter(datastream *data.DataStream, w io.Writer) *ArrowDataWriter

func (*ArrowDataWriter) Close added in v0.21.0

func (aw *ArrowDataWriter) Close() error

func (*ArrowDataWriter) CreateBatchWriter added in v0.21.0

func (aw *ArrowDataWriter) CreateBatchWriter() data.BatchWriter

func (*ArrowDataWriter) Flush added in v0.21.0

func (aw *ArrowDataWriter) Flush() error

type AzureBlob added in v0.16.0

type AzureBlob struct {
	*AzureBlobConfig
	// contains filtered or unexported fields
}

func (*AzureBlob) Close added in v0.16.0

func (a *AzureBlob) Close() error

func (*AzureBlob) Write added in v0.16.0

func (a *AzureBlob) Write(p []byte) (n int, err error)

type AzureBlobConfig added in v0.16.0

type AzureBlobConfig struct {
	// contains filtered or unexported fields
}

func ParseAzureBlobURL added in v0.16.0

func ParseAzureBlobURL(blobUrl *url.URL) (*AzureBlobConfig, error)

func ParseAzurite added in v0.16.0

func ParseAzurite(blobUrl *url.URL) (*AzureBlobConfig, error)

func (*AzureBlobConfig) GetWriter added in v0.16.0

func (a *AzureBlobConfig) GetWriter(ctx context.Context) (*AzureBlob, error)

type BufferedWriter added in v0.16.0

type BufferedWriter struct {
	// contains filtered or unexported fields
}

func NewBufferedWriter added in v0.16.0

func NewBufferedWriter(writer io.Writer, closers ...io.WriteCloser) *BufferedWriter

func (*BufferedWriter) Close added in v0.16.0

func (b *BufferedWriter) Close() error

func (*BufferedWriter) Write added in v0.16.0

func (b *BufferedWriter) Write(p []byte) (n int, err error)

type CSVBatchWriter added in v0.13.0

type CSVBatchWriter struct {
	// contains filtered or unexported fields
}

func (*CSVBatchWriter) WriteBatch added in v0.13.0

func (cb *CSVBatchWriter) WriteBatch(batch data.Batch) error

type CSVDataWriter

type CSVDataWriter struct {
	// contains filtered or unexported fields
}

func AddCSV

func AddCSV(ds *data.DataStream, writer io.WriteCloser) *CSVDataWriter

func NewCSVDataWriter

func NewCSVDataWriter(datastream *data.DataStream, writer io.WriteCloser) *CSVDataWriter

func (*CSVDataWriter) Close

func (cw *CSVDataWriter) Close() error

func (*CSVDataWriter) CreateBatchWriter added in v0.13.0

func (cw *CSVDataWriter) CreateBatchWriter() data.BatchWriter

func (*CSVDataWriter) Flush

func (cw *CSVDataWriter) Flush() error

func (*CSVDataWriter) ProcessRow added in v0.13.0

func (cw *CSVDataWriter) ProcessRow(row []any) ([]string, error)

type ColumnMetadata added in v0.18.0

type ColumnMetadata struct {
	Name   string `json:"name"`
	Type   string `json:"type"`
	Length int32  `json:"length"`
}

type DecimalType added in v0.6.0

type DecimalType int
const (
	UnknownDecimal DecimalType = iota
	Int32Decimal
	Int64Decimal
	ByteArrayDecimal
)

type JSONLBatchWriter added in v0.13.0

type JSONLBatchWriter struct {
	// contains filtered or unexported fields
}

func (*JSONLBatchWriter) WriteBatch added in v0.13.0

func (bw *JSONLBatchWriter) WriteBatch(batch data.Batch) error

type JSONLWriter

type JSONLWriter struct {
	// contains filtered or unexported fields
}

func AddJSONL

func AddJSONL(ds *data.DataStream, writer io.WriteCloser) *JSONLWriter

func NewJSONLWriter added in v0.13.0

func NewJSONLWriter(ds *data.DataStream, writer io.Writer) *JSONLWriter

func (*JSONLWriter) Close

func (w *JSONLWriter) Close() error

func (*JSONLWriter) CreateBatchWriter added in v0.13.0

func (w *JSONLWriter) CreateBatchWriter() data.BatchWriter

func (*JSONLWriter) Flush

func (w *JSONLWriter) Flush() error

func (*JSONLWriter) ProcessRow added in v0.13.0

func (w *JSONLWriter) ProcessRow(row []any) ([]byte, error)

type MappedType added in v0.6.0

type MappedType struct {
	ParquetType parquet.Type
	LogicalType schema.LogicalType
	GoType      reflect.Type
}

type ParquetBatchWriter added in v0.13.0

type ParquetBatchWriter struct {
	// contains filtered or unexported fields
}

func (*ParquetBatchWriter) WriteBatch added in v0.13.0

func (pb *ParquetBatchWriter) WriteBatch(batch data.Batch) error

type ParquetDataWriter

type ParquetDataWriter struct {
	// contains filtered or unexported fields
}

func AddParquet

func AddParquet(ds *data.DataStream, writer io.WriteCloser) *ParquetDataWriter

func NewParquetDataWriter

func NewParquetDataWriter(datastream *data.DataStream, ioWriter io.Writer) *ParquetDataWriter

func (*ParquetDataWriter) Close

func (pw *ParquetDataWriter) Close() error

func (*ParquetDataWriter) CreateBatchWriter added in v0.13.0

func (pw *ParquetDataWriter) CreateBatchWriter() data.BatchWriter

func (*ParquetDataWriter) Flush

func (pw *ParquetDataWriter) Flush() error

type ParquetDecimal added in v0.6.0

type ParquetDecimal struct {
	Type         DecimalType // Discriminator to indicate the active type
	Int32Val     int32
	Int64Val     int64
	ByteArrayVal []byte
}

type WriteCloseBuffer added in v0.16.0

type WriteCloseBuffer struct {
	*bytes.Buffer
}

func NewWriteCloseBuffer added in v0.16.0

func NewWriteCloseBuffer(buf *bytes.Buffer) *WriteCloseBuffer

NewWriteCloseBuffer initializes and returns a new WriteCloseBuffer.

func (*WriteCloseBuffer) Close added in v0.16.0

func (wcb *WriteCloseBuffer) Close() error

func (*WriteCloseBuffer) Write added in v0.16.0

func (wcb *WriteCloseBuffer) Write(p []byte) (n int, err error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL