utils

func ArrowBatchToJSON ¶

func ArrowBatchToJSON(batch arrow.Record) (string, error)

ArrowBatchToJSON converts an Arrow record batch to a JSON string.

func ArrowRecordToString ¶

func ArrowRecordToString(record arrow.Record) (string, error)

ArrowRecordToString converts an Arrow record to a string representation.

func CloneSourceStream ¶

func CloneSourceStream(sourceChan <-chan arrow.Record, numClones int) ([]<-chan arrow.Record, error)

CloneSourceStream creates multiple clones of an Arrow record source channel.

func ConvertSchemaSPB ¶

func ConvertSchemaSPB(schema *arrow.Schema) *storagepb.ProtoSchema

func DecodeToMap ¶

func DecodeToMap(s *pb.Struct) map[string]interface{}

DecodeToMap decodes a protobuf Struct to a Go map.

func IpcReaderToChannel ¶

func IpcReaderToChannel(reader *ipc.Reader) (<-chan arrow.Record, error)

IpcReaderToChannel reads records from an IPC reader and sends them to a channel.

func LoadEnv ¶

func LoadEnv()

func MapArrowTypeToProtoType ¶

func MapArrowTypeToProtoType(dataType arrow.DataType) *descriptorpb.FieldDescriptorProto_Type

MapArrowTypeToProtoType maps an Arrow data type to a protobuf field type.

func PrintRecordBatch ¶

func PrintRecordBatch(record arrow.Record) error

PrintRecordBatch prints the contents of an Arrow record batch.

func ProcessStreams ¶

func ProcessStreams(readErrChan <-chan error, writeErrChan <-chan error) error

ProcessStreams processes errors from read and write error channels concurrently.