Documentation
¶
Index ¶
Constants ¶
View Source
const ( OrcFormatName = "ORC" ParquetFormatName = "Parquet" )
Variables ¶
Functions ¶
This section is empty.
Types ¶
type FileReader ¶
type FileReader interface {
MetadataReader
Read(n int) ([]*InventoryObject, error)
}
type IReader ¶
type IReader interface {
GetFileReader(format string, bucket string, key string) (FileReader, error)
GetMetadataReader(format string, bucket string, key string) (MetadataReader, error)
}
type InventoryObject ¶
type InventoryObject struct {
Bucket string
Key string
IsLatest bool
IsDeleteMarker bool
Size int64
LastModified *time.Time
Checksum string
}
func NewInventoryObject ¶
func NewInventoryObject() *InventoryObject
func (*InventoryObject) GetPhysicalAddress ¶
func (o *InventoryObject) GetPhysicalAddress() string
type MetadataReader ¶
type OrcFile ¶
func DownloadOrc ¶
func DownloadOrc(ctx context.Context, svc s3iface.S3API, logger logging.Logger, bucket string, key string, tailOnly bool) (*OrcFile, error)
DownloadOrc downloads a file from s3 and returns a ReaderSeeker to it. If tailOnly is set to true, download only the tail (metadata+footer) by trying the last `orcInitialReadSize` bytes of the file. Then, check the last byte to see if the whole tail was downloaded. If not, download again with the actual tail length.
type OrcInventoryFileReader ¶
type OrcInventoryFileReader struct {
// contains filtered or unexported fields
}
func (*OrcInventoryFileReader) Close ¶
func (r *OrcInventoryFileReader) Close() error
func (*OrcInventoryFileReader) FirstObjectKey ¶
func (r *OrcInventoryFileReader) FirstObjectKey() string
func (*OrcInventoryFileReader) GetNumRows ¶
func (r *OrcInventoryFileReader) GetNumRows() int64
func (*OrcInventoryFileReader) LastObjectKey ¶
func (r *OrcInventoryFileReader) LastObjectKey() string
func (*OrcInventoryFileReader) Read ¶
func (r *OrcInventoryFileReader) Read(n int) ([]*InventoryObject, error)
type ParquetInventoryFileReader ¶
type ParquetInventoryFileReader struct {
*reader.ParquetReader
// contains filtered or unexported fields
}
func NewParquetInventoryFileReader ¶
func NewParquetInventoryFileReader(parquetReader *reader.ParquetReader) (*ParquetInventoryFileReader, error)
func (*ParquetInventoryFileReader) Close ¶
func (p *ParquetInventoryFileReader) Close() error
func (*ParquetInventoryFileReader) FirstObjectKey ¶
func (p *ParquetInventoryFileReader) FirstObjectKey() string
func (*ParquetInventoryFileReader) LastObjectKey ¶
func (p *ParquetInventoryFileReader) LastObjectKey() string
func (*ParquetInventoryFileReader) Read ¶
func (p *ParquetInventoryFileReader) Read(n int) ([]*InventoryObject, error)
type Reader ¶
type Reader struct {
// contains filtered or unexported fields
}
func (*Reader) GetFileReader ¶
func (*Reader) GetMetadataReader ¶
Click to show internal directories.
Click to hide internal directories.