dataexporter

package
v1.51.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 11, 2025 License: MIT Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source
const (
	DefaultSizeLimit      = "5TB"
	DefaultPartSize       = "500GB"
	DefaultChunkSize      = "250MB"
	DefaultBufferSize     = "32MB"
	DefaultReportPeriod   = time.Second
	DefaultConcurrentJobs = 10
)

Variables

This section is empty.

Functions

func GetDirSize

func GetDirSize(path string) (datasize.ByteSize, error)

Types

type DeleteOption

type DeleteOption func(*DeleteOptions)

func WithConcurrentDeleteJobs

func WithConcurrentDeleteJobs(concurrentJobs int) DeleteOption

type DeleteOptions

type DeleteOptions struct {
	ConcurrentJobs int
}

type Exporter

type Exporter interface {
	Provider() Provider
	Upload(dir, bucket, name string, opts ...UploadOption) error
	Delete(bucket, name string, opts ...DeleteOption) error
}

func FromProvider

func FromProvider(p Provider) (Exporter, error)

type GcsExporter

type GcsExporter struct {
	// contains filtered or unexported fields
}

func NewGcsExporter

func NewGcsExporter() (*GcsExporter, error)

func (*GcsExporter) Delete

func (gcs *GcsExporter) Delete(bucket, name string, opts ...DeleteOption) error

func (*GcsExporter) Provider

func (gcs *GcsExporter) Provider() Provider

func (*GcsExporter) Upload

func (gcs *GcsExporter) Upload(dir, bucket, name string, opts ...UploadOption) error

type Provider

type Provider string
const (
	GCS                   Provider = "gcs"
	CompositionBatchLimit          = 32
)

type UploadOption

type UploadOption func(*UploadOptions)

func WithBufferSize

func WithBufferSize(size string) UploadOption

func WithChunkSize

func WithChunkSize(size string) UploadOption

func WithConcurrentUploadJobs

func WithConcurrentUploadJobs(concurrentJobs int) UploadOption

func WithPartSize

func WithPartSize(size string) UploadOption

func WithReportPeriod

func WithReportPeriod(period time.Duration) UploadOption

func WithSizeLimit

func WithSizeLimit(size string) UploadOption

type UploadOptions

type UploadOptions struct {
	PartSize       datasize.ByteSize
	ChunkSize      datasize.ByteSize
	SizeLimit      datasize.ByteSize
	ReportPeriod   time.Duration
	ConcurrentJobs int
	BufferSize     datasize.ByteSize
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL