client

package
v0.1.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 7, 2025 License: AGPL-3.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// DefaultChunkSize is the default chunk size for downloads (5MB)
	DefaultChunkSize = 5 * 1024 * 1024
)
View Source 
const (
	// MaxChunkSize is the maximum chunk size for streaming downloads (5MB)
	MaxChunkSize = 5 * 1024 * 1024
)

Variables

This section is empty.

Functions

This section is empty.

Types

type AggregateOptions added in v0.1.3 

type AggregateOptions struct {
	MaxParallelism   int              // Maximum number of concurrent downloads/uploads (default: 4)
	MaxRetries       int              // Maximum number of retry attempts per operation (default: 3)
	ProgressCallback ProgressCallback // Optional progress callback
}

AggregateOptions configures the aggregation behavior

func DefaultAggregateOptions added in v0.1.3 

func DefaultAggregateOptions() *AggregateOptions

DefaultAggregateOptions returns sensible default options for aggregation

type Client 

type Client struct {
	// contains filtered or unexported fields
}

func NewClient 

func NewClient(baseURL string) *Client

func (*Client) AddBucket 

func (c *Client) AddBucket(ctx context.Context, bucket *bucket.BucketInfo) error

func (*Client) AddDatas3t 

func (c *Client) AddDatas3t(ctx context.Context, datas3t *datas3t.AddDatas3tRequest) error

func (*Client) AggregateDataRanges added in v0.1.3 

func (c *Client) AggregateDataRanges(ctx context.Context, datas3tName string, firstDatapointIndex, lastDatapointIndex uint64, opts *AggregateOptions) error

AggregateDataRanges combines multiple existing dataranges into a single aggregate datarange

func (*Client) CancelAggregate added in v0.1.3 

func (c *Client) CancelAggregate(ctx context.Context, r *dataranges.CancelAggregateRequest) error

func (*Client) CancelDatarangeUpload 

func (c *Client) CancelDatarangeUpload(ctx context.Context, r *dataranges.CancelUploadRequest) error

func (*Client) CompleteAggregate added in v0.1.3 

func (c *Client) CompleteAggregate(ctx context.Context, r *dataranges.CompleteAggregateRequest) error

func (*Client) CompleteDatarangeUpload 

func (c *Client) CompleteDatarangeUpload(ctx context.Context, r *dataranges.CompleteUploadRequest) error

func (*Client) DatapointIterator added in v0.1.1 

func (c *Client) DatapointIterator(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64) iter.Seq2[[]byte, error]

DatapointIterator creates an iterator that progressively loads chunks of max 5MB and yields individual datapoint file contents from the tar stream

func (*Client) DeleteDatarange 

func (c *Client) DeleteDatarange(ctx context.Context, r *dataranges.DeleteDatarangeRequest) error

func (*Client) DownloadDatapointsTar 

func (c *Client) DownloadDatapointsTar(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64, outputPath string) error

DownloadDatapointsTar downloads a range of datapoints as a TAR file and saves it to the specified path

func (*Client) DownloadDatapointsTarWithOptions 

func (c *Client) DownloadDatapointsTarWithOptions(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64, outputPath string, opts *DownloadOptions) error

DownloadDatapointsTarWithOptions downloads a range of datapoints as a TAR file with configurable options

func (*Client) GetDatapointsBitmap added in v0.1.2 

func (c *Client) GetDatapointsBitmap(ctx context.Context, datas3tName string) (*roaring64.Bitmap, error)

func (*Client) ImportDatas3t added in v0.1.3 

func (c *Client) ImportDatas3t(ctx context.Context, req *datas3t.ImportDatas3tRequest) (*datas3t.ImportDatas3tResponse, error)

func (*Client) ListBuckets 

func (c *Client) ListBuckets(ctx context.Context) ([]*bucket.BucketListInfo, error)

func (*Client) ListDatas3ts 

func (c *Client) ListDatas3ts(ctx context.Context) ([]datas3t.Datas3tInfo, error)

func (*Client) PreSignDownloadForDatapoints 

func (c *Client) PreSignDownloadForDatapoints(ctx context.Context, r *download.PreSignDownloadForDatapointsRequest) (*download.PreSignDownloadForDatapointsResponse, error)

func (*Client) StartAggregate added in v0.1.3 

func (c *Client) StartAggregate(ctx context.Context, r *dataranges.StartAggregateRequest) (*dataranges.StartAggregateResponse, error)

func (*Client) StartDatarangeUpload 

func (c *Client) StartDatarangeUpload(ctx context.Context, r *dataranges.UploadDatarangeRequest) (*dataranges.UploadDatarangeResponse, error)

func (*Client) UploadDataRangeFile 

func (c *Client) UploadDataRangeFile(ctx context.Context, datas3tName string, file io.ReaderAt, size int64, opts *UploadOptions) error

type DownloadOptions 

type DownloadOptions struct {
	MaxParallelism int   // Maximum number of concurrent downloads (default: 4)
	MaxRetries     int   // Maximum number of retry attempts per chunk (default: 3)
	ChunkSize      int64 // Size of each chunk in bytes (default: 5MB)
}

DownloadOptions configures the download behavior

func DefaultDownloadOptions 

func DefaultDownloadOptions() *DownloadOptions

DefaultDownloadOptions returns sensible default options

type ProgressCallback 

type ProgressCallback func(info ProgressInfo)

ProgressCallback is called to report upload progress

type ProgressInfo 

type ProgressInfo struct {
	Phase           ProgressPhase
	TotalBytes      int64
	CompletedBytes  int64
	PercentComplete float64
	CurrentStep     string
	TotalSteps      int
	CompletedSteps  int
	EstimatedETA    time.Duration
	Speed           float64 // bytes per second
	StartTime       time.Time
}

ProgressInfo contains information about the upload progress

type ProgressPhase 

type ProgressPhase string

ProgressPhase represents the current phase of the upload process 

const (
	PhaseStartingAggregate   ProgressPhase = "starting_aggregate"
	PhaseDownloadingSources  ProgressPhase = "downloading_sources"
	PhaseMergingTars         ProgressPhase = "merging_tars"
	PhaseUploadingAggregate  ProgressPhase = "uploading_aggregate"
	PhaseCompletingAggregate ProgressPhase = "completing_aggregate"
)

Additional progress phases for aggregation 

const (
	PhaseAnalyzing      ProgressPhase = "analyzing"
	PhaseIndexing       ProgressPhase = "indexing"
	PhaseStarting       ProgressPhase = "starting"
	PhaseUploading      ProgressPhase = "uploading"
	PhaseUploadingIndex ProgressPhase = "uploading_index"
	PhaseCompleting     ProgressPhase = "completing"
)

type TarInfo 

type TarInfo struct {
	FirstDatapointIndex int64
	NumDatapoints       int
}

TarInfo contains metadata extracted from analyzing the TAR file

type UploadOptions 

type UploadOptions struct {
	MaxParallelism   int              // Maximum number of concurrent uploads (default: 4)
	MaxRetries       int              // Maximum number of retry attempts per chunk (default: 3)
	ProgressCallback ProgressCallback // Optional progress callback
}

UploadOptions configures the upload behavior

func DefaultUploadOptions 

func DefaultUploadOptions() *UploadOptions

DefaultUploadOptions returns sensible default options

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.