client

package
v0.1.12 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 13, 2025 License: AGPL-3.0 Imports: 21 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// DefaultChunkSize is the default chunk size for downloads (5MB)
	DefaultChunkSize = 5 * 1024 * 1024
)
View Source
const (
	// MaxChunkSize is the maximum chunk size for streaming downloads (5MB)
	MaxChunkSize = 5 * 1024 * 1024
)

Variables

This section is empty.

Functions

func IsEndpointTLS added in v0.1.4

func IsEndpointTLS(endpoint string) bool

IsEndpointTLS determines if an endpoint uses TLS based on its protocol

Types

type AddDatas3tRequest added in v0.1.4

type AddDatas3tRequest struct {
	Name   string `json:"name"`
	Bucket string `json:"bucket"`
}

func (*AddDatas3tRequest) Validate added in v0.1.4

func (r *AddDatas3tRequest) Validate() error

Validate validates the AddDatas3tRequest struct

type AggregateOptions added in v0.1.3

type AggregateOptions struct {
	MaxParallelism   int              // Maximum number of concurrent downloads/uploads (default: 4)
	MaxRetries       int              // Maximum number of retry attempts per operation (default: 3)
	ProgressCallback ProgressCallback // Optional progress callback
	TempDir          string           // Directory for temporary files (default: os.TempDir())
}

AggregateOptions configures the aggregation behavior

func DefaultAggregateOptions added in v0.1.3

func DefaultAggregateOptions() *AggregateOptions

DefaultAggregateOptions returns sensible default options for aggregation

type BucketInfo added in v0.1.4

type BucketInfo struct {
	Name      string `json:"name"`
	Endpoint  string `json:"endpoint"`
	Bucket    string `json:"bucket"`
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
}

func (*BucketInfo) Validate added in v0.1.4

func (r *BucketInfo) Validate() error

Validate validates the BucketInfo struct

type BucketListInfo added in v0.1.4

type BucketListInfo struct {
	Name     string `json:"name"`
	Endpoint string `json:"endpoint"`
	Bucket   string `json:"bucket"`
}

type CancelAggregateRequest added in v0.1.4

type CancelAggregateRequest struct {
	AggregateUploadID int64 `json:"aggregate_upload_id"`
}

type CancelUploadRequest added in v0.1.4

type CancelUploadRequest struct {
	DatarangeUploadID int64 `json:"datarange_upload_id"`
}

type ClearDatas3tRequest added in v0.1.9

type ClearDatas3tRequest struct {
	Name string `json:"name"`
}

func (*ClearDatas3tRequest) Validate added in v0.1.9

func (r *ClearDatas3tRequest) Validate() error

Validate validates the ClearDatas3tRequest struct

type ClearDatas3tResponse added in v0.1.9

type ClearDatas3tResponse struct {
	DatarangesDeleted int `json:"dataranges_deleted"`
	ObjectsScheduled  int `json:"objects_scheduled"`
}

type Client

type Client struct {
	// contains filtered or unexported fields
}

func NewClient

func NewClient(baseURL string) *Client

func (*Client) AddBucket

func (c *Client) AddBucket(ctx context.Context, bucket *BucketInfo) error

func (*Client) AddDatas3t

func (c *Client) AddDatas3t(ctx context.Context, datas3t *AddDatas3tRequest) error

func (*Client) AggregateDataRanges added in v0.1.3

func (c *Client) AggregateDataRanges(ctx context.Context, datas3tName string, firstDatapointIndex, lastDatapointIndex uint64, opts *AggregateOptions) error

AggregateDataRanges combines multiple existing dataranges into a single aggregate datarange

func (*Client) CancelAggregate added in v0.1.3

func (c *Client) CancelAggregate(ctx context.Context, r *CancelAggregateRequest) error

func (*Client) CancelDatarangeUpload

func (c *Client) CancelDatarangeUpload(ctx context.Context, r *CancelUploadRequest) error

func (*Client) ClearDatas3t added in v0.1.9

func (c *Client) ClearDatas3t(ctx context.Context, req *ClearDatas3tRequest) (*ClearDatas3tResponse, error)

func (*Client) CompleteAggregate added in v0.1.3

func (c *Client) CompleteAggregate(ctx context.Context, r *CompleteAggregateRequest) error

func (*Client) CompleteDatarangeUpload

func (c *Client) CompleteDatarangeUpload(ctx context.Context, r *CompleteUploadRequest) error

func (*Client) DatapointIterator added in v0.1.1

func (c *Client) DatapointIterator(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64) iter.Seq2[[]byte, error]

DatapointIterator creates an iterator that progressively loads chunks of max 5MB and yields individual datapoint file contents from the tar stream

func (*Client) DeleteDatarange

func (c *Client) DeleteDatarange(ctx context.Context, r *DeleteDatarangeRequest) error

func (*Client) DownloadDatapointsTar

func (c *Client) DownloadDatapointsTar(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64, outputPath string) error

DownloadDatapointsTar downloads a range of datapoints as a TAR file and saves it to the specified path

func (*Client) DownloadDatapointsTarWithOptions

func (c *Client) DownloadDatapointsTarWithOptions(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64, outputPath string, opts *DownloadOptions) error

DownloadDatapointsTarWithOptions downloads a range of datapoints as a TAR file with configurable options

func (*Client) GetDatapointsBitmap added in v0.1.2

func (c *Client) GetDatapointsBitmap(ctx context.Context, datas3tName string) (*roaring64.Bitmap, error)

func (*Client) ImportDatas3t added in v0.1.3

func (c *Client) ImportDatas3t(ctx context.Context, req *ImportDatas3tRequest) (*ImportDatas3tResponse, error)

func (*Client) ListBuckets

func (c *Client) ListBuckets(ctx context.Context) ([]*BucketListInfo, error)

func (*Client) ListDataranges added in v0.1.5

func (c *Client) ListDataranges(ctx context.Context, datas3tName string) ([]DatarangeInfo, error)

func (*Client) ListDatas3ts

func (c *Client) ListDatas3ts(ctx context.Context) ([]Datas3tInfo, error)

func (*Client) StartAggregate added in v0.1.3

func (*Client) StartDatarangeUpload

func (c *Client) StartDatarangeUpload(ctx context.Context, r *UploadDatarangeRequest) (*UploadDatarangeResponse, error)

func (*Client) UploadDataRangeFile

func (c *Client) UploadDataRangeFile(ctx context.Context, datas3tName string, file io.ReaderAt, size int64, opts *UploadOptions) error

type CompleteAggregateRequest added in v0.1.4

type CompleteAggregateRequest struct {
	AggregateUploadID int64    `json:"aggregate_upload_id"`
	UploadIDs         []string `json:"upload_ids,omitempty"` // For multipart uploads
}

type CompleteUploadRequest added in v0.1.4

type CompleteUploadRequest struct {
	DatarangeUploadID int64    `json:"datarange_upload_id"`
	UploadIDs         []string `json:"upload_ids,omitempty"` // For multipart uploads
}

type DatarangeDownloadURL added in v0.1.4

type DatarangeDownloadURL struct {
	DatarangeID       int64  `json:"datarange_id"`
	DataObjectKey     string `json:"data_object_key"`
	IndexObjectKey    string `json:"index_object_key"`
	MinDatapointKey   int64  `json:"min_datapoint_key"`
	MaxDatapointKey   int64  `json:"max_datapoint_key"`
	SizeBytes         int64  `json:"size_bytes"`
	PresignedDataURL  string `json:"presigned_data_url"`
	PresignedIndexURL string `json:"presigned_index_url"`
}

type DatarangeInfo added in v0.1.5

type DatarangeInfo struct {
	DatarangeID     int64  `json:"datarange_id"`
	DataObjectKey   string `json:"data_object_key"`
	IndexObjectKey  string `json:"index_object_key"`
	MinDatapointKey int64  `json:"min_datapoint_key"`
	MaxDatapointKey int64  `json:"max_datapoint_key"`
	SizeBytes       int64  `json:"size_bytes"`
}

type Datas3tInfo added in v0.1.4

type Datas3tInfo struct {
	Datas3tName      string `json:"datas3t_name"`
	BucketName       string `json:"bucket_name"`
	DatarangeCount   int64  `json:"datarange_count"`
	TotalDatapoints  int64  `json:"total_datapoints"`
	LowestDatapoint  int64  `json:"lowest_datapoint"`
	HighestDatapoint int64  `json:"highest_datapoint"`
	TotalBytes       int64  `json:"total_bytes"`
}

type DeleteDatarangeRequest added in v0.1.4

type DeleteDatarangeRequest struct {
	Datas3tName       string `json:"datas3t_name"`
	FirstDatapointKey uint64 `json:"first_datapoint_key"`
	LastDatapointKey  uint64 `json:"last_datapoint_key"`
}

func (*DeleteDatarangeRequest) Validate added in v0.1.4

func (r *DeleteDatarangeRequest) Validate() error

Validate validates the DeleteDatarangeRequest struct

type DownloadOptions

type DownloadOptions struct {
	MaxParallelism int   // Maximum number of concurrent downloads (default: 4)
	MaxRetries     int   // Maximum number of retry attempts per chunk (default: 3)
	ChunkSize      int64 // Size of each chunk in bytes (default: 5MB)
}

DownloadOptions configures the download behavior

func DefaultDownloadOptions

func DefaultDownloadOptions() *DownloadOptions

DefaultDownloadOptions returns sensible default options

type DownloadSegment added in v0.1.4

type DownloadSegment struct {
	PresignedURL string `json:"presigned_url"`
	Range        string `json:"range"`
}

type ImportDatas3tRequest added in v0.1.4

type ImportDatas3tRequest struct {
	BucketName string `json:"bucket_name"`
}

func (*ImportDatas3tRequest) Validate added in v0.1.4

func (r *ImportDatas3tRequest) Validate() error

Validate validates the ImportDatas3tRequest struct

type ImportDatas3tResponse added in v0.1.4

type ImportDatas3tResponse struct {
	ImportedDatas3ts []string `json:"imported_datas3ts"`
	ImportedCount    int      `json:"imported_count"`
}

type ListDatarangesRequest added in v0.1.5

type ListDatarangesRequest struct {
	Datas3tName string `json:"datas3t_name"`
}

func (*ListDatarangesRequest) Validate added in v0.1.5

func (r *ListDatarangesRequest) Validate() error

Validate validates the ListDatarangesRequest struct

type ListDatarangesResponse added in v0.1.5

type ListDatarangesResponse struct {
	Dataranges []DatarangeInfo `json:"dataranges"`
}

type PreSignDownloadForDatapointsRequest added in v0.1.4

type PreSignDownloadForDatapointsRequest struct {
	Datas3tName    string `json:"datas3t_name"`
	FirstDatapoint uint64 `json:"first_datapoint"`
	LastDatapoint  uint64 `json:"last_datapoint"`
}

func (*PreSignDownloadForDatapointsRequest) Validate added in v0.1.4

Validate validates the PreSignDownloadForDatapointsRequest struct

type PreSignDownloadForDatapointsResponse added in v0.1.4

type PreSignDownloadForDatapointsResponse struct {
	DownloadSegments []DownloadSegment `json:"download_segments"`
}

type ProgressCallback

type ProgressCallback func(info ProgressInfo)

ProgressCallback is called to report upload progress

type ProgressInfo

type ProgressInfo struct {
	Phase           ProgressPhase
	TotalBytes      int64
	CompletedBytes  int64
	PercentComplete float64
	CurrentStep     string
	TotalSteps      int
	CompletedSteps  int
	EstimatedETA    time.Duration
	Speed           float64 // bytes per second
	StartTime       time.Time
}

ProgressInfo contains information about the upload progress

type ProgressPhase

type ProgressPhase string

ProgressPhase represents the current phase of the upload process

const (
	PhaseStartingAggregate   ProgressPhase = "starting_aggregate"
	PhaseDownloadingSources  ProgressPhase = "downloading_sources"
	PhaseMergingTars         ProgressPhase = "merging_tars"
	PhaseUploadingAggregate  ProgressPhase = "uploading_aggregate"
	PhaseCompletingAggregate ProgressPhase = "completing_aggregate"
)

Additional progress phases for aggregation

const (
	PhaseAnalyzing      ProgressPhase = "analyzing"
	PhaseIndexing       ProgressPhase = "indexing"
	PhaseStarting       ProgressPhase = "starting"
	PhaseUploading      ProgressPhase = "uploading"
	PhaseUploadingIndex ProgressPhase = "uploading_index"
	PhaseCompleting     ProgressPhase = "completing"
)

type StartAggregateRequest added in v0.1.4

type StartAggregateRequest struct {
	Datas3tName         string `json:"datas3t_name"`
	FirstDatapointIndex uint64 `json:"first_datapoint_index"`
	LastDatapointIndex  uint64 `json:"last_datapoint_index"`
}

func (*StartAggregateRequest) Validate added in v0.1.4

func (r *StartAggregateRequest) Validate() error

Validate validates the StartAggregateRequest struct

type StartAggregateResponse added in v0.1.4

type StartAggregateResponse struct {
	AggregateUploadID int64  `json:"aggregate_upload_id"`
	ObjectKey         string `json:"object_key"`

	// Source datarange information
	SourceDatarangeDownloadURLs []DatarangeDownloadURL `json:"source_datarange_download_urls"`

	// Upload configuration
	UseDirectPut                    bool     `json:"use_direct_put"`
	PresignedMultipartUploadPutURLs []string `json:"presigned_multipart_upload_urls,omitempty"`
	PresignedDataPutURL             string   `json:"presigned_data_put_url,omitempty"`
	PresignedIndexPutURL            string   `json:"presigned_index_put_url"`
}

type TarInfo

type TarInfo struct {
	FirstDatapointIndex int64
	NumDatapoints       int
}

TarInfo contains metadata extracted from analyzing the TAR file

type UploadDatarangeRequest added in v0.1.4

type UploadDatarangeRequest struct {
	Datas3tName         string `json:"datas3t_name"`
	DataSize            uint64 `json:"data_size"`
	NumberOfDatapoints  uint64 `json:"number_of_datapoints"`
	FirstDatapointIndex uint64 `json:"first_datapoint_index"`
}

func (*UploadDatarangeRequest) Validate added in v0.1.4

func (r *UploadDatarangeRequest) Validate() error

Validate validates the UploadDatarangeRequest struct

type UploadDatarangeResponse added in v0.1.4

type UploadDatarangeResponse struct {
	DatarangeID         int64  `json:"datarange_id"`
	ObjectKey           string `json:"object_key"`
	FirstDatapointIndex uint64 `json:"first_datapoint_index"`
	UseDirectPut        bool   `json:"use_direct_put"`

	// For multipart uploads
	PresignedMultipartUploadPutURLs []string `json:"presigned_multipart_upload_urls,omitempty"`

	// For direct PUT uploads
	PresignedDataPutURL string `json:"presigned_data_put_url,omitempty"`

	// Common fields
	PresignedIndexPutURL string `json:"presigned_index_put_url"`
}

type UploadOptions

type UploadOptions struct {
	MaxParallelism   int              // Maximum number of concurrent uploads (default: 4)
	MaxRetries       int              // Maximum number of retry attempts per chunk (default: 3)
	ProgressCallback ProgressCallback // Optional progress callback
}

UploadOptions configures the upload behavior

func DefaultUploadOptions

func DefaultUploadOptions() *UploadOptions

DefaultUploadOptions returns sensible default options

type ValidationError added in v0.1.4

type ValidationError error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL