Documentation
¶
Index ¶
- Constants
- func IsEndpointTLS(endpoint string) bool
- type AddDatas3tRequest
- type AggregateOptions
- type BucketInfo
- type BucketListInfo
- type CancelAggregateRequest
- type CancelUploadRequest
- type ClearDatas3tRequest
- type ClearDatas3tResponse
- type Client
- func (c *Client) AddBucket(ctx context.Context, bucket *BucketInfo) error
- func (c *Client) AddDatas3t(ctx context.Context, datas3t *AddDatas3tRequest) error
- func (c *Client) AggregateDataRanges(ctx context.Context, datas3tName string, ...) (err error)
- func (c *Client) CancelAggregate(ctx context.Context, r *CancelAggregateRequest) error
- func (c *Client) CancelDatarangeUpload(ctx context.Context, r *CancelUploadRequest) error
- func (c *Client) ClearDatas3t(ctx context.Context, req *ClearDatas3tRequest) (*ClearDatas3tResponse, error)
- func (c *Client) CompleteAggregate(ctx context.Context, r *CompleteAggregateRequest) error
- func (c *Client) CompleteDatarangeUpload(ctx context.Context, r *CompleteUploadRequest) error
- func (c *Client) DatapointIterator(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64) iter.Seq2[[]byte, error]
- func (c *Client) DeleteDatarange(ctx context.Context, r *DeleteDatarangeRequest) error
- func (c *Client) DownloadDatapointsTar(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64, ...) error
- func (c *Client) DownloadDatapointsTarWithOptions(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64, ...) error
- func (c *Client) GetDatapointsBitmap(ctx context.Context, datas3tName string) (*roaring64.Bitmap, error)
- func (c *Client) ImportDatas3t(ctx context.Context, req *ImportDatas3tRequest) (*ImportDatas3tResponse, error)
- func (c *Client) ListBuckets(ctx context.Context) ([]*BucketListInfo, error)
- func (c *Client) ListDataranges(ctx context.Context, datas3tName string) ([]DatarangeInfo, error)
- func (c *Client) ListDatas3ts(ctx context.Context) ([]Datas3tInfo, error)
- func (c *Client) PreSignDownloadForDatapoints(ctx context.Context, r *PreSignDownloadForDatapointsRequest) (*PreSignDownloadForDatapointsResponse, error)
- func (c *Client) StartAggregate(ctx context.Context, r *StartAggregateRequest) (*StartAggregateResponse, error)
- func (c *Client) StartDatarangeUpload(ctx context.Context, r *UploadDatarangeRequest) (*UploadDatarangeResponse, error)
- func (c *Client) UploadDataRangeFile(ctx context.Context, datas3tName string, file io.ReaderAt, size int64, ...) error
- type CompleteAggregateRequest
- type CompleteUploadRequest
- type DatarangeDownloadURL
- type DatarangeInfo
- type Datas3tInfo
- type DeleteDatarangeRequest
- type DownloadOptions
- type DownloadSegment
- type ImportDatas3tRequest
- type ImportDatas3tResponse
- type ListDatarangesRequest
- type ListDatarangesResponse
- type PreSignDownloadForDatapointsRequest
- type PreSignDownloadForDatapointsResponse
- type ProgressCallback
- type ProgressInfo
- type ProgressPhase
- type StartAggregateRequest
- type StartAggregateResponse
- type TarInfo
- type UploadDatarangeRequest
- type UploadDatarangeResponse
- type UploadOptions
- type ValidationError
Constants ¶
const (
// DefaultChunkSize is the default chunk size for downloads (5MB)
DefaultChunkSize = 5 * 1024 * 1024
)
const (
// MaxChunkSize is the maximum chunk size for streaming downloads (5MB)
MaxChunkSize = 5 * 1024 * 1024
)
Variables ¶
This section is empty.
Functions ¶
func IsEndpointTLS ¶ added in v0.1.4
IsEndpointTLS determines if an endpoint uses TLS based on its protocol
Types ¶
type AddDatas3tRequest ¶ added in v0.1.4
func (*AddDatas3tRequest) Validate ¶ added in v0.1.4
func (r *AddDatas3tRequest) Validate() error
Validate validates the AddDatas3tRequest struct
type AggregateOptions ¶ added in v0.1.3
type AggregateOptions struct {
MaxParallelism int // Maximum number of concurrent downloads/uploads (default: 4)
MaxRetries int // Maximum number of retry attempts per operation (default: 3)
ProgressCallback ProgressCallback // Optional progress callback
TempDir string // Directory for temporary files (default: os.TempDir())
}
AggregateOptions configures the aggregation behavior
func DefaultAggregateOptions ¶ added in v0.1.3
func DefaultAggregateOptions() *AggregateOptions
DefaultAggregateOptions returns sensible default options for aggregation
type BucketInfo ¶ added in v0.1.4
type BucketInfo struct {
Name string `json:"name"`
Endpoint string `json:"endpoint"`
Bucket string `json:"bucket"`
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
}
func (*BucketInfo) Validate ¶ added in v0.1.4
func (r *BucketInfo) Validate() error
Validate validates the BucketInfo struct
type BucketListInfo ¶ added in v0.1.4
type CancelAggregateRequest ¶ added in v0.1.4
type CancelAggregateRequest struct {
AggregateUploadID int64 `json:"aggregate_upload_id"`
}
type CancelUploadRequest ¶ added in v0.1.4
type CancelUploadRequest struct {
DatarangeUploadID int64 `json:"datarange_upload_id"`
}
type ClearDatas3tRequest ¶ added in v0.1.9
type ClearDatas3tRequest struct {
Name string `json:"name"`
}
func (*ClearDatas3tRequest) Validate ¶ added in v0.1.9
func (r *ClearDatas3tRequest) Validate() error
Validate validates the ClearDatas3tRequest struct
type ClearDatas3tResponse ¶ added in v0.1.9
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
func (*Client) AddBucket ¶
func (c *Client) AddBucket(ctx context.Context, bucket *BucketInfo) error
func (*Client) AddDatas3t ¶
func (c *Client) AddDatas3t(ctx context.Context, datas3t *AddDatas3tRequest) error
func (*Client) AggregateDataRanges ¶ added in v0.1.3
func (c *Client) AggregateDataRanges(ctx context.Context, datas3tName string, firstDatapointIndex, lastDatapointIndex uint64, opts *AggregateOptions) (err error)
AggregateDataRanges combines multiple existing dataranges into a single aggregate datarange
func (*Client) CancelAggregate ¶ added in v0.1.3
func (c *Client) CancelAggregate(ctx context.Context, r *CancelAggregateRequest) error
func (*Client) CancelDatarangeUpload ¶
func (c *Client) CancelDatarangeUpload(ctx context.Context, r *CancelUploadRequest) error
func (*Client) ClearDatas3t ¶ added in v0.1.9
func (c *Client) ClearDatas3t(ctx context.Context, req *ClearDatas3tRequest) (*ClearDatas3tResponse, error)
func (*Client) CompleteAggregate ¶ added in v0.1.3
func (c *Client) CompleteAggregate(ctx context.Context, r *CompleteAggregateRequest) error
func (*Client) CompleteDatarangeUpload ¶
func (c *Client) CompleteDatarangeUpload(ctx context.Context, r *CompleteUploadRequest) error
func (*Client) DatapointIterator ¶ added in v0.1.1
func (c *Client) DatapointIterator(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64) iter.Seq2[[]byte, error]
DatapointIterator creates an iterator that progressively loads chunks of max 5MB and yields individual datapoint file contents from the tar stream
func (*Client) DeleteDatarange ¶
func (c *Client) DeleteDatarange(ctx context.Context, r *DeleteDatarangeRequest) error
func (*Client) DownloadDatapointsTar ¶
func (c *Client) DownloadDatapointsTar(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64, outputPath string) error
DownloadDatapointsTar downloads a range of datapoints as a TAR file and saves it to the specified path
func (*Client) DownloadDatapointsTarWithOptions ¶
func (c *Client) DownloadDatapointsTarWithOptions(ctx context.Context, datas3tName string, firstDatapoint, lastDatapoint uint64, outputPath string, opts *DownloadOptions) error
DownloadDatapointsTarWithOptions downloads a range of datapoints as a TAR file with configurable options
func (*Client) GetDatapointsBitmap ¶ added in v0.1.2
func (*Client) ImportDatas3t ¶ added in v0.1.3
func (c *Client) ImportDatas3t(ctx context.Context, req *ImportDatas3tRequest) (*ImportDatas3tResponse, error)
func (*Client) ListBuckets ¶
func (c *Client) ListBuckets(ctx context.Context) ([]*BucketListInfo, error)
func (*Client) ListDataranges ¶ added in v0.1.5
func (*Client) ListDatas3ts ¶
func (c *Client) ListDatas3ts(ctx context.Context) ([]Datas3tInfo, error)
func (*Client) PreSignDownloadForDatapoints ¶
func (c *Client) PreSignDownloadForDatapoints(ctx context.Context, r *PreSignDownloadForDatapointsRequest) (*PreSignDownloadForDatapointsResponse, error)
func (*Client) StartAggregate ¶ added in v0.1.3
func (c *Client) StartAggregate(ctx context.Context, r *StartAggregateRequest) (*StartAggregateResponse, error)
func (*Client) StartDatarangeUpload ¶
func (c *Client) StartDatarangeUpload(ctx context.Context, r *UploadDatarangeRequest) (*UploadDatarangeResponse, error)
type CompleteAggregateRequest ¶ added in v0.1.4
type CompleteUploadRequest ¶ added in v0.1.4
type DatarangeDownloadURL ¶ added in v0.1.4
type DatarangeDownloadURL struct {
DatarangeID int64 `json:"datarange_id"`
DataObjectKey string `json:"data_object_key"`
IndexObjectKey string `json:"index_object_key"`
MinDatapointKey int64 `json:"min_datapoint_key"`
MaxDatapointKey int64 `json:"max_datapoint_key"`
SizeBytes int64 `json:"size_bytes"`
PresignedDataURL string `json:"presigned_data_url"`
PresignedIndexURL string `json:"presigned_index_url"`
}
type DatarangeInfo ¶ added in v0.1.5
type Datas3tInfo ¶ added in v0.1.4
type Datas3tInfo struct {
Datas3tName string `json:"datas3t_name"`
BucketName string `json:"bucket_name"`
DatarangeCount int64 `json:"datarange_count"`
TotalDatapoints int64 `json:"total_datapoints"`
LowestDatapoint int64 `json:"lowest_datapoint"`
HighestDatapoint int64 `json:"highest_datapoint"`
TotalBytes int64 `json:"total_bytes"`
}
type DeleteDatarangeRequest ¶ added in v0.1.4
type DeleteDatarangeRequest struct {
Datas3tName string `json:"datas3t_name"`
FirstDatapointKey uint64 `json:"first_datapoint_key"`
LastDatapointKey uint64 `json:"last_datapoint_key"`
}
func (*DeleteDatarangeRequest) Validate ¶ added in v0.1.4
func (r *DeleteDatarangeRequest) Validate() error
Validate validates the DeleteDatarangeRequest struct
type DownloadOptions ¶
type DownloadOptions struct {
MaxParallelism int // Maximum number of concurrent downloads (default: 4)
MaxRetries int // Maximum number of retry attempts per chunk (default: 3)
ChunkSize int64 // Size of each chunk in bytes (default: 5MB)
}
DownloadOptions configures the download behavior
func DefaultDownloadOptions ¶
func DefaultDownloadOptions() *DownloadOptions
DefaultDownloadOptions returns sensible default options
type DownloadSegment ¶ added in v0.1.4
type ImportDatas3tRequest ¶ added in v0.1.4
type ImportDatas3tRequest struct {
BucketName string `json:"bucket_name"`
}
func (*ImportDatas3tRequest) Validate ¶ added in v0.1.4
func (r *ImportDatas3tRequest) Validate() error
Validate validates the ImportDatas3tRequest struct
type ImportDatas3tResponse ¶ added in v0.1.4
type ListDatarangesRequest ¶ added in v0.1.5
type ListDatarangesRequest struct {
Datas3tName string `json:"datas3t_name"`
}
func (*ListDatarangesRequest) Validate ¶ added in v0.1.5
func (r *ListDatarangesRequest) Validate() error
Validate validates the ListDatarangesRequest struct
type ListDatarangesResponse ¶ added in v0.1.5
type ListDatarangesResponse struct {
Dataranges []DatarangeInfo `json:"dataranges"`
}
type PreSignDownloadForDatapointsRequest ¶ added in v0.1.4
type PreSignDownloadForDatapointsRequest struct {
Datas3tName string `json:"datas3t_name"`
FirstDatapoint uint64 `json:"first_datapoint"`
LastDatapoint uint64 `json:"last_datapoint"`
}
func (*PreSignDownloadForDatapointsRequest) Validate ¶ added in v0.1.4
func (r *PreSignDownloadForDatapointsRequest) Validate() error
Validate validates the PreSignDownloadForDatapointsRequest struct
type PreSignDownloadForDatapointsResponse ¶ added in v0.1.4
type PreSignDownloadForDatapointsResponse struct {
DownloadSegments []DownloadSegment `json:"download_segments"`
}
type ProgressCallback ¶
type ProgressCallback func(info ProgressInfo)
ProgressCallback is called to report upload progress
type ProgressInfo ¶
type ProgressInfo struct {
Phase ProgressPhase
TotalBytes int64
CompletedBytes int64
PercentComplete float64
CurrentStep string
TotalSteps int
CompletedSteps int
EstimatedETA time.Duration
Speed float64 // bytes per second
StartTime time.Time
}
ProgressInfo contains information about the upload progress
type ProgressPhase ¶
type ProgressPhase string
ProgressPhase represents the current phase of the upload process
const ( PhaseStartingAggregate ProgressPhase = "starting_aggregate" PhaseDownloadingSources ProgressPhase = "downloading_sources" PhaseMergingTars ProgressPhase = "merging_tars" PhaseUploadingAggregate ProgressPhase = "uploading_aggregate" PhaseCompletingAggregate ProgressPhase = "completing_aggregate" )
Additional progress phases for aggregation
const ( PhaseAnalyzing ProgressPhase = "analyzing" PhaseIndexing ProgressPhase = "indexing" PhaseStarting ProgressPhase = "starting" PhaseUploading ProgressPhase = "uploading" PhaseUploadingIndex ProgressPhase = "uploading_index" PhaseCompleting ProgressPhase = "completing" )
type StartAggregateRequest ¶ added in v0.1.4
type StartAggregateRequest struct {
Datas3tName string `json:"datas3t_name"`
FirstDatapointIndex uint64 `json:"first_datapoint_index"`
LastDatapointIndex uint64 `json:"last_datapoint_index"`
}
func (*StartAggregateRequest) Validate ¶ added in v0.1.4
func (r *StartAggregateRequest) Validate() error
Validate validates the StartAggregateRequest struct
type StartAggregateResponse ¶ added in v0.1.4
type StartAggregateResponse struct {
AggregateUploadID int64 `json:"aggregate_upload_id"`
ObjectKey string `json:"object_key"`
// Source datarange information
SourceDatarangeDownloadURLs []DatarangeDownloadURL `json:"source_datarange_download_urls"`
// Upload configuration
UseDirectPut bool `json:"use_direct_put"`
PresignedMultipartUploadPutURLs []string `json:"presigned_multipart_upload_urls,omitempty"`
PresignedDataPutURL string `json:"presigned_data_put_url,omitempty"`
PresignedIndexPutURL string `json:"presigned_index_put_url"`
}
type UploadDatarangeRequest ¶ added in v0.1.4
type UploadDatarangeRequest struct {
Datas3tName string `json:"datas3t_name"`
DataSize uint64 `json:"data_size"`
NumberOfDatapoints uint64 `json:"number_of_datapoints"`
FirstDatapointIndex uint64 `json:"first_datapoint_index"`
}
func (*UploadDatarangeRequest) Validate ¶ added in v0.1.4
func (r *UploadDatarangeRequest) Validate() error
Validate validates the UploadDatarangeRequest struct
type UploadDatarangeResponse ¶ added in v0.1.4
type UploadDatarangeResponse struct {
DatarangeID int64 `json:"datarange_id"`
ObjectKey string `json:"object_key"`
FirstDatapointIndex uint64 `json:"first_datapoint_index"`
UseDirectPut bool `json:"use_direct_put"`
// For multipart uploads
PresignedMultipartUploadPutURLs []string `json:"presigned_multipart_upload_urls,omitempty"`
// For direct PUT uploads
PresignedDataPutURL string `json:"presigned_data_put_url,omitempty"`
// Common fields
PresignedIndexPutURL string `json:"presigned_index_put_url"`
}
type UploadOptions ¶
type UploadOptions struct {
MaxParallelism int // Maximum number of concurrent uploads (default: 4)
MaxRetries int // Maximum number of retry attempts per chunk (default: 3)
ProgressCallback ProgressCallback // Optional progress callback
}
UploadOptions configures the upload behavior
func DefaultUploadOptions ¶
func DefaultUploadOptions() *UploadOptions
DefaultUploadOptions returns sensible default options
type ValidationError ¶ added in v0.1.4
type ValidationError error
Source Files
¶
- add_bucket.go
- add_datas3t.go
- aggregate_dataranges.go
- cancel_aggregate.go
- cancel_datarange_upload.go
- clear_datas3t.go
- client.go
- complete_aggregate.go
- complete_datarange_upload.go
- datarange_iterator.go
- datarange_reader.go
- delete_datarange.go
- download_datapoints_tar.go
- get_datapoints_bitmap.go
- import_datas3t.go
- list_buckets.go
- list_dataranges.go
- list_datas3ts.go
- presign_download_for_datapoints.go
- start_aggregate.go
- start_datarange_upload.go
- types.go
- upload_datarange_file.go