Documentation
¶
Index ¶
- Variables
- func DoUpload(ctx context.Context, req request.Requester, urlStr string, body io.Reader, ...) (string, error)
- func GenericDownload(ctx context.Context, req request.Requester, signedURL string, ...) (*http.Response, error)
- func NonRetryable(err error) error
- func RetryAction(ctx context.Context, logger TransferLogger, strategy RetryStrategy, ...) error
- type Backend
- type Chunk
- type ChunkStatus
- type Downloader
- type ExponentialBackoff
- type MultipartBackend
- type MultipartPart
- type NoOpLogger
- func (NoOpLogger) Debug(string, ...any)
- func (NoOpLogger) DebugContext(context.Context, string, ...any)
- func (NoOpLogger) DeleteFromFailedLog(string)
- func (NoOpLogger) Error(string, ...any)
- func (NoOpLogger) ErrorContext(context.Context, string, ...any)
- func (NoOpLogger) Failed(string, string, common.FileMetadata, string, int, bool)
- func (NoOpLogger) FailedContext(context.Context, string, string, common.FileMetadata, string, int, bool)
- func (NoOpLogger) GetFailedLogMap() map[string]common.RetryObject
- func (NoOpLogger) GetSucceededLogMap() map[string]string
- func (NoOpLogger) Info(string, ...any)
- func (NoOpLogger) InfoContext(context.Context, string, ...any)
- func (NoOpLogger) Printf(string, ...any)
- func (NoOpLogger) Println(...any)
- func (NoOpLogger) Scoreboard() *logs.Scoreboard
- func (NoOpLogger) Slog() *slog.Logger
- func (NoOpLogger) Succeeded(string, string)
- func (NoOpLogger) SucceededContext(context.Context, string, string)
- func (NoOpLogger) Warn(string, ...any)
- func (NoOpLogger) WarnContext(context.Context, string, ...any)
- type ObjectDeleter
- type ObjectMetadata
- type PartSigner
- type Planner
- type Provider
- type ReadBackend
- type ResolvedObject
- type Resolver
- type RetryStrategy
- type Service
- type SignedURL
- type TransferLogger
- type TransferPlan
- type TransferRequest
- type TransferStrategy
- type Uploader
- type Validator
- type WriteBackend
Constants ¶
This section is empty.
Variables ¶
var ( // ErrRangeIgnored indicates a server ignored a Range header and sent the full body. ErrRangeIgnored = fmt.Errorf("server ignored range request and returned full body") )
Functions ¶
func DoUpload ¶
func DoUpload(ctx context.Context, req request.Requester, urlStr string, body io.Reader, size int64) (string, error)
DoUpload performs a presigned PUT request and returns ETag when available.
func GenericDownload ¶
func GenericDownload(ctx context.Context, req request.Requester, signedURL string, rangeStart, rangeEnd *int64) (*http.Response, error)
GenericDownload performs GET (optionally ranged) against a signed URL.
func NonRetryable ¶ added in v0.2.8
func RetryAction ¶
func RetryAction(ctx context.Context, logger TransferLogger, strategy RetryStrategy, maxRetries int, action func() error) error
RetryAction is a helper that executes an action with retries according to a strategy.
Types ¶
type Backend ¶
type Backend interface {
ReadBackend
MultipartBackend
ObjectDeleter
Validator
}
Backend embeds capabilities and includes bucket validation.
type ChunkStatus ¶
type ChunkStatus string
const ( ChunkPending ChunkStatus = "pending" ChunkCompleted ChunkStatus = "completed" ChunkFailed ChunkStatus = "failed" )
type Downloader ¶
type Downloader interface {
Service
ResolveDownloadURL(ctx context.Context, guid string, accessID string) (string, error)
Download(ctx context.Context, url string, rangeStart, rangeEnd *int64) (*http.Response, error)
}
Downloader is the signed URL resolution and byte download surface.
type ExponentialBackoff ¶
type ExponentialBackoff struct {
MaxWaitSeconds int64
}
ExponentialBackoff implements a standard exponential backoff.
type MultipartBackend ¶ added in v0.2.7
type MultipartBackend interface {
WriteBackend
MultipartInit(ctx context.Context, guid string) (string, error)
MultipartPart(ctx context.Context, guid string, uploadID string, partNum int, body io.Reader) (string, error)
MultipartComplete(ctx context.Context, guid string, uploadID string, parts []MultipartPart) error
}
MultipartBackend enables specialized multipart uploads (init/part/complete).
type MultipartPart ¶
MultipartPart represents a single uploaded segment.
type NoOpLogger ¶
type NoOpLogger struct{}
NoOpLogger satisfies TransferLogger without emitting output.
func (NoOpLogger) Debug ¶
func (NoOpLogger) Debug(string, ...any)
func (NoOpLogger) DebugContext ¶
func (NoOpLogger) DebugContext(context.Context, string, ...any)
func (NoOpLogger) DeleteFromFailedLog ¶
func (NoOpLogger) DeleteFromFailedLog(string)
func (NoOpLogger) Error ¶
func (NoOpLogger) Error(string, ...any)
func (NoOpLogger) ErrorContext ¶
func (NoOpLogger) ErrorContext(context.Context, string, ...any)
func (NoOpLogger) Failed ¶
func (NoOpLogger) Failed(string, string, common.FileMetadata, string, int, bool)
func (NoOpLogger) FailedContext ¶
func (NoOpLogger) FailedContext(context.Context, string, string, common.FileMetadata, string, int, bool)
func (NoOpLogger) GetFailedLogMap ¶
func (NoOpLogger) GetFailedLogMap() map[string]common.RetryObject
func (NoOpLogger) GetSucceededLogMap ¶
func (NoOpLogger) GetSucceededLogMap() map[string]string
func (NoOpLogger) Info ¶
func (NoOpLogger) Info(string, ...any)
func (NoOpLogger) InfoContext ¶
func (NoOpLogger) InfoContext(context.Context, string, ...any)
func (NoOpLogger) Printf ¶
func (NoOpLogger) Printf(string, ...any)
func (NoOpLogger) Println ¶
func (NoOpLogger) Println(...any)
func (NoOpLogger) Scoreboard ¶
func (NoOpLogger) Scoreboard() *logs.Scoreboard
func (NoOpLogger) Slog ¶
func (NoOpLogger) Slog() *slog.Logger
func (NoOpLogger) Succeeded ¶
func (NoOpLogger) Succeeded(string, string)
func (NoOpLogger) SucceededContext ¶
func (NoOpLogger) SucceededContext(context.Context, string, string)
func (NoOpLogger) Warn ¶
func (NoOpLogger) Warn(string, ...any)
func (NoOpLogger) WarnContext ¶
func (NoOpLogger) WarnContext(context.Context, string, ...any)
type ObjectDeleter ¶
ObjectDeleter handles cleanup.
type ObjectMetadata ¶
type ObjectMetadata struct {
Size int64
Checksums []hash.HashInfo
MD5 string
AcceptRanges bool
Provider string
}
ObjectMetadata carries provider-agnostic information about a storage target.
type PartSigner ¶
type PartSigner interface {
GetDownloadPartURL(ctx context.Context, id string, start, end int64) (*SignedURL, error)
Logger() TransferLogger
}
PartSigner is a generic interface for components that can sign byte ranges.
type Planner ¶
type Planner interface {
PlanDownload(ctx context.Context, obj ResolvedObject) (*TransferPlan, error)
PlanUpload(ctx context.Context, obj ResolvedObject) (*TransferPlan, error)
}
Planner defines how to analyze a resolved object into a transfer plan.
type Provider ¶
type Provider interface {
GetStorageLocation(ctx context.Context, guid string) (bucket, key string, err error)
}
Provider resolves the bucket/key for a GUID.
type ReadBackend ¶ added in v0.2.7
type ReadBackend interface {
Service
Stat(ctx context.Context, guid string) (*ObjectMetadata, error)
GetReader(ctx context.Context, guid string) (io.ReadCloser, error)
GetRangeReader(ctx context.Context, guid string, offset, length int64) (io.ReadCloser, error)
}
ReadBackend provides metadata, single-stream reads, and ranged reads.
type ResolvedObject ¶
type ResolvedObject struct {
Id string
Name string
Size int64
Checksums hash.HashInfo
ProviderURL string
AccessMethod string
}
ResolvedObject is the outcome of the resolution layer.
type Resolver ¶
type Resolver interface {
Resolve(ctx context.Context, id string) (*ResolvedObject, error)
}
Resolver handles logical-to-physical mapping.
type RetryStrategy ¶
RetryStrategy defines how to wait between retries.
func DefaultBackoff ¶
func DefaultBackoff() RetryStrategy
DefaultBackoff returns the standard syfon backoff strategy.
type Service ¶
type Service interface {
Name() string
Logger() TransferLogger
}
Service provides high-level identity and logging access.
type TransferLogger ¶
type TransferLogger interface {
Slog() *slog.Logger
Info(msg string, args ...any)
InfoContext(ctx context.Context, msg string, args ...any)
Error(msg string, args ...any)
ErrorContext(ctx context.Context, msg string, args ...any)
Warn(msg string, args ...any)
WarnContext(ctx context.Context, msg string, args ...any)
Debug(msg string, args ...any)
DebugContext(ctx context.Context, msg string, args ...any)
Printf(format string, v ...any)
Println(v ...any)
Failed(filePath, filename string, metadata common.FileMetadata, guid string, retryCount int, multipart bool)
FailedContext(ctx context.Context, filePath, filename string, metadata common.FileMetadata, guid string, retryCount int, multipart bool)
Succeeded(filePath, guid string)
SucceededContext(ctx context.Context, filePath, guid string)
GetSucceededLogMap() map[string]string
GetFailedLogMap() map[string]common.RetryObject
DeleteFromFailedLog(path string)
Scoreboard() *logs.Scoreboard
}
TransferLogger is the minimal logging surface used by the transfer engines.
type TransferPlan ¶
type TransferPlan struct {
Strategy TransferStrategy
TotalSize int64
Chunks []Chunk
CheckpointPath string
}
TransferPlan is the blueprint for a specific data movement.
type TransferRequest ¶
type TransferRequest struct {
SourcePath string
ObjectKey string
GUID string
Bucket string
Metadata common.FileMetadata
ForceMultipart bool
}
TransferRequest represents a request to move a single file.
type TransferStrategy ¶
type TransferStrategy string
TransferStrategy defines how the engine intends to move the data.
const ( StrategySingleStream TransferStrategy = "single_stream" StrategyParallelRanges TransferStrategy = "parallel_ranges" StrategyMultipart TransferStrategy = "multipart" )
type Uploader ¶
type Uploader interface {
Service
ResolveUploadURL(ctx context.Context, guid string, filename string, metadata common.FileMetadata, bucket string) (string, error)
InitMultipartUpload(ctx context.Context, guid string, filename string, bucket string) (uploadID string, key string, err error)
GetMultipartUploadURL(ctx context.Context, key string, uploadID string, partNumber int32, bucket string) (string, error)
CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []internalapi.InternalMultipartPart, bucket string) error
Upload(ctx context.Context, url string, body io.Reader, size int64) error
UploadPart(ctx context.Context, url string, body io.Reader, size int64) (string, error)
DeleteFile(ctx context.Context, guid string) (string, error)
CanonicalObjectURL(signedURL, bucketHint, fallbackDID string) (string, error)
}
Uploader is the signed URL and multipart upload surface.