transfer

package
v0.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 8, 2026 License: MIT Imports: 17 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	// ErrRangeIgnored indicates a server ignored a Range header and sent the full body.
	ErrRangeIgnored = fmt.Errorf("server ignored range request and returned full body")
)

Functions

func DoUpload

func DoUpload(ctx context.Context, req request.Requester, urlStr string, body io.Reader, size int64) (string, error)

DoUpload performs a presigned PUT request and returns ETag when available.

func GenericDownload

func GenericDownload(ctx context.Context, req request.Requester, signedURL string, rangeStart, rangeEnd *int64) (*http.Response, error)

GenericDownload performs GET (optionally ranged) against a signed URL.

func NonRetryable added in v0.2.8

func NonRetryable(err error) error

func RetryAction

func RetryAction(ctx context.Context, logger TransferLogger, strategy RetryStrategy, maxRetries int, action func() error) error

RetryAction is a helper that executes an action with retries according to a strategy.

Types

type Backend

Backend embeds capabilities and includes bucket validation.

type Chunk

type Chunk struct {
	Index  int
	Offset int64
	Length int64
	Status ChunkStatus
}

type ChunkStatus

type ChunkStatus string
const (
	ChunkPending   ChunkStatus = "pending"
	ChunkCompleted ChunkStatus = "completed"
	ChunkFailed    ChunkStatus = "failed"
)

type Downloader

type Downloader interface {
	Service
	ResolveDownloadURL(ctx context.Context, guid string, accessID string) (string, error)
	Download(ctx context.Context, url string, rangeStart, rangeEnd *int64) (*http.Response, error)
}

Downloader is the signed URL resolution and byte download surface.

type ExponentialBackoff

type ExponentialBackoff struct {
	MaxWaitSeconds int64
}

ExponentialBackoff implements a standard exponential backoff.

func (*ExponentialBackoff) WaitTime

func (e *ExponentialBackoff) WaitTime(retryCount int) time.Duration

type MultipartBackend added in v0.2.7

type MultipartBackend interface {
	WriteBackend
	MultipartInit(ctx context.Context, guid string) (string, error)
	MultipartPart(ctx context.Context, guid string, uploadID string, partNum int, body io.Reader) (string, error)
	MultipartComplete(ctx context.Context, guid string, uploadID string, parts []MultipartPart) error
}

MultipartBackend enables specialized multipart uploads (init/part/complete).

type MultipartPart

type MultipartPart struct {
	PartNumber int32
	ETag       string
}

MultipartPart represents a single uploaded segment.

type NoOpLogger

type NoOpLogger struct{}

NoOpLogger satisfies TransferLogger without emitting output.

func (NoOpLogger) Debug

func (NoOpLogger) Debug(string, ...any)

func (NoOpLogger) DebugContext

func (NoOpLogger) DebugContext(context.Context, string, ...any)

func (NoOpLogger) DeleteFromFailedLog

func (NoOpLogger) DeleteFromFailedLog(string)

func (NoOpLogger) Error

func (NoOpLogger) Error(string, ...any)

func (NoOpLogger) ErrorContext

func (NoOpLogger) ErrorContext(context.Context, string, ...any)

func (NoOpLogger) Failed

func (NoOpLogger) FailedContext

func (NoOpLogger) GetFailedLogMap

func (NoOpLogger) GetFailedLogMap() map[string]common.RetryObject

func (NoOpLogger) GetSucceededLogMap

func (NoOpLogger) GetSucceededLogMap() map[string]string

func (NoOpLogger) Info

func (NoOpLogger) Info(string, ...any)

func (NoOpLogger) InfoContext

func (NoOpLogger) InfoContext(context.Context, string, ...any)

func (NoOpLogger) Printf

func (NoOpLogger) Printf(string, ...any)

func (NoOpLogger) Println

func (NoOpLogger) Println(...any)

func (NoOpLogger) Scoreboard

func (NoOpLogger) Scoreboard() *logs.Scoreboard

func (NoOpLogger) Slog

func (NoOpLogger) Slog() *slog.Logger

func (NoOpLogger) Succeeded

func (NoOpLogger) Succeeded(string, string)

func (NoOpLogger) SucceededContext

func (NoOpLogger) SucceededContext(context.Context, string, string)

func (NoOpLogger) Warn

func (NoOpLogger) Warn(string, ...any)

func (NoOpLogger) WarnContext

func (NoOpLogger) WarnContext(context.Context, string, ...any)

type ObjectDeleter

type ObjectDeleter interface {
	Delete(ctx context.Context, guid string) error
}

ObjectDeleter handles cleanup.

type ObjectMetadata

type ObjectMetadata struct {
	Size         int64
	Checksums    []hash.HashInfo
	MD5          string
	AcceptRanges bool
	Provider     string
}

ObjectMetadata carries provider-agnostic information about a storage target.

type PartSigner

type PartSigner interface {
	GetDownloadPartURL(ctx context.Context, id string, start, end int64) (*SignedURL, error)
	Logger() TransferLogger
}

PartSigner is a generic interface for components that can sign byte ranges.

type Planner

type Planner interface {
	PlanDownload(ctx context.Context, obj ResolvedObject) (*TransferPlan, error)
	PlanUpload(ctx context.Context, obj ResolvedObject) (*TransferPlan, error)
}

Planner defines how to analyze a resolved object into a transfer plan.

type Provider

type Provider interface {
	GetStorageLocation(ctx context.Context, guid string) (bucket, key string, err error)
}

Provider resolves the bucket/key for a GUID.

type ReadBackend added in v0.2.7

type ReadBackend interface {
	Service
	Stat(ctx context.Context, guid string) (*ObjectMetadata, error)
	GetReader(ctx context.Context, guid string) (io.ReadCloser, error)
	GetRangeReader(ctx context.Context, guid string, offset, length int64) (io.ReadCloser, error)
}

ReadBackend provides metadata, single-stream reads, and ranged reads.

type ResolvedObject

type ResolvedObject struct {
	Id           string
	Name         string
	Size         int64
	Checksums    hash.HashInfo
	ProviderURL  string
	AccessMethod string
}

ResolvedObject is the outcome of the resolution layer.

type Resolver

type Resolver interface {
	Resolve(ctx context.Context, id string) (*ResolvedObject, error)
}

Resolver handles logical-to-physical mapping.

type RetryStrategy

type RetryStrategy interface {
	WaitTime(retryCount int) time.Duration
}

RetryStrategy defines how to wait between retries.

func DefaultBackoff

func DefaultBackoff() RetryStrategy

DefaultBackoff returns the standard syfon backoff strategy.

type Service

type Service interface {
	Name() string
	Logger() TransferLogger
}

Service provides high-level identity and logging access.

type SignedURL

type SignedURL struct {
	URL     string
	Headers map[string]string
}

SignedURL contains a URL and any mandatory headers for access.

type TransferLogger

type TransferLogger interface {
	Slog() *slog.Logger
	Info(msg string, args ...any)
	InfoContext(ctx context.Context, msg string, args ...any)
	Error(msg string, args ...any)
	ErrorContext(ctx context.Context, msg string, args ...any)
	Warn(msg string, args ...any)
	WarnContext(ctx context.Context, msg string, args ...any)
	Debug(msg string, args ...any)
	DebugContext(ctx context.Context, msg string, args ...any)
	Printf(format string, v ...any)
	Println(v ...any)
	Failed(filePath, filename string, metadata common.FileMetadata, guid string, retryCount int, multipart bool)
	FailedContext(ctx context.Context, filePath, filename string, metadata common.FileMetadata, guid string, retryCount int, multipart bool)
	Succeeded(filePath, guid string)
	SucceededContext(ctx context.Context, filePath, guid string)
	GetSucceededLogMap() map[string]string
	GetFailedLogMap() map[string]common.RetryObject
	DeleteFromFailedLog(path string)
	Scoreboard() *logs.Scoreboard
}

TransferLogger is the minimal logging surface used by the transfer engines.

type TransferPlan

type TransferPlan struct {
	Strategy       TransferStrategy
	TotalSize      int64
	Chunks         []Chunk
	CheckpointPath string
}

TransferPlan is the blueprint for a specific data movement.

type TransferRequest

type TransferRequest struct {
	SourcePath     string
	ObjectKey      string
	GUID           string
	Bucket         string
	Metadata       common.FileMetadata
	ForceMultipart bool
}

TransferRequest represents a request to move a single file.

type TransferStrategy

type TransferStrategy string

TransferStrategy defines how the engine intends to move the data.

const (
	StrategySingleStream   TransferStrategy = "single_stream"
	StrategyParallelRanges TransferStrategy = "parallel_ranges"
	StrategyMultipart      TransferStrategy = "multipart"
)

type Uploader

type Uploader interface {
	Service
	ResolveUploadURL(ctx context.Context, guid string, filename string, metadata common.FileMetadata, bucket string) (string, error)
	InitMultipartUpload(ctx context.Context, guid string, filename string, bucket string) (uploadID string, key string, err error)
	GetMultipartUploadURL(ctx context.Context, key string, uploadID string, partNumber int32, bucket string) (string, error)
	CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []internalapi.InternalMultipartPart, bucket string) error
	Upload(ctx context.Context, url string, body io.Reader, size int64) error
	UploadPart(ctx context.Context, url string, body io.Reader, size int64) (string, error)
	DeleteFile(ctx context.Context, guid string) (string, error)
	CanonicalObjectURL(signedURL, bucketHint, fallbackDID string) (string, error)
}

Uploader is the signed URL and multipart upload surface.

type Validator added in v0.2.7

type Validator interface {
	Validate(ctx context.Context, bucket string) error
}

Validator checks bucket/provider configuration.

type WriteBackend added in v0.2.7

type WriteBackend interface {
	Service
	Upload(ctx context.Context, guid string, body io.Reader, size int64) error
}

WriteBackend provides single-stream write access.

Directories

Path Synopsis
providers
gcs
s3

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL