common

package
v0.2.7 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 5, 2026 License: MIT Imports: 12 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// B is bytes
	B int64 = 1
	// KB is kilobytes
	KB int64 = 1024 * B
	// MB is megabytes
	MB int64 = 1024 * KB
	// GB is gigabytes
	GB int64 = 1024 * MB
	// TB is terabytes
	TB int64 = 1024 * GB
)
View Source
const (
	// IndexdIndexEndpoint is the endpoint postfix for INDEXD index
	IndexdIndexEndpoint = "/index"

	// DataUserEndpoint is the endpoint postfix for FENCE user
	DataUserEndpoint = "/user/user"

	// DataEndpoint is the canonical endpoint prefix for upload/delete flows
	DataEndpoint = "/data/upload"

	// DataAccessTokenEndpoint is the endpoint postfix for FENCE access token
	DataAccessTokenEndpoint = "/user/credentials/api/access_token"

	// DataUploadEndpoint is the endpoint postfix for upload init/presigned-url
	DataUploadEndpoint = DataEndpoint

	// DataDownloadEndpoint is the endpoint postfix for download presigned-url
	DataDownloadEndpoint = "/data/download"

	// DataDownloadPartEndpoint is the endpoint postfix for download part presigned-url
	DataDownloadPartEndpoint = "/data/download/%s/part"

	// DataMultipartInitEndpoint is the endpoint postfix for multipart init
	DataMultipartInitEndpoint = "/data/multipart/init"

	// DataMultipartUploadEndpoint is the endpoint postfix for multipart upload
	DataMultipartUploadEndpoint = "/data/multipart/upload"

	// DataMultipartCompleteEndpoint is the endpoint postfix for multipart complete
	DataMultipartCompleteEndpoint = "/data/multipart/complete"

	// PathSeparator is os dependent path separator char
	PathSeparator = string(os.PathSeparator)

	// DefaultTimeout is used for standard metadata/API requests
	DefaultTimeout = 60 * time.Second
	// DataTimeout is used specifically for large data transfers (uploads/downloads)
	DataTimeout = 5 * time.Minute

	HeaderContentType   = "Content-Type"
	MIMEApplicationJSON = "application/json"

	// FileSizeLimit is the maximum single file size for non-multipart upload (5GB)
	FileSizeLimit = 5 * GB

	// MultipartFileSizeLimit is the maximum single file size for multipart upload (5TB)
	MultipartFileSizeLimit = 5 * TB
	MinMultipartChunkSize  = 10 * MB

	// MaxRetryCount is the maximum retry number per record
	MaxRetryCount = 5
	MaxWaitTime   = 300

	MaxMultipartParts    = 10000
	MaxConcurrentUploads = 10
	MaxRetries           = 5

	OnProgressThreshold = 1 * MB

	// API Endpoints
	HealthzEndpoint = "/healthz"

	IndexdIndexBulkEndpoint               = "/index/bulk"
	IndexdIndexBulkHashesEndpoint         = "/index/bulk/hashes"
	IndexdIndexBulkDeleteEndpoint         = "/index/bulk/delete"
	IndexdIndexBulkSHA256ValidityEndpoint = "/index/bulk/sha256/validity"
	IndexdIndexBulkDocumentsEndpoint      = "/index/bulk/documents"
	IndexdIndexRecordEndpointTemplate     = "/index/%s"

	DataUploadBulkEndpoint             = "/data/upload/bulk"
	DataRecordEndpointTemplate         = "/data/upload/%s"
	DataDownloadRecordEndpointTemplate = "/data/download/%s"

	DataBucketsEndpoint                = "/data/buckets"
	DataBucketsRecordsEndpointTemplate = "/data/buckets/%s"
	DataBucketsScopesEndpointTemplate  = "/data/buckets/%s/scopes"

	MetricsSummaryEndpoint      = "/index/v1/metrics/summary"
	MetricsFilesEndpoint        = "/index/v1/metrics/files"
	MetricsFileEndpointTemplate = "/index/v1/metrics/files/%s"

	GA4GHDRSObjectAccessEndpointTemplate = "/ga4gh/drs/v1/objects/%s/access/%s"

	// Query Parameters
	QueryParamHash         = "hash"
	QueryParamOrganization = "organization"
	QueryParamProject      = "project"
	QueryParamLimit        = "limit"
	QueryParamOffset       = "offset"
	QueryParamPage         = "page"
	QueryParamURL          = "url"
	QueryParamHashType     = "hash_type"
	QueryParamBucket       = "bucket"
	QueryParamFileName     = "file_name"
	QueryParamExpiresIn    = "expires_in"
	QueryParamRedirect     = "redirect"
	QueryParamInactiveDays = "inactive_days"
	QueryParamStart        = "start"
	QueryParamEnd          = "end"
)

Variables

View Source
var (
	// MinChunkSize is the lower bound for multipart chunk sizing.
	MinChunkSize = 10 * MB
)

Functions

func CanDownloadFile

func CanDownloadFile(signedURL string) error

func GetAbsolutePath

func GetAbsolutePath(filePath string) (string, error)

func GetOid

func GetOid(ctx context.Context) string

func IsCloudPresignedURL

func IsCloudPresignedURL(raw string) bool

func IsHidden

func IsHidden(filePath string) (bool, error)

func LoadFailedLog

func LoadFailedLog(path string) (map[string]RetryObject, error)

func ParseFilePaths

func ParseFilePaths(filePath string, metadataEnabled bool) ([]string, error)

func ParseRootPath

func ParseRootPath(filePath string) (string, error)

func ResponseBodyError

func ResponseBodyError(resp *http.Response, prefix string) error

ResponseBodyError formats a non-2xx HTTP response with a bounded body preview.

func ToJSONReader

func ToJSONReader(payload any) (io.Reader, error)

func WithOid

func WithOid(ctx context.Context, oid string) context.Context

func WithProgress

func WithProgress(ctx context.Context, cb ProgressCallback) context.Context

func WithTransferCompletion added in v0.2.5

func WithTransferCompletion(ctx context.Context, cb TransferCompletionCallback) context.Context

Types

type FileMetadata

type FileMetadata struct {
	Authorizations map[string][]string `json:"authorizations,omitempty"`
	Aliases        []string            `json:"aliases"`
	Metadata       map[string]any      `json:"metadata"`
}

type ProgressCallback

type ProgressCallback func(ProgressEvent) error

func GetProgress

func GetProgress(ctx context.Context) ProgressCallback

type ProgressEvent

type ProgressEvent struct {
	Event          string         `json:"event"`
	Oid            string         `json:"oid"`
	BytesSoFar     int64          `json:"bytesSoFar"`
	BytesSinceLast int64          `json:"bytesSinceLast"`
	Message        string         `json:"message,omitempty"`
	Level          string         `json:"level,omitempty"`
	Attrs          map[string]any `json:"attrs,omitempty"`
}

type RetryObject

type RetryObject struct {
	SourcePath   string
	ObjectKey    string
	FileMetadata FileMetadata
	GUID         string
	RetryCount   int
	Multipart    bool
	Bucket       string
}

type TransferCompletionCallback added in v0.2.5

type TransferCompletionCallback func(TransferCompletionEvent) error

func GetTransferCompletion added in v0.2.5

func GetTransferCompletion(ctx context.Context) TransferCompletionCallback

type TransferCompletionEvent added in v0.2.5

type TransferCompletionEvent struct {
	Direction  string
	GUID       string
	RangeStart int64
	RangeEnd   int64
	Bytes      int64
	PartNumber int
	Strategy   string
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL