s3

package
v1.4.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 15, 2026 License: MIT Imports: 19 Imported by: 0

Documentation

Overview

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.

Package s3 provides Amazon S3 compatibility layer

  • Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source
const (
	// AWS URL params
	QparamVersioning        = "versioning" // Configure or retrieve bucket versioning settings
	QparamLifecycle         = "lifecycle"
	QparamCORS              = "cors"
	QparamPolicy            = "policy"
	QparamACL               = "acl"
	QparamMultiDelete       = "delete"             // Delete multiple objects in a single request
	QparamMaxKeys           = "max-keys"           // Maximum number of objects to return in listing
	QparamPrefix            = "prefix"             // Filter objects by key prefix
	QparamContinuationToken = "continuation-token" // Pagination token for continued listing
	QparamStartAfter        = "start-after"        // Start listing after this object key
	QparamDelimiter         = "delimiter"          // Delimiter for grouping object keys

	// multipart
	QparamMptUploads        = "uploads"    // Start multipart upload or list active uploads
	QparamMptUploadID       = "uploadId"   // Complete, abort, or list parts of specific multipart upload
	QparamMptPartNo         = "partNumber" // Part number for multipart upload
	QparamMptMaxUploads     = "max-uploads"
	QparamMptUploadIDMarker = "upload-id-marker"

	QparamAccessKeyID = "AWSAccessKeyId"
	QparamExpires     = "Expires"
	QparamSignature   = "Signature"
	QparamXID         = "x-id"

	HeaderPrefix = "X-Amz-"

	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata
	HeaderMetaPrefix = cmn.AwsHeaderMetaPrefix

	HeaderCredentials   = "X-Amz-Credential"     //nolint:gosec // This is just a header name definition...
	HeaderSecurityToken = "X-Amz-Security-Token" // AWS temporary security token (used for JWT in compatibility mode)

	DefaultPartSize = 128 * cos.MiB

	AISRegion = "ais"
	AISServer = "AIStore"
)
View Source
const (
	InvName   = ".inventory"
	InvSrcExt = ".csv.gz"
	InvDstExt = ".csv"
)
View Source
const ErrPrefix = "aws-error"

Variables

This section is empty.

Functions

func DecodeXML added in v1.4.0

func DecodeXML[T any](body []byte) (result T, _ error)

func EnforceCompleteAllParts added in v1.4.0

func EnforceCompleteAllParts(req *CompleteMptUpload, count int) (int, error)

validate that the caller requests completion of exactly all uploaded parts: the set {1..count} in ANY order on success, normalize req.Parts in-place on error return: - 501 when partial completion is requested (len != count) - 400 for malformed input (nil part number, out of range, duplicates)

func FillLsoMsg added in v1.3.22

func FillLsoMsg(query url.Values, msg *apc.LsoMsg)

func InvPrefObjname added in v1.3.23

func InvPrefObjname(bck *cmn.Bck, name, id string) (prefix, objName string)

func ListParts

func ListParts(manifest *core.Ufest) (parts []types.CompletedPart, ecode int, err error)

func NewErrNoSuchUpload added in v1.3.31

func NewErrNoSuchUpload(uploadID string, err error) error

func ObjName

func ObjName(items []string) string

func SetS3Headers added in v1.3.26

func SetS3Headers(hdr http.Header, lom *core.LOM)

func WriteErr

func WriteErr(w http.ResponseWriter, r *http.Request, err error, ecode int)

func WriteMptErr added in v1.3.22

func WriteMptErr(w http.ResponseWriter, r *http.Request, err error, ecode int, lom *core.LOM, uploadID string)

with user-friendly tip

Types

type BckOwner

type BckOwner struct {
	ID   string `xml:"ID"`
	Name string `xml:"DisplayName"`
}

type Bucket

type Bucket struct {
	Name    string `xml:"Name"`
	Created string `xml:"CreationDate"`
	String  string `xml:"String"`
}

type CommonPrefix added in v1.3.16

type CommonPrefix struct {
	Prefix string `xml:"Prefix"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

type CompleteMptUpload

type CompleteMptUpload struct {
	Parts []types.CompletedPart `xml:"Part"`
}

Multipart upload completion request

type CompleteMptUploadResult

type CompleteMptUploadResult struct {
	Bucket string `xml:"Bucket"`
	Key    string `xml:"Key"`
	ETag   string `xml:"ETag"`
}

Multipart upload completion response

func (*CompleteMptUploadResult) MustMarshal

func (r *CompleteMptUploadResult) MustMarshal(sgl *memsys.SGL)

type CopyObjectResult

type CopyObjectResult struct {
	LastModified string `xml:"LastModified"` // e.g. <LastModified>2009-10-12T17:50:30.000Z</LastModified>
	ETag         string `xml:"ETag"`
}

Response for object copy request

func (*CopyObjectResult) MustMarshal

func (r *CopyObjectResult) MustMarshal(sgl *memsys.SGL)

type Delete

type Delete struct {
	Object []*DeleteObjectInfo `xml:"Object"`
	Quiet  bool                `xml:"Quiet"`
}

Multiple object delete request

type DeleteObjectInfo

type DeleteObjectInfo struct {
	Key     string `xml:"Key"`
	Version string `xml:"Version"`
}

type DeleteResult added in v1.3.16

type DeleteResult struct {
	Objs []DeletedObjInfo `xml:"Deleted"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

func (*DeleteResult) MustMarshal added in v1.3.16

func (r *DeleteResult) MustMarshal(sgl *memsys.SGL)

type DeletedObjInfo added in v1.3.16

type DeletedObjInfo struct {
	Key string `xml:"Key"`
}

Deleted result: list of deleted objects and errors

type Error

type Error struct {
	Code      string `xml:"Code"`
	Message   string `xml:"Message"`
	Resource  string `xml:"Resource"`
	RequestID string `xml:"RequestId"`
}

See https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html e.g. XML: <Error> <Code>NoSuchKey</Code> <Message>The resource you requested does not exist</Message> <Resource>/mybucket/myfoto.jpg</Resource> <RequestId>4442587FB7D0A2F9</RequestId> </Error>

type InitiateMptUploadResult

type InitiateMptUploadResult struct {
	Bucket   string `xml:"Bucket"`
	Key      string `xml:"Key"`
	UploadID string `xml:"UploadId"`
}

Multipart upload start response

func (*InitiateMptUploadResult) MustMarshal

func (r *InitiateMptUploadResult) MustMarshal(sgl *memsys.SGL)

type ListBucketResult

type ListBucketResult struct {
	Ns      string    `xml:"xmlns,attr"`
	Owner   BckOwner  `xml:"Owner"`
	Buckets []*Bucket `xml:"Buckets>Bucket"`
}

List bucket response

func NewListBucketResult

func NewListBucketResult() (r *ListBucketResult)

func (*ListBucketResult) Add

func (r *ListBucketResult) Add(bck *meta.Bck)

func (*ListBucketResult) MustMarshal

func (r *ListBucketResult) MustMarshal(sgl *memsys.SGL)

type ListMptUploadsResult

type ListMptUploadsResult struct {
	Bucket         string             `xml:"Bucket"`
	UploadIDMarker string             `xml:"UploadIdMarker"`
	Uploads        []UploadInfoResult `xml:"Upload"`
	MaxUploads     int                `xml:"MaxUploads"`
	IsTruncated    bool               `xml:"IsTruncated"`
}

List of active multipart uploads response

func ListUploads

func ListUploads(all []*core.Ufest, bckName, idMarker string, maxUploads int) *ListMptUploadsResult

func (*ListMptUploadsResult) MustMarshal

func (r *ListMptUploadsResult) MustMarshal(sgl *memsys.SGL)

type ListObjectResult

type ListObjectResult struct {
	Name                  string          `xml:"Name"`
	Ns                    string          `xml:"xmlns,attr"`
	Prefix                string          `xml:"Prefix"`
	ContinuationToken     string          `xml:"ContinuationToken"`               // original
	NextContinuationToken string          `xml:"NextContinuationToken,omitempty"` // to read the next page
	Contents              []*ObjInfo      `xml:"Contents"`                        // list of object
	CommonPrefixes        []*CommonPrefix `xml:"CommonPrefixes,omitempty"`        // list of dirs (used with `apc.LsNoRecursion`)
	KeyCount              int             `xml:"KeyCount"`                        // number of object names in the response
	MaxKeys               int             `xml:"MaxKeys"`                         // "The maximum number of keys returned ..."
	IsTruncated           bool            `xml:"IsTruncated"`                     // true if there are more pages to read
}

List objects response

func NewListObjectResult

func NewListObjectResult(bucket string) *ListObjectResult

func (*ListObjectResult) FromLsoResult added in v1.3.22

func (r *ListObjectResult) FromLsoResult(lst *cmn.LsoRes)

func (*ListObjectResult) MustMarshal

func (r *ListObjectResult) MustMarshal(sgl *memsys.SGL)

type ListPartsResult

type ListPartsResult struct {
	Bucket   string                `xml:"Bucket"`
	Key      string                `xml:"Key"`
	UploadID string                `xml:"UploadId"`
	Parts    []types.CompletedPart `xml:"Part"`
}

Multipart uploaded parts response

func (*ListPartsResult) MustMarshal

func (r *ListPartsResult) MustMarshal(sgl *memsys.SGL)

type ObjInfo

type ObjInfo struct {
	Key          string `xml:"Key"`
	LastModified string `xml:"LastModified"`
	ETag         string `xml:"ETag"`
	Class        string `xml:"StorageClass"`
	Size         int64  `xml:"Size"`
}

NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.

type PresignedReq added in v1.3.23

type PresignedReq struct {
	// contains filtered or unexported fields
}

func NewPresignedReq added in v1.3.23

func NewPresignedReq(oreq *http.Request, lom *core.LOM, body io.ReadCloser, q url.Values) *PresignedReq

func (*PresignedReq) Do added in v1.3.23

func (pts *PresignedReq) Do(client *http.Client) (*PresignedResp, error)

Do request/response. Always return `PresignedResp` (on error - with `StatusCode = 400`)

func (*PresignedReq) DoHead added in v1.3.24

func (pts *PresignedReq) DoHead(client *http.Client) (*PresignedResp, error)

func (*PresignedReq) DoReader added in v1.3.23

func (pts *PresignedReq) DoReader(client *http.Client) (*PresignedResp, error)

DoReader sends request and returns opened body/reader if successful. Caller is responsible for closing the reader.

NOTE: If error occurs `PresignedResp` with `StatusCode` will be set. NOTE: `BodyR` will be set only if `err` is `nil`.

type PresignedResp added in v1.3.23

type PresignedResp struct {
	BodyR      io.ReadCloser // Set when invoked `Do` with `async` option.
	Header     http.Header
	Body       []byte
	Size       int64
	StatusCode int
}

func (*PresignedResp) ObjAttrs added in v1.3.23

func (resp *PresignedResp) ObjAttrs() (oa *cmn.ObjAttrs)

(compare w/ cmn/objattrs FromHeader)

type UploadInfoResult

type UploadInfoResult struct {
	Initiated time.Time `xml:"Initiated"`
	Key       string    `xml:"Key"`
	UploadID  string    `xml:"UploadId"`
}

Active upload info

type VersioningConfiguration

type VersioningConfiguration struct {
	Status string `xml:"Status"`
}

Bucket versioning

func NewVersioningConfiguration

func NewVersioningConfiguration(enabled bool) *VersioningConfiguration

func (*VersioningConfiguration) Enabled

func (r *VersioningConfiguration) Enabled() bool

func (*VersioningConfiguration) MustMarshal

func (r *VersioningConfiguration) MustMarshal(sgl *memsys.SGL)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL