Documentation
¶
Overview ¶
Package s3 provides Amazon S3 compatibility layer
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package s3 provides Amazon S3 compatibility layer
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package s3 provides Amazon S3 compatibility layer
- Copyright (c) 2022-2025, NVIDIA CORPORATION. All rights reserved.
Package s3 provides Amazon S3 compatibility layer
- Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.
Package s3 provides Amazon S3 compatibility layer
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package s3 provides Amazon S3 compatibility layer
- Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.
Package s3 provides Amazon S3 compatibility layer
- Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.
Index ¶
- Constants
- func DecodeXML[T any](body []byte) (result T, _ error)
- func EnforceCompleteAllParts(req *CompleteMptUpload, count int) (int, error)
- func FillLsoMsg(query url.Values, msg *apc.LsoMsg)
- func InvPrefObjname(bck *cmn.Bck, name, id string) (prefix, objName string)
- func ListParts(manifest *core.Ufest) (parts []types.CompletedPart, ecode int, err error)
- func NewErrNoSuchUpload(uploadID string, err error) error
- func ObjName(items []string) string
- func SetS3Headers(hdr http.Header, lom *core.LOM)
- func WriteErr(w http.ResponseWriter, r *http.Request, err error, ecode int)
- func WriteMptErr(w http.ResponseWriter, r *http.Request, err error, ecode int, lom *core.LOM, ...)
- type BckOwner
- type Bucket
- type CommonPrefix
- type CompleteMptUpload
- type CompleteMptUploadResult
- type CopyObjectResult
- type Delete
- type DeleteObjectInfo
- type DeleteResult
- type DeletedObjInfo
- type Error
- type InitiateMptUploadResult
- type ListBucketResult
- type ListMptUploadsResult
- type ListObjectResult
- type ListPartsResult
- type ObjInfo
- type PresignedReq
- type PresignedResp
- type UploadInfoResult
- type VersioningConfiguration
Constants ¶
const ( // AWS URL params QparamVersioning = "versioning" // Configure or retrieve bucket versioning settings QparamLifecycle = "lifecycle" QparamCORS = "cors" QparamPolicy = "policy" QparamACL = "acl" QparamMultiDelete = "delete" // Delete multiple objects in a single request QparamMaxKeys = "max-keys" // Maximum number of objects to return in listing QparamPrefix = "prefix" // Filter objects by key prefix QparamContinuationToken = "continuation-token" // Pagination token for continued listing QparamStartAfter = "start-after" // Start listing after this object key QparamDelimiter = "delimiter" // Delimiter for grouping object keys // multipart QparamMptUploads = "uploads" // Start multipart upload or list active uploads QparamMptUploadID = "uploadId" // Complete, abort, or list parts of specific multipart upload QparamMptPartNo = "partNumber" // Part number for multipart upload QparamMptMaxUploads = "max-uploads" QparamMptUploadIDMarker = "upload-id-marker" QparamAccessKeyID = "AWSAccessKeyId" QparamExpires = "Expires" QparamSignature = "Signature" QparamXID = "x-id" HeaderPrefix = "X-Amz-" // https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata HeaderMetaPrefix = cmn.AwsHeaderMetaPrefix HeaderCredentials = "X-Amz-Credential" //nolint:gosec // This is just a header name definition... HeaderSecurityToken = "X-Amz-Security-Token" // AWS temporary security token (used for JWT in compatibility mode) DefaultPartSize = 128 * cos.MiB AISRegion = "ais" AISServer = "AIStore" )
const ( InvName = ".inventory" InvSrcExt = ".csv.gz" InvDstExt = ".csv" )
const ErrPrefix = "aws-error"
Variables ¶
This section is empty.
Functions ¶
func EnforceCompleteAllParts ¶ added in v1.4.0
func EnforceCompleteAllParts(req *CompleteMptUpload, count int) (int, error)
validate that the caller requests completion of exactly all uploaded parts: the set {1..count} in ANY order on success, normalize req.Parts in-place on error return: - 501 when partial completion is requested (len != count) - 400 for malformed input (nil part number, out of range, duplicates)
func InvPrefObjname ¶ added in v1.3.23
func NewErrNoSuchUpload ¶ added in v1.3.31
Types ¶
type CommonPrefix ¶ added in v1.3.16
type CommonPrefix struct {
Prefix string `xml:"Prefix"`
}
NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.
type CompleteMptUpload ¶
type CompleteMptUpload struct {
Parts []types.CompletedPart `xml:"Part"`
}
Multipart upload completion request
type CompleteMptUploadResult ¶
type CompleteMptUploadResult struct {
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
ETag string `xml:"ETag"`
}
Multipart upload completion response
func (*CompleteMptUploadResult) MustMarshal ¶
func (r *CompleteMptUploadResult) MustMarshal(sgl *memsys.SGL)
type CopyObjectResult ¶
type CopyObjectResult struct {
LastModified string `xml:"LastModified"` // e.g. <LastModified>2009-10-12T17:50:30.000Z</LastModified>
ETag string `xml:"ETag"`
}
Response for object copy request
func (*CopyObjectResult) MustMarshal ¶
func (r *CopyObjectResult) MustMarshal(sgl *memsys.SGL)
type Delete ¶
type Delete struct {
Object []*DeleteObjectInfo `xml:"Object"`
Quiet bool `xml:"Quiet"`
}
Multiple object delete request
type DeleteObjectInfo ¶
type DeleteResult ¶ added in v1.3.16
type DeleteResult struct {
Objs []DeletedObjInfo `xml:"Deleted"`
}
NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.
func (*DeleteResult) MustMarshal ¶ added in v1.3.16
func (r *DeleteResult) MustMarshal(sgl *memsys.SGL)
type DeletedObjInfo ¶ added in v1.3.16
type DeletedObjInfo struct {
Key string `xml:"Key"`
}
Deleted result: list of deleted objects and errors
type Error ¶
type Error struct {
Code string `xml:"Code"`
Message string `xml:"Message"`
Resource string `xml:"Resource"`
RequestID string `xml:"RequestId"`
}
See https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html e.g. XML: <Error> <Code>NoSuchKey</Code> <Message>The resource you requested does not exist</Message> <Resource>/mybucket/myfoto.jpg</Resource> <RequestId>4442587FB7D0A2F9</RequestId> </Error>
type InitiateMptUploadResult ¶
type InitiateMptUploadResult struct {
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
}
Multipart upload start response
func (*InitiateMptUploadResult) MustMarshal ¶
func (r *InitiateMptUploadResult) MustMarshal(sgl *memsys.SGL)
type ListBucketResult ¶
type ListBucketResult struct {
Ns string `xml:"xmlns,attr"`
Owner BckOwner `xml:"Owner"`
Buckets []*Bucket `xml:"Buckets>Bucket"`
}
List bucket response
func NewListBucketResult ¶
func NewListBucketResult() (r *ListBucketResult)
func (*ListBucketResult) Add ¶
func (r *ListBucketResult) Add(bck *meta.Bck)
func (*ListBucketResult) MustMarshal ¶
func (r *ListBucketResult) MustMarshal(sgl *memsys.SGL)
type ListMptUploadsResult ¶
type ListMptUploadsResult struct {
Bucket string `xml:"Bucket"`
UploadIDMarker string `xml:"UploadIdMarker"`
Uploads []UploadInfoResult `xml:"Upload"`
MaxUploads int `xml:"MaxUploads"`
IsTruncated bool `xml:"IsTruncated"`
}
List of active multipart uploads response
func ListUploads ¶
func ListUploads(all []*core.Ufest, bckName, idMarker string, maxUploads int) *ListMptUploadsResult
func (*ListMptUploadsResult) MustMarshal ¶
func (r *ListMptUploadsResult) MustMarshal(sgl *memsys.SGL)
type ListObjectResult ¶
type ListObjectResult struct {
Name string `xml:"Name"`
Ns string `xml:"xmlns,attr"`
Prefix string `xml:"Prefix"`
ContinuationToken string `xml:"ContinuationToken"` // original
NextContinuationToken string `xml:"NextContinuationToken,omitempty"` // to read the next page
Contents []*ObjInfo `xml:"Contents"` // list of object
CommonPrefixes []*CommonPrefix `xml:"CommonPrefixes,omitempty"` // list of dirs (used with `apc.LsNoRecursion`)
KeyCount int `xml:"KeyCount"` // number of object names in the response
MaxKeys int `xml:"MaxKeys"` // "The maximum number of keys returned ..."
IsTruncated bool `xml:"IsTruncated"` // true if there are more pages to read
}
List objects response
func NewListObjectResult ¶
func NewListObjectResult(bucket string) *ListObjectResult
func (*ListObjectResult) FromLsoResult ¶ added in v1.3.22
func (r *ListObjectResult) FromLsoResult(lst *cmn.LsoRes)
func (*ListObjectResult) MustMarshal ¶
func (r *ListObjectResult) MustMarshal(sgl *memsys.SGL)
type ListPartsResult ¶
type ListPartsResult struct {
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
Parts []types.CompletedPart `xml:"Part"`
}
Multipart uploaded parts response
func (*ListPartsResult) MustMarshal ¶
func (r *ListPartsResult) MustMarshal(sgl *memsys.SGL)
type ObjInfo ¶
type ObjInfo struct {
Key string `xml:"Key"`
LastModified string `xml:"LastModified"`
ETag string `xml:"ETag"`
Class string `xml:"StorageClass"`
Size int64 `xml:"Size"`
}
NOTE: do not rename structs that have `xml` tags. The names of those structs become a top level tag of resulting XML, and those tags S3-compatible clients require.
type PresignedReq ¶ added in v1.3.23
type PresignedReq struct {
// contains filtered or unexported fields
}
func NewPresignedReq ¶ added in v1.3.23
func NewPresignedReq(oreq *http.Request, lom *core.LOM, body io.ReadCloser, q url.Values) *PresignedReq
func (*PresignedReq) Do ¶ added in v1.3.23
func (pts *PresignedReq) Do(client *http.Client) (*PresignedResp, error)
Do request/response. Always return `PresignedResp` (on error - with `StatusCode = 400`)
func (*PresignedReq) DoHead ¶ added in v1.3.24
func (pts *PresignedReq) DoHead(client *http.Client) (*PresignedResp, error)
func (*PresignedReq) DoReader ¶ added in v1.3.23
func (pts *PresignedReq) DoReader(client *http.Client) (*PresignedResp, error)
DoReader sends request and returns opened body/reader if successful. Caller is responsible for closing the reader.
NOTE: If error occurs `PresignedResp` with `StatusCode` will be set. NOTE: `BodyR` will be set only if `err` is `nil`.
type PresignedResp ¶ added in v1.3.23
type PresignedResp struct {
BodyR io.ReadCloser // Set when invoked `Do` with `async` option.
Header http.Header
Body []byte
Size int64
StatusCode int
}
func (*PresignedResp) ObjAttrs ¶ added in v1.3.23
func (resp *PresignedResp) ObjAttrs() (oa *cmn.ObjAttrs)
(compare w/ cmn/objattrs FromHeader)
type UploadInfoResult ¶
type UploadInfoResult struct {
Initiated time.Time `xml:"Initiated"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
}
Active upload info
type VersioningConfiguration ¶
type VersioningConfiguration struct {
Status string `xml:"Status"`
}
Bucket versioning
func NewVersioningConfiguration ¶
func NewVersioningConfiguration(enabled bool) *VersioningConfiguration
func (*VersioningConfiguration) Enabled ¶
func (r *VersioningConfiguration) Enabled() bool
func (*VersioningConfiguration) MustMarshal ¶
func (r *VersioningConfiguration) MustMarshal(sgl *memsys.SGL)