s3

package
v0.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 11, 2025 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// Default configuration values
	DefaultTimeout            = time.Minute * 5 // Increased to 5 minutes for large uploads
	DefaultRegion             = "eu-west-1"
	DefaultMultipartThreshold = int64(100 * 1024 * 1024) // 100MB
	DefaultPartSize           = int64(10 * 1024 * 1024)  // Increased to 10MB for better performance
	DefaultMaxUploadParts     = 10000
	DefaultMaxRetries         = 3
	DefaultUploadTimeout      = time.Minute * 30 // 30 minutes for large file uploads

	// Minimum and maximum part sizes for multipart uploads
	MinPartSize = int64(5 * 1024 * 1024)        // 5MB
	MaxPartSize = int64(5 * 1024 * 1024 * 1024) // 5GB

	// Server-Side Encryption types
	SSEAlgorithmAES256  = "AES256"
	SSEAlgorithmKMS     = "aws:kms"
	SSEAlgorithmKMSDSSE = "aws:kms:dsse"

	// Customer-provided encryption algorithm
	SSECAlgorithmAES256 = "AES256"
)
View Source
const (
	ErrNilConfig           = utils.Error("Config is nil")
	ErrMissingEndpoint     = utils.Error("missing endpoint")
	ErrMissingRegion       = utils.Error("missing region")
	ErrInvalidTimeout      = utils.Error("invalid timeout")
	ErrInvalidPartSize     = utils.Error("invalid part size")
	ErrInvalidThreshold    = utils.Error("invalid multipart threshold")
	ErrBucketNotFound      = utils.Error("bucket not found")
	ErrBucketAlreadyExists = utils.Error("bucket already exists")
	ErrObjectNotFound      = utils.Error("object not found")
	ErrInvalidBucketName   = utils.Error("invalid bucket bucketName")
	ErrInvalidObjectKey    = utils.Error("invalid object key")
	ErrClientNotConnected  = utils.Error("client not connected")
)

Error constants

Variables

This section is empty.

Functions

This section is empty.

Types

type Bucket

type Bucket struct {
	*Client
	// contains filtered or unexported fields
}

func NewBucket

func NewBucket(client *Client, bucketName string) (*Bucket, error)

func (*Bucket) CopyObject

func (b *Bucket) CopyObject(ctx context.Context, srcName, dstBucket, dstName string, opts ...ObjectOptions) error

CopyObject copies an object within S3

func (*Bucket) Create

func (b *Bucket) Create(ctx context.Context, opts ...BucketOptions) error

Create attempt to create current bucket

func (*Bucket) Delete

func (b *Bucket) Delete(ctx context.Context) error

Delete attempt to delete current bucket

func (*Bucket) DeleteObject

func (b *Bucket) DeleteObject(ctx context.Context, name string) error

DeleteObject deletes an object from S3

func (*Bucket) Exists

func (b *Bucket) Exists(ctx context.Context) (bool, error)

Exists check if current bucket exists

func (*Bucket) GetObject

func (b *Bucket) GetObject(ctx context.Context, name string) (io.ReadCloser, error)

GetObject downloads an object from S3 Note: no object is actually transfered; it returns a ReadCloser that will perform the read; if name does not exist, no error is returned; use ObjectExists()/StatObject() instead to check for existence

func (*Bucket) GetObjectAdvanced

func (b *Bucket) GetObjectAdvanced(ctx context.Context, objectName string, writer io.Writer, opts DownloadOptions) error

GetObjectAdvanced provides advanced download functionality with simplified control Note: Complex download manager removed - uses simple range downloads instead

func (*Bucket) GetObjectRange

func (b *Bucket) GetObjectRange(ctx context.Context, objectName string, start, end int64) (io.ReadCloser, error)

GetObjectRange downloads a specific range of bytes from an object

func (*Bucket) GetObjectStream

func (b *Bucket) GetObjectStream(ctx context.Context, objectName string, writer io.Writer) error

GetObjectStream downloads an object and writes it directly to a writer

func (*Bucket) GetObjectStreamRange

func (b *Bucket) GetObjectStreamRange(ctx context.Context, objectName string, writer io.Writer, start, end int64) error

GetObjectStreamRange downloads a range of bytes from an object to a writer

func (*Bucket) HeadObject

func (b *Bucket) HeadObject(ctx context.Context, name string) (*ObjectInfo, error)

HeadObject gets object metadata

func (*Bucket) ListObjects

func (b *Bucket) ListObjects(ctx context.Context, opts ...ListOptions) ([]ObjectInfo, error)

ListObjects lists objects in a bucket

func (*Bucket) Name

func (b *Bucket) Name() string

func (*Bucket) ObjectExists

func (b *Bucket) ObjectExists(ctx context.Context, name string) (bool, error)

ObjectExists checks if an object exists

func (*Bucket) PresignDeleteObject

func (b *Bucket) PresignDeleteObject(ctx context.Context, objectName string, expiry time.Duration) (string, error)

PresignDeleteObject generates a pre-signed URL for deleting an object Note: MinIO-Go does not support presigned DELETE URLs

func (*Bucket) PresignGetObject

func (b *Bucket) PresignGetObject(ctx context.Context, objectName string, expiry time.Duration) (string, error)

PresignGetObject generates a pre-signed URL for downloading an object

func (*Bucket) PresignHeadObject

func (b *Bucket) PresignHeadObject(ctx context.Context, objectName string, expiry time.Duration) (string, error)

PresignHeadObject generates a pre-signed URL for getting object metadata

func (*Bucket) PresignPutObject

func (b *Bucket) PresignPutObject(ctx context.Context, objectName string, expiry time.Duration, opts ...ObjectOptions) (string, error)

PresignPutObject generates a pre-signed URL for uploading an object

func (*Bucket) PutObject

func (b *Bucket) PutObject(ctx context.Context, objectName string, reader io.Reader, size int64, opts ...ObjectOptions) error

PutObject uploads an object to S3

func (*Bucket) PutObjectAdvanced

func (b *Bucket) PutObjectAdvanced(ctx context.Context, objectName string, reader io.Reader, size int64, opts UploadOptions) error

PutObjectAdvanced provides advanced upload functionality with detailed control

func (*Bucket) PutObjectMultipart

func (b *Bucket) PutObjectMultipart(ctx context.Context, objectName string, reader io.Reader, size int64, opts ...ObjectOptions) error

PutObjectMultipart uploads an object using multipart upload with progress tracking

func (*Bucket) PutObjectStream

func (b *Bucket) PutObjectStream(ctx context.Context, objectName string, reader io.Reader, opts ...ObjectOptions) error

PutObjectStream uploads an object using streaming (no size required)

type BucketInfo

type BucketInfo struct {
	Name         string
	CreationDate time.Time
}

BucketInfo represents information about an S3 bucket

type BucketInterface

type BucketInterface interface {
	// Object operations
	PutObject(ctx context.Context, bucket, key string, reader io.Reader, size int64, opts ...ObjectOptions) error
	GetObject(ctx context.Context, bucket, key string) (io.ReadCloser, error)
	DeleteObject(ctx context.Context, bucket, key string) error
	ListObjects(ctx context.Context, bucket string, opts ...ListOptions) ([]ObjectInfo, error)
	ObjectExists(ctx context.Context, bucket, key string) (bool, error)
	HeadObject(ctx context.Context, bucket, key string) (*ObjectInfo, error)

	// Advanced operations
	PutObjectStream(ctx context.Context, bucket, key string, reader io.Reader, opts ...ObjectOptions) error
	GetObjectStream(ctx context.Context, bucket, key string, writer io.Writer) error
	CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKey string, opts ...ObjectOptions) error

	// Advanced download operations
	GetObjectRange(ctx context.Context, bucket, key string, start, end int64) (io.ReadCloser, error)
	GetObjectStreamRange(ctx context.Context, bucket, key string, writer io.Writer, start, end int64) error
	GetObjectAdvanced(ctx context.Context, bucket, key string, writer io.Writer, opts DownloadOptions) error

	// Multipart upload operations
	PutObjectMultipart(ctx context.Context, bucket, key string, reader io.Reader, size int64, opts ...ObjectOptions) error
	PutObjectAdvanced(ctx context.Context, bucket, key string, reader io.Reader, size int64, opts UploadOptions) error

	// Pre-signed URLs
	PresignGetObject(ctx context.Context, bucket, key string, expiry time.Duration) (string, error)
	PresignPutObject(ctx context.Context, bucket, key string, expiry time.Duration, opts ...ObjectOptions) (string, error)
}

type BucketOptions

type BucketOptions struct {
	Region string
	ACL    string
}

BucketOptions represents options for bucket operations

type Client

type Client struct {
	// contains filtered or unexported fields
}

func NewClient

func NewClient(cfg *Config, logger *log.Logger) (*Client, error)

NewClient creates a new S3 client

func (*Client) Bucket

func (c *Client) Bucket(bucketName string) (*Bucket, error)

Bucket create bucket object

func (*Client) BucketExists

func (c *Client) BucketExists(ctx context.Context, bucket string) (bool, error)

BucketExists checks if a bucket exists

func (*Client) Close

func (c *Client) Close() error

Close closes the S3 client connection

func (*Client) Connect

func (c *Client) Connect(ctx context.Context) error

Connect establishes connection to S3

func (*Client) CreateBucket

func (c *Client) CreateBucket(ctx context.Context, bucket string, opts ...BucketOptions) error

CreateBucket creates a new bucket

func (*Client) DeleteBucket

func (c *Client) DeleteBucket(ctx context.Context, bucket string) error

DeleteBucket deletes a bucket

func (*Client) IsConnected

func (c *Client) IsConnected() bool

IsConnected returns true if the client is connected

func (*Client) ListBuckets

func (c *Client) ListBuckets(ctx context.Context) ([]BucketInfo, error)

ListBuckets lists all S3 buckets

func (*Client) MinioClient

func (c *Client) MinioClient() *minio.Client

MinioClient returns the underlying MinIO client

type ClientInterface

type ClientInterface interface {
	// Connection management
	Connect(ctx context.Context) error
	MinioClient() *minio.Client
	Close() error
	IsConnected() bool

	// Bucket operations
	CreateBucket(ctx context.Context, bucket string, opts ...BucketOptions) error
	DeleteBucket(ctx context.Context, bucket string) error
	ListBuckets(ctx context.Context) ([]BucketInfo, error)
	BucketExists(ctx context.Context, bucket string) (bool, error)
}

type Config

type Config struct {
	// Connection settings
	Endpoint    string `json:"endpoint"`    // Custom endpoint for S3-compatible services
	Region      string `json:"region"`      // AWS region
	AccessKeyID string `json:"accessKeyId"` // AWS access key ID

	// Secret access key using secure credential handling
	secure.DefaultCredentialConfig

	// Optional bucket default
	DefaultBucket string `json:"defaultBucket"`

	// Behavior settings
	ForcePathStyle bool `json:"forcePathStyle"` // Force path-style addressing (for MinIO, etc.)
	UseAccelerate  bool `json:"useAccelerate"`  // Use S3 transfer acceleration
	UseSSL         bool `json:"useSSL"`         // Use SSL/TLS (default: true)

	// Timeout settings
	TimeoutSeconds       int `json:"timeoutSeconds"`       // Request timeout in seconds
	UploadTimeoutSeconds int `json:"uploadTimeoutSeconds"` // Upload timeout in seconds (for large files)

	// Multipart upload settings
	MultipartThreshold int64 `json:"multipartThreshold"` // Threshold for multipart uploads in bytes
	PartSize           int64 `json:"partSize"`           // Size of each part in multipart uploads
	MaxUploadParts     int   `json:"maxUploadParts"`     // Maximum number of parts in multipart upload
	Concurrency        int   `json:"concurrency"`        // Number of concurrent uploads

	// TLS configuration
	tls.ClientConfig

	// connection polling
	MaxIdleConns        int           `json:"maxIdleConns"`        // Total idle connections
	MaxIdleConnsPerHost int           `json:"maxIdleConnsPerHost"` // Per-host idle connections
	MaxConnsPerHost     int           `json:"maxConnsPerHost"`     // Max connections per host
	IdleConnTimeout     time.Duration `json:"idleConnTimeout"`

	// Retry settings
	MaxRetries int    `json:"maxRetries"` // Maximum number of retries
	RetryMode  string `json:"retryMode"`  // Retry mode: "standard" or "adaptive"
}

Config represents the S3 client configuration

func NewConfig

func NewConfig() *Config

NewConfig creates a new configuration with default values

func (*Config) GetEndpointURL

func (c *Config) GetEndpointURL() string

GetEndpointURL returns the full endpoint URL with protocol

func (*Config) IsCustomEndpoint

func (c *Config) IsCustomEndpoint() bool

IsCustomEndpoint returns true if a custom endpoint is configured

func (*Config) Validate

func (c *Config) Validate() error

Validate validates the configuration

type DownloadOptions

type DownloadOptions struct {
	// Range specification
	StartByte *int64
	EndByte   *int64

	// Concurrency control
	Concurrency int

	// Part size for multipart downloads
	PartSize int64
}

DownloadOptions provides options for download operations

type ListOptions

type ListOptions struct {
	Prefix     string
	Delimiter  string
	MaxKeys    int32
	StartAfter string
}

ListOptions represents options for listing objects

type ObjectInfo

type ObjectInfo struct {
	Key          string
	Size         int64
	LastModified time.Time
	ETag         string
	StorageClass string
	ContentType  string
}

ObjectInfo represents information about an S3 object

type ObjectOptions

type ObjectOptions struct {
	ContentType        string
	CacheControl       string
	ContentDisposition string
	ContentEncoding    string
	ContentLanguage    string
	Metadata           map[string]string
	Tags               map[string]string
	StorageClass       string
	// Server-Side Encryption options
	ServerSideEncryption    string            // AES256, aws:kms, aws:kms:dsse
	SSEKMSKeyId             string            // KMS key ID for SSE-KMS
	SSEKMSEncryptionContext map[string]string // KMS encryption context
	SSECustomerAlgorithm    string            // Customer-provided encryption algorithm (AES256)
	SSECustomerKey          string            // Customer-provided encryption key (base64)
	SSECustomerKeyMD5       string            // MD5 digest of customer key
	BucketKeyEnabled        *bool             // Enable S3 Bucket Key for cost optimization
}

ObjectOptions represents options for object operations

type UploadOptions

type UploadOptions struct {
	ObjectOptions
	// Additional upload-specific options
	LeavePartsOnError bool // Leave successfully uploaded parts on error for manual recovery
	MaxUploadParts    int  // Override the default maximum number of parts
	Concurrency       int  // Override the default concurrency level
}

UploadOptions provides options for advanced upload operations

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL