s3

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jan 26, 2026 License: Apache-2.0 Imports: 35 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// ContextKeyAccessKeyID is the context key for the authenticated user's access key ID
	ContextKeyAccessKeyID contextKey = "accessKeyID"
	// ContextKeyAccountID is the context key for the authenticated user's account ID
	ContextKeyAccountID contextKey = "accountID"
)

Variables

This section is empty.

Functions

func IsValidBucketName added in v1.0.0

func IsValidBucketName(bucket string) error

IsValidBucketName validates a bucket name according to S3 naming rules https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html

func IsValidKeyName added in v1.0.0

func IsValidKeyName(key string) error

IsValidKeyName validates an object key name

Types

type ACL

type ACL struct {
	AccessKeyId     string `toml:"access_key_id"`
	SecretAccessKey string `toml:"secret_access_key`
	Owner           string `toml:"owner`
	Permissions     int    `toml:"permissions`
}

type AuthEntry

type AuthEntry struct {
	AccessKeyID     string       `toml:"access_key_id"`
	SecretAccessKey string       `toml:"secret_access_key"`
	AccountID       string       `toml:"account_id"`
	Policy          []PolicyRule `toml:"policy"`
}

Authentication and policy

type BackendType added in v1.0.0

type BackendType string

BackendType specifies the storage backend type

const (
	// BackendFilesystem uses local filesystem storage (simple, single-node)
	BackendFilesystem BackendType = "filesystem"
	// BackendDistributed uses distributed storage with erasure coding
	BackendDistributed BackendType = "distributed"
)

type BucketOwner

type BucketOwner struct {
	ID          string
	DisplayName string
}

type CompleteMultipartUpload

type CompleteMultipartUpload struct {
	XMLName xml.Name              `xml:"CompleteMultipartUpload"`
	Parts   []MultipartUploadPart `xml:"Part"`
}

type CompleteMultipartUploadResult

type CompleteMultipartUploadResult struct {
	XMLName  xml.Name `xml:"CompleteMultipartUpload"`
	Location string   `xml:"Location"`
	Bucket   string   `xml:"Bucket"`
	Key      string   `xml:"Key"`
	ETag     string   `xml:"ETag"`
	// Both of these are optional
	ChecksumCRC64NVME string `xml:"ChecksumCRC64NVME,omitempty"`
	ChecksumType      string `xml:"ChecksumType,omitempty"`
}

type Config

type Config struct {
	ConfigPath string // Path to config file
	Version    string `toml:"version"`
	Region     string `toml:"region"`

	RS    RS      `toml:"rs"`
	Nodes []Nodes `toml:"nodes"`

	// Distributed database nodes for global state
	DB []DBNode `toml:"db"`

	Buckets []S3_Buckets `toml:"buckets"`

	// TODO: Move to IAM
	Auth                  []AuthEntry `toml:"auth"`
	AllowAnonymousListing bool        `toml:"allow_anonymous_listing"`
	AllowAnonymousAccess  bool        `toml:"allow_anonymous_access"`

	// Only needed on local (filesystem)
	Port int    `toml:"port"`
	Host string `toml:"host"`

	Debug          bool   `toml:"debug"`
	BasePath       string `toml:"base_path"`
	DisableLogging bool   `toml:"disable_logging"`

	// Distributed backend config (deprecated - use DB nodes instead)
	BadgerDir string `toml:"badger_dir"`
}

func New

func New(ConfigSettings *Config) *Config

func (*Config) BucketConfig

func (s3 *Config) BucketConfig(bucket string) (S3_Buckets, error)

func (*Config) ReadConfig

func (s3 *Config) ReadConfig() (err error)

func (*Config) ToFilesystemConfig added in v1.0.0

func (s3 *Config) ToFilesystemConfig() interface{}

ToFilesystemConfig converts s3.Config to filesystem backend config

type CreateBucketConfiguration added in v1.0.0

type CreateBucketConfiguration struct {
	XMLName            xml.Name `xml:"CreateBucketConfiguration"`
	LocationConstraint string   `xml:"LocationConstraint"`
}

CreateBucketConfiguration is the request body for CreateBucket

type CreateBucketResult added in v1.0.0

type CreateBucketResult struct {
	XMLName  xml.Name `xml:"CreateBucketResult"`
	Location string   `xml:"Location"`
}

CreateBucketResult is the response for CreateBucket

type DBNode added in v1.0.0

type DBNode struct {
	ID              int    `toml:"id"`
	Host            string `toml:"host"`
	Port            int    `toml:"port"`
	RaftPort        int    `toml:"raft_port"` // Port for Raft consensus (default: Port + 1000)
	Path            string `toml:"path"`
	AccessKeyID     string `toml:"access_key_id"`
	SecretAccessKey string `toml:"secret_access_key"`
	Leader          bool   `toml:"leader"`
	Epoch           int    `toml:"epoch"`
}

DBNode represents a distributed database node configuration

type HTTP2Server added in v1.0.0

type HTTP2Server struct {
	// contains filtered or unexported fields
}

HTTP2Server is an HTTP/2 compatible S3 server using net/http

func NewHTTP2Server added in v1.0.0

func NewHTTP2Server(config *Config) *HTTP2Server

NewHTTP2Server creates a new HTTP/2 compatible S3 server

func NewHTTP2ServerWithBackend added in v1.0.0

func NewHTTP2ServerWithBackend(config *Config, be backend.Backend) *HTTP2Server

NewHTTP2ServerWithBackend creates a new HTTP/2 server with an existing backend

func (*HTTP2Server) GetHandler added in v1.0.0

func (s *HTTP2Server) GetHandler() http.Handler

GetHandler returns the HTTP handler for testing with httptest

func (*HTTP2Server) GetRouter added in v1.0.0

func (s *HTTP2Server) GetRouter() chi.Router

GetRouter returns the chi router for testing

func (*HTTP2Server) ListenAndServe added in v1.0.0

func (s *HTTP2Server) ListenAndServe(addr, certFile, keyFile string) error

ListenAndServe starts the HTTP/2 server with TLS

func (*HTTP2Server) ListenAndServeAsync added in v1.0.0

func (s *HTTP2Server) ListenAndServeAsync(addr, certFile, keyFile string) error

ListenAndServeAsync starts the server in a goroutine

func (*HTTP2Server) Shutdown added in v1.0.0

func (s *HTTP2Server) Shutdown(ctx context.Context) error

Shutdown gracefully shuts down the server

type InitiateMultipartUploadResult

type InitiateMultipartUploadResult struct {
	XMLName  xml.Name `xml:"InitiateMultipartUploadResult"`
	Bucket   string   `xml:"Bucket"`
	Key      string   `xml:"Key"`
	UploadId string   `xml:"UploadId"`
}

type ListBucket

type ListBucket struct {
	CreationDate time.Time
	Name         string
}

type ListBuckets

type ListBuckets struct {
	XMLName xml.Name `xml:"ListAllMyBucketsResult"`
	Owner   BucketOwner
	Buckets []ListBucket `xml:"Buckets>Bucket"`
}

type ListObjectsV2

type ListObjectsV2 struct {
	XMLName        xml.Name `xml:"ListBucketResult"`
	Name           string
	Prefix         string
	KeyCount       int
	MaxKeys        int
	IsTruncated    bool
	Contents       *[]ListObjectsV2_Contents
	CommonPrefixes *[]ListObjectsV2_Dir
}

type ListObjectsV2_Contents

type ListObjectsV2_Contents struct {
	Key          string
	LastModified time.Time
	ETag         string
	Size         int64
	StorageClass string
}

type ListObjectsV2_Dir

type ListObjectsV2_Dir struct {
	Prefix string
}

type MultipartUploadPart

type MultipartUploadPart struct {
	PartNumber int    `xml:"PartNumber"`
	ETag       string `xml:"ETag"`
}

type Nodes added in v1.0.0

type Nodes struct {
	ID     int    `toml:"id"`
	Host   string `toml:"host"`
	Port   int    `toml:"port"`
	Path   string `toml:"path"`
	DB     bool   `toml:"db"`
	DBPort int    `toml:"dbport"`
	DBPath string `toml:"dbpath"`
	Leader bool   `toml:"leader"`
	Epoch  int    `toml:"epoch"`
}

type Option added in v1.0.0

type Option func(*Server) error

Option configures a Server

func WithAddress added in v1.0.0

func WithAddress(host string, port int) Option

WithAddress sets the server host and port

func WithBackend added in v1.0.0

func WithBackend(backendType BackendType) Option

WithBackend sets the storage backend type. Use BackendFilesystem for simple single-node storage. Use BackendDistributed for distributed storage with erasure coding. If empty string is passed, the default (BackendFilesystem) is used.

func WithBasePath added in v1.0.0

func WithBasePath(path string) Option

WithBasePath sets the base directory for data storage

func WithConfigPath added in v1.0.0

func WithConfigPath(path string) Option

WithConfigPath sets the path to the TOML configuration file

func WithDebug added in v1.0.0

func WithDebug(enabled bool) Option

WithDebug enables debug logging

func WithNodeID added in v1.0.0

func WithNodeID(nodeID int) Option

WithNodeID sets the node ID for distributed mode. Use -1 (default) for dev mode which runs all nodes locally. Use a specific ID to run only that node (for production deployments).

func WithPprof added in v1.0.0

func WithPprof(enabled bool, outputPath string) Option

WithPprof enables CPU profiling. The profile is written to a temp file during operation and saved to outputPath on shutdown. If outputPath is empty, it defaults to /tmp/predastore-cpu.prof

func WithTLS added in v1.0.0

func WithTLS(certPath, keyPath string) Option

WithTLS sets the TLS certificate and key paths

type PolicyRule

type PolicyRule struct {
	Bucket  string   `toml:"bucket"`  // Can be "*" or bucket name
	Actions []string `toml:"actions"` // Like "s3:GetObject", "s3:PutObject", or "s3:*"
}

type RS added in v1.0.0

type RS struct {
	Data   int `toml:"data"`
	Parity int `toml:"parity"`
}

type S3Error

type S3Error struct {
	XMLName    xml.Name `xml:"Error"`
	Code       string   `xml:"Code"`
	Message    string   `xml:"Message"`
	BucketName string   `xml:"BucketName"`
	RequestId  string   `xml:"RequestId"`
	HostId     string   `xml:"HostId"`
}

type S3_Buckets

type S3_Buckets struct {
	Name       string `toml:"name"`
	Region     string `toml:"region"`
	Type       string `toml:"type"`
	Pathname   string `toml:"pathname"`
	Public     bool   `toml:"public"`
	Encryption string `toml:"encryption"`
}

type Server added in v1.0.0

type Server struct {
	// contains filtered or unexported fields
}

Server encapsulates the S3-compatible server with all its components

func NewServer added in v1.0.0

func NewServer(opts ...Option) (*Server, error)

NewServer creates a new S3 server with the given options. By default, it uses the filesystem backend which is simple and requires no additional setup.

For distributed storage with erasure coding, use WithBackend(BackendDistributed).

func (*Server) ListenAndServe added in v1.0.0

func (s *Server) ListenAndServe() error

ListenAndServe starts the server and blocks until shutdown

func (*Server) ListenAndServeAsync added in v1.0.0

func (s *Server) ListenAndServeAsync() error

ListenAndServeAsync starts the server in a goroutine

func (*Server) Shutdown added in v1.0.0

func (s *Server) Shutdown(ctx context.Context) error

Shutdown gracefully shuts down the server

func (*Server) WaitForShutdownSignal added in v1.0.0

func (s *Server) WaitForShutdownSignal()

WaitForShutdownSignal blocks until SIGINT or SIGTERM is received

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL