object

package
v1.4.0-beta1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 29, 2026 License: Apache-2.0 Imports: 134 Imported by: 20

Documentation

Index

Constants

View Source
const (
	// AsyncWriteBack writes the object asynchronously to the backend.
	AsyncWriteBack = iota

	// WriteBack writes the object synchronously to the backend.
	WriteBack

	// Ephemeral only writes the object to the dfdaemon.
	// It is only provided for creating temporary objects between peers,
	// and users are not allowed to use this mode.
	Ephemeral
)
View Source
const (
	// HeaderDragonflyObjectMetaLastModifiedTime is used for last modified time of object storage.
	HeaderDragonflyObjectMetaLastModifiedTime = "X-Dragonfly-Object-Meta-Last-Modified-Time"

	// HeaderDragonflyObjectMetaStorageClass is used for storage class of object storage.
	HeaderDragonflyObjectMetaStorageClass = "X-Dragonfly-Object-Meta-Storage-Class"

	// HeaderDragonflyObjectOperation is used for object storage operation.
	HeaderDragonflyObjectOperation = "X-Dragonfly-Object-Operation"
)
View Source
const (
	// Upper limit of maxGetObjectMetadatas.
	MaxGetObjectMetadatasLimit = 1000

	// DefaultMaxReplicas is the default value of maxReplicas.
	DefaultMaxReplicas = 0

	// Upper limit of maxReplicas.
	MaxReplicasLimit = 100
)
View Source
const (
	// FilterOSS is the filter of oss url for generating task id.
	FilterOSS = "Expires&Signature"

	// FilterS3 is the filter of s3 url for generating task id.
	FilterS3 = "X-Amz-Algorithm&X-Amz-Credential&X-Amz-Date&X-Amz-Expires&X-Amz-SignedHeaders&X-Amz-Signature"

	// FilterOBS is the filter of obs url for generating task id.
	FilterOBS = "X-Amz-Algorithm&X-Amz-Credential&X-Amz-Date&X-Obs-Date&X-Amz-Expires&X-Amz-SignedHeaders&X-Amz-Signature"
)
View Source
const (
	AES256GCM_RSA = "aes256gcm-rsa"
	CHACHA20_RSA  = "chacha20-rsa"
	SM4GCM        = "sm4gcm"
)
View Source
const (
	// CopyOperation is the operation of copying object.
	CopyOperation = "copy"
)
View Source
const DefaultStorageClass = "STANDARD"

Variables

View Source
var ErrKeyNeedPasswd = errors.New("passphrase is required to private key")
View Source
var HEADER_NAMES = []string{"Content-MD5", "Content-Type", "Date"}
View Source
var OVHCompileRegexp = `^s3\.(\w*)(\.\w*)?\.cloud\.ovh\.net$`
View Source
var PutInplace bool
View Source
var TryCFR bool // try copy_file_range
View Source
var UserAgent = "JuiceFS"

Functions

func ExportRsaPrivateKeyToPem

func ExportRsaPrivateKeyToPem(key *rsa.PrivateKey, passphrase string) string

func GetHttpClient

func GetHttpClient() *http.Client

func IsErrReplicating

func IsErrReplicating(err error) bool

func IsFileSystem

func IsFileSystem(object ObjectStorage) bool

func ListAll

func ListAll(ctx context.Context, store ObjectStorage, prefix, marker string, followLink, sort bool) (<-chan Object, error)

ListAll lists all keys that starts at marker from object storage.

func ListAllWithDelimiter

func ListAllWithDelimiter(ctx context.Context, store ObjectStorage, prefix, start, end string, followLink bool) (<-chan Object, error)

func MarshalObject

func MarshalObject(o Object) map[string]interface{}

func NewDataEncryptor

func NewDataEncryptor(keyEncryptor Encryptor, algo string) (*dataEncryptor, error)

func ParsePrivateKeyFromPem

func ParsePrivateKeyFromPem(enc []byte, passphrase []byte) (any, error)

func ParseRsaPrivateKeyFromPath

func ParseRsaPrivateKeyFromPath(path, passphrase string) (any, error)

func Register

func Register(name string, register Creator)

func Shutdown

func Shutdown(o ObjectStorage)

func TmpFilePath

func TmpFilePath(parent, name string) string

Types

type AttrGetter

type AttrGetter func(attrs *ResponseAttrs)

func WithRequestID

func WithRequestID(id *string) AttrGetter

func WithRequestSize

func WithRequestSize(size *int64) AttrGetter

func WithStorageClass

func WithStorageClass(sc *string) AttrGetter

type COS

type COS struct {
	// contains filtered or unexported fields
}

func (*COS) AbortUpload

func (c *COS) AbortUpload(ctx context.Context, key string, uploadID string)

func (*COS) CompleteUpload

func (c *COS) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error

func (*COS) Copy

func (c *COS) Copy(ctx context.Context, dst, src string) error

func (*COS) Create

func (c *COS) Create(ctx context.Context) error

func (*COS) CreateMultipartUpload

func (c *COS) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error)

func (*COS) Delete

func (c *COS) Delete(ctx context.Context, key string, getters ...AttrGetter) error

func (*COS) Get

func (c *COS) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)

func (*COS) GetStorageClass

func (b *COS) GetStorageClass(ctx context.Context) string

func (*COS) Head

func (c *COS) Head(ctx context.Context, key string) (Object, error)

func (*COS) Limits

func (c *COS) Limits() Limits

func (*COS) List

func (c *COS) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error)

func (*COS) ListAll

func (c *COS) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error)

func (*COS) ListUploads

func (c *COS) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error)

func (*COS) Put

func (c *COS) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error

func (*COS) Restore

func (c *COS) Restore(ctx context.Context, key string) error

func (*COS) SetStorageClass

func (c *COS) SetStorageClass(sc string) error

func (*COS) SetTier

func (b *COS) SetTier(init Tiers)

func (*COS) String

func (c *COS) String() string

func (*COS) UploadPart

func (c *COS) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error)

func (*COS) UploadPartCopy

func (c *COS) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error)

type CommonPrefixesItem added in v0.1.3

type CommonPrefixesItem struct {
	Prefix string
}

type ContentsItem added in v0.1.3

type ContentsItem struct {
	Key          string
	Size         string
	LastModified int
	CreateTime   int
	StorageClass string
	ETag         string
}

type Creator

type Creator func(bucket, accessKey, secretKey, token string) (ObjectStorage, error)

type DefaultObjectStorage

type DefaultObjectStorage struct{}

func (DefaultObjectStorage) AbortUpload

func (s DefaultObjectStorage) AbortUpload(ctx context.Context, key string, uploadID string)

func (DefaultObjectStorage) CompleteUpload

func (s DefaultObjectStorage) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error

func (DefaultObjectStorage) Copy

func (s DefaultObjectStorage) Copy(ctx context.Context, dst, src string) error

func (DefaultObjectStorage) Create

func (s DefaultObjectStorage) Create(ctx context.Context) error

func (DefaultObjectStorage) CreateMultipartUpload

func (s DefaultObjectStorage) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error)

func (DefaultObjectStorage) Head

func (s DefaultObjectStorage) Head(key string) (Object, error)

func (DefaultObjectStorage) Limits

func (s DefaultObjectStorage) Limits() Limits

func (DefaultObjectStorage) List

func (s DefaultObjectStorage) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error)

func (DefaultObjectStorage) ListAll

func (s DefaultObjectStorage) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error)

func (DefaultObjectStorage) ListUploads

func (s DefaultObjectStorage) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error)

func (DefaultObjectStorage) Restore

func (s DefaultObjectStorage) Restore(ctx context.Context, key string) error

func (DefaultObjectStorage) UploadPart

func (s DefaultObjectStorage) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error)

func (DefaultObjectStorage) UploadPartCopy

func (s DefaultObjectStorage) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error)

type Encryptor

type Encryptor interface {
	Encrypt(plaintext []byte) ([]byte, error)
	Decrypt(ciphertext []byte) ([]byte, error)
}

func NewKeyEncryptor

func NewKeyEncryptor(privKey any) Encryptor

func NewRSAEncryptor

func NewRSAEncryptor(privKey *rsa.PrivateKey) Encryptor

func NewSM2Encryptor

func NewSM2Encryptor(privKey *sm2.PrivateKey) Encryptor

type File

type File interface {
	Object
	Owner() string
	Group() string
	Mode() os.FileMode
}

type FileSystem

type FileSystem interface {
	MtimeChanger
	Chmod(path string, mode os.FileMode) error
	Chown(path string, owner, group string) error
}

type Limits

type Limits struct {
	IsSupportMultipartUpload bool
	IsSupportUploadPartCopy  bool
	MinPartSize              int
	MaxPartSize              int64
	MaxPartCount             int
}

type MtimeChanger

type MtimeChanger interface {
	Chtimes(path string, mtime time.Time) error
}

type MultipartUpload

type MultipartUpload struct {
	MinPartSize int
	MaxCount    int
	UploadID    string
}

type Object

type Object interface {
	Key() string
	Size() int64
	Mtime() time.Time
	IsDir() bool
	IsSymlink() bool
	StorageClass() string
	Status() string
}

func UnmarshalObject

func UnmarshalObject(m map[string]interface{}) Object

type ObjectMetadata

type ObjectMetadata struct {
	// Key is object key.
	Key string

	// ContentDisposition is Content-Disposition header.
	ContentDisposition string

	// ContentEncoding is Content-Encoding header.
	ContentEncoding string

	// ContentLanguage is Content-Language header.
	ContentLanguage string

	// ContentLength is Content-Length header.
	ContentLength int64

	// ContentType is Content-Type header.
	ContentType string

	// ETag is ETag header.
	ETag string

	// Digest is object digest.
	Digest string

	// LastModifiedTime is last modified time.
	LastModifiedTime time.Time

	// StorageClass is object storage class.
	StorageClass string
}

ObjectMetadata is the object metadata.

type ObjectMetadatas

type ObjectMetadatas struct {
	// CommonPrefixes are similar prefixes in object storage.
	CommonPrefixes []string `json:"CommonPrefixes"`

	// Metadatas are object metadata.
	Metadatas []*ObjectMetadata `json:"Metadatas"`
}

ObjectMetadatas is the object metadata list.

type ObjectStorage

type ObjectStorage interface {
	// Description of the object storage.
	String() string
	// Limits of the object storage.
	Limits() Limits
	// Create the bucket if not existed.
	Create(ctx context.Context) error
	// Get the data for the given object specified by key.
	Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)
	// Put data read from a reader to an object specified by key.
	Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error
	// Copy an object from src to dst.
	Copy(ctx context.Context, dst, src string) error
	// Delete a object.
	Delete(ctx context.Context, key string, getters ...AttrGetter) error

	// Head returns some information about the object or an error if not found.
	Head(ctx context.Context, key string) (Object, error)
	// List returns a list of objects using ListObjectV2.
	List(ctx context.Context, prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error)
	// ListAll returns all the objects as a channel.
	ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error)

	// CreateMultipartUpload starts to upload a large object part by part.
	CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error)
	// UploadPart upload a part of an object.
	UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error)
	// UploadPartCopy Uploads a part by copying data from an existing object as data source.
	UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error)
	// AbortUpload abort a multipart upload.
	AbortUpload(ctx context.Context, key string, uploadID string)
	// CompleteUpload finish a multipart upload.
	CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error
	// ListUploads lists existing multipart uploads.
	ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error)
	// Restore restores an archived object to be available for read.
	Restore(ctx context.Context, key string) error
}

ObjectStorage is the interface for object storage. all of these API should be idempotent.

func CreateStorage

func CreateStorage(name, endpoint, accessKey, secretKey, token string) (ObjectStorage, error)

func NewChunkedEncrypted

func NewChunkedEncrypted(o ObjectStorage, enc *dataEncryptor) ObjectStorage

func NewEncrypted

func NewEncrypted(o ObjectStorage, enc Encryptor) ObjectStorage

NewEncrypted returns a encrypted object storage

func NewSharded

func NewSharded(name, endpoint, ak, sk, token string, shards int) (ObjectStorage, error)

func WithPrefix

func WithPrefix(os ObjectStorage, prefix string) ObjectStorage

WithPrefix return an object storage that add a prefix to keys.

type ObjectStorageMetadata

type ObjectStorageMetadata struct {
	// Name is object storage name of type, it can be s3, oss or obs.
	Name string

	// Region is storage region.
	Region string

	// Endpoint is datacenter endpoint.
	Endpoint string
}

ObjectStorageMetadata is the object storage metadata.

type Part

type Part struct {
	Num  int
	Size int
	ETag string
}

type PendingPart

type PendingPart struct {
	Key      string
	UploadID string
	Created  time.Time
}

type ResponseAttrs

type ResponseAttrs struct {
	// contains filtered or unexported fields
}

A generic way to get attributes from different object storage clients

func ApplyGetters

func ApplyGetters(getters ...AttrGetter) ResponseAttrs

func (*ResponseAttrs) GetRequestSize

func (r *ResponseAttrs) GetRequestSize() int64

func (*ResponseAttrs) SetRequestID

func (r *ResponseAttrs) SetRequestID(id string) *ResponseAttrs

func (*ResponseAttrs) SetStorageClass

func (r *ResponseAttrs) SetStorageClass(sc string) *ResponseAttrs

type RestfulStorage

type RestfulStorage struct {
	DefaultObjectStorage
	// contains filtered or unexported fields
}

func (*RestfulStorage) Copy

func (s *RestfulStorage) Copy(ctx context.Context, dst, src string) error

func (*RestfulStorage) Delete

func (s *RestfulStorage) Delete(ctx context.Context, key string, getters ...AttrGetter) error

func (*RestfulStorage) Get

func (s *RestfulStorage) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)

func (*RestfulStorage) Head

func (s *RestfulStorage) Head(ctx context.Context, key string) (Object, error)

func (*RestfulStorage) List

func (s *RestfulStorage) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error)

func (*RestfulStorage) Put

func (u *RestfulStorage) Put(ctx context.Context, key string, body io.Reader, getters ...AttrGetter) error

func (*RestfulStorage) String

func (s *RestfulStorage) String() string

type SectionReaderCloser

type SectionReaderCloser struct {
	*io.SectionReader
	io.Closer
}

type Shutdownable

type Shutdownable interface {
	Shutdown()
}

type SupportStorageClass

type SupportStorageClass interface {
	SetStorageClass(sc string) error
}
type SupportSymlink interface {
	// Symlink create a symbolic link
	Symlink(oldName, newName string) error
	// Readlink read a symbolic link
	Readlink(name string) (string, error)
}

type SupportTier

type SupportTier interface {
	SetTier(init Tiers)
	GetStorageClass(ctx context.Context) string
}

type Tier

type Tier struct {
	ID uint8  `json:"ID"`
	Sc string `json:"StorageClass"`
}

func (Tier) GetHumanSc

func (t Tier) GetHumanSc() string

func (Tier) MarshalJSON

func (t Tier) MarshalJSON() ([]byte, error)

func (*Tier) UnmarshalJSON

func (t *Tier) UnmarshalJSON(data []byte) error

type TierKey

type TierKey struct{}

type Tiers

type Tiers map[uint8]Tier

func NewTiers

func NewTiers() Tiers

func (Tiers) GetID

func (t Tiers) GetID(sc string) (uint8, bool)

func (Tiers) GetSc

func (t Tiers) GetSc(id uint8) (string, bool)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL