Documentation
¶
Index ¶
- Constants
- Variables
- func ExportRsaPrivateKeyToPem(key *rsa.PrivateKey, passphrase string) string
- func GetHttpClient() *http.Client
- func IsErrReplicating(err error) bool
- func IsFileSystem(object ObjectStorage) bool
- func ListAll(ctx context.Context, store ObjectStorage, prefix, marker string, ...) (<-chan Object, error)
- func ListAllWithDelimiter(ctx context.Context, store ObjectStorage, prefix, start, end string, ...) (<-chan Object, error)
- func MarshalObject(o Object) map[string]interface{}
- func NewDataEncryptor(keyEncryptor Encryptor, algo string) (*dataEncryptor, error)
- func ParsePrivateKeyFromPem(enc []byte, passphrase []byte) (any, error)
- func ParseRsaPrivateKeyFromPath(path, passphrase string) (any, error)
- func Register(name string, register Creator)
- func Shutdown(o ObjectStorage)
- func TmpFilePath(parent, name string) string
- type AttrGetter
- type COS
- func (c *COS) AbortUpload(ctx context.Context, key string, uploadID string)
- func (c *COS) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error
- func (c *COS) Copy(ctx context.Context, dst, src string) error
- func (c *COS) Create(ctx context.Context) error
- func (c *COS) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error)
- func (c *COS) Delete(ctx context.Context, key string, getters ...AttrGetter) error
- func (c *COS) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)
- func (b *COS) GetStorageClass(ctx context.Context) string
- func (c *COS) Head(ctx context.Context, key string) (Object, error)
- func (c *COS) Limits() Limits
- func (c *COS) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, ...) ([]Object, bool, string, error)
- func (c *COS) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error)
- func (c *COS) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error)
- func (c *COS) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error
- func (c *COS) Restore(ctx context.Context, key string, days int32) error
- func (b *COS) SetTier(init Tiers) error
- func (c *COS) String() string
- func (c *COS) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error)
- func (c *COS) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, ...) (*Part, error)
- type CommonPrefixesItem
- type ContentsItem
- type Creator
- type DefaultObjectStorage
- func (s DefaultObjectStorage) AbortUpload(ctx context.Context, key string, uploadID string)
- func (s DefaultObjectStorage) CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error
- func (s DefaultObjectStorage) Copy(ctx context.Context, dst, src string) error
- func (s DefaultObjectStorage) Create(ctx context.Context) error
- func (s DefaultObjectStorage) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error)
- func (s DefaultObjectStorage) Head(key string) (Object, error)
- func (s DefaultObjectStorage) Limits() Limits
- func (s DefaultObjectStorage) List(ctx context.Context, prefix, start, token, delimiter string, limit int64, ...) ([]Object, bool, string, error)
- func (s DefaultObjectStorage) ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error)
- func (s DefaultObjectStorage) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error)
- func (s DefaultObjectStorage) Restore(ctx context.Context, key string, days int32) error
- func (s DefaultObjectStorage) UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error)
- func (s DefaultObjectStorage) UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, ...) (*Part, error)
- type Encryptor
- type File
- type FileSystem
- type Limits
- type MtimeChanger
- type MultipartUpload
- type Object
- type ObjectMetadata
- type ObjectMetadatas
- type ObjectStorage
- func CreateStorage(name, endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
- func DirStorage(s ObjectStorage) ObjectStorage
- func NewChunkedEncrypted(o ObjectStorage, enc *dataEncryptor) ObjectStorage
- func NewEncrypted(o ObjectStorage, enc Encryptor) ObjectStorage
- func NewSharded(name, endpoint, ak, sk, token string, shards int) (ObjectStorage, error)
- func WithPrefix(os ObjectStorage, prefix string) ObjectStorage
- type ObjectStorageMetadata
- type Part
- type PendingPart
- type ResponseAttrs
- type RestfulStorage
- func (s *RestfulStorage) Copy(ctx context.Context, dst, src string) error
- func (s *RestfulStorage) Delete(ctx context.Context, key string, getters ...AttrGetter) error
- func (s *RestfulStorage) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)
- func (s *RestfulStorage) Head(ctx context.Context, key string) (Object, error)
- func (s *RestfulStorage) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, ...) ([]Object, bool, string, error)
- func (u *RestfulStorage) Put(ctx context.Context, key string, body io.Reader, getters ...AttrGetter) error
- func (s *RestfulStorage) String() string
- type SectionReaderCloser
- type Shutdownable
- type SupportSymlink
- type SupportTier
- type Tier
- type TierKey
- type Tiers
Constants ¶
const ( // AsyncWriteBack writes the object asynchronously to the backend. AsyncWriteBack = iota // WriteBack writes the object synchronously to the backend. WriteBack // Ephemeral only writes the object to the dfdaemon. // It is only provided for creating temporary objects between peers, // and users are not allowed to use this mode. Ephemeral )
const ( // HeaderDragonflyObjectMetaLastModifiedTime is used for last modified time of object storage. HeaderDragonflyObjectMetaLastModifiedTime = "X-Dragonfly-Object-Meta-Last-Modified-Time" // HeaderDragonflyObjectMetaStorageClass is used for storage class of object storage. HeaderDragonflyObjectMetaStorageClass = "X-Dragonfly-Object-Meta-Storage-Class" // HeaderDragonflyObjectOperation is used for object storage operation. HeaderDragonflyObjectOperation = "X-Dragonfly-Object-Operation" )
const ( // Upper limit of maxGetObjectMetadatas. MaxGetObjectMetadatasLimit = 1000 // DefaultMaxReplicas is the default value of maxReplicas. DefaultMaxReplicas = 0 // Upper limit of maxReplicas. MaxReplicasLimit = 100 )
const ( // FilterOSS is the filter of oss url for generating task id. FilterOSS = "Expires&Signature" // FilterS3 is the filter of s3 url for generating task id. FilterS3 = "X-Amz-Algorithm&X-Amz-Credential&X-Amz-Date&X-Amz-Expires&X-Amz-SignedHeaders&X-Amz-Signature" // FilterOBS is the filter of obs url for generating task id. FilterOBS = "X-Amz-Algorithm&X-Amz-Credential&X-Amz-Date&X-Obs-Date&X-Amz-Expires&X-Amz-SignedHeaders&X-Amz-Signature" )
const ( AES256GCM_RSA = "aes256gcm-rsa" CHACHA20_RSA = "chacha20-rsa" SM4GCM = "sm4gcm" )
const (
// CopyOperation is the operation of copying object.
CopyOperation = "copy"
)
const DefaultRestoreDays = 3
const DefaultStorageClass = "STANDARD"
Variables ¶
var ErrKeyNeedPasswd = errors.New("passphrase is required to private key")
var HEADER_NAMES = []string{"Content-MD5", "Content-Type", "Date"}
var OVHCompileRegexp = `^s3\.(\w*)(\.\w*)?\.cloud\.ovh\.net$`
var PutInplace bool
var TryCFR bool // try copy_file_range
var UserAgent = "JuiceFS"
Functions ¶
func ExportRsaPrivateKeyToPem ¶
func ExportRsaPrivateKeyToPem(key *rsa.PrivateKey, passphrase string) string
func GetHttpClient ¶
func IsErrReplicating ¶
func IsFileSystem ¶
func IsFileSystem(object ObjectStorage) bool
func ListAll ¶
func ListAll(ctx context.Context, store ObjectStorage, prefix, marker string, followLink, sort bool) (<-chan Object, error)
ListAll lists all keys that starts at marker from object storage.
func ListAllWithDelimiter ¶
func MarshalObject ¶
func NewDataEncryptor ¶
func ParsePrivateKeyFromPem ¶
func Shutdown ¶
func Shutdown(o ObjectStorage)
func TmpFilePath ¶
Types ¶
type AttrGetter ¶
type AttrGetter func(attrs *ResponseAttrs)
func WithRequestID ¶
func WithRequestID(id *string) AttrGetter
func WithRequestSize ¶
func WithRequestSize(size *int64) AttrGetter
func WithStorageClass ¶
func WithStorageClass(sc *string) AttrGetter
type COS ¶
type COS struct {
// contains filtered or unexported fields
}
func (*COS) AbortUpload ¶
func (*COS) CompleteUpload ¶
func (*COS) CreateMultipartUpload ¶
func (*COS) Get ¶
func (c *COS) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)
func (*COS) GetStorageClass ¶
func (*COS) ListUploads ¶
func (*COS) UploadPart ¶
type CommonPrefixesItem ¶ added in v0.1.3
type CommonPrefixesItem struct {
Prefix string
}
type ContentsItem ¶ added in v0.1.3
type Creator ¶
type Creator func(bucket, accessKey, secretKey, token string) (ObjectStorage, error)
type DefaultObjectStorage ¶
type DefaultObjectStorage struct{}
func (DefaultObjectStorage) AbortUpload ¶
func (s DefaultObjectStorage) AbortUpload(ctx context.Context, key string, uploadID string)
func (DefaultObjectStorage) CompleteUpload ¶
func (DefaultObjectStorage) Copy ¶
func (s DefaultObjectStorage) Copy(ctx context.Context, dst, src string) error
func (DefaultObjectStorage) Create ¶
func (s DefaultObjectStorage) Create(ctx context.Context) error
func (DefaultObjectStorage) CreateMultipartUpload ¶
func (s DefaultObjectStorage) CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error)
func (DefaultObjectStorage) Limits ¶
func (s DefaultObjectStorage) Limits() Limits
func (DefaultObjectStorage) ListUploads ¶
func (s DefaultObjectStorage) ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error)
func (DefaultObjectStorage) UploadPart ¶
type Encryptor ¶
type Encryptor interface {
Encrypt(plaintext []byte) ([]byte, error)
Decrypt(ciphertext []byte) ([]byte, error)
}
func NewKeyEncryptor ¶
func NewRSAEncryptor ¶
func NewRSAEncryptor(privKey *rsa.PrivateKey) Encryptor
func NewSM2Encryptor ¶
func NewSM2Encryptor(privKey *sm2.PrivateKey) Encryptor
type FileSystem ¶
type MultipartUpload ¶
type Object ¶
type Object interface {
Key() string
Size() int64
Mtime() time.Time
IsDir() bool
IsSymlink() bool
StorageClass() string
Status() string
}
func UnmarshalObject ¶
type ObjectMetadata ¶
type ObjectMetadata struct {
// Key is object key.
Key string
// ContentDisposition is Content-Disposition header.
ContentDisposition string
// ContentEncoding is Content-Encoding header.
ContentEncoding string
// ContentLanguage is Content-Language header.
ContentLanguage string
// ContentLength is Content-Length header.
ContentLength int64
// ContentType is Content-Type header.
ContentType string
// ETag is ETag header.
ETag string
// Digest is object digest.
Digest string
// LastModifiedTime is last modified time.
LastModifiedTime time.Time
// StorageClass is object storage class.
StorageClass string
}
ObjectMetadata is the object metadata.
type ObjectMetadatas ¶
type ObjectMetadatas struct {
// CommonPrefixes are similar prefixes in object storage.
CommonPrefixes []string `json:"CommonPrefixes"`
// Metadatas are object metadata.
Metadatas []*ObjectMetadata `json:"Metadatas"`
}
ObjectMetadatas is the object metadata list.
type ObjectStorage ¶
type ObjectStorage interface {
// Description of the object storage.
String() string
// Limits of the object storage.
Limits() Limits
// Create the bucket if not existed.
Create(ctx context.Context) error
// Get the data for the given object specified by key.
Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)
// Put data read from a reader to an object specified by key.
Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) error
// Copy an object from src to dst.
Copy(ctx context.Context, dst, src string) error
// Delete a object.
Delete(ctx context.Context, key string, getters ...AttrGetter) error
// Head returns some information about the object or an error if not found.
Head(ctx context.Context, key string) (Object, error)
// List returns a list of objects using ListObjectV2.
List(ctx context.Context, prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error)
// ListAll returns all the objects as a channel.
ListAll(ctx context.Context, prefix, marker string, followLink bool) (<-chan Object, error)
// CreateMultipartUpload starts to upload a large object part by part.
CreateMultipartUpload(ctx context.Context, key string) (*MultipartUpload, error)
// UploadPart upload a part of an object.
UploadPart(ctx context.Context, key string, uploadID string, num int, body []byte) (*Part, error)
// UploadPartCopy Uploads a part by copying data from an existing object as data source.
UploadPartCopy(ctx context.Context, key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error)
// AbortUpload abort a multipart upload.
AbortUpload(ctx context.Context, key string, uploadID string)
// CompleteUpload finish a multipart upload.
CompleteUpload(ctx context.Context, key string, uploadID string, parts []*Part) error
// ListUploads lists existing multipart uploads.
ListUploads(ctx context.Context, marker string) ([]*PendingPart, string, error)
// Restore restores an archived object to be available for read.
Restore(ctx context.Context, key string, days int32) error
}
ObjectStorage is the interface for object storage. all of these API should be idempotent.
func CreateStorage ¶
func CreateStorage(name, endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
func DirStorage ¶
func DirStorage(s ObjectStorage) ObjectStorage
DirStorage returns an ObjectStorage representing the parent directory of s. If s already represents a directory (String() ends with "/"), it is returned unchanged. For file-like storages, returns a new storage rooted at the parent directory.
func NewChunkedEncrypted ¶
func NewChunkedEncrypted(o ObjectStorage, enc *dataEncryptor) ObjectStorage
func NewEncrypted ¶
func NewEncrypted(o ObjectStorage, enc Encryptor) ObjectStorage
NewEncrypted returns a encrypted object storage
func NewSharded ¶
func NewSharded(name, endpoint, ak, sk, token string, shards int) (ObjectStorage, error)
func WithPrefix ¶
func WithPrefix(os ObjectStorage, prefix string) ObjectStorage
WithPrefix return an object storage that add a prefix to keys.
type ObjectStorageMetadata ¶
type ObjectStorageMetadata struct {
// Name is object storage name of type, it can be s3, oss or obs.
Name string
// Region is storage region.
Region string
// Endpoint is datacenter endpoint.
Endpoint string
}
ObjectStorageMetadata is the object storage metadata.
type ResponseAttrs ¶
type ResponseAttrs struct {
// contains filtered or unexported fields
}
A generic way to get attributes from different object storage clients
func ApplyGetters ¶
func ApplyGetters(getters ...AttrGetter) ResponseAttrs
func (*ResponseAttrs) GetRequestSize ¶
func (r *ResponseAttrs) GetRequestSize() int64
func (*ResponseAttrs) SetRequestID ¶
func (r *ResponseAttrs) SetRequestID(id string) *ResponseAttrs
func (*ResponseAttrs) SetStorageClass ¶
func (r *ResponseAttrs) SetStorageClass(sc string) *ResponseAttrs
type RestfulStorage ¶
type RestfulStorage struct {
DefaultObjectStorage
// contains filtered or unexported fields
}
func (*RestfulStorage) Copy ¶
func (s *RestfulStorage) Copy(ctx context.Context, dst, src string) error
func (*RestfulStorage) Delete ¶
func (s *RestfulStorage) Delete(ctx context.Context, key string, getters ...AttrGetter) error
func (*RestfulStorage) Get ¶
func (s *RestfulStorage) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)
func (*RestfulStorage) Put ¶
func (u *RestfulStorage) Put(ctx context.Context, key string, body io.Reader, getters ...AttrGetter) error
func (*RestfulStorage) String ¶
func (s *RestfulStorage) String() string
type SectionReaderCloser ¶
type SectionReaderCloser struct {
*io.SectionReader
io.Closer
}
type Shutdownable ¶
type Shutdownable interface {
Shutdown()
}
type SupportSymlink ¶
type SupportTier ¶
Source Files
¶
- azure.go
- b2.go
- bos.go
- checksum.go
- cifs.go
- cos.go
- dragonfly.go
- encrypt.go
- encrypt_chunked.go
- eos.go
- etcd.go
- file.go
- file_linux.go
- file_unix.go
- gs.go
- hdfs.go
- hdfs_kerberos.go
- ibmcos.go
- interface.go
- ks3.go
- mem.go
- minio.go
- nfs.go
- object_storage.go
- obs.go
- oos.go
- oss.go
- prefix.go
- qingstor.go
- qiniu.go
- redis.go
- response_attrs.go
- restful.go
- s3.go
- scw.go
- sftp.go
- sharding.go
- space.go
- sql.go
- sql_mysql.go
- sql_pg.go
- sql_sqlite.go
- storj.go
- swift.go
- tikv.go
- tos.go
- ufile.go
- wasabi.go
- webdav.go