core

package
v1.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 6, 2025 License: Apache-2.0 Imports: 60 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ADL2_CLIENT_REQUEST_ID = "X-Ms-Client-Request-Id"
	ADL2_REQUEST_ID        = "X-Ms-Request-Id"
)
View Source
const (
	AzuriteEndpoint               = "http://127.0.0.1:8080/devstoreaccount1/"
	AzureDirBlobMetadataKey       = "hdi_isfolder"
	AzureBlobMetaDataHeaderPrefix = "x-ms-meta-"
)
View Source
const (
	CGROUP_PATH           = "/proc/self/cgroup"
	CGROUP_FOLDER_PREFIX  = "/sys/fs/cgroup/memory"
	MEM_LIMIT_FILE_SUFFIX = "/memory.limit_in_bytes"
	MEM_USAGE_FILE_SUFFIX = "/memory.usage_in_bytes"
)
View Source
const (
	N_INODES  = fuseops.InodeID(1 << 32)
	N_HANDLES = fuseops.HandleID(1 << 32)

	STEAL_INODE_BACKOFF = 200 * time.Millisecond

	UNKNOWN_OWNER = 0
)
View Source
const (
	SRC_NODE_ID_METADATA_KEY = "src-node-id"
	DST_NODE_ID_METADATA_KEY = "dst-node-id"
)
View Source
const (
	IOV_MAX            = 1024
	READ_BUF_SIZE      = 128 * 1024
	MAX_FLUSH_PRIORITY = 3
)

On Linux and MacOS, IOV_MAX = 1024

View Source
const (
	FALLOC_FL_KEEP_SIZE      = uint32(0x01)
	FALLOC_FL_PUNCH_HOLE     = uint32(0x02)
	FALLOC_FL_COLLAPSE_RANGE = uint32(0x08)
	FALLOC_FL_ZERO_RANGE     = uint32(0x10)
	FALLOC_FL_INSERT_RANGE   = uint32(0x20)
)
View Source
const (
	ST_CACHED   int32 = 0
	ST_DEAD     int32 = 1
	ST_CREATED  int32 = 2
	ST_MODIFIED int32 = 3
	ST_DELETED  int32 = 4
)
View Source
const (
	XATTR_CREATE  = unix.XATTR_CREATE
	XATTR_REPLACE = unix.XATTR_REPLACE
	ENOATTR       = unix.ENODATA
)
View Source
const ADL1_REQUEST_ID = "X-Ms-Request-Id"
View Source
const INIT_ERR_BLOB = "mount.err"
View Source
const MAX_BUF = 2 * 1024 * 1024
View Source
const OUTSTAGE_TIMEOUT = 10 * time.Second
View Source
const READY_OWNER_BACKOFF = 100 * time.Millisecond
View Source
const STAT_PRINT_INTERVAL = 1 * time.Second

Variables

View Source
var (
	ErrBufferIsMissing = errors.New("tried to read from a missing buffer")
	ErrBufferIsLoading = errors.New("tried to read from a loading buffer")
)
View Source
var SmallActionsGate = make(chan int, 100)
View Source
var TIME_MAX = time.Unix(1<<63-62135596801, 999999999)

Functions

func Dup

func Dup(value []byte) []byte

func IsADLv1Endpoint

func IsADLv1Endpoint(endpoint string) bool

func IsADLv2Endpoint

func IsADLv2Endpoint(endpoint string) bool

func LogClientInterceptor

func LogClientInterceptor(ctx context.Context, method string, req, resp interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error

func LogServerInterceptor

func LogServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error)

func MaxInt

func MaxInt(a, b int) int

func MaxInt64

func MaxInt64(a, b int64) int64

func MaxUInt32

func MaxUInt32(a, b uint32) uint32

func MaxUInt64

func MaxUInt64(a, b uint64) uint64

func MinInt

func MinInt(a, b int) int

func MinInt64

func MinInt64(a, b int64) int64

func MinUInt32

func MinUInt32(a, b uint32) uint32

func MinUInt64

func MinUInt64(a, b uint64) uint64

func MountCluster

func MountCluster(
	ctx context.Context,
	bucketName string,
	flags *cfg.FlagStorage,
) (*Goofys, MountedFS, error)

func MountFuse

func MountFuse(
	ctx context.Context,
	bucketName string,
	flags *cfg.FlagStorage,
) (fs *Goofys, mfs MountedFS, err error)

Mount the file system based on the supplied arguments, returning a fuse.MountedFileSystem that can be joined to wait for unmounting.

func NilInt64

func NilInt64(v *int64) int64

func NilStr

func NilStr(v *string) string

func NilUInt32

func NilUInt32(v *uint32) uint32

func PBool

func PBool(v bool) *bool

func PInt32

func PInt32(v int32) *int32

func PInt64

func PInt64(v int64) *int64

func PMetadata

func PMetadata(m map[string]string) map[string]*string

func PString

func PString(v string) *string

func PTime

func PTime(v time.Time) *time.Time

func PUInt32

func PUInt32(v uint32) *uint32

func PUInt64

func PUInt64(v uint64) *uint64

func ReadBackoff

func ReadBackoff(flags *cfg.FlagStorage, try func(attempt int) error) (err error)

FIXME: Add similar write backoff (now it's handled by file/dir code)

func S3Debug

func S3Debug(l *log.LogHandle, params any, msg string)

func SignV2

func SignV2(req *request.Request)

Sign requests with signature version 2.

Will sign the requests with the service config's Credentials object Signing is skipped if the credentials is the credentials.AnonymousCredentials object.

func TryUnmount

func TryUnmount(mountPoint string) (err error)

Types

type ADL2Error

type ADL2Error struct {
	adl2.DataLakeStorageError
}

func (ADL2Error) Error

func (e ADL2Error) Error() string

type ADLv1

type ADLv1 struct {
	// contains filtered or unexported fields
}

func NewADLv1

func NewADLv1(bucket string, flags *cfg.FlagStorage, config *cfg.ADLv1Config) (*ADLv1, error)

func (*ADLv1) Bucket

func (b *ADLv1) Bucket() string

func (*ADLv1) Capabilities

func (b *ADLv1) Capabilities() *Capabilities

func (*ADLv1) CopyBlob

func (b *ADLv1) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)

func (*ADLv1) Delegate

func (b *ADLv1) Delegate() interface{}

func (*ADLv1) DeleteBlob

func (b *ADLv1) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)

func (*ADLv1) DeleteBlobs

func (b *ADLv1) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)

func (*ADLv1) GetBlob

func (b *ADLv1) GetBlob(param *GetBlobInput) (*GetBlobOutput, error)

func (*ADLv1) HeadBlob

func (b *ADLv1) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)

func (*ADLv1) Init

func (b *ADLv1) Init(key string) error

func (*ADLv1) ListBlobs

func (b *ADLv1) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*ADLv1) MakeBucket

func (b *ADLv1) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)

func (*ADLv1) MultipartBlobAbort

func (b *ADLv1) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)

func (*ADLv1) MultipartBlobAdd

func (b *ADLv1) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)

func (*ADLv1) MultipartBlobBegin

func (b *ADLv1) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)

func (*ADLv1) MultipartBlobCommit

func (b *ADLv1) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)

func (*ADLv1) MultipartBlobCopy

func (b *ADLv1) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

func (*ADLv1) MultipartExpire

func (b *ADLv1) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)

func (*ADLv1) PatchBlob

func (s *ADLv1) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

func (*ADLv1) PutBlob

func (b *ADLv1) PutBlob(param *PutBlobInput) (*PutBlobOutput, error)

func (*ADLv1) RemoveBucket

func (b *ADLv1) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)

func (*ADLv1) RenameBlob

func (b *ADLv1) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)

type ADLv1Err

type ADLv1Err struct {
	RemoteException struct {
		Exception     string
		Message       string
		JavaClassName string
	}
	// contains filtered or unexported fields
}

func (ADLv1Err) Error

func (err ADLv1Err) Error() string

type ADLv1MultipartBlobCommitInput

type ADLv1MultipartBlobCommitInput struct {
	Size uint64
}

type ADLv2

type ADLv2 struct {
	// contains filtered or unexported fields
}

func NewADLv2

func NewADLv2(bucket string, flags *cfg.FlagStorage, config *cfg.ADLv2Config) (*ADLv2, error)

func (*ADLv2) Bucket

func (b *ADLv2) Bucket() string

func (*ADLv2) Capabilities

func (b *ADLv2) Capabilities() *Capabilities

func (*ADLv2) CopyBlob

func (b *ADLv2) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)

func (*ADLv2) Delegate

func (b *ADLv2) Delegate() interface{}

func (*ADLv2) DeleteBlob

func (b *ADLv2) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)

func (*ADLv2) DeleteBlobs

func (b *ADLv2) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)

func (*ADLv2) GetBlob

func (b *ADLv2) GetBlob(param *GetBlobInput) (*GetBlobOutput, error)

func (*ADLv2) HeadBlob

func (b *ADLv2) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)

func (*ADLv2) Init

func (b *ADLv2) Init(key string) (err error)

func (*ADLv2) ListBlobs

func (b *ADLv2) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*ADLv2) MakeBucket

func (b *ADLv2) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)

func (*ADLv2) MultipartBlobAbort

func (b *ADLv2) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)

func (*ADLv2) MultipartBlobAdd

func (b *ADLv2) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)

func (*ADLv2) MultipartBlobBegin

func (b *ADLv2) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)

adlv2 doesn't have atomic multipart upload, instead we will hold a lease, replace the object, then release the lease

func (*ADLv2) MultipartBlobCommit

func (b *ADLv2) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)

func (*ADLv2) MultipartBlobCopy

func (b *ADLv2) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

func (*ADLv2) MultipartExpire

func (b *ADLv2) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)

func (*ADLv2) PatchBlob

func (s *ADLv2) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

func (*ADLv2) PutBlob

func (b *ADLv2) PutBlob(param *PutBlobInput) (*PutBlobOutput, error)

func (*ADLv2) RemoveBucket

func (b *ADLv2) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)

func (*ADLv2) RenameBlob

func (b *ADLv2) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)

type ADLv2MultipartBlobCommitInput

type ADLv2MultipartBlobCommitInput struct {
	Size           uint64
	ContentType    string
	RenewLeaseStop chan bool
}

type AZBlob

type AZBlob struct {
	// contains filtered or unexported fields
}

func NewAZBlob

func NewAZBlob(container string, config *cfg.AZBlobConfig) (*AZBlob, error)

func (*AZBlob) Bucket

func (b *AZBlob) Bucket() string

func (*AZBlob) Capabilities

func (b *AZBlob) Capabilities() *Capabilities

func (*AZBlob) CopyBlob

func (b *AZBlob) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)

func (*AZBlob) Delegate

func (b *AZBlob) Delegate() interface{}

func (*AZBlob) DeleteBlob

func (b *AZBlob) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)

func (*AZBlob) DeleteBlobs

func (b *AZBlob) DeleteBlobs(param *DeleteBlobsInput) (ret *DeleteBlobsOutput, deleteError error)

func (*AZBlob) GetBlob

func (b *AZBlob) GetBlob(param *GetBlobInput) (*GetBlobOutput, error)

func (*AZBlob) HeadBlob

func (b *AZBlob) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)

func (*AZBlob) Init

func (b *AZBlob) Init(key string) error

func (*AZBlob) ListBlobs

func (b *AZBlob) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*AZBlob) MakeBucket

func (b *AZBlob) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)

func (*AZBlob) MultipartBlobAbort

func (b *AZBlob) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)

func (*AZBlob) MultipartBlobAdd

func (b *AZBlob) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)

func (*AZBlob) MultipartBlobBegin

func (b *AZBlob) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)

func (*AZBlob) MultipartBlobCommit

func (b *AZBlob) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)

func (*AZBlob) MultipartBlobCopy

func (b *AZBlob) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

func (*AZBlob) MultipartExpire

func (b *AZBlob) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)

func (*AZBlob) PatchBlob

func (s *AZBlob) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

func (*AZBlob) PutBlob

func (b *AZBlob) PutBlob(param *PutBlobInput) (*PutBlobOutput, error)

func (*AZBlob) RemoveBucket

func (b *AZBlob) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)

func (*AZBlob) RenameBlob

func (b *AZBlob) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)

type BlobItemOutput

type BlobItemOutput struct {
	Key          *string
	ETag         *string
	LastModified *time.Time
	Size         uint64
	StorageClass *string
	Metadata     map[string]*string
	Content      []byte
}

func (BlobItemOutput) String

func (b BlobItemOutput) String() string

type BlobPrefixOutput

type BlobPrefixOutput struct {
	Prefix *string
}

func (BlobPrefixOutput) String

func (b BlobPrefixOutput) String() string

type BucketSpec

type BucketSpec struct {
	Scheme string
	Bucket string
	Prefix string
}

func ParseBucketSpec

func ParseBucketSpec(bucket string) (spec BucketSpec, err error)

type BufferList

type BufferList struct {
	// contains filtered or unexported fields
}

func (*BufferList) Add

func (l *BufferList) Add(offset uint64, data []byte, state BufferState, copyData bool) (allocated int64)

func (*BufferList) AddLoading

func (l *BufferList) AddLoading(offset, size uint64)

func (*BufferList) AddLoadingFromDisk

func (l *BufferList) AddLoadingFromDisk(offset, size uint64) (readRanges []Range)

func (*BufferList) AnyFlushed

func (l *BufferList) AnyFlushed(offset, size uint64) (flushed bool)

func (*BufferList) AnyUnclean

func (l *BufferList) AnyUnclean() bool

func (*BufferList) Ascend

func (l *BufferList) Ascend(offset uint64, iter func(end uint64, b *FileBuffer) (cont bool, changed bool))

func (*BufferList) Count

func (l *BufferList) Count() int

func (*BufferList) DebugCheckHoles

func (l *BufferList) DebugCheckHoles(s string)

func (*BufferList) Dump

func (l *BufferList) Dump(offset, size uint64) string

Left here for the ease of debugging

func (*BufferList) EvictFromMemory

func (l *BufferList) EvictFromMemory(buf *FileBuffer) (allocated int64, deleted bool)

func (*BufferList) Get

func (l *BufferList) Get(end uint64) *FileBuffer

func (*BufferList) GetData

func (l *BufferList) GetData(offset, size uint64, returnIds bool) (data [][]byte, ids map[uint64]bool, err error)

func (*BufferList) GetHoles

func (l *BufferList) GetHoles(offset, size uint64) (holes []Range, loading bool, flushCleared bool)

func (*BufferList) IterateDirtyParts

func (l *BufferList) IterateDirtyParts(cb func(partNum uint64) bool)

func (*BufferList) RemoveLoading

func (l *BufferList) RemoveLoading(offset, size uint64)

func (*BufferList) RemoveRange

func (l *BufferList) RemoveRange(removeOffset, removeSize uint64, filter func(b *FileBuffer) bool) (allocated int64)

Remove buffers in range (offset..size)

func (*BufferList) ReviveFromDisk

func (l *BufferList) ReviveFromDisk(offset uint64, data []byte)

func (*BufferList) Select

func (l *BufferList) Select(start, end uint64, cb func(buf *FileBuffer) (good bool)) (bufs []*FileBuffer)

func (*BufferList) SetFlushedClean

func (l *BufferList) SetFlushedClean()

func (*BufferList) SetState

func (l *BufferList) SetState(offset, size uint64, ids map[uint64]bool, state BufferState)

func (*BufferList) SplitAt

func (l *BufferList) SplitAt(offset uint64)

func (*BufferList) ZeroRange

func (l *BufferList) ZeroRange(offset, size uint64) (zeroed bool, allocated int64)

type BufferListHelpers

type BufferListHelpers interface {
	PartNum(uint64) uint64
	QueueCleanBuffer(*FileBuffer)
	UnqueueCleanBuffer(*FileBuffer)
}

type BufferOrZero

type BufferOrZero struct {
	// contains filtered or unexported fields
}

type BufferPointer

type BufferPointer struct {
	// contains filtered or unexported fields
}

Several FileBuffers may be slices of the same array, but we want to track memory usage, so we have to refcount them... O_o

type BufferPool

type BufferPool struct {
	FreeSomeCleanBuffers func(size int64) (int64, bool)
	// contains filtered or unexported fields
}

BufferPool tracks memory used by cache buffers

func NewBufferPool

func NewBufferPool(limit int64, gcInterval uint64) *BufferPool

func (*BufferPool) Use

func (pool *BufferPool) Use(size int64, ignoreMemoryLimit bool) (err error)

func (*BufferPool) UseUnlocked

func (pool *BufferPool) UseUnlocked(size int64, ignoreMemoryLimit bool) error

type BufferQueue

type BufferQueue struct {
	// contains filtered or unexported fields
}

func (*BufferQueue) Add

func (l *BufferQueue) Add(inode *Inode, b *FileBuffer)

func (*BufferQueue) Delete

func (l *BufferQueue) Delete(b *FileBuffer)

func (*BufferQueue) NextClean

func (l *BufferQueue) NextClean(minQueueId uint64) (inode *Inode, end, nextQueueId uint64)

type BufferState

type BufferState int16
const (
	// Buffer is clean
	BUF_CLEAN BufferState = 1
	// Buffer is modified locally
	BUF_DIRTY BufferState = 2
	// Buffer is flushed to the server as a full part, but multipart upload is not finalized yet
	BUF_FLUSHED_FULL BufferState = 3
	// Buffer is flushed to the server as an undersized part
	// (and multipart upload is not finalized yet)
	BUF_FLUSHED_CUT BufferState = 4
	// Buffer is flushed to the server and then removed from memory
	// (which is only possible for BUF_FLUSHED_FULL buffers)
	// (and multipart upload is not finalized yet)
	BUF_FL_CLEARED BufferState = 5
)

Yes I know this is against Go style. But it's easier to grep and distinguish visually so fuck off. :-)

type Capabilities

type Capabilities struct {
	MaxMultipartSize uint64
	// indicates that the blob store has native support for directories
	DirBlob bool
	Name    string
}

type ClusterFs

type ClusterFs struct {
	Flags  *cfg.FlagStorage
	Conns  *ConnPool
	Goofys *Goofys
	// contains filtered or unexported fields
}

func (*ClusterFs) Join

func (fs *ClusterFs) Join(ctx context.Context) error

func (*ClusterFs) StatPrinter

func (fs *ClusterFs) StatPrinter()

func (*ClusterFs) Unmount

func (fs *ClusterFs) Unmount() error

type ClusterFsFuse

type ClusterFsFuse struct {
	fuseutil.NotImplementedFileSystem
	*ClusterFs
}

func (*ClusterFsFuse) CreateFile

func (fs *ClusterFsFuse) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) (err error)
func (fs *ClusterFsFuse) CreateSymlink(ctx context.Context, op *fuseops.CreateSymlinkOp) (err error)

func (*ClusterFsFuse) ForgetInode

func (fs *ClusterFsFuse) ForgetInode(ctx context.Context, op *fuseops.ForgetInodeOp) (err error)

func (*ClusterFsFuse) GetInodeAttributes

func (fs *ClusterFsFuse) GetInodeAttributes(ctx context.Context, op *fuseops.GetInodeAttributesOp) (err error)

func (*ClusterFsFuse) LookUpInode

func (fs *ClusterFsFuse) LookUpInode(ctx context.Context, op *fuseops.LookUpInodeOp) (err error)

func (*ClusterFsFuse) MkDir

func (fs *ClusterFsFuse) MkDir(ctx context.Context, op *fuseops.MkDirOp) (err error)

func (*ClusterFsFuse) OpenDir

func (fs *ClusterFsFuse) OpenDir(ctx context.Context, op *fuseops.OpenDirOp) (err error)

func (*ClusterFsFuse) OpenFile

func (fs *ClusterFsFuse) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) (err error)

func (*ClusterFsFuse) ReadDir

func (fs *ClusterFsFuse) ReadDir(ctx context.Context, op *fuseops.ReadDirOp) (err error)

func (*ClusterFsFuse) ReadFile

func (fs *ClusterFsFuse) ReadFile(ctx context.Context, op *fuseops.ReadFileOp) (err error)
func (fs *ClusterFsFuse) ReadSymlink(ctx context.Context, op *fuseops.ReadSymlinkOp) (err error)

func (*ClusterFsFuse) ReleaseDirHandle

func (fs *ClusterFsFuse) ReleaseDirHandle(ctx context.Context, op *fuseops.ReleaseDirHandleOp) (err error)

func (*ClusterFsFuse) ReleaseFileHandle

func (fs *ClusterFsFuse) ReleaseFileHandle(ctx context.Context, op *fuseops.ReleaseFileHandleOp) (err error)

func (*ClusterFsFuse) RmDir

func (fs *ClusterFsFuse) RmDir(ctx context.Context, op *fuseops.RmDirOp) (err error)

func (*ClusterFsFuse) SetInodeAttributes

func (fs *ClusterFsFuse) SetInodeAttributes(ctx context.Context, op *fuseops.SetInodeAttributesOp) (err error)

func (*ClusterFsFuse) StatFS

func (fs *ClusterFsFuse) StatFS(ctx context.Context, op *fuseops.StatFSOp) error
func (fs *ClusterFsFuse) Unlink(ctx context.Context, op *fuseops.UnlinkOp) (err error)

func (*ClusterFsFuse) WriteFile

func (fs *ClusterFsFuse) WriteFile(ctx context.Context, op *fuseops.WriteFileOp) (err error)

type ClusterFsGrpc

type ClusterFsGrpc struct {
	pb.UnimplementedFsGrpcServer
	*ClusterFs
}

func (*ClusterFsGrpc) CreateFile

func (*ClusterFsGrpc) ForgetInode

func (*ClusterFsGrpc) ForgetInode2

func (*ClusterFsGrpc) GetInodeAttributes

func (*ClusterFsGrpc) LookUpInode

func (*ClusterFsGrpc) LookUpInode2

func (*ClusterFsGrpc) MkDir

func (fs *ClusterFsGrpc) MkDir(ctx context.Context, req *pb.MkDirRequest) (*pb.MkDirResponse, error)

func (*ClusterFsGrpc) OpenDir

func (*ClusterFsGrpc) OpenFile

func (*ClusterFsGrpc) ReadDir

func (*ClusterFsGrpc) ReadFile

func (*ClusterFsGrpc) ReleaseDirHandle

func (*ClusterFsGrpc) ReleaseFileHandle

func (*ClusterFsGrpc) RmDir

func (fs *ClusterFsGrpc) RmDir(ctx context.Context, req *pb.RmDirRequest) (*pb.RmDirResponse, error)

func (*ClusterFsGrpc) SetInodeAttributes

func (*ClusterFsGrpc) TryStealInodeOwnership

func (fs *ClusterFsGrpc) Unlink(ctx context.Context, req *pb.UnlinkRequest) (*pb.UnlinkResponse, error)

func (*ClusterFsGrpc) WriteFile

type CommonPrefix

type CommonPrefix struct {
	Prefix *string `type:"string"`
}

type ConnPool

type ConnPool struct {
	// contains filtered or unexported fields
}

func NewConnPool

func NewConnPool(flags *cfg.FlagStorage) *ConnPool

func (*ConnPool) Broad

func (conns *ConnPool) Broad(
	makeRequst Request,
) (errs map[NodeId]error)

func (*ConnPool) BroadConfigurable

func (conns *ConnPool) BroadConfigurable(
	makeRequst Request,
	unmountOnError bool,
) (errs map[NodeId]error)

func (*ConnPool) Unary

func (conns *ConnPool) Unary(
	nodeId NodeId,
	makeRequst Request,
) (err error)

func (*ConnPool) UnaryConfiguarble

func (conns *ConnPool) UnaryConfiguarble(
	nodeId NodeId,
	makeRequst Request,
	unmountOnError bool,
) (err error)

type CopyBlobInput

type CopyBlobInput struct {
	Source      string
	Destination string

	Size         *uint64
	ETag         *string            // if non-nil, do conditional copy
	Metadata     map[string]*string // if nil, copy from Source
	StorageClass *string            // if nil, copy from Source
}

type CopyBlobOutput

type CopyBlobOutput struct {
	RequestId string
}

type Delegator

type Delegator interface {
	Delegate() interface{}
}

type DeleteBlobInput

type DeleteBlobInput struct {
	Key string
}

type DeleteBlobOutput

type DeleteBlobOutput struct {
	RequestId string
}

type DeleteBlobsInput

type DeleteBlobsInput struct {
	Items []string
}

type DeleteBlobsOutput

type DeleteBlobsOutput struct {
	RequestId string
}

type DirHandle

type DirHandle struct {
	// contains filtered or unexported fields
}

func NewDirHandle

func NewDirHandle(inode *Inode) (dh *DirHandle)

func (*DirHandle) CloseDir

func (dh *DirHandle) CloseDir() error

func (*DirHandle) Next

func (dh *DirHandle) Next(name string)

LOCKS_REQUIRED(dh.mu)

func (*DirHandle) ReadDir

func (dh *DirHandle) ReadDir() (inode *Inode, err error)

LOCKS_REQUIRED(dh.mu) LOCKS_EXCLUDED(dh.inode.mu) LOCKS_EXCLUDED(dh.inode.fs)

func (*DirHandle) Seek

func (dh *DirHandle) Seek(newOffset fuseops.DirOffset)

LOCKS_REQUIRED(dh.mu)

type DirInodeData

type DirInodeData struct {
	DirTime     time.Time
	ImplicitDir bool

	ModifiedChildren int64

	Children        []*Inode
	DeletedChildren map[string]*Inode
	Gaps            []*SlurpGap
	// contains filtered or unexported fields
}

type FDQueue

type FDQueue struct {
	// contains filtered or unexported fields
}

func NewFDQueue

func NewFDQueue(maxCount int) *FDQueue

func (*FDQueue) CloseExtra

func (l *FDQueue) CloseExtra()

Close unneeded cache FDs

func (*FDQueue) DeleteFD

func (l *FDQueue) DeleteFD(inode *Inode)

func (*FDQueue) InsertFD

func (l *FDQueue) InsertFD(inode *Inode)

func (*FDQueue) NextFD

func (l *FDQueue) NextFD(minQueueID uint64) (inode *Inode, nextQueueId uint64)

func (*FDQueue) UseFD

func (l *FDQueue) UseFD(inode *Inode)

type FileBuffer

type FileBuffer struct {
	// contains filtered or unexported fields
}

func (*FileBuffer) Append

func (buf *FileBuffer) Append(data []byte) int64

type FileHandle

type FileHandle struct {
	// contains filtered or unexported fields
}

func NewFileHandle

func NewFileHandle(inode *Inode) *FileHandle

NewFileHandle returns a new file handle for the given `inode`

func (*FileHandle) ReadFile

func (fh *FileHandle) ReadFile(sOffset int64, sLen int64) (data [][]byte, bytesRead int, err error)

func (*FileHandle) Release

func (fh *FileHandle) Release()

func (*FileHandle) WriteFile

func (fh *FileHandle) WriteFile(offset int64, data []byte, copyData bool) (err error)

type FuseMfsWrapper

type FuseMfsWrapper struct {
	*fuse.MountedFileSystem
	// contains filtered or unexported fields
}

func (*FuseMfsWrapper) Unmount

func (m *FuseMfsWrapper) Unmount() error

type GCPCredResponse

type GCPCredResponse struct {
	AccessToken string `json:"access_token"`
	TokenType   string `json:"token_type"`
	ExpiresIn   int    `json:"expires_in"`
}

type GCS3

type GCS3 struct {
	*S3Backend
	// contains filtered or unexported fields
}

GCS variant of S3

func NewGCS3

func NewGCS3(bucket string, flags *cfg.FlagStorage, config *cfg.S3Config) (*GCS3, error)

func (*GCS3) Delegate

func (s *GCS3) Delegate() interface{}

func (*GCS3) DeleteBlobs

func (s *GCS3) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)

func (*GCS3) ListBlobs

func (s *GCS3) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*GCS3) MultipartBlobCopy

func (s *GCS3) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

FIXME GCS doesn't have UploadPartCopy, so optimized modification flushing doesn't work You can either reupload the whole object or use some other way of making multipart objects For example, Composite Objects are even better than multipart uploads but intermediate objects should be filtered out from List responses so they don't appear as separate files then

func (*GCS3) PatchBlob

func (s *GCS3) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

type GetBlobInput

type GetBlobInput struct {
	Key     string
	Start   uint64
	Count   uint64
	IfMatch *string
}

type GetBlobOutput

type GetBlobOutput struct {
	HeadBlobOutput

	Body io.ReadCloser

	RequestId string
}

type Goofys

type Goofys struct {
	NotifyCallback func(notifications []interface{})
	// contains filtered or unexported fields
}

func NewClusterGoofys

func NewClusterGoofys(ctx context.Context, bucket string, flags *cfg.FlagStorage, conns *ConnPool) (*Goofys, error)

func NewGoofys

func NewGoofys(ctx context.Context, bucketName string, flags *cfg.FlagStorage) (*Goofys, error)

func (*Goofys) AddDirHandle

func (fs *Goofys) AddDirHandle(dh *DirHandle) fuseops.HandleID

func (*Goofys) AddFileHandle

func (fs *Goofys) AddFileHandle(fh *FileHandle) fuseops.HandleID

func (*Goofys) EvictEntry

func (fs *Goofys) EvictEntry(id fuseops.InodeID) bool

func (*Goofys) FDCloser

func (fs *Goofys) FDCloser()

Close unneeded cache FDs

func (*Goofys) Flusher

func (fs *Goofys) Flusher()

Flusher goroutine. Overall algorithm:

  1. File opened => reads and writes just populate cache
  2. File closed => flush it Created or fully overwritten => => Less than 5 MB => upload in a single part => More than 5 MB => upload using multipart Updated => CURRENTLY: => Less than 5 MB => upload in a single part => More than 5 MB => update using multipart copy Also we can't update less than 5 MB because it's the minimal part size
  3. Fsync triggered => intermediate full flush (same algorithm)
  4. Dirty memory limit reached => without on-disk cache we have to flush the whole object. With on-disk cache we can unload some dirty buffers to disk.

func (*Goofys) FreeSomeCleanBuffers

func (fs *Goofys) FreeSomeCleanBuffers(origSize int64) (int64, bool)

Try to reclaim some clean buffers

func (*Goofys) LookupParent

func (fs *Goofys) LookupParent(path string) (parent *Inode, child string, err error)

func (*Goofys) LookupPath

func (fs *Goofys) LookupPath(path string) (inode *Inode, err error)

func (*Goofys) MetaEvictor

func (fs *Goofys) MetaEvictor()

func (*Goofys) Mount

func (fs *Goofys) Mount(mount *Mount)

func (*Goofys) MountAll

func (fs *Goofys) MountAll(mounts []*Mount)

func (*Goofys) RefreshInodeCache

func (fs *Goofys) RefreshInodeCache(inode *Inode) error

func (*Goofys) ScheduleRetryFlush

func (fs *Goofys) ScheduleRetryFlush()

func (*Goofys) Shutdown

func (fs *Goofys) Shutdown()

func (*Goofys) SigUsr1

func (fs *Goofys) SigUsr1()

func (*Goofys) StatPrinter

func (fs *Goofys) StatPrinter()

func (*Goofys) SyncTree

func (fs *Goofys) SyncTree(parent *Inode)

func (*Goofys) Unmount

func (fs *Goofys) Unmount(mountPoint string)

func (*Goofys) WakeupFlusher

func (fs *Goofys) WakeupFlusher()

func (*Goofys) WakeupFlusherAndWait

func (fs *Goofys) WakeupFlusherAndWait(wait bool)

type GoofysFuse

type GoofysFuse struct {
	fuseutil.NotImplementedFileSystem
	*Goofys
	// contains filtered or unexported fields
}

func NewGoofysFuse

func NewGoofysFuse(fs *Goofys) *GoofysFuse

func (*GoofysFuse) CreateFile

func (fs *GoofysFuse) CreateFile(
	ctx context.Context,
	op *fuseops.CreateFileOp,
) (err error)
func (fs *GoofysFuse) CreateSymlink(ctx context.Context,
	op *fuseops.CreateSymlinkOp,
) (err error)

func (*GoofysFuse) Fallocate

func (fs *GoofysFuse) Fallocate(
	ctx context.Context,
	op *fuseops.FallocateOp,
) (err error)

func (*GoofysFuse) FlushFile

func (fs *GoofysFuse) FlushFile(
	ctx context.Context,
	op *fuseops.FlushFileOp,
) (err error)

func (*GoofysFuse) ForgetInode

func (fs *GoofysFuse) ForgetInode(
	ctx context.Context,
	op *fuseops.ForgetInodeOp,
) (err error)

func (*GoofysFuse) GetInodeAttributes

func (fs *GoofysFuse) GetInodeAttributes(
	ctx context.Context,
	op *fuseops.GetInodeAttributesOp,
) (err error)

func (*GoofysFuse) GetXattr

func (fs *GoofysFuse) GetXattr(ctx context.Context,
	op *fuseops.GetXattrOp,
) (err error)

func (*GoofysFuse) ListXattr

func (fs *GoofysFuse) ListXattr(ctx context.Context,
	op *fuseops.ListXattrOp,
) (err error)

func (*GoofysFuse) LookUpInode

func (fs *GoofysFuse) LookUpInode(
	ctx context.Context,
	op *fuseops.LookUpInodeOp,
) (err error)

func (*GoofysFuse) MkDir

func (fs *GoofysFuse) MkDir(
	ctx context.Context,
	op *fuseops.MkDirOp,
) (err error)

func (*GoofysFuse) MkNode

func (fs *GoofysFuse) MkNode(
	ctx context.Context,
	op *fuseops.MkNodeOp,
) (err error)

MkNode is required for NFS even with regular files because kernel nfsd uses vfs_create() -> fuse_create() -> fuse_mknod() and then separate fuse_open() for file creation instead of fuse_create_open()

func (*GoofysFuse) OpenDir

func (fs *GoofysFuse) OpenDir(
	ctx context.Context,
	op *fuseops.OpenDirOp,
) (err error)

func (*GoofysFuse) OpenFile

func (fs *GoofysFuse) OpenFile(
	ctx context.Context,
	op *fuseops.OpenFileOp,
) (err error)

func (*GoofysFuse) ReadDir

func (fs *GoofysFuse) ReadDir(
	ctx context.Context,
	op *fuseops.ReadDirOp,
) (err error)

func (*GoofysFuse) ReadFile

func (fs *GoofysFuse) ReadFile(
	ctx context.Context,
	op *fuseops.ReadFileOp,
) (err error)
func (fs *GoofysFuse) ReadSymlink(ctx context.Context,
	op *fuseops.ReadSymlinkOp,
) (err error)

func (*GoofysFuse) ReleaseDirHandle

func (fs *GoofysFuse) ReleaseDirHandle(
	ctx context.Context,
	op *fuseops.ReleaseDirHandleOp,
) (err error)

func (*GoofysFuse) ReleaseFileHandle

func (fs *GoofysFuse) ReleaseFileHandle(
	ctx context.Context,
	op *fuseops.ReleaseFileHandleOp,
) (err error)

func (*GoofysFuse) RemoveXattr

func (fs *GoofysFuse) RemoveXattr(ctx context.Context,
	op *fuseops.RemoveXattrOp,
) (err error)

func (*GoofysFuse) Rename

func (fs *GoofysFuse) Rename(
	ctx context.Context,
	op *fuseops.RenameOp,
) (err error)

rename("from", "to") causes the kernel to send lookup of "from" and "to" prior to sending rename to us

func (*GoofysFuse) RmDir

func (fs *GoofysFuse) RmDir(
	ctx context.Context,
	op *fuseops.RmDirOp,
) (err error)

func (*GoofysFuse) SetConnection

func (fs *GoofysFuse) SetConnection(conn *fuse.Connection)

func (*GoofysFuse) SetInodeAttributes

func (fs *GoofysFuse) SetInodeAttributes(
	ctx context.Context,
	op *fuseops.SetInodeAttributesOp,
) (err error)

func (*GoofysFuse) SetXattr

func (fs *GoofysFuse) SetXattr(ctx context.Context,
	op *fuseops.SetXattrOp,
) (err error)

func (*GoofysFuse) StatFS

func (fs *GoofysFuse) StatFS(
	ctx context.Context,
	op *fuseops.StatFSOp,
) (err error)

func (*GoofysFuse) SyncFS

func (fs *GoofysFuse) SyncFS(
	ctx context.Context,
	op *fuseops.SyncFSOp,
) (err error)

func (*GoofysFuse) SyncFile

func (fs *GoofysFuse) SyncFile(
	ctx context.Context,
	op *fuseops.SyncFileOp,
) (err error)
func (fs *GoofysFuse) Unlink(
	ctx context.Context,
	op *fuseops.UnlinkOp,
) (err error)

func (*GoofysFuse) WriteFile

func (fs *GoofysFuse) WriteFile(
	ctx context.Context,
	op *fuseops.WriteFileOp,
) (err error)

type GrpcServer

type GrpcServer struct {
	*grpc.Server
	// contains filtered or unexported fields
}

func NewGrpcServer

func NewGrpcServer(flags *cfg.FlagStorage) *GrpcServer

func (*GrpcServer) Start

func (srv *GrpcServer) Start() error

type HeadBlobInput

type HeadBlobInput struct {
	Key string
}

type HeadBlobOutput

type HeadBlobOutput struct {
	BlobItemOutput

	ContentType *string
	IsDirBlob   bool

	RequestId string
}

func RetryHeadBlob

func RetryHeadBlob(flags *cfg.FlagStorage, cloud StorageBackend, req *HeadBlobInput) (resp *HeadBlobOutput, err error)

type IMDSv1Response

type IMDSv1Response struct {
	Code       string
	Token      string
	Expiration time.Time
}

type Inode

type Inode struct {
	Id   fuseops.InodeID
	Name string

	Attributes InodeAttributes
	// It is generally safe to read `AttrTime` without locking because if some other
	// operation is modifying `AttrTime`, in most cases the reader is okay with working with
	// stale data. But Time is a struct and modifying it is not atomic. However
	// in practice (until the year 2157) we should be okay because
	// - Almost all uses of AttrTime will be about comparisons (AttrTime < x, AttrTime > x)
	// - Time object will have Time::monotonic bit set (until the year 2157) => the time
	//   comparision just compares Time::ext field
	// Ref: https://github.com/golang/go/blob/e42ae65a8507/src/time/time.go#L12:L56
	AttrTime   time.Time
	ExpireTime time.Time

	// We are not very consistent about enforcing locks for `Parent` because, the
	// parent field very very rarely changes and it is generally fine to operate on
	// stale parent information
	Parent *Inode

	// cached/buffered data
	CacheState int32

	DiskFDQueueID uint64
	DiskCacheFD   *os.File
	OnDisk        bool

	IsFlushing int
	// contains filtered or unexported fields
}

func NewInode

func NewInode(fs *Goofys, parent *Inode, name string) (inode *Inode)

func (*Inode) ChangeOwnerLock

func (inode *Inode) ChangeOwnerLock()

func (*Inode) ChangeOwnerUnlock

func (inode *Inode) ChangeOwnerUnlock()

func (*Inode) CheckLoadRange

func (inode *Inode) CheckLoadRange(offset, size, readAheadSize uint64, ignoreMemoryLimit bool) (bool, error)

func (*Inode) Create

func (parent *Inode) Create(name string) (*Inode, *FileHandle, error)

func (*Inode) CreateOrOpen

func (parent *Inode) CreateOrOpen(name string, open bool) (inode *Inode, fh *FileHandle, err error)
func (parent *Inode) CreateSymlink(
	name string, target string,
) (inode *Inode, err error)

func (*Inode) DeRef

func (inode *Inode) DeRef(n int64) (stale bool)

LOCKS_REQUIRED(inode.mu) LOCKS_EXCLUDED(fs.mu)

func (*Inode) DowngradeToKeepOwnerLock

func (inode *Inode) DowngradeToKeepOwnerLock()

Only for inode in StateLock

func (*Inode) DumpThis

func (inode *Inode) DumpThis(fn string, withBuffers bool, noLock bool) (children []*Inode)

func (*Inode) DumpTree

func (inode *Inode) DumpTree(fn string, withBuffers bool, noLock bool)

func (*Inode) FullName

func (inode *Inode) FullName() string

func (*Inode) GetAttributes

func (inode *Inode) GetAttributes() *fuseops.InodeAttributes

LOCKS_EXCLUDED(inode.mu)

func (*Inode) GetXattr

func (inode *Inode) GetXattr(name string) ([]byte, error)

func (*Inode) InflateAttributes

func (inode *Inode) InflateAttributes() (attr fuseops.InodeAttributes)

func (*Inode) IsRangeLocked

func (inode *Inode) IsRangeLocked(offset uint64, size uint64, onlyFlushing bool) bool

func (*Inode) KeepOwnerLock

func (inode *Inode) KeepOwnerLock()

func (*Inode) KeepOwnerUnlock

func (inode *Inode) KeepOwnerUnlock()

func (*Inode) ListXattr

func (inode *Inode) ListXattr() ([]string, error)

func (*Inode) LoadRange

func (inode *Inode) LoadRange(offset, size uint64, readAheadSize uint64, ignoreMemoryLimit bool) (miss bool, err error)

Load some inode data into memory Must be called with inode.mu taken Loaded range should be guarded against eviction by adding it into inode.readRanges

func (*Inode) LockRange

func (inode *Inode) LockRange(offset uint64, size uint64, flushing bool)

LockRange/UnlockRange could be moved into buffer_list.go, but they still have to be stored separately from buffers and can't be a refcount - otherwise an overwrite would reset the reference count and break locking

func (*Inode) LookUp

func (parent *Inode) LookUp(name string, doSlurp bool) (*Inode, error)

func (*Inode) LookUpCached

func (parent *Inode) LookUpCached(name string) (inode *Inode, err error)

func (*Inode) LookUpInodeMaybeDir

func (parent *Inode) LookUpInodeMaybeDir(name string) (*BlobItemOutput, error)

func (*Inode) MkDir

func (parent *Inode) MkDir(
	name string,
) (inode *Inode, err error)

func (*Inode) OpenCacheFD

func (inode *Inode) OpenCacheFD() error

func (*Inode) OpenDir

func (inode *Inode) OpenDir() (dh *DirHandle)

func (*Inode) OpenFile

func (inode *Inode) OpenFile() (fh *FileHandle, err error)

func (*Inode) PartNum

func (inode *Inode) PartNum(offset uint64) uint64

For BufferListHelpers

func (*Inode) QueueCleanBuffer

func (inode *Inode) QueueCleanBuffer(buf *FileBuffer)

For BufferListHelpers

func (inode *Inode) ReadSymlink() (target string, err error)

func (*Inode) Ref

func (inode *Inode) Ref()

func (*Inode) RemoveXattr

func (inode *Inode) RemoveXattr(name string) error

func (*Inode) Rename

func (parent *Inode) Rename(from string, newParent *Inode, to string) (err error)

semantic of rename: rename("any", "not_exists") = ok rename("file1", "file2") = ok rename("empty_dir1", "empty_dir2") = ok rename("nonempty_dir1", "empty_dir2") = ok rename("nonempty_dir1", "nonempty_dir2") = ENOTEMPTY rename("file", "dir") = EISDIR rename("dir", "file") = ENOTDIR LOCKS_EXCLUDED(parent.mu) LOCKS_EXCLUDED(newParent.mu)

func (*Inode) ResetForUnmount

func (inode *Inode) ResetForUnmount()

ResetForUnmount resets the Inode as part of unmounting a storage backend mounted at the given inode. ACQUIRES_LOCK(inode.mu)

func (*Inode) ResizeUnlocked

func (inode *Inode) ResizeUnlocked(newSize uint64, finalizeFlushed bool)

func (*Inode) RmDir

func (parent *Inode) RmDir(name string) (err error)

func (*Inode) SendDelete

func (inode *Inode) SendDelete()

func (*Inode) SendMkDir

func (dir *Inode) SendMkDir()

func (*Inode) SetAttrTime

func (inode *Inode) SetAttrTime(tm time.Time)

LOCKS_REQUIRED(inode.mu) LOCKS_EXCLUDED(inode.fs.mu)

func (*Inode) SetAttributes

func (inode *Inode) SetAttributes(size *uint64, mode *os.FileMode,
	mtime *time.Time, uid *uint32, gid *uint32,
) (err error)

func (*Inode) SetCacheState

func (inode *Inode) SetCacheState(state int32)

LOCKS_REQUIRED(inode.mu)

func (*Inode) SetExpireLocked

func (inode *Inode) SetExpireLocked(tm time.Time)

LOCKS_EXCLUDED(inode.mu) LOCKS_EXCLUDED(inode.fs.mu)

func (*Inode) SetExpireTime

func (inode *Inode) SetExpireTime(tm time.Time)

LOCKS_REQUIRED(inode.mu) LOCKS_EXCLUDED(inode.fs.mu)

func (*Inode) SetFromBlobItem

func (inode *Inode) SetFromBlobItem(item *BlobItemOutput)

LOCKS_EXCLUDED(inode.mu)

func (*Inode) SetXattr

func (inode *Inode) SetXattr(name string, value []byte, flags uint32) error

func (*Inode) StateLock

func (inode *Inode) StateLock()

func (*Inode) StateUnlock

func (inode *Inode) StateUnlock()

func (*Inode) SyncFile

func (inode *Inode) SyncFile() (err error)

func (*Inode) ToDir

func (inode *Inode) ToDir()

func (*Inode) TryFlush

func (inode *Inode) TryFlush(priority int) bool
func (parent *Inode) Unlink(name string) (err error)

func (*Inode) UnlockRange

func (inode *Inode) UnlockRange(offset uint64, size uint64, flushing bool)

func (*Inode) UnqueueCleanBuffer

func (inode *Inode) UnqueueCleanBuffer(buf *FileBuffer)

For BufferListHelpers

func (*Inode) UpgradeToStateLock

func (inode *Inode) UpgradeToStateLock()

Only for inode in KeepOwnerLock

type InodeAttributes

type InodeAttributes struct {
	Size  uint64
	Mtime time.Time
	Ctime time.Time
	Uid   uint32
	Gid   uint32
	Rdev  uint32
	Mode  os.FileMode
}

type InodeQueue

type InodeQueue struct {
	// contains filtered or unexported fields
}

func (*InodeQueue) Add

func (l *InodeQueue) Add(inodeID uint64) (queueID uint64)

func (*InodeQueue) Delete

func (l *InodeQueue) Delete(queueID uint64)

func (*InodeQueue) Next

func (l *InodeQueue) Next(minQueueID uint64) (inodeID, nextQueueID uint64)

func (*InodeQueue) Size

func (l *InodeQueue) Size() int

type Joinable

type Joinable interface {
	Join(ctx context.Context) error
}

type ListBlobsInput

type ListBlobsInput struct {
	Prefix            *string
	Delimiter         *string
	MaxKeys           *uint32
	StartAfter        *string // XXX: not supported by Azure
	ContinuationToken *string
}

type ListBlobsOutput

type ListBlobsOutput struct {
	Prefixes              []BlobPrefixOutput
	Items                 []BlobItemOutput
	NextContinuationToken *string
	IsTruncated           bool

	RequestId string
}

func RetryListBlobs

func RetryListBlobs(flags *cfg.FlagStorage, cloud StorageBackend, req *ListBlobsInput) (resp *ListBlobsOutput, err error)

type ListObjectsV2Output

type ListObjectsV2Output struct {
	CommonPrefixes        []*CommonPrefix `type:"list" flattened:"true"`
	Contents              []*Object       `type:"list" flattened:"true"`
	ContinuationToken     *string         `type:"string"`
	Delimiter             *string         `type:"string"`
	EncodingType          *string         `type:"string" enum:"EncodingType"`
	IsTruncated           *bool           `type:"boolean"`
	KeyCount              *int64          `type:"integer"`
	MaxKeys               *int64          `type:"integer"`
	Name                  *string         `type:"string"`
	NextContinuationToken *string         `type:"string"`
	Prefix                *string         `type:"string"`
	RequestCharged        *string         `location:"header" locationName:"x-amz-request-charged" type:"string" enum:"RequestCharged"`
	StartAfter            *string         `type:"string"`
	// contains filtered or unexported fields
}

type MPUPart

type MPUPart struct {
	Num    uint32
	Offset uint64
	Size   uint64
	ETag   string
}

type MakeBucketInput

type MakeBucketInput struct{}

type MakeBucketOutput

type MakeBucketOutput struct {
	RequestId string
}

type MetadataEntry

type MetadataEntry struct {
	Name  string
	Value string
}

type Mount

type Mount struct {
	// contains filtered or unexported fields
}

type MountedFS

type MountedFS interface {
	Join(ctx context.Context) error
	Unmount() error
}

type MultiReader

type MultiReader struct {
	// contains filtered or unexported fields
}

func NewMultiReader

func NewMultiReader() *MultiReader

func (*MultiReader) AddBuffer

func (r *MultiReader) AddBuffer(buf []byte)

func (*MultiReader) AddZero

func (r *MultiReader) AddZero(size uint64)

func (*MultiReader) Len

func (r *MultiReader) Len() uint64

func (*MultiReader) Read

func (r *MultiReader) Read(buf []byte) (n int, err error)

func (*MultiReader) Seek

func (r *MultiReader) Seek(offset int64, whence int) (newOffset int64, err error)

type MultipartBlobAbortOutput

type MultipartBlobAbortOutput struct {
	RequestId string
}

type MultipartBlobAddInput

type MultipartBlobAddInput struct {
	Commit     *MultipartBlobCommitInput
	PartNumber uint32

	Body io.ReadSeeker

	Size   uint64 // GCS wants to know part size
	Offset uint64 // ADLv2 needs to know offset
}

type MultipartBlobAddOutput

type MultipartBlobAddOutput struct {
	RequestId string
	PartId    *string
}

type MultipartBlobBeginInput

type MultipartBlobBeginInput struct {
	Key         string
	Metadata    map[string]*string
	ContentType *string
}

type MultipartBlobCommitInput

type MultipartBlobCommitInput struct {
	Key *string

	Metadata map[string]*string
	UploadId *string
	Parts    []*string
	NumParts uint32
	// contains filtered or unexported fields
}

type MultipartBlobCommitOutput

type MultipartBlobCommitOutput struct {
	ETag         *string
	LastModified *time.Time
	StorageClass *string

	RequestId string
}

type MultipartBlobCopyInput

type MultipartBlobCopyInput struct {
	Commit     *MultipartBlobCommitInput
	PartNumber uint32
	CopySource string
	Offset     uint64
	Size       uint64
}

type MultipartBlobCopyOutput

type MultipartBlobCopyOutput struct {
	RequestId string
	PartId    *string
}

type MultipartExpireInput

type MultipartExpireInput struct{}

type MultipartExpireOutput

type MultipartExpireOutput struct {
	RequestId string
}

type NodeId

type NodeId uint64

type Object

type Object struct {
	ChecksumAlgorithm []*string       `type:"list" flattened:"true" enum:"ChecksumAlgorithm"`
	ETag              *string         `type:"string"`
	Key               *string         `min:"1" type:"string"`
	LastModified      *time.Time      `type:"timestamp"`
	Owner             *Owner          `type:"structure"`
	RestoreStatus     *RestoreStatus  `type:"structure"`
	Size              *int64          `type:"long"`
	StorageClass      *string         `type:"string" enum:"ObjectStorageClass"`
	Metadata          []MetadataEntry `type:"list" flattened:"true"`
	Data              xsdBase64Binary `type:"blob"`
}

type OpStats

type OpStats struct {
	// contains filtered or unexported fields
}

type Owner

type Owner struct {
	DisplayName *string `type:"string"`
	ID          *string `type:"string"`
}

type PatchBlobInput

type PatchBlobInput struct {
	Key            string
	Offset         uint64
	Size           uint64
	AppendPartSize int64

	Body io.ReadSeeker
}

type PatchBlobOutput

type PatchBlobOutput struct {
	ETag         *string
	LastModified *time.Time

	RequestId string
}

type Peer

type Peer struct {
	// contains filtered or unexported fields
}

type PutBlobInput

type PutBlobInput struct {
	Key         string
	Metadata    map[string]*string
	ContentType *string
	DirBlob     bool

	Body io.ReadSeeker
	Size *uint64
}

type PutBlobOutput

type PutBlobOutput struct {
	ETag         *string
	LastModified *time.Time
	StorageClass *string

	RequestId string
}

type QueuedBuffer

type QueuedBuffer struct {
	// contains filtered or unexported fields
}

type Range

type Range struct {
	Start, End uint64
}

type ReadRange

type ReadRange struct {
	Offset   uint64
	Size     uint64
	Flushing bool
}

type ReadSeekerCloser

type ReadSeekerCloser struct {
	io.ReadSeeker
}

func (*ReadSeekerCloser) Close

func (r *ReadSeekerCloser) Close() error

type Recovery

type Recovery struct {
	pb.UnimplementedRecoveryServer
	Flags *cfg.FlagStorage
}

func (*Recovery) Unmount

func (rec *Recovery) Unmount(ctx context.Context, req *pb.UnmountRequest) (*pb.UnmountResponse, error)

type RemoveBucketInput

type RemoveBucketInput struct{}

type RemoveBucketOutput

type RemoveBucketOutput struct {
	RequestId string
}

type RenameBlobInput

type RenameBlobInput struct {
	Source      string
	Destination string
}

type RenameBlobOutput

type RenameBlobOutput struct {
	RequestId string
}

type Request

type Request func(ctx context.Context, conn *grpc.ClientConn) error

type RestoreStatus

type RestoreStatus struct {
	IsRestoreInProgress *bool      `type:"boolean"`
	RestoreExpiryDate   *time.Time `type:"timestamp"`
}

type S3Backend

type S3Backend struct {
	*s3.S3
	// contains filtered or unexported fields
}

func NewS3

func NewS3(bucket string, flags *cfg.FlagStorage, config *cfg.S3Config) (*S3Backend, error)

func (*S3Backend) Bucket

func (s *S3Backend) Bucket() string

func (*S3Backend) Capabilities

func (s *S3Backend) Capabilities() *Capabilities

func (*S3Backend) CopyBlob

func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)

func (*S3Backend) Delegate

func (s *S3Backend) Delegate() interface{}

func (*S3Backend) DeleteBlob

func (s *S3Backend) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)

func (*S3Backend) DeleteBlobs

func (s *S3Backend) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)

func (*S3Backend) GetBlob

func (s *S3Backend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error)

func (*S3Backend) HeadBlob

func (s *S3Backend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)

func (*S3Backend) Init

func (s *S3Backend) Init(key string) error

func (*S3Backend) ListBlobs

func (s *S3Backend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)

func (*S3Backend) MakeBucket

func (s *S3Backend) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)

func (*S3Backend) MultipartBlobAbort

func (s *S3Backend) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)

func (*S3Backend) MultipartBlobAdd

func (s *S3Backend) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)

func (*S3Backend) MultipartBlobBegin

func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)

func (*S3Backend) MultipartBlobCommit

func (s *S3Backend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)

func (*S3Backend) MultipartBlobCopy

func (s *S3Backend) MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)

func (*S3Backend) MultipartExpire

func (s *S3Backend) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)

func (*S3Backend) PatchBlob

func (s *S3Backend) PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)

func (*S3Backend) PutBlob

func (s *S3Backend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error)

func (*S3Backend) RefreshIAM

func (s *S3Backend) RefreshIAM()

func (*S3Backend) RemoveBucket

func (s *S3Backend) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)

func (*S3Backend) RenameBlob

func (s *S3Backend) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)

func (*S3Backend) TryIAM

func (s *S3Backend) TryIAM() (err error)

type SlurpGap

type SlurpGap struct {
	// contains filtered or unexported fields
}

type Stat

type Stat struct {
	// contains filtered or unexported fields
}

type StorageBackend

type StorageBackend interface {
	Init(key string) error
	Capabilities() *Capabilities
	// typically this would return bucket/prefix
	Bucket() string
	HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error)
	ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error)
	DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error)
	DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error)
	RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error)
	CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error)
	GetBlob(param *GetBlobInput) (*GetBlobOutput, error)
	PutBlob(param *PutBlobInput) (*PutBlobOutput, error)
	PatchBlob(param *PatchBlobInput) (*PatchBlobOutput, error)
	MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error)
	MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error)
	MultipartBlobCopy(param *MultipartBlobCopyInput) (*MultipartBlobCopyOutput, error)
	MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error)
	MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error)
	MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error)
	RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error)
	MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error)
	Delegate() interface{}
}

Implementations of all the functions here are expected to be concurrency-safe, except for:

Init() is called exactly once before any other functions are called.

Capabilities()/Bucket() are expected to be const

func NewBackend

func NewBackend(bucket string, flags *cfg.FlagStorage) (cloud StorageBackend, err error)

type StorageBackendInitError

type StorageBackendInitError struct {
	// contains filtered or unexported fields
}

func (StorageBackendInitError) Bucket

func (s StorageBackendInitError) Bucket() string

func (StorageBackendInitError) Capabilities

func (e StorageBackendInitError) Capabilities() *Capabilities

func (StorageBackendInitError) CopyBlob

func (StorageBackendInitError) Delegate

func (e StorageBackendInitError) Delegate() interface{}

func (StorageBackendInitError) DeleteBlob

func (StorageBackendInitError) DeleteBlobs

func (StorageBackendInitError) GetBlob

func (StorageBackendInitError) HeadBlob

func (StorageBackendInitError) Init

func (e StorageBackendInitError) Init(key string) error

func (StorageBackendInitError) ListBlobs

func (StorageBackendInitError) MakeBucket

func (StorageBackendInitError) MultipartBlobAbort

func (StorageBackendInitError) MultipartBlobAdd

func (StorageBackendInitError) MultipartBlobBegin

func (StorageBackendInitError) MultipartBlobCommit

func (StorageBackendInitError) MultipartBlobCopy

func (StorageBackendInitError) MultipartExpire

func (StorageBackendInitError) PatchBlob

func (StorageBackendInitError) PutBlob

func (StorageBackendInitError) RemoveBucket

func (StorageBackendInitError) RenameBlob

type StorageBackendInitWrapper

type StorageBackendInitWrapper struct {
	StorageBackend
	// contains filtered or unexported fields
}

func (*StorageBackendInitWrapper) Bucket

func (s *StorageBackendInitWrapper) Bucket() string

func (*StorageBackendInitWrapper) Capabilities

func (s *StorageBackendInitWrapper) Capabilities() *Capabilities

func (*StorageBackendInitWrapper) CopyBlob

func (*StorageBackendInitWrapper) DeleteBlob

func (*StorageBackendInitWrapper) DeleteBlobs

func (*StorageBackendInitWrapper) GetBlob

func (*StorageBackendInitWrapper) HeadBlob

func (*StorageBackendInitWrapper) Init

func (s *StorageBackendInitWrapper) Init(key string) error

func (*StorageBackendInitWrapper) ListBlobs

func (*StorageBackendInitWrapper) MakeBucket

func (*StorageBackendInitWrapper) MultipartBlobAbort

func (*StorageBackendInitWrapper) MultipartBlobAdd

func (*StorageBackendInitWrapper) MultipartBlobBegin

func (*StorageBackendInitWrapper) MultipartBlobCommit

func (*StorageBackendInitWrapper) MultipartBlobCopy

func (*StorageBackendInitWrapper) MultipartExpire

func (*StorageBackendInitWrapper) PatchBlob

func (*StorageBackendInitWrapper) PutBlob

func (*StorageBackendInitWrapper) RemoveBucket

func (*StorageBackendInitWrapper) RenameBlob

Directories

Path Synopsis
System permissions-related code.
System permissions-related code.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL