Documentation
¶
Overview ¶
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Index ¶
- Constants
- Variables
- func AllMpathBcks(opts *WalkOpts) (bcks []cmn.Bck, err error)
- func CapStatusGetWhat() (fsInfo apc.CapacityInfo)
- func CountPersisted(fname string) (cnt int)
- func CreateBucket(op string, bck *cmn.Bck, nilbmd bool) (errs []error)
- func Decommission(mdOnly bool)
- func DestroyBucket(op string, bck *cmn.Bck, bid uint64) (err error)
- func DirectOpen(path string, flag int, perm os.FileMode) (*os.File, error)
- func FillDiskStats(m ios.AllDiskStats)
- func Get() (MPI, MPI)
- func GetAllMpathUtils() (utils *ios.MpathUtil)
- func GetMpathUtil(mpath string) int64
- func GetTotalDisksSize() (uint64, error)
- func GetXattr(fqn, attrName string) ([]byte, error)
- func GetXattrBuf(fqn, attrName string, buf []byte) (b []byte, err error)
- func IsDirEmpty(dir string) (names []string, empty bool, err error)
- func LcacheIdx(digest uint64) int
- func LoadNodeID(mpaths cos.StrSet) (mDaeID string, err error)
- func MarkerExists(marker string) bool
- func MountpathsToLists() (mpl *apc.MountpathList)
- func New(allowSharedDisksAndNoDisks bool)
- func NumAvail() int
- func PersistMarker(marker string) (fatalErr, writeErr error)
- func PersistOnMpaths(fname, backupName string, meta jsp.Opts, atMost int, b []byte, sgl *memsys.SGL) (cnt, availCnt int)
- func PutMPI(available, disabled MPI)
- func RemoveDaemonIDs()
- func RemoveMarker(marker string) (err error)
- func RenameBucketDirs(bidFrom uint64, bckFrom, bckTo *cmn.Bck) (err error)
- func SetXattr(fqn, attrName string, data []byte) (err error)
- func TestDisableValidation()
- func TestNew(iostater ios.IOStater)
- func ValidateNCopies(tname string, copies int) (err error)
- func Walk(opts *WalkOpts) error
- func WalkBck(opts *WalkBckOpts) error
- func WalkDir(dir string, ucb func(string, DirEntry) error) error
- type CapStatus
- type Capacity
- type ContentInfo
- type ContentResolver
- type DirEntry
- type ECMetaContentResolver
- type ECSliceContentResolver
- func (*ECSliceContentResolver) GenUniqueFQN(base, _ string) string
- func (*ECSliceContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
- func (*ECSliceContentResolver) PermToEvict() bool
- func (*ECSliceContentResolver) PermToMove() bool
- func (*ECSliceContentResolver) PermToProcess() bool
- type ErrMountpathNoDisks
- type ErrStorageIntegrity
- type FilesystemInfo
- type MPCap
- type MPI
- type MountedFS
- type MountpathInfo
- func Add(mpath, tid string) (mi *MountpathInfo, err error)
- func AddMpath(mpath, tid string, cb func(), force bool) (mi *MountpathInfo, err error)
- func BeginDD(action string, flags uint64, mpath string) (mi *MountpathInfo, numAvail int, err error)
- func Disable(mpath string, cb ...func()) (disabledMpath *MountpathInfo, err error)
- func Enable(mpath string) (enabledMpath *MountpathInfo, err error)
- func EnableMpath(mpath, tid string, cb func()) (enabledMpath *MountpathInfo, err error)
- func FQN2Mpath(fqn string) (found *MountpathInfo, relativePath string, err error)
- func NewMountpath(mpath string) (mi *MountpathInfo, err error)
- func Path2Mpath(path string) (found *MountpathInfo, err error)
- func Remove(mpath string, cb ...func()) (*MountpathInfo, error)
- func (mi *MountpathInfo) AddDisabled(disabledPaths MPI)
- func (mi *MountpathInfo) AddEnabled(tid string, availablePaths MPI, config *cmn.Config) (err error)
- func (mi *MountpathInfo) CheckDisks() (err error)
- func (mi *MountpathInfo) ClearDD()
- func (mi *MountpathInfo) ClearMDs()
- func (mi *MountpathInfo) CreateMissingBckDirs(bck *cmn.Bck) (err error)
- func (mi *MountpathInfo) DeletedRoot() string
- func (mi *MountpathInfo) EvictLomCache()
- func (mi *MountpathInfo) IsAnySet(flags uint64) bool
- func (mi *MountpathInfo) IsIdle(config *cmn.Config) bool
- func (mi *MountpathInfo) LomCache(idx int) *sync.Map
- func (mi *MountpathInfo) MakePathBck(bck *cmn.Bck) string
- func (mi *MountpathInfo) MakePathCT(bck *cmn.Bck, contentType string) string
- func (mi *MountpathInfo) MakePathFQN(bck *cmn.Bck, contentType, objName string) string
- func (mi *MountpathInfo) MoveToDeleted(dir string) (err error)
- func (mi *MountpathInfo) Remove(path string) error
- func (mi *MountpathInfo) RemoveDeleted(who string) (rerr error)
- func (mi *MountpathInfo) SetDaemonIDXattr(tid string) error
- func (mi *MountpathInfo) String() string
- func (mi *MountpathInfo) TempDir(dir string) string
- type ObjectContentResolver
- type ParsedFQN
- type PartsFQN
- type WalkBckOpts
- type WalkOpts
- type WorkfileContentResolver
- func (*WorkfileContentResolver) GenUniqueFQN(base, prefix string) string
- func (*WorkfileContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
- func (*WorkfileContentResolver) PermToEvict() bool
- func (*WorkfileContentResolver) PermToMove() bool
- func (*WorkfileContentResolver) PermToProcess() bool
Constants ¶
const ( ObjectType = "ob" WorkfileType = "wk" ECSliceType = "ec" ECMetaType = "mt" )
const ( SieMpathIDMismatch = (1 + iota) * 10 SieTargetIDMismatch SieNotEqVMD SieMetaCorrupted SieFsDiffers SieMpathNotFound )
const ( // prefixes for workfiles created by various services WorkfileRemote = "remote" // getting object from neighbor target when rebalancing WorkfileColdget = "cold" // object GET: coldget WorkfilePut = "put" // object PUT WorkfileCopy = "copy" // copy object WorkfileAppend = "append" // APPEND to object (as file) WorkfileAppendToArch = "append-to-arch" // APPEND to existing archive WorkfileCreateArch = "create-arch" // CREATE multi-object archive )
const ( FlagBeingDisabled uint64 = 1 << iota FlagBeingDetached )
enum MountpathInfo.Flags
const FlagWaitingDD = FlagBeingDisabled | FlagBeingDetached
Variables ¶
var CSM *contentSpecMgr
Functions ¶
func CapStatusGetWhat ¶
func CapStatusGetWhat() (fsInfo apc.CapacityInfo)
func CountPersisted ¶
func Decommission ¶
func Decommission(mdOnly bool)
func DirectOpen ¶
DirectOpen opens a file with direct disk access (with OS caching disabled).
func FillDiskStats ¶
func FillDiskStats(m ios.AllDiskStats)
func GetMpathUtil ¶
func GetTotalDisksSize ¶
func GetXattrBuf ¶
GetXattr gets xattr by name via provided buffer
func LoadNodeID ¶
traverses all mountpaths to load and validate node ID
func MarkerExists ¶
func MountpathsToLists ¶
func MountpathsToLists() (mpl *apc.MountpathList)
func PersistMarker ¶
func PersistOnMpaths ¶
func PersistOnMpaths(fname, backupName string, meta jsp.Opts, atMost int, b []byte, sgl *memsys.SGL) (cnt, availCnt int)
PersistOnMpaths persists `what` on mountpaths under "mountpath.Path/path" filename. It does it on maximum `atMost` mountPaths. If `atMost == 0`, it does it on every mountpath. If `backupPath != ""`, it removes files from `backupPath` and moves files from `path` to `backupPath`. Returns how many times it has successfully stored a file.
func RemoveDaemonIDs ¶
func RemoveDaemonIDs()
func RemoveMarker ¶
func TestDisableValidation ¶
func TestDisableValidation()
TestDisableValidation disables fsid checking and allows mountpaths without disks (testing-only)
func ValidateNCopies ¶
func WalkBck ¶
func WalkBck(opts *WalkBckOpts) error
Types ¶
type CapStatus ¶
type CapStatus struct {
Err error
TotalUsed uint64 // bytes
TotalAvail uint64 // bytes
PctAvg int32 // used average (%)
PctMax int32 // max used (%)
OOS bool
}
func CapPeriodic ¶
func CapPeriodic(now int64, config *cmn.Config, mpcap MPCap) (cs CapStatus, updated bool, err error)
NOTE: Is called only and exclusively by `stats.Trunner` providing
`config.Periodic.StatsTime` tick.
func GetCapStatus ¶
func GetCapStatus() (cs CapStatus)
func RefreshCapStatus ¶
type ContentInfo ¶
type ContentResolver ¶
type ContentResolver interface {
// When set to true, services like rebalance have permission to move
// content for example to another target because it is misplaced (HRW).
PermToMove() bool
// When set to true, services like LRU have permission to evict/delete content
PermToEvict() bool
// When set to true, content can be checksumed, shown or processed in other ways.
PermToProcess() bool
// Generates unique base name for original one. This function may add
// additional information to the base name.
// prefix - user-defined marker
GenUniqueFQN(base, prefix string) (ufqn string)
// Parses generated unique fqn to the original one.
ParseUniqueFQN(base string) (orig string, old, ok bool)
}
type ECMetaContentResolver ¶
type ECMetaContentResolver struct{}
func (*ECMetaContentResolver) GenUniqueFQN ¶
func (*ECMetaContentResolver) GenUniqueFQN(base, _ string) string
func (*ECMetaContentResolver) ParseUniqueFQN ¶
func (*ECMetaContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
func (*ECMetaContentResolver) PermToEvict ¶
func (*ECMetaContentResolver) PermToEvict() bool
func (*ECMetaContentResolver) PermToMove ¶
func (*ECMetaContentResolver) PermToMove() bool
func (*ECMetaContentResolver) PermToProcess ¶
func (*ECMetaContentResolver) PermToProcess() bool
type ECSliceContentResolver ¶
type ECSliceContentResolver struct{}
func (*ECSliceContentResolver) GenUniqueFQN ¶
func (*ECSliceContentResolver) GenUniqueFQN(base, _ string) string
func (*ECSliceContentResolver) ParseUniqueFQN ¶
func (*ECSliceContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
func (*ECSliceContentResolver) PermToEvict ¶
func (*ECSliceContentResolver) PermToEvict() bool
func (*ECSliceContentResolver) PermToMove ¶
func (*ECSliceContentResolver) PermToMove() bool
func (*ECSliceContentResolver) PermToProcess ¶
func (*ECSliceContentResolver) PermToProcess() bool
type ErrMountpathNoDisks ¶
type ErrMountpathNoDisks struct {
Mi *MountpathInfo
}
func (*ErrMountpathNoDisks) Error ¶
func (e *ErrMountpathNoDisks) Error() string
type ErrStorageIntegrity ¶
func (*ErrStorageIntegrity) Error ¶
func (sie *ErrStorageIntegrity) Error() string
type FilesystemInfo ¶
func (*FilesystemInfo) Equal ¶
func (fsi *FilesystemInfo) Equal(otherFsi FilesystemInfo) bool
type MPI ¶
type MPI map[string]*MountpathInfo
type MountedFS ¶
type MountedFS struct {
// contains filtered or unexported fields
}
MountedFS holds all mountpaths for the target.
type MountpathInfo ¶
type MountpathInfo struct {
Path string // clean path
FilesystemInfo // underlying filesystem
Disks []string // owned disks (ios.FsDisks map => slice)
PathDigest uint64 // (HRW logic)
// contains filtered or unexported fields
}
func Add ¶
func Add(mpath, tid string) (mi *MountpathInfo, err error)
(used only in _unit_ tests - compare with AddMpath below)
func AddMpath ¶
func AddMpath(mpath, tid string, cb func(), force bool) (mi *MountpathInfo, err error)
Add adds new mountpath to the target's `availablePaths` TODO: extend `force=true` to disregard "filesystem sharing"
func BeginDD ¶
func BeginDD(action string, flags uint64, mpath string) (mi *MountpathInfo, numAvail int, err error)
begin (disable | detach) transaction: CoW-mark the corresponding mountpath
func Disable ¶
func Disable(mpath string, cb ...func()) (disabledMpath *MountpathInfo, err error)
Disables a mountpath, i.e., removes it from usage but keeps in the volume (for possible future re-enablement). If successful, returns the disabled mountpath. Otherwise, returns nil (also in the case if the mountpath was already disabled).
func Enable ¶
func Enable(mpath string) (enabledMpath *MountpathInfo, err error)
(used only in tests - compare with EnableMpath below)
func EnableMpath ¶
func EnableMpath(mpath, tid string, cb func()) (enabledMpath *MountpathInfo, err error)
Enable enables previously disabled mountpath. enabled is set to true if mountpath has been moved from disabled to available and exists is set to true if such mountpath even exists.
func FQN2Mpath ¶
func FQN2Mpath(fqn string) (found *MountpathInfo, relativePath string, err error)
FQN2Mpath matches FQN to mountpath and returns the mountpath and the relative path.
func NewMountpath ¶
func NewMountpath(mpath string) (mi *MountpathInfo, err error)
func Path2Mpath ¶
func Path2Mpath(path string) (found *MountpathInfo, err error)
Path2Mpath takes in any file path (e.g., ../../a/b/c) and returns the matching `mpathInfo`, if exists
func Remove ¶
func Remove(mpath string, cb ...func()) (*MountpathInfo, error)
Remove removes mountpaths from the target's mountpaths. It searches for the mountpath in `available` and, if not found, in `disabled`.
func (*MountpathInfo) AddDisabled ¶
func (mi *MountpathInfo) AddDisabled(disabledPaths MPI)
func (*MountpathInfo) AddEnabled ¶
func (*MountpathInfo) CheckDisks ¶
func (mi *MountpathInfo) CheckDisks() (err error)
func (*MountpathInfo) ClearDD ¶
func (mi *MountpathInfo) ClearDD()
func (*MountpathInfo) ClearMDs ¶
func (mi *MountpathInfo) ClearMDs()
func (*MountpathInfo) CreateMissingBckDirs ¶
func (mi *MountpathInfo) CreateMissingBckDirs(bck *cmn.Bck) (err error)
func (*MountpathInfo) DeletedRoot ¶
func (mi *MountpathInfo) DeletedRoot() string
func (*MountpathInfo) EvictLomCache ¶
func (mi *MountpathInfo) EvictLomCache()
func (*MountpathInfo) IsAnySet ¶
func (mi *MountpathInfo) IsAnySet(flags uint64) bool
func (*MountpathInfo) MakePathBck ¶
func (mi *MountpathInfo) MakePathBck(bck *cmn.Bck) string
func (*MountpathInfo) MakePathCT ¶
func (mi *MountpathInfo) MakePathCT(bck *cmn.Bck, contentType string) string
func (*MountpathInfo) MakePathFQN ¶
func (mi *MountpathInfo) MakePathFQN(bck *cmn.Bck, contentType, objName string) string
func (*MountpathInfo) MoveToDeleted ¶
func (mi *MountpathInfo) MoveToDeleted(dir string) (err error)
MoveToDeleted removes directory in steps: 1. Synchronously gets temporary directory name 2. Synchronously renames old folder to temporary directory
func (*MountpathInfo) Remove ¶
func (mi *MountpathInfo) Remove(path string) error
func (*MountpathInfo) RemoveDeleted ¶
func (mi *MountpathInfo) RemoveDeleted(who string) (rerr error)
func (*MountpathInfo) SetDaemonIDXattr ¶
func (mi *MountpathInfo) SetDaemonIDXattr(tid string) error
func (*MountpathInfo) String ¶
func (mi *MountpathInfo) String() string
func (*MountpathInfo) TempDir ¶
func (mi *MountpathInfo) TempDir(dir string) string
type ObjectContentResolver ¶
type ObjectContentResolver struct{}
func (*ObjectContentResolver) GenUniqueFQN ¶
func (*ObjectContentResolver) GenUniqueFQN(base, _ string) string
func (*ObjectContentResolver) ParseUniqueFQN ¶
func (*ObjectContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
func (*ObjectContentResolver) PermToEvict ¶
func (*ObjectContentResolver) PermToEvict() bool
func (*ObjectContentResolver) PermToMove ¶
func (*ObjectContentResolver) PermToMove() bool
func (*ObjectContentResolver) PermToProcess ¶
func (*ObjectContentResolver) PermToProcess() bool
type ParsedFQN ¶
type PartsFQN ¶
type PartsFQN interface {
ObjectName() string
Bucket() *cmn.Bck
MpathInfo() *MountpathInfo
CacheIdx() int
}
type WalkBckOpts ¶
type WalkBckOpts struct {
ValidateCallback walkFunc // should return filepath.SkipDir to skip directory without an error
WalkOpts
}
type WorkfileContentResolver ¶
type WorkfileContentResolver struct{}
func (*WorkfileContentResolver) GenUniqueFQN ¶
func (*WorkfileContentResolver) GenUniqueFQN(base, prefix string) string
func (*WorkfileContentResolver) ParseUniqueFQN ¶
func (*WorkfileContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
func (*WorkfileContentResolver) PermToEvict ¶
func (*WorkfileContentResolver) PermToEvict() bool
func (*WorkfileContentResolver) PermToMove ¶
func (*WorkfileContentResolver) PermToMove() bool
func (*WorkfileContentResolver) PermToProcess ¶
func (*WorkfileContentResolver) PermToProcess() bool