Documentation
¶
Overview ¶
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
- Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
Index ¶
- Constants
- Variables
- func AllMpathBcks(opts *WalkOpts) (bcks []cmn.Bck, err error)
- func CapStatusGetWhat() (fsInfo apc.CapacityInfo)
- func Clblk()
- func CleanPathErr(err error)
- func ComputeDiskSize()
- func CountPersisted(fname string) (cnt int)
- func CreateBucket(bck *cmn.Bck, nilbmd bool) (errs []error)
- func Decommission(mdOnly bool)
- func DestroyBucket(op string, bck *cmn.Bck, bid uint64) (err error)
- func DirectOpen(path string, flag int, perm os.FileMode) (*os.File, error)
- func ExpireCapCache()
- func FillDiskStats(m ios.AllDiskStats)
- func Get() (MPI, MPI)
- func GetAllMpathUtils() (utils *ios.MpathUtil)
- func GetDiskSize() uint64
- func GetMpathUtil(mpath string) int64
- func GetXattr(fqn, attrName string) ([]byte, error)
- func GetXattrBuf(fqn, attrName string, buf []byte) (b []byte, err error)
- func InitCDF(tcdf *TargetCDF)
- func IsDirEmpty(dir string) (names []string, empty bool, err error)
- func LcacheIdx(digest uint64) int
- func LoadNodeID(mpaths cos.StrKVs) (mDaeID string, err error)
- func MarkerExists(marker string) bool
- func MountpathsToLists() (mpl *apc.MountpathList)
- func New(num int)
- func NoneShared(numMpaths int) bool
- func NumAvail() int
- func OnDiskSize(bck *cmn.Bck, prefix string) (size uint64)
- func PersistMarker(marker string) (fatalErr, writeErr error)
- func PersistOnMpaths(fname, backupName string, meta jsp.Opts, atMost int, b []byte, sgl *memsys.SGL) (cnt, availCnt int)
- func PutMPI(available, disabled MPI)
- func RemoveAll(dir string) (err error)
- func RemoveMarker(marker string) (err error)
- func RenameBucketDirs(bckFrom, bckTo *cmn.Bck) (err error)
- func SetXattr(fqn, attrName string, data []byte) (err error)
- func TestNew(iostater ios.IOS)
- func ValidateNCopies(tname string, copies int) (err error)
- func Walk(opts *WalkOpts) error
- func WalkBck(opts *WalkBckOpts) error
- func WalkDir(dir string, ucb func(string, DirEntry) error) error
- type CDF
- type CDFv322
- type CapStatus
- type Capacity
- type ContentInfo
- type ContentResolver
- type DirEntry
- type ECMetaContentResolver
- type ECSliceContentResolver
- func (*ECSliceContentResolver) GenUniqueFQN(base, _ string) string
- func (*ECSliceContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
- func (*ECSliceContentResolver) PermToEvict() bool
- func (*ECSliceContentResolver) PermToMove() bool
- func (*ECSliceContentResolver) PermToProcess() bool
- type ErrMountpathNoDisks
- type ErrStorageIntegrity
- type MFS
- type MPI
- type Mountpath
- func Add(mpath, tid string) (mi *Mountpath, err error)
- func AddMpath(tid, mpath string, label ios.Label, cb func()) (mi *Mountpath, err error)
- func BeginDD(action string, flags uint64, mpath string) (mi *Mountpath, numAvail int, noResil bool, err error)
- func Disable(mpath string, cb ...func()) (disabledMpath *Mountpath, err error)
- func Enable(mpath string) (enabledMpath *Mountpath, err error)
- func EnableMpath(mpath, tid string, cb func()) (enabledMpath *Mountpath, err error)
- func FQN2Mpath(fqn string) (found *Mountpath, relativePath string, err error)
- func Hrw(uname string) (mi *Mountpath, digest uint64, err error)
- func NewMountpath(mpath string, label ios.Label) (*Mountpath, error)
- func Path2Mpath(path string) (found *Mountpath, err error)
- func Remove(mpath string, cb ...func()) (*Mountpath, error)
- func (mi *Mountpath) AddDisabled(disabled MPI)
- func (mi *Mountpath) AddEnabled(tid string, avail MPI, config *cmn.Config) (err error)
- func (mi *Mountpath) ClearDD()
- func (mi *Mountpath) ClearMDs(inclBMD bool) (rerr error)
- func (mi *Mountpath) CreateMissingBckDirs(bck *cmn.Bck) (err error)
- func (mi *Mountpath) DeletedRoot() string
- func (mi *Mountpath) IsAnySet(flags uint64) bool
- func (mi *Mountpath) IsIdle(config *cmn.Config) bool
- func (mi *Mountpath) LomCache(idx int) *sync.Map
- func (mi *Mountpath) MakePathBck(bck *cmn.Bck) string
- func (mi *Mountpath) MakePathCT(bck *cmn.Bck, contentType string) string
- func (mi *Mountpath) MakePathFQN(bck *cmn.Bck, contentType, objName string) string
- func (mi *Mountpath) MoveToDeleted(dir string) (err error)
- func (mi *Mountpath) RemoveDeleted(who string) (rerr error)
- func (mi *Mountpath) SetDaemonIDXattr(tid string) error
- func (mi *Mountpath) String() string
- func (mi *Mountpath) TempDir(dir string) string
- type ObjectContentResolver
- type ParsedFQN
- type PartsFQN
- type TargetCDF
- type TargetCDFv322
- type WalkBckOpts
- type WalkOpts
- type WorkfileContentResolver
- func (*WorkfileContentResolver) GenUniqueFQN(base, prefix string) string
- func (*WorkfileContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
- func (*WorkfileContentResolver) PermToEvict() bool
- func (*WorkfileContentResolver) PermToMove() bool
- func (*WorkfileContentResolver) PermToProcess() bool
Constants ¶
const ( ObjectType = "ob" WorkfileType = "wk" ECSliceType = "ec" ECMetaType = "mt" )
const ( SieMpathIDMismatch = (1 + iota) * 10 SieTargetIDMismatch SieNotEqVMD SieMetaCorrupted SieFsDiffers SieMpathNotFound )
const ( // prefixes for workfiles created by various services WorkfileRemote = "remote" // getting object from neighbor target when rebalancing WorkfileColdget = "cold" // object GET: coldget WorkfilePut = "put" // object PUT WorkfileCopy = "copy" // copy object WorkfileAppend = "append" // APPEND to object (as file) WorkfileAppendToArch = "append-to-arch" // APPEND to existing archive WorkfileCreateArch = "create-arch" // CREATE multi-object archive )
const ( FlagBeingDisabled uint64 = 1 << iota FlagBeingDetached )
enum Mountpath.Flags
const FlagWaitingDD = FlagBeingDisabled | FlagBeingDetached
Variables ¶
var CSM *contentSpecMgr
Functions ¶
func CapStatusGetWhat ¶
func CapStatusGetWhat() (fsInfo apc.CapacityInfo)
func CleanPathErr ¶ added in v1.3.18
func CleanPathErr(err error)
func CountPersisted ¶
func Decommission ¶
func Decommission(mdOnly bool)
func DestroyBucket ¶
NOTE: caller must make sure to evict LOM cache
func DirectOpen ¶
DirectOpen opens a file with direct disk access (with OS caching disabled).
func ExpireCapCache ¶ added in v1.3.23
func ExpireCapCache()
func FillDiskStats ¶
func FillDiskStats(m ios.AllDiskStats)
func GetAllMpathUtils ¶
func GetDiskSize ¶ added in v1.3.21
func GetDiskSize() uint64
func GetMpathUtil ¶
func GetXattrBuf ¶
GetXattr gets xattr by name via provided buffer
func LoadNodeID ¶
traverses all mountpaths to load and validate node ID
func MarkerExists ¶
func MountpathsToLists ¶
func MountpathsToLists() (mpl *apc.MountpathList)
func NoneShared ¶ added in v1.3.23
func OnDiskSize ¶ added in v1.3.21
bucket and bucket+prefix on-disk sizing
func PersistMarker ¶
func PersistOnMpaths ¶
func PersistOnMpaths(fname, backupName string, meta jsp.Opts, atMost int, b []byte, sgl *memsys.SGL) (cnt, availCnt int)
PersistOnMpaths persists `what` on mountpaths under "mountpath.Path/path" filename. It does it on maximum `atMost` mountPaths. If `atMost == 0`, it does it on every mountpath. If `backupPath != ""`, it removes files from `backupPath` and moves files from `path` to `backupPath`. Returns how many times it has successfully stored a file.
func RemoveMarker ¶
func RenameBucketDirs ¶
func ValidateNCopies ¶
func WalkBck ¶
func WalkBck(opts *WalkBckOpts) error
Types ¶
type CDF ¶ added in v1.3.16
type CDF struct {
Capacity
Disks []string `json:"disks"` // owned or shared disks (ios.FsDisks map => slice)
Label ios.Label `json:"mountpath_label"`
FS cos.FS `json:"fs"`
}
Capacity, Disks, Filesystem (CDF)
type CapStatus ¶
type CapStatus struct {
// config
HighWM int64
OOS int64
// metrics
TotalUsed uint64 // bytes
TotalAvail uint64 // bytes
PctAvg int32 // average used (%)
PctMax int32 // max used (%)
PctMin int32 // max used (%)
}
func CapPeriodic ¶
func CapPeriodic(now int64, config *cmn.Config, tcdf *TargetCDF) (cs CapStatus, updated bool, err, errCap error)
called only and exclusively by `stats.Trunner` providing `config.Periodic.StatsTime` tick
func CapRefresh ¶ added in v1.3.16
sum up && compute %% capacities while skipping already _counted_ filesystems
type ContentInfo ¶
type ContentResolver ¶
type ContentResolver interface {
// When set to true, services like rebalance have permission to move
// content for example to another target because it is misplaced (HRW).
PermToMove() bool
// When set to true, services like LRU have permission to evict/delete content
PermToEvict() bool
// When set to true, content can be checksumed, shown or processed in other ways.
PermToProcess() bool
// Generates unique base name for original one. This function may add
// additional information to the base name.
// prefix - user-defined marker
GenUniqueFQN(base, prefix string) (ufqn string)
// Parses generated unique fqn to the original one.
ParseUniqueFQN(base string) (orig string, old, ok bool)
}
type ECMetaContentResolver ¶
type ECMetaContentResolver struct{}
func (*ECMetaContentResolver) GenUniqueFQN ¶
func (*ECMetaContentResolver) GenUniqueFQN(base, _ string) string
func (*ECMetaContentResolver) ParseUniqueFQN ¶
func (*ECMetaContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
func (*ECMetaContentResolver) PermToEvict ¶
func (*ECMetaContentResolver) PermToEvict() bool
func (*ECMetaContentResolver) PermToMove ¶
func (*ECMetaContentResolver) PermToMove() bool
func (*ECMetaContentResolver) PermToProcess ¶
func (*ECMetaContentResolver) PermToProcess() bool
type ECSliceContentResolver ¶
type ECSliceContentResolver struct{}
func (*ECSliceContentResolver) GenUniqueFQN ¶
func (*ECSliceContentResolver) GenUniqueFQN(base, _ string) string
func (*ECSliceContentResolver) ParseUniqueFQN ¶
func (*ECSliceContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
func (*ECSliceContentResolver) PermToEvict ¶
func (*ECSliceContentResolver) PermToEvict() bool
func (*ECSliceContentResolver) PermToMove ¶
func (*ECSliceContentResolver) PermToMove() bool
func (*ECSliceContentResolver) PermToProcess ¶
func (*ECSliceContentResolver) PermToProcess() bool
type ErrMountpathNoDisks ¶
type ErrMountpathNoDisks struct {
Mi *Mountpath
}
func (*ErrMountpathNoDisks) Error ¶
func (e *ErrMountpathNoDisks) Error() string
type ErrStorageIntegrity ¶
func (*ErrStorageIntegrity) Error ¶
func (sie *ErrStorageIntegrity) Error() string
type Mountpath ¶ added in v1.3.16
type Mountpath struct {
Path string // clean path
Label ios.Label // (disk sharing; help resolve lsblk; storage class; user-defined grouping)
cos.FS // underlying filesystem
Disks []string // owned disks (ios.FsDisks map => slice)
PathDigest uint64 // (HRW logic)
// contains filtered or unexported fields
}
func BeginDD ¶
func BeginDD(action string, flags uint64, mpath string) (mi *Mountpath, numAvail int, noResil bool, err error)
begin (disable | detach) transaction: CoW-mark the corresponding mountpath
func Disable ¶
Disables a mountpath, i.e., removes it from usage but keeps in the volume (for possible future re-enablement). If successful, returns the disabled mountpath. Otherwise, returns nil (also in the case if the mountpath was already disabled).
func EnableMpath ¶
Enable enables previously disabled mountpath. enabled is set to true if mountpath has been moved from disabled to available and exists is set to true if such mountpath even exists.
func Path2Mpath ¶
Path2Mpath takes in any file path (e.g., ../../a/b/c) and returns the matching `mi`, if exists
func Remove ¶
Remove removes mountpaths from the target's mountpaths. It searches for the mountpath in `available` and, if not found, in `disabled`.
func (*Mountpath) AddDisabled ¶ added in v1.3.16
func (*Mountpath) AddEnabled ¶ added in v1.3.16
func (*Mountpath) CreateMissingBckDirs ¶ added in v1.3.16
func (*Mountpath) DeletedRoot ¶ added in v1.3.16
func (*Mountpath) MakePathBck ¶ added in v1.3.16
func (*Mountpath) MakePathCT ¶ added in v1.3.16
func (*Mountpath) MakePathFQN ¶ added in v1.3.16
func (*Mountpath) MoveToDeleted ¶ added in v1.3.16
MoveToDeleted removes directory in steps: 1. Synchronously gets temporary directory name 2. Synchronously renames old folder to temporary directory
func (*Mountpath) RemoveDeleted ¶ added in v1.3.16
func (*Mountpath) SetDaemonIDXattr ¶ added in v1.3.16
type ObjectContentResolver ¶
type ObjectContentResolver struct{}
func (*ObjectContentResolver) GenUniqueFQN ¶
func (*ObjectContentResolver) GenUniqueFQN(base, _ string) string
func (*ObjectContentResolver) ParseUniqueFQN ¶
func (*ObjectContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
func (*ObjectContentResolver) PermToEvict ¶
func (*ObjectContentResolver) PermToEvict() bool
func (*ObjectContentResolver) PermToMove ¶
func (*ObjectContentResolver) PermToMove() bool
func (*ObjectContentResolver) PermToProcess ¶
func (*ObjectContentResolver) PermToProcess() bool
type ParsedFQN ¶
type TargetCDF ¶ added in v1.3.16
type TargetCDF struct {
Mountpaths map[string]*CDF // mpath => [Capacity, Disks, FS (CDF)]
TotalUsed uint64 `json:"total_used,string"` // bytes
TotalAvail uint64 `json:"total_avail,string"` // bytes
PctMax int32 `json:"pct_max"` // max used (%)
PctAvg int32 `json:"pct_avg"` // avg used (%)
PctMin int32 `json:"pct_min"` // min used (%)
CsErr string `json:"cs_err"` // OOS or high-wm error message
}
Target (cumulative) CDF
type TargetCDFv322 ¶ added in v1.3.23
type TargetCDFv322 struct {
Mountpaths map[string]*CDFv322
PctMax int32 `json:"pct_max"`
PctAvg int32 `json:"pct_avg"`
PctMin int32 `json:"pct_min"`
CsErr string `json:"cs_err"`
}
[backward compatibility]: v3.22 cdf* structures
type WalkBckOpts ¶
type WalkBckOpts struct {
ValidateCb walkFunc // should return filepath.SkipDir to skip directory without an error
WalkOpts
}
type WorkfileContentResolver ¶
type WorkfileContentResolver struct{}
func (*WorkfileContentResolver) GenUniqueFQN ¶
func (*WorkfileContentResolver) GenUniqueFQN(base, prefix string) string
func (*WorkfileContentResolver) ParseUniqueFQN ¶
func (*WorkfileContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)
func (*WorkfileContentResolver) PermToEvict ¶
func (*WorkfileContentResolver) PermToEvict() bool
func (*WorkfileContentResolver) PermToMove ¶
func (*WorkfileContentResolver) PermToMove() bool
func (*WorkfileContentResolver) PermToProcess ¶
func (*WorkfileContentResolver) PermToProcess() bool