core

package
v1.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 5, 2025 License: MIT Imports: 39 Imported by: 2

Documentation

Overview

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source 
const (
	RemoteDeletedDelCount = "remote.deleted.del.n"

	// lcache stats
	LcacheCollisionCount = "lcache.collision.n"
	LcacheEvictedCount   = "lcache.evicted.n"
	LcacheErrCount       = "err.lcache.n" // errPrefix + "lcache.n"
	LcacheFlushColdCount = "lcache.flush.cold.n"
)

core stats

View Source 
const (
	// backward compat v3.31 and prior
	MetaverLOM_V1 = 1 //nolint:revive // readability

	// current
	MetaverLOM = 2
)

LOM

View Source 
const (
	Targets = iota // 0 (core.Targets) used as default value for NewStreamBundle
	Proxies
	AllNodes
	SelectedNodes
)
View Source 
const (
	UponTerm     = Upon(1 << iota) // success or fail is separately provided via error
	UponProgress                   // periodic (BytesCount, ObjCount)
)

enum: when to notify

View Source 
const (
	QuiInactiveCB     = QuiRes(iota) // e.g., no pending requests (NOTE: used exclusively by `quicb` callbacks)
	QuiActive                        // active (e.g., receiving data)
	QuiActiveRet                     // active that immediately breaks waiting for quiecscence
	QuiActiveDontBump                // active that does not increase inactivity duration (ie., keeps initial setting)
	QuiDone                          // all done
	QuiAborted                       // aborted
	QuiTimeout                       // timeout
	Quiescent                        // idle => quiescent
)
View Source 
const (
	AisBID = cos.MSB64
)
View Source 
const (
	DumpLomEnvVar = "AIS_DUMP_LOM"
)

cmd/xmeta support

View Source 
const (

	// MaxChunkCount is the maximum number of chunks allowed per object (limited by uint16 chunk numbering)
	MaxChunkCount = 9999
)

Variables

This section is empty.

Functions

func DrainLIF added in v1.3.30 

func DrainLIF(workCh chan LIF) (n int)

non-blocking drain LIF workCh

func FreeLOM 

func FreeLOM(lom *LOM)

func FreePutParams 

func FreePutParams(a *PutParams)

func HrwFQN 

func HrwFQN(bck *cmn.Bck, contentType, objName string) (fqn string, digest uint64, err error)

func InMaintOrDecomm 

func InMaintOrDecomm(smap *meta.Smap, tsi *meta.Snode, xact Xact) error

func LcacheClear added in v1.3.28 

func LcacheClear()

func LcacheClearBcks added in v1.3.28 

func LcacheClearBcks(wg *sync.WaitGroup, bcks ...*meta.Bck) bool

func LcacheClearMpath added in v1.3.28 

func LcacheClearMpath(mi *fs.Mountpath)

func NewBID added in v1.3.26 

func NewBID(serial uint64, isAis bool) uint64

func ParseObjLoc 

func ParseObjLoc(loc string) (tname, mpname string)

func Pinit 

func Pinit()

func ResolveFQN 

func ResolveFQN(fqn string, parsed *fs.ParsedFQN) (hrwFQN string, err error)

func Term 

func Term()

func Tinit 

func Tinit(t Target, config *cmn.Config, runHK bool)

Types

type AllRunningInOut 

type AllRunningInOut struct {
	Kind    string
	Running []string
	Idle    []string // NOTE: returning only when not nil
}

type Backend added in v1.3.23 

type Backend interface {
	Provider() string
	MetricName(string) string

	CreateBucket(bck *meta.Bck) (ecode int, err error)
	ListBuckets(qbck cmn.QueryBcks) (bcks cmn.Bcks, ecode int, err error)

	// list-objects
	ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes) (ecode int, err error)
	ListObjectsInv(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes, ctx *LsoInvCtx) error

	PutObj(ctx context.Context, r io.ReadCloser, lom *LOM, origReq *http.Request) (ecode int, err error)
	DeleteObj(ctx context.Context, lom *LOM) (ecode int, err error)

	// head
	HeadBucket(ctx context.Context, bck *meta.Bck) (bckProps cos.StrKVs, ecode int, err error)
	HeadObj(ctx context.Context, lom *LOM, origReq *http.Request) (objAttrs *cmn.ObjAttrs, ecode int, err error)

	// get (exclusively via GetCold; calls GetObjReader)
	GetObj(ctx context.Context, lom *LOM, owt cmn.OWT, origReq *http.Request) (ecode int, err error)
	// get (jobs; REST)
	GetObjReader(ctx context.Context, lom *LOM, offset, length int64) GetReaderResult

	// bucket inventory
	GetBucketInv(bck *meta.Bck, ctx *LsoInvCtx) (ecode int, err error)

	// multipart upload
	// TODO: implement these methods for remote AIS, GCP and Azure backends
	StartMpt(lom *LOM, r *http.Request) (uploadID string, ecode int, err error)
	PutMptPart(lom *LOM, reader io.ReadCloser, r *http.Request, uploadID string, size int64, partNum int32) (etag string, ecode int, err error)
	CompleteMpt(lom *LOM, r *http.Request, uploadID string, body []byte, parts apc.MptCompletedParts) (version, etag string, ecode int, err error)
	AbortMpt(lom *LOM, r *http.Request, uploadID string) (ecode int, err error)
}

type BlobParams added in v1.3.23 

type BlobParams struct {
	Lom *LOM
	Msg *apc.BlobMsg

	// When `RespWriter` is set, `XactBlobDl` not only downloads chunks into the cluster,
	// but also stitches them together and sequentially writes to `RespWriter`.
	// This makes the blob downloading job synchronous and blocking until all chunks are written.
	// Only set this if you need to simultaneously download and write to the response writer (e.g., for streaming blob GET).
	RespWriter io.Writer
}

intra-cluster data path: control structures and types

type CRMD 

type CRMD struct {
	Err      error
	ObjAttrs *cmn.ObjAttrs
	ErrCode  int
	Eq       bool
}

returned by lom.CheckRemoteMD

type CT 

type CT struct {
	// contains filtered or unexported fields
}

func NewCTFromBO 

func NewCTFromBO(bck *meta.Bck, objName, ctType string, extras ...string) (ct *CT, err error)

costruct, init bucket and compute HRW (similar to lom.InitBck())

func NewCTFromFQN 

func NewCTFromFQN(fqn string, b meta.Bowner) (ct *CT, err error)

full construction and init, including: - parse fqn - init bucket - HRW

func NewCTFromLOM 

func NewCTFromLOM(lom *LOM, ctType string, extras ...string) (ct *CT)

bare minimum: from LOM

func NewCTFromParsed added in v1.4.0 

func NewCTFromParsed(parsed *fs.ParsedFQN, fqn string) *CT

bare minimum: from fs.ParsedFQN

func NewDsortCT added in v1.4.0 

func NewDsortCT(bck *cmn.Bck, objName string) (ct *CT, err error)

usage: dsort only

func (*CT) Bck 

func (ct *CT) Bck() *meta.Bck

func (*CT) Bucket 

func (ct *CT) Bucket() *cmn.Bck

func (*CT) Clone 

func (ct *CT) Clone(ctType string) (clone *CT)

bare minimum: from CT

func (*CT) Cname added in v1.3.24 

func (ct *CT) Cname() string

func (*CT) ContentType 

func (ct *CT) ContentType() string

func (*CT) Digest 

func (ct *CT) Digest() uint64

func (*CT) FQN 

func (ct *CT) FQN() string

func (*CT) GenFQN added in v1.4.0 

func (ct *CT) GenFQN(cttype string, extras ...string) string

func (*CT) LoadSliceFromFS added in v1.3.24 

func (ct *CT) LoadSliceFromFS() error

func (*CT) Lock 

func (ct *CT) Lock(exclusive bool)

func (*CT) Lsize added in v1.3.24 

func (ct *CT) Lsize() int64

func (*CT) Mountpath 

func (ct *CT) Mountpath() *fs.Mountpath

func (*CT) MtimeUnix 

func (ct *CT) MtimeUnix() int64

func (*CT) ObjectName 

func (ct *CT) ObjectName() string

func (*CT) UnamePtr added in v1.3.24 

func (ct *CT) UnamePtr() *string

func (*CT) Unlock 

func (ct *CT) Unlock(exclusive bool)

func (*CT) Write 

func (ct *CT) Write(reader io.Reader, size int64, workFQN string) (err error)

Save CT to local drives. If workFQN is set, it saves in two steps: first, save to workFQN; second, rename workFQN to ct.fqn. If unset, it writes directly to ct.fqn

type ETLArgs added in v1.3.31 

type ETLArgs struct {
	TransformArgs string          // optional and ETL-specific; can be used to indicate transformation on a per-object basis
	Pipeline      apc.ETLPipeline // intermediate ETL pod's address or destination target's address for direct put
}

GetROC defines a function that retrieves an object based on the given `lom` and flags. If `Pipeline` are provided, the implementation may choose to deliver the object directly, in which case the returned ReadResp will have R = nil and Ecode = 204.

Implementations include: - `core.DefaultGetROC`: fetches from local or remote backend - `etl.HTTPCommunicator.OfflineTransform`: fetches transformed object from ETL pod through HTTP response - `etl.statefulCommnicator.Transform`: fetches transformed object from ETL pod through WebSocket message

type GetROC added in v1.3.28 

type GetROC func(lom *LOM, latestVer, sync bool, args *ETLArgs) ReadResp

type GetReaderResult 

type GetReaderResult struct {
	R        io.ReadCloser
	Err      error
	ExpCksum *cos.Cksum
	Size     int64
	ErrCode  int
}

type GetStats added in v1.4.1 

type GetStats interface {
	Objs() int64
	ObjsAdd(int, int64)    // locally processed
	OutObjsAdd(int, int64) // transmit
	InObjsAdd(int, int64)  // receive
	InBytes() int64
	OutBytes() int64
}

type GfnParams added in v1.4.0 

type GfnParams struct {
	Lom      *LOM
	Tsi      *meta.Snode
	Config   *cmn.Config
	ArchPath string
	Size     int64
}

intra-cluster data path: control structures and types

type LIF 

type LIF struct {
	// contains filtered or unexported fields
}

LOM In Flight (LIF)

func (*LIF) CacheIdx 

func (lif *LIF) CacheIdx() int

func (*LIF) LOM 

func (lif *LIF) LOM() (lom *LOM, err error)

LIF => LOM with a check for bucket existence

func (*LIF) Name added in v1.3.28 

func (lif *LIF) Name() string

func (*LIF) Unlock 

func (lif *LIF) Unlock(exclusive bool)

type LOM 

type LOM struct {
	ObjName string
	FQN     string
	HrwFQN  *string // (=> main replica)
	// contains filtered or unexported fields
}

func AllocLOM 

func AllocLOM(objName string) *LOM

func (*LOM) AddCopy 

func (lom *LOM) AddCopy(copyFQN string, mpi *fs.Mountpath) error

NOTE: used only in tests

func (*LOM) AppendWork added in v1.3.24 

func (*LOM) AppendWork(wfqn string) (fh cos.LomWriter, err error)

append

func (*LOM) Atime 

func (lom *LOM) Atime() time.Time

func (*LOM) AtimeUnix 

func (lom *LOM) AtimeUnix() int64

func (*LOM) Bck 

func (lom *LOM) Bck() *meta.Bck

assorted _convenient_ accessors

func (*LOM) Bprops 

func (lom *LOM) Bprops() *cmn.Bprops

func (*LOM) Bucket 

func (lom *LOM) Bucket() *cmn.Bck

func (*LOM) CacheIdx 

func (lom *LOM) CacheIdx() int

func (*LOM) CheckEq added in v1.3.24 

func (lom *LOM) CheckEq(rem cos.OAH) error

LOM == remote-object equality check

func (*LOM) CheckRemoteMD 

func (lom *LOM) CheckRemoteMD(locked, sync bool, origReq *http.Request) (res CRMD)

NOTE: - [PRECONDITION]: `versioning.validate_warm_get` || QparamLatestVer - [Sync] when Sync option is used (via bucket config and/or `sync` argument) caller MUST take wlock or rlock - [MAY] delete remotely-deleted (non-existing) object and increment associated stats counter

also returns `NotFound` after removing local replica - the Sync option

func (*LOM) Checksum 

func (lom *LOM) Checksum() *cos.Cksum

func (*LOM) CksumConf 

func (lom *LOM) CksumConf() *cmn.CksumConf

func (*LOM) CksumType 

func (lom *LOM) CksumType() string

more cksum conf

func (*LOM) Clone added in v1.3.30 

func (lom *LOM) Clone() *LOM

func (*LOM) CloneTo added in v1.3.30 

func (lom *LOM) CloneTo(fqn string) *LOM

allocates and copies metadata (in particular, atime and uname) NOTE: cloned lom.bid() == 0 is possible - copying/transforming scenarios

func (*LOM) Cname 

func (lom *LOM) Cname() string

see also: transport.ObjHdr.Cname()

func (*LOM) CompleteUfest added in v1.4.0 

func (lom *LOM) CompleteUfest(u *Ufest, locked bool) (err error)

func (*LOM) ComputeCksum 

func (lom *LOM) ComputeCksum(cksumType string, locked bool) (cksum *cos.CksumHash, _ error)

func (*LOM) ComputeSetCksum 

func (lom *LOM) ComputeSetCksum(locked bool) (*cos.Cksum, error)

func (*LOM) Copy 

func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) error

increment the object's num copies by (well) copying the former (compare with lom.Copy2FQN below)

func (*LOM) Copy2FQN 

func (lom *LOM) Copy2FQN(dstFQN string, buf []byte) (dst *LOM, err error)

copy object => any local destination recommended for copying between different buckets (compare with lom.Copy() above) NOTE: `lom` source and `dst` must be w-locked

func (*LOM) CopyAttrs 

func (lom *LOM) CopyAttrs(oah cos.OAH, skipCksum bool)

func (*LOM) CopyVersion added in v1.3.24 

func (lom *LOM) CopyVersion(oah cos.OAH)

func (*LOM) Create added in v1.3.24 

func (lom *LOM) Create() (cos.LomWriter, error)

func (*LOM) CreatePart added in v1.3.24 

func (lom *LOM) CreatePart(wfqn string) (*os.File, error)

func (*LOM) CreateSlice added in v1.3.24 

func (lom *LOM) CreateSlice(wfqn string) (*os.File, error)

func (*LOM) CreateWork added in v1.3.24 

func (lom *LOM) CreateWork(wfqn string) (cos.LomWriter, error)

func (*LOM) DelAllCopies 

func (lom *LOM) DelAllCopies() (err error)

func (*LOM) DelCopies 

func (lom *LOM) DelCopies(copiesFQN ...string) (err error)

func (*LOM) DelCustomKey added in v1.4.0 

func (lom *LOM) DelCustomKey(key string)

func (*LOM) DelExtraCopies 

func (lom *LOM) DelExtraCopies(fqn ...string) (removed bool, err error)

DelExtraCopies deletes obj replicas that are not part of the lom.md.copies metadata (cleanup)

func (*LOM) Digest 

func (lom *LOM) Digest() uint64

func (*LOM) DowngradeLock 

func (lom *LOM) DowngradeLock()

func (*LOM) ECEnabled 

func (lom *LOM) ECEnabled() bool

func (*LOM) EqCksum 

func (lom *LOM) EqCksum(cksum *cos.Cksum) bool

func (*LOM) FromFS 

func (lom *LOM) FromFS() error

func (*LOM) Fstat added in v1.3.24 

func (lom *LOM) Fstat(getAtime bool) (size, atimefs int64, mtime time.Time, _ error)

low-level access to the os.FileInfo of a chunk or whole file

func (*LOM) GenFQN added in v1.4.0 

func (lom *LOM) GenFQN(cttype string, extras ...string) string

func (*LOM) GetCopies 

func (lom *LOM) GetCopies() fs.MPI

GetCopies returns all copies - copies include lom.FQN aka "main repl." - caller must take a lock

func (*LOM) GetCustomKey 

func (lom *LOM) GetCustomKey(key string) (string, bool)

func (*LOM) GetCustomMD 

func (lom *LOM) GetCustomMD() cos.StrKVs

custom metadata

func (*LOM) GetROC added in v1.3.28 

func (lom *LOM) GetROC(latestVer, sync bool) (resp ReadResp)

func (*LOM) GetXattr added in v1.3.30 

func (lom *LOM) GetXattr(buf []byte) ([]byte, error)

func (*LOM) GetXattrN added in v1.3.30 

func (lom *LOM) GetXattrN(name string) ([]byte, error)

func (*LOM) HasCopies 

func (lom *LOM) HasCopies() bool

func (*LOM) HrwTarget 

func (lom *LOM) HrwTarget(smap *meta.Smap) (tsi *meta.Snode, local bool, err error)

func (*LOM) IncVersion 

func (lom *LOM) IncVersion() error

func (*LOM) InitBck 

func (lom *LOM) InitBck(bck *cmn.Bck) (err error)

func (*LOM) InitCT 

func (lom *LOM) InitCT(ct *CT)

func (*LOM) InitFQN 

func (lom *LOM) InitFQN(fqn string, expbck *cmn.Bck) error

func (*LOM) IsChunked added in v1.3.24 

func (lom *LOM) IsChunked(special ...bool) bool

func (*LOM) IsCopy 

func (lom *LOM) IsCopy() bool

given an existing (on-disk) object, determines whether it is a _copy_ (compare with isMirror below)

func (*LOM) IsFeatureSet added in v1.3.23 

func (lom *LOM) IsFeatureSet(f feat.Flags) bool

func (*LOM) IsFntl added in v1.3.26 

func (lom *LOM) IsFntl() bool

func (*LOM) IsHRW 

func (lom *LOM) IsHRW() bool

subj to resilvering

func (*LOM) IsLocked 

func (lom *LOM) IsLocked() int

returns {apc.LockNone, ...} enum

func (*LOM) LIF 

func (lom *LOM) LIF() (lif LIF)

LOM => LIF constructor

func (*LOM) LeastUtilNoCopy 

func (lom *LOM) LeastUtilNoCopy() (mi *fs.Mountpath)

returns the least-utilized mountpath that does _not_ have a copy of this `lom` yet (compare with leastUtilCopy())

func (*LOM) Load 

func (lom *LOM) Load(cacheit, locked bool) error

no lock is taken when locked by an immediate caller, or otherwise is known to be locked otherwise, try Rlock temporarily _if and only when_ reading from fs

(compare w/ LoadUnsafe() below)

func (*LOM) LoadLatest 

func (lom *LOM) LoadLatest(latest bool) (oa *cmn.ObjAttrs, deleted bool, err error)

NOTE: Sync is false (ie., not deleting)

func (*LOM) LoadMetaFromFS 

func (lom *LOM) LoadMetaFromFS() error

NOTE usage: tests and `xmeta` only; ignores `dirty`

func (*LOM) Location 

func (lom *LOM) Location() string

to report via list-objects and HEAD()

func (*LOM) Lock 

func (lom *LOM) Lock(exclusive bool)

func (*LOM) Lsize added in v1.3.24 

func (lom *LOM) Lsize(special ...bool) int64

special a) when a new version is being created b) for usage in unit tests

func (*LOM) MirrorConf 

func (lom *LOM) MirrorConf() *cmn.MirrorConf

func (*LOM) MirrorPaths added in v1.3.30 

func (lom *LOM) MirrorPaths() []string

func (*LOM) Mountpath 

func (lom *LOM) Mountpath() *fs.Mountpath

func (*LOM) MtimeUTC added in v1.4.0 

func (lom *LOM) MtimeUTC() (time.Time, error)

func (*LOM) NewArchpathReader added in v1.3.30 

func (lom *LOM) NewArchpathReader(lh cos.LomReader, archpath, mime string) (csl cos.ReadCloseSizer, err error)

extract a single file from a (.tar, .tgz or .tar.gz, .zip, .tar.lz4) shard uses the provided `mime` or lom.ObjName to detect formatting (empty = auto-detect)

func (*LOM) NewDeferROC 

func (lom *LOM) NewDeferROC(loaded bool) (cos.ReadOpenCloser, error)

is called under rlock; unlocks on fail NOTE: compare w/ lom.Open() returning cos.LomReader

func (*LOM) NewHandle added in v1.3.30 

func (lom *LOM) NewHandle(loaded bool) (*LomHandle, error)

func (*LOM) NewUfestReader added in v1.4.0 

func (lom *LOM) NewUfestReader() (cos.LomReader, error)

func (*LOM) NumCopies 

func (lom *LOM) NumCopies() int

func (*LOM) ObjAttrs 

func (lom *LOM) ObjAttrs() *cmn.ObjAttrs

func (*LOM) ObjectName 

func (lom *LOM) ObjectName() string

as fs.PartsFQN

func (*LOM) Open added in v1.3.24 

func (lom *LOM) Open() (lh cos.LomReader, err error)

open read-only, return a reader see also: lom.NewDeferROC() see also: lom.GetROC()

func (*LOM) OpenCopy added in v1.4.0 

func (lom *LOM) OpenCopy() (cos.LomReader, string)

load-balanced GET from replicated lom - picks least-utilized mountpath - returns (open reader + its FQN) or (nil, "")

func (*LOM) OrigFntl added in v1.3.26 

func (lom *LOM) OrigFntl() []string

func (*LOM) Persist 

func (lom *LOM) Persist() error

(caller must set atime; compare with the above)

func (*LOM) PersistMain 

func (lom *LOM) PersistMain(isChunked bool) error

func (*LOM) PopFntl added in v1.3.26 

func (lom *LOM) PopFntl(saved []string)

func (*LOM) PostInit added in v1.3.23 

func (lom *LOM) PostInit() error

func (*LOM) PreInit added in v1.3.23 

func (lom *LOM) PreInit(fqn string) error

NOTE: to facilitate fast path filtering-out

func (*LOM) PushFntl added in v1.3.26 

func (lom *LOM) PushFntl(short []string) (saved []string)

func (*LOM) Recache 

func (lom *LOM) Recache()

store new or refresh existing

func (*LOM) RemoveMain added in v1.3.24 

func (lom *LOM) RemoveMain() error

func (*LOM) RemoveObj added in v1.3.24 

func (lom *LOM) RemoveObj(force ...bool) (err error)

func (*LOM) RenameFinalize added in v1.3.24 

func (lom *LOM) RenameFinalize(wfqn string) error

func (*LOM) RenameMainTo added in v1.3.24 

func (lom *LOM) RenameMainTo(wfqn string) error

func (*LOM) RenameToMain added in v1.3.24 

func (lom *LOM) RenameToMain(wfqn string) error

func (*LOM) RestoreToLocation 

func (lom *LOM) RestoreToLocation() (exists bool)

RestoreObjectFromAny tries to restore the object at its default location. Returns true if object exists, false otherwise TODO: locking vs concurrent restore: consider (read-lock object + write-lock meta) split

func (*LOM) SetAtimeUnix 

func (lom *LOM) SetAtimeUnix(tu int64)

func (*LOM) SetCksum 

func (lom *LOM) SetCksum(cksum *cos.Cksum)

func (*LOM) SetCustomKey 

func (lom *LOM) SetCustomKey(key, value string)

func (*LOM) SetCustomMD 

func (lom *LOM) SetCustomMD(md cos.StrKVs)

func (*LOM) SetSize 

func (lom *LOM) SetSize(size int64)

func (*LOM) SetVersion 

func (lom *LOM) SetVersion(ver string)

func (*LOM) SetXattr added in v1.3.30 

func (lom *LOM) SetXattr(data []byte) error

func (*LOM) SetXattrN added in v1.3.30 

func (lom *LOM) SetXattrN(name string, data []byte) error

func (*LOM) ShortenFntl added in v1.3.26 

func (lom *LOM) ShortenFntl() []string

(compare with fs/content Gen())

func (*LOM) String 

func (lom *LOM) String() string

func (*LOM) TestAtime added in v1.3.26 

func (lom *LOM) TestAtime() error

usage: unit tests only

func (*LOM) ToMpath 

func (lom *LOM) ToMpath() (mi *fs.Mountpath, fixHrw bool)

must be called under w-lock returns mountpath to relocate or copy this lom, or nil if none required/available return fixHrw = true when lom is currently misplaced - checks hrw location first, and - checks copies (if any) against the current configuration and available mountpaths; - does not check `fstat` in either case (TODO: configurable or scrub);

func (*LOM) TryLock 

func (lom *LOM) TryLock(exclusive bool) bool

func (*LOM) Uname 

func (lom *LOM) Uname() string

func (*LOM) UnamePtr added in v1.3.24 

func (lom *LOM) UnamePtr() *string

func (*LOM) Uncache 

func (lom *LOM) Uncache()

func (*LOM) UncacheDel added in v1.3.28 

func (lom *LOM) UncacheDel()

upon: error; RemoveObj

func (*LOM) UncacheUnless 

func (lom *LOM) UncacheUnless()

remove from cache unless dirty

func (*LOM) Unlock 

func (lom *LOM) Unlock(exclusive bool)

func (*LOM) UpgradeLock 

func (lom *LOM) UpgradeLock() (finished bool)

func (*LOM) ValidateColdGet added in v1.4.0 

func (lom *LOM) ValidateColdGet() bool

func (*LOM) ValidateContentChecksum 

func (lom *LOM) ValidateContentChecksum(locked bool) (err error)

ValidateDiskChecksum validates if checksum stored in lom's in-memory metadata matches object's content checksum. Use lom.ValidateMetaChecksum() to check lom's checksum vs on-disk metadata.

func (*LOM) ValidateMetaChecksum 

func (lom *LOM) ValidateMetaChecksum() error

ValidateMetaChecksum validates whether checksum stored in lom's in-memory metadata matches checksum stored on disk. Use lom.ValidateContentChecksum() to recompute and check object's content checksum.

func (*LOM) ValidateWarmGet 

func (lom *LOM) ValidateWarmGet() bool

func (*LOM) Version 

func (lom *LOM) Version(special ...bool) string

func (*LOM) VersionConf 

func (lom *LOM) VersionConf() cmn.VersionConf

func (*LOM) VersionPtr added in v1.3.24 

func (lom *LOM) VersionPtr() *string

func (*LOM) WritePolicy 

func (lom *LOM) WritePolicy() (p apc.WritePolicy)

type LomHandle added in v1.3.30 

type LomHandle struct {
	cos.LomReader
	// contains filtered or unexported fields
}

func (*LomHandle) Open added in v1.3.30 

func (lh *LomHandle) Open() (cos.ReadOpenCloser, error)

type LsoInvCtx added in v1.3.23 

type LsoInvCtx struct {
	Lmfh   cos.LomReader
	Lom    *LOM
	SGL    *memsys.SGL
	Name   string
	ID     string
	Schema []string
	Size   int64
	EOF    bool
}

type NLP 

type NLP interface {
	Lock()
	TryLock(timeout time.Duration) bool
	TryRLock(timeout time.Duration) bool
	Unlock()
}

pair

func NewNLP 

func NewNLP(name []byte) NLP

NOTE: currently, is only used to lock buckets

type Node 

type Node interface {
	SID() string
	String() string
	Snode() *meta.Snode

	Bowner() meta.Bowner
	Sowner() meta.Sowner

	ClusterStarted() bool
	NodeStarted() bool

	StatsUpdater() cos.StatsUpdater

	// Memory allocators
	PageMM() *memsys.MMSA
	ByteMM() *memsys.MMSA
}

cluster node

type Notif 

type Notif interface {
	OnFinishedCB() func(Notif, error, bool)
	OnProgressCB() func(Notif)
	NotifyInterval() time.Duration // notify interval in secs
	LastNotifTime() int64          // time last notified
	SetLastNotified(now int64)
	Upon(u Upon) bool
	Subscribers() []string
	ToNotifMsg(aborted bool) NotifMsg
}

intra-cluster notification interface

type NotifMsg 

type NotifMsg struct {
	UUID     string `json:"uuid"`    // xaction UUID
	NodeID   string `json:"node_id"` // notifier node ID
	Kind     string `json:"kind"`    // xaction `Kind`
	ErrMsg   string `json:"err"`     // error.Error()
	Data     []byte `json:"message"` // (e.g. usage: custom progress stats)
	AbortedX bool   `json:"aborted"` // true if aborted (see related: Snap.AbortedX)
}

intra-cluster notification message

func (*NotifMsg) String 

func (msg *NotifMsg) String() (s string)

type PromoteParams 

type PromoteParams struct {
	Bck             *meta.Bck   // destination bucket
	Cksum           *cos.Cksum  // checksum to validate
	Config          *cmn.Config // during xaction
	Xact            Xact        // responsible xaction
	apc.PromoteArgs             // all of the above
}

intra-cluster data path: control structures and types

type PutParams 

type PutParams struct {
	Reader    io.ReadCloser
	Cksum     *cos.Cksum // checksum to check
	Atime     time.Time
	Xact      Xact
	WorkTag   string // (=> work fqn)
	Size      int64
	ChunkSize int64 // if set, the object will be chunked with this size regardless of the bucket's chunk properties
	OWT       cmn.OWT
	SkipEC    bool // don't erasure-code when finalizing
	ColdGET   bool // this PUT is in fact a cold-GET
	Locked    bool // true if the LOM is already locked by the caller
}

intra-cluster data path: control structures and types

func AllocPutParams 

func AllocPutParams() (a *PutParams)

type PutWOC added in v1.3.30 

type PutWOC func(lom *LOM, latestVer, sync bool, woc io.WriteCloser, args *ETLArgs) (written int64, ecode int, err error)

type QuiCB 

type QuiCB func(elapsed time.Duration) QuiRes // see enum below

type QuiRes 

type QuiRes int

type ReadResp added in v1.3.28 

type ReadResp struct {
	R      cos.ReadOpenCloser // Reader for the object content, may be nil if already delivered
	OAH    cos.OAH
	Err    error
	Ecode  int
	Remote bool
}

ReadResp represents the status, attributes, and reader of an object from the source provider - Ecode 200 (http.StatusOK): the object is successfully retrieved; content available in `R`. - Ecode 204 (http.StatusNoContent): object has already been delivered directly to `daddr`, R is nil.

func GetDefaultROC added in v1.3.28 

func GetDefaultROC(lom *LOM, latestVer, sync bool, _ *ETLArgs) ReadResp

type Snap 

type Snap struct {
	// xaction-specific stats counters
	Ext any `json:"ext"`

	// common static props
	StartTime time.Time `json:"start-time"`
	EndTime   time.Time `json:"end-time"`
	Bck       cmn.Bck   `json:"bck"`
	SrcBck    cmn.Bck   `json:"src-bck"`
	DstBck    cmn.Bck   `json:"dst-bck"`
	ID        string    `json:"id"`
	Kind      string    `json:"kind"`
	CtlMsg    string    `json:"ctlmsg,omitempty"` // initiating control msg (a.k.a. "run options"; added v3.26)

	// extended error info
	AbortErr string `json:"abort-err"`
	Err      string `json:"err"`

	// packed field: number of workers, et al.
	Packed int64 `json:"glob.id,string"`

	// common runtime: stats counters (above) and state
	Stats    Stats `json:"stats"`
	AbortedX bool  `json:"aborted"`
	IdleX    bool  `json:"is_idle"`
}

func (*Snap) IsAborted 

func (xsnap *Snap) IsAborted() bool

func (*Snap) IsFinished added in v1.4.1 

func (xsnap *Snap) IsFinished() bool

func (*Snap) IsIdle 

func (xsnap *Snap) IsIdle() bool

func (*Snap) IsRunning added in v1.4.1 

func (xsnap *Snap) IsRunning() bool

func (*Snap) Pack added in v1.3.28 

func (xsnap *Snap) Pack(njoggers, nworkers int, chanFull int64)

func (*Snap) Started 

func (xsnap *Snap) Started() bool

func (*Snap) Unpack added in v1.3.28 

func (xsnap *Snap) Unpack() (njoggers, nworkers, chanFull int)

type Stats 

type Stats struct {
	Objs     int64 `json:"loc-objs,string"`  // locally processed
	Bytes    int64 `json:"loc-bytes,string"` //
	OutObjs  int64 `json:"out-objs,string"`  // transmit
	OutBytes int64 `json:"out-bytes,string"` //
	InObjs   int64 `json:"in-objs,string"`   // receive
	InBytes  int64 `json:"in-bytes,string"`
}

type Target 

type Target interface {
	TargetLoc

	// target <=> target & target => backend (no streams)
	DataClient() *http.Client

	// core object (+ PutObject above)
	FinalizeObj(lom *LOM, workFQN string, xctn Xact, owt cmn.OWT) (ecode int, err error)
	EvictObject(lom *LOM) (ecode int, err error)
	DeleteObject(lom *LOM, evict bool) (ecode int, err error)

	GetCold(ctx context.Context, lom *LOM, xkind string, owt cmn.OWT) (ecode int, err error)

	HeadCold(lom *LOM, origReq *http.Request) (objAttrs *cmn.ObjAttrs, ecode int, err error)

	Promote(params *PromoteParams) (ecode int, err error)
	HeadObjT2T(lom *LOM, si *meta.Snode) bool

	ECRestoreReq(ct *CT, si *meta.Snode, uuid string) error

	BMDVersionFixup(r *http.Request, bck ...cmn.Bck)

	GetFromNeighbor(params *GfnParams) (*http.Response, error)
}

all of the above; for implementations, see `ais/tgtimpl.go` and `ais/htrun.go` 

var (
	T Target
)

target only

type TargetLoc 

type TargetLoc interface {
	TargetPut

	fs.HC

	// backend
	Backend(*meta.Bck) Backend

	// Node health
	Health(si *meta.Snode, timeout time.Duration, query url.Values) (body []byte, ecode int, err error)
}

local target node

type TargetPut 

type TargetPut interface {
	Node

	// Space
	OOS(*fs.CapStatus, *cmn.Config, *fs.Tcdf) fs.CapStatus

	// xactions (jobs) now
	GetAllRunning(inout *AllRunningInOut, periodic bool)

	// PUT params.Reader => lom
	PutObject(lom *LOM, params *PutParams) (err error)

	// utilize blob downloader to cold-GET => (lom | custom write callback)
	GetColdBlob(params *BlobParams, oa *cmn.ObjAttrs) (xctn Xact, err error)
}

a node that can also write objects

type Uchunk added in v1.4.0 

type Uchunk struct {
	ETag string // S3/legacy
	MD5  []byte // ditto
	// contains filtered or unexported fields
}

func (*Uchunk) Num added in v1.4.0 

func (c *Uchunk) Num() uint16

func (*Uchunk) Path added in v1.4.0 

func (c *Uchunk) Path() string

func (*Uchunk) SetCksum added in v1.4.0 

func (c *Uchunk) SetCksum(cksum *cos.Cksum)

validate and set

func (*Uchunk) SetETag added in v1.4.0 

func (c *Uchunk) SetETag(etag string)

func (*Uchunk) Size added in v1.4.0 

func (c *Uchunk) Size() int64

readonly

type Ufest added in v1.4.0 

type Ufest struct {
	// contains filtered or unexported fields
}

func NewUfest added in v1.4.0 

func NewUfest(id string, lom *LOM, mustExist bool) (*Ufest, error)

func (*Ufest) Abort added in v1.4.0 

func (u *Ufest) Abort(lom *LOM)

func (*Ufest) Add added in v1.4.0 

func (u *Ufest) Add(c *Uchunk, size, num int64) error

func (*Ufest) Check added in v1.4.0 

func (u *Ufest) Check() error

func (*Ufest) Completed added in v1.4.0 

func (u *Ufest) Completed() bool

immutable once completed

func (*Ufest) ComputeWholeChecksum added in v1.4.0 

func (u *Ufest) ComputeWholeChecksum(cksumH *cos.CksumHash) error

reread all chunk payloads to compute a checksum of the given type TODO: avoid the extra pass by accumulating during AddPart/StorePartial or by caching a tree-hash see also: s3/mpt for ListParts

func (*Ufest) Count added in v1.4.0 

func (u *Ufest) Count() int

func (*Ufest) Created added in v1.4.0 

func (u *Ufest) Created() time.Time

func (*Ufest) ETagS3 added in v1.4.0 

func (u *Ufest) ETagS3() (string, error)

func (*Ufest) GetChunk added in v1.4.0 

func (u *Ufest) GetChunk(num int, locked bool) *Uchunk

func (*Ufest) ID added in v1.4.0 

func (u *Ufest) ID() string

func (*Ufest) LoadCompleted added in v1.4.0 

func (u *Ufest) LoadCompleted(lom *LOM) error

must be called under lom (r)lock

func (*Ufest) LoadPartial added in v1.4.0 

func (u *Ufest) LoadPartial(lom *LOM) error

func (*Ufest) Lock added in v1.4.0 

func (u *Ufest) Lock()

func (*Ufest) Lom added in v1.4.0 

func (u *Ufest) Lom() *LOM

func (*Ufest) NewChunk added in v1.4.0 

func (u *Ufest) NewChunk(num int, lom *LOM) (*Uchunk, error)

construct a new chunk and its pathname - `num` is not an index (must start from 1) - chunk #1 stays on the object's mountpath - other chunks get HRW distributed

func (*Ufest) NewReader added in v1.4.0 

func (u *Ufest) NewReader() (*UfestReader, error)

func (*Ufest) Size added in v1.4.0 

func (u *Ufest) Size() int64

func (*Ufest) StorePartial added in v1.4.0 

func (u *Ufest) StorePartial(lom *LOM, locked bool) error

func (*Ufest) Unlock added in v1.4.0 

func (u *Ufest) Unlock()

type UfestReader added in v1.4.0 

type UfestReader struct {
	// contains filtered or unexported fields
}

func (*UfestReader) Close added in v1.4.0 

func (r *UfestReader) Close() error

func (*UfestReader) Read added in v1.4.0 

func (r *UfestReader) Read(p []byte) (n int, err error)

func (*UfestReader) ReadAt added in v1.4.0 

func (r *UfestReader) ReadAt(p []byte, off int64) (n int, err error)

consistent with io.ReaderAt semantics: - do not use or modify reader's offset (r.goff) or any other state - open the needed chunk(s) transiently and close them immediately - return io.EOF when less than len(p)

type Upon 

type Upon int

type Xact 

type Xact interface {
	Run(*sync.WaitGroup)
	ID() string
	Kind() string
	Bck() *meta.Bck
	FromTo() (*meta.Bck, *meta.Bck)
	StartTime() time.Time
	EndTime() time.Time
	IsDone() bool
	IsRunning() bool
	IsIdle() bool
	Quiesce(time.Duration, QuiCB) QuiRes

	// abrt
	IsAborted() bool
	AbortErr() error
	AbortedAfter(time.Duration) error
	ChanAbort() <-chan error
	// err (info)
	AddErr(error, ...int)

	// to support api.QueryXactionSnaps
	CtlMsg() string
	Snap() *Snap // (struct below)

	// reporting: log, err
	String() string
	Name() string
	Cname() string

	// modifiers
	Finish()
	Abort(error) bool
	AddNotif(n Notif)

	// common stats
	GetStats
}

Source Files

View all Source files

Directories

Path Synopsis
Package meta: cluster-level metadata
 Package meta: cluster-level metadata
Package mock provides a variety of mock implementations used for testing.
 Package mock provides a variety of mock implementations used for testing.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.