core

package
v1.4.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 15, 2026 License: MIT Imports: 39 Imported by: 2

Documentation

Overview

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.

Package core provides core metadata and in-cluster API

  • Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source 
const (
	RemoteDeletedDelCount = "remote.deleted.del.n"

	// lcache stats
	LcacheCollisionCount = "lcache.collision.n"
	LcacheEvictedCount   = "lcache.evicted.n"
	LcacheErrCount       = "err.lcache.n" // errPrefix + "lcache.n"
	LcacheFlushColdCount = "lcache.flush.cold.n"
)

core stats

View Source 
const (
	// backward compat v3.31 and prior
	MetaverLOM_V1 = 1 //nolint:revive // readability

	// current
	MetaverLOM = 2
)

LOM

View Source 
const (
	Targets = iota // 0 (core.Targets) used as default value for NewStreamBundle
	Proxies
	AllNodes
	SelectedNodes
)
View Source 
const (
	UponTerm     = Upon(1 << iota) // success or fail is separately provided via error
	UponProgress                   // periodic (BytesCount, ObjCount)
)

enum: when to notify

View Source 
const (
	QuiInactiveCB     = QuiRes(iota) // e.g., no pending requests (NOTE: used exclusively by `quicb` callbacks)
	QuiActive                        // active (e.g., receiving data)
	QuiActiveRet                     // active that immediately breaks waiting for quiecscence
	QuiActiveDontBump                // active that does not increase inactivity duration (ie., keeps initial setting)
	QuiDone                          // all done
	QuiAborted                       // aborted
	QuiTimeout                       // timeout
	Quiescent                        // idle => quiescent
)
View Source 
const (
	AisBID = cos.MSB64
)
View Source 
const (
	DumpLomEnvVar = "AIS_DUMP_LOM"
)

cmd/xmeta support

View Source 
const (

	// MaxChunkCount is the maximum number of chunks allowed per object (limited by uint16 chunk numbering)
	MaxChunkCount = 9999
)

Variables

This section is empty.

Functions

func DrainLIF added in v1.3.30 

func DrainLIF(workCh chan LIF) (n int)

non-blocking drain LIF workCh

func FreeLOM 

func FreeLOM(lom *LOM)

func FreePutParams 

func FreePutParams(a *PutParams)

func HrwFQN 

func HrwFQN(bck *cmn.Bck, contentType, objName string) (fqn string, digest uint64, err error)

func InMaintOrDecomm 

func InMaintOrDecomm(smap *meta.Smap, tsi *meta.Snode, xact Xact) error

func LcacheClear added in v1.3.28 

func LcacheClear()

func LcacheClearBcks added in v1.3.28 

func LcacheClearBcks(wg *sync.WaitGroup, bcks ...*meta.Bck) bool

func LcacheClearMpath added in v1.3.28 

func LcacheClearMpath(mi *fs.Mountpath)

func NewBID added in v1.3.26 

func NewBID(serial uint64, isAis bool) uint64

func ParseObjLoc 

func ParseObjLoc(loc string) (tname, mpname string)

func Pinit 

func Pinit()

func ResolveFQN 

func ResolveFQN(fqn string, parsed *fs.ParsedFQN) (hrwFQN string, err error)

func Term 

func Term()

func Tinit 

func Tinit(t Target, config *cmn.Config, runHK bool)

Types

type AllRunningInOut 

type AllRunningInOut struct {
	Kind    string
	Running []string
	Idle    []string // NOTE: returning only when not nil
}

type Backend added in v1.3.23 

type Backend interface {
	Provider() string
	MetricName(string) string

	CreateBucket(bck *meta.Bck) (ecode int, err error)
	ListBuckets(qbck cmn.QueryBcks) (bcks cmn.Bcks, ecode int, err error)

	// list-objects
	ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes) (ecode int, err error)
	ListObjectsInv(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.LsoRes, ctx *LsoInvCtx) error

	PutObj(ctx context.Context, r io.ReadCloser, lom *LOM, origReq *http.Request) (ecode int, err error)
	DeleteObj(ctx context.Context, lom *LOM) (ecode int, err error)

	// head
	HeadBucket(ctx context.Context, bck *meta.Bck) (bckProps cos.StrKVs, ecode int, err error)
	HeadObj(ctx context.Context, lom *LOM, origReq *http.Request) (objAttrs *cmn.ObjAttrs, ecode int, err error)

	// get (exclusively via GetCold; calls GetObjReader)
	GetObj(ctx context.Context, lom *LOM, owt cmn.OWT, origReq *http.Request) (ecode int, err error)
	// get (jobs; REST)
	GetObjReader(ctx context.Context, lom *LOM, offset, length int64) GetReaderResult

	// bucket inventory
	GetBucketInv(bck *meta.Bck, ctx *LsoInvCtx) (ecode int, err error)

	// multipart upload
	StartMpt(lom *LOM, r *http.Request) (uploadID string, ecode int, err error)
	PutMptPart(lom *LOM, reader cos.ReadOpenCloser, r *http.Request, uploadID string, size int64, partNum int32) (etag string, ecode int, err error)
	CompleteMpt(lom *LOM, r *http.Request, uploadID string, body []byte, parts apc.MptCompletedParts) (version, etag string, ecode int, err error)
	AbortMpt(lom *LOM, r *http.Request, uploadID string) (ecode int, err error)
}

type BlobParams added in v1.3.23 

type BlobParams struct {
	Lom *LOM
	Msg *apc.BlobMsg

	// When `RespWriter` is set, `XactBlobDl` not only downloads chunks into the cluster,
	// but also stitches them together and sequentially writes to `RespWriter`.
	// This makes the blob downloading job synchronous and blocking until all chunks are written.
	// Only set this if you need to simultaneously download and write to the response writer (e.g., for streaming blob GET).
	RespWriter io.Writer
}

intra-cluster data path: control structures and types

type CRMD 

type CRMD struct {
	Err      error
	ObjAttrs *cmn.ObjAttrs
	ErrCode  int
	Eq       bool
}

returned by lom.CheckRemoteMD

type CT 

type CT struct {
	// contains filtered or unexported fields
}

func NewCTFromBO 

func NewCTFromBO(bck *meta.Bck, objName, ctType string, extras ...string) (ct *CT, err error)

construct, init bucket and compute HRW (similar to lom.InitBck())

func NewCTFromFQN 

func NewCTFromFQN(fqn string, b meta.Bowner) (ct *CT, err error)

full construction and init, including: - parse fqn - init bucket - HRW

func NewCTFromLOM 

func NewCTFromLOM(lom *LOM, ctType string, extras ...string) (ct *CT)

bare minimum: from LOM

func NewCTFromParsed added in v1.4.0 

func NewCTFromParsed(parsed *fs.ParsedFQN, fqn string) *CT

bare minimum: from fs.ParsedFQN

func NewDsortCT added in v1.4.0 

func NewDsortCT(bck *cmn.Bck, objName string) (ct *CT, err error)

usage: dsort only

func (*CT) Bck 

func (ct *CT) Bck() *meta.Bck

func (*CT) Bucket 

func (ct *CT) Bucket() *cmn.Bck

func (*CT) Clone 

func (ct *CT) Clone(ctType string) (clone *CT)

bare minimum: from CT

func (*CT) Cname added in v1.3.24 

func (ct *CT) Cname() string

func (*CT) ContentType 

func (ct *CT) ContentType() string

func (*CT) Digest 

func (ct *CT) Digest() uint64

func (*CT) FQN 

func (ct *CT) FQN() string

func (*CT) GenFQN added in v1.4.0 

func (ct *CT) GenFQN(cttype string, extras ...string) string

func (*CT) LoadSliceFromFS added in v1.3.24 

func (ct *CT) LoadSliceFromFS() error

func (*CT) Lock 

func (ct *CT) Lock(exclusive bool)

func (*CT) Lsize added in v1.3.24 

func (ct *CT) Lsize() int64

func (*CT) Mountpath 

func (ct *CT) Mountpath() *fs.Mountpath

func (*CT) MtimeUnix 

func (ct *CT) MtimeUnix() int64

func (*CT) ObjectName 

func (ct *CT) ObjectName() string

func (*CT) UnamePtr added in v1.3.24 

func (ct *CT) UnamePtr() *string

func (*CT) Unlock 

func (ct *CT) Unlock(exclusive bool)

func (*CT) Write 

func (ct *CT) Write(reader io.Reader, size int64, workFQN string) (err error)

Save CT to local drives. If workFQN is set, it saves in two steps: first, save to workFQN; second, rename workFQN to ct.fqn. If unset, it writes directly to ct.fqn

type ETLArgs added in v1.3.31 

type ETLArgs struct {
	TransformArgs string          // optional and ETL-specific; can be used to indicate transformation on a per-object basis
	Pipeline      apc.ETLPipeline // intermediate ETL pod's address or destination target's address for direct put
}

GetROC defines a function that retrieves an object based on the given `lom` and flags. If `Pipeline` are provided, the implementation may choose to deliver the object directly, in which case the returned ReadResp will have R = nil and Ecode = 204.

Implementations include: - `core.DefaultGetROC`: fetches from local or remote backend - `etl.HTTPCommunicator.OfflineTransform`: fetches transformed object from ETL pod through HTTP response - `etl.statefulCommnicator.Transform`: fetches transformed object from ETL pod through WebSocket message

type GetROC added in v1.3.28 

type GetROC func(lom *LOM, latestVer, sync bool, args *ETLArgs) ReadResp

type GetReaderResult 

type GetReaderResult struct {
	R        io.ReadCloser
	Err      error
	ExpCksum *cos.Cksum
	Size     int64
	ErrCode  int
}

type GetStats added in v1.4.1 

type GetStats interface {
	Objs() int64
	ObjsAdd(int, int64)    // locally processed
	OutObjsAdd(int, int64) // transmit
	InObjsAdd(int, int64)  // receive
	InBytes() int64
	OutBytes() int64
}

type GfnParams added in v1.4.0 

type GfnParams struct {
	Lom      *LOM
	Tsi      *meta.Snode
	Config   *cmn.Config
	ArchPath string
	Size     int64
}

intra-cluster data path: control structures and types

type LIF 

type LIF struct {
	// contains filtered or unexported fields
}

LOM In Flight (LIF)

func (*LIF) CacheIdx 

func (lif *LIF) CacheIdx() int

func (*LIF) LOM 

func (lif *LIF) LOM() (lom *LOM, err error)

LIF => LOM with a check for bucket existence

func (*LIF) Name added in v1.3.28 

func (lif *LIF) Name() string

func (*LIF) Unlock 

func (lif *LIF) Unlock(exclusive bool)

type LOM 

type LOM struct {
	ObjName string
	FQN     string
	HrwFQN  *string // (=> main replica)
	// contains filtered or unexported fields
}

func AllocLOM 

func AllocLOM(objName string) *LOM

func (*LOM) AddCopy 

func (lom *LOM) AddCopy(copyFQN string, mpi *fs.Mountpath) error

NOTE: used only in tests

func (*LOM) AppendWork added in v1.3.24 

func (*LOM) AppendWork(wfqn string) (fh cos.LomWriter, err error)

append

func (*LOM) Atime 

func (lom *LOM) Atime() time.Time

func (*LOM) AtimeUnix 

func (lom *LOM) AtimeUnix() int64

func (*LOM) Bck 

func (lom *LOM) Bck() *meta.Bck

assorted _convenient_ accessors

func (*LOM) Bprops 

func (lom *LOM) Bprops() *cmn.Bprops

func (*LOM) Bucket 

func (lom *LOM) Bucket() *cmn.Bck

func (*LOM) CacheIdx 

func (lom *LOM) CacheIdx() int

func (*LOM) CheckEq added in v1.3.24 

func (lom *LOM) CheckEq(rem cos.OAH) error

LOM == remote-object equality check

func (*LOM) CheckRemoteMD 

func (lom *LOM) CheckRemoteMD(locked, sync bool, origReq *http.Request) (res CRMD)

NOTE: - [PRECONDITION]: `versioning.validate_warm_get` || QparamLatestVer - [Sync] when Sync option is used (via bucket config and/or `sync` argument) caller MUST take wlock or rlock - [MAY] delete remotely-deleted (non-existing) object and increment associated stats counter

also returns `NotFound` after removing local replica - the Sync option

func (*LOM) Checksum 

func (lom *LOM) Checksum() *cos.Cksum

func (*LOM) CksumConf 

func (lom *LOM) CksumConf() *cmn.CksumConf

func (*LOM) CksumType 

func (lom *LOM) CksumType() string

more cksum conf

func (*LOM) CleanupCopies added in v1.4.2 

func (lom *LOM) CleanupCopies(avail fs.MPI) (exp, got int)

remove stale copy metadata entries (copies on unavailable or disabled mountpaths) and return the expected number of copies per config and the actual number of valid copies remaining; must be called under w-lock (usage: resilvering)

func (*LOM) Clone added in v1.3.30 

func (lom *LOM) Clone() *LOM

func (*LOM) CloneTo added in v1.3.30 

func (lom *LOM) CloneTo(fqn string) *LOM

allocates and copies metadata (in particular, atime and uname) NOTE: cloned lom.bid() == 0 is possible - copying/transforming scenarios

func (*LOM) Cname 

func (lom *LOM) Cname() string

see also: transport.ObjHdr.Cname()

func (*LOM) CompleteUfest added in v1.4.0 

func (lom *LOM) CompleteUfest(u *Ufest, locked bool) (err error)

func (*LOM) ComputeCksum 

func (lom *LOM) ComputeCksum(cksumType string, locked bool) (cksum *cos.CksumHash, _ error)

func (*LOM) ComputeSetCksum 

func (lom *LOM) ComputeSetCksum(locked bool) (*cos.Cksum, error)

func (*LOM) Copy 

func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) error

increment the object's num copies by (well) copying the former (compare with lom.Copy2FQN below) TODO: add T.FSHC() calls _after_ having disambiguated source vs destination (to blame)

func (*LOM) Copy2FQN 

func (lom *LOM) Copy2FQN(dstFQN string, buf []byte) (dst *LOM, err error)

copy object => any local destination usage: copying between different buckets (compare with lom.Copy() above) compare w/ Ufest.Relocate(lom)

func (*LOM) CopyAttrs 

func (lom *LOM) CopyAttrs(oah cos.OAH, skipCksum bool)

func (*LOM) CopyVersion added in v1.3.24 

func (lom *LOM) CopyVersion(oah cos.OAH)

func (*LOM) Create added in v1.3.24 

func (lom *LOM) Create() (cos.LomWriter, error)

func (*LOM) CreatePart added in v1.3.24 

func (lom *LOM) CreatePart(wfqn string) (*os.File, error)

func (*LOM) CreateSlice added in v1.3.24 

func (lom *LOM) CreateSlice(wfqn string) (*os.File, error)

func (*LOM) CreateWork added in v1.3.24 

func (lom *LOM) CreateWork(wfqn string) (cos.LomWriter, error)

func (*LOM) DelAllCopies 

func (lom *LOM) DelAllCopies() (err error)

func (*LOM) DelCopies 

func (lom *LOM) DelCopies(copiesFQN ...string) (err error)

func (*LOM) DelCustomKey added in v1.4.0 

func (lom *LOM) DelCustomKey(key string)

func (*LOM) DelExtraCopies 

func (lom *LOM) DelExtraCopies(fqn ...string) (removed bool, err error)

DelExtraCopies deletes obj replicas that are not part of the lom.md.copies metadata (cleanup)

func (*LOM) Digest 

func (lom *LOM) Digest() uint64

func (*LOM) DowngradeLock 

func (lom *LOM) DowngradeLock()

func (*LOM) ECEnabled 

func (lom *LOM) ECEnabled() bool

func (*LOM) ETag added in v1.4.2 

func (lom *LOM) ETag(mtime time.Time, allowSyscall bool) string

allowGen=false: return only existing (custom or MD5) values allowGen=true: as the name implies

func (*LOM) EqCksum 

func (lom *LOM) EqCksum(cksum *cos.Cksum) bool

func (*LOM) FromFS 

func (lom *LOM) FromFS() error

func (*LOM) Fstat added in v1.3.24 

func (lom *LOM) Fstat(getAtime bool) (size, atimefs int64, mtime time.Time, _ error)

low-level access to the os.FileInfo of a chunk or whole file

func (*LOM) GenFQN added in v1.4.0 

func (lom *LOM) GenFQN(cttype string, extras ...string) string

func (*LOM) GetCopies 

func (lom *LOM) GetCopies() fs.MPI

GetCopies returns all copies - copies include lom.FQN aka "main repl." - caller must take a lock

func (*LOM) GetCustomKey 

func (lom *LOM) GetCustomKey(key string) (string, bool)

func (*LOM) GetCustomMD 

func (lom *LOM) GetCustomMD() cos.StrKVs

custom metadata

func (*LOM) GetROC added in v1.3.28 

func (lom *LOM) GetROC(latestVer, sync bool) (resp ReadResp)

func (*LOM) GetXattr added in v1.3.30 

func (lom *LOM) GetXattr(buf []byte) ([]byte, error)

func (*LOM) GetXattrN added in v1.3.30 

func (lom *LOM) GetXattrN(name string) ([]byte, error)

func (*LOM) HasCopies 

func (lom *LOM) HasCopies() bool

func (*LOM) HrwTarget 

func (lom *LOM) HrwTarget(smap *meta.Smap) (tsi *meta.Snode, local bool, err error)

func (*LOM) IncVersion 

func (lom *LOM) IncVersion() error

func (*LOM) InitBck 

func (lom *LOM) InitBck(bck *meta.Bck) (err error)

func (*LOM) InitCT 

func (lom *LOM) InitCT(ct *CT)

func (*LOM) InitCmnBck added in v1.4.2 

func (lom *LOM) InitCmnBck(bck *cmn.Bck) (err error)

func (*LOM) InitFQN 

func (lom *LOM) InitFQN(fqn string, expbck *cmn.Bck) error

func (*LOM) IsChunked added in v1.3.24 

func (lom *LOM) IsChunked(special ...bool) bool

func (*LOM) IsCopy 

func (lom *LOM) IsCopy() bool

given an existing (on-disk) object, determines whether it is a _copy_ (compare with isMirror below)

func (*LOM) IsFeatureSet added in v1.3.23 

func (lom *LOM) IsFeatureSet(f feat.Flags) bool

func (*LOM) IsFntl added in v1.3.26 

func (lom *LOM) IsFntl() bool

func (*LOM) IsHRW 

func (lom *LOM) IsHRW() bool

func (*LOM) IsLocked 

func (lom *LOM) IsLocked() int

returns {apc.LockNone, ...} enum

func (*LOM) IsPrimaryCopy added in v1.4.2 

func (lom *LOM) IsPrimaryCopy(avail fs.MPI, hmi *fs.Mountpath, sentinel string) (isPrimary, mainExists bool)

deterministically pick a copy to restore any missing ones must be called under w-lock (usage: resilvering)

func (*LOM) LIF 

func (lom *LOM) LIF() (lif LIF)

LOM => LIF constructor

func (*LOM) LastModified added in v1.4.2 

func (lom *LOM) LastModified() (time.Time, error)

func (*LOM) LastModifiedLso added in v1.4.2 

func (lom *LOM) LastModifiedLso() (string, time.Time)

NOTE:

  • (list-objects, S3): prefer stored RFC3339 (cmn.LsoLastModified);
  • else convert Last-Modified (http.TimeFormat) to RFC3339;
  • else (perf) fall back to atime when available
  • clients requiring strict S3 semantics with ais:// buckets must use HEAD(object) (which is allowed to execute syscalls)

func (*LOM) LastModifiedStr added in v1.4.2 

func (lom *LOM) LastModifiedStr() (string, time.Time)

(same as above when callers want a string)

func (*LOM) LeastUtilNoCopy 

func (lom *LOM) LeastUtilNoCopy() (mi *fs.Mountpath)

returns the least-utilized mountpath that does _not_ have a copy of this `lom` yet (compare with leastUtilCopy())

func (*LOM) Load 

func (lom *LOM) Load(cacheit, locked bool) error

no lock is taken when locked by an immediate caller, or otherwise is known to be locked otherwise, try Rlock temporarily _if and only when_ reading from fs

(compare w/ LoadUnsafe() below)

func (*LOM) LoadLatest 

func (lom *LOM) LoadLatest(latest bool) (oa *cmn.ObjAttrs, deleted bool, err error)

NOTE: Sync is false (ie., not deleting)

func (*LOM) LoadMetaFromFS 

func (lom *LOM) LoadMetaFromFS() error

NOTE usage: tests and `xmeta` only; ignores `dirty`

func (*LOM) Location 

func (lom *LOM) Location() string

to report via list-objects and HEAD()

func (*LOM) Lock 

func (lom *LOM) Lock(exclusive bool)

func (*LOM) Lsize added in v1.3.24 

func (lom *LOM) Lsize(special ...bool) int64

special a) when a new version is being created b) for usage in unit tests

func (*LOM) MirrorConf 

func (lom *LOM) MirrorConf() *cmn.MirrorConf

func (*LOM) MirrorPaths added in v1.3.30 

func (lom *LOM) MirrorPaths() []string

func (*LOM) Mountpath 

func (lom *LOM) Mountpath() *fs.Mountpath

func (*LOM) NewArchpathReader added in v1.3.30 

func (lom *LOM) NewArchpathReader(lh cos.LomReader, archpath, mime string) (csl cos.ReadCloseSizer, err error)

extract a single file from a (.tar, .tgz or .tar.gz, .zip, .tar.lz4) shard uses the provided `mime` or lom.ObjName to detect formatting (empty = auto-detect)

func (*LOM) NewDeferROC 

func (lom *LOM) NewDeferROC(loaded bool) (cos.ReadOpenCloser, error)

is called under rlock; unlocks on fail NOTE: compare w/ lom.Open() returning cos.LomReader

func (*LOM) NewHandle added in v1.3.30 

func (lom *LOM) NewHandle(loaded bool) (*LomHandle, error)

func (*LOM) NewUfestReader added in v1.4.0 

func (lom *LOM) NewUfestReader() (cos.LomReader, error)

func (*LOM) NumCopies 

func (lom *LOM) NumCopies() int

func (*LOM) ObjAttrs 

func (lom *LOM) ObjAttrs() *cmn.ObjAttrs

func (*LOM) ObjectName 

func (lom *LOM) ObjectName() string

as fs.PartsFQN

func (*LOM) Open added in v1.3.24 

func (lom *LOM) Open() (lh cos.LomReader, err error)

open read-only, return a reader see also: lom.NewDeferROC() see also: lom.GetROC()

func (*LOM) OpenCopy added in v1.4.0 

func (lom *LOM) OpenCopy() (cos.LomReader, string)

load-balanced GET from replicated lom - picks least-utilized mountpath - returns (open reader + its FQN) or (nil, "")

func (*LOM) OrigFntl added in v1.3.26 

func (lom *LOM) OrigFntl() []string

func (*LOM) Persist 

func (lom *LOM) Persist() error

(caller must set atime; compare with the above)

func (*LOM) PersistMain 

func (lom *LOM) PersistMain(isChunked bool) error

func (*LOM) PopFntl added in v1.3.26 

func (lom *LOM) PopFntl(saved []string)

func (*LOM) PostInit added in v1.3.23 

func (lom *LOM) PostInit() error

func (*LOM) PreInit added in v1.3.23 

func (lom *LOM) PreInit(fqn string) error

NOTE: to facilitate fast path filtering-out

func (*LOM) PushFntl added in v1.3.26 

func (lom *LOM) PushFntl(short []string) (saved []string)

func (*LOM) Recache 

func (lom *LOM) Recache()

store new or refresh existing

func (*LOM) RemoveMain added in v1.3.24 

func (lom *LOM) RemoveMain() error

func (*LOM) RemoveObj added in v1.3.24 

func (lom *LOM) RemoveObj(force ...bool) (err error)

func (*LOM) RenameFinalize added in v1.3.24 

func (lom *LOM) RenameFinalize(wfqn string) error

func (*LOM) RenameMainTo added in v1.3.24 

func (lom *LOM) RenameMainTo(wfqn string) error

func (*LOM) RenameToMain added in v1.3.24 

func (lom *LOM) RenameToMain(wfqn string) error

func (*LOM) RestoreToLocation 

func (lom *LOM) RestoreToLocation() (exists bool)

RestoreObjectFromAny tries to restore the object at its default location. Returns true if object exists, false otherwise TODO: locking vs concurrent restore: consider (read-lock object + write-lock meta) split

func (*LOM) SetAtimeUnix 

func (lom *LOM) SetAtimeUnix(tu int64)

func (*LOM) SetCksum 

func (lom *LOM) SetCksum(cksum *cos.Cksum)

func (*LOM) SetCustomKey 

func (lom *LOM) SetCustomKey(key, value string)

func (*LOM) SetCustomMD 

func (lom *LOM) SetCustomMD(md cos.StrKVs)

func (*LOM) SetSize 

func (lom *LOM) SetSize(size int64)

func (*LOM) SetVersion 

func (lom *LOM) SetVersion(ver string)

func (*LOM) SetXattr added in v1.3.30 

func (lom *LOM) SetXattr(data []byte) error

func (*LOM) SetXattrN added in v1.3.30 

func (lom *LOM) SetXattrN(name string, data []byte) error

func (*LOM) ShortenFntl added in v1.3.26 

func (lom *LOM) ShortenFntl() []string

(compare with fs/content Gen())

func (*LOM) String 

func (lom *LOM) String() string

func (*LOM) TestAtime added in v1.3.26 

func (lom *LOM) TestAtime() error

usage: unit tests only

func (*LOM) ToMpath 

func (lom *LOM) ToMpath(avail fs.MPI) (*fs.Mountpath, bool)

must be called under w-lock returns mountpath to relocate or copy this lom, or nil if none required/available return fixHrw = true when lom is currently misplaced - checks hrw location first, and - checks copies (if any) against the current configuration and available mountpaths; - does not check `fstat` in either case (usage: resilvering)

func (*LOM) TryLock 

func (lom *LOM) TryLock(exclusive bool) bool

func (*LOM) Uname 

func (lom *LOM) Uname() string

func (*LOM) UnamePtr added in v1.3.24 

func (lom *LOM) UnamePtr() *string

func (*LOM) Uncache 

func (lom *LOM) Uncache()

func (*LOM) UncacheDel added in v1.3.28 

func (lom *LOM) UncacheDel()

upon: error; RemoveObj

func (*LOM) UncacheUnless 

func (lom *LOM) UncacheUnless()

remove from cache unless dirty

func (*LOM) Unlock 

func (lom *LOM) Unlock(exclusive bool)

func (*LOM) UpgradeLock 

func (lom *LOM) UpgradeLock() (finished bool)

func (*LOM) ValidateColdGet added in v1.4.0 

func (lom *LOM) ValidateColdGet() bool

func (*LOM) ValidateContentChecksum 

func (lom *LOM) ValidateContentChecksum(locked bool) (err error)

ValidateDiskChecksum validates if checksum stored in lom's in-memory metadata matches object's content checksum. Use lom.ValidateMetaChecksum() to check lom's checksum vs on-disk metadata.

func (*LOM) ValidateMetaChecksum 

func (lom *LOM) ValidateMetaChecksum() error

ValidateMetaChecksum validates whether checksum stored in lom's in-memory metadata matches checksum stored on disk. Use lom.ValidateContentChecksum() to recompute and check object's content checksum.

func (*LOM) ValidateWarmGet 

func (lom *LOM) ValidateWarmGet() bool

func (*LOM) Version 

func (lom *LOM) Version(special ...bool) string

func (*LOM) VersionConf 

func (lom *LOM) VersionConf() cmn.VersionConf

func (*LOM) VersionPtr added in v1.3.24 

func (lom *LOM) VersionPtr() *string

func (*LOM) WritePolicy 

func (lom *LOM) WritePolicy() (p apc.WritePolicy)

type LomHandle added in v1.3.30 

type LomHandle struct {
	cos.LomReader
	// contains filtered or unexported fields
}

func (*LomHandle) Open added in v1.3.30 

func (lh *LomHandle) Open() (cos.ReadOpenCloser, error)

type LsoInvCtx added in v1.3.23 

type LsoInvCtx struct {
	Lmfh   cos.LomReader
	Lom    *LOM
	SGL    *memsys.SGL
	Name   string
	ID     string
	Schema []string
	Size   int64
	EOF    bool
}

type NLP 

type NLP interface {
	Lock()
	TryLock(timeout time.Duration) bool
	TryRLock(timeout time.Duration) bool
	Unlock()
}

pair

func NewNLP 

func NewNLP(name []byte) NLP

NOTE: currently, is only used to lock buckets

type Node 

type Node interface {
	SID() string
	String() string
	Snode() *meta.Snode

	Bowner() meta.Bowner
	Sowner() meta.Sowner

	ClusterStarted() bool
	NodeStarted() bool

	StatsUpdater() cos.StatsUpdater

	// Memory allocators
	PageMM() *memsys.MMSA
	ByteMM() *memsys.MMSA
}

cluster node

type Notif 

type Notif interface {
	OnFinishedCB() func(Notif, error, bool)
	OnProgressCB() func(Notif)
	NotifyInterval() time.Duration // notify interval in secs
	LastNotifTime() int64          // time last notified
	SetLastNotified(now int64)
	Upon(u Upon) bool
	Subscribers() []string
	ToNotifMsg(aborted bool) NotifMsg
}

intra-cluster notification interface

type NotifMsg 

type NotifMsg struct {
	UUID     string `json:"uuid"`    // xaction UUID
	NodeID   string `json:"node_id"` // notifier node ID
	Kind     string `json:"kind"`    // xaction `Kind`
	ErrMsg   string `json:"err"`     // error.Error()
	Data     []byte `json:"message"` // (e.g. usage: custom progress stats)
	AbortedX bool   `json:"aborted"` // true if aborted (see related: Snap.AbortedX)
}

intra-cluster notification message

func (*NotifMsg) String 

func (msg *NotifMsg) String() (s string)

type PromoteParams 

type PromoteParams struct {
	Bck             *meta.Bck   // destination bucket
	Cksum           *cos.Cksum  // checksum to validate
	Config          *cmn.Config // during xaction
	Xact            Xact        // responsible xaction
	apc.PromoteArgs             // all of the above
}

intra-cluster data path: control structures and types

type PutParams 

type PutParams struct {
	Reader    io.ReadCloser
	Cksum     *cos.Cksum // checksum to check
	Atime     time.Time
	Xact      Xact
	WorkTag   string // (=> work fqn)
	Size      int64
	ChunkSize int64 // if set, the object will be chunked with this size regardless of the bucket's chunk properties
	OWT       cmn.OWT
	SkipEC    bool // don't erasure-code when finalizing
	ColdGET   bool // this PUT is in fact a cold-GET
	Locked    bool // true if the LOM is already locked by the caller
}

intra-cluster data path: control structures and types

func AllocPutParams 

func AllocPutParams() (a *PutParams)

type PutWOC added in v1.3.30 

type PutWOC func(lom *LOM, latestVer, sync bool, woc io.WriteCloser, args *ETLArgs) (written int64, ecode int, err error)

type QuiCB 

type QuiCB func(elapsed time.Duration) QuiRes // see enum below

type QuiRes 

type QuiRes int

type ReadResp added in v1.3.28 

type ReadResp struct {
	R      cos.ReadOpenCloser // Reader for the object content, may be nil if already delivered
	OAH    cos.OAH
	Err    error
	Ecode  int
	Remote bool
}

ReadResp represents the status, attributes, and reader of an object from the source provider - Ecode 200 (http.StatusOK): the object is successfully retrieved; content available in `R`. - Ecode 204 (http.StatusNoContent): object has already been delivered directly to `daddr`, R is nil.

func GetDefaultROC added in v1.3.28 

func GetDefaultROC(lom *LOM, latestVer, sync bool, _ *ETLArgs) ReadResp

type Snap 

type Snap struct {
	// xaction-specific stats counters
	Ext any `json:"ext"`

	// common static props
	StartTime time.Time `json:"start-time"`
	EndTime   time.Time `json:"end-time"`
	Bck       cmn.Bck   `json:"bck"`
	SrcBck    cmn.Bck   `json:"src-bck"`
	DstBck    cmn.Bck   `json:"dst-bck"`
	ID        string    `json:"id"`
	Kind      string    `json:"kind"`
	CtlMsg    string    `json:"ctlmsg,omitempty"` // initiating control msg (a.k.a. "run options"; added v3.26)

	// extended error info
	AbortErr string `json:"abort-err"`
	Err      string `json:"err"`

	// packed field: number of workers, et al.
	Packed int64 `json:"glob.id,string"`

	// common runtime: stats counters (above) and state
	Stats    Stats `json:"stats"`
	AbortedX bool  `json:"aborted"`
	IdleX    bool  `json:"is_idle"`
}

func (*Snap) IsAborted 

func (xsnap *Snap) IsAborted() bool

func (*Snap) IsFinished added in v1.4.1 

func (xsnap *Snap) IsFinished() bool

func (*Snap) IsIdle 

func (xsnap *Snap) IsIdle() bool

func (*Snap) IsRunning added in v1.4.1 

func (xsnap *Snap) IsRunning() bool

func (*Snap) Pack added in v1.3.28 

func (xsnap *Snap) Pack(njoggers, nworkers int, chanFull int64)

func (*Snap) Started 

func (xsnap *Snap) Started() bool

func (*Snap) Unpack added in v1.3.28 

func (xsnap *Snap) Unpack() (njoggers, nworkers, chanFull int)

type Stats 

type Stats struct {
	Objs     int64 `json:"loc-objs,string"`  // locally processed
	Bytes    int64 `json:"loc-bytes,string"` //
	OutObjs  int64 `json:"out-objs,string"`  // transmit
	OutBytes int64 `json:"out-bytes,string"` //
	InObjs   int64 `json:"in-objs,string"`   // receive
	InBytes  int64 `json:"in-bytes,string"`
}

type Target 

type Target interface {
	TargetLoc

	// target <=> target & target => backend (no streams)
	DataClient() *http.Client

	// core object (+ PutObject above)
	FinalizeObj(lom *LOM, workFQN string, xctn Xact, owt cmn.OWT) (ecode int, err error)
	EvictObject(lom *LOM) (ecode int, err error)
	DeleteObject(lom *LOM, evict bool) (ecode int, err error)

	GetCold(ctx context.Context, lom *LOM, xkind string, owt cmn.OWT) (ecode int, err error)

	HeadCold(lom *LOM, origReq *http.Request) (objAttrs *cmn.ObjAttrs, ecode int, err error)

	Promote(params *PromoteParams) (ecode int, err error)
	HeadObjT2T(lom *LOM, si *meta.Snode) bool

	ECRestoreReq(ct *CT, si *meta.Snode, uuid string) error

	BMDVersionFixup(r *http.Request, bck ...cmn.Bck)

	GetFromNeighbor(params *GfnParams) (*http.Response, error)
}

all of the above; for implementations, see `ais/tgtimpl.go` and `ais/htrun.go` 

var (
	T Target
)

target only

type TargetLoc 

type TargetLoc interface {
	TargetPut

	fs.HC

	// backend
	Backend(*meta.Bck) Backend

	// Node health
	Health(si *meta.Snode, timeout time.Duration, query url.Values) (body []byte, ecode int, err error)
}

local target node

type TargetPut 

type TargetPut interface {
	Node

	// Space
	OOS(*fs.CapStatus, *cmn.Config, *fs.Tcdf) fs.CapStatus

	// xactions (jobs) now
	GetAllRunning(inout *AllRunningInOut, periodic bool)

	// PUT params.Reader => lom
	PutObject(lom *LOM, params *PutParams) (err error)

	// utilize blob downloader to cold-GET => (lom | custom write callback)
	GetColdBlob(params *BlobParams, oa *cmn.ObjAttrs) (xctn Xact, err error)
}

a node that can also write objects

type Uchunk added in v1.4.0 

type Uchunk struct {
	ETag string // S3/legacy
	MD5  []byte // ditto
	// contains filtered or unexported fields
}

func (*Uchunk) Num added in v1.4.0 

func (c *Uchunk) Num() uint16

func (*Uchunk) Path added in v1.4.0 

func (c *Uchunk) Path() string

func (*Uchunk) SetCksum added in v1.4.0 

func (c *Uchunk) SetCksum(cksum *cos.Cksum)

validate and set

func (*Uchunk) SetETag added in v1.4.0 

func (c *Uchunk) SetETag(etag string)

func (*Uchunk) Size added in v1.4.0 

func (c *Uchunk) Size() int64

readonly

type Ufest added in v1.4.0 

type Ufest struct {
	// contains filtered or unexported fields
}

func NewUfest added in v1.4.0 

func NewUfest(id string, lom *LOM, mustExist bool) (*Ufest, error)

func (*Ufest) Abort added in v1.4.0 

func (u *Ufest) Abort(lom *LOM)

func (*Ufest) Add added in v1.4.0 

func (u *Ufest) Add(c *Uchunk, size, num int64) error

(when manifest may be shared: locking is caller's responsibility)

func (*Ufest) Check added in v1.4.0 

func (u *Ufest) Check(completed bool) error

func (*Ufest) Completed added in v1.4.0 

func (u *Ufest) Completed() bool

immutable once completed

func (*Ufest) ComputeWholeChecksum added in v1.4.0 

func (u *Ufest) ComputeWholeChecksum(cksumH *cos.CksumHash) error

reread all chunk payloads to compute a checksum of the given type TODO: avoid the extra pass by accumulating during AddPart/StorePartial or by caching a tree-hash see also: s3/mpt for ListParts

func (*Ufest) Count added in v1.4.0 

func (u *Ufest) Count() int

func (*Ufest) Created added in v1.4.0 

func (u *Ufest) Created() time.Time

func (*Ufest) ETagS3 added in v1.4.0 

func (u *Ufest) ETagS3() (string, error)

func (*Ufest) GetChunk added in v1.4.0 

func (u *Ufest) GetChunk(num int) (*Uchunk, error)

(when manifest may be shared: locking is caller's responsibility)

func (*Ufest) ID added in v1.4.0 

func (u *Ufest) ID() string

func (*Ufest) IsHRW added in v1.4.2 

func (u *Ufest) IsHRW(avail fs.MPI) (bool, error)

check whether completed chunks are properly located

func (*Ufest) LoadCompleted added in v1.4.0 

func (u *Ufest) LoadCompleted(lom *LOM) error

must be called under lom (r)lock

func (*Ufest) LoadPartial added in v1.4.0 

func (u *Ufest) LoadPartial(lom *LOM) error

func (*Ufest) Lock added in v1.4.0 

func (u *Ufest) Lock()

func (*Ufest) Lom added in v1.4.0 

func (u *Ufest) Lom() *LOM

func (*Ufest) NewChunk added in v1.4.0 

func (u *Ufest) NewChunk(num int, lom *LOM) (*Uchunk, error)

construct a new chunk and its pathname - arg `num` is chunk.num (starts from 1) - chunk #1 stays on the object's mountpath - see chunk1Path() - other chunks get HRW distributed

func (*Ufest) NewReader added in v1.4.0 

func (u *Ufest) NewReader() (*UfestReader, error)

func (*Ufest) Relocate added in v1.4.2 

func (u *Ufest) Relocate(hrwMi *fs.Mountpath, buf []byte) (*LOM, error)

func (*Ufest) Size added in v1.4.0 

func (u *Ufest) Size() int64

func (*Ufest) StorePartial added in v1.4.0 

func (u *Ufest) StorePartial(lom *LOM, locked bool) error

func (*Ufest) Unlock added in v1.4.0 

func (u *Ufest) Unlock()

func (*Ufest) ValidateChunkLocations added in v1.4.2 

func (u *Ufest) ValidateChunkLocations(completed bool) error

note: used only in tests and asserts

type UfestReader added in v1.4.0 

type UfestReader struct {
	// contains filtered or unexported fields
}

func (*UfestReader) Close added in v1.4.0 

func (r *UfestReader) Close() error

func (*UfestReader) Read added in v1.4.0 

func (r *UfestReader) Read(p []byte) (n int, err error)

func (*UfestReader) ReadAt added in v1.4.0 

func (r *UfestReader) ReadAt(p []byte, off int64) (n int, err error)

consistent with io.ReaderAt semantics: - do not use or modify reader's offset (r.goff) or any other state - open the needed chunk(s) transiently and close them immediately - return io.EOF when less than len(p)

type Upon 

type Upon int

type Xact 

type Xact interface {
	Run(*sync.WaitGroup)
	ID() string
	Kind() string
	Bck() *meta.Bck
	FromTo() (*meta.Bck, *meta.Bck)
	StartTime() time.Time
	EndTime() time.Time
	IsDone() bool
	IsRunning() bool
	IsIdle() bool
	Quiesce(time.Duration, QuiCB) QuiRes

	// abrt
	IsAborted() bool
	AbortErr() error
	AbortedAfter(time.Duration) error
	ChanAbort() <-chan error
	// err (info)
	AddErr(error, ...int)

	// to support api.QueryXactionSnaps
	CtlMsg() string
	Snap() *Snap // (struct below)

	// reporting: log, err
	String() string
	Name() string
	Cname() string

	// modifiers
	Finish()
	Abort(error) bool
	AddNotif(n Notif)

	// common stats
	GetStats
}

Source Files

View all Source files

Directories

Path Synopsis
Package meta: cluster-level metadata
 Package meta: cluster-level metadata
Package mock provides a variety of mock implementations used for testing.
 Package mock provides a variety of mock implementations used for testing.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.