Documentation
¶
Overview ¶
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
* Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
* Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package ec provides erasure coding (EC) based data protection for AIStore.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Index ¶
- Constants
- Variables
- func AllocLomFromHdr(hdr *transport.ObjHdr) (*core.LOM, error)
- func Init()
- func IsECCopy(size int64, ecConf *cmn.ECConf) bool
- func SliceSize(fileSize int64, slices int) int64
- func WriteReplicaAndMeta(lom *core.LOM, args *WriteArgs) error
- func WriteSliceAndMeta(hdr *transport.ObjHdr, args *WriteArgs) error
- type ExtECGetStats
- type ExtECPutStats
- type Manager
- func (mgr *Manager) BMDChanged() error
- func (mgr *Manager) CleanupObject(lom *core.LOM)
- func (mgr *Manager) CloseStreams(justRefc bool)
- func (mgr *Manager) EncodeObject(lom *core.LOM, cb onFin) error
- func (mgr *Manager) IsActive() bool
- func (mgr *Manager) NewGetXact(bck *cmn.Bck) *XactGet
- func (mgr *Manager) NewPutXact(bck *cmn.Bck) *XactPut
- func (mgr *Manager) NewRespondXact(bck *cmn.Bck) *XactRespond
- func (mgr *Manager) OpenStreams(withRefc bool)
- func (mgr *Manager) Recover(lom *core.LOM) error
- func (*Manager) RestoreBckGetXact(bck *meta.Bck) *XactGet
- func (*Manager) RestoreBckPutXact(bck *meta.Bck) *XactPut
- func (*Manager) RestoreBckRespXact(bck *meta.Bck) *XactRespond
- type Metadata
- func LoadMetadata(fqn string) (*Metadata, error)
- func MetaFromReader(reader io.Reader, size int64) (*Metadata, error)
- func NewMetadata() *Metadata
- func ObjectMetadata(bck *meta.Bck, objName string) (*Metadata, error)
- func RequestECMeta(bck *cmn.Bck, objName string, si *meta.Snode, client *http.Client) (*Metadata, error)
- type RequestsControlMsg
- type Stats
- type WriteArgs
- type XactBckEncode
- type XactGet
- type XactPut
- type XactRespond
Constants ¶
const ( ActSplit = "split" ActRestore = "restore" ActDelete = "delete" RespStreamName = "ec-resp" ReqStreamName = "ec-req" ActClearRequests = "clear-requests" ActEnableRequests = "enable-requests" URLMeta = "meta" /// .. - metadata requests )
const MDVersionLast = 1 // current version of metadata
Variables ¶
Functions ¶
func AllocLomFromHdr ¶ added in v1.3.23
lom <= transport.ObjHdr (NOTE: caller must call freeLOM)
func WriteReplicaAndMeta ¶
WriteReplicaAndMeta saves replica and its metafile
Types ¶
type ExtECGetStats ¶
type ExtECGetStats struct {
AvgTime cos.Duration `json:"ec.decode.ns"`
ErrCount int64 `json:"ec.decode.err.n,string"`
AvgObjTime cos.Duration `json:"ec.obj.process.ns"`
AvgQueueLen float64 `json:"ec.queue.len.f"`
IsIdle bool `json:"is_idle"`
}
extended x-ec-get statistics
type ExtECPutStats ¶
type ExtECPutStats struct {
AvgEncodeTime cos.Duration `json:"ec.encode.ns"`
AvgDeleteTime cos.Duration `json:"ec.delete.ns"`
EncodeCount int64 `json:"ec.encode.n,string"`
DeleteCount int64 `json:"ec.delete.n,string"`
EncodeSize int64 `json:"ec.encode.size,string"`
EncodeErrCount int64 `json:"ec.encode.err.n,string"`
DeleteErrCount int64 `json:"ec.delete.err.n,string"`
AvgObjTime cos.Duration `json:"ec.obj.process.ns"`
AvgQueueLen float64 `json:"ec.queue.len.f"`
IsIdle bool `json:"is_idle"`
}
extended x-ec-put statistics
type Manager ¶
type Manager struct {
// contains filtered or unexported fields
}
var (
ECM *Manager
)
func (*Manager) BMDChanged ¶ added in v1.3.22
func (*Manager) CleanupObject ¶
func (*Manager) CloseStreams ¶ added in v1.3.24
func (*Manager) EncodeObject ¶
EncodeObject generates slices using Reed-Solom algorithm:
- lom - object to encode
- intra - if true, it is internal request and has low priority
- cb - optional callback that is called after the object is encoded
func (*Manager) NewRespondXact ¶
func (mgr *Manager) NewRespondXact(bck *cmn.Bck) *XactRespond
func (*Manager) OpenStreams ¶ added in v1.3.24
func (*Manager) RestoreBckRespXact ¶
func (*Manager) RestoreBckRespXact(bck *meta.Bck) *XactRespond
type Metadata ¶
type Metadata struct {
Size int64 `json:"obj_size"` // obj size (after EC'ing sum size of slices differs from the original)
Generation int64 `json:"generation"` // Timestamp when the object was EC'ed
ObjCksum string `json:"obj_cksum"` // checksum of the original object
ObjVersion string `json:"obj_version"` // object version
CksumType string `json:"cksum_type"` // slice checksum type
CksumValue string `json:"slice_cksum"` // slice checksum of the slice if EC is used
FullReplica string `json:"replica_node"` // daemon ID where full(main) replica is
Daemons cos.MapStrUint16 `json:"nodes"` // Locations of all slices: DaemonID <-> SliceID
Data int `json:"data_slices"` // the number of data slices
Parity int `json:"parity_slices"` // the number of parity slices
SliceID int `json:"slice_id"` // 0 for full replica, 1 to N for slices
MDVersion uint32 `json:"md_version"` // Metadata format version
IsCopy bool `json:"is_copy"` // object is replicated(true) or encoded(false)
}
Metadata - EC information stored in metafiles for every encoded object
func LoadMetadata ¶
LoadMetadata loads and parses EC metadata from a file
func NewMetadata ¶
func NewMetadata() *Metadata
func ObjectMetadata ¶
ObjectMetadata returns metadata for an object or its slice if any exists
func RequestECMeta ¶
func RequestECMeta(bck *cmn.Bck, objName string, si *meta.Snode, client *http.Client) (*Metadata, error)
RequestECMeta returns an EC metadata found on a remote target.
func (*Metadata) PackedSize ¶
func (*Metadata) RemoteTargets ¶
RemoteTargets returns list of Snodes that contain a slice or replica. This target(`t`) is removed from the list.
type RequestsControlMsg ¶
type RequestsControlMsg struct {
Action string
}
type Stats ¶
type Stats struct {
// mpathrunner(not ecrunner) queue len
QueueLen float64
// time between ecrunner receives an object and mpathrunner starts processing it
WaitTime time.Duration
// EC encoding time (for both EC'ed and replicated objects)
EncodeTime time.Duration
// size of a file put into encode queue
EncodeSize int64
// total number of errors while encoding objects
EncodeErr int64
// total number of errors while restoring objects
DecodeErr int64
// time to restore an object(for both EC'ed and replicated objects)
DecodeTime time.Duration
// time to cleanup object's slices(for both EC'ed and replicated objects)
DeleteTime time.Duration
// total number of errors while cleaning up object slices
DeleteErr int64
// total object processing time: from putting to ecrunner queue to
// completing the request by mpathrunner
ObjTime time.Duration
// total number of cleanup requests
DelReq int64
// total number of restore requests
GetReq int64
// total number of encode requests
PutReq int64
// name of the bucket
Bck cmn.Bck
// xaction state: working or waiting for commands
IsIdle bool
}
Stats are EC-specific stats for clients-side apps - calculated from raw counters All numbers except number of errors and requests are average ones
type XactBckEncode ¶
func (*XactBckEncode) CtlMsg ¶ added in v1.4.1
func (r *XactBckEncode) CtlMsg() (s string)
func (*XactBckEncode) RecvRecover ¶ added in v1.3.26
func (r *XactBckEncode) RecvRecover(lom *core.LOM)
func (*XactBckEncode) Run ¶
func (r *XactBckEncode) Run(gowg *sync.WaitGroup)
func (*XactBckEncode) Snap ¶ added in v1.3.16
func (r *XactBckEncode) Snap() (snap *core.Snap)
type XactGet ¶
type XactGet struct {
// contains filtered or unexported fields
}
Erasure coding runner: accepts requests and dispatches them to a correct mountpath runner. Runner uses dedicated to EC memory manager inherited by dependent mountpath runners
func (*XactGet) ClearRequests ¶
func (r *XactGet) ClearRequests()
ClearRequests disables receiving new EC requests, they will be terminated with error Then it starts draining a channel from pending EC requests It does not enable receiving new EC requests, it has to be done explicitly, when EC is enabled again
func (*XactGet) EnableRequests ¶
func (r *XactGet) EnableRequests()
type XactPut ¶
type XactPut struct {
// contains filtered or unexported fields
}
Erasure coding runner: accepts requests and dispatches them to a correct mountpath runner. Runner uses dedicated to EC memory manager inherited by dependent mountpath runners
func (*XactPut) ClearRequests ¶
func (r *XactPut) ClearRequests()
ClearRequests disables receiving new EC requests, they will be terminated with error Then it starts draining a channel from pending EC requests It does not enable receiving new EC requests, it has to be done explicitly, when EC is enabled again
func (*XactPut) EnableRequests ¶
func (r *XactPut) EnableRequests()
type XactRespond ¶
type XactRespond struct {
// contains filtered or unexported fields
}
Xaction responsible for responding to EC requests of other targets. Should not be stopped if number of known targets is small.
func (*XactRespond) CtlMsg ¶ added in v1.4.1
func (*XactRespond) CtlMsg() string
func (*XactRespond) Run ¶
func (r *XactRespond) Run(gowg *sync.WaitGroup)
func (*XactRespond) Snap ¶
func (r *XactRespond) Snap() *core.Snap
func (*XactRespond) Stop ¶
func (r *XactRespond) Stop(err error)