Documentation
¶
Overview ¶
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package cmn provides common low-level types and utilities for all aistore projects
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
Index ¶
- Constants
- func AbortReq(jobID string) cmn.HreqArgs
- func CompareObjects(lom *core.LOM, dst *DstElement) (bool, error)
- func Init(db kvdb.Driver, clientConf *cmn.ClientConf)
- func IsType(a string) bool
- func ListJobs(regex *regexp.Regexp, onlyActive bool) (any, int, error)
- func NormalizeObjName(objName string) (string, error)
- func ParseStartRequest(bck *meta.Bck, id string, dlb Body, xdl *Xact) (jobif, error)
- type AdminBody
- type BackendBody
- type BackendResource
- type Base
- type Body
- type DiffResolver
- func (dr *DiffResolver) Abort(err error)
- func (dr *DiffResolver) CloseDst()
- func (dr *DiffResolver) CloseSrc()
- func (dr *DiffResolver) Next() (DiffResolverResult, error)
- func (dr *DiffResolver) PushDst(v any)
- func (dr *DiffResolver) PushSrc(v any)
- func (dr *DiffResolver) Start()
- func (dr *DiffResolver) Stop()
- func (dr *DiffResolver) Stopped() bool
- type DiffResolverCtx
- type DiffResolverResult
- type DlPostResp
- type DstElement
- type Job
- type JobInfos
- type Limits
- type MultiBody
- type NotifDownload
- type NotifDownloadListerner
- type RangeBody
- type SingleBody
- type SingleObj
- type StatusResp
- type TaskDlInfo
- type TaskErrByName
- type TaskErrInfo
- type TaskInfoByName
- type Type
- type WebResource
- type Xact
- func (xld *Xact) AbortJob(id string) (resp any, statusCode int, err error)
- func (*Xact) CtlMsg() string
- func (xld *Xact) Download(job jobif) (resp any, statusCode int, err error)
- func (xld *Xact) JobStatus(id string, onlyActive bool) (resp any, statusCode int, err error)
- func (xld *Xact) RemoveJob(id string) (resp any, statusCode int, err error)
- func (xld *Xact) Run(*sync.WaitGroup)
- func (xld *Xact) Snap() *core.Snap
Constants ¶
const ( DiffResolverSend = iota DiffResolverRecv DiffResolverDelete DiffResolverSkip DiffResolverErr DiffResolverEOF )
const DownloadProgressInterval = 10 * time.Second
const PrefixJobID = "dnl-"
Variables ¶
This section is empty.
Functions ¶
func CompareObjects ¶
func CompareObjects(lom *core.LOM, dst *DstElement) (bool, error)
Use all available metadata including {size, version, ETag, MD5, CRC} to compare local object with its remote counterpart (source).
func NormalizeObjName ¶
Removes everything that goes after '?', eg. "?query=key..." so it will not be part of final object name.
Types ¶
type AdminBody ¶
type BackendBody ¶
type BackendBody struct {
Base
Prefix string `json:"prefix"`
Suffix string `json:"suffix"`
Sync bool `json:"synchronize"`
}
func (*BackendBody) Describe ¶
func (b *BackendBody) Describe() string
func (*BackendBody) Validate ¶
func (b *BackendBody) Validate() error
type BackendResource ¶
type BackendResource struct {
ObjName string
}
type Base ¶
type Base struct {
Description string `json:"description"`
Bck cmn.Bck `json:"bucket"`
Timeout string `json:"timeout"`
ProgressInterval string `json:"progress_interval"`
Limits Limits `json:"limits"`
Headers http.Header `json:"headers,omitempty"`
// ETL fields
ETLName string `json:"etl_name,omitempty"`
ETLArgs string `json:"etl_args,omitempty"`
}
type Body ¶
type Body struct {
Type Type `json:"type"`
json.RawMessage
}
NOTE: Changing this structure requires changes in `MarshalJSON` and `UnmarshalJSON` methods.
func (Body) MarshalJSON ¶
func (*Body) UnmarshalJSON ¶
type DiffResolver ¶
type DiffResolver struct {
// contains filtered or unexported fields
}
DiffResolver is entity that computes difference between two streams of objects. The streams are expected to be in sorted order.
func NewDiffResolver ¶
func NewDiffResolver(ctx DiffResolverCtx) *DiffResolver
TODO: configurable burst size of the channels, plus `chanFull` check
func (*DiffResolver) Abort ¶
func (dr *DiffResolver) Abort(err error)
func (*DiffResolver) CloseDst ¶
func (dr *DiffResolver) CloseDst()
func (*DiffResolver) CloseSrc ¶
func (dr *DiffResolver) CloseSrc()
func (*DiffResolver) Next ¶
func (dr *DiffResolver) Next() (DiffResolverResult, error)
func (*DiffResolver) PushDst ¶
func (dr *DiffResolver) PushDst(v any)
func (*DiffResolver) PushSrc ¶
func (dr *DiffResolver) PushSrc(v any)
func (*DiffResolver) Start ¶
func (dr *DiffResolver) Start()
func (*DiffResolver) Stop ¶
func (dr *DiffResolver) Stop()
func (*DiffResolver) Stopped ¶
func (dr *DiffResolver) Stopped() bool
type DiffResolverCtx ¶
type DiffResolverResult ¶
type DiffResolverResult struct {
Err error
Src *core.LOM
Dst *DstElement
Action uint8
}
type DlPostResp ¶
type DlPostResp struct {
ID string `json:"id"`
}
Download POST result returned to the user
type DstElement ¶
type Job ¶
type Job struct {
ID string `json:"id"`
XactID string `json:"xaction_id"`
Description string `json:"description"`
StartedTime time.Time `json:"started_time"`
FinishedTime time.Time `json:"finished_time"`
FinishedCnt int `json:"finished_cnt"`
ScheduledCnt int `json:"scheduled_cnt"` // tasks being processed or already processed by dispatched
SkippedCnt int `json:"skipped_cnt"` // number of tasks skipped
ErrorCnt int `json:"error_cnt"`
Total int `json:"total"` // total number of tasks, negative if unknown
AllDispatched bool `json:"all_dispatched"` // if true, dispatcher has already scheduled all tasks for given job
Aborted bool `json:"aborted"`
}
func (*Job) DoneCnt ¶
DoneCnt returns number of tasks that have finished (either successfully or with an error).
func (*Job) JobFinished ¶
func (*Job) JobRunning ¶
func (*Job) PendingCnt ¶
PendingCnt returns number of tasks which are currently being processed.
type NotifDownload ¶
func (*NotifDownload) ToNotifMsg ¶
func (nd *NotifDownload) ToNotifMsg(aborted bool) core.NotifMsg
type NotifDownloadListerner ¶
type NotifDownloadListerner struct {
nl.ListenerBase
}
func NewDownloadNL ¶
func (*NotifDownloadListerner) QueryArgs ¶
func (nd *NotifDownloadListerner) QueryArgs() cmn.HreqArgs
func (*NotifDownloadListerner) UnmarshalStats ¶
func (*NotifDownloadListerner) UnmarshalStats(rawMsg []byte) (stats any, finished, aborted bool, err error)
type SingleBody ¶
func (*SingleBody) Describe ¶
func (b *SingleBody) Describe() string
func (*SingleBody) ExtractPayload ¶
func (b *SingleBody) ExtractPayload() (cos.StrKVs, error)
func (*SingleBody) String ¶
func (b *SingleBody) String() string
func (*SingleBody) Validate ¶
func (b *SingleBody) Validate() error
type SingleObj ¶
type StatusResp ¶
type StatusResp struct {
Job
CurrentTasks []TaskDlInfo `json:"current_tasks,omitempty"`
FinishedTasks []TaskDlInfo `json:"finished_tasks,omitempty"`
Errs []TaskErrInfo `json:"download_errors,omitempty"`
}
func (*StatusResp) Aggregate ¶
func (d *StatusResp) Aggregate(rhs *StatusResp) *StatusResp
type TaskDlInfo ¶
type TaskErrByName ¶
type TaskErrByName []TaskErrInfo
func (TaskErrByName) Len ¶
func (t TaskErrByName) Len() int
func (TaskErrByName) Less ¶
func (t TaskErrByName) Less(i, j int) bool
func (TaskErrByName) Swap ¶
func (t TaskErrByName) Swap(i, j int)
type TaskErrInfo ¶
type TaskInfoByName ¶
type TaskInfoByName []TaskDlInfo
func (TaskInfoByName) Len ¶
func (t TaskInfoByName) Len() int
func (TaskInfoByName) Less ¶
func (t TaskInfoByName) Less(i, j int) bool
func (TaskInfoByName) Swap ¶
func (t TaskInfoByName) Swap(i, j int)