Documentation
¶
Overview ¶
Package transfer defines data structures and functions for transferring data between local and 0g storage. It contains end-to-end interfaces for uploading and downloading data. These interfaces internally utilize parallelism to leverage computational resources.
Index ¶
- Variables
- func BuildFileTree(ctx context.Context, downloader IDownloader, root string, proof bool) (*dir.FsNode, error)
- func DownloadDir(ctx context.Context, downloader IDownloader, root, filename string, ...) error
- type BatchUploadOption
- type Downloader
- type FileSegmentUploader
- type FileSegmentsWithProof
- type FinalityRequirement
- type IDownloader
- type SelectedNodes
- type SubmitLogEntryOption
- type UploadOption
- type Uploader
- func (uploader *Uploader) BatchUpload(ctx context.Context, datas []core.IterableData, option ...BatchUploadOption) (common.Hash, []common.Hash, error)
- func (uploader *Uploader) ParseLogs(ctx context.Context, logs []*types.Log) ([]uint64, error)
- func (uploader *Uploader) SplitableUpload(ctx context.Context, data core.IterableData, fragmentSize int64, ...) ([]common.Hash, []common.Hash, error)
- func (uploader *Uploader) SubmitLogEntry(ctx context.Context, datas []core.IterableData, tags [][]byte, ...) (common.Hash, *types.Receipt, error)
- func (uploader *Uploader) Upload(ctx context.Context, data core.IterableData, option ...UploadOption) (common.Hash, common.Hash, error)
- func (uploader *Uploader) UploadDir(ctx context.Context, folder string, option ...UploadOption) (txnHash, rootHash common.Hash, _ error)
- func (uploader *Uploader) UploadFile(ctx context.Context, path string, option ...UploadOption) (txnHash common.Hash, rootHash common.Hash, err error)
- func (uploader *Uploader) WithRoutines(routines int) *Uploader
Constants ¶
This section is empty.
Variables ¶
var (
ErrFileAlreadyExists = errors.New("File already exists")
)
Functions ¶
func BuildFileTree ¶
func BuildFileTree(ctx context.Context, downloader IDownloader, root string, proof bool) (*dir.FsNode, error)
BuildFileTree downloads directory metadata from the ZeroGStorage network and decodes it into an FsNode structure. This function retrieves the metadata of a directory, which is stored in the ZeroGStorage network, and then decodes the metadata to construct a file tree representation of the directory.
Parameters:
- ctx: Context to manage request deadlines, cancellation, and timeout.
- downloader: The interface responsible for downloading files from the ZeroGStorage network.
- root: The root hash of the directory's metadata.
- proof: Whether to download with Merkle proof validation.
Returns:
- *dir.FsNode: A pointer to the decoded file tree structure representing the directory.
- error: An error if downloading or decoding the directory metadata fails.
func DownloadDir ¶
func DownloadDir(ctx context.Context, downloader IDownloader, root, filename string, withProof bool) error
DownloadDir downloads files within a directory recursively from the ZeroGStorage network. It first builds a file tree from the directory metadata, then downloads each file in the directory, and finally seals the directory when the download is complete.
Parameters:
- ctx: Context for managing request timeouts and cancellations.
- downloader: The interface responsible for downloading files from the ZeroGStorage network.
- root: The root hash of the directory to be downloaded.
- filename: The name of the local directory to store the downloaded files.
- withProof: Whether to download the files with a Merkle proof for validation.
Returns:
- error: An error if any part of the download or file creation process fails.
Types ¶
type BatchUploadOption ¶
type BatchUploadOption struct {
Fee *big.Int // fee in neuron
Nonce *big.Int // nonce for transaction
MaxGasPrice *big.Int // max gas price for transaction
NRetries int // number of retries for uploading
Step int64 // step for uploading
TaskSize uint // number of files to upload simutanously
Method string // method for selecting nodes, can be "max", "random" or certain positive number in string
FullTrusted bool // whether to use full trusted nodes
DataOptions []UploadOption // upload option for single file, nonce and fee are ignored
}
BatchUploadOption upload option for a batching
type Downloader ¶
type Downloader struct {
// contains filtered or unexported fields
}
Downloader downloader to download file to storage nodes
func NewDownloader ¶
NewDownloader Initialize a new downloader.
func (*Downloader) Download ¶
func (downloader *Downloader) Download(ctx context.Context, root, filename string, withProof bool) error
Download download data from storage nodes.
func (*Downloader) DownloadFragments ¶
func (*Downloader) WithRoutines ¶
func (downloader *Downloader) WithRoutines(routines int) *Downloader
type FileSegmentUploader ¶
type FileSegmentUploader struct {
// contains filtered or unexported fields
}
func NewFileSegmentUploader ¶
func NewFileSegmentUploader(clients []*node.ZgsClient, opts ...zg_common.LogOption) *FileSegmentUploader
func (*FileSegmentUploader) Upload ¶
func (uploader *FileSegmentUploader) Upload(ctx context.Context, fileSeg FileSegmentsWithProof, option ...UploadOption) error
Upload uploads file segments with proof to the storage nodes parallelly. Note: only `ExpectedReplica` and `TaskSize` are used from UploadOption.
type FileSegmentsWithProof ¶
type FileSegmentsWithProof struct {
*node.FileInfo
Segments []node.SegmentWithProof
}
FileSegmentsWithProof wraps segments with proof and file info
type FinalityRequirement ¶
type FinalityRequirement uint
const ( FileFinalized FinalityRequirement = iota // wait for file finalization TransactionPacked // wait for transaction packed )
type IDownloader ¶
type SelectedNodes ¶
SelectedNodes holds the selected trusted and discovered nodes.
type SubmitLogEntryOption ¶
type SubmitLogEntryOption struct {
Fee *big.Int
Nonce *big.Int
MaxGasPrice *big.Int
NRetries int
Step int64
}
SubmitLogEntryOption option for submitting log entry
type UploadOption ¶
type UploadOption struct {
Tags []byte // transaction tags
FinalityRequired FinalityRequirement // finality setting
TaskSize uint // number of segment to upload in single rpc request
ExpectedReplica uint // expected number of replications
SkipTx bool // skip sending transaction on chain, this can set to true only if the data has already settled on chain before
Fee *big.Int // fee in neuron
Nonce *big.Int // nonce for transaction
MaxGasPrice *big.Int // max gas price for transaction
NRetries int // number of retries for uploading
Step int64 // step for uploading
Method string // method for selecting nodes, can be "max", "random" or certain positive number in string
FullTrusted bool // whether to use full trusted nodes
}
UploadOption upload option for a file
type Uploader ¶
type Uploader struct {
// contains filtered or unexported fields
}
Uploader uploader to upload file to 0g storage, send on-chain transactions and transfer data to storage nodes.
func NewUploader ¶
func NewUploader(ctx context.Context, w3Client *web3go.Client, clients *SelectedNodes, opts ...zg_common.LogOption) (*Uploader, error)
NewUploader Initialize a new uploader.
func (*Uploader) BatchUpload ¶
func (uploader *Uploader) BatchUpload(ctx context.Context, datas []core.IterableData, option ...BatchUploadOption) (common.Hash, []common.Hash, error)
BatchUpload submit multiple data to 0g storage contract batchly in single on-chain transaction, then transfer the data to the storage nodes. The nonce for upload transaction will be the first non-nil nonce in given upload options, the protocol fee is the sum of fees in upload options.
func (*Uploader) SplitableUpload ¶
func (uploader *Uploader) SplitableUpload(ctx context.Context, data core.IterableData, fragmentSize int64, option ...UploadOption) ([]common.Hash, []common.Hash, error)
SplitableUpload submit data to 0g storage contract and large data will be splited to reduce padding cost.
func (*Uploader) SubmitLogEntry ¶
func (uploader *Uploader) SubmitLogEntry(ctx context.Context, datas []core.IterableData, tags [][]byte, submitOption SubmitLogEntryOption) (common.Hash, *types.Receipt, error)
SubmitLogEntry submit the data to 0g storage contract by sending a transaction
func (*Uploader) Upload ¶
func (uploader *Uploader) Upload(ctx context.Context, data core.IterableData, option ...UploadOption) (common.Hash, common.Hash, error)
Upload submit data to 0g storage contract, then transfer the data to the storage nodes. returns the submission transaction hash and the hash will be zero if transaction is skipped.