transfer

package
v0.4.80 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 30, 2026 License: BSD-3-Clause Imports: 18 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var BufferSize int64 = 8 * 1024 * 1024
View Source 
var CopyBufferDelay time.Duration
View Source 
var ErrChecksumMismatch = errors.New("checksum mismatch")
View Source 
var MinimumRangeSize int64 = 32 * 1024 * 1024

Functions

func ProgressLabel added in v0.4.13 

func ProgressLabel(local, remote string) string

func Sha256Checksum added in v0.3.11 

func Sha256Checksum(ctx context.Context, local string) ([]byte, error)

Sha256Checksum computes the sha256 checksum of a local file in a goroutine, so that it can be canceled with the context. The checksum is computed in a goroutine, so that it can be canceled with the context. The function returns the checksum as a byte slice, or an error if either the context is canceled or the checksum computation fails.

func VerifyLocalToRemote added in v0.4.56 

func VerifyLocalToRemote(a *api.API, progressHandler func(Progress)) func(ctx context.Context, local, remote string, localInfo, remoteInfo os.FileInfo) ([]byte, []byte, error)

Verify checks the checksum of a local file against the checksum of a remote file

func VerifyRemoteToLocal added in v0.4.56 

func VerifyRemoteToLocal(a *api.API, progressHandler func(Progress)) func(ctx context.Context, local, remote string, localInfo, remoteInfo os.FileInfo) ([]byte, []byte, error)

func VerifyRemoteToRemote added in v0.4.56 

func VerifyRemoteToRemote(a *api.API, progressHandler func(Progress)) func(ctx context.Context, remote1, remote2 string, remote1Info, remote2Info os.FileInfo) ([]byte, []byte, error)

VerifyRemote checks the checksum of two remote files

Types

type Action added in v0.4.12 

type Action int
const (
	CreateDirectory Action = iota
	TransferFile
	RemoveFile
	RemoveDirectory
	ComputeChecksum
	SetModificationTime
)

func (Action) Format added in v0.4.13 

func (a Action) Format(label string) string

type ChecksumReader added in v0.4.56 

type ChecksumReader interface {
	Reader
	Checksum(ctx context.Context) ([]byte, error)
}

type ChecksumWriter added in v0.4.56 

type ChecksumWriter interface {
	Writer
	Checksum(ctx context.Context) ([]byte, error)
}

type CircularReader added in v0.4.37 

type CircularReader struct {
	ReadSeekCloser io.ReadSeekCloser
	MaxThreads     int
	Reopen         func() (io.ReadSeekCloser, error)
	Size           int64
	// contains filtered or unexported fields
}

func (*CircularReader) Close added in v0.4.37 

func (r *CircularReader) Close() error

func (*CircularReader) Read added in v0.4.37 

func (r *CircularReader) Read(data []byte) (int, error)

type CircularWriter added in v0.4.37 

type CircularWriter struct {
	WriteSeekCloser WriteSeekCloser
	MaxThreads      int
	Reopen          func() (WriteSeekCloser, error)
	// contains filtered or unexported fields
}

func (*CircularWriter) Close added in v0.4.37 

func (w *CircularWriter) Close() error

func (*CircularWriter) Write added in v0.4.37 

func (w *CircularWriter) Write(data []byte) (int, error)

type Direction added in v0.4.12 

type Direction int
const (
	LocalToRemote Direction = iota
	RemoteToLocal
)

type Options added in v0.3.11 

type Options struct {
	// Do not overwrite existing files
	Exclusive bool
	// Delete unrecognized files at the target when downloading or uploading a directory
	Delete bool
	// Don't update files in place. If set, a data object will be removed
	// and a new data object will be created, instead of updating the original object
	DisableUpdateInPlace bool
	// SkipTrash indicates whether files should be moved to the trash or not
	SkipTrash bool
	// Sync modification time
	SyncModTime bool
	// MaxThreads indicates the maximum threads per transferred file
	MaxThreads int
	// MaxQueued indicates the maximum number of queued files
	// when uploading or downloading a directory
	MaxQueued int
	// OnlyIfNewer indicates whether files should only be transferred
	// if the source file is newer than the destination file,
	// based on the modification time, when syncing directories (UploadDir, DownloadDir, CopyDir).
	OnlyIfNewer bool
	// CompareChecksums indicates whether checksums should be verified
	// to compare two existing file when syncing directories (UploadDir, DownloadDir, CopyDir).
	CompareChecksums bool
	// IntegrityChecksums indicates whether checksums should be computed before
	// and after the transfer to verify the integrity of the transfer (Upload, Download, UploadDir, DownloadDir, CopyDir).
	IntegrityChecksums bool
	// DryRun will only print actions. No actual transfers will be performed. Progress and errors will still be reported.
	DryRun bool
	// IgnorePatterns indicates patterns to ignore when uploading, downloading or copying a directory (UploadDir, DownloadDir, CopyDir).
	// The pattern syntax is the same as filepath.Match.
	IgnorePatterns []string
	// Output will, if set, display a progress bar and occurring errors
	// If ErrorHandler or ProgressHandler is set, this option is ignored
	Output io.Writer
	// Progress handler, can be used to track the progress of the transfers
	ProgressHandler func(progress Progress)
	// Error handler, called when an error occurs
	// If this callback is not set or returns an error, the worker will stop and Wait() will return the error
	ErrorHandler func(local, remote string, err error) error
}

type PB added in v0.3.11 

type PB struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func ProgressBar added in v0.3.11 

func ProgressBar(w io.Writer) *PB

func (*PB) Close added in v0.3.11 

func (pb *PB) Close() error

func (*PB) Elapsed added in v0.3.12 

func (pb *PB) Elapsed() time.Duration

func (*PB) ErrorHandler added in v0.3.12 

func (pb *PB) ErrorHandler(path, irodsPath string, err error) error

func (*PB) Handler added in v0.3.11 

func (pb *PB) Handler(progress Progress)

func (*PB) ScanCompleted added in v0.3.11 

func (pb *PB) ScanCompleted()

func (*PB) Write added in v0.3.12 

func (pb *PB) Write(buf []byte) (int, error)

type Progress 

type Progress struct {
	Action      Action
	Label       string
	Size        int64
	Transferred int64
	Increment   int64
	StartedAt   time.Time
	FinishedAt  time.Time
}

type RangeReader 

type RangeReader interface {
	Range(offset, length int64) io.Reader
}

type RangeWriter 

type RangeWriter interface {
	Range(offset, length int64) io.Writer
}

type Reader added in v0.3.12 

type Reader interface {
	Name() string
	Size() int64
	ModTime() time.Time
	io.ReaderAt
	io.Closer
}

type ReaderAtRangeReader 

type ReaderAtRangeReader struct {
	io.ReaderAt
}

func (*ReaderAtRangeReader) Range 

func (r *ReaderAtRangeReader) Range(offset, length int64) io.Reader

type ReopenRangeReader 

type ReopenRangeReader struct {
	io.ReadSeekCloser
	Reopen func() (io.ReadSeekCloser, error)

	sync.Mutex
	// contains filtered or unexported fields
}

func (*ReopenRangeReader) Close 

func (r *ReopenRangeReader) Close() error

func (*ReopenRangeReader) Range 

func (r *ReopenRangeReader) Range(offset, length int64) io.Reader

type ReopenRangeWriter 

type ReopenRangeWriter struct {
	WriteSeekCloser
	Reopen func() (WriteSeekCloser, error)

	sync.Mutex
	// contains filtered or unexported fields
}

func (*ReopenRangeWriter) Close 

func (r *ReopenRangeWriter) Close() error

func (*ReopenRangeWriter) Range 

func (r *ReopenRangeWriter) Range(offset, length int64) io.Writer

type SynchronizeOptions added in v0.4.17 

type SynchronizeOptions struct {
	DisableUpdateInPlace bool
}

type Task added in v0.4.12 

type Task struct {
	Action          Action
	Path, IrodsPath string
	Size            int64
	Checksum        []byte
	ModTime         time.Time
}

type Worker 

type Worker struct {
	IndexPool    *api.API
	TransferPool *api.API
	// contains filtered or unexported fields
}

func New 

func New(indexPool, transferPool *api.API, options Options) *Worker

func (*Worker) ComputeChecksums added in v0.4.56 

func (worker *Worker) ComputeChecksums(ctx context.Context, remote string)

ComputeChecksums computes the checksums of all files in a directory on the iRODS server. It handles the recursion client side, but individual files are processed server-side only. The call blocks until the source directory has been completely scanned.

func (*Worker) CopyDir added in v0.4.16 

func (worker *Worker) CopyDir(ctx context.Context, remote1, remote2 string)

CopyDir copies one directory to another on the iRODS server. It handles the recursion client side, but individual files are copied server-side only. The call blocks until the source directory has been completely scanned.

func (*Worker) Download added in v0.3.11 

func (worker *Worker) Download(ctx context.Context, local, remote string)

Download schedules the download of a remote file from the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) DownloadDir added in v0.3.11 

func (worker *Worker) DownloadDir(ctx context.Context, local, remote string)

DownloadDir downloads a remote directory from the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the source directory has been completely scanned.

func (*Worker) Error added in v0.3.11 

func (worker *Worker) Error(local, remote string, err error)

Error schedules an error

func (*Worker) FromReader added in v0.3.12 

func (worker *Worker) FromReader(ctx context.Context, r Reader, remote string)

FromReader schedules the upload of a reader to the iRODS server using parallel transfers. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) FromStream added in v0.4.37 

func (worker *Worker) FromStream(ctx context.Context, name string, r io.Reader, remote string, appendToFile bool)

FromStream schedules the upload of a io.Reader to the iRODS server using parallel transfers. In contrast to FromReader, FromStream will block until the full file has been uploaded. The remote file refers to an iRODS path.

func (*Worker) Progress added in v0.4.56 

func (worker *Worker) Progress(progress Progress)

Progress triggers the progress handler with the given progress.

func (*Worker) RemoveDir added in v0.4.13 

func (worker *Worker) RemoveDir(ctx context.Context, remote string)

RemoveDir removes a directory from the iRODS server entirely, but instead of calling the recursive RemoveCollection API, it handles the recursion client side. This allows the caller to track the deletion progress better. The call blocks until the source directory has been completely scanned.

func (*Worker) SynchronizeDir added in v0.4.12 

func (worker *Worker) SynchronizeDir(ctx context.Context, local, remote string, direction Direction, queue chan<- Task, opts SynchronizeOptions) error

SynchronizeDir schedules tasks to synchronize a local directory to or from a remote collection on the iRODS server. All individual actions are appended to the queue. The local file refers to the local file system. The remote file refers to an iRODS path. The Direction dictates the order of deletes and transfers. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. The call blocks until the source directory has been completely scanned.

func (*Worker) SynchronizeRemoteDir added in v0.4.16 

func (worker *Worker) SynchronizeRemoteDir(ctx context.Context, remote1, remote2 string, queue chan<- Task, opts SynchronizeOptions) error

SynchronizeRemoteDir schedules tasks to synchronize a remote collection to another remote collection on the iRODS server. All individual actions are appended to the queue. The deleteFirst parameter indicates whether to delete files first before retransferring, or whether files might be updated in place. The call blocks until the source directory has been completely scanned.

func (*Worker) ToStream added in v0.4.37 

func (worker *Worker) ToStream(ctx context.Context, name string, w io.Writer, remote string)

ToStream schedules the download into a io.Writer from the iRODS server. The remote file refers to an iRODS path. The call blocks until the transfer has started.

func (*Worker) ToWriter added in v0.3.12 

func (worker *Worker) ToWriter(ctx context.Context, w Writer, remote string)

Download schedules the download of a remote file from the iRODS server using parallel transfers. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) Upload added in v0.3.11 

func (worker *Worker) Upload(ctx context.Context, local, remote string)

Upload schedules the upload of a local file to the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the transfer of all chunks has started.

func (*Worker) UploadDir added in v0.3.11 

func (worker *Worker) UploadDir(ctx context.Context, local, remote string)

UploadDir uploads a local directory to the iRODS server using parallel transfers. The local file refers to the local file system. The remote file refers to an iRODS path. The call blocks until the source directory has been completely scanned.

func (*Worker) Wait 

func (worker *Worker) Wait() error

Wait for all transfers to finish

type WriteSeekCloser 

type WriteSeekCloser interface {
	io.WriteSeeker
	io.Closer
}

type Writer added in v0.3.12 

type Writer interface {
	Name() string
	io.WriterAt
	io.Closer
	Remove() error
	Touch(mtime time.Time) error
}

type WriterAtRangeWriter 

type WriterAtRangeWriter struct {
	io.WriterAt
}

func (*WriterAtRangeWriter) Range 

func (r *WriterAtRangeWriter) Range(offset, length int64) io.Writer

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.