Documentation
¶
Index ¶
- Constants
- Variables
- func CleanConfig(tempPath string)
- func CleanOperation(op *Operation)
- func Copy(remotes *pb.RcloneRemotes, srcStr, dstStr string) error
- func CopyFilesWithProgress(srcFs fs.Fs, srcPath string, dstFs fs.Fs, dstPath string) error
- func GetMD5(f fs.DirEntry) string
- func Info(remotes *pb.RcloneRemotes, srcStr string) (fs.DirEntry, error)
- func IsDir(f fs.DirEntry) bool
- func IsFile(f fs.DirEntry) bool
- func Join(path, file string) string
- func List(remotes *pb.RcloneRemotes, srcStr string) ([]string, error)
- func Purge(ctx context.Context, uriStr string) error
- func RawList(remotes *pb.RcloneRemotes, srcStr string) (fs.DirEntries, error)
- type AsperaBackend
- type FastqBackend
- type FileSystemInterface
- type LocalBackend
- type MetaFileSystem
- type Operation
- type RcloneBackend
- type URI
Constants ¶
const DefaultRcloneConfig = "/etc/rclone.conf"
Variables ¶
var LoadedRemotes []string
FetchContext holds global configuration settings for the fetch package.
Functions ¶
func CleanConfig ¶
func CleanConfig(tempPath string)
func CleanOperation ¶
func CleanOperation(op *Operation)
func CopyFilesWithProgress ¶
func Purge ¶ added in v0.9.3
Purge recursively removes everything under the given URI. Used by workflow-lifetime output cleanup (spec: outputs.lifetime = workflow): once a workflow reaches S, the server sweeps the workspace copies of intermediate outputs the author declared as ephemeral.
Purge is idempotent-ish: rclone returns fs.ErrorDirNotFound (or the backend's equivalent) when the directory is already gone, which we treat as success. Any other error is bubbled up so the caller can log it / raise an event.
The URI is resolved through the same ParseURI + uriToRcloneString path the copy code uses, so any backend rclone knows about (via /etc/rclone.conf) works. The trailing slash convention on the URI doesn't matter — Purge operates on a directory prefix either way.
func RawList ¶
func RawList(remotes *pb.RcloneRemotes, srcStr string) (fs.DirEntries, error)
Types ¶
type AsperaBackend ¶
type AsperaBackend struct{}
AsperaBackend represents an Aspera (FASP) backend that supports only copying to LocalBackend.
func NewAsperaBackend ¶
func NewAsperaBackend() *AsperaBackend
NewAsperaBackend initializes an AsperaBackend.
func (*AsperaBackend) Copy ¶
func (ab *AsperaBackend) Copy(otherFs FileSystemInterface, src, dst URI, selfIsSource bool) error
Copy implements the Copy method for AsperaBackend. Supports only copying to a local backend.
func (*AsperaBackend) List ¶
func (ab *AsperaBackend) List(path string) (fs.DirEntries, error)
func (*AsperaBackend) Mkdir ¶
func (rb *AsperaBackend) Mkdir(path string) error
type FastqBackend ¶
type FastqBackend struct{}
FastqBackend handles downloading FASTQ files using FTP, Aspera, or SRA.
func NewFastqBackend ¶
func NewFastqBackend() *FastqBackend
NewFastqBackend initializes a FastqBackend.
func (*FastqBackend) Copy ¶
func (fb *FastqBackend) Copy(otherFs FileSystemInterface, src, dst URI, selfIsSource bool) error
Copy implements the Copy method for FastqBackend.
func (*FastqBackend) List ¶
func (fb *FastqBackend) List(path string) (fs.DirEntries, error)
List, Mkdir and Info are not supported for FastqBackend
func (*FastqBackend) Mkdir ¶
func (fb *FastqBackend) Mkdir(path string) error
type FileSystemInterface ¶
type FileSystemInterface interface {
Copy(otherFs FileSystemInterface, src, dst URI, isSelfSource bool) error
List(path string) (fs.DirEntries, error)
Mkdir(path string) error
Info(path string) (fs.DirEntry, error)
}
FileSystemInterface defines the methods for file operations.
type LocalBackend ¶
type LocalBackend struct {
RcloneBackend
// contains filtered or unexported fields
}
LocalBackend implements a local FileSystemInterface using rclone but is also capable of working with other backends
func NewLocalBackend ¶
func NewLocalBackend(ctx context.Context, component string) (*LocalBackend, error)
NewLocalBackend creates a new LocalBackend.
func (LocalBackend) AbsolutePath ¶
func (lb LocalBackend) AbsolutePath(path string) (string, error)
type MetaFileSystem ¶
type MetaFileSystem struct {
// contains filtered or unexported fields
}
MetaFileSystem provides common functionality for file operations using any FileSystemInterface.
func NewMetaFileSystem ¶
func NewMetaFileSystem(fs FileSystemInterface, err error) (*MetaFileSystem, error)
NewMetaFileSystem creates a new MetaFileSystem with the given FileSystemInterface.
func (*MetaFileSystem) RecursiveCopy ¶
func (mfs *MetaFileSystem) RecursiveCopy(otherFs MetaFileSystem, src, dst URI, isSelfSource bool) error
RecursiveCopy performs a recursive copy using the embedded FileSystemInterface.
type Operation ¶
type Operation struct {
// contains filtered or unexported fields
}
func NewOperation ¶
func NewOperation(remotes *pb.RcloneRemotes, srcStr, dstStr string) (*Operation, error)
type RcloneBackend ¶
type RcloneBackend struct {
// contains filtered or unexported fields
}
RcloneBackend implements the FileSystemInterface using rclone.
func NewRcloneBackend ¶
func NewRcloneBackend(ctx context.Context, remote string) (*RcloneBackend, error)
NewRcloneBackend creates a new RcloneBackend.
func (*RcloneBackend) Copy ¶
func (rb *RcloneBackend) Copy(otherFsInterface FileSystemInterface, src, dst URI, selfIsSource bool) error
Copy implements the Copy method for RcloneBackend.
func (*RcloneBackend) List ¶
func (rb *RcloneBackend) List(path string) (fs.DirEntries, error)
func (*RcloneBackend) Mkdir ¶
func (rb *RcloneBackend) Mkdir(path string) error
Mkdir implements the Mkdir method for RcloneBackend.
type URI ¶
type URI struct {
Proto string
Options []string
User string
Password string
Component string
Path string
File string
Separator string
Actions []string
}
func ParseURI ¶
ParseURI parses a URI using a regular expression and returns the appropriate backend and path.
Source Files