Documentation
¶
Index ¶
- func Download(ctx context.Context, r Downloader, key string, fn string) error
- func DownloadAll(ctx context.Context, r Store, outDir string, prefix string, ...) ([]string, error)
- func Upload(ctx context.Context, r Uploader, fn string, key string) error
- func UploadAll(ctx context.Context, r Uploader, srcDir string, prefix string, ...) error
- func WithAllowFTP(req *Request)
- func WithAllowLocal(req *Request)
- func WithAllowS3(req *Request)
- func WithAuth(secret dmfr.Secret, auth dmfr.FeedAuthorization) func(req *Request)
- type Az
- func (r Az) CreateSignedUrl(ctx context.Context, key string, contentDisposition string) (string, error)
- func (r Az) Download(ctx context.Context, key string) (io.ReadCloser, int, error)
- func (r Az) DownloadAuth(ctx context.Context, key string, auth dmfr.FeedAuthorization) (io.ReadCloser, int, error)
- func (r Az) ListKeys(ctx context.Context, prefix string) ([]string, error)
- func (r *Az) SetSecret(secret dmfr.Secret) error
- func (r Az) Upload(ctx context.Context, key string, uploadFile io.Reader) error
- type CanSetSecret
- type Downloader
- type FetchResponse
- type Ftp
- type Http
- type Local
- func (r Local) Download(ctx context.Context, key string) (io.ReadCloser, int, error)
- func (r Local) DownloadAuth(ctx context.Context, key string, auth dmfr.FeedAuthorization) (io.ReadCloser, int, error)
- func (r Local) Exists(ctx context.Context, key string) bool
- func (r *Local) ListKeys(ctx context.Context, prefix string) ([]string, error)
- func (r *Local) SetSecret(secret dmfr.Secret) error
- func (r Local) Upload(ctx context.Context, key string, uploadFile io.Reader) error
- type Presigner
- type Request
- type RequestOption
- type S3
- func (r S3) CreateSignedUrl(ctx context.Context, key string, contentDisposition string) (string, error)
- func (r S3) Download(ctx context.Context, key string) (io.ReadCloser, int, error)
- func (r S3) DownloadAuth(ctx context.Context, key string, auth dmfr.FeedAuthorization) (io.ReadCloser, int, error)
- func (r S3) ListKeys(ctx context.Context, prefix string) ([]string, error)
- func (r *S3) SetAcl(acl string)
- func (r *S3) SetSecret(secret dmfr.Secret) error
- func (r S3) Upload(ctx context.Context, key string, uploadFile io.Reader) error
- type Store
- type Uploader
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func DownloadAll ¶ added in v1.0.0
func WithAllowFTP ¶
func WithAllowFTP(req *Request)
func WithAllowLocal ¶
func WithAllowLocal(req *Request)
func WithAllowS3 ¶
func WithAllowS3(req *Request)
Types ¶
type Az ¶
func NewAzFromUrl ¶
func (Az) CreateSignedUrl ¶
func (Az) DownloadAuth ¶ added in v1.0.0
func (r Az) DownloadAuth(ctx context.Context, key string, auth dmfr.FeedAuthorization) (io.ReadCloser, int, error)
type CanSetSecret ¶ added in v1.0.0
type Downloader ¶
type FetchResponse ¶
type FetchResponse struct {
ResponseSize int
ResponseCode int
ResponseTimeMs int
ResponseTtfbMs int
ResponseSHA1 string
FetchError error
}
func AuthenticatedRequest ¶
func AuthenticatedRequest(ctx context.Context, out io.Writer, address string, opts ...RequestOption) (FetchResponse, error)
AuthenticatedRequestContext fetches a url using a secret and auth description.
func AuthenticatedRequestDownload ¶
func AuthenticatedRequestDownload(ctx context.Context, address string, opts ...RequestOption) (string, FetchResponse, error)
AuthenticatedRequestDownload is similar to AuthenticatedRequest but writes to a temporary file. Fatal errors will be returned as the error; non-fatal errors as FetchResponse.FetchError
type Ftp ¶
type Ftp struct {
// contains filtered or unexported fields
}
func (Ftp) DownloadAuth ¶ added in v1.0.0
func (r Ftp) DownloadAuth(ctx context.Context, ustr string, auth dmfr.FeedAuthorization) (io.ReadCloser, int, error)
type Http ¶
type Http struct {
// MaxRetries sets the maximum number of retry attempts for a request.
// If MaxRetries is zero or negative, a default value (defaultMaxRetries) is used.
MaxRetries int
// BackoffSchedule defines the backoff duration for each retry attempt.
// If nil or empty, defaultBackoffSchedule is used.
BackoffSchedule []time.Duration
// contains filtered or unexported fields
}
func (Http) DownloadAuth ¶ added in v1.0.0
func (r Http) DownloadAuth(ctx context.Context, ustr string, auth dmfr.FeedAuthorization) (io.ReadCloser, int, error)
type Local ¶
type Local struct {
Directory string
}
func (Local) DownloadAuth ¶ added in v1.0.0
func (r Local) DownloadAuth(ctx context.Context, key string, auth dmfr.FeedAuthorization) (io.ReadCloser, int, error)
type Request ¶
type Request struct {
URL string
AllowFTP bool
AllowLocal bool
AllowS3 bool
MaxSize uint64
Secret dmfr.Secret
Auth dmfr.FeedAuthorization
}
func NewRequest ¶
func NewRequest(address string, opts ...RequestOption) *Request
type RequestOption ¶
type RequestOption func(*Request)
func WithMaxSize ¶
func WithMaxSize(s uint64) RequestOption
type S3 ¶
func NewS3FromUrl ¶
func (S3) CreateSignedUrl ¶
func (S3) DownloadAuth ¶ added in v1.0.0
func (r S3) DownloadAuth(ctx context.Context, key string, auth dmfr.FeedAuthorization) (io.ReadCloser, int, error)
type Store ¶
type Store interface {
Downloader
Uploader
CanSetSecret
ListKeys(ctx context.Context, prefix string) ([]string, error)
}
Click to show internal directories.
Click to hide internal directories.