storage

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 1, 2020 License: MIT Imports: 28 Imported by: 3

Documentation

Overview

Package storage implements operations for s3 and fs.

Index

Constants

This section is empty.

Variables

View Source 
var (
	// ErrGivenObjectNotFound indicates a specified object is not found.
	ErrGivenObjectNotFound = fmt.Errorf("given object not found")

	// ErrNoObjectFound indicates there are no objects found from a given directory.
	ErrNoObjectFound = fmt.Errorf("no object found")
)

Functions

func Init 

func Init(opts S3Options) error

func IsCancelationError 

func IsCancelationError(err error) bool

func ShouldProcessUrl 

func ShouldProcessUrl(url *url.URL, followSymlinks bool) bool

ShouldProcessUrl returns true if follow symlinks is enabled. If follow symlinks is disabled we should not process the url. (this check is needed only for local files)

Types

type Bucket 

type Bucket struct {
	CreationDate time.Time `json:"created_at"`
	Name         string    `json:"name"`
}

Bucket is a container for storage objects.

func (Bucket) JSON 

func (b Bucket) JSON() string

JSON returns the JSON representation of Bucket.

func (Bucket) String 

func (b Bucket) String() string

String returns the string representation of Bucket.

type CustomRetryer 

type CustomRetryer struct {
	client.DefaultRetryer
}

CustomRetryer wraps the SDK's built in DefaultRetryer adding additional error codes. Such as, retry for S3 InternalError code.

func NewCustomRetryer 

func NewCustomRetryer(maxRetries int) *CustomRetryer

func (*CustomRetryer) ShouldRetry 

func (c *CustomRetryer) ShouldRetry(req *request.Request) bool

ShouldRetry overrides the SDK's built in DefaultRetryer adding customization to retry S3 InternalError code.

type Filesystem 

type Filesystem struct{}

func NewFilesystem 

func NewFilesystem() *Filesystem

func (*Filesystem) Copy 

func (f *Filesystem) Copy(ctx context.Context, src, dst *url.URL, _ map[string]string) error

func (*Filesystem) Delete 

func (f *Filesystem) Delete(ctx context.Context, url *url.URL) error

func (*Filesystem) Get 

func (f *Filesystem) Get(_ context.Context, _ *url.URL, _ io.WriterAt, _ int, _ int64) (int64, error)

func (*Filesystem) List 

func (f *Filesystem) List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

func (*Filesystem) ListBuckets 

func (f *Filesystem) ListBuckets(_ context.Context, _ string) ([]Bucket, error)

func (*Filesystem) MakeBucket 

func (f *Filesystem) MakeBucket(_ context.Context, _ string) error

func (*Filesystem) MultiDelete 

func (f *Filesystem) MultiDelete(ctx context.Context, urlch <-chan *url.URL) <-chan *Object

func (*Filesystem) Put 

func (f *Filesystem) Put(_ context.Context, _ io.Reader, _ *url.URL, _ map[string]string, _ int, _ int64) error

func (*Filesystem) Stat 

func (f *Filesystem) Stat(ctx context.Context, url *url.URL) (*Object, error)

type MockStorage 

type MockStorage struct {
	mock.Mock
}

MockStorage is an autogenerated mock type for the Storage type

func (*MockStorage) Copy 

func (_m *MockStorage) Copy(ctx context.Context, src *url.URL, dst *url.URL, metadata map[string]string) error

Copy provides a mock function with given fields: ctx, src, dst, metadata

func (*MockStorage) Delete 

func (_m *MockStorage) Delete(ctx context.Context, src *url.URL) error

Delete provides a mock function with given fields: ctx, src

func (*MockStorage) Get 

func (_m *MockStorage) Get(ctx context.Context, src *url.URL, dst io.WriterAt, concurrency int, partSize int64) (int64, error)

Get provides a mock function with given fields: ctx, src, dst, concurrency, partSize

func (*MockStorage) List 

func (_m *MockStorage) List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

List provides a mock function with given fields: ctx, src, followSymlinks

func (*MockStorage) ListBuckets 

func (_m *MockStorage) ListBuckets(ctx context.Context, prefix string) ([]Bucket, error)

ListBuckets provides a mock function with given fields: ctx, prefix

func (*MockStorage) MakeBucket 

func (_m *MockStorage) MakeBucket(ctx context.Context, bucket string) error

MakeBucket provides a mock function with given fields: ctx, bucket

func (*MockStorage) MultiDelete 

func (_m *MockStorage) MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object

MultiDelete provides a mock function with given fields: ctx, urls

func (*MockStorage) Put 

func (_m *MockStorage) Put(ctx context.Context, src io.Reader, dst *url.URL, metadata map[string]string, concurrency int, partSize int64) error

Put provides a mock function with given fields: ctx, src, dst, metadata, concurrency, partSize

func (*MockStorage) Stat 

func (_m *MockStorage) Stat(ctx context.Context, src *url.URL) (*Object, error)

Stat provides a mock function with given fields: ctx, src

type Object 

type Object struct {
	URL          *url.URL     `json:"key,omitempty"`
	Etag         string       `json:"etag,omitempty"`
	ModTime      *time.Time   `json:"last_modified,omitempty"`
	Type         ObjectType   `json:"type,omitempty"`
	Size         int64        `json:"size,omitempty"`
	StorageClass StorageClass `json:"storage_class,omitempty"`
	Err          error        `json:"error,omitempty"`
}

Object is a generic type which contains metadata for storage items.

func (*Object) JSON 

func (o *Object) JSON() string

JSON returns the JSON representation of Object.

func (*Object) String 

func (o *Object) String() string

String returns the string representation of Object.

type ObjectType 

type ObjectType struct {
	// contains filtered or unexported fields
}

ObjectType is the type of Object.

func (ObjectType) IsDir 

func (o ObjectType) IsDir() bool

IsDir checks if the object is a directory. 

func (o ObjectType) IsSymlink() bool

IsSymlink checks if the object is a symbolic link.

func (ObjectType) MarshalJSON 

func (o ObjectType) MarshalJSON() ([]byte, error)

MarshalJSON returns the stringer of ObjectType as a marshalled json.

func (ObjectType) String 

func (o ObjectType) String() string

String returns the string representation of ObjectType.

type S3 

type S3 struct {
	// contains filtered or unexported fields
}

S3 is a storage type which interacts with S3API, DownloaderAPI and UploaderAPI.

func NewS3Storage 

func NewS3Storage(opts S3Options) (*S3, error)

NewS3Storage creates new S3 session.

func (*S3) Copy 

func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata map[string]string) error

Copy is a single-object copy operation which copies objects to S3 destination from another S3 source.

func (*S3) Delete 

func (s *S3) Delete(ctx context.Context, url *url.URL) error

Delete is a single object delete operation.

func (*S3) Get 

func (s *S3) Get(
	ctx context.Context,
	from *url.URL,
	to io.WriterAt,
	concurrency int,
	partSize int64,
) (int64, error)

Get is a multipart download operation which downloads S3 objects into any destination that implements io.WriterAt interface. Makes a single 'GetObject' call if 'concurrency' is 1 and ignores 'partSize'.

func (*S3) List 

func (s *S3) List(ctx context.Context, url *url.URL, _ bool) <-chan *Object

List is a non-blocking S3 list operation which paginates and filters S3 keys. If no object found or an error is encountered during this period, it sends these errors to object channel.

func (*S3) ListBuckets 

func (s *S3) ListBuckets(ctx context.Context, prefix string) ([]Bucket, error)

ListBuckets is a blocking list-operation which gets bucket list and returns the buckets that match with given prefix.

func (*S3) MakeBucket 

func (s *S3) MakeBucket(ctx context.Context, name string) error

MakeBucket creates an S3 bucket with the given name.

func (*S3) MultiDelete 

func (s *S3) MultiDelete(ctx context.Context, urlch <-chan *url.URL) <-chan *Object

MultiDelete is a asynchronous removal operation for multiple objects. It reads given url channel, creates multiple chunks and run these chunks in parallel. Each chunk may have at most 1000 objects since DeleteObjects API has a limitation. See: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html.

func (*S3) Put 

func (s *S3) Put(
	ctx context.Context,
	reader io.Reader,
	to *url.URL,
	metadata map[string]string,
	concurrency int,
	partSize int64,
) error

Put is a multipart upload operation to upload resources, which implements io.Reader interface, into S3 destination.

func (*S3) Stat 

func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error)

Stat retrieves metadata from S3 object without returning the object itself.

type S3Options 

type S3Options struct {
	MaxRetries  int
	Endpoint    string
	Region      string
	NoVerifySSL bool
}

S3Options stores configuration for S3 storage.

type Storage 

type Storage interface {
	// Stat returns the Object structure describing object. If src is not
	// found, ErrGivenObjectNotFound is returned.
	Stat(ctx context.Context, src *url.URL) (*Object, error)

	// List the objects and directories/prefixes in the src.
	List(ctx context.Context, src *url.URL, followSymlinks bool) <-chan *Object

	// Copy src to dst, optionally setting the given metadata. Src and dst
	// arguments are of the same type. If src is a remote type, server side
	// copying will be used.
	Copy(ctx context.Context, src, dst *url.URL, metadata map[string]string) error

	// Get reads object content from src and writes to dst in parallel.
	Get(ctx context.Context, src *url.URL, dst io.WriterAt, concurrency int, partSize int64) (int64, error)

	// Put reads from src and writes content to dst.
	Put(ctx context.Context, src io.Reader, dst *url.URL, metadata map[string]string, concurrency int, partSize int64) error

	// Delete deletes the given src.
	Delete(ctx context.Context, src *url.URL) error

	// MultiDelete deletes all items returned from given urls in batches.
	MultiDelete(ctx context.Context, urls <-chan *url.URL) <-chan *Object

	// ListBuckets returns bucket list. If prefix is given, results will be
	// filtered.
	ListBuckets(ctx context.Context, prefix string) ([]Bucket, error)

	// MakeBucket creates given bucket.
	MakeBucket(ctx context.Context, bucket string) error
}

Storage is an interface for storage operations.

func NewClient 

func NewClient(url *url.URL) Storage

NewClient returns new Storage client from given url. Storage implementation is inferred from the url.

type StorageClass 

type StorageClass string
const (
	// StorageUnknown is a placeholder class if the class is not valid.
	StorageInvalid StorageClass = "UNKNOWN"

	// StorageStandard is a standard storage class type.
	StorageStandard StorageClass = "STANDARD"

	// StorageReducedRedundancy is a reduced redundancy storage class type.
	StorageReducedRedundancy StorageClass = "REDUCED_REDUNDANCY"

	// StorageGlacier is a glacier storage class type.
	StorageGlacier StorageClass = "GLACIER"

	// StorageStandardIA is a Standard Infrequent-Access storage
	// class type.
	StorageStandardIA StorageClass = "STANDARD_IA"
)

func LookupClass 

func LookupClass(s string) StorageClass

func (StorageClass) ShortCode 

func (s StorageClass) ShortCode() string

ShortCode returns the short code of Storage Class.

Source Files

View all Source files

Directories

Path Synopsis
Package url abstracts local and remote file URLs.
 Package url abstracts local and remote file URLs.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.