Documentation
¶
Overview ¶
Example (LocalFileSystem) ¶
// First, clear any files we may have there already
fmt.Printf("Setup: %v\n", os.RemoveAll("./test-output/"))
// Now run the tests
runTest(&FSAccess{}, "./test-output")
// NOTE: test output must match the output from S3 (except cleanup steps)
Output: Setup: <nil> JSON: <nil> JSON no-indent: <nil> Exists1: false|<nil> Binary: <nil> Exists2: true|<nil> Copy: <nil> Copy bad path, got not found error: true Read JSON: <nil>, {Hello 778 World} Read JSON no-indent: <nil>, {Hello 778 World} Read Binary: <nil>, [250 130 10 0 33] Read bad path, got not found error: true Read bad JSON: invalid character 'ú' looking for beginning of value Not a "not found" error: true Listing: <nil>, [the-files/data.bin the-files/pretty.json the-files/subdir/copied.json the-files/subdir/ugly.json] Listing subdir: <nil>, [the-files/subdir/copied.json the-files/subdir/ugly.json] Listing with prefix: <nil>, [the-files/subdir/ugly.json] Listing bad path: <nil>, [] Delete copy: <nil> Delete bin: <nil> Listing2: <nil>, [the-files/pretty.json the-files/subdir/ugly.json] Listing subdir2: <nil>, [the-files/subdir/ugly.json] Empty dir: <nil> Listing subdir3: <nil>, []
Example (S3) ¶
rand.Seed(time.Now().UnixNano())
sess, err := awsutil.GetSessionWithRegion("us-east-1")
if err != nil {
fmt.Println("Failed to get AWS session")
return
}
s3svc, err := awsutil.GetS3(sess)
if err != nil {
fmt.Println("Failed to get S3")
return
}
fmt.Printf("Setup: %v\n", err)
fs := MakeS3Access(s3svc)
// Create test S3 bucket for this purpose
testBucket := "api-fileaccess-s3-test-" + utils.RandStringBytesMaskImpr(10)
_, err = s3svc.CreateBucket(
&s3.CreateBucketInput{
Bucket: aws.String(testBucket),
//CreateBucketConfiguration:
},
)
if err != nil {
fmt.Printf("Failed to create test S3 bucket: %v\n", err)
return
}
defer func() {
_, err := s3svc.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(testBucket)})
fmt.Printf("Delete bucket errors: %v\n", err)
}()
// Now run the tests
runTest(fs, testBucket)
// NOTE: test output must match the output from local file system (except cleanup steps)
Output: Setup: <nil> JSON: <nil> JSON no-indent: <nil> Exists1: false|<nil> Binary: <nil> Exists2: true|<nil> Copy: <nil> Copy bad path, got not found error: true Read JSON: <nil>, {Hello 778 World} Read JSON no-indent: <nil>, {Hello 778 World} Read Binary: <nil>, [250 130 10 0 33] Read bad path, got not found error: true Read bad JSON: invalid character 'ú' looking for beginning of value Not a "not found" error: true Listing: <nil>, [the-files/data.bin the-files/pretty.json the-files/subdir/copied.json the-files/subdir/ugly.json] Listing subdir: <nil>, [the-files/subdir/copied.json the-files/subdir/ugly.json] Listing with prefix: <nil>, [the-files/subdir/ugly.json] Listing bad path: <nil>, [] Delete copy: <nil> Delete bin: <nil> Listing2: <nil>, [the-files/pretty.json the-files/subdir/ugly.json] Listing subdir2: <nil>, [the-files/subdir/ugly.json] Empty dir: <nil> Listing subdir3: <nil>, [] Delete bucket errors: <nil>
Index ¶
- func CopyFileLocally(srcPath string, dstPath string) error
- func IsValidObjectName(name string) bool
- func MakeEmptyLocalDirectory(root string, subdir string) (string, error)
- func MakeValidObjectName(name string, allowSpace bool) string
- type FSAccess
- func (fs *FSAccess) CopyObject(srcRootPath string, srcPath string, dstRootPath string, dstPath string) error
- func (fs *FSAccess) DeleteObject(rootPath string, path string) error
- func (fs *FSAccess) EmptyObjects(rootPath string) error
- func (fs *FSAccess) IsNotFoundError(err error) bool
- func (fs *FSAccess) ListObjects(rootPath string, prefix string) ([]string, error)
- func (fs *FSAccess) ObjectExists(rootPath string, path string) (bool, error)
- func (fs *FSAccess) ReadJSON(rootPath string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error
- func (fs *FSAccess) ReadObject(rootPath string, path string) ([]byte, error)
- func (fs *FSAccess) WriteJSON(rootPath string, s3Path string, itemsPtr interface{}) error
- func (fs *FSAccess) WriteJSONNoIndent(rootPath string, s3Path string, itemsPtr interface{}) error
- func (fs *FSAccess) WriteObject(rootPath string, path string, data []byte) error
- type FileAccess
- type Mock
- type S3Access
- func (s3Access S3Access) CopyObject(srcBucket string, srcPath string, dstBucket string, dstPath string) error
- func (s3Access S3Access) DeleteObject(bucket string, path string) error
- func (s3Access S3Access) EmptyObjects(targetBucket string) error
- func (s3Access S3Access) IsNotFoundError(err error) bool
- func (s3Access S3Access) ListObjects(bucket string, prefix string) ([]string, error)
- func (s3Access S3Access) ObjectExists(bucket string, path string) (bool, error)
- func (s3Access S3Access) ReadJSON(bucket string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error
- func (s3Access S3Access) ReadObject(bucket string, path string) ([]byte, error)
- func (s3Access S3Access) WriteJSON(bucket string, s3Path string, itemsPtr interface{}) error
- func (s3Access S3Access) WriteJSONNoIndent(bucket string, s3Path string, itemsPtr interface{}) error
- func (s3Access S3Access) WriteObject(bucket string, path string, data []byte) error
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CopyFileLocally ¶
func IsValidObjectName ¶
Is this string a valid name to use as an AWS object name?
func MakeEmptyLocalDirectory ¶
Creates a directory under the specified root, ensures it's empty (eg if it already existed)
func MakeValidObjectName ¶
Types ¶
type FSAccess ¶
type FSAccess struct {
}
Implementation of file access using local file system
func (*FSAccess) CopyObject ¶
func (*FSAccess) DeleteObject ¶
func (*FSAccess) EmptyObjects ¶
func (*FSAccess) IsNotFoundError ¶
func (*FSAccess) ListObjects ¶
func (*FSAccess) ObjectExists ¶
func (*FSAccess) ReadObject ¶
func (*FSAccess) WriteJSONNoIndent ¶
type FileAccess ¶
type FileAccess interface {
ListObjects(bucket string, prefix string) ([]string, error)
ObjectExists(rootPath string, path string) (bool, error)
ReadObject(bucket string, path string) ([]byte, error)
WriteObject(bucket string, path string, data []byte) error
ReadJSON(bucket string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error
WriteJSON(bucket string, s3Path string, itemsPtr interface{}) error
// A few places in the code need to write non-pretty-printed JSON for those
// files to work with Athena queries. Instead of adding a flag to WriteJSON
// this is easier to implement. Searching for WriteJSON still returns these!
WriteJSONNoIndent(bucket string, s3Path string, itemsPtr interface{}) error
DeleteObject(bucket string, path string) error
CopyObject(srcBucket string, srcPath string, dstBucket string, dstPath string) error
EmptyObjects(targetBucket string) error
IsNotFoundError(err error) bool
}
type S3Access ¶
type S3Access struct {
// contains filtered or unexported fields
}
Implementation of file access using AWS S3
func MakeS3Access ¶
func (S3Access) CopyObject ¶
func (S3Access) DeleteObject ¶
func (S3Access) EmptyObjects ¶
func (S3Access) IsNotFoundError ¶
func (S3Access) ListObjects ¶
ListObjects - calls AWS ListObjectsV2 and if a continuation token is returned this keeps looping and storing more items until no more continuation tokens are left.
func (S3Access) ObjectExists ¶
func (S3Access) ReadObject ¶
func (S3Access) WriteJSONNoIndent ¶
Click to show internal directories.
Click to hide internal directories.