Documentation
¶
Index ¶
- Constants
- Variables
- type AbstractFileAdapter
- type DummyFlattener
- type FileAdapter
- type FileConfig
- type Flattener
- type FlattenerImpl
- type GoogleCloudStorage
- func (gcs *GoogleCloudStorage) Close() error
- func (gcs *GoogleCloudStorage) DeleteObject(key string) (err error)
- func (gcs *GoogleCloudStorage) Download(key string) (fileBytes []byte, err error)
- func (gcs *GoogleCloudStorage) GetObjectURL(fileName string) (string, error)
- func (gcs *GoogleCloudStorage) Ping() error
- func (gcs *GoogleCloudStorage) Type() string
- func (gcs *GoogleCloudStorage) Upload(fileName string, fileReader io.ReadSeeker) (err error)
- func (gcs *GoogleCloudStorage) UploadBytes(fileName string, fileBytes []byte) error
- func (gcs *GoogleCloudStorage) ValidateWritePermission() error
- type GoogleConfig
- type S3
- func (a *S3) Close() error
- func (a *S3) DeleteObject(key string) error
- func (a *S3) Download(fileName string) ([]byte, error)
- func (a *S3) GetObjectURL(fileName string) (string, error)
- func (a *S3) Ping() (err error)
- func (a *S3) Type() string
- func (a *S3) Upload(fileName string, fileReader io.ReadSeeker) error
- func (a *S3) UploadBytes(fileName string, fileBytes []byte) error
- func (a *S3) ValidateWritePermission() error
- type S3Config
Constants ¶
const GCSBulkerTypeId = "gcs"
const (
S3BulkerTypeId = "s3"
)
Variables ¶
var ErrMalformedBQDataset = errors.New("bq_dataset must be alphanumeric (plus underscores) and must be at most 1024 characters long")
Functions ¶
This section is empty.
Types ¶
type AbstractFileAdapter ¶
type AbstractFileAdapter struct {
// contains filtered or unexported fields
}
func (*AbstractFileAdapter) AddFileExtension ¶
func (a *AbstractFileAdapter) AddFileExtension(fileName string) string
func (*AbstractFileAdapter) Compression ¶
func (a *AbstractFileAdapter) Compression() types.FileCompression
func (*AbstractFileAdapter) Format ¶
func (a *AbstractFileAdapter) Format() types.FileFormat
func (*AbstractFileAdapter) Path ¶
func (a *AbstractFileAdapter) Path(fileName string) string
type DummyFlattener ¶
type DummyFlattener struct {
}
func NewDummyFlattener ¶
func NewDummyFlattener() *DummyFlattener
func (*DummyFlattener) FlattenObject ¶
func (df *DummyFlattener) FlattenObject(object map[string]any, sqlTypeHints types.SQLTypes) (map[string]any, error)
FlattenObject return the same json object
type FileAdapter ¶
type FileAdapter interface {
io.Closer
Type() string
Ping() error
UploadBytes(fileName string, fileBytes []byte) error
Upload(fileName string, fileReader io.ReadSeeker) error
// GetObjectURL returns http URL to uploaded object.
// It is short-lived signed URL if the corresponding setting (e.g. S3Config.UsePresignedURL ) is enabled on adapter config
// Otherwise, it returns URL build with public URL template which won't work if the object is private
GetObjectURL(fileName string) (url string, err error)
Download(fileName string) ([]byte, error)
DeleteObject(key string) error
Path(fileName string) string
AddFileExtension(fileName string) string
Format() types.FileFormat
Compression() types.FileCompression
}
type FileConfig ¶
type FileConfig struct {
Folder string `mapstructure:"folder" json:"folder,omitempty" yaml:"folder,omitempty"`
Format types.FileFormat `mapstructure:"format,omitempty" json:"format,omitempty" yaml:"format,omitempty"`
Compression types.FileCompression `mapstructure:"compression,omitempty" json:"compression,omitempty" yaml:"compression,omitempty"`
}
type Flattener ¶
type FlattenerImpl ¶
type FlattenerImpl struct {
// contains filtered or unexported fields
}
func (*FlattenerImpl) FlattenObject ¶
func (f *FlattenerImpl) FlattenObject(object types.Object, notFlatteningKeys types2.Set[string]) (types.Object, error)
FlattenObject flatten object e.g. from {"key1":{"key2":123}} to {"key1_key2":123} from {"$key1":1} to {"_key1":1} from {"(key1)":1} to {"_key1_":1}
type GoogleCloudStorage ¶
type GoogleCloudStorage struct {
AbstractFileAdapter
// contains filtered or unexported fields
}
func NewGoogleCloudStorage ¶
func NewGoogleCloudStorage(config *GoogleConfig) (*GoogleCloudStorage, error)
func (*GoogleCloudStorage) Close ¶
func (gcs *GoogleCloudStorage) Close() error
Close closes gcp client and returns err if occurred
func (*GoogleCloudStorage) DeleteObject ¶
func (gcs *GoogleCloudStorage) DeleteObject(key string) (err error)
DeleteObject deletes object from google cloud storage bucket
func (*GoogleCloudStorage) Download ¶
func (gcs *GoogleCloudStorage) Download(key string) (fileBytes []byte, err error)
Download downloads file from google cloud storage bucket
func (*GoogleCloudStorage) GetObjectURL ¶
func (gcs *GoogleCloudStorage) GetObjectURL(fileName string) (string, error)
func (*GoogleCloudStorage) Ping ¶
func (gcs *GoogleCloudStorage) Ping() error
func (*GoogleCloudStorage) Type ¶
func (gcs *GoogleCloudStorage) Type() string
func (*GoogleCloudStorage) Upload ¶
func (gcs *GoogleCloudStorage) Upload(fileName string, fileReader io.ReadSeeker) (err error)
UploadBytes creates named file on google cloud storage with payload
func (*GoogleCloudStorage) UploadBytes ¶
func (gcs *GoogleCloudStorage) UploadBytes(fileName string, fileBytes []byte) error
func (*GoogleCloudStorage) ValidateWritePermission ¶
func (gcs *GoogleCloudStorage) ValidateWritePermission() error
ValidateWritePermission tries to create temporary file and remove it. returns nil if file creation was successful.
type GoogleConfig ¶
type GoogleConfig struct {
FileConfig `mapstructure:",squash" json:",inline" yaml:",inline"`
Bucket string `mapstructure:"gcsBucket,omitempty" json:"gcsBucket,omitempty" yaml:"gcsBucket,omitempty"`
Project string `mapstructure:"project,omitempty" json:"project,omitempty" yaml:"project,omitempty"`
Dataset string `mapstructure:"bqDataset,omitempty" json:"bqDataset,omitempty" yaml:"bqDataset,omitempty"`
KeyFile any `mapstructure:"keyFile,omitempty" json:"keyFile,omitempty" yaml:"keyFile,omitempty"`
//will be set on validation
Credentials option.ClientOption
}
func (*GoogleConfig) Validate ¶
func (gc *GoogleConfig) Validate() error
type S3 ¶
type S3 struct {
AbstractFileAdapter
// contains filtered or unexported fields
}
S3 is a S3 adapter for uploading/deleting files
func (*S3) DeleteObject ¶
DeleteObject deletes object from s3 bucket by key
func (*S3) Upload ¶
func (a *S3) Upload(fileName string, fileReader io.ReadSeeker) error
Upload creates named file on s3 with payload
func (*S3) ValidateWritePermission ¶
ValidateWritePermission tries to create temporary file and remove it. returns nil if file creation was successful.
type S3Config ¶
type S3Config struct {
FileConfig `mapstructure:",squash" json:",inline" yaml:",inline"`
AuthenticationMethod string `mapstructure:"authenticationMethod,omitempty" json:"authenticationMethod,omitempty" yaml:"authenticationMethod,omitempty"`
AccessKeyID string `mapstructure:"accessKeyId,omitempty" json:"accessKeyId,omitempty" yaml:"accessKeyId,omitempty"`
SecretAccessKey string `mapstructure:"secretAccessKey,omitempty" json:"secretAccessKey,omitempty" yaml:"secretAccessKey,omitempty"`
Bucket string `mapstructure:"bucket,omitempty" json:"bucket,omitempty" yaml:"bucket,omitempty"`
Region string `mapstructure:"region,omitempty" json:"region,omitempty" yaml:"region,omitempty"`
Endpoint string `mapstructure:"endpoint,omitempty" json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
UsePresignedURL bool `mapstructure:"usePresignedURL,omitempty" json:"usePresignedURL,omitempty" yaml:"usePresignedURL,omitempty"`
RoleARN string `mapstructure:"roleARN" json:"roleARN" yaml:"roleARN"`
RoleARNExpiry time.Duration `json:"roleARNExpiry"` // default: 15m
ExternalID string `mapstructure:"externalID" json:"externalID" yaml:"externalID"`
}
S3Config is a dto for config deserialization