Documentation
¶
Overview ¶
Package storage provides basic storage interfaces for storage providers to write / read objects to and from
Index ¶
- Constants
- Variables
- func DetectContentType(reader io.ReadSeeker) (string, error)
- func ParseDocument(reader io.Reader, mimeType string) (any, error)
- type CredentialSyncConfig
- type Disk
- func (d *Disk) Close() error
- func (d *Disk) Delete(_ context.Context, key string) error
- func (d *Disk) Download(_ context.Context, opts *objects.DownloadFileOptions) (*objects.DownloadFileMetadata, error)
- func (d *Disk) GetPresignedURL(key string, _ time.Duration) (string, error)
- func (d *Disk) GetScheme() *string
- func (d *Disk) ListBuckets() ([]string, error)
- func (d *Disk) ManagerUpload(_ context.Context, files [][]byte) error
- func (d *Disk) Upload(_ context.Context, r io.Reader, opts *objects.UploadFileOptions) (*objects.UploadedFileMetadata, error)
- type DiskOption
- type DiskOptions
- type DownloadOptions
- type DownloadedMetadata
- type ErrResponseHandler
- type File
- type FileMetadata
- type Files
- type NameGeneratorFunc
- type ObjectService
- func (s *ObjectService) Delete(ctx context.Context, provider Provider, file *storagetypes.File, ...) error
- func (s *ObjectService) Download(ctx context.Context, provider Provider, file *storagetypes.File, ...) (*DownloadedMetadata, error)
- func (s *ObjectService) ErrorResponseHandler() ErrResponseHandler
- func (s *ObjectService) GetPresignedURL(ctx context.Context, provider Provider, file *storagetypes.File, ...) (string, error)
- func (s *ObjectService) IgnoreNonExistentKeys() bool
- func (s *ObjectService) Keys() []string
- func (s *ObjectService) MaxMemory() int64
- func (s *ObjectService) MaxSize() int64
- func (s *ObjectService) Skipper() SkipperFunc
- func (s *ObjectService) Upload(ctx context.Context, provider Provider, reader io.Reader, opts *UploadOptions) (*File, error)
- func (s *ObjectService) WithUploader(uploader UploaderFunc) *ObjectService
- func (s *ObjectService) WithValidation(validationFunc ValidationFunc) *ObjectService
- type ParentObject
- type Provider
- type ProviderConfig
- type ProviderConfigs
- type ProviderCredentials
- type ProviderHints
- type ProviderOption
- func WithBasePath(path string) ProviderOption
- func WithBucket(bucket string) ProviderOption
- func WithCredentials(creds ProviderCredentials) ProviderOption
- func WithEndpoint(endpoint string) ProviderOption
- func WithExtra(key string, value any) ProviderOption
- func WithLocalURL(url string) ProviderOption
- func WithRegion(region string) ProviderOption
- type ProviderOptions
- type ProviderType
- type Providers
- type S3Option
- func WithS3AWSConfig(cfg aws.Config) S3Option
- func WithS3AccessKeyID(accessKeyID string) S3Option
- func WithS3Bucket(bucket string) S3Option
- func WithS3Endpoint(endpoint string) S3Option
- func WithS3PathStyle(v bool) S3Option
- func WithS3PresignedURLTimeout(timeout int) S3Option
- func WithS3Region(region string) S3Option
- func WithS3SecretAccessKey(secretAccessKey string) S3Option
- func WithS3UseSSL(useSSL bool) S3Option
- type S3Options
- type S3Store
- func (s *S3Store) Close() error
- func (s *S3Store) Delete(ctx context.Context, key string) error
- func (s *S3Store) Download(ctx context.Context, opts *objects.DownloadFileOptions) (*objects.DownloadFileMetadata, error)
- func (s *S3Store) Exists(ctx context.Context, key string) (bool, error)
- func (s *S3Store) GetPresignedURL(key string, expires time.Duration) (string, error)
- func (s *S3Store) GetScheme() *string
- func (s *S3Store) GetTags(ctx context.Context, key string) (map[string]string, error)
- func (s *S3Store) HeadObj(ctx context.Context, key string) (*s3.HeadObjectOutput, error)
- func (s *S3Store) ListBuckets() ([]string, error)
- func (s *S3Store) ManagerUpload(ctx context.Context, files [][]byte) error
- func (s *S3Store) Tag(ctx context.Context, key string, tags map[string]string) error
- func (s *S3Store) Upload(ctx context.Context, r io.Reader, opts *objects.UploadFileOptions) (*objects.UploadedFileMetadata, error)
- type SkipperFunc
- type UploadOptions
- type UploadedMetadata
- type UploaderFunc
- type ValidationFunc
Constants ¶
const ( S3Provider = storagetypes.S3Provider R2Provider = storagetypes.R2Provider GCSProvider = storagetypes.GCSProvider DiskProvider = storagetypes.DiskProvider DatabaseProvider = storagetypes.DatabaseProvider )
Provider type constants so we can range, switch, etc
const ( DefaultMaxFileSize = 32 << 20 // 32MB DefaultMaxMemory = 32 << 20 // 32MB DefaultUploadFileKey = "uploadFile" )
Configuration constants
const (
// ErrNoFilesUploaded is returned when no files were uploaded in the request
ErrNoFilesUploaded = errorMsg("objects: no uploadable files found in request")
)
const (
// MIMEDetectionBufferSize defines the buffer size for MIME type detection
MIMEDetectionBufferSize = 512
)
Variables ¶
var ( // ErrInvalidS3Bucket is returned when an invalid s3 bucket is provided ErrInvalidS3Bucket = errors.New("invalid s3 bucket provided") // ErrInvalidFolderPath is returned when an invalid folder path is provided ErrInvalidFolderPath = errors.New("invalid folder path provided") // ErrMissingRequiredAWSParams is returned when required AWS parameters are missing ErrMissingRequiredAWSParams = errors.New("missing required AWS parameters") // ErrMissingLocalURL = errors.New("missing local URL in disk storage options" ErrMissingLocalURL = errors.New("missing local URL in disk storage options") // ErrJSONParseFailed is returned when JSON parsing fails ErrJSONParseFailed = errors.New("failed to parse JSON") // ErrYAMLParseFailed is returned when YAML parsing fails ErrYAMLParseFailed = errors.New("failed to parse YAML") // ErrProviderResolutionRequired is returned when provider resolution is required but not available ErrProviderResolutionRequired = errors.New("provider resolution required - use external orchestration layer") )
var ( DefaultValidationFunc ValidationFunc = func(_ File) error { return nil } DefaultNameGeneratorFunc = func(originalName string) string { return originalName } DefaultSkipper = func(_ *http.Request) bool { return false } DefaultErrorResponseHandler = func(err error, statusCode int) http.HandlerFunc { return func(w http.ResponseWriter, _ *http.Request) { http.Error(w, err.Error(), statusCode) } } )
Default function implementations
var ProviderDisk = "disk"
ProviderDisk is the provider for the disk storage
var ProviderS3 = "s3"
ProviderS3 is the provider for the S3 storage
Functions ¶
func DetectContentType ¶ added in v0.38.1
func DetectContentType(reader io.ReadSeeker) (string, error)
DetectContentType detects the MIME type of the provided reader using gabriel-vasile/mimetype library
Types ¶
type CredentialSyncConfig ¶ added in v0.38.1
type CredentialSyncConfig struct {
// Enabled indicates whether credential synchronization runs on startup
Enabled bool `json:"enabled" koanf:"enabled" default:"true"`
}
CredentialSyncConfig controls whether provider credentials are synchronized into the database
type Disk ¶
type Disk struct {
// Scheme is the scheme of the storage backend
Scheme string
// Opts is the options for the disk storage
Opts *DiskOptions
// contains filtered or unexported fields
}
func NewDiskStorage ¶
func NewDiskStorage(opts *DiskOptions) (*Disk, error)
NewDiskStorage creates a new Disk storage backend
func (*Disk) Download ¶
func (d *Disk) Download(_ context.Context, opts *objects.DownloadFileOptions) (*objects.DownloadFileMetadata, error)
Download is used to download a file from the storage backend
func (*Disk) GetPresignedURL ¶
GetPresignedURL is used to get a presigned URL for a file in the storage backend
func (*Disk) ListBuckets ¶ added in v0.3.1
ListBuckets lists the local bucket if it exists
func (*Disk) ManagerUpload ¶
ManagerUpload uploads multiple files to disk
type DiskOption ¶ added in v0.3.1
type DiskOption func(*DiskOptions)
DiskOption is an options function for the DiskOptions
func WithDiskLocalBucket ¶ added in v0.38.1
func WithDiskLocalBucket(bucket string) DiskOption
WithLocalBucket is a DiskOption that sets the bucket for the disk storage which equates to a folder on the file system
func WithDiskLocalKey ¶ added in v0.38.1
func WithDiskLocalKey(key string) DiskOption
WithLocalKey specifies the name of the file in the local folder
func WithDiskLocalURL ¶ added in v0.38.1
func WithDiskLocalURL(url string) DiskOption
WithLocalURL specifies the URL to use for the "presigned" URL for the file
type DiskOptions ¶ added in v0.3.1
type DiskOptions struct {
Bucket string
Key string
// LocalURL is the URL to use for the "presigned" URL for the file
// e.g for local development, this can be http://localhost:17608/files/
LocalURL string
}
DiskOptions are options for the disk storage
func NewDiskOptions ¶ added in v0.3.1
func NewDiskOptions(opts ...DiskOption) *DiskOptions
NewDiskOptions returns a new DiskOptions struct
type DownloadOptions ¶ added in v0.38.1
type DownloadOptions = storagetypes.DownloadFileOptions
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type DownloadedMetadata ¶ added in v0.38.1
type DownloadedMetadata = storagetypes.DownloadedFileMetadata
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type ErrResponseHandler ¶ added in v0.38.1
type ErrResponseHandler func(err error, statusCode int) http.HandlerFunc
ErrResponseHandler is a custom error handler for upload failures
type File ¶ added in v0.38.1
type File = storagetypes.File
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
func NewUploadFile ¶ added in v0.38.1
NewUploadFile creates a new File from a file path
type FileMetadata ¶ added in v0.38.1
type FileMetadata = storagetypes.FileMetadata
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type NameGeneratorFunc ¶ added in v0.38.1
NameGeneratorFunc generates names for uploaded files
type ObjectService ¶ added in v0.38.1
type ObjectService struct {
// contains filtered or unexported fields
}
ObjectService provides pure object management functionality without provider resolution
func NewObjectService ¶ added in v0.38.1
func NewObjectService() *ObjectService
NewObjectService creates a new object service instance with default configuration
func (*ObjectService) Delete ¶ added in v0.38.1
func (s *ObjectService) Delete(ctx context.Context, provider Provider, file *storagetypes.File, opts *storagetypes.DeleteFileOptions) error
Delete deletes a file using a specific storage provider client
func (*ObjectService) Download ¶ added in v0.38.1
func (s *ObjectService) Download(ctx context.Context, provider Provider, file *storagetypes.File, opts *DownloadOptions) (*DownloadedMetadata, error)
Download downloads a file using a specific storage provider client
func (*ObjectService) ErrorResponseHandler ¶ added in v0.38.1
func (s *ObjectService) ErrorResponseHandler() ErrResponseHandler
ErrorResponseHandler returns the configured error response handler
func (*ObjectService) GetPresignedURL ¶ added in v0.38.1
func (s *ObjectService) GetPresignedURL(ctx context.Context, provider Provider, file *storagetypes.File, opts *storagetypes.PresignedURLOptions) (string, error)
GetPresignedURL gets a presigned URL for a file using a specific storage provider client
func (*ObjectService) IgnoreNonExistentKeys ¶ added in v0.38.1
func (s *ObjectService) IgnoreNonExistentKeys() bool
IgnoreNonExistentKeys returns whether to ignore non-existent form keys
func (*ObjectService) Keys ¶ added in v0.38.1
func (s *ObjectService) Keys() []string
Keys returns the configured form keys
func (*ObjectService) MaxMemory ¶ added in v0.38.1
func (s *ObjectService) MaxMemory() int64
MaxMemory returns the configured maximum memory for multipart forms
func (*ObjectService) MaxSize ¶ added in v0.38.1
func (s *ObjectService) MaxSize() int64
MaxSize returns the configured maximum file size
func (*ObjectService) Skipper ¶ added in v0.38.1
func (s *ObjectService) Skipper() SkipperFunc
Skipper returns the configured skipper function
func (*ObjectService) Upload ¶ added in v0.38.1
func (s *ObjectService) Upload(ctx context.Context, provider Provider, reader io.Reader, opts *UploadOptions) (*File, error)
Upload uploads a file using a specific storage provider client
func (*ObjectService) WithUploader ¶ added in v0.38.1
func (s *ObjectService) WithUploader(uploader UploaderFunc) *ObjectService
WithUploader returns a new ObjectService with the specified uploader function
func (*ObjectService) WithValidation ¶ added in v0.38.1
func (s *ObjectService) WithValidation(validationFunc ValidationFunc) *ObjectService
WithValidation returns a new ObjectService with the specified validation function
type ParentObject ¶ added in v0.38.1
type ParentObject = storagetypes.ParentObject
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type Provider ¶ added in v0.38.1
type Provider = storagetypes.Provider
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type ProviderConfig ¶ added in v0.38.1
type ProviderConfig struct {
// Enabled indicates if object storage is enabled
Enabled bool `json:"enabled" koanf:"enabled" default:"true"`
// Keys are the form field keys that will be processed for uploads
Keys []string `json:"keys" koanf:"keys" default:"[\"uploadFile\"]"`
// MaxSizeMB is the maximum file size allowed in MB
MaxSizeMB int64 `json:"maxSizeMB" koanf:"maxSizeMB"`
// MaxMemoryMB is the maximum memory to use for file uploads in MB
MaxMemoryMB int64 `json:"maxMemoryMB" koanf:"maxMemoryMB"`
// DevMode enables simple file upload handling for local development and testing
DevMode bool `json:"devMode" koanf:"devMode" default:"false"`
// EnsureAvailable enforces provider availability before completing server startup
EnsureAvailable bool `json:"ensureAvailable" koanf:"ensureAvailable" default:"false"`
// CredentialSync contains options for synchronizing provider credentials into the database
CredentialSync CredentialSyncConfig `json:"credentialSync" koanf:"credentialSync"`
// Providers contains configuration for each storage provider
Providers Providers `json:"providers" koanf:"providers"`
}
ProviderConfig contains configuration for object storage providers
type ProviderConfigs ¶ added in v0.38.1
type ProviderConfigs struct {
// Enabled indicates if this provider is enabled
Enabled bool `json:"enabled" koanf:"enabled"`
// Region for cloud providers
Region string `json:"region" koanf:"region"`
// Bucket name for cloud providers
Bucket string `json:"bucket" koanf:"bucket"`
// Endpoint for custom endpoints
Endpoint string `json:"endpoint" koanf:"endpoint"`
// Credentials contains the credentials for accessing the provider
Credentials ProviderCredentials `json:"credentials" koanf:"credentials"`
}
ProviderConfigs contains configuration for all storage providers This is structured to allow easy extension for additional providers in the future
type ProviderCredentials ¶ added in v0.38.1
type ProviderCredentials struct {
// AccessKeyID for cloud providers
AccessKeyID string `json:"accessKeyID" koanf:"accessKeyID"`
// SecretAccessKey for cloud providers
SecretAccessKey string `json:"secretAccessKey" koanf:"accessKeySecret"`
// Endpoint for custom endpoints
Endpoint string `json:"endpoint" koanf:"endpoint"`
// ProjectID for GCS
ProjectID string `json:"projectID" koanf:"projectID"`
// AccountID for Cloudflare R2
AccountID string `json:"accountID" koanf:"accountID"`
// APIToken for Cloudflare R2
APIToken string `json:"apiToken" koanf:"apiToken"`
}
ProviderCredentials contains credentials for a storage provider
type ProviderHints ¶ added in v0.38.1
type ProviderHints = storagetypes.ProviderHints
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type ProviderOption ¶ added in v0.38.1
type ProviderOption func(*ProviderOptions)
ProviderOption configures runtime provider options
func WithBasePath ¶ added in v0.38.1
func WithBasePath(path string) ProviderOption
WithBasePath sets the local base path for disk providers
func WithBucket ¶ added in v0.3.1
func WithBucket(bucket string) ProviderOption
WithBucket sets the bucket/path value
func WithCredentials ¶ added in v0.38.1
func WithCredentials(creds ProviderCredentials) ProviderOption
WithCredentials sets provider credentials
func WithEndpoint ¶ added in v0.3.1
func WithEndpoint(endpoint string) ProviderOption
WithEndpoint sets the custom endpoint
func WithExtra ¶ added in v0.38.1
func WithExtra(key string, value any) ProviderOption
WithExtra attaches provider specific metadata
func WithLocalURL ¶ added in v0.6.10
func WithLocalURL(url string) ProviderOption
WithLocalURL sets the local URL used for presigned links
func WithRegion ¶ added in v0.3.1
func WithRegion(region string) ProviderOption
WithRegion sets the region value
type ProviderOptions ¶ added in v0.38.1
type ProviderOptions struct {
Credentials ProviderCredentials
Bucket string
Region string
Endpoint string
BasePath string
LocalURL string
// contains filtered or unexported fields
}
ProviderOptions captures runtime configuration shared across storage providers
func NewProviderOptions ¶ added in v0.38.1
func NewProviderOptions(opts ...ProviderOption) *ProviderOptions
NewProviderOptions constructs ProviderOptions applying the supplied options
func (*ProviderOptions) Apply ¶ added in v0.38.1
func (p *ProviderOptions) Apply(opts ...ProviderOption)
Apply applies option functions to ProviderOptions
func (*ProviderOptions) Clone ¶ added in v0.38.1
func (p *ProviderOptions) Clone() *ProviderOptions
Clone returns a deep copy of ProviderOptions
type ProviderType ¶ added in v0.38.1
type ProviderType = storagetypes.ProviderType
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type Providers ¶ added in v0.38.1
type Providers struct {
// S3 provider configuration
S3 ProviderConfigs `json:"s3" koanf:"s3"`
// CloudflareR2 provider configuration
CloudflareR2 ProviderConfigs `json:"cloudflareR2" koanf:"cloudflareR2"`
// GCS provider configuration
GCS ProviderConfigs `json:"gcs" koanf:"gcs"`
// Disk provider configuration
Disk ProviderConfigs `json:"disk" koanf:"disk"`
// Database provider configuration
Database ProviderConfigs `json:"database" koanf:"database"`
}
type S3Option ¶ added in v0.3.1
type S3Option func(*S3Options)
S3Option is a function that modifies S3Options
func WithS3AWSConfig ¶ added in v0.38.1
WithAWSConfig sets the AWS configuration for S3Options
func WithS3AccessKeyID ¶ added in v0.38.1
WithAccessKeyID sets the access key ID for S3Options
func WithS3Bucket ¶ added in v0.38.1
WithBucket sets the bucket for S3Options
func WithS3Endpoint ¶ added in v0.38.1
WithEndpoint sets the endpoint for S3Options
func WithS3PathStyle ¶ added in v0.38.1
WithPathStyle allows you set the path style. This is useful for other compatible s3 storage systems
func WithS3PresignedURLTimeout ¶ added in v0.38.1
WithPresignedURLTimeout sets the presigned URL timeout for S3Options
func WithS3Region ¶ added in v0.38.1
WithRegion sets the region for S3Options
func WithS3SecretAccessKey ¶ added in v0.38.1
WithSecretAccessKey sets the secret access key for S3Options
func WithS3UseSSL ¶ added in v0.38.1
WithUseSSL sets the use SSL flag for S3Options
type S3Options ¶
type S3Options struct {
// Bucket to store objects in
Bucket string
// DebugMode will log all requests and responses
DebugMode bool
// UsePathStyle allows you to enable the client to use path-style addressing, i.e., https://s3.amazonaws.com/BUCKET/KEY .
// by default, the S3 client will use virtual hosted bucket addressing when possible( https://BUCKET.s3.amazonaws.com/KEY ).
UsePathStyle bool
// ACL should only be used if the bucket supports ACL
ACL types.ObjectCannedACL
// Region is the region to use for the S3 client
Region string
// AccessKeyID is the access key ID to use for the S3 client
AccessKeyID string
// SecretAccessKey is the secret access key to use for the S3 client
SecretAccessKey string
// Endpoint is the endpoint to use for the S3 client
Endpoint string
// UseSSL is a flag to determine if the S3 client should use SSL
UseSSL bool
// PresignURLTimeout is the timeout for presigned URLs
PresignedURLTimeout int
// AWSConfig is the AWS configuration to use for the S3 client
AWSConfig aws.Config
}
S3Options is used to configure the S3Store
func NewS3Options ¶ added in v0.3.1
NewS3Options creates a new S3Options instance with the provided options
type S3Store ¶
type S3Store struct {
Client *s3.Client
Opts *S3Options
PresignClient *s3.PresignClient
Downloader *manager.Downloader
Uploader *manager.Uploader
ObjExistsWaiter *s3.ObjectExistsWaiter
ObjNotExistsWaiter *s3.ObjectNotExistsWaiter
Scheme string
}
S3Store is a store that uses S3 as the backend
func NewS3FromConfig ¶
NewS3FromConfig creates a new S3Store from the provided configuration
func (*S3Store) Download ¶
func (s *S3Store) Download(ctx context.Context, opts *objects.DownloadFileOptions) (*objects.DownloadFileMetadata, error)
Download an object from S3 and return the metadata and a reader - the reader must be closed after use and is the responsibility of the caller
func (*S3Store) GetPresignedURL ¶
GetPresignedURL returns a URL that provides access to a file for the set duration if no duration is provided, it will default to 15 minutes
func (*S3Store) ListBuckets ¶ added in v0.3.1
ListBuckets lists the buckets in the current account.
func (*S3Store) ManagerUpload ¶
ManagerUpload uploads multiple files to S3
type SkipperFunc ¶ added in v0.38.1
SkipperFunc defines a function to skip middleware processing
type UploadOptions ¶ added in v0.38.1
type UploadOptions = storagetypes.UploadFileOptions
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type UploadedMetadata ¶ added in v0.38.1
type UploadedMetadata = storagetypes.UploadedFileMetadata
Alias types from storage/types to maintain clean imports having a bunch of smaller subpackages seemed to just complicate things
type UploaderFunc ¶ added in v0.38.1
UploaderFunc handles the file upload process and returns uploaded files
type ValidationFunc ¶ added in v0.38.1
ValidationFunc is a type that can be used to dynamically validate a file
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
providers
|
|
|
disk
Package disk is the local disk storage provider for objects service
|
Package disk is the local disk storage provider for objects service |
|
r2
Package r2 is the Cloudflare R2 storage provider for objects service
|
Package r2 is the Cloudflare R2 storage provider for objects service |
|
s3
Package s3 is the AWS S3 storage provider for objects service
|
Package s3 is the AWS S3 storage provider for objects service |
|
Package storagetypes contains types used across the storage package
|
Package storagetypes contains types used across the storage package |