storage

type CachingConfig struct {
	// Maximum size of the cache where the Blob store data is cached in-memory
	// Refer to https://github.com/coocood/freecache to understand how to set the value
	// If not specified or set to 0, cache is not used
	// NOTE: if Object sizes are larger than 1/1024 of the cache size, the entry will not be written to the cache
	// Also refer to https://github.com/coocood/freecache/issues/17 to understand how to set the cache
	MaxSizeMegabytes int `` /* 149-byte string literal not displayed */
	// sets the garbage collection target percentage:
	// a collection is triggered when the ratio of freshly allocated data
	// to live data remaining after the previous collection reaches this percentage.
	// refer to https://golang.org/pkg/runtime/debug/#SetGCPercent
	// If not specified or set to 0, GC percent is not tweaked
	TargetGCPercent int `json:"target_gc_percent" pflag:",Sets the garbage collection target percentage."`
}

type ComposedProtobufStore interface {
	RawStore
	ProtobufStore
}

type Config struct {
	Type Type `json:"type" pflag:",Sets the type of storage [s3/minio/local/mem/stow/redis]."`
	// Deprecated: Please use StowConfig instead
	Connection ConnectionConfig `json:"connection"`
	Stow       StowConfig       `json:"stow,omitempty" pflag:",Storage config for stow backend."`
	Redis      RedisConfig      `json:"redis,omitempty" pflag:"-,Storage config for the redis backend."`

	// Container here is misleading, it refers to a Bucket (AWS S3) like blobstore entity. In some terms it could be a table
	InitContainer string `json:"container" pflag:",Initial container (in s3 a bucket) to create -if it doesn't exist-.'"`
	// By default if this is not enabled, multiple containers are not supported by the storage layer. Only the configured `container` InitContainer will be allowed to requests data from. But, if enabled then data will be loaded to written to any
	// container specified in the DataReference.
	MultiContainerEnabled bool `` /* 213-byte string literal not displayed */
	// Schemes provides optional per-scheme backend overrides for the multi-scheme DataStore. It is
	// keyed by URL scheme (e.g. "s3", "gs", "abfs", "redis"). The DataStore lazily instantiates a
	// backend the first time a reference with a given scheme is seen and memoizes it for reuse, so a
	// single DataStore can serve any scheme/container thrown at it. A scheme absent from this map is
	// dialed with ambient credentials (the provider's default credential chain — IAM/instance profile,
	// GCP ADC, workload identity, env vars). The primary scheme (derived from Type) continues to be
	// configured by the top-level Connection/Stow/Redis fields and owns GetBaseContainerFQN.
	Schemes map[string]SchemeConfig `json:"schemes,omitempty" pflag:"-,Optional per-scheme backend overrides keyed by URL scheme."`
	// Caching is recommended to improve the performance of underlying systems. It caches the metadata and resolving
	// inputs is accelerated. The size of the cache is large so understand how to configure the cache.
	// TODO provide some default config choices
	// If this section is skipped, Caching is disabled
	Cache             CachingConfig    `json:"cache"`
	Limits            LimitsConfig     `json:"limits" pflag:",Sets limits for stores."`
	DefaultHTTPClient HTTPClientConfig `json:"defaultHttpClient" pflag:",Sets the default http client config."`
	SignedURL         SignedURLConfig  `json:"signedUrl" pflag:",Sets config for SignedURL."`
}

type ConnectionConfig struct {
	Endpoint   config.URL `json:"endpoint" pflag:",URL for storage client to connect to."`
	AuthType   string     `json:"auth-type" pflag:",Auth Type to use [iam,accesskey]."`
	AccessKey  string     `json:"access-key" pflag:",Access key to use. Only required when authtype is set to accesskey."`
	SecretKey  string     `json:"secret-key" pflag:",Secret to use when accesskey is set."`
	Region     string     `json:"region" pflag:",Region to connect to."`
	DisableSSL bool       `json:"disable-ssl" pflag:",Disables SSL connection. Should only be used for development."`
}

type Cursor struct {
	// contains filtered or unexported fields
}

type CursorState int

type DataReference string

type DataStore struct {
	ComposedProtobufStore
	ReferenceConstructor
	// contains filtered or unexported fields
}

type DefaultProtobufStore struct {
	RawStore
	// contains filtered or unexported fields
}

type HTTPClientConfig struct {
	Headers map[string][]string `json:"headers" pflag:"-,Sets http headers to set on the http client."`
	Timeout config.Duration     `json:"timeout" pflag:",Sets time out on the http client."`
	// Zero values for the transport settings below fall back to the http.DefaultTransport values.
	MaxIdleConns        int             `` /* 129-byte string literal not displayed */
	MaxIdleConnsPerHost int             `` /* 128-byte string literal not displayed */
	MaxConnsPerHost     int             `json:"maxConnsPerHost" pflag:",Maximum number of connections per host; new requests block at the limit. Zero means no limit."`
	IdleConnTimeout     config.Duration `` /* 135-byte string literal not displayed */
}

type InMemoryStore struct {
	// contains filtered or unexported fields
}

type LimitsConfig struct {
	GetLimitMegabytes int64 `json:"maxDownloadMBs" pflag:",Maximum allowed download size (in MBs) per call."`
}

type MemoryMetadata struct {
	// contains filtered or unexported fields
}

type Metadata interface {
	Exists() bool
	Size() int64
	Etag() string
	// ContentMD5 retrieves the value of a special metadata tag added by the system that
	// contains the MD5 of the uploaded file. If there is no metadata attached
	// or that `FlyteContentMD5` key isn't set, ContentMD5 will return empty.
	ContentMD5() string
}

type Options struct {
	Metadata map[string]interface{}
}

type ProtobufStore interface {
	// ReadProtobuf retrieves the entire blob from blobstore and unmarshals it to the passed protobuf
	ReadProtobuf(ctx context.Context, reference DataReference, msg proto.Message) error

	// WriteProtobuf serializes and stores the protobuf.
	WriteProtobuf(ctx context.Context, reference DataReference, opts Options, msg proto.Message) error
}

type RawStore interface {
	// GetBaseContainerFQN returns a FQN DataReference with the configured base init container
	GetBaseContainerFQN(ctx context.Context) DataReference

	// CreateSignedURL creates a signed url with the provided properties.
	CreateSignedURL(ctx context.Context, reference DataReference, properties SignedURLProperties) (SignedURLResponse, error)

	// Head gets metadata about the reference. This should generally be a light weight operation.
	Head(ctx context.Context, reference DataReference) (Metadata, error)

	// List gets a list of items (relative path to the reference input) given a prefix, using a paginated API
	List(ctx context.Context, reference DataReference, maxItems int, cursor Cursor) ([]DataReference, Cursor, error)

	// ReadRaw retrieves a byte array from the Blob store or an error
	ReadRaw(ctx context.Context, reference DataReference) (io.ReadCloser, error)

	// WriteRaw stores a raw byte array.
	WriteRaw(ctx context.Context, reference DataReference, size int64, opts Options, raw io.Reader) error

	// CopyRaw copies from source to destination.
	CopyRaw(ctx context.Context, source, destination DataReference, opts Options) error

	// Delete removes the referenced data from the blob store.
	Delete(ctx context.Context, reference DataReference) error
}

type RedisConfig struct {
	// Addr is the host:port of the redis server.
	Addr string `json:"addr"`
	// Username for redis ACL authentication (redis 6+), if required.
	Username string `json:"username"`
	// Password for redis authentication, if required.
	Password string `json:"password"`
	// DB is the logical database to select after connecting.
	DB int `json:"db"`
}

type RedisStore struct {
	// contains filtered or unexported fields
}

type ReferenceConstructor interface {
	// ConstructReference creates a new dataReference that matches the storage structure.
	ConstructReference(ctx context.Context, reference DataReference, nestedKeys ...string) (DataReference, error)
}

type SchemeConfig struct {
	// Kind overrides the stow kind for this scheme. Defaults to the kind derived from the scheme
	// (s3->s3, gs->google, abfs/abfss->azure, os->oracle, sw->swift, file->local). Ignored for redis.
	Kind string `json:"kind,omitempty"`
	// Config is the stow ConfigMap (credentials, region, endpoint, ...) passed to stow.Dial for this
	// scheme. When empty, the scheme is dialed with ambient credentials.
	Config map[string]string `json:"config,omitempty"`
	// Redis configures the connection when the scheme is redis. When nil, the redis address is taken
	// from the top-level Redis.Addr, falling back to the host portion of the reference.
	Redis *RedisConfig `json:"redis,omitempty"`
}

type SignedURLConfig struct {
	StowConfigOverride map[string]string `json:"stowConfigOverride,omitempty" pflag:"-,Configuration for stow backend. Refer to github/flyteorg/stow"`
}

type SignedURLProperties struct {
	// Scope defines the permission level allowed for the generated URL.
	Scope stow.ClientMethod
	// ExpiresIn defines the expiration duration for the URL. It's strongly recommended setting it.
	ExpiresIn time.Duration
	// ContentMD5 defines the expected Base64-encoded 128-bit MD5 hash of the generated file.
	// It's strongly recommended setting it.for data integrity checks on the storage backend.
	ContentMD5 string
	// AddContentMD5Metadata Add ContentMD5 to the metadata of signed URL if true.
	AddContentMD5Metadata bool
}

type SignedURLResponse struct {
	URL                    url.URL
	RequiredRequestHeaders map[string]string
}

type StowConfig struct {
	Kind   string            `json:"kind,omitempty" pflag:",Kind of Stow backend to use. Refer to github/flyteorg/stow"`
	Config map[string]string `json:"config,omitempty" pflag:",Configuration for stow backend. Refer to github/flyteorg/stow"`
}

type StowMetadata struct {
	// contains filtered or unexported fields
}

type StowStore struct {
	// contains filtered or unexported fields
}

type Type = string

type URLPathConstructor struct {
}