fileservice

type AliyunSDK struct {
	// contains filtered or unexported fields
}

type AppConfig struct {
	Name string
	GCFn func(filePath string, fs FileService) (neesGC bool, err error)
}

type AppFS struct {
	// contains filtered or unexported fields
}

type AwsSDKv2 struct {
	// contains filtered or unexported fields
}

type Bytes struct {
	// contains filtered or unexported fields
}

type CacheCallbackFunc = func(fscache.CacheKey, fscache.Data)

type CacheCallbacks struct {
	PostGet   []CacheCallbackFunc
	PostSet   []CacheCallbackFunc
	PostEvict []CacheCallbackFunc
}

type CacheConfig struct {
	MemoryCapacity       *toml.ByteSize `toml:"memory-capacity" user_setting:"advanced"`
	DiskPath             *string        `toml:"disk-path"`
	DiskCapacity         *toml.ByteSize `toml:"disk-capacity"`
	DiskMinEvictInterval *toml.Duration `toml:"disk-min-evict-interval"`
	DiskEvictTarget      *float64       `toml:"disk-evict-target"`
	RemoteCacheEnabled   bool           `toml:"remote-cache-enabled"`
	RPC                  morpc.Config   `toml:"rpc"`
	CheckOverlaps        bool           `toml:"check-overlaps"`

	QueryClient      client.QueryClient            `json:"-"`
	KeyRouterFactory KeyRouterFactory[pb.CacheKey] `json:"-"`
	KeyRouter        client.KeyRouter[pb.CacheKey] `json:"-"`
	InitKeyRouter    *sync.Once                    `json:"-"`
	CacheCallbacks   `json:"-"`
	// contains filtered or unexported fields
}

type CacheDataAllocator interface {
	AllocateCacheData(ctx context.Context, size int) fscache.Data
	AllocateCacheDataWithHint(ctx context.Context, size int, hints malloc.Hints) fscache.Data
	CopyToCacheData(ctx context.Context, data []byte) fscache.Data
}

type CachingFileService interface {
	FileService

	// FlushCache flushes cache
	FlushCache(ctx context.Context)

	// SetAsyncUpdate sets cache update operation to async mode
	SetAsyncUpdate(bool)
}

type Config struct {
	// Name name of fileservice, describe what an instance of fileservice is used for
	Name string `toml:"name"`
	// Backend fileservice backend. [MEM|DISK|DISK-ETL|S3|MINIO]
	Backend string `toml:"backend"`
	// S3 used to create fileservice using s3 as the backend
	S3 ObjectStorageArguments `toml:"s3"`
	// Cache specifies configs for cache
	Cache CacheConfig `toml:"cache"`
	// DataDir used to create fileservice using DISK as the backend
	DataDir string `toml:"data-dir"`
	// FixMissing inidicates the file service to try its best to fix missing files
	FixMissing bool `toml:"fix-missing"`
}

type CostAttr struct {
	// List is the cost of List from FileService
	List CostItem
}

type CostItem uint8

type DirEntry struct {
	// file name, not full path
	Name  string
	IsDir bool
	Size  int64
}

type DiskCache struct {
	// contains filtered or unexported fields
}

type DiskCacheCallbacks struct {
	OnWritten []OnDiskCacheWrittenFunc
	OnEvict   []OnDiskCacheEvictFunc
	// contains filtered or unexported fields
}

type ETLFileService interface {
	FileService

	// ETLCompatible marks the implementation to be compatible to ETL operations
	// implementations must save file contents as-is
	ETLCompatible()
}

type FileCache interface {
	SetFile(
		ctx context.Context,
		path string,
		openReader func(context.Context) (io.ReadCloser, error),
	) error
}

type FileLike interface {
	io.ReadWriteSeeker
	io.WriterAt
	io.ReaderAt
}

type FileService interface {
	// Name is file service's name
	// service name is case-insensitive
	Name() string

	// Write writes a new file
	// returns ErrFileExisted if file already existed
	// returns ErrSizeNotMatch if provided size does not match data
	// entries in vector should be written atomically. if write failed, following reads must not succeed.
	Write(ctx context.Context, vector IOVector) error

	// Read reads a file to fill IOEntries
	// returns ErrFileNotFound if requested file not found
	// returns ErrUnexpectedEOF if less data is read than requested size
	// returns ErrEmptyRange if no data at specified offset and size
	// returns ErrEmptyVector if no IOEntry is passed
	Read(ctx context.Context, vector *IOVector) error

	// ReadCache reads cached data if any
	// if cache hit, IOEntry.CachedData will be set
	ReadCache(ctx context.Context, vector *IOVector) error

	// List lists sub-entries in a dir
	List(ctx context.Context, dirPath string) iter.Seq2[*DirEntry, error]

	// Delete deletes multi file
	// returns ErrFileNotFound if requested file not found
	Delete(ctx context.Context, filePaths ...string) error

	// Stat returns infomations about a file
	// returns ErrFileNotFound if requested file not found
	StatFile(ctx context.Context, filePath string) (*DirEntry, error)

	// PrefetchFile prefetches a file
	PrefetchFile(ctx context.Context, filePath string) error

	// Cost returns the cost attr of the file service
	Cost() *CostAttr

	Close(ctx context.Context)
}

type FileServiceWriter struct {
	Reader      *io.PipeReader
	Writer      *io.PipeWriter
	Group       *errgroup.Group
	Filepath    string
	FileService FileService
}

type FileServices struct {
	// contains filtered or unexported fields
}

type FileWithChecksum[T FileLike] struct {
	// contains filtered or unexported fields
}

type HDFS struct {
	// contains filtered or unexported fields
}

type IOEntry struct {
	// offset in file
	// when writing or mutating, offset can be arbitrary value, gaps between provided data are zero-filled
	// when reading, valid offsets are in range [0, len(file) - 1]
	Offset int64

	// number of bytes to read or write, [1, len(file)]
	// when reading, pass -1 to read to the end of file
	Size int64

	// raw content
	// when reading, if len(Data) < Size, a new Size-lengthed byte slice will be allocated
	Data []byte

	// when reading, if Writer is not nil, write data to it instead of setting Data field
	WriterForRead io.Writer

	// when reading, if ReadCloser is not nil, set an io.ReadCloser instead of setting Data field
	ReadCloserForRead *io.ReadCloser

	// when writing, if Reader is not nil, read data from it instead of reading Data field
	// number of bytes to be read is specified by Size field
	// if number of bytes is unknown, set Size field to -1
	ReaderForWrite io.Reader

	// When reading, if the ToCacheData field is not nil, the returning object's byte slice will be set to this field
	// Data, WriterForRead, ReadCloserForRead may be empty if CachedData is not null
	// if ToCacheData is provided, caller should always read CachedData instead of Data, WriterForRead or ReadCloserForRead
	CachedData fscache.Data

	// ToCacheData constructs an object byte slice from entry contents
	// reader or data must not be retained after returns
	// reader always contains entry contents
	// data may contains entry contents if available
	// if data is empty, the io.Reader must be fully read before returning nil error
	ToCacheData func(ctx context.Context, reader io.Reader, data []byte, allocator CacheDataAllocator) (cacheData fscache.Data, err error)
	// contains filtered or unexported fields
}

type IOMergeKey struct {
	Path       string
	Offset     int64
	End        int64
	FullObject bool
	Policy     Policy
}

type IOMerger struct {
	// contains filtered or unexported fields
}

type IOVector struct {

	// FilePath indicates where to find the file
	// a path has two parts, service name and file name, separated by ':'
	// service name is optional, if omitted, the receiver FileService will use the default name of the service
	// file name parts are separated by '/'
	// valid characters in file name: 0-9 a-z A-Z / ! - _ . * ' ( )
	// and all printable non-ASCII characters
	// example:
	// s3:a/b/c S3:a/b/c represents the same file 'a/b/c' located in 'S3' service
	FilePath string

	// io entries
	// empty Entries is not allowed
	// when writing, overlapping Entries is not allowed
	Entries []IOEntry

	// ExpireAt specifies the expire time of the file
	// implementations may or may not delete the file after this time
	// zero value means no expire
	ExpireAt time.Time

	// Policy controls policy for the vector
	Policy Policy

	// Caches indicates extra caches to operate on
	Caches []IOVectorCache
}

type IOVectorCache interface {
	Read(
		ctx context.Context,
		vector *IOVector,
	) error

	Update(
		ctx context.Context,
		vector *IOVector,
		async bool,
	) error

	Flush(ctx context.Context)

	//TODO file contents may change in TAE that violates the immutibility assumption
	// before they fix this, we still need this sh**.
	DeletePaths(
		ctx context.Context,
		paths []string,
	) error

	// Evict triggers eviction
	// if done is not nil, when eviction finish, target size will be send to the done chan
	Evict(ctx context.Context, done chan int64)

	Close(ctx context.Context)
}

type KeyRouterFactory[T comparable] func() client.KeyRouter[T]

type LocalETLFS struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

type LocalETLFSMutator struct {
	// contains filtered or unexported fields
}

type LocalFS struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

type LocalFSMutator struct {
	// contains filtered or unexported fields
}

type MemCache struct {
	// contains filtered or unexported fields
}

type MemoryFS struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

type MinioSDK struct {
	// contains filtered or unexported fields
}

type MutableFileService interface {
	FileService

	// NewMutator creates a new mutator
	NewMutator(ctx context.Context, filePath string) (Mutator, error)

	// Raw os.File API
	EnsureDir(ctx context.Context, filePath string) error
	OpenFile(ctx context.Context, filePath string) (*os.File, error)
	CreateFile(ctx context.Context, filePath string) (*os.File, error)
	RemoveFile(ctx context.Context, filePath string) error
	CreateAndRemoveFile(ctx context.Context, filePath string) (*os.File, error)
}

type Mutator interface {

	// Mutate mutates file contents
	Mutate(ctx context.Context, entries ...IOEntry) error

	// Append appends data to file
	// all IOEntry.Offset is base on the end of file position
	// for example, passing IOEntry{Offset: 0, Len:1, Data: []byte("a")} will append "a" to the end of file
	Append(ctx context.Context, entries ...IOEntry) error

	// Close closes the mutator
	// Must be called after finishing mutation
	Close() error
}

type NewFileServicesFunc = func(defaultName string) (*FileServices, error)

type ObjectStorage interface {
	// List lists objects with specified prefix
	List(
		ctx context.Context,
		prefix string,
	) iter.Seq2[*DirEntry, error]

	// Stat returns informations about an object
	Stat(
		ctx context.Context,
		key string,
	) (
		size int64,
		err error,
	)

	// Exists reports whether specified object exists
	Exists(
		ctx context.Context,
		key string,
	) (
		bool,
		error,
	)

	// Write writes an object
	Write(
		ctx context.Context,
		key string,
		r io.Reader,
		sizeHint *int64,
		expire *time.Time,
	) (
		err error,
	)

	// Read returns an io.Reader for specified object range
	Read(
		ctx context.Context,
		key string,
		min *int64,
		max *int64,
	) (
		r io.ReadCloser,
		err error,
	)

	// Delete deletes objects
	Delete(
		ctx context.Context,
		keys ...string,
	) (
		err error,
	)
}

type ObjectStorageArguments struct {
	// misc
	Name                 string       `toml:"name"`
	KeyPrefix            string       `toml:"key-prefix"`
	SharedConfigProfile  string       `toml:"shared-config-profile"`
	NoDefaultCredentials bool         `toml:"no-default-credentials"`
	NoBucketValidation   bool         `toml:"no-bucket-validation"`
	Concurrency          int64        `toml:"concurrency"`
	MaxConnsPerHost      int          `toml:"max-conns-per-host"`
	ParallelMode         ParallelMode `toml:"parallel-mode"`

	// s3
	Bucket    string   `toml:"bucket"`
	Endpoint  string   `toml:"endpoint"`
	IsMinio   bool     `toml:"is-minio"`
	Region    string   `toml:"region"`
	CertFiles []string `toml:"cert-files"`

	// credentials
	RoleARN         string `json:"-" toml:"role-arn"`
	BearerToken     string `json:"-" toml:"bearer-token"`
	ExternalID      string `json:"-" toml:"external-id"`
	KeyID           string `json:"-" toml:"key-id"`
	KeySecret       string `json:"-" toml:"key-secret"`
	RAMRole         string `json:"-" toml:"ram-role"`
	RoleSessionName string `json:"-" toml:"role-session-name"`
	SecurityToken   string `json:"-" toml:"security-token"`
	SessionToken    string `json:"-" toml:"session-token"`

	// HDFS
	IsHDFS                       bool   `toml:"is-hdfs"`
	User                         string `toml:"user"`
	KerberosServicePrincipleName string `toml:"kerberos-service-principle-name"`
	KerberosUsername             string `toml:"kerberos-username"`
	KerberosRealm                string `toml:"kerberos-realm"`
	KerberosPassword             string `json:"-" toml:"kerberos-password"`
	KerberosKeytabPath           string `toml:"kerberos-keytab-path"`
}

type OnDiskCacheEvictFunc = func(
	diskFilePath string,
)

type OnDiskCacheWrittenFunc = func(
	filePath string,
	entry IOEntry,
)

type ParallelMode uint8

type ParallelMultipartOption struct {
	// PartSize configures each part size; defaults to 64MB if zero.
	PartSize int64
	// Concurrency configures worker count; defaults to runtime.NumCPU() if zero.
	Concurrency int
	// Expire sets object expiration.
	Expire *time.Time
}

type ParallelMultipartWriter interface {
	SupportsParallelMultipart() bool
	WriteMultipartParallel(
		ctx context.Context,
		key string,
		r io.Reader,
		sizeHint *int64,
		opt *ParallelMultipartOption,
	) error
}

type Path struct {
	Service          string
	ServiceArguments []string
	File             string
}

type Policy uint64

type Pool[T any] struct {
	// contains filtered or unexported fields
}

type PutBack[T any] struct {
	// contains filtered or unexported fields
}

type QCloudSDK struct {
	// contains filtered or unexported fields
}

type ReaderWriterFileService interface {
	FileService

	// NewReader creates an io.ReadCloser for reading the content of the file
	NewReader(ctx context.Context, filePath string) (io.ReadCloser, error)

	// NewWriter creates an io.WriteCloser for writing to the file
	NewWriter(ctx context.Context, filePath string) (io.WriteCloser, error)
}

type RemoteCache struct {
	// contains filtered or unexported fields
}

type ReplaceableFileService interface {
	FileService

	Replace(ctx context.Context, vector IOVector) error
}

type S3FS struct {
	// contains filtered or unexported fields
}

type TargetCacheKeys map[string][]*query.RequestCacheKey

type TmpFileService struct {
	FileService
	// contains filtered or unexported fields
}