Documentation
¶
Overview ¶
nolint
Index ¶
- Constants
- Variables
- func BackGroundReaderWait(n int) error
- func Clean(path string) string
- func CopyFile(srcFile, dstFile string, opt ...FSOption) (written int64, err error)
- func CopyFileFromDFVToOBS(srcPath, dstPath string, opt ...FSOption) error
- func CreateTime(name string) (*time.Time, error)
- func DecodeObsPath(dir, path string) (string, error)
- func DecryptObsSk(sk string) string
- func DeleteObsPath(path string, obsOpts *obs.ObsOptions) error
- func DirNotExists(err error) bool
- func EnableMmapRead(en bool)
- func EnableReadDataCache(en uint64)
- func EnableReadMetaCache(en uint64)
- func EncodeObsPath(endpoint, bucket, path, ak, sk string) string
- func Fadvise(fd int, offset int64, length int64, advice int) (err error)
- func Fdatasync(file File) (err error)
- func FindDir(dirs []fs.FileInfo, dirName string) fs.FileInfo
- func GetAllFilesSizeInPath(path string) (int64, int64, int64, error)
- func GetLocalFileName(fName string) (string, error)
- func GetOBSTmpFileName(srcPath string, obsOption *obs.ObsOptions, opt ...FSOption) string
- func GetOBSTmpIndexFileName(srcPath string, obsOption *obs.ObsOptions, opt ...FSOption) string
- func GetRemoteDataPath(obsOpt *obs.ObsOptions, dataPath string) string
- func GetRemotePrefixPath(obsOpt *obs.ObsOptions) string
- func GetSubDirFiles(fis []fs.FileInfo, prefixPath string) []fs.FileInfo
- func GetSubDirNamesForObsReadDirs(fis []fs.FileInfo, prefixPath string) []string
- func Glob(pattern string) ([]string, error)
- func InitWriterPool(size int)
- func IsObsFile(path string) (bool, error)
- func Join(path ...string) string
- func MUnmap(data []byte) error
- func Mkdir(path string, perm os.FileMode, opt ...FSOption) error
- func MkdirAll(path string, perm os.FileMode, opt ...FSOption) error
- func Mmap(fd int, offset int64, length int) (data []byte, err error)
- func MmapRW(fd int, offset int64, length int) (data []byte, err error)
- func NewFileReader(f File, lock *string) *fileReader
- func NewFileWriter(lw NameReadWriterCloser, bufferSize int, lockPath *string) *fileWriter
- func NormalizeDirPath(path string) string
- func PutObsClient(conf *obsConf, client ObsClient)
- func ReadDir(dirname string) ([]fs.FileInfo, error)
- func ReadFile(filename string, opt ...FSOption) ([]byte, error)
- func RecoverLease(lock string) error
- func Remove(name string, opt ...FSOption) error
- func RemoveAll(path string, opt ...FSOption) error
- func RemoveAllWithOutDir(path string, opt ...FSOption) error
- func RemoveLocal(localName string, opt ...FSOption) error
- func RemoveLocalEnabled(localName string, obsOpt *obs.ObsOptions) bool
- func RenameFile(oldPath, newPath string, opt ...FSOption) error
- func SetBackgroundReadLimiter(limiter int)
- func SetLogger()
- func SetMetaPageList(confMetaPageList []string)
- func SetPageSize(confPageSize string)
- func Stat(name string) (os.FileInfo, error)
- func StreamConfUpdate(propertyName, propertyValue string) error
- func Truncate(name string, size int64) error
- func UpdateObsAkSk(ak, sk string) error
- func WriteFile(filename string, data []byte, perm os.FileMode, opt ...FSOption) error
- type BasicFileReader
- type BasicFileWriter
- type FSOption
- type File
- func Create(name string, opt ...FSOption) (File, error)
- func CreateV1(name string, opt ...FSOption) (File, error)
- func CreateV2(name string, opt ...FSOption) (File, error)
- func Open(name string, opt ...FSOption) (File, error)
- func OpenFile(name string, flag int, perm os.FileMode, opt ...FSOption) (File, error)
- func OpenObsFile(path, fileName string, obsOpts *obs.ObsOptions, onlyRead bool) (File, error)
- type FileLockOption
- type FilePriorityOption
- type FileWriter
- type FsType
- type LimitWriter
- type Limiter
- type MemFile
- type MetaWriter
- type NameReadWriterCloser
- type ObsClient
- type Range
- type RangeRequest
- type VFS
Constants ¶
const ( HdfsPrefix = "hdfs://" ObsPrefixTS = "obs:/" ObsPrefixLogKeeper = "obs://" )
const ( FADV_NORMAL = 0x0 FADV_RANDOM = 0x1 FADV_SEQUENTIAL = 0x2 FADV_WILLNEED = 0x3 FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 )
const ( DefaultWriterBufferSize = 1024 * 1024 DefaultBufferSize = 256 * 1024 )
const ( ObsClientCacheSize = 1024 ObsClientCacheTTL = 24 * time.Hour )
const ( ObsReadRetryTimes = 3 ObsWriteRetryTimes = 3 ObsOpenRetryTimes = 3 )
Variables ¶
var ( IO_PRIORITY_ULTRA_HIGH = 0 IO_PRIORITY_HIGH = 1 IO_PRIORITY_NORMAL = 2 IO_PRIORITY_LOW = 3 IO_PRIORITY_LOW_READ = 4 )
var (
BackgroundReadLimiter = NewLimiter(64*1024*1024, 64*1024*1024)
)
var MmapEn = false
var ObsPrefix = ObsPrefixTS
var ObsSingleRequestSize int64 = 9 * 1024 * 1024
var ReadDataCacheEn = false
var ReadMetaCacheEn = false
Functions ¶
func BackGroundReaderWait ¶ added in v1.1.0
func CopyFile ¶
CopyFile copys file content from srcFile to dstFile until either EOF is reached on srcFile or an errors accurs. the optional opt is: (FileLockOption,FilePriorityOption)
func CopyFileFromDFVToOBS ¶ added in v1.2.0
CopyFileFromDFVToOBS copy a file from DFV TO OBS when use streamfs
func DecodeObsPath ¶ added in v1.3.0
func DecryptObsSk ¶ added in v1.4.0
func DeleteObsPath ¶ added in v1.3.0
func DeleteObsPath(path string, obsOpts *obs.ObsOptions) error
func DirNotExists ¶ added in v1.5.0
func EnableMmapRead ¶ added in v1.1.0
func EnableMmapRead(en bool)
func EnableReadDataCache ¶ added in v1.2.0
func EnableReadDataCache(en uint64)
func EnableReadMetaCache ¶ added in v1.2.0
func EnableReadMetaCache(en uint64)
func EncodeObsPath ¶ added in v1.2.0
func GetAllFilesSizeInPath ¶ added in v1.3.0
func GetLocalFileName ¶ added in v1.3.0
func GetOBSTmpFileName ¶ added in v1.3.0
func GetOBSTmpFileName(srcPath string, obsOption *obs.ObsOptions, opt ...FSOption) string
func GetOBSTmpIndexFileName ¶ added in v1.5.0
func GetOBSTmpIndexFileName(srcPath string, obsOption *obs.ObsOptions, opt ...FSOption) string
func GetRemoteDataPath ¶ added in v1.3.0
func GetRemoteDataPath(obsOpt *obs.ObsOptions, dataPath string) string
func GetRemotePrefixPath ¶ added in v1.3.0
func GetRemotePrefixPath(obsOpt *obs.ObsOptions) string
func GetSubDirFiles ¶ added in v1.5.0
func GetSubDirNamesForObsReadDirs ¶ added in v1.5.0
func Glob ¶
Glob returns the names of all files matching pattern or nil if there is no matching file.
func InitWriterPool ¶ added in v1.1.0
func InitWriterPool(size int)
func IsObsFile ¶ added in v1.2.0
IsObsFile determines whether a file is stored in OBS when use streamfs
func Mkdir ¶
Mkdir creates a directory named path, it's parents directory must exist. the optional opt is: FileLockOption
func MkdirAll ¶
MkdirAll creates a directory named path, along with any necessary parents the optional opt is: FileLockOption
func NewFileReader ¶ added in v1.1.0
func NewFileWriter ¶ added in v1.1.0
func NewFileWriter(lw NameReadWriterCloser, bufferSize int, lockPath *string) *fileWriter
func NormalizeDirPath ¶ added in v1.4.0
func PutObsClient ¶ added in v1.2.0
func PutObsClient(conf *obsConf, client ObsClient)
func ReadDir ¶
ReadDir reads the directory named by dirname and returns a list of fs.FileInfo for the directory's contents, sorted by filename.
func ReadFile ¶
ReadFile reads the file named by filename and returns the contents. the optional opt is: FilePriorityOption
func RecoverLease ¶ added in v1.0.0
func Remove ¶
Remove removes the named file or (empty) directory. the optional opt is: FileLockOption
func RemoveAll ¶
RemoveAll removes path and any children it contains. the optional opt is: FileLockOption
func RemoveAllWithOutDir ¶ added in v1.5.0
RemoveAllWithOutDir Delete the file and retain the file directory. the optional opt is: FileLockOption
func RemoveLocal ¶ added in v1.3.0
func RemoveLocalEnabled ¶ added in v1.3.0
func RemoveLocalEnabled(localName string, obsOpt *obs.ObsOptions) bool
func RenameFile ¶
RenameFile renames (moves) oldPath to newPath. If newPath already exists and is not a directory, Rename replaces it. the optional opt is: FileLockOption
func SetBackgroundReadLimiter ¶ added in v1.1.0
func SetBackgroundReadLimiter(limiter int)
func SetMetaPageList ¶ added in v1.4.0
func SetMetaPageList(confMetaPageList []string)
func SetPageSize ¶ added in v1.2.0
func SetPageSize(confPageSize string)
func StreamConfUpdate ¶ added in v1.5.0
func UpdateObsAkSk ¶ added in v1.5.0
Types ¶
type BasicFileReader ¶ added in v1.1.0
type BasicFileReader interface {
Name() string
Size() (int64, error)
ReadAt(off int64, size uint32, dst *[]byte, ioPriority int) ([]byte, error)
StreamReadBatch(off, length []int64, c chan *request.StreamReader, limit int, isStat bool)
Rename(newName string) error
RenameOnObs(newName string, tmp bool, obsOpt *obs.ObsOptions) error
ReOpen() error
IsMmapRead() bool
IsOpen() bool
FreeFileHandle() error
Close() error
ReadAll(dst []byte) ([]byte, error)
}
type BasicFileWriter ¶ added in v1.1.0
type File ¶
type File interface {
io.Closer
io.Reader
io.Seeker
io.Writer
io.ReaderAt
Name() string
Truncate(size int64) error
Sync() error
Stat() (os.FileInfo, error)
SyncUpdateLength() error
Fd() uintptr
Size() (int64, error)
StreamReadBatch([]int64, []int64, int64, chan *request.StreamReader, int, bool)
}
func Create ¶
Create creates or truncates the named file. If the file already exists, it is truncated. If the file does not exist, it is created with mode 0666 the optional opt is: (FileLockOption,FilePriorityOption)
func Open ¶
Open opens the named file with specified options. the optional opt is: (FileLockOption,FilePriorityOption)
func OpenFile ¶
OpenFile opens the named file with specified flag and other options. the optional opt is: (FileLockOption,FilePriorityOption)
func OpenObsFile ¶ added in v1.3.0
type FileLockOption ¶
type FileLockOption string
func (FileLockOption) Parameter ¶
func (opt FileLockOption) Parameter() interface{}
type FilePriorityOption ¶
type FilePriorityOption int
func (FilePriorityOption) Parameter ¶
func (opt FilePriorityOption) Parameter() interface{}
type FileWriter ¶ added in v1.1.0
type FileWriter interface {
WriteData(b []byte) (int, error)
WriteChunkMeta(b []byte) (int, error)
Close() error
DataSize() int64
ChunkMetaSize() int64
ChunkMetaBlockSize() int64
GetFileWriter() BasicFileWriter
AppendChunkMetaToData() error
SwitchMetaBuffer() (int, error)
MetaDataBlocks(dst [][]byte) [][]byte
Name() string
}
type LimitWriter ¶ added in v1.1.0
type LimitWriter struct {
// contains filtered or unexported fields
}
func (*LimitWriter) Close ¶ added in v1.1.0
func (w *LimitWriter) Close() error
func (*LimitWriter) Name ¶ added in v1.1.0
func (w *LimitWriter) Name() string
type Limiter ¶ added in v1.1.0
type Limiter interface {
SetBurst(newBurst int)
SetLimit(newLimit rate.Limit)
WaitN(ctx context.Context, n int) (err error)
Limit() rate.Limit
Burst() int
}
func NewLimiter ¶ added in v1.1.0
type MemFile ¶ added in v1.5.0
type MemFile struct {
// contains filtered or unexported fields
}
func NewMemFile ¶ added in v1.5.0
type MetaWriter ¶ added in v1.2.0
type MetaWriter interface {
WriteMetaIndex(b []byte) (int, error)
WritePrimaryKey(b []byte) (int, error)
WritePrimaryKeyMeta(b []byte) (int, error)
WriteBloomFilter(bfIdx int, b []byte) (int, error)
GetMetaIndexSize() int64
GetPrimaryKeySize() int64
GetPrimaryKeyMetaSize() int64
GetBloomFilterSize(bfIdx int) int64
GetPrimaryKeyHandler() File
GetPrimaryKeyMetaHandler() File
Close() error
}
type NameReadWriterCloser ¶ added in v1.1.0
type NameReadWriterCloser interface {
Name() string
io.ReadWriteCloser
}
func NewLimitWriter ¶ added in v1.1.0
func NewLimitWriter(w NameReadWriterCloser, l Limiter) NameReadWriterCloser
type ObsClient ¶ added in v1.2.0
type ObsClient interface {
ListBuckets(input *obs.ListBucketsInput) (output *obs.ListBucketsOutput, err error)
ListObjects(input *obs.ListObjectsInput) (output *obs.ListObjectsOutput, err error)
GetObject(input *obs.GetObjectInput) (output *obs.GetObjectOutput, err error)
DeleteObject(input *obs.DeleteObjectInput) (output *obs.DeleteObjectOutput, err error)
DeleteObjects(input *obs.DeleteObjectsInput) (output *obs.DeleteObjectsOutput, err error)
ModifyObject(input *obs.ModifyObjectInput) (output *obs.ModifyObjectOutput, err error)
PutObject(input *obs.PutObjectInput) (output *obs.PutObjectOutput, err error)
GetObjectMetadata(input *obs.GetObjectMetadataInput) (output *obs.GetObjectMetadataOutput, err error)
RenameFile(input *obs.RenameFileInput) (*obs.RenameFileOutput, error)
IsObsFile(input *obs.HeadObjectInput) (output *obs.BaseModel, err error)
Do(r *http.Request) (*http.Response, error)
}
func GetObsClient ¶ added in v1.2.0
type RangeRequest ¶ added in v1.2.0
type RangeRequest struct {
// contains filtered or unexported fields
}
func NewObsReadRequest ¶ added in v1.2.0
func NewObsRetryReadRequest ¶ added in v1.2.0
func NewObsRetryReadRequest(retryRange map[int64]int64, obsRangeSize int) []*RangeRequest
func NewRangeRequest ¶ added in v1.2.0
func NewRangeRequest() *RangeRequest
func (*RangeRequest) AddReadAction ¶ added in v1.2.0
func (r *RangeRequest) AddReadAction(offset int64, length int64)
func (*RangeRequest) GetRangeString ¶ added in v1.2.0
func (r *RangeRequest) GetRangeString() string
type VFS ¶
type VFS interface {
// Open opens the named file with specified options.
// the optional opt is: (FileLockOption,FilePriorityOption)
Open(name string, opt ...FSOption) (File, error)
// OpenFile opens the named file with specified flag and other options.
// the optional opt is: (FileLockOption,FilePriorityOption)
OpenFile(name string, flag int, perm os.FileMode, opt ...FSOption) (File, error)
// Create creates or truncates the named file. If the file already exists, it is truncated.
// If the file does not exist, it is created with mode 0666
// the optional opt is: (FileLockOption,FilePriorityOption)
Create(name string, opt ...FSOption) (File, error)
CreateV1(name string, opt ...FSOption) (File, error)
// CreateV2 create a new file in OBS when use streamfs
CreateV2(name string, opt ...FSOption) (File, error)
// Remove removes the named file or (empty) directory.
// the optional opt is: FileLockOption
Remove(name string, opt ...FSOption) error
// RemoveLocal removes the named file on local storage
// the optional opt is: FileLockOption
RemoveLocal(name string, opt ...FSOption) error
// RemoveLocalEnabled return whether remove the named file on local storage
RemoveLocalEnabled(obsOptValid bool) bool
// RemoveAll removes path and any children it contains.
// the optional opt is: FileLockOption
RemoveAll(path string, opt ...FSOption) error
// RemoveAllWithOutDir Delete the file and retain the file directory.
// the optional opt is: FileLockOption
RemoveAllWithOutDir(path string, opt ...FSOption) error
// Mkdir creates a directory named path, it's parents directory must exist.
// the optional opt is: FileLockOption
Mkdir(path string, perm os.FileMode, opt ...FSOption) error
// MkdirAll creates a directory named path, along with any necessary parents
// the optional opt is: FileLockOption
MkdirAll(path string, perm os.FileMode, opt ...FSOption) error
// NormalizeDirPath returns a standardized directory path
NormalizeDirPath(path string) string
// ReadDir reads the directory named by dirname and returns
// a list of fs.FileInfo for the directory's contents, sorted by filename.
ReadDir(dirname string) ([]fs.FileInfo, error)
// Glob returns the names of all files matching pattern or nil if there is no matching file.
Glob(pattern string) ([]string, error)
// RenameFile renames (moves) oldPath to newPath.
// If newPath already exists and is not a directory, Rename replaces it.
// the optional opt is: FileLockOption
RenameFile(oldPath, newPath string, opt ...FSOption) error
// Stat returns a FileInfo describing the named file.
Stat(name string) (os.FileInfo, error)
// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm
// the optional opt is: (FileLockOption,FilePriorityOption)
WriteFile(filename string, data []byte, perm os.FileMode, opt ...FSOption) error
// ReadFile reads the file named by filename and returns the contents.
// the optional opt is: FilePriorityOption
ReadFile(filename string, opt ...FSOption) ([]byte, error)
// CopyFile copys file content from srcFile to dstFile until either EOF is reached on srcFile or an errors accurs.
// the optional opt is: (FileLockOption,FilePriorityOption)
CopyFile(srcFile, dstFile string, opt ...FSOption) (written int64, err error)
CreateTime(name string) (*time.Time, error)
// Truncate changes the size of the file to size.
// the optional opt is: (FileLockOption)
Truncate(name string, size int64, opt ...FSOption) error
// IsObsFile determines whether a file is stored in OBS when use streamfs
IsObsFile(path string) (bool, error)
// CopyFileFromDFVToOBS copy a file from DFV TO OBS when use streamfs
CopyFileFromDFVToOBS(srcPath, dstPath string, opt ...FSOption) error
// GetAllFilesSizeInPath return totalSize, dfvSize, obsSize
GetAllFilesSizeInPath(path string) (int64, int64, int64, error)
// GetOBSTmpFileName return tmp file name on obs
GetOBSTmpFileName(path string, obsOption *obs.ObsOptions) string
// GetOBSTmpIndexFileName return tmp index file name on obs
GetOBSTmpIndexFileName(path string, obsOption *obs.ObsOptions) string
// DecodeRemotePathToLocal return remote key path
DecodeRemotePathToLocal(path string) (string, error)
// UpdateObsAkSk updates the cold storage ak and sk
UpdateObsAkSk(ak, sk string) error
// StreamConfUpdate
StreamConfUpdate(propertyName, propertyValue string) error
}