Documentation
¶
Index ¶
- Constants
- Variables
- func EstimateBlockSize(dataSize int64) int64
- func WalFileName(dir string, fid uint32, ext string) string
- type ChunkHeader
- type ChunkIterator
- type ChunkPos
- type ChunkType
- type F
- type File
- type FileIterator
- type LogFile
- func (log *LogFile) Close() error
- func (log *LogFile) Fid() uint32
- func (log *LogFile) Iterator() (ChunkIterator, error)
- func (log *LogFile) MarkImmutable()
- func (log *LogFile) Read(block uint32, chunkOffset int64) ([]byte, error)
- func (log *LogFile) Remove() error
- func (log *LogFile) Size() int64
- func (log *LogFile) Sync() error
- func (log *LogFile) Write(data []byte) (ChunkPos, error)
- func (log *LogFile) WriteAll(data [][]byte) ([]ChunkPos, error)
- type LogFileChunkIterator
- type Option
- type Pending
- type Stat
- type Wal
- func (w *Wal) ActiveFid() uint32
- func (w *Wal) Close() error
- func (w *Wal) IsEmpty() bool
- func (w *Wal) Iterator(minFid, maxFid uint32, pos ChunkPos) (FileIterator, error)
- func (w *Wal) Pending(capacity int) *Pending
- func (w *Wal) Purge() error
- func (w *Wal) Read(pos ChunkPos) ([]byte, error)
- func (w *Wal) Rotate() error
- func (w *Wal) Stat() Stat
- func (w *Wal) Sync() error
- func (w *Wal) Write(data []byte) (ChunkPos, error)
- func (w *Wal) WriteAll(datas [][]byte, needSync bool) ([]ChunkPos, error)
Constants ¶
const ChunkHeaderSize = 7
ChunkHeaderSize +-----------+--------------+------------+------------+ | check sum | chunk length | chunk type | chunk data | +-----------+--------------+------------+------------+ | 4 Bytes | 2 Bytes | 1 Bytes | <= 32KB-7B | +-----------+--------------+------------+------------+
const DefaultWalSuffix = "wal"
DefaultWalSuffix is the extension name of wal file, default is wal. eg: 001.wal
const MaxBlockSize = 32 * types.KB
Variables ¶
var ( ErrAlreadyClosed = errors.New("file already closed") ErrBlockExceeded = errors.New("block size exceeded") ErrWriteToImmutable = errors.New("write to immutable") )
var ( ErrFileNotFound = errors.New("wal file not found") ErrDataExceedFile = errors.New("data size has exceeded the max file limit") )
Functions ¶
func EstimateBlockSize ¶ added in v0.3.0
EstimateBlockSize estimate the possible max size of data will be written in wal file if the first written block has already contains chunks, maybe occurs the first data chunk could be hold by the block but no left space to hold the second chunk header, in that case, the block need padding to fill the left space. padding + block * header + data size
Types ¶
type ChunkHeader ¶
type ChunkIterator ¶
type ChunkIterator interface {
// File returns file that iterator belongs to
File() File
// Index returns current chunk position
Index() ChunkPos
// Next returns the next chunk data bytes, chunk info, and error
Next() ([]byte, ChunkPos, error)
}
ChunkIterator iterate chunk data for a log file
type File ¶
type File interface {
// Fid return file id
Fid() uint32
// Read bytes with specified block and offset
// chunkOffset is offset by the start of the specified block
Read(block uint32, chunkOffset int64) ([]byte, error)
// Write bytes and returns ChunkPos
Write(data []byte) (ChunkPos, error)
// WriteAll write bytes in batch and returns ChunkPos
WriteAll(data [][]byte) ([]ChunkPos, error)
// Iterator iterate per chunk in file
Iterator() (ChunkIterator, error)
// Size return size of file
Size() int64
// Remove remove file
Remove() error
// Sync buffer to disk
Sync() error
// MarkImmutable mark this file as immutable
MarkImmutable()
// Close file and release resource
Close() error
}
File represents a wal file
type FileIterator ¶
type FileIterator struct {
// contains filtered or unexported fields
}
FileIterator iterates over all the files and chunks
func (*FileIterator) IndexFid ¶
func (f *FileIterator) IndexFid() uint32
IndexFid returns the current fid
func (*FileIterator) IndexPos ¶
func (f *FileIterator) IndexPos() ChunkPos
IndexPos returns curren index position
func (*FileIterator) NextFile ¶
func (f *FileIterator) NextFile()
type LogFile ¶
type LogFile struct {
// contains filtered or unexported fields
}
func (*LogFile) Iterator ¶
func (log *LogFile) Iterator() (ChunkIterator, error)
func (*LogFile) MarkImmutable ¶
func (log *LogFile) MarkImmutable()
type LogFileChunkIterator ¶
type LogFileChunkIterator struct {
// contains filtered or unexported fields
}
func (*LogFileChunkIterator) File ¶
func (l *LogFileChunkIterator) File() File
func (*LogFileChunkIterator) Index ¶
func (l *LogFileChunkIterator) Index() ChunkPos
type Option ¶
type Option struct {
// data dir
DataDir string
// max log file size
MaxFileSize int64
// log file extension
Ext string
// how many block will be cached, off if it is 0
BlockCache uint32
// sync buffer to disk per write, if not enabled data maybe not save when machine crashed
FsyncPerWrite bool
// specify the written threshold for triggering sync
FsyncThreshold int64
}
func DefaultOption ¶ added in v0.2.0
type Pending ¶ added in v0.2.0
type Pending struct {
// contains filtered or unexported fields
}
Pending provides a way to write data in batches
type Wal ¶
type Wal struct {
// contains filtered or unexported fields
}
Wal instance represents a data directory and its sub-files that store write-ahead-log format data
func (*Wal) Iterator ¶
func (w *Wal) Iterator(minFid, maxFid uint32, pos ChunkPos) (FileIterator, error)
Iterator returns an FileIterator maxFid determines the file range for the iterator pos determines the chunk position for the iterator
Source Files