storage

package
v0.0.0-...-3f4335e Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 5, 2025 License: MIT Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source
const (
	OpTypePut    byte = 1
	OpTypeDelete byte = 2
)

WAL operation types

Variables

This section is empty.

Functions

This section is empty.

Types

type Checkpoint

type Checkpoint struct {
	// contains filtered or unexported fields
}

Checkpoint represents a snapshot of the memory table

func NewCheckpoint

func NewCheckpoint(baseDir string) (*Checkpoint, error)

NewCheckpoint creates a new checkpoint manager

func (*Checkpoint) GetLastWALTimestamp

func (c *Checkpoint) GetLastWALTimestamp() int64

GetLastWALTimestamp returns the last WAL timestamp included in the checkpoint

func (*Checkpoint) Load

func (c *Checkpoint) Load() (map[string][]byte, int64, int64, error)

Load loads the memory table from a checkpoint file

func (*Checkpoint) Save

func (c *Checkpoint) Save(memTable map[string][]byte, memTableSize int64, lastWALTimestamp int64) error

Save saves the current memory table to a checkpoint file

type CheckpointData

type CheckpointData struct {
	// Timestamp when the checkpoint was created
	Timestamp int64 `json:"timestamp"`

	// Last WAL entry timestamp included in this checkpoint
	LastWALTimestamp int64 `json:"last_wal_timestamp"`

	// Memory table data
	MemTable map[string][]byte `json:"mem_table"`

	// Memory table size
	MemTableSize int64 `json:"mem_table_size"`
}

CheckpointData represents the data stored in a checkpoint file

type CompactionManager

type CompactionManager struct {
	// contains filtered or unexported fields
}

CompactionManager handles background compaction of LSM tree levels

func NewCompactionManager

func NewCompactionManager(tree *LSMTree, dataDir string, numWorkers int) *CompactionManager

NewCompactionManager creates a new compaction manager

func (*CompactionManager) GetStats

func (c *CompactionManager) GetStats() CompactionStats

GetStats returns the current compaction statistics

func (*CompactionManager) RunCompaction

func (c *CompactionManager) RunCompaction() error

RunCompaction runs a compaction cycle

func (*CompactionManager) ScheduleCompaction

func (c *CompactionManager) ScheduleCompaction(sourceLevel, targetLevel int, blocks []blockInfo)

ScheduleCompaction schedules a compaction task

func (*CompactionManager) Start

func (c *CompactionManager) Start()

Start starts the compaction workers

func (*CompactionManager) Stop

func (c *CompactionManager) Stop()

Stop stops the compaction workers

type CompactionStats

type CompactionStats struct {
	// Number of compactions performed
	CompactionCount int

	// Number of blocks compacted
	BlocksCompacted int

	// Number of bytes read
	BytesRead int64

	// Number of bytes written
	BytesWritten int64

	// Total compaction time
	TotalTime time.Duration

	// CPU usage percentage (0-100)
	CPUUsagePercent float64

	// Number of compaction tasks in queue
	TasksInQueue int

	// Number of compaction tasks dropped due to queue full
	TasksDropped int

	// Last compaction timestamp
	LastCompactionTime time.Time

	// Compaction throughput (bytes/second)
	CompactionThroughput float64
}

CompactionStats tracks statistics about compaction operations

type Engine

type Engine struct {
	// contains filtered or unexported fields
}

Engine is the main storage engine that integrates LSM tree, WAL, and compaction

func NewEngine

func NewEngine(baseDir string) (*Engine, error)

NewEngine creates a new storage engine

func (*Engine) Close

func (e *Engine) Close() error

Close closes the storage engine and releases resources

func (*Engine) Delete

func (e *Engine) Delete(key []byte) error

Delete removes a key-value pair

func (*Engine) Get

func (e *Engine) Get(key []byte) ([]byte, error)

Get retrieves a value for a key

func (*Engine) GetStats

func (e *Engine) GetStats() Stats

GetStats returns statistics about the storage engine

func (*Engine) Put

func (e *Engine) Put(key, value []byte) error

Put stores a key-value pair

func (*Engine) RunCompaction

func (e *Engine) RunCompaction() error

RunCompaction manually triggers a compaction cycle

type FileData

type FileData struct {
	// File path
	Path string `json:"path"`

	// File size
	Size int64 `json:"size"`

	// Timestamp when the file was created
	Timestamp int64 `json:"timestamp"`

	// Min key in the file
	MinKey string `json:"min_key"`

	// Max key in the file
	MaxKey string `json:"max_key"`

	// Number of entries in the file
	EntryCount int `json:"entry_count"`
}

FileData represents data about a file in the LSM tree

type LSMTree

type LSMTree struct {
	// contains filtered or unexported fields
}

LSMTree implements a Log-Structured Merge Tree for efficient storage with level-triggered compaction.

func NewLSMTree

func NewLSMTree(dataDir string) (*LSMTree, error)

NewLSMTree creates a new LSM tree with the given data directory

func (*LSMTree) Close

func (t *LSMTree) Close() error

Close closes the LSM tree and releases resources

func (*LSMTree) Read

func (t *LSMTree) Read(key []byte) ([]byte, error)

Read reads data from the LSM tree, searching through all levels

func (*LSMTree) StartCompactionWorker

func (t *LSMTree) StartCompactionWorker()

StartCompactionWorker starts a background goroutine for compaction

func (*LSMTree) Write

func (t *LSMTree) Write(b *block.Block) error

Write adds a new block to the LSM tree (level 0)

type LevelData

type LevelData struct {
	// Level number
	Level int `json:"level"`

	// Files in this level
	Files []FileData `json:"files"`
}

LevelData represents data about a level in the LSM tree

type Manifest

type Manifest struct {
	// contains filtered or unexported fields
}

Manifest represents the state of the LSM tree

func NewManifest

func NewManifest(baseDir string) (*Manifest, error)

NewManifest creates a new manifest

func (*Manifest) GetCurrentWAL

func (m *Manifest) GetCurrentWAL() string

GetCurrentWAL returns the current WAL file

func (*Manifest) GetLastCheckpoint

func (m *Manifest) GetLastCheckpoint() int64

GetLastCheckpoint returns the last checkpoint timestamp

func (*Manifest) GetLevelFiles

func (m *Manifest) GetLevelFiles(level int) ([]FileData, error)

GetLevelFiles returns the files in a level

func (*Manifest) Save

func (m *Manifest) Save() error

Save saves the manifest to disk

func (*Manifest) UpdateCurrentWAL

func (m *Manifest) UpdateCurrentWAL(walFile string) error

UpdateCurrentWAL updates the current WAL file

func (*Manifest) UpdateLastCheckpoint

func (m *Manifest) UpdateLastCheckpoint(timestamp int64) error

UpdateLastCheckpoint updates the last checkpoint timestamp

func (*Manifest) UpdateLevel

func (m *Manifest) UpdateLevel(level int, files []FileData) error

UpdateLevel updates the files in a level

type ManifestData

type ManifestData struct {
	// Timestamp when the manifest was created
	Timestamp int64 `json:"timestamp"`

	// LSM tree levels
	Levels []LevelData `json:"levels"`

	// Current WAL file
	CurrentWAL string `json:"current_wal"`

	// Last checkpoint timestamp
	LastCheckpoint int64 `json:"last_checkpoint"`
}

ManifestData represents the data stored in a manifest file

type MmapBlock

type MmapBlock struct {
	// contains filtered or unexported fields
}

MmapBlock represents a memory-mapped block file with index

func NewMmapBlock

func NewMmapBlock(path string) (*MmapBlock, error)

NewMmapBlock creates a new memory-mapped block

func (*MmapBlock) Close

func (b *MmapBlock) Close() error

Close closes the block and releases resources

func (*MmapBlock) Get

func (b *MmapBlock) Get(key []byte) ([]byte, error)

Get retrieves a value for a key from the block

func (*MmapBlock) MaxKey

func (b *MmapBlock) MaxKey() []byte

MaxKey returns the maximum key in the block

func (*MmapBlock) MinKey

func (b *MmapBlock) MinKey() []byte

MinKey returns the minimum key in the block

func (*MmapBlock) Size

func (b *MmapBlock) Size() int64

Size returns the size of the block in bytes

type MmapFile

type MmapFile struct {
	// contains filtered or unexported fields
}

MmapFile represents a memory-mapped file for zero-copy reads

func NewMmapFile

func NewMmapFile(path string) (*MmapFile, error)

NewMmapFile creates a new memory-mapped file

func (*MmapFile) Close

func (m *MmapFile) Close() error

Close closes the memory-mapped file and releases resources

func (*MmapFile) Data

func (m *MmapFile) Data() ([]byte, error)

Data returns the entire memory-mapped data as a byte slice This is a zero-copy operation

func (*MmapFile) Read

func (m *MmapFile) Read(offset, length int64) ([]byte, error)

Read reads data from the memory-mapped file without copying

func (*MmapFile) ReadAt

func (m *MmapFile) ReadAt(p []byte, offset int64) (int, error)

ReadAt reads data from the memory-mapped file at a specific offset

func (*MmapFile) Size

func (m *MmapFile) Size() int64

Size returns the size of the memory-mapped file

type Stats

type Stats struct {
	// Memory table size
	MemTableSize int64

	// Number of keys in memory table
	MemTableKeys int

	// Compaction statistics
	CompactionStats CompactionStats

	// LSM tree level sizes
	LevelSizes [7]int64

	// LSM tree level block counts
	LevelBlocks [7]int
}

Stats returns statistics about the storage engine

type WAL

type WAL struct {
	// contains filtered or unexported fields
}

WAL (Write-Ahead Log) provides durability guarantees by logging operations before they are applied to the main data structure.

func NewWAL

func NewWAL(walDir string) (*WAL, error)

NewWAL creates a new WAL with the given directory

func (*WAL) AppendDelete

func (w *WAL) AppendDelete(key []byte) error

AppendDelete appends a DELETE operation to the WAL

func (*WAL) AppendPut

func (w *WAL) AppendPut(key, value []byte) error

AppendPut appends a PUT operation to the WAL

func (*WAL) Close

func (w *WAL) Close() error

Close closes the WAL and releases resources

func (*WAL) Replay

func (w *WAL) Replay(callback func(entry WALEntry) error) error

Replay replays the WAL entries and applies them to the given callback function

func (*WAL) ReplayFrom

func (w *WAL) ReplayFrom(fromTimestamp int64, callback func(entry WALEntry) error) error

ReplayFrom replays the WAL entries from the given timestamp and applies them to the given callback function

type WALEntry

type WALEntry struct {
	// Timestamp of the entry
	Timestamp int64

	// Type of operation (e.g., PUT, DELETE)
	OpType byte

	// Key and value
	Key, Value []byte
}

WALEntry represents a single entry in the WAL

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL