storage

package
v0.28.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 5, 2025 License: Apache-2.0 Imports: 8 Imported by: 0

Documentation

Overview

internal/storage/dedup.go

internal/storage/garbage_collector.go

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Block

type Block struct {
	ID        string
	Size      int64
	CreatedAt time.Time
}

Block represents a data block

type BlockRef

type BlockRef struct {
	Filename  string
	BlockHash string
	Size      int
}

BlockRef represents a reference to a deduplicated block

type Chunk

type Chunk struct {
	Data   []byte
	Hash   string
	Offset int
	Size   int
}

Chunk represents a data chunk

type ContentChunker

type ContentChunker struct {
	// contains filtered or unexported fields
}

ContentChunker performs content-aware chunking

func NewContentChunker

func NewContentChunker(minSize, maxSize int) *ContentChunker

NewContentChunker creates a content-aware chunker

func (*ContentChunker) Split

func (c *ContentChunker) Split(data []byte) []Chunk

Split divides data into content-aware chunks

type DedupStats

type DedupStats struct {
	UniqueBlocks    int
	TotalReferences int
	StoredSize      int
	DedupRatio      float64
}

DedupStats contains deduplication statistics

type DedupStore

type DedupStore struct {
	// contains filtered or unexported fields
}

DedupStore manages deduplicated storage

func NewDedupStore

func NewDedupStore(name string) *DedupStore

NewDedupStore creates a deduplicated store

func (*DedupStore) Get

func (ds *DedupStore) Get(filename string) ([]byte, error)

Get retrieves data by filename

func (*DedupStore) Stats

func (ds *DedupStore) Stats() DedupStats

Stats returns deduplication statistics

func (*DedupStore) Store

func (ds *DedupStore) Store(filename string, data []byte) (*BlockRef, error)

Store saves data with deduplication

func (*DedupStore) UniqueBlocks

func (ds *DedupStore) UniqueBlocks() int

UniqueBlocks returns count of unique blocks

type Deduplicator

type Deduplicator struct {
	// contains filtered or unexported fields
}

Deduplicator tracks blocks for deduplication

func NewDeduplicator

func NewDeduplicator(blockSize int) *Deduplicator

NewDeduplicator creates a new deduplicator

func (*Deduplicator) CheckBlock

func (d *Deduplicator) CheckBlock(data []byte) (string, bool)

CheckBlock checks if block is duplicate

type Delta

type Delta struct {
	Data   []byte
	Offset int
	Type   string // "xor-compressed", "full"
}

Delta represents a change between versions

type DeltaEncoder

type DeltaEncoder struct{}

DeltaEncoder handles delta encoding between versions

func NewDeltaEncoder

func NewDeltaEncoder() *DeltaEncoder

NewDeltaEncoder creates a new delta encoder

func (*DeltaEncoder) ApplyDelta

func (de *DeltaEncoder) ApplyDelta(original []byte, delta *Delta) []byte

ApplyDelta applies a delta to reconstruct the modified version

func (*DeltaEncoder) CreateDelta

func (de *DeltaEncoder) CreateDelta(original, modified []byte) *Delta

CreateDelta creates a delta between original and modified

type FileMetadata

type FileMetadata struct {
	ID           string
	Filename     string
	Size         int64
	Tier         string
	LastAccessed time.Time
	AccessCount  int
}

FileMetadata tracks file information

type GCStats

type GCStats struct {
	TotalBlocks    int
	OrphanedBlocks int
	ExpiredBlocks  int
	TotalSize      int64
	OrphanedSize   int64
	ExpiredSize    int64
}

GCStats contains garbage collection statistics

type GarbageCollector

type GarbageCollector struct {
	// contains filtered or unexported fields
}

GarbageCollector manages cleanup of unused data

func NewGarbageCollector

func NewGarbageCollector() *GarbageCollector

NewGarbageCollector creates a new garbage collector

func (*GarbageCollector) AddBlock

func (gc *GarbageCollector) AddBlock(id string, size int64)

AddBlock registers a block

func (*GarbageCollector) AddBlockWithTime

func (gc *GarbageCollector) AddBlockWithTime(id string, size int64, createdAt time.Time)

AddBlockWithTime registers a block with specific time

func (*GarbageCollector) AddReference

func (gc *GarbageCollector) AddReference(fileID, blockID string)

AddReference adds a reference from file to block

func (*GarbageCollector) Cleanup

func (gc *GarbageCollector) Cleanup() int64

Cleanup removes expired blocks and returns reclaimed space

func (*GarbageCollector) FindExpired

func (gc *GarbageCollector) FindExpired() []string

FindExpired finds blocks older than TTL

func (*GarbageCollector) FindOrphaned

func (gc *GarbageCollector) FindOrphaned() []string

FindOrphaned finds blocks with no references

func (*GarbageCollector) SetTTL

func (gc *GarbageCollector) SetTTL(ttl time.Duration)

SetTTL sets the time-to-live for blocks

func (*GarbageCollector) Stats

func (gc *GarbageCollector) Stats() GCStats

Stats returns garbage collection statistics

type Tier

type Tier struct {
	Name     string
	Capacity int64
	Used     int64
	Files    map[string]*FileMetadata
}

Tier represents a storage tier

type TierLevel

type TierLevel string

TierLevel represents storage tier levels

const (
	HotTier  TierLevel = "hot"
	WarmTier TierLevel = "warm"
	ColdTier TierLevel = "cold"
)

type TierManager

type TierManager struct {
	// contains filtered or unexported fields
}

TierManager manages data across tiers

func NewTierManager

func NewTierManager() *TierManager

NewTierManager creates a tier manager

func (*TierManager) AddTier

func (tm *TierManager) AddTier(name string, capacity int64)

AddTier adds a storage tier

func (*TierManager) GetFileTier

func (tm *TierManager) GetFileTier(id string) string

GetFileTier returns the tier of a file

func (*TierManager) RecordAccess

func (tm *TierManager) RecordAccess(id string)

RecordAccess records file access

func (*TierManager) SimulateAging

func (tm *TierManager) SimulateAging(duration time.Duration)

SimulateAging simulates time passing (for testing)

func (*TierManager) Store

func (tm *TierManager) Store(filename string, data []byte) (string, error)

Store stores data in appropriate tier

type TieringEngine

type TieringEngine struct {
	// contains filtered or unexported fields
}

TieringEngine classifies data by access patterns

func NewTieringEngine

func NewTieringEngine() *TieringEngine

NewTieringEngine creates a new tiering engine

func (*TieringEngine) GetTier

func (te *TieringEngine) GetTier(filename string) TierLevel

GetTier determines the appropriate tier for a file

func (*TieringEngine) RecordAccess

func (te *TieringEngine) RecordAccess(filename string, timestamp time.Time)

RecordAccess records an access event

type Version

type Version struct {
	ID       string
	Filename string
	Size     int
	IsDelta  bool
	BaseID   string // For deltas, the base version
}

Version represents a stored version

type VersionStore

type VersionStore struct {
	// contains filtered or unexported fields
}

VersionStore manages versioned storage with deltas

func NewVersionStore

func NewVersionStore() *VersionStore

NewVersionStore creates a new version store

func (*VersionStore) GetVersion

func (vs *VersionStore) GetVersion(filename string, versionID string) ([]byte, error)

GetVersion retrieves a specific version

func (*VersionStore) Store

func (vs *VersionStore) Store(filename string, content []byte) (string, error)

Store saves the initial version

func (*VersionStore) TotalSize

func (vs *VersionStore) TotalSize() int

TotalSize returns total storage used

func (*VersionStore) Update

func (vs *VersionStore) Update(filename string, content []byte) (string, error)

Update stores a new version as a delta

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL