fracmanager

package
v0.63.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 15, 2025 License: Apache-2.0 Imports: 37 Imported by: 0

Documentation

Index

Constants

View Source 
const StateFile = "storage-state.json"

Variables

View Source 
var ErrSealingFractionSuicided = errors.New("sealing fraction is suicided")

Functions

func NewFracInfoCache added in v0.62.1 

func NewFracInfoCache(filePath string) *fracInfoCache

func NewFracInfoCacheFromDisk added in v0.62.1 

func NewFracInfoCacheFromDisk(filePath string) *fracInfoCache

Types

type CacheMaintainer 

type CacheMaintainer struct {
	// contains filtered or unexported fields
}

func NewCacheMaintainer 

func NewCacheMaintainer(totalCacheSize, sortCacheSize uint64, metrics *CacheMaintainerMetrics) *CacheMaintainer

func (*CacheMaintainer) CreateDocBlockCache 

func (cm *CacheMaintainer) CreateDocBlockCache() *cache.Cache[[]byte]

func (*CacheMaintainer) CreateIndexCache 

func (cm *CacheMaintainer) CreateIndexCache() *frac.IndexCache

func (*CacheMaintainer) CreateSortDocsCache 

func (cm *CacheMaintainer) CreateSortDocsCache() *cache.Cache[[]byte]

func (*CacheMaintainer) Reset 

func (cm *CacheMaintainer) Reset()

Reset is used in tests only

func (*CacheMaintainer) RunCleanLoop 

func (cm *CacheMaintainer) RunCleanLoop(done <-chan struct{}, cleanupInterval, gcInterval time.Duration)

type CacheMaintainerMetrics 

type CacheMaintainerMetrics struct {
	HitsTotal       *prometheus.CounterVec
	MissTotal       *prometheus.CounterVec
	PanicsTotal     *prometheus.CounterVec
	LockWaitsTotal  *prometheus.CounterVec
	WaitsTotal      *prometheus.CounterVec
	ReattemptsTotal *prometheus.CounterVec
	SizeRead        *prometheus.CounterVec
	SizeOccupied    *prometheus.CounterVec
	SizeReleased    *prometheus.CounterVec
	MapsRecreated   *prometheus.CounterVec
	MissLatency     *prometheus.CounterVec

	Oldest            *prometheus.GaugeVec
	AddBuckets        *prometheus.CounterVec
	DelBuckets        *prometheus.CounterVec
	CleanGenerations  *prometheus.CounterVec
	ChangeGenerations *prometheus.CounterVec
}

func (*CacheMaintainerMetrics) GetCleanerMetrics 

func (m *CacheMaintainerMetrics) GetCleanerMetrics(cleanerLabel string) *cache.CleanerMetrics

func (*CacheMaintainerMetrics) GetLayerMetrics 

func (m *CacheMaintainerMetrics) GetLayerMetrics(layerName string) *cache.Metrics

type Config 

type Config struct {
	DataDir string

	FracSize  uint64
	TotalSize uint64
	CacheSize uint64

	ReplayWorkers     int
	MaintenanceDelay  time.Duration
	CacheCleanupDelay time.Duration
	CacheGCDelay      time.Duration
	SealParams        common.SealParams
	SortCacheSize     uint64 // size for docs cache for active fraction
	Fraction          frac.Config

	OffloadingEnabled   bool
	OffloadingRetention time.Duration
}

func FillConfigWithDefault 

func FillConfigWithDefault(config *Config) *Config

type Fetcher 

type Fetcher struct {
	// contains filtered or unexported fields
}

func NewFetcher 

func NewFetcher(maxWorkersNum int) *Fetcher

func (*Fetcher) FetchDocs 

func (f *Fetcher) FetchDocs(ctx context.Context, fracs List, ids []seq.IDSource) ([][]byte, error)

type FracManager 

type FracManager struct {
	// contains filtered or unexported fields
}

func New added in v0.62.5 

func New(ctx context.Context, cfg *Config, s3cli *s3.Client) (*FracManager, error)

func (*FracManager) Active 

func (fm *FracManager) Active() frac.Fraction

func (*FracManager) Append 

func (fm *FracManager) Append(ctx context.Context, docs, metas storage.DocBlock) error

func (*FracManager) Flags added in v0.62.3 

func (fm *FracManager) Flags() *StateManager

func (*FracManager) Fractions added in v0.62.5 

func (fm *FracManager) Fractions() (fracs List)

Fractions returns a list of known fracs (local and remote).

While working with this list, it may become irrelevant (factions may, for example, be deleted). This is a valid situation, because access to the data of these factions (search and fetch) occurs under blocking (see DataProvider). This way we avoid the race.

Accessing the deleted faction data just will return an empty result.

func (*FracManager) Oldest added in v0.62.5 

func (fm *FracManager) Oldest() uint64

func (*FracManager) SealForcedForTests 

func (fm *FracManager) SealForcedForTests()

func (*FracManager) Start 

func (fm *FracManager) Start()

func (*FracManager) Stop 

func (fm *FracManager) Stop()

func (*FracManager) WaitIdleForTests added in v0.62.5 

func (fm *FracManager) WaitIdleForTests()

func (*FracManager) Writer 

func (fm *FracManager) Writer() *proxyFrac

type FracType added in v0.60.0 

type FracType int
const (
	FracTypeLocal FracType = 1 << iota
	FracTypeRemote
)

type List 

type List []frac.Fraction

func (List) FilterInRange 

func (l List) FilterInRange(from, to seq.MID) List

func (List) GetOldestFrac 

func (l List) GetOldestFrac() frac.Fraction

func (List) GetTotalSize 

func (l List) GetTotalSize() uint64

func (*List) Shift 

func (l *List) Shift(n int) []frac.Fraction

func (List) Sort 

func (l List) Sort(order seq.DocsOrder)

type Loader added in v0.62.3 

type Loader struct {
	// contains filtered or unexported fields
}

Loader is responsible for loading and initializing fractions from filesystem Coordinates the process of discovering, validating, and loading all fraction types

func NewLoader 

func NewLoader(config *Config, provider *fractionProvider, infoCache *fracInfoCache) *Loader

NewLoader creates a new fraction loader Initialized at system startup to prepare data

func (*Loader) Load added in v0.62.3 

func (l *Loader) Load(ctx context.Context) (*frac.Active, []*frac.Sealed, []*frac.Remote, error)

Load is the main method for loading all fractions Coordinates the entire process: discovery, validation, recovery, and ordering

type Searcher 

type Searcher struct {
	// contains filtered or unexported fields
}

func NewSearcher 

func NewSearcher(maxWorkersNum int, cfg SearcherCfg) *Searcher

func (*Searcher) SearchDocs 

func (s *Searcher) SearchDocs(ctx context.Context, fracs []frac.Fraction, params processor.SearchParams) (*seq.QPR, error)

type SearcherCfg 

type SearcherCfg struct {
	MaxFractionHits       int // the maximum number of fractions used in the search
	FractionsPerIteration int
}

type StateManager added in v0.62.3 

type StateManager struct {
	// contains filtered or unexported fields
}

StateManager manages storage state with thread safety

func NewStateManager added in v0.62.3 

func NewStateManager(dataDir string, defaultState StorageState) (*StateManager, error)

NewStateManager creates a new storage state manager

func (*StateManager) IsCapacityExceeded added in v0.62.3 

func (m *StateManager) IsCapacityExceeded() bool

IsCapacityExceeded returns storage capacity exceeded flag

type StorageState added in v0.62.3 

type StorageState struct {
	CapacityExceeded bool `json:"capacity_exceeded"` // Storage capacity exceeded flag
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.