common

package
v0.0.0-debug-20260702 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 2, 2026 License: Apache-2.0 Imports: 26 Imported by: 0

Documentation

Overview

A few allocators for TAE

Index

Constants

View Source 
const (
	DefaultMinOsizeQualifiedMB    = 90 // MB
	DefaultMinOsizeQualifiedBytes = DefaultMinOsizeQualifiedMB * Const1MBytes
	DefaultMaxOsizeObjMB          = 128 // MB
	DefaultMaxOsizeObjBytes       = DefaultMaxOsizeObjMB * Const1MBytes
	DefaultMinCNMergeSize         = 80000 // MB
	DefaultCNMergeMemControlHint  = 8192  // MB
	DefaultMaxMergeObjN           = 16

	Const1GBytes = 1 << 30
	Const1MBytes = 1 << 20
)
View Source 
const (
	CloseIntervalSize int64 = int64(unsafe.Sizeof(ClosedInterval{}))
)
View Source 
const DefaultMaxRowsToPrint = 3
View Source 
const (
	IDSize int64 = int64(unsafe.Sizeof(ID{}))
)

Variables

View Source 
var (
	RuntimeMaxMergeObjN        atomic.Int32
	RuntimeOsizeRowsQualified  atomic.Uint32
	RuntimeMaxObjOsize         atomic.Uint32
	RuntimeMinCNMergeSize      atomic.Uint64
	RuntimeCNMergeMemControl   atomic.Uint64
	RuntimeCNTakeOverAll       atomic.Bool
	IsStandaloneBoost          atomic.Bool
	ShouldStandaloneCNTakeOver atomic.Bool
	RuntimeDisableZMBasedMerge atomic.Bool

	RuntimeOverallFlushMemCap atomic.Uint64

	FlushMemCapacity atomic.Int32
)
View Source 
var AnyField = zap.Any
View Source 
var CheckpointAllocator *mpool.MPool
View Source 
var DebugAllocator *mpool.MPool
View Source 
var DefaultAllocator *mpool.MPool

A few allocators for TAE

View Source 
var (
	GlobalSeqNum atomic.Uint64
)
View Source 
var ISCPAllocator *mpool.MPool
View Source 
var LogtailAllocator *mpool.MPool
View Source 
var MergeAllocator *mpool.MPool
View Source 
var MutMemAllocator *mpool.MPool
View Source 
var PublicationAllocator *mpool.MPool
View Source 
var SmallAllocator *mpool.MPool
View Source 
var StringerField = zap.Stringer
View Source 
var WorkspaceAllocator *mpool.MPool

Functions

func ApiBatchToString added in v1.0.0 

func ApiBatchToString(apiBat *api.Batch, printN int) string

func BitmapEqual added in v0.8.0 

func BitmapEqual(v1, v2 *nulls.Bitmap) bool

func ContextField 

func ContextField(format string, a ...any) zap.Field

func CountField 

func CountField(val int) zap.Field

func DebugMoBatch added in v0.8.0 

func DebugMoBatch(moBat *batch.Batch) string

func DoIfDebugEnabled added in v0.8.0 

func DoIfDebugEnabled(fn func())

func DoIfErrorEnabled added in v0.8.0 

func DoIfErrorEnabled(fn func())

func DoIfFatalEnabled added in v0.8.0 

func DoIfFatalEnabled(fn func())

func DoIfInfoEnabled added in v0.8.0 

func DoIfInfoEnabled(fn func())

func DoIfWarnEnabled added in v0.8.0 

func DoIfWarnEnabled(fn func())

func DurationField 

func DurationField(val time.Duration) zap.Field

func EncodeCloseInterval added in v0.8.0 

func EncodeCloseInterval(i *ClosedInterval) []byte

func EncodeID added in v0.8.0 

func EncodeID(id *ID) []byte

func EntityField 

func EntityField(val any) zap.Field

func ErrorField 

func ErrorField(val error) zap.Field

func ExceptionField 

func ExceptionField(val any) zap.Field

func FormatFiled 

func FormatFiled(key string, format string, a ...any) zap.Field

func GetGlobalSeqNum 

func GetGlobalSeqNum() uint64

func HumanReadableBytes added in v1.0.0 

func HumanReadableBytes(bytes int) string

func IDField 

func IDField(val int) zap.Field

func InitTAEMPool added in v0.6.0 

func InitTAEMPool()

func LoopGenericSortedDList added in v0.6.0 

func LoopGenericSortedDList[T any](head *GenericDLNode[T],
	fn func(node *GenericDLNode[T]) bool,
	reverse bool)

Loop the list and apply fn on each node head is the head node of a list fn is operation applied to each node during iterating. if fn(node) returns false, stop iterating. reverse is true to loop in reversed way.

func MoBatchToString added in v1.0.0 

func MoBatchToString(moBat *batch.Batch, printN int) string

func MoVectorToString added in v1.0.0 

func MoVectorToString(v *vector.Vector, printN int, opts ...TypePrintOpt) string

func NameSpaceField 

func NameSpaceField(val string) zap.Field

func NextGlobalSeqNum 

func NextGlobalSeqNum() uint64

func OperandField 

func OperandField(val any) zap.Field

func OperandNameSpace 

func OperandNameSpace() zap.Field

func OperationField 

func OperationField(val string) zap.Field

func ReasonField 

func ReasonField(val string) zap.Field

func RepeatStr 

func RepeatStr(str string, times int) string

func ReprerField 

func ReprerField(key string, val Reprer) zap.Field

func RetryWithInterval 

func RetryWithInterval(
	ctx context.Context,
	op WaitOp,
	interval time.Duration,
) (err error)

func RoaringToMOBitmap added in v0.8.0 

func RoaringToMOBitmap(bm *roaring.Bitmap) *nulls.Bitmap

func TimestampField 

func TimestampField(val any) zap.Field

func TypeStringValue added in v0.6.0 

func TypeStringValue(t types.Type, v any, isNull bool, opts ...TypePrintOpt) string

Types

type ClosedInterval 

type ClosedInterval struct {
	Start, End uint64
}

func (*ClosedInterval) Append 

func (i *ClosedInterval) Append(id uint64) error

func (*ClosedInterval) AtomicUpdateEnd 

func (i *ClosedInterval) AtomicUpdateEnd(v uint64)

func (*ClosedInterval) Contains 

func (i *ClosedInterval) Contains(o ClosedInterval) bool

func (*ClosedInterval) GT 

func (i *ClosedInterval) GT(o *ClosedInterval) bool

func (*ClosedInterval) IsCoveredByInt 

func (i *ClosedInterval) IsCoveredByInt(idx uint64) bool

func (*ClosedInterval) LT 

func (i *ClosedInterval) LT(o *ClosedInterval) bool

func (*ClosedInterval) String 

func (i *ClosedInterval) String() string

func (*ClosedInterval) TryMerge 

func (i *ClosedInterval) TryMerge(o ClosedInterval) bool

func (*ClosedInterval) Uint64Contains added in v1.0.0 

func (i *ClosedInterval) Uint64Contains(start, end uint64) bool

type ClosedIntervals 

type ClosedIntervals struct {
	Intervals []*ClosedInterval
}

func NewClosedIntervals 

func NewClosedIntervals() *ClosedIntervals

func NewClosedIntervalsByInt 

func NewClosedIntervalsByInt(i uint64) *ClosedIntervals

func NewClosedIntervalsByInterval 

func NewClosedIntervalsByInterval(i *ClosedInterval) *ClosedIntervals

func NewClosedIntervalsByIntervals 

func NewClosedIntervalsByIntervals(i *ClosedIntervals) *ClosedIntervals

func NewClosedIntervalsBySlice 

func NewClosedIntervalsBySlice(array []uint64) *ClosedIntervals

func (*ClosedIntervals) Contains 

func (intervals *ClosedIntervals) Contains(o ClosedIntervals) bool

func (*ClosedIntervals) ContainsInt added in v0.6.0 

func (intervals *ClosedIntervals) ContainsInt(n uint64) bool

func (*ClosedIntervals) ContainsInterval 

func (intervals *ClosedIntervals) ContainsInterval(oIntervals ClosedInterval) bool

func (*ClosedIntervals) Equal 

func (intervals *ClosedIntervals) Equal(o *ClosedIntervals) bool

Equal is for test

func (*ClosedIntervals) GetCardinality 

func (intervals *ClosedIntervals) GetCardinality() int

func (*ClosedIntervals) GetMax added in v0.6.0 

func (intervals *ClosedIntervals) GetMax() uint64

func (*ClosedIntervals) GetMin added in v0.6.0 

func (intervals *ClosedIntervals) GetMin() uint64

func (*ClosedIntervals) IsCoveredByInt 

func (intervals *ClosedIntervals) IsCoveredByInt(i uint64) bool

func (*ClosedIntervals) ReadFrom 

func (intervals *ClosedIntervals) ReadFrom(r io.Reader) (n int64, err error)

func (*ClosedIntervals) TryMerge 

func (intervals *ClosedIntervals) TryMerge(o *ClosedIntervals)

func (*ClosedIntervals) WriteTo 

func (intervals *ClosedIntervals) WriteTo(w io.Writer) (n int64, err error)

type GenericDLNode added in v0.6.0 

type GenericDLNode[T any] struct {
	// contains filtered or unexported fields
}

Doubly sorted linked-list node

func InsertGenericDLNode added in v0.6.0 

func InsertGenericDLNode[T any](payload T,
	head *GenericDLNode[T],
	compare func(T, T) int) (node, nhead, ntail *GenericDLNode[T])

Insert a wrapped object into a list specified by a head node node is the inserted dlnode nhead is the new head node ntail is the new tail node. If ntail is not nil, tail is updated. Else tail is not updated

func (*GenericDLNode[T]) GetNext added in v0.6.0 

func (l *GenericDLNode[T]) GetNext() *GenericDLNode[T]

func (*GenericDLNode[T]) GetPayload added in v0.6.0 

func (l *GenericDLNode[T]) GetPayload() T

func (*GenericDLNode[T]) GetPrev added in v0.6.0 

func (l *GenericDLNode[T]) GetPrev() *GenericDLNode[T]

func (*GenericDLNode[T]) KeepSorted added in v0.6.0 

func (l *GenericDLNode[T]) KeepSorted(compare func(T, T) int) (head, tail *GenericDLNode[T])

Keep node be sorted in the list

type GenericSortedDList added in v0.6.0 

type GenericSortedDList[T any] struct {
	// contains filtered or unexported fields
}

Sorted doubly linked-list

func NewGenericSortedDList added in v0.6.0 

func NewGenericSortedDList[T any](compare func(T, T) int) *GenericSortedDList[T]

func (*GenericSortedDList[T]) Delete added in v0.6.0 

func (l *GenericSortedDList[T]) Delete(n *GenericDLNode[T])

Given a node and remove it from the list 

Delete [node]

[prev node] <-> [node] <-> [next node] =============> [prev node] <-> [next node]

func (*GenericSortedDList[T]) Depth added in v0.6.0 

func (l *GenericSortedDList[T]) Depth() int

Get the length of the list

func (*GenericSortedDList[T]) GetHead added in v0.6.0 

func (l *GenericSortedDList[T]) GetHead() *GenericDLNode[T]

Get the head node

func (*GenericSortedDList[T]) GetTail added in v0.6.0 

func (l *GenericSortedDList[T]) GetTail() *GenericDLNode[T]

Get the tail node

func (*GenericSortedDList[T]) Insert added in v0.6.0 

func (l *GenericSortedDList[T]) Insert(payload T) *GenericDLNode[T]

Insert a object and wrap it as GenericDLNode into the list The inserted object must be instance of interface NodePayload [List]: [1,x1] <-> [5,x5] <-> [10,x10] Insert a node [7,x7] [List]: [1,x1] <-> [5,x5] <-> [7,x7] <-> [10,x10]

func (*GenericSortedDList[T]) Loop added in v0.6.0 

func (l *GenericSortedDList[T]) Loop(fn func(n *GenericDLNode[T]) bool, reverse bool)

Loop the list and apply fn on each node

func (*GenericSortedDList[T]) Update added in v0.6.0 

func (l *GenericSortedDList[T]) Update(n *GenericDLNode[T])

Update the node to keep the list be sorted

[List] [1,x1] <-> [3,x3] <-> [10,x10] <-> [20,x20] 

|

[Node] [10,x10]

--------- UPDATE [10,x10] TO [2, x10]--------------

[List] [1,x1] <-> [2,x10] <-> [3,x3] <-> [20,x20]

type GenericSortedDListIt added in v0.6.0 

type GenericSortedDListIt[T any] struct {
	// contains filtered or unexported fields
}

Sorted doubly linked-list iterator

func NewGenericSortedDListIt added in v0.6.0 

func NewGenericSortedDListIt[T any](linkLocker *sync.RWMutex,
	dlist *GenericSortedDList[T],
	reverse bool) *GenericSortedDListIt[T]

linkLocker is the outer locker to guard dlist access

func (*GenericSortedDListIt[T]) Get added in v0.6.0 

func (it *GenericSortedDListIt[T]) Get() *GenericDLNode[T]

func (*GenericSortedDListIt[T]) Next added in v0.6.0 

func (it *GenericSortedDListIt[T]) Next()

func (*GenericSortedDListIt[T]) Valid added in v0.6.0 

func (it *GenericSortedDListIt[T]) Valid() bool

type ID 

type ID struct {
	// Internal db id
	DbID uint64
	// Internal table id
	TableID uint64
	// Internal block id
	BlockID types.Blockid
}

ID is the general identifier type shared by different types like table, object, block, etc.

We could wrap info from upper level via ID, for instance, get the table id, object id, and the block id for one block by ID.AsBlockID, which made the resource management easier.

func (*ID) BlockString 

func (id *ID) BlockString() string

func (*ID) DBString added in v0.8.0 

func (id *ID) DBString() string

func (*ID) ObjectID added in v1.1.0 

func (id *ID) ObjectID() *types.Objectid

func (*ID) ObjectString added in v1.1.0 

func (id *ID) ObjectString() string

func (*ID) SegmentID 

func (id *ID) SegmentID() *types.Segmentid

func (*ID) SetBlockOffset added in v1.2.0 

func (id *ID) SetBlockOffset(blkn uint16)

func (*ID) SetObjectID added in v1.1.0 

func (id *ID) SetObjectID(oid *types.Objectid)

func (*ID) SetSegmentID added in v0.8.0 

func (id *ID) SetSegmentID(sid *types.Segmentid)

func (*ID) String 

func (id *ID) String() string

func (*ID) TableString 

func (id *ID) TableString() string

type IRef 

type IRef interface {
	RefCount() int64
	// RefIfHasRef increment refcnt if existing cnt > 0 and return true,
	// return false if cnt is zero. Note: the update is atomic
	RefIfHasRef() bool
	Ref()
	Unref()
}

IRef is the general representation of the resources that should be managed with a reference count. Once the reference count reached 0, the OnZeroCB would be called.

type ISLLNode 

type ISLLNode interface {
	IRef
}

type ISSLLNode 

type ISSLLNode interface {
	Insert(ISSLLNode)
	GetNext() ISSLLNode
	SetNext(ISSLLNode)
	ReleaseNextNode() ISSLLNode
	ReleaseFollowing() ISSLLNode
}

type IdAllocator 

type IdAllocator struct {
	// contains filtered or unexported fields
}

func NewIdAllocator 

func NewIdAllocator(from uint64) *IdAllocator

func (*IdAllocator) Alloc 

func (alloc *IdAllocator) Alloc() uint64

func (*IdAllocator) Get 

func (alloc *IdAllocator) Get() uint64

func (*IdAllocator) SetStart 

func (alloc *IdAllocator) SetStart(start uint64)

type Map added in v1.2.0 

type Map[K comparable, V any] struct {
	// contains filtered or unexported fields
}

func NewMap added in v1.2.0 

func NewMap[K comparable, V any](shardCount int) *Map[K, V]

func (*Map[K, V]) Delete added in v1.2.0 

func (m *Map[K, V]) Delete(key K)

func (*Map[K, V]) DeleteIf added in v1.2.0 

func (m *Map[K, V]) DeleteIf(f func(key K, value V) bool)

func (*Map[K, V]) Load added in v1.2.0 

func (m *Map[K, V]) Load(key K) (value V, ok bool)

func (*Map[K, V]) LoadAndDelete added in v1.2.0 

func (m *Map[K, V]) LoadAndDelete(key K) (value V, loaded bool)

func (*Map[K, V]) Range added in v1.2.0 

func (m *Map[K, V]) Range(f func(key K, value V) bool)

func (*Map[K, V]) Store added in v1.2.0 

func (m *Map[K, V]) Store(key K, value V)

type OnZeroCB 

type OnZeroCB func()

type PPLevel 

type PPLevel int8
const (
	PPL0 PPLevel = iota
	PPL1
	PPL2
	PPL3
	PPL4
)

type PinnedItem added in v0.6.0 

type PinnedItem[T IRef] struct {
	Val T
}

func (*PinnedItem[T]) Close added in v0.6.0 

func (item *PinnedItem[T]) Close()

func (*PinnedItem[T]) Item added in v0.6.0 

func (item *PinnedItem[T]) Item() T

type RefHelper 

type RefHelper struct {
	Refs     atomic.Int64
	OnZeroCB OnZeroCB
}

func (*RefHelper) Ref 

func (helper *RefHelper) Ref()

func (*RefHelper) RefCount 

func (helper *RefHelper) RefCount() int64

func (*RefHelper) RefIfHasRef added in v0.7.0 

func (helper *RefHelper) RefIfHasRef() bool

func (*RefHelper) Unref 

func (helper *RefHelper) Unref()

type Reprer 

type Reprer interface {
	Repr() string
}

type RetryOp 

type RetryOp = func() error

type SLLNode 

type SLLNode struct {
	RefHelper
	*sync.RWMutex
	Next ISLLNode
}

SLLNode represent a single node in linked list. It is thread-safe.

func NewSLLNode 

func NewSLLNode(mu *sync.RWMutex) *SLLNode

func (*SLLNode) GetNextNode 

func (l *SLLNode) GetNextNode() ISLLNode

func (*SLLNode) Insert 

func (l *SLLNode) Insert(n ISLLNode)

func (*SLLNode) ReleaseNextNode 

func (l *SLLNode) ReleaseNextNode()

func (*SLLNode) SetNextNode 

func (l *SLLNode) SetNextNode(next ISLLNode)

func (*SLLNode) SetNextNodeNoLock 

func (l *SLLNode) SetNextNodeNoLock(next ISLLNode)

type SSLLNode 

type SSLLNode struct {
	// contains filtered or unexported fields
}

func NewSSLLNode 

func NewSSLLNode() *SSLLNode

func (*SSLLNode) GetNext 

func (n *SSLLNode) GetNext() ISSLLNode

func (*SSLLNode) Insert 

func (n *SSLLNode) Insert(nn ISSLLNode)

func (*SSLLNode) ReleaseFollowing 

func (n *SSLLNode) ReleaseFollowing() ISSLLNode

func (*SSLLNode) ReleaseNextNode 

func (n *SSLLNode) ReleaseNextNode() ISSLLNode

func (*SSLLNode) SetNext 

func (n *SSLLNode) SetNext(next ISSLLNode)

type TableCompactStat added in v1.0.0 

type TableCompactStat struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewTableCompactStatWithRandomMergeTime 

func NewTableCompactStatWithRandomMergeTime() TableCompactStat

func (*TableCompactStat) GetFlushDeadline 

func (s *TableCompactStat) GetFlushDeadline() time.Time

func (*TableCompactStat) GetLastMergeTime 

func (s *TableCompactStat) GetLastMergeTime() time.Time

func (*TableCompactStat) Init 

func (s *TableCompactStat) Init(maxFlushInterval time.Duration)

func (*TableCompactStat) ResetDeadline 

func (s *TableCompactStat) ResetDeadline(maxFlushInterval time.Duration)

func (*TableCompactStat) SetLastMergeTime 

func (s *TableCompactStat) SetLastMergeTime()

type TxnIDAllocator added in v0.6.0 

type TxnIDAllocator struct {
}

func NewTxnIDAllocator added in v0.6.0 

func NewTxnIDAllocator() *TxnIDAllocator

func (*TxnIDAllocator) Alloc added in v0.6.0 

func (alloc *TxnIDAllocator) Alloc() []byte

type TypePrintOpt added in v0.6.0 

type TypePrintOpt interface {
	// contains filtered or unexported methods
}

type WaitOp added in v0.7.0 

type WaitOp = func() (ok bool, err error)

type WithDoNotPrintBin added in v0.6.0 

type WithDoNotPrintBin struct{}

type WithIsComposite 

type WithIsComposite struct{}

type WithSpecialRowid added in v0.6.0 

type WithSpecialRowid struct{}

type ZonemapPrintKind added in v1.2.0 

type ZonemapPrintKind int
const (
	ZonemapPrintKindNormal ZonemapPrintKind = iota
	ZonemapPrintKindCompose
	ZonemapPrintKindHex
)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.