objectio

package
v0.0.0-debug-20260702 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 2, 2026 License: Apache-2.0 Imports: 46 Imported by: 0

Documentation

Overview

Copyright 2021 Matrix Origin

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

Constants

View Source 
const (
	ArenaSmall = 0 // flush tasks (single-block writers)
	ArenaLarge = 1 // merge, compaction, and sinker tasks
)

Arena size tiers. Small arenas serve flush tasks and sinkers that process modest data volumes. Large arenas serve merge/compaction tasks that may aggregate up to arenaMaxSize (128 MB). Keeping the tiers separate prevents small callers from inflating arena sizes and saves ~50% of permanent RSS on large-core machines.

View Source 
const (
	BitmapBitsInPool = BlockMaxRows
	BitmapPoolSize   = 64
)
View Source 
const (
	BlockInfoType = uint16(1)

	V1 = uint16(1)
)
View Source 
const (
	IOET_Empty   = 0
	IOET_ObjMeta = 1
	IOET_ColData = 2
	IOET_BF      = 3
	IOET_ZM      = 4
)
View Source 
const (
	BlockMaxRows = 8192

	SEQNUM_UPPER    = math.MaxUint16 - 5 // reserved 5 column for special committs、committs etc.
	SEQNUM_ROWID    = math.MaxUint16
	SEQNUM_ABORT    = math.MaxUint16 - 1
	SEQNUM_COMMITTS = math.MaxUint16 - 2
)
View Source 
const (
	TombstoneAttr_Rowid_Idx = 0
	TombstoneAttr_PK_Idx    = 1

	// Appendable
	TombstoneAttr_A_PhyAddr_Idx  = 2
	TombstoneAttr_A_CommitTs_Idx = 3
	TombstoneAttr_A_Abort_Idx    = 4

	// non-Appendable tn created
	TombstoneAttr_NA_CommitTs_Idx = 2

	TombstoneAttr_Rowid_SeqNum = TombstoneAttr_Rowid_Idx
	TombstoneAttr_PK_SeqNum    = TombstoneAttr_PK_Idx

	TombstoneAttr_A_PhyAddr_SeqNum = TombstoneAttr_A_PhyAddr_Idx
	TombstoneAttr_CommitTs_SeqNum  = SEQNUM_COMMITTS
	TombstoneAttr_Abort_SeqNum     = SEQNUM_ABORT

	TombstonePrimaryKeyIdx = TombstoneAttr_Rowid_Idx
)
View Source 
const (
	PhysicalAddr_Attr    = "__mo_rowid"
	DefaultCommitTS_Attr = "__mo_%1_commit_time"
	DefaultAbort_Attr    = "__mo_%1_abort"

	TombstoneAttr_Rowid_Attr    = "__mo_%1_delete_rowid"
	TombstoneAttr_PK_Attr       = "__mo_%1_pk_val"
	TombstoneAttr_CommitTs_Attr = DefaultCommitTS_Attr
	TombstoneAttr_Abort_Attr    = DefaultAbort_Attr
)
View Source 
const (
	SegmentIdSize = types.UuidSize
	ObjectIDSize  = types.ObjectidSize
)
View Source 
const (
	FJ_EmptyDB  = ""
	FJ_EmptyTBL = ""
)
View Source 
const (
	FJ_CommitDelete  = "fj/commit/delete"
	FJ_CommitSlowLog = "fj/commit/slowlog"
	FJ_CommitWait    = "fj/commit/wait"
	FJ_TransferSlow  = "fj/transfer/slow"
	FJ_FlushTimeout  = "fj/flush/timeout"
	FJ_FlushEntry    = "fj/flush/entry"

	FJ_CheckpointSave = "fj/checkpoint/save"
	FJ_GCKPWait1      = "fj/gckp/wait1"

	FJ_TraceRanges         = "fj/trace/ranges"
	FJ_TracePartitionState = "fj/trace/partitionstate"
	FJ_PrefetchThreshold   = "fj/prefetch/threshold"

	FJ_Debug19524 = "fj/debug/19524"

	FJ_CNRecvErr        = "fj/cn/recv/err"
	FJ_CNSubSysErr      = "fj/cn/recv/subsyserr"
	FJ_CNReplayCacheErr = "fj/cn/recv/rcacheerr"
	FJ_CNGCDumpTable    = "fj/cn/gc/dumptable"

	FJ_LogReader    = "fj/log/reader"
	FJ_LogWorkspace = "fj/log/workspace"

	FJ_CronJobsOpen = "fj/cronjobs/open"
	FJ_CDCRecordTxn = "fj/cdc/recordtxn"

	FJ_CDCExecutor  = "fj/cdc/executor"
	FJ_CDCScanTable = "fj/cdc/scantable"

	FJ_PublicationSnapshotFinished = "fj/publication/snapshot/finished"

	FJ_UpstreamSQLHelper = "fj/publication/upstream/sqlhelper"

	FJ_WALReplayFailed = "fj/wal/replay/failed"

	FJ_CDCHandleSlow             = "fj/cdc/handleslow"
	FJ_CDCHandleErr              = "fj/cdc/handleerr"
	FJ_CDCScanTableErr           = "fj/cdc/scantableerr"
	FJ_CDCAddExecErr             = "fj/cdc/addexecerr"
	FJ_CDCAddExecConsumeTruncate = "fj/cdc/addexecconsumetruncate"

	FJ_CNFlushSmallObjs      = "fj/cn/flush_small_objs"
	FJ_CNSubscribeTableFail  = "fj/cn/subscribe_table_fail"
	FJ_CNWorkspaceForceFlush = "fj/cn/workspace_force_flush"

	FJ_CNCLONEFailed                    = "fj/cn/clone_fails"
	FJ_CNCommitAfterWorkspaceDumpFailed = "fj/cn/commit_after_workspace_dump_fails"
	FJ_CNNeedRetryError                 = "fj/cn/need_retry_error"
)
View Source 
const (
	FJ_LogLevel0 = iota
	FJ_LogLevel1
	FJ_LogLevel2
	FJ_LogLevel3
)
View Source 
const (
	ExtentOff   = ObjectNameLen
	ExtentLen   = ExtentSize
	RowsOff     = ExtentOff + ExtentLen
	RowsLen     = 4
	BlockIDOff  = RowsOff + RowsLen
	BlockIDLen  = 2
	LocationLen = BlockIDOff + BlockIDLen
)
View Source 
const (
	FileNumOff         = SegmentIdSize
	FileNumLen         = 2
	NameStringOff      = FileNumOff + FileNumLen
	NameStringLen      = 42 //uuid[36]+_[1]+filename[5]
	ObjectNameLen      = NameStringOff + NameStringLen
	ObjectNameShortLen = NameStringOff
)
View Source 
const (
	ObjectFlag_Appendable = 1 << iota
	ObjectFlag_Sorted
	ObjectFlag_CNCreated
)
View Source 
const (
	IOET_ObjectMeta_V1  = 1
	IOET_ObjectMeta_V2  = 2
	IOET_ObjectMeta_V3  = 3
	IOET_ColumnData_V1  = 1
	IOET_ColumnData_V2  = 2
	IOET_BloomFilter_V1 = 1
	IOET_BloomFilter_V2 = 2
	IOET_ZoneMap_V1     = 1

	IOET_ObjectMeta_CurrVer  = IOET_ObjectMeta_V3
	IOET_ColumnData_CurrVer  = IOET_ColumnData_V2
	IOET_BloomFilter_CurrVer = IOET_BloomFilter_V2
	IOET_ZoneMap_CurrVer     = IOET_ZoneMap_V1
)
View Source 
const (
	WriterNormal = iota
	WriterCheckpoint
	WriterQueryResult
	WriterGC
	WriterETL
	WriterTmp
	WriterDumpTable
)
View Source 
const (
	BlockInfoSize = int(unsafe.Sizeof(EmptyBlockInfo))
)
View Source 
const (
	ExtentSize = extentOriginOff + extentOriginLen
)
View Source 
const FSName = "local"
View Source 
const FooterSize = 64
View Source 
const HeaderSize = 64
View Source 
const IOEntryHeaderSize = 4
View Source 
const Magic = 0xFFFFFFFF
View Source 
const (
	ObjectStatsLen = reservedOffset + reservedLen
)
View Source 
const Version = 1
View Source 
const ZoneMapSize = index.ZMSize

Variables

View Source 
var (
	EmptyBlockInfo      = BlockInfo{}
	EmptyBlockInfoBytes = EncodeBlockInfo(&EmptyBlockInfo)
)
View Source 
var (
	TombstoneSeqnums_CN_Created         = []uint16{0, 1}
	TombstoneSeqnums_CN_Created_PhyAddr = []uint16{0, 1, SEQNUM_ROWID}
	TombstoneSeqnums_DN_Created         = []uint16{0, 1, TombstoneAttr_CommitTs_SeqNum}
	TombstoneSeqnums_DN_Created_PhyAddr = []uint16{0, 1, TombstoneAttr_CommitTs_SeqNum, SEQNUM_ROWID}

	TombstoneColumns_CN_Created         = []int{0, 1}
	TombstoneColumns_CN_Created_PhyAddr = []int{0, 1, SEQNUM_ROWID}
	TombstoneColumns_TN_Created         = []int{0, 1, TombstoneAttr_CommitTs_SeqNum}
	TombstoneColumns_TN_Created_PhyAddr = []int{0, 1, TombstoneAttr_CommitTs_SeqNum, SEQNUM_ROWID}

	TombstoneAttrs_CN_Created         = []string{TombstoneAttr_Rowid_Attr, TombstoneAttr_PK_Attr}
	TombstoneAttrs_CN_Created_PhyAddr = []string{TombstoneAttr_Rowid_Attr, TombstoneAttr_PK_Attr, PhysicalAddr_Attr}
	TombstoneAttrs_TN_Created         = []string{TombstoneAttr_Rowid_Attr, TombstoneAttr_PK_Attr, TombstoneAttr_CommitTs_Attr}
	TombstoneAttrs_TN_Created_PhyAddr = []string{TombstoneAttr_Rowid_Attr, TombstoneAttr_PK_Attr, TombstoneAttr_CommitTs_Attr, PhysicalAddr_Attr}
)
View Source 
var (
	VarcharType types.Type
	RowidType   types.Type
	TSType      types.Type
	Uint64Type  types.Type
)
View Source 
var (
	BuildObjectMeta        = buildObjectDataMetaV1
	NewObjectWriterSpecial = newObjectWriterSpecialV1
	NewObjectWriter        = newObjectWriterV1
	NewObjectReaderWithStr = newObjectReaderWithStrV1
	NewObjectReader        = newObjectReaderV1
)
View Source 
var BitmapPool = fileservice.NewPool(
	BitmapPoolSize,
	func() *bitmap.Bitmap {
		var bm bitmap.Bitmap
		bm.InitWithSize(BitmapBitsInPool)
		return &bm
	},
	func(bm *bitmap.Bitmap) {
		bm.Clear()
	},
	nil,
)
View Source 
var BlkReadStats = newBlockReadStats()
View Source 
var BuildZM = index.BuildZM
View Source 
var EmptyZm = [64]byte{}
View Source 
var GlobalCacheCapacityHint atomic.Int64
View Source 
var NameCheckPoint [NameStringOff]byte
View Source 
var NameCopyTable [NameStringOff]byte
View Source 
var NameDiskCleaner [NameStringOff]byte
View Source 
var NameDumpTable [NameStringOff]byte
View Source 
var NameETL [NameStringOff]byte
View Source 
var NameNormal [NameStringOff]byte
View Source 
var NameQueryResult [NameStringOff]byte
View Source 
var NameTmp [NameStringOff]byte
View Source 
var NewBlockidWithObjectID = types.NewBlockidWithObjectID
View Source 
var NewObjectid = types.NewObjectid
View Source 
var NewRowIDWithObjectIDBlkNumAndRowID = types.NewRowIDWithObjectIDBlkNumAndRowID
View Source 
var NewRowid = types.NewRowid
View Source 
var NewZM = index.NewZM
View Source 
var ObjectSizeLimit = 3 * mpool.GB

make it mutable in ut

Functions

func BitmapPoolReport 

func BitmapPoolReport() string

func BuildLocationTo 

func BuildLocationTo(
	name ObjectName,
	extent Extent,
	rows uint32,
	id uint16,
	toLoc []byte,
)

func BuildMetaData added in v0.8.0 

func BuildMetaData(blkCount, colCount uint16) objectDataMetaV1

test used

func CDCAddExecConsumeTruncateInjected 

func CDCAddExecConsumeTruncateInjected() bool

func CDCAddExecErrInjected 

func CDCAddExecErrInjected() bool

func CDCHandleErrInjected 

func CDCHandleErrInjected() bool

func CDCHandleSlowInjected 

func CDCHandleSlowInjected() (sleepSeconds int64, injected bool)

func CDCRecordTxnInjected 

func CDCRecordTxnInjected(dbName, tableName string) (bool, int)

func CDCScanTableErrInjected 

func CDCScanTableErrInjected() bool

func CDCScanTableInjected 

func CDCScanTableInjected() (string, bool)

func CNCommitAfterWorkspaceDumpFailedInjected 

func CNCommitAfterWorkspaceDumpFailedInjected() (string, bool)

func CNRecvErrInjected 

func CNRecvErrInjected() (bool, int)

func CNReplayCacheErrInjected 

func CNReplayCacheErrInjected() (bool, int)

func CNSubSysErrInjected 

func CNSubSysErrInjected() (bool, int)

func CNWorkspaceForceFlushInjected 

func CNWorkspaceForceFlushInjected() bool

func CheckpointSaveInjected 

func CheckpointSaveInjected() (string, bool)

func CommitWaitInjected 

func CommitWaitInjected() (string, bool)

func ConstructRowidColumn added in v0.8.0 

func ConstructRowidColumn(
	id *Blockid, start, length uint32, mp *mpool.MPool,
) (vec *vector.Vector, err error)

func ConstructRowidColumnTo added in v0.8.0 

func ConstructRowidColumnTo(
	vec *vector.Vector,
	id *Blockid, start, length uint32, mp *mpool.MPool,
) (err error)

func ConstructRowidColumnToWithSels added in v0.8.0 

func ConstructRowidColumnToWithSels(
	vec *vector.Vector,
	id *Blockid,
	sels []int64,
	mp *mpool.MPool,
) (err error)

func ConvertToCkpIdx added in v1.0.0 

func ConvertToCkpIdx(dataType uint16) uint16

func Debug19524Injected 

func Debug19524Injected() bool

func Decode added in v0.8.0 

func Decode(buf []byte) (any, error)

func DecodeColumnDataV1 added in v0.8.0 

func DecodeColumnDataV1(buf []byte) (ioe any, err error)

NOTE: Break by MustVector. Need to update MustVector to support new version.

func DecodeColumnDataV2 

func DecodeColumnDataV2(buf []byte) (ioe any, err error)

NOTE: Break by MustVector. Need to update MustVector to support new version.

func DecodeObjectMetaV1 added in v1.0.0 

func DecodeObjectMetaV1(buf []byte) (ioe any, err error)

func DecodeObjectMetaV2 added in v1.0.0 

func DecodeObjectMetaV2(buf []byte) (ioe any, err error)

func DecodeObjectMetaV3 added in v1.0.0 

func DecodeObjectMetaV3(buf []byte) (ioe any, err error)

func EncodeBlockInfo added in v1.2.0 

func EncodeBlockInfo(info *BlockInfo) []byte

func EncodeColumnDataV1 added in v0.8.0 

func EncodeColumnDataV1(ioe any) (buf []byte, err error)

func EncodeIOEntryHeader added in v0.8.0 

func EncodeIOEntryHeader(h *IOEntryHeader) []byte

func EncodeInfoHeader added in v1.2.0 

func EncodeInfoHeader(h InfoHeader) uint32

func EncodeLocation 

func EncodeLocation(location Location) []byte

func EstimateS3Input 

func EstimateS3Input(writtenRows int64) float64

Estimating S3input using the number of written lines, Maintained by @LeftHandCold

func EstimateS3InputForDeleteRows 

func EstimateS3InputForDeleteRows(deletedRows int64) float64

func EvictCache 

func EvictCache(ctx context.Context) (target int64)

func EvictCacheToCapacityPercent 

func EvictCacheToCapacityPercent(ctx context.Context, percent int64) (used int64)

func FillBlockidWithNameAndSeq 

func FillBlockidWithNameAndSeq(name ObjectName, sequence uint16, to []byte)

func ForeachBlkInObjStatsList 

func ForeachBlkInObjStatsList(
	next bool,
	dataMeta ObjectDataMeta,
	onBlock func(blk BlockInfo, blkMeta BlockObject) bool,
	objects ...ObjectStats,
)

func ForeachObjectStats added in v1.1.0 

func ForeachObjectStats(onStats func(stats *ObjectStats) bool, statsList ...ObjectStats)

ForeachObjectStats executes onStats on each object stats until onStats returns false or all object stats have been visited

func GCDumpTableInjected 

func GCDumpTableInjected() (string, bool)

func GetIOEntryCodec added in v0.8.0 

func GetIOEntryCodec(h IOEntryHeader) (codec ioEntryCodec)

func GetTombstoneAttrs 

func GetTombstoneAttrs(hidden HiddenColumnSelection) []string

func GetTombstoneCommitTSAttrIdx 

func GetTombstoneCommitTSAttrIdx(columnCnt uint16) uint16

func GetTombstoneSchema 

func GetTombstoneSchema(
	pk types.Type, hidden HiddenColumnSelection,
) (attrs []string, attrTypes []types.Type)

func GetTombstoneSeqnums 

func GetTombstoneSeqnums(hidden HiddenColumnSelection) []uint16

func GetTombstoneTypes 

func GetTombstoneTypes(
	pk types.Type, hidden HiddenColumnSelection,
) []types.Type

func ISCPExecutorInjected 

func ISCPExecutorInjected() (string, bool)

func InitMetaCache added in v0.8.0 

func InitMetaCache(size int64)

func InjectCDCExecutor 

func InjectCDCExecutor(msg string) (rmFault func() (bool, error), err error)

func InjectCDCRecordTxn 

func InjectCDCRecordTxn(
	databaseName string,
	tableName string,
	level int,
) (rmFault func(), err error)

func InjectCDCScanTable 

func InjectCDCScanTable(msg string) (rmFault func() (bool, error), err error)

func InjectCheckpointSave 

func InjectCheckpointSave(msg string) (rmFault func() (bool, error), err error)

func InjectCommitWait 

func InjectCommitWait(msg string) (rmFault func() (bool, error), err error)

func InjectGCDumpTable 

func InjectGCDumpTable(msg string) (rmFault func() (bool, error), err error)

func InjectLog1 

func InjectLog1(
	tableName string,
	level int,
) (rmFault func(), err error)

inject log reader and partition state `name` is the table name

func InjectLogPartitionState 

func InjectLogPartitionState(
	databaseName string,
	tableName string,
	level int,
) (rmFault func(), err error)

func InjectLogRanges 

func InjectLogRanges(
	ctx context.Context,
	tableName string,
) (rmFault func(), err error)

func InjectLogging 

func InjectLogging(
	key string,
	databaseName string,
	tableName string,
	level int,
	isEqual bool,
) (rmFault func(), err error)

func InjectNotify 

func InjectNotify(key, target string) (rmFault func(), err error)

func InjectPrefetchThreshold 

func InjectPrefetchThreshold(threshold int) (rmFault func(), err error)

func InjectPrintFlushEntry 

func InjectPrintFlushEntry(msg string) (rmFault func() (bool, error), err error)

func InjectPublicationSnapshotFinished 

func InjectPublicationSnapshotFinished(msg string) (rmFault func() (bool, error), err error)

func InjectUpstreamSQLHelper 

func InjectUpstreamSQLHelper(msg string) (rmFault func() (bool, error), err error)

func InjectWALReplayFailed 

func InjectWALReplayFailed(msg string) (rmFault func() (bool, error), err error)

func InjectWait 

func InjectWait(key string) (rmFault func(), err error)

func IsBlockInObject added in v0.8.0 

func IsBlockInObject(blkID *types.Blockid, objID *ObjectName) bool

func IsEmptyBlkid added in v0.8.0 

func IsEmptyBlkid(id *Blockid) bool

func IsEmptySegid added in v0.8.0 

func IsEmptySegid(id *Segmentid) bool

func IsPhysicalAddr 

func IsPhysicalAddr(attr string) bool

func IsSameObjectLocVsMeta added in v0.8.0 

func IsSameObjectLocVsMeta(location Location, meta ObjectDataMeta) bool

func IsSameObjectLocVsShort added in v0.8.0 

func IsSameObjectLocVsShort(location Location, short *ObjectNameShort) bool

func LogCNCloneFailedInjected 

func LogCNCloneFailedInjected(args ...string) (bool, int)

func LogCNFlushSmallObjsInjected 

func LogCNFlushSmallObjsInjected(args ...string) (bool, int)

func LogCNNeedRetryErrorInjected 

func LogCNNeedRetryErrorInjected(args ...string) (bool, int)

func LogCNSubscribeTableFailInjected 

func LogCNSubscribeTableFailInjected(args ...string) (bool, int)

func LogReaderInjected 

func LogReaderInjected(args ...string) (bool, int)

`name` is the table name return injected, logLevel

func LogWorkspaceInjected 

func LogWorkspaceInjected(args ...string) (bool, int)

func MakeInjectDBAndTableLoggingIntArg 

func MakeInjectDBAndTableLoggingIntArg(level int) int

func MakeInjectDBLoggingIntArg 

func MakeInjectDBLoggingIntArg(level int, isEqual bool) int

func MakeInjectTableLoggingIntArg 

func MakeInjectTableLoggingIntArg(level int, isEqual bool) int

func MockOneObj_MulBlks_Rowids 

func MockOneObj_MulBlks_Rowids(
	blkCnt int,
	rowsPerBlk int,
	reversed bool,
	mp *mpool.MPool,
) (vec *vector.Vector, err error)

func MustGetPhysicalColumnPosition 

func MustGetPhysicalColumnPosition(seqnums []uint16, colTypes []types.Type) int

func MustVectorTo 

func MustVectorTo(toVec *vector.Vector, buf []byte) (err error)

NOTE: hack way to get vector

func NewArrayVector added in v1.1.0 

func NewArrayVector[T types.RealNumbers](n int, typ types.Type, m *mpool.MPool, random bool, vs [][]T) *vector.Vector

func NewBatch added in v1.1.0 

func NewBatch(ts []types.Type, random bool, n int, m *mpool.MPool) *batch.Batch

func NewBlockidVector added in v1.1.0 

func NewBlockidVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.Blockid) *vector.Vector

func NewBoolVector added in v1.1.0 

func NewBoolVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []bool) *vector.Vector

func NewDateVector added in v1.1.0 

func NewDateVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewDatetimeVector added in v1.1.0 

func NewDatetimeVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewDecimal64Vector added in v1.1.0 

func NewDecimal64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []types.Decimal64) *vector.Vector

func NewDecimal128Vector added in v1.1.0 

func NewDecimal128Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []types.Decimal128) *vector.Vector

func NewEnumVector added in v1.2.2 

func NewEnumVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []types.Enum) *vector.Vector

func NewFloat32Vector added in v1.1.0 

func NewFloat32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []float32) *vector.Vector

func NewFloat64Vector added in v1.1.0 

func NewFloat64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []float64) *vector.Vector

func NewInt8Vector added in v1.1.0 

func NewInt8Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int8) *vector.Vector

func NewInt16Vector added in v1.1.0 

func NewInt16Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int16) *vector.Vector

func NewInt32Vector added in v1.1.0 

func NewInt32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int32) *vector.Vector

func NewInt64Vector added in v1.1.0 

func NewInt64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int64) *vector.Vector

func NewJsonVector added in v1.1.0 

func NewJsonVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []string) *vector.Vector

func NewOfflineFS 

func NewOfflineFS(
	ctx context.Context,
	dir string,
	fromS3 bool,
) (fs fileservice.FileService, err error)

NewOfflineFS creates a new offline file service. It is used to read data from a local directory. There are two types of offline files: 1. files generated by MO w/o S3 storage 2. files generated by MO with S3 storage

func NewRowidVector added in v1.1.0 

func NewRowidVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.Rowid) *vector.Vector

func NewStringVector added in v1.1.0 

func NewStringVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewTimeVector added in v1.1.0 

func NewTimeVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewTimestampVector added in v1.1.0 

func NewTimestampVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector

func NewTsVector added in v1.1.0 

func NewTsVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.TS) *vector.Vector

func NewUInt8Vector added in v1.1.0 

func NewUInt8Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint8) *vector.Vector

func NewUInt16Vector added in v1.1.0 

func NewUInt16Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint16) *vector.Vector

func NewUInt32Vector added in v1.1.0 

func NewUInt32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint32) *vector.Vector

func NewUInt64Vector added in v1.1.0 

func NewUInt64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint64) *vector.Vector

func NewVector added in v1.1.0 

func NewVector(n int, typ types.Type, m *mpool.MPool, random bool, Values interface{}) *vector.Vector

func NotifyInjected 

func NotifyInjected(key string)

func ParseLoggingIntArgs 

func ParseLoggingIntArgs(iarg int) (logLevel int, funcId int)

ParseFJIntArgs parses the injected fault integer argument and returns the log level and function id logLevel = iarg % 10 and funcId = iarg / 10 iarg quick mapping: 0->3: logLevel = 0->3, database name equal 10->13: logLevel = 0->3, database name contains 20->23: logLevel = 0->3, table name equal 30->33: logLevel = 0->3, table name contains 40->43: logLevel = 0->3, database name and table name contains

func ParseLoggingSArgs 

func ParseLoggingSArgs(sarg string, funcId int, args ...string) bool

func PartitionStateInjected 

func PartitionStateInjected(dbName, tableName string) (bool, int)

func PrefetchMetaThresholdInjected 

func PrefetchMetaThresholdInjected() (bool, int)

bool: injected or not int: threshold. 1 means 1 millisecond

func PrintFlushEntryInjected 

func PrintFlushEntryInjected() (string, bool)

func PublicationSnapshotFinishedInjected 

func PublicationSnapshotFinishedInjected() (string, bool)

func PutArena 

func PutArena(a *WriteArena)

PutArena pushes a WriteArena back into its tier's free list, auto-routing based on sizeLimit.

func RangesLogInjected 

func RangesLogInjected(dbName, tableName string) (bool, int)

func ReadAllBlocksWithMeta added in v0.8.0 

func ReadAllBlocksWithMeta(
	ctx context.Context,
	meta *ObjectDataMeta,
	name string,
	cols []uint16,
	policy fileservice.Policy,
	m *mpool.MPool,
	fs fileservice.FileService,
	factory CacheConstructorFactory,
) (ioVec fileservice.IOVector, err error)

func ReadBytes added in v0.8.0 

func ReadBytes(r io.Reader) (buf []byte, n int64, err error)

func ReadExtent added in v0.8.0 

func ReadExtent(
	ctx context.Context,
	name string,
	extent *Extent,
	policy fileservice.Policy,
	fs fileservice.FileService,
	factory CacheConstructorFactory,
) (buf []byte, err error)

func ReadOneBlock added in v0.8.0 

func ReadOneBlock(
	ctx context.Context,
	meta *ObjectDataMeta,
	name string,
	blk uint16,
	seqnums []uint16,
	typs []types.Type,
	m *mpool.MPool,
	fs fileservice.FileService,
	policy fileservice.Policy,
) (ioVec fileservice.IOVector, err error)

func ReadOneBlockAllColumns added in v1.1.0 

func ReadOneBlockAllColumns(
	ctx context.Context,
	meta *ObjectDataMeta,
	name string,
	id uint32,
	cols []uint16,
	cachePolicy fileservice.Policy,
	fs fileservice.FileService,
) (bat *batch.Batch, err error)

func ReadOneBlockWithMeta added in v0.8.0 

func ReadOneBlockWithMeta(
	ctx context.Context,
	meta *ObjectDataMeta,
	name string,
	blk uint16,
	seqnums []uint16,
	typs []types.Type,
	m *mpool.MPool,
	fs fileservice.FileService,
	factory CacheConstructorFactory,
	policy fileservice.Policy,
) (ioVec fileservice.IOVector, err error)

func ReadString added in v0.8.0 

func ReadString(r io.Reader) (str string, n int64, err error)

func RegisterIOEnrtyCodec added in v0.8.0 

func RegisterIOEnrtyCodec(h IOEntryHeader, encFn IOEncodeFunc, decFn IODecodeFunc)

func ReleaseIOVector added in v1.2.0 

func ReleaseIOVector(vector *fileservice.IOVector)

func ScheduleArenaDrain 

func ScheduleArenaDrain()

ScheduleArenaDrain debounces arena pool draining. Each call resets the timer to arenaDrainDelay from now. During steady-state checkpointing (every ~60 s), the timer never fires and pools stay warm. Once activity ceases, the timer fires and RSS is reclaimed.

func SetMetaCachePressureTargetPercent 

func SetMetaCachePressureTargetPercent(percent int64, until time.Time)

func SetObjectSizeLimit 

func SetObjectSizeLimit(limit int)

SetObjectSizeLimit set ObjectSizeLimit to limit Bytes

func SetObjectStats 

func SetObjectStats(stats, o *ObjectStats) error

func SetObjectStatsBlkCnt added in v1.1.0 

func SetObjectStatsBlkCnt(stats *ObjectStats, cnt uint32) error

func SetObjectStatsExtent added in v1.1.0 

func SetObjectStatsExtent(stats *ObjectStats, extent Extent) error

func SetObjectStatsLocation added in v1.1.0 

func SetObjectStatsLocation(stats *ObjectStats, location Location) error

func SetObjectStatsObjectName added in v1.1.0 

func SetObjectStatsObjectName(stats *ObjectStats, name ObjectName) error

func SetObjectStatsOriginSize added in v1.1.0 

func SetObjectStatsOriginSize(stats *ObjectStats, size uint32) error

func SetObjectStatsRowCnt added in v1.1.0 

func SetObjectStatsRowCnt(stats *ObjectStats, cnt uint32) error

func SetObjectStatsShortName added in v1.1.0 

func SetObjectStatsShortName(stats *ObjectStats, name *ObjectNameShort) error

func SetObjectStatsSize added in v1.1.0 

func SetObjectStatsSize(stats *ObjectStats, size uint32) error

func SetObjectStatsSortKeyZoneMap added in v1.1.0 

func SetObjectStatsSortKeyZoneMap(stats *ObjectStats, zoneMap ZoneMap) error

func SimpleInject 

func SimpleInject(key string) (rmFault func(), err error)

func SimpleInjected 

func SimpleInjected(key string) bool

func SumSizeOfBlocks 

func SumSizeOfBlocks(blocks []BlockObject) (size, osize uint32)

func TmpNewFileservice 

func TmpNewFileservice(ctx context.Context, dir string) fileservice.FileService

func TmpNewSharedFileservice 

func TmpNewSharedFileservice(ctx context.Context, dir string) fileservice.FileService

func TotalMem 

func TotalMem() uint64

func UpstreamSQLHelperInjected 

func UpstreamSQLHelperInjected() (string, bool)

func WALReplayFailedExecutorInjected 

func WALReplayFailedExecutorInjected() (string, bool)

func WaitInjected 

func WaitInjected(key string)

func WriteBytes added in v0.8.0 

func WriteBytes(b []byte, w io.Writer) (n int64, err error)

func WriteString added in v0.8.0 

func WriteString(str string, w io.Writer) (n int64, err error)

Types

type BackupObject added in v1.2.0 

type BackupObject struct {
	Location Location
	CrateTS  types.TS
	DropTS   types.TS
	NeedCopy bool
}

type Bitmap 

type Bitmap struct {
	// contains filtered or unexported fields
}
var NullBitmap Bitmap

func GetNoReuseBitmap 

func GetNoReuseBitmap() Bitmap

func GetReusableBitmap 

func GetReusableBitmap() Bitmap

func (*Bitmap) Add 

func (r *Bitmap) Add(i uint64)

func (*Bitmap) Bitmap 

func (r *Bitmap) Bitmap() *bitmap.Bitmap

func (*Bitmap) Clear 

func (r *Bitmap) Clear()

func (*Bitmap) Contains 

func (r *Bitmap) Contains(i uint64) bool

func (*Bitmap) Count 

func (r *Bitmap) Count() int

func (*Bitmap) Idx 

func (r *Bitmap) Idx() int

func (*Bitmap) IsEmpty 

func (r *Bitmap) IsEmpty() bool

func (*Bitmap) IsValid 

func (r *Bitmap) IsValid() bool

func (*Bitmap) Or 

func (r *Bitmap) Or(o Bitmap)

func (*Bitmap) OrBitmap 

func (r *Bitmap) OrBitmap(o *bitmap.Bitmap)

func (*Bitmap) PreExtend 

func (r *Bitmap) PreExtend(nbits int)

func (*Bitmap) Release 

func (r *Bitmap) Release()

func (*Bitmap) Reusable 

func (r *Bitmap) Reusable() bool

func (*Bitmap) ToArray 

func (r *Bitmap) ToArray() []uint64

func (*Bitmap) ToI64Array 

func (r *Bitmap) ToI64Array(out *[]int64) []int64

type BlockHeader 

type BlockHeader []byte

func BuildBlockHeader added in v0.8.0 

func BuildBlockHeader() BlockHeader

func (BlockHeader) Appendable added in v1.1.0 

func (bh BlockHeader) Appendable() bool

func (BlockHeader) BFExtent added in v0.8.0 

func (bh BlockHeader) BFExtent() Extent

func (BlockHeader) BlockID added in v0.8.0 

func (bh BlockHeader) BlockID() *Blockid

func (BlockHeader) BloomFilterType 

func (bh BlockHeader) BloomFilterType() uint8

func (BlockHeader) ColumnCount added in v0.8.0 

func (bh BlockHeader) ColumnCount() uint16

func (BlockHeader) IsEmpty added in v0.8.0 

func (bh BlockHeader) IsEmpty() bool

func (BlockHeader) MaxSeqnum added in v0.8.0 

func (bh BlockHeader) MaxSeqnum() uint16

func (BlockHeader) MetaColumnCount added in v0.8.0 

func (bh BlockHeader) MetaColumnCount() uint16

func (BlockHeader) MetaLocation added in v0.8.0 

func (bh BlockHeader) MetaLocation() Extent

func (BlockHeader) Rows added in v0.8.0 

func (bh BlockHeader) Rows() uint32

func (BlockHeader) Sequence added in v0.8.0 

func (bh BlockHeader) Sequence() uint16

func (BlockHeader) SetAppendable added in v1.1.0 

func (bh BlockHeader) SetAppendable(appendable bool)

func (BlockHeader) SetBFExtent added in v0.8.0 

func (bh BlockHeader) SetBFExtent(location Extent)

func (BlockHeader) SetBlockID added in v0.8.0 

func (bh BlockHeader) SetBlockID(id *Blockid)

func (BlockHeader) SetBloomFilterType 

func (bh BlockHeader) SetBloomFilterType(typ uint8)

func (BlockHeader) SetColumnCount added in v0.8.0 

func (bh BlockHeader) SetColumnCount(count uint16)

func (BlockHeader) SetMaxSeqnum added in v0.8.0 

func (bh BlockHeader) SetMaxSeqnum(seqnum uint16)

func (BlockHeader) SetMetaColumnCount added in v0.8.0 

func (bh BlockHeader) SetMetaColumnCount(count uint16)

func (BlockHeader) SetMetaLocation added in v0.8.0 

func (bh BlockHeader) SetMetaLocation(location Extent)

func (BlockHeader) SetRows added in v0.8.0 

func (bh BlockHeader) SetRows(rows uint32)

func (BlockHeader) SetSequence added in v0.8.0 

func (bh BlockHeader) SetSequence(seq uint16)

func (BlockHeader) SetSortKey added in v1.1.0 

func (bh BlockHeader) SetSortKey(idx uint16)

func (BlockHeader) SetStartID added in v1.0.0 

func (bh BlockHeader) SetStartID(id uint16)

func (BlockHeader) SetTableID added in v0.8.0 

func (bh BlockHeader) SetTableID(id uint64)

func (BlockHeader) SetZoneMapArea added in v0.8.0 

func (bh BlockHeader) SetZoneMapArea(location Extent)

func (BlockHeader) ShortName added in v0.8.0 

func (bh BlockHeader) ShortName() *ObjectNameShort

func (BlockHeader) SortKey added in v1.1.0 

func (bh BlockHeader) SortKey() uint16

func (BlockHeader) StartID added in v1.0.0 

func (bh BlockHeader) StartID() uint16

func (BlockHeader) TableID added in v0.8.0 

func (bh BlockHeader) TableID() uint64

func (BlockHeader) ZoneMapArea added in v0.8.0 

func (bh BlockHeader) ZoneMapArea() Extent

type BlockIndex added in v0.8.0 

type BlockIndex []byte

func BuildBlockIndex added in v0.8.0 

func BuildBlockIndex(count uint32) BlockIndex

func (BlockIndex) BlockCount added in v0.8.0 

func (oh BlockIndex) BlockCount() uint32

func (BlockIndex) BlockMetaPos added in v0.8.0 

func (oh BlockIndex) BlockMetaPos(BlockID uint32) (uint32, uint32)

func (BlockIndex) Length added in v0.8.0 

func (oh BlockIndex) Length() uint32

func (BlockIndex) SetBlockCount added in v0.8.0 

func (oh BlockIndex) SetBlockCount(cnt uint32)

func (BlockIndex) SetBlockMetaPos added in v0.8.0 

func (oh BlockIndex) SetBlockMetaPos(BlockID uint32, offset, length uint32)

type BlockInfo added in v1.2.0 

type BlockInfo struct {
	BlockID     types.Blockid
	MetaLoc     ObjectLocation
	ObjectFlags int8

	PartitionIdx int32
}

func DecodeBlockInfo added in v1.2.0 

func DecodeBlockInfo(buf []byte) *BlockInfo

func (*BlockInfo) ConstructBlockID 

func (b *BlockInfo) ConstructBlockID(name ObjectName, sequence uint16)

func (*BlockInfo) IsAppendable 

func (b *BlockInfo) IsAppendable() bool

func (*BlockInfo) IsCNCreated 

func (b *BlockInfo) IsCNCreated() bool

func (*BlockInfo) IsMemBlk 

func (b *BlockInfo) IsMemBlk() bool

func (*BlockInfo) IsSorted 

func (b *BlockInfo) IsSorted() bool

func (*BlockInfo) MarshalWithBuf 

func (b *BlockInfo) MarshalWithBuf(w *bytes.Buffer) (uint32, error)

func (*BlockInfo) MetaLocation added in v1.2.0 

func (b *BlockInfo) MetaLocation() Location

func (*BlockInfo) SetFlagByObjStats 

func (b *BlockInfo) SetFlagByObjStats(stats *ObjectStats)

func (*BlockInfo) SetMetaLocation added in v1.2.0 

func (b *BlockInfo) SetMetaLocation(metaLoc Location)

func (*BlockInfo) String 

func (b *BlockInfo) String() string

func (*BlockInfo) Unmarshal 

func (b *BlockInfo) Unmarshal(buf []byte) error

type BlockInfoSlice added in v1.2.0 

type BlockInfoSlice []byte

func MakeBlockInfoSlice 

func MakeBlockInfoSlice(cnt int) BlockInfoSlice

func MultiObjectStatsToBlockInfoSlice 

func MultiObjectStatsToBlockInfoSlice(objs []ObjectStats, withFirstEmpty bool) BlockInfoSlice

func ObjectStatsToBlockInfoSlice 

func ObjectStatsToBlockInfoSlice(stats *ObjectStats, withFirstEmpty bool) BlockInfoSlice

func PreAllocBlockInfoSlice 

func PreAllocBlockInfoSlice(preAllocBlocks int) BlockInfoSlice

func (*BlockInfoSlice) Append added in v1.2.0 

func (s *BlockInfoSlice) Append(bs []byte)

func (*BlockInfoSlice) AppendBlockInfo added in v1.2.0 

func (s *BlockInfoSlice) AppendBlockInfo(info *BlockInfo)

func (*BlockInfoSlice) Get added in v1.2.0 

func (s *BlockInfoSlice) Get(i int) *BlockInfo

func (*BlockInfoSlice) GetAllBytes added in v1.2.0 

func (s *BlockInfoSlice) GetAllBytes() []byte

func (*BlockInfoSlice) GetBytes added in v1.2.0 

func (s *BlockInfoSlice) GetBytes(i int) []byte

func (*BlockInfoSlice) Len added in v1.2.0 

func (s *BlockInfoSlice) Len() int

func (*BlockInfoSlice) Set added in v1.2.0 

func (s *BlockInfoSlice) Set(i int, info *BlockInfo)

func (*BlockInfoSlice) SetBytes added in v1.2.0 

func (s *BlockInfoSlice) SetBytes(bs []byte)

func (*BlockInfoSlice) Size added in v1.2.0 

func (s *BlockInfoSlice) Size() int

func (*BlockInfoSlice) Slice added in v1.2.0 

func (s *BlockInfoSlice) Slice(i, j int) []byte

func (*BlockInfoSlice) String added in v1.2.0 

func (s *BlockInfoSlice) String() string

type BlockObject 

type BlockObject []byte

func BuildBlockMeta added in v0.8.0 

func BuildBlockMeta(count uint16) BlockObject

func NewBlock 

func NewBlock(seqnums *Seqnums) BlockObject

func (BlockObject) AddColumnMeta added in v0.8.0 

func (bm BlockObject) AddColumnMeta(idx uint16, col ColumnMeta)

func (BlockObject) BlockHeader added in v0.8.0 

func (bm BlockObject) BlockHeader() BlockHeader

func (BlockObject) ColumnMeta added in v0.8.0 

func (bm BlockObject) ColumnMeta(seqnum uint16) ColumnMeta

ColumnMeta is for internal use only, it didn't consider the block does not contain the seqnum

func (BlockObject) GenerateBlockInfo 

func (bm BlockObject) GenerateBlockInfo(objName ObjectName, sorted bool) BlockInfo

func (BlockObject) GetBlockID added in v1.0.0 

func (bm BlockObject) GetBlockID(name ObjectName) *Blockid

func (BlockObject) GetColumnCount added in v0.8.0 

func (bm BlockObject) GetColumnCount() uint16

func (BlockObject) GetExtent 

func (bm BlockObject) GetExtent() Extent

func (BlockObject) GetID 

func (bm BlockObject) GetID() uint16

func (BlockObject) GetMaxSeqnum added in v0.8.0 

func (bm BlockObject) GetMaxSeqnum() uint16

func (BlockObject) GetMeta 

func (bm BlockObject) GetMeta() BlockObject

func (BlockObject) GetMetaColumnCount added in v0.8.0 

func (bm BlockObject) GetMetaColumnCount() uint16

func (BlockObject) GetRows 

func (bm BlockObject) GetRows() uint32

func (BlockObject) IsEmpty added in v0.8.0 

func (bm BlockObject) IsEmpty() bool

func (BlockObject) MustGetColumn added in v0.8.0 

func (bm BlockObject) MustGetColumn(seqnum uint16) ColumnMeta

MustGetColumn is for general use. it return a empty ColumnMeta if the block does not contain the seqnum

func (BlockObject) SetBlockMetaHeader added in v0.8.0 

func (bm BlockObject) SetBlockMetaHeader(header BlockHeader)

func (BlockObject) ToColumnZoneMaps added in v0.8.0 

func (bm BlockObject) ToColumnZoneMaps(seqnums []uint16) []ZoneMap

type BlockReadFilter 

type BlockReadFilter struct {
	HasFakePK          bool
	Valid              bool
	SortedSearchFunc   ReadFilterSearchFuncType
	UnSortedSearchFunc ReadFilterSearchFuncType
	Cleanup            func() // Cleanup function to release resources (e.g., reusableTempVec)
}

func (BlockReadFilter) DecideSearchFunc 

func (f BlockReadFilter) DecideSearchFunc(isSortedBlk bool) ReadFilterSearchFuncType

type BlockReadStats added in v1.0.0 

type BlockReadStats struct {
	// using this we can collect the number of blks have read and hit among them
	BlkCacheHitStats hitStats
	// using this we can collect the number of entries have read and hit among them
	EntryCacheHitStats hitStats
	// using this we can collect the number of blks each reader will read
	BlksByReaderStats hitStats
	CounterSet        *perfcounter.CounterSet
}

BlockReadStats collect blk read related cache statistics, include mem and disk

type Blockid added in v0.8.0 

type Blockid = types.Blockid

func BuildObjectBlockid added in v0.8.0 

func BuildObjectBlockid(name ObjectName, sequence uint16) *Blockid

func NewBlockid added in v0.8.0 

func NewBlockid(segid *Segmentid, fnum, blknum uint16) *Blockid

type BloomFilter 

type BloomFilter []byte

func FastLoadBF added in v1.1.0 

func FastLoadBF(
	ctx context.Context,
	location Location,
	isPrefetch bool,
	fs fileservice.FileService,
) (BloomFilter, error)

func LoadBFWithMeta added in v1.0.0 

func LoadBFWithMeta(
	ctx context.Context,
	meta ObjectDataMeta,
	location Location,
	fs fileservice.FileService,
) (BloomFilter, error)

func ReadBloomFilter added in v0.8.0 

func ReadBloomFilter(
	ctx context.Context,
	name string,
	extent *Extent,
	policy fileservice.Policy,
	fs fileservice.FileService,
) (filters BloomFilter, err error)

func (BloomFilter) BlockCount added in v0.8.0 

func (bf BloomFilter) BlockCount() uint32

func (BloomFilter) GetBloomFilter added in v0.8.0 

func (bf BloomFilter) GetBloomFilter(BlockID uint32) []byte

func (BloomFilter) GetObjectBloomFilter added in v0.8.0 

func (bf BloomFilter) GetObjectBloomFilter() []byte

func (BloomFilter) Size added in v0.8.0 

func (bf BloomFilter) Size() int

type CacheConfig added in v0.8.0 

type CacheConfig struct {
	MemoryCapacity toml.ByteSize `toml:"memory-capacity"`
}

type CacheConstructor added in v0.8.0 

type CacheConstructor = func(ctx context.Context, r io.Reader, buf []byte, allocator fileservice.CacheDataAllocator) (fscache.Data, error)

type CacheConstructorFactory added in v0.8.0 

type CacheConstructorFactory = func(size int64, algo uint8) CacheConstructor

type ColumnMeta 

type ColumnMeta []byte

func BuildColumnMeta added in v0.8.0 

func BuildColumnMeta() ColumnMeta

func BuildObjectColumnMeta added in v0.8.0 

func BuildObjectColumnMeta() ColumnMeta

func GetColumnMeta added in v0.8.0 

func GetColumnMeta(idx uint16, data []byte) ColumnMeta

func GetObjectColumnMeta added in v0.8.0 

func GetObjectColumnMeta(seqnum uint16, data []byte) ColumnMeta

caller makes sure the data has column meta fot the given seqnum

func (ColumnMeta) Checksum added in v0.8.0 

func (cm ColumnMeta) Checksum() uint32

func (ColumnMeta) DataType added in v0.8.0 

func (cm ColumnMeta) DataType() uint8

func (ColumnMeta) Idx added in v0.8.0 

func (cm ColumnMeta) Idx() uint16

func (ColumnMeta) IsEmpty added in v0.8.0 

func (cm ColumnMeta) IsEmpty() bool

func (ColumnMeta) Location added in v0.8.0 

func (cm ColumnMeta) Location() Extent

func (ColumnMeta) Ndv added in v0.8.0 

func (cm ColumnMeta) Ndv() uint32

func (ColumnMeta) NullCnt added in v0.8.0 

func (cm ColumnMeta) NullCnt() uint32

func (ColumnMeta) SetNdv added in v0.8.0 

func (cm ColumnMeta) SetNdv(cnt uint32)

func (ColumnMeta) SetNullCnt added in v0.8.0 

func (cm ColumnMeta) SetNullCnt(cnt uint32)

func (ColumnMeta) SetZoneMap added in v0.8.0 

func (cm ColumnMeta) SetZoneMap(zm ZoneMap)

func (ColumnMeta) ZoneMap added in v0.8.0 

func (cm ColumnMeta) ZoneMap() ZoneMap

type ColumnMetaFetcher added in v0.8.0 

type ColumnMetaFetcher interface {
	MustGetColumn(seqnum uint16) ColumnMeta
}

type CreateBlockOpt added in v0.8.0 

type CreateBlockOpt struct {
	Loc *struct {
		Metaloc  Location
		Deltaloc Location
	}

	Id *struct {
		Filen uint16
		Blkn  uint16
	}
}

func (*CreateBlockOpt) WithBlkIdx added in v0.8.0 

func (o *CreateBlockOpt) WithBlkIdx(s uint16) *CreateBlockOpt

func (*CreateBlockOpt) WithDetaloc added in v0.8.0 

func (o *CreateBlockOpt) WithDetaloc(s Location) *CreateBlockOpt

func (*CreateBlockOpt) WithFileIdx added in v0.8.0 

func (o *CreateBlockOpt) WithFileIdx(s uint16) *CreateBlockOpt

func (*CreateBlockOpt) WithMetaloc added in v0.8.0 

func (o *CreateBlockOpt) WithMetaloc(s Location) *CreateBlockOpt

type CreateObjOpt added in v1.1.0 

type CreateObjOpt struct {
	Stats       *ObjectStats
	IsTombstone bool
}

func (*CreateObjOpt) WithIsTombstone 

func (o *CreateObjOpt) WithIsTombstone(tombstone bool) *CreateObjOpt

func (*CreateObjOpt) WithObjectStats 

func (o *CreateObjOpt) WithObjectStats(stats *ObjectStats) *CreateObjOpt

type DataMetaType added in v1.0.0 

type DataMetaType uint16
const (
	SchemaData      DataMetaType = 0
	SchemaTombstone DataMetaType = 1

	CkpMetaStart DataMetaType = 2

	// CkpMetaEnd = CkpMetaStart + `MaxIDX`
	CkpMetaEnd DataMetaType = 28 + 2
)

func ConvertToSchemaType added in v1.0.0 

func ConvertToSchemaType(ckpIdx uint16) DataMetaType

type Extent 

type Extent []byte

Alg | Offset | Length | OriginSize ----|--------|--------|------------ 1 | 4 | 4 | 4 Alg: Specifies the compression algorithm Offset: The offset of the compressed data in the file Length: The length of the compressed data OriginSize: The length of the original data

func NewExtent 

func NewExtent(alg uint8, offset, length, originSize uint32) Extent

func NewRandomExtent 

func NewRandomExtent() Extent

func (Extent) Alg added in v0.8.0 

func (ex Extent) Alg() uint8

func (Extent) End 

func (ex Extent) End() uint32

func (Extent) Length 

func (ex Extent) Length() uint32

func (Extent) Offset 

func (ex Extent) Offset() uint32

func (Extent) OriginSize 

func (ex Extent) OriginSize() uint32

func (Extent) SetAlg added in v0.8.0 

func (ex Extent) SetAlg(alg uint8)

func (Extent) SetLength added in v0.8.0 

func (ex Extent) SetLength(length uint32)

func (Extent) SetOffset added in v0.8.0 

func (ex Extent) SetOffset(offset uint32)

func (Extent) SetOriginSize added in v0.8.0 

func (ex Extent) SetOriginSize(originSize uint32)

func (Extent) String added in v0.8.0 

func (ex Extent) String() string

type Float64Heap 

type Float64Heap []float64

func (Float64Heap) Len 

func (h Float64Heap) Len() int

func (Float64Heap) Less 

func (h Float64Heap) Less(i, j int) bool

func (*Float64Heap) Pop 

func (h *Float64Heap) Pop() any

func (*Float64Heap) Push 

func (h *Float64Heap) Push(x any)

func (Float64Heap) Swap 

func (h Float64Heap) Swap(i, j int)
type Footer struct {
	// contains filtered or unexported fields
}

func (Footer) Marshal added in v0.8.0 

func (f Footer) Marshal() []byte
type Header []byte

func BuildHeader added in v0.8.0 

func BuildHeader() Header

func (Header) Extent added in v0.8.0 

func (h Header) Extent() Extent

func (Header) SchemaVersion added in v0.8.0 

func (h Header) SchemaVersion(ver uint32)

func (Header) SetExtent added in v0.8.0 

func (h Header) SetExtent(location Extent)

func (Header) SetSchemaVersion added in v0.8.0 

func (h Header) SetSchemaVersion(ver uint32)

type HiddenColumnSelection 

type HiddenColumnSelection uint64
const (
	HiddenColumnSelection_PhysicalAddr HiddenColumnSelection = 1 << iota
	HiddenColumnSelection_CommitTS
	HiddenColumnSelection_Abort
)
const HiddenColumnSelection_None HiddenColumnSelection = 0

type IODecodeFunc added in v0.8.0 

type IODecodeFunc = func([]byte) (any, error)

type IOEncodeFunc added in v0.8.0 

type IOEncodeFunc = func(any) ([]byte, error)

type IOEntry added in v0.8.0 

type IOEntry interface {
	MarshalBinary() ([]byte, error)
	UnmarshalBinary([]byte) error
}

type IOEntryHeader added in v0.8.0 

type IOEntryHeader struct {
	Type, Version uint16
}

func DecodeIOEntryHeader added in v0.8.0 

func DecodeIOEntryHeader(buf []byte) *IOEntryHeader

func (IOEntryHeader) String added in v0.8.0 

func (h IOEntryHeader) String() string

type IndexReaderTopOp 

type IndexReaderTopOp struct {
	Typ          types.T
	MetricType   metric.MetricType
	ColPos       int32
	NumVec       []byte
	Limit        uint64
	OrderedLimit bool
	Desc         bool

	LowerBoundType plan.BoundType
	UpperBoundType plan.BoundType
	LowerBound     float64
	UpperBound     float64

	DistHeap Float64Heap
}

type InfoHeader added in v1.2.0 

type InfoHeader struct {
	Type, Version uint16
}

func DecodeInfoHeader added in v1.2.0 

func DecodeInfoHeader(h uint32) InfoHeader

type Location added in v0.8.0 

type Location []byte

Location is a fixed-length unmodifiable byte array. Layout: ObjectName | Extent | Rows(uint32) | ID(uint16)

func BuildLocation added in v0.8.0 

func BuildLocation(name ObjectName, extent Extent, rows uint32, id uint16) Location

func DecodeLocation 

func DecodeLocation(buf []byte) *Location

func MockLocation added in v0.8.0 

func MockLocation(name ObjectName) Location

func NewRandomLocation 

func NewRandomLocation(id uint16, rows uint32) Location

func StringToLocation 

func StringToLocation(info string) (Location, error)

StringToLocation Generate a metaloc from an info string

func (Location) Clone 

func (l Location) Clone() Location

func (Location) Extent added in v0.8.0 

func (l Location) Extent() Extent

func (Location) ID added in v0.8.0 

func (l Location) ID() uint16

func (Location) IsEmpty added in v0.8.0 

func (l Location) IsEmpty() bool

func (Location) Name added in v0.8.0 

func (l Location) Name() ObjectName

func (Location) ObjectId 

func (l Location) ObjectId() ObjectId

func (Location) Rows added in v0.8.0 

func (l Location) Rows() uint32

func (Location) SetID added in v1.0.0 

func (l Location) SetID(id uint16)

func (Location) SetRows added in v1.2.0 

func (l Location) SetRows(rows uint32)

func (Location) ShortName added in v0.8.0 

func (l Location) ShortName() *ObjectNameShort

func (Location) String added in v0.8.0 

func (l Location) String() string

type LocationSlice 

type LocationSlice []byte

func (*LocationSlice) Append 

func (s *LocationSlice) Append(bs []byte)

func (*LocationSlice) AppendLocation 

func (s *LocationSlice) AppendLocation(location Location)

func (*LocationSlice) Get 

func (s *LocationSlice) Get(i int) *Location

func (*LocationSlice) GetAllBytes 

func (s *LocationSlice) GetAllBytes() []byte

func (*LocationSlice) GetBytes 

func (s *LocationSlice) GetBytes(i int) []byte

func (*LocationSlice) Len 

func (s *LocationSlice) Len() int

func (*LocationSlice) Set 

func (s *LocationSlice) Set(i int, location *Location)

func (*LocationSlice) SetBytes 

func (s *LocationSlice) SetBytes(bs []byte)

func (*LocationSlice) Size 

func (s *LocationSlice) Size() int

func (*LocationSlice) Slice 

func (s *LocationSlice) Slice(i, j int) []byte

func (*LocationSlice) String 

func (s *LocationSlice) String() string

type Object 

type Object struct {
	// contains filtered or unexported fields
}

func NewObject 

func NewObject(name string, fs fileservice.FileService) *Object

func (*Object) GetFs added in v1.0.0 

func (o *Object) GetFs() fileservice.FileService

type ObjectBuffer 

type ObjectBuffer struct {
	// contains filtered or unexported fields
}

ObjectBuffer is the buffer prepared before writing to the object file, all data written to the object needs to be filled in it, and then written to the object file at one time

func NewObjectBuffer 

func NewObjectBuffer(name string) *ObjectBuffer

func (*ObjectBuffer) GetData 

func (b *ObjectBuffer) GetData() fileservice.IOVector

func (*ObjectBuffer) SetDataOptions added in v0.7.0 

func (b *ObjectBuffer) SetDataOptions(items ...WriteOptions)

func (*ObjectBuffer) Write 

func (b *ObjectBuffer) Write(buf []byte, items ...WriteOptions) (int, int)

type ObjectDataMeta added in v1.0.0 

type ObjectDataMeta = objectDataMetaV1

type ObjectDescriber added in v1.1.0 

type ObjectDescriber interface {
	DescribeObject() (ObjectStats, error)
}

type ObjectEntry 

type ObjectEntry struct {
	ObjectStats
	CreateTime types.TS
	DeleteTime types.TS
}

func (ObjectEntry) IsEmpty 

func (o ObjectEntry) IsEmpty() bool

func (ObjectEntry) Location 

func (o ObjectEntry) Location() Location

func (ObjectEntry) ObjectDTSIndexLess 

func (o ObjectEntry) ObjectDTSIndexLess(than ObjectEntry) bool

ObjectDTSIndexLess has the order: 1. if the delete time is empty, let it be the max ts 2. ascending object with delete ts. 3. ascending object with createts when same dts.

sort by DELETE time and then CREATE time

func (ObjectEntry) ObjectNameIndexLess 

func (o ObjectEntry) ObjectNameIndexLess(than ObjectEntry) bool

func (ObjectEntry) StatsValid 

func (o ObjectEntry) StatsValid() bool

func (ObjectEntry) String 

func (o ObjectEntry) String() string

func (ObjectEntry) Visible 

func (o ObjectEntry) Visible(ts types.TS) bool

type ObjectId added in v1.1.0 

type ObjectId = types.Objectid

func NewObjectidWithSegmentIDAndNum added in v1.1.0 

func NewObjectidWithSegmentIDAndNum(sid *Segmentid, num uint16) *ObjectId

type ObjectIter 

type ObjectIter interface {
	Next() bool
	Close() error
	Entry() ObjectEntry
}

type ObjectList 

type ObjectList interface {
	NewObjectsIter(
		ts types.TS,
		onlyVisible bool,
		visitTombstone bool,
	) (ObjectIter, error)

	IsNil() bool
}

type ObjectLocation added in v1.2.0 

type ObjectLocation [LocationLen]byte

type ObjectMeta added in v0.8.0 

type ObjectMeta []byte

func FastLoadObjectMeta added in v0.8.0 

func FastLoadObjectMeta(
	ctx context.Context,
	location *Location,
	prefetch bool,
	fs fileservice.FileService,
) (ObjectMeta, error)

func LoadObjectMetaByExtent added in v0.8.0 

func LoadObjectMetaByExtent(
	ctx context.Context,
	name *ObjectName,
	extent *Extent,
	prefetch bool,
	policy fileservice.Policy,
	fs fileservice.FileService,
) (meta ObjectMeta, err error)

func MustObjectMeta 

func MustObjectMeta(buffer []byte) ObjectMeta

func ReadObjectMeta added in v0.8.0 

func ReadObjectMeta(
	ctx context.Context,
	name string,
	extent *Extent,
	policy fileservice.Policy,
	fs fileservice.FileService,
) (meta ObjectMeta, err error)

func (ObjectMeta) DataMeta added in v1.0.0 

func (e ObjectMeta) DataMeta() (ObjectDataMeta, bool)

func (ObjectMeta) DataMetaCount added in v1.0.0 

func (e ObjectMeta) DataMetaCount() uint16

func (ObjectMeta) MustDataMeta added in v1.0.0 

func (e ObjectMeta) MustDataMeta() ObjectDataMeta

func (ObjectMeta) MustGetMeta added in v1.0.0 

func (e ObjectMeta) MustGetMeta(metaType DataMetaType) ObjectDataMeta

func (ObjectMeta) MustTombstoneMeta added in v1.0.0 

func (e ObjectMeta) MustTombstoneMeta() ObjectDataMeta

func (ObjectMeta) SubMeta added in v1.0.0 

func (e ObjectMeta) SubMeta(pos uint16) (ObjectDataMeta, bool)

type ObjectName added in v0.8.0 

type ObjectName []byte
 ObjectName is a fixed-length unmodifiable byte array.
 Layout: SegmentId     |   Num    |  NameStr
	     (types.Uuid)    (uint16)   (string)
 Size:	   16B         +   2B     +  42B

func BuildCheckpointName added in v0.8.0 

func BuildCheckpointName() ObjectName

func BuildDiskCleanerName added in v0.8.0 

func BuildDiskCleanerName() ObjectName

func BuildDumpTableName 

func BuildDumpTableName() ObjectName

func BuildETLName added in v0.8.0 

func BuildETLName() ObjectName

func BuildNormalName added in v0.8.0 

func BuildNormalName() ObjectName

func BuildObjectName added in v0.8.0 

func BuildObjectName(segid *Segmentid, num uint16) ObjectName

func BuildObjectNameWithObjectID added in v1.1.0 

func BuildObjectNameWithObjectID(id *ObjectId) ObjectName

func BuildQueryResultName added in v0.8.0 

func BuildQueryResultName() ObjectName

func BuildTmpName added in v1.2.3 

func BuildTmpName() ObjectName

func MockObjectName added in v0.8.0 

func MockObjectName() ObjectName

func (ObjectName) Equal added in v0.8.0 

func (o ObjectName) Equal(a ObjectName) bool

func (ObjectName) Num added in v0.8.0 

func (o ObjectName) Num() uint16

func (ObjectName) ObjectId added in v1.1.0 

func (o ObjectName) ObjectId() *ObjectId

func (ObjectName) SegmentId added in v0.8.0 

func (o ObjectName) SegmentId() Segmentid

func (ObjectName) Short added in v0.8.0 

func (o ObjectName) Short() *ObjectNameShort

func (ObjectName) String added in v0.8.0 

func (o ObjectName) String() string

func (ObjectName) UnsafeString 

func (o ObjectName) UnsafeString() string

type ObjectNameShort added in v0.8.0 

type ObjectNameShort [ObjectNameShortLen]byte

func ShortName added in v1.0.0 

func ShortName(b *Blockid) *ObjectNameShort

func (*ObjectNameShort) Equal added in v0.8.0 

func (s *ObjectNameShort) Equal(o []byte) bool

func (*ObjectNameShort) Num added in v0.8.0 

func (s *ObjectNameShort) Num() uint16

func (*ObjectNameShort) Segmentid added in v0.8.0 

func (s *ObjectNameShort) Segmentid() *Segmentid

func (*ObjectNameShort) ShortString 

func (s *ObjectNameShort) ShortString() string

type ObjectReader 

type ObjectReader = objectReaderV1

type ObjectStats added in v1.1.0 

type ObjectStats [ObjectStatsLen]byte

ObjectStats has format: +------------------------------------------------------------------------------------------------+--------+ |object_name(60B)|extent(13B)|row_cnt(4B)|block_cnt(4B)|zone_map(64B)|objectSize|objectOriginSize|reserved| +------------------------------------------------------------------------------------------------+--------+ 

var ZeroObjectStats ObjectStats

func NewObjectStats added in v1.1.0 

func NewObjectStats() *ObjectStats

func NewObjectStatsWithObjectID 

func NewObjectStatsWithObjectID(id *ObjectId, appendable, sorted, cnCreated bool) *ObjectStats

func (*ObjectStats) BlkCnt added in v1.1.0 

func (des *ObjectStats) BlkCnt() uint32

func (*ObjectStats) BlockLocation 

func (des *ObjectStats) BlockLocation(blk uint16, maxRows uint32) Location

func (*ObjectStats) BlockLocationTo 

func (des *ObjectStats) BlockLocationTo(
	blk uint16,
	maxRows uint32,
	toLoc []byte,
)

func (*ObjectStats) Clone added in v1.1.0 

func (des *ObjectStats) Clone() *ObjectStats

Clone deep copies the stats and returns its pointer

func (*ObjectStats) ConstructBlockId 

func (des *ObjectStats) ConstructBlockId(id uint16) Blockid

func (*ObjectStats) ConstructBlockInfo 

func (des *ObjectStats) ConstructBlockInfo(id uint16) BlockInfo

func (*ObjectStats) ConstructBlockInfoTo 

func (des *ObjectStats) ConstructBlockInfoTo(id uint16, blk *BlockInfo)

func (*ObjectStats) Extent added in v1.1.0 

func (des *ObjectStats) Extent() Extent

func (*ObjectStats) FlagString 

func (des *ObjectStats) FlagString() string

func (*ObjectStats) GetAppendable 

func (des *ObjectStats) GetAppendable() bool

func (*ObjectStats) GetCNCreated 

func (des *ObjectStats) GetCNCreated() bool

func (*ObjectStats) GetFlag 

func (des *ObjectStats) GetFlag() int8

func (*ObjectStats) GetLevel 

func (des *ObjectStats) GetLevel() int8

func (*ObjectStats) GetSorted 

func (des *ObjectStats) GetSorted() bool

func (*ObjectStats) IsZero added in v1.1.0 

func (des *ObjectStats) IsZero() bool

func (*ObjectStats) Marshal added in v1.1.0 

func (des *ObjectStats) Marshal() []byte

func (*ObjectStats) ObjectLocation added in v1.1.0 

func (des *ObjectStats) ObjectLocation() Location

func (*ObjectStats) ObjectName added in v1.1.0 

func (des *ObjectStats) ObjectName() ObjectName

func (*ObjectStats) ObjectShortName added in v1.1.0 

func (des *ObjectStats) ObjectShortName() *ObjectNameShort

func (*ObjectStats) OriginSize added in v1.1.0 

func (des *ObjectStats) OriginSize() uint32

func (*ObjectStats) Rows added in v1.1.0 

func (des *ObjectStats) Rows() uint32

func (*ObjectStats) SetLevel 

func (des *ObjectStats) SetLevel(level int8)

use the b5~b7 to store the level

func (*ObjectStats) Size added in v1.1.0 

func (des *ObjectStats) Size() uint32

func (*ObjectStats) SortKeyZoneMap added in v1.1.0 

func (des *ObjectStats) SortKeyZoneMap() ZoneMap

func (*ObjectStats) String added in v1.1.0 

func (des *ObjectStats) String() string

func (*ObjectStats) UnMarshal added in v1.1.0 

func (des *ObjectStats) UnMarshal(data []byte)

func (*ObjectStats) ZMIsEmpty added in v1.1.0 

func (des *ObjectStats) ZMIsEmpty() bool

type ObjectStatsOptions 

type ObjectStatsOptions func(*ObjectStats)

func WithAppendable 

func WithAppendable() ObjectStatsOptions

func WithCNCreated 

func WithCNCreated() ObjectStatsOptions

func WithSorted 

func WithSorted() ObjectStatsOptions

type ObjectStatsSlice 

type ObjectStatsSlice []byte

func (*ObjectStatsSlice) Append 

func (o *ObjectStatsSlice) Append(stats []byte)

func (*ObjectStatsSlice) Get 

func (o *ObjectStatsSlice) Get(i int) *ObjectStats

func (*ObjectStatsSlice) Len 

func (o *ObjectStatsSlice) Len() int

func (*ObjectStatsSlice) Set 

func (o *ObjectStatsSlice) Set(i int, stats []byte)

func (*ObjectStatsSlice) Size 

func (o *ObjectStatsSlice) Size() int

func (*ObjectStatsSlice) Slice 

func (o *ObjectStatsSlice) Slice(i, j int) []byte

type ObjectWriter 

type ObjectWriter = objectWriterV1

type ReadBlockOptions added in v0.8.0 

type ReadBlockOptions struct {
	Id       uint16
	DataType uint16
	Idxes    map[uint16]bool
}

type ReadFilterSearchFuncType 

type ReadFilterSearchFuncType func(containers.Vectors) []int64

type Reader 

type Reader interface {
	// Read is to read columns data of a block from fileservice at one time
	// extent is location of the block meta
	// idxs is the column serial number of the data to be read
	Read(ctx context.Context,
		extent *Extent, idxs []uint16,
		id uint32,
		m *mpool.MPool,
		readFunc CacheConstructorFactory) (*fileservice.IOVector, error)

	ReadAll(
		ctx context.Context,
		extent *Extent,
		idxs []uint16,
		m *mpool.MPool,
		readFunc CacheConstructorFactory,
	) (*fileservice.IOVector, error)

	ReadBlocks(ctx context.Context,
		extent *Extent,
		ids map[uint32]*ReadBlockOptions,
		m *mpool.MPool,
		readFunc CacheConstructorFactory) (*fileservice.IOVector, error)

	// ReadMeta is the meta that reads a block
	// extent is location of the block meta
	ReadMeta(ctx context.Context, extent *Extent, m *mpool.MPool) (ObjectDataMeta, error)

	// ReadAllMeta is read the meta of all blocks in an object
	ReadAllMeta(ctx context.Context, m *mpool.MPool) (ObjectDataMeta, error)

	GetObject() *Object
}

Reader is to read data from fileservice

type ReaderOptionFunc added in v0.8.0 

type ReaderOptionFunc func(opt *ReaderOptions)

func WithDataCachePolicyOption added in v1.0.0 

func WithDataCachePolicyOption(noLRUCache fileservice.Policy) ReaderOptionFunc

func WithMetaCachePolicyOption added in v1.0.0 

func WithMetaCachePolicyOption(noLRUCache fileservice.Policy) ReaderOptionFunc

type ReaderOptions added in v0.8.0 

type ReaderOptions struct {
	// contains filtered or unexported fields
}

type Rowid added in v0.8.0 

type Rowid = types.Rowid

type Segmentid added in v0.8.0 

type Segmentid = types.Segmentid

func NewSegmentid added in v0.8.0 

func NewSegmentid() *Segmentid

type Seqnums added in v0.8.0 

type Seqnums struct {
	Seqs       []uint16
	MaxSeq     uint16 // do not consider special column like rowid and committs
	MetaColCnt uint16 // include special columns
}

func NewSeqnums added in v0.8.0 

func NewSeqnums(seqs []uint16) *Seqnums

func (*Seqnums) InitWithColCnt added in v0.8.0 

func (s *Seqnums) InitWithColCnt(colcnt int)

type StaticFilter added in v0.8.0 

type StaticFilter = index.StaticFilter

type StatsBlkIter 

type StatsBlkIter struct {
	// contains filtered or unexported fields
}

func NewStatsBlkIter 

func NewStatsBlkIter(stats *ObjectStats, meta ObjectDataMeta) *StatsBlkIter

func (*StatsBlkIter) Entry 

func (i *StatsBlkIter) Entry() BlockInfo

func (*StatsBlkIter) Next 

func (i *StatsBlkIter) Next() bool

type SubMetaIndex added in v1.0.0 

type SubMetaIndex []byte

func BuildSubMetaIndex added in v1.0.0 

func BuildSubMetaIndex(count uint16) SubMetaIndex

func (SubMetaIndex) Length added in v1.0.0 

func (oh SubMetaIndex) Length() uint32

func (SubMetaIndex) SetSchemaMeta added in v1.0.0 

func (oh SubMetaIndex) SetSchemaMeta(pos uint16, st uint16, count uint16, offset uint32)

func (SubMetaIndex) SetSubMetaCount added in v1.0.0 

func (oh SubMetaIndex) SetSubMetaCount(cnt uint16)

func (SubMetaIndex) SubMetaCount added in v1.0.0 

func (oh SubMetaIndex) SubMetaCount() uint16

type WriteArena 

type WriteArena struct {
	// contains filtered or unexported fields
}

func GetArena 

func GetArena(tier int) *WriteArena

GetArena pops a WriteArena from the requested tier's free list, or creates a pre-warmed arena when the list is empty.

func NewArena 

func NewArena(size int) *WriteArena

func (*WriteArena) Alloc 

func (a *WriteArena) Alloc(size int) []byte

Alloc returns a slice of exactly size bytes. When the arena has insufficient space it falls back to a plain make so callers are never blocked. totalRequested is always updated so that Reset can grow the backing array for the next cycle, eliminating the fallback for future rounds of similar demand.

func (*WriteArena) CompressBuf 

func (a *WriteArena) CompressBuf(minSize int) []byte

CompressBuf returns a scratch buffer for LZ4 compression, growing as needed. The buffer is retained across arena Reset() calls. We round up to the next power of 2 so that minor size variations don't trigger repeated allocations.

func (*WriteArena) FreeBuffers 

func (a *WriteArena) FreeBuffers()

FreeBuffers releases the off-heap data and compressBuf allocations. Call this before dropping an arena that won't be returned to the pool.

func (*WriteArena) Reset 

func (a *WriteArena) Reset()

Reset prepares the arena for a new write cycle. If the previous cycle overflowed (totalRequested > len(data)) and the required capacity is within sizeLimit, the backing array is grown to a power-of-two capacity large enough to hold an equivalent cycle without any fallback allocations.

type WriteOptions added in v0.7.0 

type WriteOptions struct {
	Type WriteType
	Val  any
}

type WriteType added in v0.7.0 

type WriteType int8
const (
	WriteTS WriteType = iota
)

type Writer 

type Writer interface {
	// Write writes one batch to the Buffer at a time,
	// one batch corresponds to a virtual block,
	// and returns the handle of the block.
	Write(batch *batch.Batch) (BlockObject, error)

	// Write metadata for every column of all blocks
	WriteObjectMeta(ctx context.Context, totalRow uint32, metas []ColumnMeta)

	// WriteEnd is to write multiple batches written to
	// the buffer to the fileservice at one time
	WriteEnd(ctx context.Context, items ...WriteOptions) ([]BlockObject, error)
}

Writer is to virtualize batches into multiple blocks and write them into filefservice at one time

type WriterType added in v0.8.0 

type WriterType int8

type ZoneMap 

type ZoneMap = index.ZM

type ZoneMapArea added in v0.8.0 

type ZoneMapArea []byte

func (ZoneMapArea) BlockCount added in v0.8.0 

func (zma ZoneMapArea) BlockCount() uint32

func (ZoneMapArea) GetZoneMap added in v0.8.0 

func (zma ZoneMapArea) GetZoneMap(idx uint16, BlockID uint32) ZoneMap

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.