aggexec

package
v0.0.0-debug-20260702 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 2, 2026 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source
const (
	AggBatchSize = 8192
)
View Source
const (
	BlockCapacityForStrType = 8192 * 4
)

Due to concerns that the vector may consume additional memory, I set a more conservative row-capacity for each block to ensure its memory not more than 1 gigabyte.

View Source
const (
	// GroupNotMatched is a constant for the BatchFill method.
	// if the group is GroupNotMatched, the BatchFill method will ignore the row.
	GroupNotMatched = 0
)
View Source
const (
	MaxVectorLength = 262144
)

Variables

View Source
var (

	// list of special aggregation function IDs.
	AggIdOfCountColumn     = int64(-1)
	AggIdOfCountStar       = int64(-2)
	AggIdOfGroupConcat     = int64(-3)
	AggIdOfApproxCount     = int64(-4)
	AggIdOfMedian          = int64(-5)
	AggIdOfJsonArrayAgg    = int64(-6)
	AggIdOfJsonObjectAgg   = int64(-7)
	WinIdOfRowNumber       = int64(-8)
	WinIdOfRank            = int64(-9)
	WinIdOfDenseRank       = int64(-10)
	WinIdOfLag             = int64(-11)
	WinIdOfLead            = int64(-12)
	WinIdOfFirstValue      = int64(-13)
	WinIdOfLastValue       = int64(-14)
	WinIdOfNthValue        = int64(-15)
	AggIdOfSum             = int64(-16)
	AggIdOfAvg             = int64(-17)
	AggIdOfMin             = int64(-18)
	AggIdOfMax             = int64(-19)
	AggIdOfAny             = int64(-20)
	AggIdOfVarPop          = int64(-21)
	AggIdOfStdDevPop       = int64(-22)
	AggIdOfVarSample       = int64(-23)
	AggIdOfStdDevSample    = int64(-24)
	AggIdOfBitXor          = int64(-25)
	AggIdOfBitAnd          = int64(-26)
	AggIdOfBitOr           = int64(-27)
	AggIdOfBitmapConstruct = int64(-28)
	AggIdOfBitmapOr        = int64(-29)
	WinIdOfCumeDist        = int64(-30)
	WinIdOfNtile           = int64(-31)
	WinIdOfPercentRank     = int64(-32)
	AggIdOfAvgTwCache      = int64(-33)
	AggIdOfAvgTwResult     = int64(-34)
	AggIdOfHllAdd          = int64(-35)
	AggIdOfHllMerge        = int64(-36)
)
View Source
var (
	ErrInvalidLengthSerialize        = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowSerialize          = fmt.Errorf("proto: integer overflow")
	ErrUnexpectedEndOfGroupSerialize = fmt.Errorf("proto: unexpected end of group")
)
View Source
var AvgTwResultSupportedTypes = []types.T{
	types.T_char, types.T_varchar,
}
View Source
var GroupConcatUnsupportedTypes = []types.T{
	types.T_tuple,
}

Functions

func AppendMultiFixed

func AppendMultiFixed[T numeric | types.Decimal64 | types.Decimal128](vecs *Vectors[T], vals T, isNull bool, cnt int, mp *mpool.MPool) error

func AvgReturnType

func AvgReturnType(typs []types.Type) types.Type

func AvgTwCacheReturnType

func AvgTwCacheReturnType(typs []types.Type) types.Type

func AvgTwResultReturnType

func AvgTwResultReturnType(typs []types.Type) types.Type

func CumeDistReturnType

func CumeDistReturnType(_ []types.Type) types.Type

func FromD64ToD128

func FromD64ToD128(v types.Decimal64) types.Decimal128

func GetChunkSizeFromType

func GetChunkSizeFromType(typ types.Type) (s int)

GetChunkSizeFromType return the chunk size for the input type. This chunk size ensures that each chunk with this type does not exceed 1 GB of memory.

func GroupConcatReturnType

func GroupConcatReturnType(args []types.Type) types.Type

func IsGroupConcatSupported

func IsGroupConcatSupported(t types.Type) bool

func MedianDecimal64

func MedianDecimal64(vs *Vectors[types.Decimal64]) (types.Decimal128, error)

func MedianDecimal128

func MedianDecimal128(vs *Vectors[types.Decimal128]) (types.Decimal128, error)

func MedianNumeric

func MedianNumeric[T numeric](vs *Vectors[T]) (float64, error)

func MedianReturnType

func MedianReturnType(args []types.Type) types.Type

func ReadBytes

func ReadBytes(r io.Reader) (buf []byte, n int64, err error)

func RegisterAny

func RegisterAny(id int64)

func RegisterApproxCountAgg

func RegisterApproxCountAgg(id int64)

func RegisterAvg

func RegisterAvg(id int64)

func RegisterAvgTwCache

func RegisterAvgTwCache(id int64)

func RegisterAvgTwResult

func RegisterAvgTwResult(id int64)

func RegisterBitAndAgg

func RegisterBitAndAgg(id int64)

func RegisterBitOrAgg

func RegisterBitOrAgg(id int64)

func RegisterBitXorAgg

func RegisterBitXorAgg(id int64)

func RegisterBitmapConstruct

func RegisterBitmapConstruct(id int64)

func RegisterBitmapOr

func RegisterBitmapOr(id int64)

func RegisterCountColumnAgg

func RegisterCountColumnAgg(id int64)

func RegisterCountStarAgg

func RegisterCountStarAgg(id int64)

func RegisterCumeDistWin

func RegisterCumeDistWin(id int64)

func RegisterDenseRankWin

func RegisterDenseRankWin(id int64)

func RegisterFirstValueWin

func RegisterFirstValueWin(id int64)

func RegisterGroupConcatAgg

func RegisterGroupConcatAgg(id int64, sep string)

func RegisterHllAddAgg

func RegisterHllAddAgg(id int64)

func RegisterHllMergeAgg

func RegisterHllMergeAgg(id int64)

func RegisterJsonArrayAgg

func RegisterJsonArrayAgg(id int64)

func RegisterJsonObjectAgg

func RegisterJsonObjectAgg(id int64)

func RegisterLagWin

func RegisterLagWin(id int64)

func RegisterLastValueWin

func RegisterLastValueWin(id int64)

func RegisterLeadWin

func RegisterLeadWin(id int64)

func RegisterMax

func RegisterMax(id int64)

func RegisterMedian

func RegisterMedian(id int64)

func RegisterMin

func RegisterMin(id int64)

func RegisterNthValueWin

func RegisterNthValueWin(id int64)

func RegisterNtileWin

func RegisterNtileWin(id int64)

func RegisterPercentRankWin

func RegisterPercentRankWin(id int64)

func RegisterRankWin

func RegisterRankWin(id int64)

func RegisterRowNumberWin

func RegisterRowNumberWin(id int64)

func RegisterStdDevPop

func RegisterStdDevPop(id int64)

func RegisterStdDevSample

func RegisterStdDevSample(id int64)

func RegisterSum

func RegisterSum(id int64)

func RegisterVarPop

func RegisterVarPop(id int64)

func RegisterVarSample

func RegisterVarSample(id int64)

func SingleWindowReturnType

func SingleWindowReturnType(_ []types.Type) types.Type

func SumReturnType

func SumReturnType(typs []types.Type) types.Type

func SyncAggregatorsToChunkSize

func SyncAggregatorsToChunkSize(as []AggFuncExec, syncLimit int)

func WriteBytes

func WriteBytes(b []byte, w io.Writer) (n int64, err error)

Types

type AggFuncExec

type AggFuncExec interface {
	GetOptResult() SplitResult

	AggID() int64
	IsDistinct() bool

	// TypesInfo return the argument types and return type of the function.
	TypesInfo() ([]types.Type, types.Type)

	// GroupGrow increases the number of groups in the aggregation.
	GroupGrow(more int) error

	// PreAllocateGroups pre-allocates more additional groups to reduce garbage collection overhead.
	PreAllocateGroups(more int) error

	// XXX: WTF.
	Fill(groupIndex int, row int, vectors []*vector.Vector) error
	BulkFill(groupIndex int, vectors []*vector.Vector) error

	// BatchFill : add values to the aggregation for multiple groups at once.
	BatchFill(offset int, groups []uint64, vectors []*vector.Vector) error

	// XXX: WTF.
	Merge(next AggFuncExec, groupIdx1, groupIdx2 int) error

	// BatchMerge combines the aggregation result of multiple couples.
	// next: offset + i
	// self: groups[i] - 1
	BatchMerge(next AggFuncExec, offset int, groups []uint64) error

	// SetExtraInformation sets additional information for the aggregation executor,
	// such as partial results.
	//
	// but for the 'group_concat', it was a bad hack to use the method to set the separator.
	// and for the 'cluster_centers', it was used to set the fields of this agg.
	// todo: the old implementation is not good, we should use the vector.Vector to replace the any.
	//  and the hacks should be removed.
	//  but for first version, I will keep it.
	SetExtraInformation(partialResult any, groupIndex int) (err error)

	// Flush return the aggregation result.
	Flush() ([]*vector.Vector, error)

	// Serialize intermediate result to bytes.
	SaveIntermediateResult(cnt int64, flags [][]uint8, buf *bytes.Buffer) error
	SaveIntermediateResultOfChunk(chunk int, buf *bytes.Buffer) error
	UnmarshalFromReader(reader io.Reader, mp *mpool.MPool) error

	Size() int64

	// Free clean the resource and reuse the aggregation if possible.
	Free()
}

AggFuncExec is an interface to do execution for aggregation.

func MakeAgg

func MakeAgg(
	mg *mpool.MPool,
	aggID int64, isDistinct bool,
	param ...types.Type,
) (AggFuncExec, error)

MakeAgg is the only exporting method to create an aggregation function executor. all the aggID should be registered before calling this function.

type AggFuncExecExpression

type AggFuncExecExpression struct {
	// contains filtered or unexported fields
}

AggFuncExecExpression is the exporting structure for the aggregation information. it is used to indicate the information of the aggregation function for the operators like 'group' or 'merge group'.

func MakeAggFunctionExpression

func MakeAggFunctionExpression(id int64, isDistinct bool, args []*plan.Expr, config []byte) AggFuncExecExpression

func (*AggFuncExecExpression) GetAggID

func (ag *AggFuncExecExpression) GetAggID() int64

func (*AggFuncExecExpression) GetArgExpressions

func (ag *AggFuncExecExpression) GetArgExpressions() []*plan.Expr

func (*AggFuncExecExpression) GetExtraConfig

func (ag *AggFuncExecExpression) GetExtraConfig() []byte

func (*AggFuncExecExpression) IsDistinct

func (ag *AggFuncExecExpression) IsDistinct() bool

func (*AggFuncExecExpression) RewriteArgExpressions

func (ag *AggFuncExecExpression) RewriteArgExpressions(rewrite func(*plan.Expr) (*plan.Expr, bool, error)) (bool, error)

func (*AggFuncExecExpression) UnmarshalFromReader

func (ag *AggFuncExecExpression) UnmarshalFromReader(r io.Reader) error

type EncodedAgg

type EncodedAgg struct {
	Info EncodedBasicInfo `protobuf:"bytes,1,opt,name=info,proto3" json:"info"`
	// agg result part.
	Result  [][]byte `protobuf:"bytes,2,rep,name=result,proto3" json:"result,omitempty"`
	Empties [][]byte `protobuf:"bytes,3,rep,name=empties,proto3" json:"empties,omitempty"`
	// agg executing context
	Groups               [][]byte `protobuf:"bytes,4,rep,name=groups,proto3" json:"groups,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*EncodedAgg) Descriptor

func (*EncodedAgg) Descriptor() ([]byte, []int)

func (*EncodedAgg) GetEmpties

func (m *EncodedAgg) GetEmpties() [][]byte

func (*EncodedAgg) GetGroups

func (m *EncodedAgg) GetGroups() [][]byte

func (*EncodedAgg) GetInfo

func (m *EncodedAgg) GetInfo() EncodedBasicInfo

func (*EncodedAgg) GetResult

func (m *EncodedAgg) GetResult() [][]byte

func (*EncodedAgg) Marshal

func (m *EncodedAgg) Marshal() (dAtA []byte, err error)

func (*EncodedAgg) MarshalTo

func (m *EncodedAgg) MarshalTo(dAtA []byte) (int, error)

func (*EncodedAgg) MarshalToSizedBuffer

func (m *EncodedAgg) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*EncodedAgg) ProtoMessage

func (*EncodedAgg) ProtoMessage()

func (*EncodedAgg) ProtoSize

func (m *EncodedAgg) ProtoSize() (n int)

func (*EncodedAgg) Reset

func (m *EncodedAgg) Reset()

func (*EncodedAgg) String

func (m *EncodedAgg) String() string

func (*EncodedAgg) Unmarshal

func (m *EncodedAgg) Unmarshal(dAtA []byte) error

func (*EncodedAgg) XXX_DiscardUnknown

func (m *EncodedAgg) XXX_DiscardUnknown()

func (*EncodedAgg) XXX_Marshal

func (m *EncodedAgg) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*EncodedAgg) XXX_Merge

func (m *EncodedAgg) XXX_Merge(src proto.Message)

func (*EncodedAgg) XXX_Size

func (m *EncodedAgg) XXX_Size() int

func (*EncodedAgg) XXX_Unmarshal

func (m *EncodedAgg) XXX_Unmarshal(b []byte) error

type EncodedBasicInfo

type EncodedBasicInfo struct {
	Id                   int64                                                        `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
	IsDistinct           bool                                                         `protobuf:"varint,2,opt,name=is_distinct,json=isDistinct,proto3" json:"is_distinct,omitempty"`
	NullEmpty            bool                                                         `protobuf:"varint,3,opt,name=null_empty,json=nullEmpty,proto3" json:"null_empty,omitempty"`
	Args                 []github_com_matrixorigin_matrixone_pkg_container_types.Type `protobuf:"bytes,4,rep,name=args,proto3,customtype=github.com/matrixorigin/matrixone/pkg/container/types.Type" json:"args"`
	Ret                  github_com_matrixorigin_matrixone_pkg_container_types.Type   `protobuf:"bytes,5,opt,name=ret,proto3,customtype=github.com/matrixorigin/matrixone/pkg/container/types.Type" json:"ret"`
	XXX_NoUnkeyedLiteral struct{}                                                     `json:"-"`
	XXX_unrecognized     []byte                                                       `json:"-"`
	XXX_sizecache        int32                                                        `json:"-"`
}

func (*EncodedBasicInfo) Descriptor

func (*EncodedBasicInfo) Descriptor() ([]byte, []int)

func (*EncodedBasicInfo) GetId

func (m *EncodedBasicInfo) GetId() int64

func (*EncodedBasicInfo) GetIsDistinct

func (m *EncodedBasicInfo) GetIsDistinct() bool

func (*EncodedBasicInfo) GetNullEmpty

func (m *EncodedBasicInfo) GetNullEmpty() bool

func (*EncodedBasicInfo) Marshal

func (m *EncodedBasicInfo) Marshal() (dAtA []byte, err error)

func (*EncodedBasicInfo) MarshalTo

func (m *EncodedBasicInfo) MarshalTo(dAtA []byte) (int, error)

func (*EncodedBasicInfo) MarshalToSizedBuffer

func (m *EncodedBasicInfo) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*EncodedBasicInfo) ProtoMessage

func (*EncodedBasicInfo) ProtoMessage()

func (*EncodedBasicInfo) ProtoSize

func (m *EncodedBasicInfo) ProtoSize() (n int)

func (*EncodedBasicInfo) Reset

func (m *EncodedBasicInfo) Reset()

func (*EncodedBasicInfo) String

func (m *EncodedBasicInfo) String() string

func (*EncodedBasicInfo) Unmarshal

func (m *EncodedBasicInfo) Unmarshal(dAtA []byte) error

func (*EncodedBasicInfo) XXX_DiscardUnknown

func (m *EncodedBasicInfo) XXX_DiscardUnknown()

func (*EncodedBasicInfo) XXX_Marshal

func (m *EncodedBasicInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*EncodedBasicInfo) XXX_Merge

func (m *EncodedBasicInfo) XXX_Merge(src proto.Message)

func (*EncodedBasicInfo) XXX_Size

func (m *EncodedBasicInfo) XXX_Size() int

func (*EncodedBasicInfo) XXX_Unmarshal

func (m *EncodedBasicInfo) XXX_Unmarshal(b []byte) error

type MarshalerUnmarshaler

type MarshalerUnmarshaler interface {
	MarshalBinary() ([]byte, error)
	UnmarshalBinary([]byte) error
	UnmarshalFromReader(io.Reader) error
}

type SplitResult

type SplitResult interface {
	// contains filtered or unexported methods
}

type Vectors

type Vectors[T numeric | types.Decimal64 | types.Decimal128] struct {
	// contains filtered or unexported fields
}

func NewVectors

func NewVectors[T numeric | types.Decimal64 | types.Decimal128](typ types.Type) *Vectors[T]

func (*Vectors[T]) Free

func (vs *Vectors[T]) Free(mp *mpool.MPool)

func (*Vectors[T]) Length

func (vs *Vectors[T]) Length() int

func (*Vectors[T]) MarshalBinary

func (vs *Vectors[T]) MarshalBinary() ([]byte, error)

func (*Vectors[T]) Size

func (vs *Vectors[T]) Size() int64

func (*Vectors[T]) Union

func (vs *Vectors[T]) Union(other *Vectors[T], mp *mpool.MPool) error

clone other

func (*Vectors[T]) Unmarshal

func (vs *Vectors[T]) Unmarshal(data []byte, typ types.Type, mp *mpool.MPool) error

func (*Vectors[T]) UnmarshalFromReader

func (vs *Vectors[T]) UnmarshalFromReader(r io.Reader, typ types.Type, mp *mpool.MPool) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL