Documentation
¶
Index ¶
- Variables
- func GetCurrentCalendarUnit(base time.Time, unit string) (start, end time.Time, err error)
- type AQLQuery
- type AQLQueryContext
- func (qc *AQLQueryContext) FindDeviceForQuery(memStore memstore.MemStore, preferredDevice int, deviceManager *DeviceManager, ...)
- func (qc *AQLQueryContext) Postprocess() queryCom.AQLTimeSeriesResult
- func (qc *AQLQueryContext) PostprocessAsHLLData() ([]byte, error)
- func (qc *AQLQueryContext) ProcessQuery(memStore memstore.MemStore)
- func (qc *AQLQueryContext) Release()
- func (qc *AQLQueryContext) ReleaseHostResultsBuffers()
- func (qc *AQLQueryContext) Rewrite(expression expr.Expr) expr.Expr
- func (qc *AQLQueryContext) SerializeHLL(dataTypes []memCom.DataType, enumDicts map[int][]string, timeDimensions []int) ([]byte, error)
- type AQLRequest
- type AQLResponse
- type DeviceInfo
- type DeviceManager
- type Dimension
- type HLLDataWriter
- type HLLQueryResults
- type Join
- type Measure
- type NumericBucketizerDef
- type OOPKContext
- type TableScanner
- type TimeFilter
Constants ¶
This section is empty.
Variables ¶
var BinaryExprTypeToCFunctorType = map[expr.Token]C.enum_BinaryFunctorType{ expr.AND: C.And, expr.OR: C.Or, expr.EQ: C.Equal, expr.NEQ: C.NotEqual, expr.LT: C.LessThan, expr.LTE: C.LessThanOrEqual, expr.GT: C.GreaterThan, expr.GTE: C.GreaterThanOrEqual, expr.ADD: C.Plus, expr.SUB: C.Minus, expr.MUL: C.Multiply, expr.DIV: C.Divide, expr.MOD: C.Mod, expr.BITWISE_AND: C.BitwiseAnd, expr.BITWISE_OR: C.BitwiseOr, expr.BITWISE_XOR: C.BitwiseXor, expr.FLOOR: C.Floor, expr.CONVERT_TZ: C.Plus, }
BinaryExprTypeToCFunctorType maps from binary operator to C BinaryFunctorType
var ColumnTypeToExprType = map[string]expr.Type{ metaCom.Bool: expr.Boolean, metaCom.Int8: expr.Signed, metaCom.Int16: expr.Signed, metaCom.Int32: expr.Signed, metaCom.Int64: expr.Signed, metaCom.Uint8: expr.Unsigned, metaCom.Uint16: expr.Unsigned, metaCom.Uint32: expr.Unsigned, metaCom.Float32: expr.Float, metaCom.SmallEnum: expr.Unsigned, metaCom.BigEnum: expr.Unsigned, metaCom.GeoPoint: expr.GeoPoint, metaCom.GeoShape: expr.GeoShape, }
ColumnTypeToExprType maps data type from the column schema format to expression AST format.
var DataTypeToCDataType = map[memCom.DataType]C.enum_DataType{ memCom.Bool: C.Bool, memCom.Int8: C.Int8, memCom.Uint8: C.Uint8, memCom.Int16: C.Int16, memCom.Uint16: C.Uint16, memCom.Int32: C.Int32, memCom.Int64: C.Int64, memCom.Uint32: C.Uint32, memCom.Float32: C.Float32, memCom.SmallEnum: C.Uint8, memCom.BigEnum: C.Uint16, memCom.GeoPoint: C.GeoPoint, memCom.UUID: C.UUID, }
DataTypeToCDataType mapps from memstore data type to c data types
var UnaryExprTypeToCFunctorType = map[expr.Token]C.enum_UnaryFunctorType{ expr.NOT: C.Not, expr.UNARY_MINUS: C.Negate, expr.IS_NULL: C.IsNull, expr.IS_NOT_NULL: C.IsNotNull, expr.BITWISE_NOT: C.BitwiseNot, expr.GET_WEEK_START: C.GetWeekStart, expr.GET_MONTH_START: C.GetMonthStart, expr.GET_QUARTER_START: C.GetQuarterStart, expr.GET_YEAR_START: C.GetYearStart, expr.GET_DAY_OF_MONTH: C.GetDayOfMonth, expr.GET_DAY_OF_YEAR: C.GetDayOfYear, expr.GET_MONTH_OF_YEAR: C.GetMonthOfYear, expr.GET_QUARTER_OF_YEAR: C.GetQuarterOfYear, expr.GET_HLL_VALUE: C.GetHLLValue, }
UnaryExprTypeToCFunctorType maps from unary operator to C UnaryFunctorType
Functions ¶
Types ¶
type AQLQuery ¶
type AQLQuery struct {
// Name of the main table.
Table string `json:"table"`
// Foreign tables to be joined.
Joins []Join `json:"joins,omitempty"`
// Dimensions to group by on.
Dimensions []Dimension `json:"dimensions,omitempty"`
// Measures/metrics to report.
Measures []Measure `json:"measures"`
// Row level filters to apply for all measures. The filters are ANDed togther.
Filters []string `json:"rowFilters,omitempty"`
// Syntax sugar for specifying a time based range filter.
TimeFilter TimeFilter `json:"timeFilter,omitempty"`
// Timezone to use when converting timestamp to calendar time, specified as:
// - -8:00
// - GMT
// - America/Los_Angeles
// - timezone(city_id)
// - region_timezone(city_id)
// - mega_region_timezone(city_id)
// - sub_region_timezone(city_id)
// - country_timezone(city_id)
Timezone string `json:"timezone,omitempty"`
// This overrides "now" (in seconds)
Now int64 `json:"now,omitempty"`
// contains filtered or unexported fields
}
AQLQuery specifies the query on top of tables.
type AQLQueryContext ¶
type AQLQueryContext struct {
// The query input.
Query *AQLQuery `json:"query"`
// Context for one-operator-per-kernel execution.
OOPK OOPKContext `json:"oopk"`
// Scanner for all tables. [0] for the main table; [1:] for tables in joins.
TableScanners []*TableScanner `json:"scanners"`
// Map from table alias to ID (index to TableScanners).
TableIDByAlias map[string]int `json:"tableIDs"`
// Map from table name to schema for convenience. In case of self join,
// only one entry is referenced here by the name of the table.
TableSchemaByName map[string]*memstore.TableSchema `json:"-"`
// Index to filters in Query.Filters that are identified as prefilters.
Prefilters []int `json:"prefilters,omitempty"`
Error error `json:"error,omitempty"`
Device int `json:"device"`
Debug bool `json:"debug,omitempty"`
Profiling string `json:"profiling,omitempty"`
Results queryCom.AQLTimeSeriesResult `json:"-"`
// whether to serialize the query result as HLLData. If ReturnHLLData is true, we will not release dimension
// vector and measure vector until serialization is done.
ReturnHLLData bool `json:"ReturnHLLData"`
HLLQueryResult []byte `json:"-"`
// contains filtered or unexported fields
}
AQLQueryContext stores all contextual data for handling an AQL query.
func (*AQLQueryContext) FindDeviceForQuery ¶
func (qc *AQLQueryContext) FindDeviceForQuery(memStore memstore.MemStore, preferredDevice int, deviceManager *DeviceManager, timeout int)
FindDeviceForQuery calls device manager to find a device for the query
func (*AQLQueryContext) Postprocess ¶
func (qc *AQLQueryContext) Postprocess() queryCom.AQLTimeSeriesResult
Postprocess converts the internal dimension and measure vector in binary format to AQLTimeSeriesResult nested result format. It also translates enum values back to their string representations.
func (*AQLQueryContext) PostprocessAsHLLData ¶
func (qc *AQLQueryContext) PostprocessAsHLLData() ([]byte, error)
PostprocessAsHLLData serializes the query result into HLLData format. It will also release the device memory after serialization.
func (*AQLQueryContext) ProcessQuery ¶
func (qc *AQLQueryContext) ProcessQuery(memStore memstore.MemStore)
ProcessQuery processes the compiled query and executes it on GPU.
func (*AQLQueryContext) Release ¶
func (qc *AQLQueryContext) Release()
Release releases all device memory it allocated. It **should only called** when any errors happens while the query is processed.
func (*AQLQueryContext) ReleaseHostResultsBuffers ¶
func (qc *AQLQueryContext) ReleaseHostResultsBuffers()
ReleaseHostResultsBuffers deletes the result buffer from host memory after postprocessing
func (*AQLQueryContext) Rewrite ¶
func (qc *AQLQueryContext) Rewrite(expression expr.Expr) expr.Expr
Rewrite walks the expresison AST and resolves data types bottom up. In addition it also translates enum strings and rewrites their predicates.
func (*AQLQueryContext) SerializeHLL ¶
func (qc *AQLQueryContext) SerializeHLL(dataTypes []memCom.DataType, enumDicts map[int][]string, timeDimensions []int) ([]byte, error)
SerializeHLL allocates buffer based on the metadata and then serializes hll data into the buffer.
type AQLRequest ¶
type AQLRequest struct {
Queries []AQLQuery `json:"queries"`
}
AQLRequest contains multiple of AQLQueries.
type AQLResponse ¶
type AQLResponse struct {
Results []queryCom.AQLTimeSeriesResult `json:"results"`
Errors []error `json:"errors,omitempty"`
QueryContext []*AQLQueryContext `json:"context,omitempty"`
}
AQLResponse contains results for multiple AQLQueries.
type DeviceInfo ¶
type DeviceInfo struct {
// device id
DeviceID int `json:"deviceID"`
// number of queries being served by device
QueryCount int `json:"queryCount"`
// device capacity.
TotalMemory int `json:"totalMemory"`
// device available capacity.
TotalAvailableMemory int `json:"totalAvailableMemory"`
// total free memory
FreeMemory int `json:"totalFreeMemory"`
// query to memory map
QueryMemoryUsageMap map[*AQLQuery]int `json:"-"`
}
DeviceInfo stores memory information per device
type DeviceManager ¶
type DeviceManager struct {
// lock to sync ops.
sync.RWMutex `json:"-"`
// device to DeviceInfo map
DeviceInfos []*DeviceInfo `json:"deviceInfos"`
// default DeviceChoosingTimeout for finding a device
Timeout int `json:"timeout"`
// Max available memory, this can be used to early determined whether a query can be satisfied or not.
MaxAvailableMemory int `json:"maxAvailableMemory"`
// contains filtered or unexported fields
}
DeviceManager has the following functionalities: 1. Keep track of number of queries being served by this device and memory usage info 2. Estimate the memory requirement for a given query and determine if a device has enough memory to process a query 3. Assign queries to chosen device according to routing strategy specified
func NewDeviceManager ¶
func NewDeviceManager(cfg common.QueryConfig) *DeviceManager
NewDeviceManager is used to init a DeviceManager.
func (*DeviceManager) FindDevice ¶
func (d *DeviceManager) FindDevice(query *AQLQuery, requiredMem int, preferredDevice int, timeout int) int
FindDevice finds a device to run a given query. If a device is not found, it will wait until the DeviceChoosingTimeout seconds elapse.
func (*DeviceManager) ReleaseReservedMemory ¶
func (d *DeviceManager) ReleaseReservedMemory(device int, query *AQLQuery)
ReleaseReservedMemory adjust total free global memory for a given device after a query is complete
type Dimension ¶
type Dimension struct {
// The SQL expression for computing the dimension.
// Expr can be empty when TimeBucketizer is specified, which implies the
// designated time column from the main table is used as the expresssion.
Expr string `json:"sqlExpression"`
// Decides how to bucketize a timestamp Dimension before grouping by.
// See https://github.com/uber/aresdb/wiki/aql#time_bucketizer
TimeBucketizer string `json:"timeBucketizer,omitempty"`
TimeUnit string `json:"timeUnit,omitempty"`
// Bucketizes numeric dimensions for integers and floating point numbers.
NumericBucketizer NumericBucketizerDef `json:"numericBucketizer,omitempty"`
// contains filtered or unexported fields
}
Dimension specifies a row level dimension for grouping by.
type HLLDataWriter ¶
HLLDataWriter is the struct to serialize HLL Data struct.
func (*HLLDataWriter) SerializeHeader ¶
func (builder *HLLDataWriter) SerializeHeader() error
SerializeHeader first serializes header into buffer the builder holds.
type HLLQueryResults ¶
type HLLQueryResults struct {
// contains filtered or unexported fields
}
HLLQueryResults holds the buffer to store multiple hll query results or errors.
func NewHLLQueryResults ¶
func NewHLLQueryResults() *HLLQueryResults
NewHLLQueryResults returns a new NewHLLQueryResults and writes the magical header and padding to underlying buffer.
func (*HLLQueryResults) GetBytes ¶
func (r *HLLQueryResults) GetBytes() []byte
GetBytes returns the underlying bytes.
func (*HLLQueryResults) WriteError ¶
func (r *HLLQueryResults) WriteError(err error)
WriteError write error to the buffer.
func (*HLLQueryResults) WriteResult ¶
func (r *HLLQueryResults) WriteResult(result []byte)
WriteResult write result to the buffer.
type Join ¶
type Join struct {
// Name of the table to join against.
Table string `json:"table"`
// Alias for the table. Empty means the table name will be used as alias.
Alias string `json:"alias"`
// Condition expressions to be ANDed together for the join.
Conditions []string `json:"conditions"`
// contains filtered or unexported fields
}
Join specifies a secondary table to be explicitly joined in the query.
type Measure ¶
type Measure struct {
// The SQL expression for computing the measure.
Expr string `json:"sqlExpression"`
// Row level filters to apply for this measure.
// The filters are ANDed togther.
Filters []string `json:"rowFilters,omitempty"`
// contains filtered or unexported fields
}
Measure specifies a group level aggregation measure.
type NumericBucketizerDef ¶
type NumericBucketizerDef struct {
// Generates equal-width buckets. BucketWidth should be positive.
// The generated buckets are:
// ... [-2w, -w), [-w, 0), [0, w), [w, 2w) ...
BucketWidth float64 `json:"bucketWidth,omitempty"`
// Generates exponential/log buckets. LogBase should be positive.
// The generated buckets are:
// ... [pow(b, -2), pow(b, -1)), [pow(b, -1), 1), [1, pow(b, 1)), [pow(b, 1), pow(b, 2)) ...
LogBase float64 `json:"logBase,omitempty"`
// Generates a fixed number of buckets using the specified partitions.
// The numbers should be in sorted order. The generated buckets are:
// [-inf, p0), [p0, p1), [p1, p2), ... [pn-1, inf)
ManualPartitions []float64 `json:"manualPartitions,omitempty"`
}
NumericBucketizerDef defines how numbers should be bucketized before being grouped by as a dimension. The returned dimension is a string in the format of `lower_bound`, representing `[lower_bound, uper_bound)`.
type OOPKContext ¶
type OOPKContext struct {
// Filters that apply to all archive and live batches.
// MainTableCommonFilters match filters with only main table columns involved
MainTableCommonFilters []expr.Expr `json:"mainTableCommonFilters,omitempty"`
// ForeignTableCommonFilters match filters with foreign table columns involved
ForeignTableCommonFilters []expr.Expr `json:"foreignTableCommonFilters,omitempty"`
// Lower bound [0] and upper bound [1] time filter. nil if not applicable.
// [0] should be applied to the first archive batch and all live batches.
// [1] should be applied to the last archive batch and all live batches.
TimeFilters [2]expr.Expr `json:"timeFilters"`
// Prefilters that only apply to live batches.
// Archiving cutoff filtering is processed directly by the query engine and not
// included here (different shards may have different cutoffs).
Prefilters []expr.Expr `json:"prefilters,omitempty"`
// Compiled and annotated ASTs for dimensions and measure.
Dimensions []expr.Expr `json:"dimensions"`
// Index of single dimension vector in global dimension vector
// Following sorted order based on bytes
DimensionVectorIndex []int `json:"dimensionVectorIndex"`
// Number of dimensions per dim width
NumDimsPerDimWidth queryCom.DimCountsPerDimWidth `json:"numDims"`
// Dim row bytes is the sum number of bytes of all dimension values
// plus validity bytes, for memory allocation convenience
DimRowBytes int `json:"dimRowBytes"`
// For one-operator-per-kernel we only support one measure per query.
Measure expr.Expr `json:"measure"`
MeasureBytes int `json:"measureBytes"`
AggregateType C.enum_AggregateFunction `json:"aggregate"`
ResultSize int `json:"resultSize"`
// For reporting purpose only.
DeviceMemoryRequirement int `json:"deviceMem"`
DurationWaitedForDevice time.Duration `json:"durationWaitedForDevice"`
// Stores the overall query stats for live batches and archive batches.
LiveBatchStats oopkQueryStats `json:"liveStats"`
ArchiveBatchStats oopkQueryStats `json:"archiveStats"`
// contains filtered or unexported fields
}
OOPKContext defines additional query context for one-operator-per-kernel execution.
type TableScanner ¶
type TableScanner struct {
// Snapshot of the table schema for convenience.
Schema *memstore.TableSchema `json:"-"`
// IDS of all table shards to be scanned on this instance.
Shards []int `json:"shards"`
// IDs of columns to be used in this query, in the following order:
// 1. Columns not from ArchivingSortColumns.
// 2. Columns from ArchivingSortColumns in reverse order.
Columns []int `json:"columns"`
// reversed mapping from columnID to column scan order index
ColumnsByIDs map[int]int `json:"-"`
// Map from column ID to its usage by the query.
ColumnUsages map[int]columnUsage `json:"columnUsage"`
// Values of equality prefilters in order. Each 4 bytes of the uint32 is used
// to store any data type other than UUID (not supported).
EqualityPrefilterValues []uint32 `json:"equalityPrefilterValues,omitempty"`
// Boundary types and values of the final range prefilter.
RangePrefilterBoundaries [2]boundaryType `json:"rangePrefilterBoundaries"`
RangePrefilterValues [2]uint32 `json:"rangePrefilterValues"`
// Range of archive batches to process: [Start, end).
// Depending on the archiving progress of each shard, live batches may be
// skipped for processing if the archiving cutoff is after the time of
// ArchiveBatchIDEnd.
ArchiveBatchIDStart int `json:"archiveBatchIDStart"`
ArchiveBatchIDEnd int `json:"archiveBatchIDEnd"`
}
TableScanner defines how data for a table should be fed to device memory for processing (scanner in a traditional terminology).
type TimeFilter ¶
type TimeFilter struct {
// A table time column in the format of column, or table_alias.column.
// When empty, it defaults to the designated time column of the main table.
Column string `json:"column"`
// The time specified in from and to are both inclusive.
// See https://github.com/uber/aresdb/wiki/aql#time_filter
From string `json:"from"`
To string `json:"to"`
}
TimeFilter is a syntax sugar for specifying time range.