Documentation
¶
Overview ¶
Package oql provides SQL-compatible query language for xolu.
OQL supports a subset of T-SQL syntax for querying and mutating data:
- SELECT with aggregates (COUNT, SUM, AVG, MIN, MAX)
- GROUP BY, HAVING, ORDER BY, TOP
- INNER, LEFT, RIGHT, and FULL OUTER JOIN (two tables, SQLite store only)
- INSERT with VALUES
- UPDATE with WHERE (required)
- DELETE with WHERE (required)
JOIN support ¶
Two-table joins are pushed to SQLite as a single SQL statement. All four standard join types are supported. Each entity is classified independently as adapted (schema registered, native columns) or blob (json_extract path); mixed joins work correctly.
Not supported: CROSS JOIN, three-or-more-table joins, subquery tables in FROM, compound ON conditions. JOIN queries require the SQLite backend.
Index ¶
- Constants
- Variables
- func ApplyOffsetFetch(records []map[string]interface{}, offset, fetch ast.Expression) []map[string]interface{}
- func ApplyTop(records []map[string]interface{}, top *ast.TopClause) []map[string]interface{}
- func EvalScalarFunction(fc *ast.FunctionCall, evalFn func(ast.Expression) interface{}) interface{}
- func EvalScalarFunctionWith(overlay map[string]ScalarFunc, fc *ast.FunctionCall, ...) interface{}
- func GetSchemaPath(schemaDir, entity string) string
- func IsScalarFunction(expr ast.Expression) bool
- func OrderBy(records []map[string]interface{}, orderBy []*ast.OrderByItem) []map[string]interface{}
- func RegisterScalarFunc(name string, fn ScalarFunc)
- func RegisterSeqGenFuncs(e *Executor)
- func ValidateSchemaDir(schemaDir string) error
- type AdaptedSQL
- type AggregateFunc
- type AggregateSQL
- type Aggregator
- func (a *Aggregator) Aggregate(records []map[string]interface{}, columns []ast.SelectColumn, ...) []map[string]interface{}
- func (a *Aggregator) EvalCondition(row map[string]interface{}, expr ast.Expression) bool
- func (a *Aggregator) SetDecimalFields(fields map[string]bool)
- func (a *Aggregator) SetExprAliases(columns []ast.SelectColumn)
- type Engine
- func (e *Engine) Execute(ctx context.Context, sql string) (*Result, error)
- func (e *Engine) ExecuteWithStore(ctx context.Context, sql string, store storage.Store) (*Result, error)
- func (e *Engine) RefreshSchema()
- func (e *Engine) SetGenDispatcher(fn func(tenantID tenant.TenantID, name string) (string, error))
- func (e *Engine) SetLimits(limits QueryLimits)
- func (e *Engine) SetProfile(profile *HardwareProfile)
- func (e *Engine) SetSeqIncrementor(fn func(tenantID tenant.TenantID, name string) (int64, error))
- type EntityChecker
- type Executor
- func (e *Executor) Execute(ctx context.Context, stmt ast.Statement) (*Result, error)
- func (e *Executor) ExecuteWithStore(ctx context.Context, stmt ast.Statement, store storage.Store) (*Result, error)
- func (e *Executor) ExecuteWithTenant(ctx context.Context, stmt ast.Statement, tenantID string) (*Result, error)
- func (e *Executor) SetGenDispatcher(fn func(tenantID tenant.TenantID, name string) (string, error))
- func (e *Executor) SetLimits(limits QueryLimits)
- func (e *Executor) SetProfile(profile *HardwareProfile)
- func (e *Executor) SetSeqIncrementor(fn func(tenantID tenant.TenantID, name string) (int64, error))
- type GeneratedSQL
- type HardwareProfile
- type Job
- type JobManager
- func (jm *JobManager) Close()
- func (jm *JobManager) ExecuteSync(ctx context.Context, query string) (*Result, error)
- func (jm *JobManager) ExecuteSyncWithStore(ctx context.Context, query string, store storage.Store) (*Result, error)
- func (jm *JobManager) GetJob(id string) *Job
- func (jm *JobManager) GetJobResult(id string) (*Result, error)
- func (jm *JobManager) SetGenDispatcher(fn func(tenantID tenant.TenantID, name string) (string, error))
- func (jm *JobManager) SetQueryTimeout(d time.Duration)
- func (jm *JobManager) SetSeqIncrementor(fn func(tenantID tenant.TenantID, name string) (int64, error))
- func (jm *JobManager) Submit(query string, store storage.Store) string
- type JobStatus
- type JoinSQL
- type Planner
- func NewPlanner() *Planner
- func NewPlannerFromDialect(dialect SQLDialect) *Planner
- func NewPlannerWithDialectAndThreshold(dialect SQLDialect, threshold int) *Planner
- func NewPlannerWithProfile(dialect SQLDialect, profile *HardwareProfile) *Planner
- func NewPlannerWithThreshold(threshold int) *Planner
- type PredicateCompileResult
- type PushDecision
- type QueryComplexity
- type QueryLimits
- type QueryPlan
- type QueryStats
- type Result
- type ResultType
- type SQLDialect
- type SQLGenerator
- type SQLiteDialect
- func (d *SQLiteDialect) BaseQuery(entity string) (string, interface{})
- func (d *SQLiteDialect) CastExpression(expr, targetType string) string
- func (d *SQLiteDialect) DefaultThreshold() int
- func (d *SQLiteDialect) JSONField(fieldPath string) string
- func (d *SQLiteDialect) JSONFieldAliased(alias, fieldPath string) string
- func (d *SQLiteDialect) JSONFieldAliasedAs(alias, fieldPath, oqlType string) string
- func (d *SQLiteDialect) JSONFieldAs(fieldPath, oqlType string) string
- func (d *SQLiteDialect) JSONFieldNumeric(fieldPath string) string
- func (d *SQLiteDialect) LimitClause(placeholder string) string
- func (d *SQLiteDialect) Name() string
- func (d *SQLiteDialect) Placeholder(_ int) string
- func (d *SQLiteDialect) ScalarFunction(name string, argSQL string) (string, bool)
- type ScalarFunc
- type SchemaValidator
- type Validator
Constants ¶
const DefaultPushDownThreshold = 200
DefaultPushDownThreshold is the fallback minimum entity count above which push-down becomes worthwhile, used only when no dialect is available to provide a backend-specific threshold. Each SQLDialect implementation provides its own threshold via DefaultThreshold() based on actual benchmark data for that backend.
Variables ¶
var ( ErrScanLimit = errors.New("query scan limit exceeded") ErrResultLimit = errors.New("query result limit exceeded") )
Sentinel errors for query limit violations.
var ( // ProfileEdge targets ARM single-board computers, industrial gateways, // and low-power edge devices (1-2 cores, 1-4 GB RAM, eMMC/SD storage). // Go's GC is proportionally expensive; SQLite's C engine has a larger // relative advantage, so push-down thresholds are lower. ProfileEdge = HardwareProfile{ Name: "edge", BlobPushThreshold: 25, NonCoveringThreshold: 500, TempBTree1Threshold: 250, TempBTree2Threshold: 1000, } // ProfileVPS targets small cloud instances (1-2 vCPU, 2-8 GB RAM, SSD). // Shared CPU with noisy neighbours. This is the default and most common // deployment target for self-hosted xolu instances. ProfileVPS = HardwareProfile{ Name: "vps", BlobPushThreshold: 50, NonCoveringThreshold: 1000, TempBTree1Threshold: 500, TempBTree2Threshold: 2000, } // ProfileDedicated targets bare metal or large instances (4+ cores, // 16+ GB RAM). Go has GC headroom and the CPU is fast enough that // push-down overhead must be justified by larger datasets. ProfileDedicated = HardwareProfile{ Name: "dedicated", BlobPushThreshold: 100, NonCoveringThreshold: 2000, TempBTree1Threshold: 1000, TempBTree2Threshold: 5000, } )
Predefined profiles for common deployment targets.
var Aggregates = map[string]AggregateFunc{
"COUNT": aggCount,
"SUM": aggSum,
"AVG": aggAvg,
"MIN": aggMin,
"MAX": aggMax,
}
Aggregates maps function names to their implementations
var DecimalAggregates = map[string]AggregateFunc{
"SUM": aggDecimalSum,
"AVG": aggDecimalAvg,
"MIN": aggDecimalMin,
"MAX": aggDecimalMax,
}
DecimalAggregates maps aggregate function names to decimal-precise implementations.
var ScalarFunctions = map[string]ScalarFunc{ "DATE_TRUNC": qs.ScalarDateTrunc, "UPPER": qs.ScalarUpper, "LOWER": qs.ScalarLower, "LEN": qs.ScalarLen, "TRIM": qs.ScalarTrim, "COALESCE": qs.ScalarCoalesce, "ISNULL": qs.ScalarCoalesce, "CONCAT": qs.ScalarConcat, "CAST": qs.ScalarCast, "ABS": qs.ScalarAbs, "ROUND": qs.ScalarRound, "FLOOR": qs.ScalarFloor, "CEILING": qs.ScalarCeiling, "GETDATE": qs.ScalarGetDate, "GETUTCDATE": qs.ScalarGetUTCDate, "YEAR": qs.ScalarYear, "MONTH": qs.ScalarMonth, "DAY": qs.ScalarDay, "DATEPART": qs.ScalarDatePart, "DATEDIFF": qs.ScalarDateDiff, "SUBSTRING": qs.ScalarSubstring, "LEFT": qs.ScalarLeft, "RIGHT": qs.ScalarRight, "REPLACE": qs.ScalarReplace, "CHARINDEX": qs.ScalarCharIndex, "NEWID": qs.ScalarNewID, "@SEQ": func(_ []interface{}) interface{} { return nil }, "@GEN": func(_ []interface{}) interface{} { return nil }, }
ScalarFunctions maps function names to their implementations. ScalarFunctions maps T-SQL function names to implementations. Implementations live in pkg/qs and are shared with Sulpher. OQL-specific aliases (e.g. ISNULL) are added here.
Functions ¶
func ApplyOffsetFetch ¶ added in v0.30.23
func ApplyOffsetFetch(records []map[string]interface{}, offset, fetch ast.Expression) []map[string]interface{}
ApplyOffsetFetch applies T-SQL's own OFFSET N ROWS FETCH NEXT M ROWS ONLY pagination clause in Go (XM-5, xoluman's own report, 2026-08-12). Root cause of the original bug: stmt.Offset/stmt.Fetch were parsed correctly into the AST -- confirmed directly, tsqlparser's own grammar handles both -- but nothing anywhere in this package ever read either field. Not a rejection, not a silent misinterpretation: a genuine no-op, the clause simply vanished after parsing. Applied unconditionally here, with no "already pushed to SQL" guard the way ApplyTop has one, since neither clause is pushed to SQL anywhere in this codebase yet -- a real efficiency gap for a large offset against a large table (the full result set is still fetched before slicing), left as a known limitation rather than solved here; correctness first. offset/fetch are each nil when the clause was omitted, matching stmt.Offset/stmt.Fetch's own zero value; a non-integer-literal expression (a bound parameter, say) is treated as absent rather than erroring, matching ApplyTop's own established behaviour for a non-integer TOP count.
func EvalScalarFunction ¶
func EvalScalarFunction(fc *ast.FunctionCall, evalFn func(ast.Expression) interface{}) interface{}
EvalScalarFunction evaluates a scalar function call against a record. The evalFn callback is used to resolve argument expressions to values. Resolution consults only the package-level defaults; executor-bound functions (@SEQ, @GEN) resolve through EvalScalarFunctionWith.
func EvalScalarFunctionWith ¶
func EvalScalarFunctionWith(overlay map[string]ScalarFunc, fc *ast.FunctionCall, evalFn func(ast.Expression) interface{}) interface{}
EvalScalarFunctionWith evaluates a scalar function call, consulting the given instance overlay before the package-level defaults. The overlay carries executor-bound functions; nil is a valid overlay.
func GetSchemaPath ¶
GetSchemaPath returns the full path to an entity's schema
func IsScalarFunction ¶
func IsScalarFunction(expr ast.Expression) bool
IsScalarFunction checks whether a function call is a scalar (non-aggregate) function. Returns true if the function name matches a known scalar.
func OrderBy ¶
func OrderBy(records []map[string]interface{}, orderBy []*ast.OrderByItem) []map[string]interface{}
OrderBy sorts records by the specified order items
func RegisterScalarFunc ¶
func RegisterScalarFunc(name string, fn ScalarFunc)
RegisterScalarFunc registers a new scalar function in the package-level OQL function map. The name is normalised to uppercase. Calling this with an existing name overwrites the previous registration.
ONLY safe to call from init() (or otherwise strictly before any Engine exists): the map is read on every query evaluation and is not internally synchronised. Runtime or per-engine registrations belong on the Executor's instance overlay instead (T-40 — a per-engine registration here crashed with concurrent map writes and cross-wired engines).
func RegisterSeqGenFuncs ¶
func RegisterSeqGenFuncs(e *Executor)
RegisterSeqGenFuncs registers @SEQ and @GEN on the given Executor. Must be called after NewExecutor and before the first query using sequences.
@SEQ('name') — session-local last value of a named sequence (no increment).
Returns nil if NEXT VALUE FOR has not been called in this session for the named sequence (XOLU-GEN006).
@GEN('name') — dispatch to any stateful generator by name.
STUB: returns nil until S10 (stateful generators) is implemented. Registered now so queries using @GEN() parse and execute without error rather than failing at the unknown-function path.
func ValidateSchemaDir ¶
ValidateSchemaDir checks if the schema directory exists
Types ¶
type AdaptedSQL ¶
type AdaptedSQL struct {
SQL string
Args []interface{}
Aliases []string // Column aliases in SELECT order
// DecimalColumns tracks which result aliases need denormalisation.
// Only populated when the backend stores decimals as scaled integers
// (i.e. SupportsNativeDecimalAggregation() returns false).
DecimalColumns map[string]int
}
AdaptedSQL holds the output of GenerateAdaptedSQL: a complete SELECT statement targeting native columns in an adapted table, plus metadata for post-query decimal denormalisation.
func GenerateAdaptedSQL ¶
func GenerateAdaptedSQL( stmt *ast.SelectStatement, entity string, tenantID string, store storage.AggregateQueryable, dialect SQLDialect, ) (*AdaptedSQL, error)
GenerateAdaptedSQL translates a complete OQL SELECT statement into adapted-table SQL. Unlike GenerateAggregateSQL (which only handles GROUP BY + aggregates), this generates full queries including:
- SELECT with scalars, aggregates, and plain columns
- WHERE with all comparison operators
- GROUP BY
- HAVING (translated to SQL, not evaluated in Go)
- ORDER BY
- DISTINCT
- LIMIT
The caller must verify isFullyTranslatable() before calling this. If any clause is untranslatable, the executor falls back to the Go pipeline via the existing paths.
type AggregateFunc ¶
type AggregateFunc func(values []interface{}) interface{}
AggregateFunc is a function that computes an aggregate over values
type AggregateSQL ¶
type AggregateSQL struct {
SQL string
Args []interface{}
Aliases []string // Column aliases in SELECT order
// DecimalColumns tracks which result aliases are decimal aggregates
// that need denormalisation. Key: alias, Value: scale.
DecimalColumns map[string]int
}
AggregateSQL holds the generated aggregate query plus metadata needed to denormalise decimal results after execution.
func GenerateAggregateSQL ¶
func GenerateAggregateSQL( stmt *ast.SelectStatement, entity string, tenantID string, store storage.AggregateQueryable, dialect SQLDialect, ) (*AggregateSQL, error)
GenerateAggregateSQL builds a GROUP BY + aggregate query for an adapted table. Instead of SELECT data, _version FROM entities WHERE ... it generates:
SELECT category, SUM(price), COUNT(*) FROM xolu_products WHERE tenant_id = $1 AND (status = $2) GROUP BY category
The dialect parameter controls placeholder syntax and type coercion. This only works for adapted entities with native columns.
type Aggregator ¶
type Aggregator struct {
// contains filtered or unexported fields
}
Aggregator handles GROUP BY and aggregate function execution
func (*Aggregator) Aggregate ¶
func (a *Aggregator) Aggregate( records []map[string]interface{}, columns []ast.SelectColumn, groupBy []ast.Expression, having ast.Expression, ) []map[string]interface{}
Aggregate groups records and applies aggregate functions
func (*Aggregator) EvalCondition ¶
func (a *Aggregator) EvalCondition(row map[string]interface{}, expr ast.Expression) bool
EvalCondition evaluates a HAVING condition against a row. Exported for use by aggregate push-down HAVING filter.
func (*Aggregator) SetDecimalFields ¶
func (a *Aggregator) SetDecimalFields(fields map[string]bool)
SetDecimalFields configures which fields should use decimal-precise aggregation. Call this before Aggregate when adapted table metadata indicates decimal columns.
func (*Aggregator) SetExprAliases ¶ added in v0.30.23
func (a *Aggregator) SetExprAliases(columns []ast.SelectColumn)
SetExprAliases configures the aggregate-expression-to-alias mapping used by evalExpr's own FunctionCall case when evaluating a HAVING condition against already-aggregated rows (e.g. rows produced by the SQL aggregate push-down path, whose own result columns carry only their declared alias, never also the raw expression string). Call this before EvalCondition/evalCondition with the SELECT list's own columns.
type Engine ¶
type Engine struct {
// contains filtered or unexported fields
}
Engine is the main OQL query engine
func NewEngineWithSchemaValidator ¶
func NewEngineWithSchemaValidator(store storage.Store, schemaDir string, sv SchemaValidator) *Engine
NewEngineWithSchemaValidator creates an OQL engine with schema validation
func (*Engine) ExecuteWithStore ¶
func (e *Engine) ExecuteWithStore(ctx context.Context, sql string, store storage.Store) (*Result, error)
ExecuteWithStore parses, validates, and executes an OQL query using a specific store. The store should already be scoped to the target tenant.
func (*Engine) RefreshSchema ¶
func (e *Engine) RefreshSchema()
RefreshSchema reloads the entity list from disk
func (*Engine) SetGenDispatcher ¶
SetGenDispatcher forwards the named-generator dispatch function to the underlying executor.
func (*Engine) SetLimits ¶
func (e *Engine) SetLimits(limits QueryLimits)
SetLimits configures query execution limits for this engine.
func (*Engine) SetProfile ¶
func (e *Engine) SetProfile(profile *HardwareProfile)
SetProfile updates the hardware profile used by the query planner. Call this during server startup after calibration or profile selection.
type EntityChecker ¶
EntityChecker is an interface for checking entity existence
type Executor ¶
type Executor struct {
// contains filtered or unexported fields
}
Executor executes OQL queries against storage
func NewExecutor ¶
func NewExecutor(store storage.Store, sv SchemaValidator) *Executor
NewExecutor creates a new executor
func (*Executor) ExecuteWithStore ¶
func (e *Executor) ExecuteWithStore(ctx context.Context, stmt ast.Statement, store storage.Store) (*Result, error)
ExecuteWithStore executes a validated AST statement using a specific store. This is the preferred method for tenant-scoped queries: the caller passes a store already scoped to the tenant. The store's TenantID is extracted and set as sqlTenantID so that push-down queries include a tenant_id WHERE clause. The Go-path does not need additional filtering because the store's List/Search methods already return only that tenant's data.
func (*Executor) ExecuteWithTenant ¶
func (e *Executor) ExecuteWithTenant(ctx context.Context, stmt ast.Statement, tenantID string) (*Result, error)
ExecuteWithTenant executes a validated AST statement with tenant scoping. When tenantID is non-empty, all operations are filtered to records where the tenant_id field matches. This ensures OQL queries respect tenant isolation.
func (*Executor) SetGenDispatcher ¶
SetGenDispatcher wires in the server's named-generator dispatch function. Call once at server startup after the Executor is created. fn receives the tenant ID and generator name, resolves the named definition in gen_definitions, and produces one value. Nil leaves @GEN returning nil.
func (*Executor) SetLimits ¶
func (e *Executor) SetLimits(limits QueryLimits)
SetLimits configures query execution limits. Non-positive values are replaced with safe defaults so the executor never holds a limit-disabling zero.
func (*Executor) SetProfile ¶
func (e *Executor) SetProfile(profile *HardwareProfile)
SetProfile updates the planner's hardware profile, which controls push-down thresholds for complex queries. Call this after NewExecutor and before serving requests — typically during server startup after calibration or profile selection.
func (*Executor) SetSeqIncrementor ¶
SetSeqIncrementor wires in the server's sequence increment function. Call once at server startup after the Executor is created. fn receives the tenant ID and sequence name and returns the new current value.
type GeneratedSQL ¶
type GeneratedSQL struct {
SQL string
Args []interface{}
}
GeneratedSQL holds the output of the SQL generator: a parameterised SQL string and the ordered argument list. The SQL is ready for execution via storage.Queryable.QueryWithPlan().
func GenerateSQL ¶
func GenerateSQL( stmt *ast.SelectStatement, entity string, tenantID string, plan QueryPlan, dialect SQLDialect, ) (*GeneratedSQL, error)
GenerateSQL translates the pushable portions of an OQL SELECT statement into backend-specific SQL. Only operations listed in plan.Push are translated; the executor handles the rest in Go.
The function is the main entry point for SQL generation and is dialect-agnostic — it delegates all syntax decisions to the dialect.
type HardwareProfile ¶
type HardwareProfile struct {
Name string // "edge", "vps", "dedicated", "calibrated", or "custom"
// BlobPushThreshold is the minimum row count at which blob entity
// queries are pushed to SQL (json_extract). Below this count, the
// Go path processes rows directly without a CountEntities round-trip.
BlobPushThreshold int
// ComplexityThresholds gate PushFull for adapted entities based on
// estimated query complexity. When estimated complexity exceeds zero,
// the planner checks row count against these thresholds before
// committing to PushFull.
//
// A query's complexity is estimated from its AST:
// - Each multi-key GROUP BY adds 1 temp B-tree
// - Each ORDER BY misaligned with GROUP BY adds 1 temp B-tree
// - Non-COUNT aggregates on columns not in GROUP BY set nonCovering
//
// Simple queries (0 temp B-trees, covering) always use PushFull.
NonCoveringThreshold int // min rows for push-down with non-covering aggregate scan
TempBTree1Threshold int // min rows for push-down with 1 temp B-tree
TempBTree2Threshold int // min rows for push-down with 2+ temp B-trees
}
HardwareProfile holds threshold values that control when the planner pushes queries to SQL versus processing in Go. The optimal thresholds depend on the relative speed of Go's JSON processing versus SQLite's C engine on the host hardware.
Three named presets cover common deployment targets. The Calibrate() function can derive a custom profile from a startup micro-benchmark.
func Calibrate ¶
func Calibrate(db *sql.DB) (*HardwareProfile, error)
Calibrate runs a short micro-benchmark against the given SQLite database to measure the relative speed of Go JSON processing versus SQLite's query engine on the current hardware. It returns a HardwareProfile with thresholds tuned to the measured ratios.
The benchmark takes several milliseconds depending on hardware and creates and drops a temporary table, so it has no lasting side effects on the database.
Cached process-wide after the first call, successful or not. Calibration measures a hardware property (relative speed of Go JSON processing vs SQLite's query engine on THIS machine, via a fixed, self-contained 200-row benchmark table -- not anything about the caller's own database's current size or content), which does not change between calls within one process. Recalibrating per Server instance, as every prior call site did unconditionally, was measured directly: 965 redundant calibration calls within a single 8-package test shard, ~17ms real cost each (measured directly, not assumed from this comment's own earlier, more conservative estimate) -- pure waste once the first call has already answered the only question calibration exists to answer. A production binary calls this once at real startup regardless, so caching changes nothing there; it only removes redundant work in any process (chiefly tests) that constructs many Server instances.
If the first call fails for any reason, it returns the VPS default profile with an error -- and that outcome is cached too, same as a success. A DB broken enough to fail calibration once is not expected to become calibratable moments later within the same process, and the fallback default profile is a safe, working one.
func DefaultProfile ¶
func DefaultProfile() HardwareProfile
DefaultProfile returns the VPS profile, which is the safest middle ground when no profile is specified.
func ProfileByName ¶
func ProfileByName(name string) *HardwareProfile
ProfileByName returns the named preset, or nil if not found. Accepts: "edge", "vps", "dedicated" (case-insensitive).
type Job ¶
type Job struct {
ID string
Query string
Status JobStatus
Result *Result
Error string
CreatedAt time.Time
UpdatedAt time.Time
// contains filtered or unexported fields
}
Job represents an async OQL query job
type JobManager ¶
type JobManager struct {
// contains filtered or unexported fields
}
JobManager manages async OQL query jobs
func NewJobManager ¶
func NewJobManager(engine *Engine, ttl time.Duration) *JobManager
NewJobManager creates a new job manager
func (*JobManager) ExecuteSync ¶
ExecuteSync executes a query synchronously
func (*JobManager) ExecuteSyncWithStore ¶
func (jm *JobManager) ExecuteSyncWithStore(ctx context.Context, query string, store storage.Store) (*Result, error)
ExecuteSyncWithStore executes a query synchronously using a specific store.
func (*JobManager) GetJobResult ¶
func (jm *JobManager) GetJobResult(id string) (*Result, error)
GetJobResult returns the result of a completed job
func (*JobManager) SetGenDispatcher ¶
func (jm *JobManager) SetGenDispatcher(fn func(tenantID tenant.TenantID, name string) (string, error))
SetGenDispatcher forwards the named-generator dispatch function to the underlying executor, enabling @GEN('name') resolution.
func (*JobManager) SetQueryTimeout ¶
func (jm *JobManager) SetQueryTimeout(d time.Duration)
SetQueryTimeout sets the maximum execution time for async queries.
func (*JobManager) SetSeqIncrementor ¶
func (jm *JobManager) SetSeqIncrementor(fn func(tenantID tenant.TenantID, name string) (int64, error))
SetSeqIncrementor wires the sequence increment function into the underlying OQL engine. Call once after the JobManager is created, when v2 is enabled.
type JoinSQL ¶
type JoinSQL struct {
SQL string
Args []interface{}
Aliases []string // Result column aliases in SELECT order
// DecimalColumns is alias -> scale for every adapted-side decimal
// column selected. XM-7b (xoluman's own report, 2026-08-12): this
// field, and the tracking that populates it, didn't exist at all
// before this fix -- generateJoinSelectColumns already called
// store.AdaptedColumnInfo (which returns scale/isDecimal) but
// discarded both, so a decimal field through a JOIN returned its
// raw, scaled-integer stored form untransformed (e.g. "333000.65"
// stored/returned elsewhere as that exact decimal string came back
// as the bare integer 33300065 through a JOIN specifically -- an
// exact x100 scale factor, not a computation error, consistent
// with the missing denormalisation step this field and
// denormaliseAggregateDecimals (already used by the adapted and
// aggregate paths, reused here unmodified) now close.
DecimalColumns map[string]int
}
JoinSQL holds the output of GenerateJoinSQL: a complete two-table SELECT statement and the metadata needed to map result rows back to OQL records.
func GenerateJoinSQL ¶
func GenerateJoinSQL( stmt *ast.SelectStatement, plan QueryPlan, tenantID string, store storage.AggregateQueryable, dialect SQLDialect, ) (*JoinSQL, error)
GenerateJoinSQL translates a two-table OQL SELECT + JOIN into a single SQL statement. The plan must contain a non-nil Join field (set by planJoin).
SQL shape depends on whether each entity is adapted or blob-stored:
Both adapted: SELECT a.<col>, b.<col> FROM <left> a JOIN <right> b ON ... Both blob: SELECT a.data, b.data FROM entities a JOIN entities b ON ... Mixed: SELECT a.<col>, json_extract(b.data, '$.x') FROM <left> a JOIN entities b ON ...
All field accesses use dialect methods — no literal json_extract strings. All placeholders use dialect.Placeholder(n) — no literal ? or $N.
type Planner ¶
type Planner struct {
// contains filtered or unexported fields
}
Planner examines parsed OQL ASTs and decides which operations to delegate to the storage engine (push-down) versus executing in Go.
The planner is conservative: it only pushes down operations when (a) the backend supports the operation, (b) the entity cardinality exceeds the threshold, and (c) the expression tree is fully translatable to SQL.
For adapted entities the planner can bypass the cardinality check entirely: if the full query is translatable to native-column SQL, push-down is always beneficial regardless of row count — unless the query's estimated complexity exceeds a hardware-dependent threshold (see EstimateComplexity and HardwareProfile).
func NewPlanner ¶
func NewPlanner() *Planner
NewPlanner creates a planner with the default push-down threshold and no hardware profile. Prefer NewPlannerFromDialect or NewPlannerWithProfile when possible.
func NewPlannerFromDialect ¶
func NewPlannerFromDialect(dialect SQLDialect) *Planner
NewPlannerFromDialect creates a planner whose threshold is derived from the backend's characteristics. An in-process SQLite has near-zero call overhead (threshold=50), while a networked backend like Postgres has connection and round-trip costs that justify a higher threshold.
func NewPlannerWithDialectAndThreshold ¶
func NewPlannerWithDialectAndThreshold(dialect SQLDialect, threshold int) *Planner
NewPlannerWithDialectAndThreshold creates a planner with both a custom threshold and a dialect. Used by tests that need adapted-entity awareness at low row counts.
func NewPlannerWithProfile ¶
func NewPlannerWithProfile(dialect SQLDialect, profile *HardwareProfile) *Planner
NewPlannerWithProfile creates a planner configured from a hardware profile. The profile's BlobPushThreshold overrides the dialect default, and its complexity thresholds gate PushFull for expensive adapted-entity queries.
func NewPlannerWithThreshold ¶
NewPlannerWithThreshold creates a planner with a custom threshold. Useful for testing with smaller datasets.
func (*Planner) Plan ¶
Plan examines a SELECT statement and the storage backend to produce a QueryPlan. The plan tells the executor which operations to push down and which to execute in Go.
For adapted entities the planner checks full translatability first, skipping the CountEntities round-trip entirely when push-down is possible. For blob entities the original threshold-based logic applies.
For non-SELECT statements (UPDATE, DELETE), use PlanMutation.
func (*Planner) PlanMutation ¶
func (p *Planner) PlanMutation(ctx context.Context, where ast.Expression, entity string, store storage.Store) QueryPlan
PlanMutation examines an UPDATE or DELETE WHERE clause against the storage backend. Returns a plan that may include PushWhere to narrow the initial record fetch.
type PredicateCompileResult ¶
type PredicateCompileResult struct {
// Preds is the set of AND-combined predicates that can be evaluated
// during JSON tokenisation. May be nil if nothing was extractable.
Preds *jsonic.PredicateSet
// Residual is the remaining WHERE expression that cannot be pushed
// into the tokeniser. Must still be evaluated in Go after extraction.
// Nil if the entire WHERE was compiled to predicates.
Residual ast.Expression
}
PredicateCompileResult holds the output of compiling a WHERE clause into a jsonic.PredicateSet for inline tokenisation filtering.
func CompilePredicates ¶
func CompilePredicates(where ast.Expression) PredicateCompileResult
CompilePredicates attempts to decompose a WHERE expression into a jsonic.PredicateSet for inline evaluation during tokenisation (B4), plus a residual expression for terms that can't be pushed down.
Only AND-combined simple comparisons are extracted. OR branches, NOT wrappers, IS NULL, BETWEEN, subqueries, function calls, and any other complex expressions remain in the residual.
The caller should check result.Preds.Len() > 0 before using the predicate path.
type PushDecision ¶
type PushDecision int
PushDecision represents an operation that can be pushed to the storage engine.
const ( PushNone PushDecision = iota // Stay in Go PushWhere // Push WHERE to storage PushOrderBy // Push ORDER BY to storage PushLimit // Push TOP/LIMIT to storage PushAggregate // Push GROUP BY + aggregates to storage (adapted tables only) PushFull // Push entire SELECT to storage (adapted tables, fully translatable) PushJoin // Push a two-table JOIN to SQLite )
func (PushDecision) String ¶
func (pd PushDecision) String() string
type QueryComplexity ¶
type QueryComplexity struct {
TempBTrees int // estimated number of temp B-tree materialisations
NonCovering bool // aggregate scan requires data page access beyond the index
}
QueryComplexity describes the estimated cost characteristics of a SELECT statement when executed via PushFull on an adapted entity. The planner uses this to decide whether push-down is worthwhile at the current row count.
func EstimateComplexity ¶
func EstimateComplexity(stmt *ast.SelectStatement) QueryComplexity
EstimateComplexity examines a SELECT statement's AST and predicts the cost characteristics of executing it via PushFull on SQLite.
This is a static analysis — no database access required. The estimation assumes single-column indexes on each adapted column (which is what xolu creates by default).
The signals detected:
Multi-key GROUP BY: SQLite cannot satisfy a two-key GROUP BY from any single-column index. It will scan one index and materialise a temp B-tree for the grouping.
ORDER BY misaligned with GROUP BY: if the query has both GROUP BY and ORDER BY, and the ORDER BY columns are not a prefix of the GROUP BY output, SQLite materialises a second temp B-tree.
Non-covering aggregates: when non-COUNT aggregates (SUM, AVG, MIN, MAX) reference columns not in the GROUP BY key, the index scan is non-covering — SQLite must fetch every data page to read the aggregated values.
func (QueryComplexity) IsSimple ¶
func (qc QueryComplexity) IsSimple() bool
IsSimple returns true if the query has no complexity factors that would penalise push-down. Simple queries always use PushFull regardless of row count.
func (QueryComplexity) Threshold ¶
func (qc QueryComplexity) Threshold(profile *HardwareProfile) int
Threshold returns the minimum row count at which PushFull is expected to be faster than the Go path for this complexity level. Returns 0 for simple queries (always push).
type QueryLimits ¶
type QueryLimits struct {
MaxRows int // Max rows returned
MaxScanRows int // Max rows scanned before abort
}
QueryLimits holds server-enforced limits for query execution. Zero values mean "use default" (set by the server from config).
type QueryPlan ¶
type QueryPlan struct {
Push []PushDecision // Which operations to push
EstimatedN int // Estimated input cardinality
BackendCaps storage.QueryCapabilities // What the backend can do
Reason string // Human-readable explanation for debug log
// Join is non-nil when Push contains PushJoin.
Join *joinSpec
LeftAdapted bool // true when the left-side entity uses an adapted table
RightAdapted bool // true when the right-side entity uses an adapted table
}
QueryPlan describes which operations the planner decided to push down to the storage engine and which remain in the Go execution path.
type QueryStats ¶
type QueryStats struct {
RowsScanned int `json:"rows_scanned"`
RowsReturned int `json:"rows_returned"`
RowsAffected int `json:"rows_affected,omitempty"`
ExecutionTime time.Duration `json:"execution_time_ms"`
}
QueryStats contains execution statistics
type Result ¶
type Result struct {
Type ResultType `json:"type"`
Rows []map[string]interface{} `json:"data,omitempty"`
Stats QueryStats `json:"stats"`
}
Result represents the result of an OQL query execution
func NewMutationResult ¶
func NewMutationResult(resultType ResultType, affected int, duration time.Duration) *Result
NewMutationResult creates a result for INSERT/UPDATE/DELETE queries
type ResultType ¶
type ResultType int
ResultType indicates the type of SQL statement executed
const ( ResultSelect ResultType = iota ResultInsert ResultUpdate ResultDelete )
func (ResultType) String ¶
func (rt ResultType) String() string
type SQLDialect ¶
type SQLDialect interface {
// JSONFieldAs extracts a field from the JSON data column and casts it
// to the requested OQL type. This is the canonical extraction method;
// all comparison and ordering sites should use it rather than the
// deprecated JSONField / JSONFieldNumeric shortcuts.
//
// oqlType must be one of: "text", "numeric", "boolean", "auto".
JSONFieldAs(fieldPath, oqlType string) string
// JSONFieldAliasedAs is JSONFieldAs for JOIN queries where the data
// column is qualified by a table alias.
JSONFieldAliasedAs(alias, fieldPath, oqlType string) string
// JSONField extracts a field from the JSON data column without an
// explicit type cast. Equivalent to JSONFieldAs(fieldPath, "auto").
//
// Deprecated: use JSONFieldAs with an explicit oqlType. JSONField is
// retained for backward compatibility with existing dialect implementations
// and will be removed when a PostgreSQL dialect is added.
JSONField(fieldPath string) string
// JSONFieldNumeric extracts a field and casts it to a numeric type.
// Equivalent to JSONFieldAs(fieldPath, "numeric").
//
// Deprecated: use JSONFieldAs(fieldPath, "numeric").
JSONFieldNumeric(fieldPath string) string
// JSONFieldAliased extracts a field from a JOIN-aliased data column
// without a type cast. Equivalent to JSONFieldAliasedAs(alias, fieldPath, "auto").
//
// Deprecated: use JSONFieldAliasedAs with an explicit oqlType.
JSONFieldAliased(alias, fieldPath string) string
// Placeholder emits a parameter placeholder for the n-th argument (1-based).
// For SQLite: ?
// For Postgres: $1, $2, ...
Placeholder(n int) string
// LimitClause emits the LIMIT/TOP equivalent.
// T-SQL uses TOP N (before columns); SQLite and Postgres use LIMIT N (at end).
LimitClause(placeholder string) string
// BaseQuery emits the initial SELECT ... FROM ... WHERE entity_type = <param>.
// Returns the SQL fragment and the initial argument (entity name).
BaseQuery(entity string) (sql string, arg interface{})
// Name returns the dialect identifier (for debug logging).
Name() string
// DefaultThreshold returns the minimum entity count at which push-down
// becomes worthwhile for this backend. Below this count, the fixed
// overhead of generating and executing backend SQL exceeds the cost
// of Go-side processing. This varies by backend: an in-process SQLite
// has near-zero call overhead, while a networked Postgres has
// connection and round-trip costs that raise the crossover point.
DefaultThreshold() int
// ScalarFunction translates an OQL scalar function name to the
// backend's equivalent SQL expression. Returns the SQL fragment
// and true, or ("", false) if the function is not supported.
// Example: ("LEN", "col") -> ("LENGTH(col)", true) on SQLite,
// ("LEN", "col") -> ("CHAR_LENGTH(col)", true) on PostgreSQL.
ScalarFunction(name string, argSQL string) (string, bool)
// CastExpression emits a CAST for the target backend.
// Example: CastExpression("price", "REAL") -> "CAST(price AS REAL)"
// on SQLite, "CAST(price AS DOUBLE PRECISION)" on PostgreSQL.
CastExpression(expr, targetType string) string
}
SQLDialect defines how to emit backend-specific SQL fragments. The generator calls dialect methods to produce the correct syntax for the target database. T-SQL arrives via tsqlparser's AST; the dialect translates it to the backend's native SQL.
OQL field types ¶
Several methods accept an oqlType string that controls how a JSON field is extracted and cast. The following tokens are defined:
"text" — extract as text; no numeric coercion.
SQLite: json_extract(data, '$.f') (returns TEXT/NULL)
PostgreSQL: (data->>'f')::text
"numeric" — extract as a number; enables numeric comparison and ordering.
SQLite: CAST(json_extract(data, '$.f') AS REAL)
PostgreSQL: (data->>'f')::numeric
"boolean" — extract as a boolean.
SQLite: json_extract(data, '$.f') (SQLite has no BOOL type;
JSON true/false are stored as 1/0)
PostgreSQL: (data->>'f')::boolean
"auto" — extract without an explicit cast; use the backend's native
return type from the JSON accessor. This is safe for SQLite
equality comparisons (json_extract returns typed values) but
must NOT be used for ordering or inequality comparisons on
PostgreSQL (where all JSON accessors return text). Prefer an
explicit type whenever the stored type is known.
type SQLGenerator ¶
type SQLGenerator struct {
// contains filtered or unexported fields
}
SQLGenerator translates pushable portions of an OQL AST into backend-specific SQL. It uses a Dialect to emit the correct syntax.
func NewSQLGenerator ¶
func NewSQLGenerator(dialect SQLDialect) *SQLGenerator
NewSQLGenerator creates a generator for the given dialect.
type SQLiteDialect ¶
type SQLiteDialect struct {
// NodesTable is the tenant-scoped blob node store table name (e.g. t0000_nodes).
// Set by the OQL executor from store.NodesTable() when the store implements
// storage.TableNamer. Defaults to "t0000_nodes" if unset.
NodesTable string
}
SQLiteDialect generates SQLite-compatible SQL using json_extract().
func (*SQLiteDialect) BaseQuery ¶
func (d *SQLiteDialect) BaseQuery(entity string) (string, interface{})
func (*SQLiteDialect) CastExpression ¶
func (d *SQLiteDialect) CastExpression(expr, targetType string) string
CastExpression emits a CAST expression for SQLite.
func (*SQLiteDialect) DefaultThreshold ¶
func (d *SQLiteDialect) DefaultThreshold() int
DefaultThreshold returns 50 for SQLite. Benchmarked crossover point: push-down is faster than Go-side even at 100 records (1.5x) because SQLite is in-process with zero network overhead. The only case where push-down is marginal is broad LIKE patterns, where the crossover is higher (~500), but a 50-record threshold captures the common case.
func (*SQLiteDialect) JSONField ¶
func (d *SQLiteDialect) JSONField(fieldPath string) string
JSONField is a deprecated shortcut. Use JSONFieldAs(fieldPath, "auto").
func (*SQLiteDialect) JSONFieldAliased ¶
func (d *SQLiteDialect) JSONFieldAliased(alias, fieldPath string) string
JSONFieldAliased is a deprecated shortcut. Use JSONFieldAliasedAs(alias, fieldPath, "auto").
func (*SQLiteDialect) JSONFieldAliasedAs ¶
func (d *SQLiteDialect) JSONFieldAliasedAs(alias, fieldPath, oqlType string) string
JSONFieldAliasedAs is JSONFieldAs for JOIN queries where the data column is qualified by a table alias.
func (*SQLiteDialect) JSONFieldAs ¶
func (d *SQLiteDialect) JSONFieldAs(fieldPath, oqlType string) string
JSONFieldAs extracts a field from the JSON data column with an explicit type cast appropriate for the requested OQL type.
SQLite mapping:
- "numeric" → CAST(json_extract(data, '$.f') AS REAL)
- "boolean" → json_extract(data, '$.f') (SQLite stores JSON booleans as integer 1/0; no separate BOOL type needed)
- "text" → json_extract(data, '$.f') (returns TEXT when stored as string)
- "auto" → json_extract(data, '$.f') (return type mirrors stored JSON type)
func (*SQLiteDialect) JSONFieldNumeric ¶
func (d *SQLiteDialect) JSONFieldNumeric(fieldPath string) string
JSONFieldNumeric is a deprecated shortcut. Use JSONFieldAs(fieldPath, "numeric").
func (*SQLiteDialect) LimitClause ¶
func (d *SQLiteDialect) LimitClause(placeholder string) string
func (*SQLiteDialect) Name ¶
func (d *SQLiteDialect) Name() string
func (*SQLiteDialect) Placeholder ¶
func (d *SQLiteDialect) Placeholder(_ int) string
func (*SQLiteDialect) ScalarFunction ¶
func (d *SQLiteDialect) ScalarFunction(name string, argSQL string) (string, bool)
ScalarFunction translates OQL scalar function names to SQLite equivalents. Returns ("", false) for functions that have no clean SQLite translation.
type ScalarFunc ¶
type ScalarFunc func(args []interface{}) interface{}
ScalarFunc is a function that operates on a single value and returns a result.
type SchemaValidator ¶
type SchemaValidator interface {
Validate(entity string, data map[string]interface{}) (bool, []string)
}
SchemaValidator validates entity data against schemas
type Validator ¶
type Validator struct {
// contains filtered or unexported fields
}
Validator validates OQL queries against schema
func NewValidator ¶
NewValidator creates a new validator that checks the filesystem
func NewValidatorWithStore ¶
func NewValidatorWithStore(schemaDir string, checker EntityChecker) *Validator
NewValidatorWithStore creates a validator that checks the store for entities
func (*Validator) EntityExists ¶
EntityExists checks if an entity exists. If the entity is not in the cache, it automatically refreshes before returning false. This ensures newly created entity types are recognised without requiring manual refresh.
func (*Validator) RefreshEntities ¶
func (v *Validator) RefreshEntities()
RefreshEntities reloads the entity list. If a store checker is configured, it queries the store. Otherwise, it scans the filesystem.