Documentation
¶
Overview ¶
Package dbrpc is the client/server DB: it serves an embedded ClassAd log (package db) over a message transport (CEDAR in production), with out-of-order request multiplexing so a slow call never head-of-line-blocks others on the same connection. See DESIGN.md.
Index ¶
- Constants
- Variables
- func ClientSessionAttrNames() []string
- func DecodeWatchAd(w []byte) (*classad.ClassAd, error)
- func OpName(op uint8) string
- func RequestPanics() int64
- func ServerSessionAttrs() map[string]string
- type AdKV
- type AdReject
- type AggFunc
- type AggRow
- type AggSpec
- type AttrTypeReport
- type CallStats
- type Catalog
- type Client
- func (c *Client) Admin(ctx context.Context, action string, args ...string) (string, error)
- func (c *Client) AdminTable(ctx context.Context, table, action string, args ...string) (string, error)
- func (c *Client) Aggregate(ctx context.Context, constraint string, groupBy []string, aggs []AggSpec) ([]AggRow, error)
- func (c *Client) AggregateBucketed(ctx context.Context, constraint string, groups []GroupCol, aggs []AggSpec) ([]AggRow, error)
- func (c *Client) AggregateBucketedTable(ctx context.Context, table, constraint string, groups []GroupCol, ...) ([]AggRow, error)
- func (c *Client) AggregateTable(ctx context.Context, table, constraint string, groupBy []string, ...) ([]AggRow, error)
- func (c *Client) ArchiveAggregate(ctx context.Context, name, constraint string, groupBy []string, aggs []AggSpec) ([]AggRow, error)
- func (c *Client) ArchiveAggregateBucketed(ctx context.Context, name, constraint string, groups []GroupCol, ...) ([]AggRow, error)
- func (c *Client) ArchiveAggregateStats(ctx context.Context, name, constraint string, groups []GroupCol, ...) ([]AggRow, error)
- func (c *Client) ArchiveAppend(ctx context.Context, name, adText string) error
- func (c *Client) ArchiveQuery(ctx context.Context, name, constraint string, limit int) ([]string, error)
- func (c *Client) ArchiveRotate(ctx context.Context, name string) (int, error)
- func (c *Client) ArchiveTables(ctx context.Context) ([]string, error)
- func (c *Client) AttrTypeProfiles(ctx context.Context, table string, sampleMax int) (*AttrTypeReport, error)
- func (c *Client) AttrTypeProfilesEvaluated(ctx context.Context, table string, sampleMax int) (*AttrTypeReport, error)
- func (c *Client) BackupKeyTable(ctx context.Context, table string) ([]byte, error)
- func (c *Client) Begin(ctx context.Context) (*Tx, error)
- func (c *Client) BeginTable(ctx context.Context, table string) (*Tx, error)
- func (c *Client) Close() error
- func (c *Client) ConvertTableToMemory(ctx context.Context, name string) error
- func (c *Client) CreateArchiveTable(ctx context.Context, name string, cfg db.ArchiveConfig) error
- func (c *Client) CreateExporter(ctx context.Context, def db.ExporterDef) error
- func (c *Client) CreateTable(ctx context.Context, name string) error
- func (c *Client) CreateTableInMemory(ctx context.Context, name string) error
- func (c *Client) CreateView(ctx context.Context, name string, spec db.ViewSpec) error
- func (c *Client) DeleteWhere(ctx context.Context, constraint string) (int, error)
- func (c *Client) DeleteWhereTable(ctx context.Context, table, constraint string) (int, error)
- func (c *Client) Diagnostics(ctx context.Context) (*Diagnostics, error)
- func (c *Client) DiagnosticsBasic(ctx context.Context) (*Diagnostics, error)
- func (c *Client) DiagnosticsBasicTable(ctx context.Context, table string) (*Diagnostics, error)
- func (c *Client) DiagnosticsTable(ctx context.Context, table string) (*Diagnostics, error)
- func (c *Client) DropExporter(ctx context.Context, name string) error
- func (c *Client) DropTable(ctx context.Context, name string) error
- func (c *Client) DropView(ctx context.Context, name string) error
- func (c *Client) Explain(ctx context.Context, constraint string) (*db.QueryExplain, error)
- func (c *Client) ExplainJoin(ctx context.Context, jr JoinRequest) (*db.JoinExplain, error)
- func (c *Client) ExplainTable(ctx context.Context, table, constraint string) (*db.QueryExplain, error)
- func (c *Client) GetExporter(ctx context.Context, name string) (db.ExporterDef, bool, error)
- func (c *Client) GetExporterState(ctx context.Context, name string) ([]byte, bool, error)
- func (c *Client) InsertRows(ctx context.Context, target, source, constraint string, attrs, names []string) (int, error)
- func (c *Client) InsertSelect(ctx context.Context, target, source, constraint string, groups []db.GroupCol, ...) (int, error)
- func (c *Client) Join(ctx context.Context, jr JoinRequest, yield func(row string) bool) error
- func (c *Client) JoinAggregate(ctx context.Context, jr JoinAggRequest) ([]db.AggRow, error)
- func (c *Client) JoinAggregateStats(ctx context.Context, jr JoinAggRequest, rep *JoinStatsReport) ([]db.AggRow, error)
- func (c *Client) JoinRows(ctx context.Context, jr JoinRequest) ([]string, error)
- func (c *Client) JoinStats(ctx context.Context, jr JoinRequest, yield func(row string) bool, ...) error
- func (c *Client) JoinWire(ctx context.Context, jr JoinRequest, redact bool, yield func(row []byte) bool) error
- func (c *Client) JoinWireStats(ctx context.Context, jr JoinRequest, redact bool, yield func(row []byte) bool, ...) error
- func (c *Client) ListExporters(ctx context.Context) ([]ExporterInfo, error)
- func (c *Client) ListViews(ctx context.Context) ([]string, error)
- func (c *Client) MatchExplain(ctx context.Context, reqTable, jobSelector, resTable, targetWhere string) (*db.MatchExplain, error)
- func (c *Client) MatchSorted(ctx context.Context, jobText string, limit int) ([]string, error)
- func (c *Client) MatchSortedTable(ctx context.Context, table, jobText string, limit int) ([]string, error)
- func (c *Client) MatchTables(ctx context.Context, reqTable, resTable, keyAttr, reqWhere, targetWhere string, ...) ([]MatchRow, error)
- func (c *Client) Ordered(ctx context.Context, index int, partition string) ([]OrderedRow, error)
- func (c *Client) OrderedTable(ctx context.Context, table string, index int, partition string) ([]OrderedRow, error)
- func (c *Client) PeerProtocol() int
- func (c *Client) PrimaryKey(ctx context.Context, table string) ([]string, error)
- func (c *Client) PutExporterState(ctx context.Context, name string, state []byte) error
- func (c *Client) Query(ctx context.Context, constraint string) ([]string, error)
- func (c *Client) QueryAsOfTable(ctx context.Context, table, constraint string, limit int, asOf time.Time) ([]string, error)
- func (c *Client) QueryKeys(ctx context.Context, constraint string, yield func(key string) bool) error
- func (c *Client) QueryKeysTable(ctx context.Context, table, constraint string) ([]string, error)
- func (c *Client) QueryKeysTableStream(ctx context.Context, table, constraint string, yield func(key string) bool) error
- func (c *Client) QueryLimit(ctx context.Context, constraint string, limit int) ([]string, error)
- func (c *Client) QueryRaw(ctx context.Context, constraint string) ([]string, error)
- func (c *Client) QueryRawProject(ctx context.Context, table, constraint string, attrs []string, limit int) ([]string, error)
- func (c *Client) QueryRawProjectRefs(ctx context.Context, table, constraint string, attrs []string, limit int) ([]string, error)
- func (c *Client) QueryRawProjectRefsStream(ctx context.Context, table, constraint string, attrs []string, limit int, ...) error
- func (c *Client) QueryRawProjectRefsStreamStats(ctx context.Context, table, constraint string, attrs []string, limit int, ...) error
- func (c *Client) QueryRawProjectStream(ctx context.Context, table, constraint string, attrs []string, limit int, ...) error
- func (c *Client) QueryRawProjectedFromSeq(ctx context.Context, table, constraint string, attrs []string, after SeqCursor, ...) ([]string, *SeqPage, error)
- func (c *Client) QueryRawProjectedFromSeqStream(ctx context.Context, table, constraint string, attrs []string, after SeqCursor, ...) (*SeqPage, error)
- func (c *Client) QueryRawTable(ctx context.Context, table, constraint string, limit int) ([]string, error)
- func (c *Client) QueryRawTableStream(ctx context.Context, table, constraint string, limit int, ...) error
- func (c *Client) QueryRawWireRefsStream(ctx context.Context, table, constraint string, attrs []string, limit int, ...) error
- func (c *Client) QueryRawWireStream(ctx context.Context, table, constraint string, attrs []string, limit int, ...) error
- func (c *Client) QueryStats(ctx context.Context, table string) (*QueryStatsReport, error)
- func (c *Client) QueryTable(ctx context.Context, table, constraint string, limit int) ([]string, error)
- func (c *Client) QueryTableStream(ctx context.Context, table, constraint string, limit int, ...) error
- func (c *Client) QueryValues(ctx context.Context, table, constraint string, attrs []string, limit int, ...) ([]ValueBatch, error)
- func (c *Client) QueryValuesStream(ctx context.Context, table, constraint string, attrs []string, limit int, ...) error
- func (c *Client) QueryValuesStreamOpts(ctx context.Context, table, constraint string, attrs []string, limit int, ...) error
- func (c *Client) ResetQueryStats(ctx context.Context, table string) (*QueryStatsReport, error)
- func (c *Client) Restore(ctx context.Context, r io.Reader) error
- func (c *Client) RestoreTable(ctx context.Context, table string, r io.Reader) error
- func (c *Client) SetEncryptedAttrs(ctx context.Context, table string, attrs ...string) (string, error)
- func (c *Client) SetPrimaryKey(ctx context.Context, table string, cols []string) error
- func (c *Client) Snapshot(ctx context.Context, w io.Writer) error
- func (c *Client) SnapshotTable(ctx context.Context, table string, w io.Writer) error
- func (c *Client) Supports(f Feature) Support
- func (c *Client) SupportsNewer(f Feature) bool
- func (c *Client) Tables(ctx context.Context) ([]string, error)
- func (c *Client) TopK(ctx context.Context, table, constraint string, attrs []string, ...) ([]string, error)
- func (c *Client) TopKRowsPerGroup(ctx context.Context, constraint string, spec TopKRowsSpec) ([]TopKWireGroup, error)
- func (c *Client) TopKRowsPerGroupTable(ctx context.Context, table, constraint string, spec TopKRowsSpec) ([]TopKWireGroup, error)
- func (c *Client) TopKStats(ctx context.Context, table, constraint string, attrs []string, ...) ([]string, error)
- func (c *Client) TruncateTable(ctx context.Context, table string) (string, error)
- func (c *Client) Watch(ctx context.Context, cursor []byte) (<-chan WatchEvent, func(), error)
- func (c *Client) WatchHead(ctx context.Context, table string) ([]byte, error)
- func (c *Client) WatchTable(ctx context.Context, table string, cursor []byte) (<-chan WatchEvent, func(), error)
- func (c *Client) WatchWire(ctx context.Context, cursor []byte) (<-chan WireWatchEvent, func(), error)
- func (c *Client) WatchWireTable(ctx context.Context, table string, cursor []byte) (<-chan WireWatchEvent, func(), error)
- type ClientOption
- type Diagnostics
- type ExporterInfo
- type Feature
- type GroupCol
- type JoinAggRequest
- type JoinRequest
- type JoinStatsReport
- type LocalSession
- type MatchRow
- type MsgConn
- type OrderedRow
- type Pool
- func (p *Pool) Begin(ctx context.Context) (*Tx, error)
- func (p *Pool) Close() error
- func (p *Pool) MatchSorted(ctx context.Context, jobText string, limit int) ([]string, error)
- func (p *Pool) Ordered(ctx context.Context, index int, partition string) ([]OrderedRow, error)
- func (p *Pool) Query(ctx context.Context, constraint string) ([]string, error)
- func (p *Pool) Watch(ctx context.Context, cursor []byte) (<-chan WatchEvent, func(), error)
- type ProposeFunc
- type QueryExemplar
- type QueryLog
- type QueryShapeStat
- type QueryStatsOptions
- type QueryStatsReport
- type SeqCursor
- type SeqPage
- type ServeOptions
- type Server
- func (s *Server) Close()
- func (s *Server) DisableQueryStats()
- func (s *Server) EnableQueryStats(opts QueryStatsOptions)
- func (s *Server) LocalSession(opts ServeOptions) *LocalSession
- func (s *Server) ServeConn(conn MsgConn) error
- func (s *Server) ServeConnOpts(conn MsgConn, opts ServeOptions) error
- func (s *Server) SessionAttrs() map[string]string
- func (s *Server) SetProposeHook(fn ProposeFunc)
- func (s *Server) StartIdemReaper(interval, maxAge time.Duration) (stop func())
- func (s *Server) StartMaintenance(interval time.Duration, opts db.MaintainOptions)
- func (s *Server) StartTxnReaper(interval, maxIdle time.Duration) (stop func())
- type ServerError
- type Support
- type TopKOrderTerm
- type TopKRowsSpec
- type TopKWireGroup
- type Tx
- func (t *Tx) Abort(ctx context.Context) error
- func (t *Tx) Commit(ctx context.Context) error
- func (t *Tx) CommitIdempotent(ctx context.Context, idemKey string) error
- func (t *Tx) DeleteAttribute(ctx context.Context, key, name string) error
- func (t *Tx) DestroyClassAd(ctx context.Context, key string) error
- func (t *Tx) KeysWhere(ctx context.Context, constraint string) ([]string, error)
- func (t *Tx) LookupAttr(ctx context.Context, key, name string) (string, bool, error)
- func (t *Tx) LookupClassAd(ctx context.Context, key string) (string, bool, error)
- func (t *Tx) NewClassAd(ctx context.Context, key, adText string) error
- func (t *Tx) NewClassAdBatch(ctx context.Context, items []AdKV) ([]AdReject, error)
- func (t *Tx) NewClassAdBatchPipelined(ctx context.Context, items []AdKV, chunkSize int) ([]AdReject, error)
- func (t *Tx) Query(ctx context.Context, constraint string, limit int) ([]string, error)
- func (t *Tx) QueryStream(ctx context.Context, constraint string, limit int, yield func(row string) bool) error
- func (t *Tx) SetAttribute(ctx context.Context, key, name, expr string) error
- type ValueBatch
- type ValueColumn
- type ValueException
- type ValueKind
- type ValueStreamOptions
- type WatchEvent
- type WireWatchEvent
- type WriteKind
- type WriteOp
Constants ¶
const ( AggCount = db.AggCount // COUNT(*) or COUNT(col) AggSum = db.AggSum AggAvg = db.AggAvg AggMin = db.AggMin AggMax = db.AggMax // AggCountDistinct is COUNT(DISTINCT col). It rides the extended opcodes (see // anyFiltered), since an older server would not recognize the function. AggCountDistinct = db.AggCountDistinct // AggApproxCountDistinct is COUNT(DISTINCT col) answered from a per-group HyperLogLog: // an ESTIMATE, within db.ApproxDistinctRelErr of the count as a standard deviation. It // rides the extended opcodes for the same reason AggCountDistinct does, so a server that // does not know it refuses rather than answering the column with something else. AggApproxCountDistinct = db.AggApproxCountDistinct )
const ( SessionAttrProtocol = "DBProtocolVersion" SessionAttrFeatures = "DBProtocolFeatures" )
Session attribute names carried in the CEDAR post-authentication ad. The application wires these through: a server publishes ServerSessionAttrs() as its cedar SecurityConfig.SessionAttrs, and a client asks for ClientSessionAttrNames() in WantSessionAttrs, then hands what it captured to NewClient via WithPeerSessionAttrs.
They are strings rather than integers because the handshake ad is read by peers of several vintages, and a string reads back identically everywhere.
const DefaultTable = "ads"
DefaultTable is the table name a single-DB server serves and the client targets when a table is not named -- the historical, single-collection view.
const ProtocolVersion = 10
ProtocolVersion is the dbrpc protocol this build speaks.
It is advertised for coarse checks and for operators reading a log; what a caller should branch on is a Feature, because a server may be built without one. Bump it when an opcode is added or an existing opcode's reply changes shape, and add a Feature for anything a client would otherwise probe for.
Variables ¶
var ErrArchiveAggregateUnsupported = errors.New("dbrpc: server does not support archive aggregation")
ErrArchiveAggregateUnsupported is returned by ArchiveAggregate against a server too old to implement the opcode (it rejects the request as a bad op), so a caller can fall back to client-side aggregation (ArchiveQuery + reduce locally).
var ErrArchiveTable = errors.New("dbrpc: table is an append-only archive")
ErrArchiveTable reports that the named table exists and is an append-only archive, and that the op the caller used does not serve one. It arrives as a *ServerError whose message names what to use instead, or why nothing applies. Deterministic: the call fails the same way on replay, so surface it rather than retry -- and do not read it as the table being absent, which is what "no such table" means and this does not.
var ErrBadRequest = errors.New("dbrpc: bad request")
ErrBadRequest is the server's stBadReq: the request was malformed -- which is also how it answers an opcode it does not implement, so a newer client can tell "this server is too old for this op" from a genuine failure without matching on message text.
var ErrBucketedUnsupported = errors.New("dbrpc: server does not support bucketed aggregation")
ErrBucketedUnsupported is returned by AggregateBucketed against a server too old to implement the opcode (it rejects the request), so a caller can fall back to client-side bucketing.
var ErrConnClosed = errors.New("dbrpc: connection closed")
ErrConnClosed reports that the dbrpc connection is no longer usable. It wraps the underlying transport cause, so errors.Is(err, ErrConnClosed) identifies the class while errors.Unwrap reaches the specific I/O error.
var ErrCursorExpired = errors.New("dbrpc: pagination cursor expired")
ErrCursorExpired reports that a paginated read's cursor names a snapshot the server has since reclaimed versions of -- its lease ran out before the listing finished, or the server restarted -- so the rest of that listing cannot be read. It arrives as a *ServerError before any row of the page. Replaying the same cursor fails the same way: restart the listing from the zero cursor, and surface the restart to whoever holds the page token rather than presenting the rows so far as the whole list.
var ErrEvalTypesUnsupported = errors.New("dbrpc: evaluated attribute type profiles unavailable")
ErrEvalTypesUnsupported reports that the peer cannot measure evaluated classes: it predates the option, and would answer the request with stored classes only.
It is returned rather than silently delivering that older answer, because the older answer is not a degraded version of this one. Stored classes rule an expression-valued attribute untypeable; a caller handed them in place of what it asked for would publish a schema missing exactly the columns the option exists to recover.
var ErrExtendedAggregateUnsupported = errors.New(
"dbrpc: server does not support per-aggregate FILTER or COUNT DISTINCT")
ErrExtendedAggregateUnsupported is returned when the server is too old to know the extended-aggregate opcodes -- the ones carrying a per-aggregate FILTER or a function it would not recognize, such as COUNT DISTINCT. The caller must NOT retry without them: the answer would be a different aggregate, which is wrong rather than merely slower.
var ErrFilteredAggregateUnsupported = ErrExtendedAggregateUnsupported
ErrFilteredAggregateUnsupported is the former name of ErrExtendedAggregateUnsupported, from when a filter was the only thing the extended opcodes carried. It is the same error value, so errors.Is against either still matches.
var ErrInsertRowsUnsupported = errors.New(
"dbrpc: the server does not implement a server-side INSERT ... SELECT row copy")
ErrInsertRowsUnsupported is returned against a server too old to implement the row-copy opcode. There is deliberately no fallback: see InsertRows.
var ErrInsertSelectUnsupported = errors.New(
"dbrpc: the server does not implement a server-side INSERT ... SELECT")
ErrInsertSelectUnsupported is returned against a server too old to implement the roll-up opcode. There is deliberately no fallback: see InsertSelect.
var ErrInternal = errors.New("dbrpc: internal server error")
ErrInternal reports that the server failed a request because serving it panicked -- a server bug, not a problem with the request. It arrives as a *ServerError. The server keeps serving, but the same request will most likely fail the same way, so surface it rather than retry.
It is a stErr with a tagged message, not a status of its own. A client that predates it treats any status but a row or an error as nothing in a stream and then reads the end of the stream as success, so a new status would hand those clients a silently short answer.
var ErrJoinAggUnsupported = errors.New("dbrpc: the server does not implement a server-side join aggregate")
ErrJoinAggUnsupported reports that the server cannot reduce a join, so the caller should take the join's rows (or the two sides) and reduce them itself. Four causes, one response: a server too old to know opJoinAgg, a server whose catalog has no reduce engine wired to the join, a chain sent to a server whose join is two tables, and a client that already knows from the handshake that the server lacks the feature and so sends nothing at all.
It is delivered BEFORE any group, always: a reduction is one answer, so there is no partial result it can arrive after. That is the one way this is easier than the row-streaming join, where an error can follow rows that were already yielded.
var ErrJoinBandUnsupported = errors.New("dbrpc: this server does not support a join band; " +
"run the join in process, or write the window as an ON condition")
ErrJoinBandUnsupported reports a peer that cannot carry a join band.
It is a statement about the PEER. A caller holding one can run the join in process, or write the window as an ON condition and accept the nested loop.
var ErrJoinUnsupported = errors.New("dbrpc: the server does not implement a server-side join")
ErrJoinUnsupported reports that the server does not implement the join opcode, so the caller should join the two sides itself. A server that predates opJoin rejects it as a bad request, which is what this maps -- a caller classifies with errors.Is rather than by matching prose.
var ErrJoinWireUnsupported = errors.New("dbrpc: wire-form joined rows unavailable for this request")
ErrJoinWireUnsupported reports that the server cannot stream wire-form joined rows, so the caller should use Join and take the text rows. It is delivered BEFORE any row, so a caller that has already relayed rows can treat it as a hard failure rather than a fallback.
Three causes, one response: a server too old to know opJoinWire, a catalog that cannot produce wire rows, and a redacted request over a side whose whole ad was asked for -- which has no projection to withhold from (see db.ErrJoinWireRedactNeedsProjection).
var ErrLocalUnsupported = errors.New("dbrpc: table cannot serve an in-process column scan")
ErrLocalUnsupported says this table cannot serve a column scan, so the caller falls back to the framed path rather than to a different answer.
It is a FALL-THROUGH, not a failure: the column source is an accelerator, and a table whose rows are not in columnar form -- an in-memory table, or one whose segments have not been rewritten yet -- is read perfectly well by the record scan. The framed op makes the same decision before it has written anything, which is what lets it fall through cleanly; this returns before yielding a batch for the same reason.
var ErrNamedAggregateUnsupported = errors.New(
"dbrpc: server does not support user-defined (named) aggregates")
ErrNamedAggregateUnsupported is returned when a request names a user-defined aggregate (db.RegisterAggregate, sent as db.AggUser plus db.AggSpec.UserFunc) and the server is too old to know the opcode that carries the name.
There is deliberately no fallback. A server that does not know the opcode refuses it before any row; retrying on an older opcode would drop the name and ask for a selector the server does not recognize, which it answers "undefined" rather than refusing -- a different aggregate reported as success.
var ErrPaginationUnsupported = errors.New("dbrpc: server does not support cursor pagination")
ErrPaginationUnsupported is returned when the server does not implement the paginated opcode (it rejects the request as a bad op), so a caller can fall back to an unpaginated read rather than failing.
var ErrPeerFeaturesUnknown = errors.New("dbrpc: this client was not given the peer's session attributes, so it cannot tell what the server supports")
ErrPeerFeaturesUnknown reports that this CLIENT was never told what the peer advertised, so it cannot say whether the server implements a request. It is a fact about the local client -- NewClient without WithPeerSessionAttrs -- and not a statement about the server.
Before it existed the two were the same error. A client built without the peer's session attributes has peerFeatures nil, every feature introduced after advertisement reads as SupportUnknown, and SupportsNewer turns that into a no; the caller was then told "the server does not implement a server-side join" about a server that does. Production is fine, because the transports that matter pass the session's attrs, but a test harness that forgets gets a confident wrong answer, and the sentinel gave it nothing to tell that from a genuinely old peer.
It is reported ALONGSIDE the request's own sentinel, never instead of it: errors.Is finds both, so a caller that already classifies ErrJoinUnsupported (and falls back to joining the sides itself) keeps working unchanged, and only a caller that asks the new question sees the difference.
var ErrProjectRefsUnsupported = errors.New("dbrpc: server does not support reference-chasing projection")
ErrProjectRefsUnsupported is returned by the refs-chasing projection calls against a server too old to implement the opcode. A caller falls back to QueryRawProject, whose results are correct for literal-valued attributes and undefined for expression-valued ones whose references were projected away.
var ErrRawWireRefsUnsupported = fmt.Errorf("%w: reference-chasing wire rows", ErrRawWireUnsupported)
ErrRawWireRefsUnsupported reports that the REFERENCE-CHASING wire-row stream is not available for this request. Two causes, one response, exactly as with ErrRawWireUnsupported (which it wraps, so a caller matching only the general case still matches): a server too old to know the opcode, and a table that cannot produce self-contained rows.
It wraps rather than replaces because the two have different best fallbacks. A caller that wanted the closure can retry the PLAIN wire stream for the whole ad and compute the closure itself -- which works against a server that has opQueryRawWire but not this one, and is what the mirror did before this op existed. That retry will itself return ErrRawWireUnsupported for a table that serves no wire rows at all, at which point the text stream is the only answer left.
Like ErrRawWireUnsupported it is always delivered BEFORE the first row.
var ErrRawWireUnsupported = errors.New("dbrpc: wire-form rows unavailable for this request")
ErrRawWireUnsupported reports that the wire-form row stream is not available for this request, and the caller should fall back to a text row stream (which every server and every table can serve). Two causes, one response: a server too old to know opQueryRawWire, and a table that cannot produce self-contained rows (an in-memory table -- see db.ErrRawWireUnsupported).
It is always delivered BEFORE the first row, so a caller that has already relayed rows can treat it as a hard failure rather than retrying.
var ErrServerClosed = errors.New("dbrpc: server closed")
ErrServerClosed is returned by ServeConn/ServeConnOpts when the Server has been closed: either the connection was handed to a server that was already closing, or a frame arrived on it after Close began. The server takes no new work once Close starts, so the requests it waits for cannot grow while it waits. Not retryable on this Server.
var ErrTableReadOnly = errors.New("dbrpc: table is read-only")
ErrTableReadOnly reports that the server refused to modify a table this connection may only read (ServeOptions.TableWritable) -- typically one an in-process writer owns, such as a mirror of another store. It arrives as a *ServerError and, like every ServerError, is deterministic: replaying the write fails the same way, so surface it, never retry.
var ErrTooDeep = db.ErrTooDeep
ErrTooDeep is db.ErrTooDeep, the same error value, as the client sees it: the server refused a write because a value in it nests deeper than the store can read back. It arrives as a *ServerError -- from NewClassAd and SetAttribute when the write is buffered, from Commit for a write only Commit could judge, and from ArchiveAppend -- and in an AdReject's Cause for a batch. Nothing of the refused write was applied, and replaying it fails the same way: drop it and move on. A caller tailing a log must not rewind or retry on it.
var ErrTopKRowsUnsupported = errors.New("dbrpc: server does not support top-k rows per group")
ErrTopKRowsUnsupported reports that the server does not implement opTopKRows -- it is older than the opcode. It is always delivered BEFORE any group, so a caller can fall back to fetching every matching row and ranking them itself, which is what it did before this existed. An empty answer would be indistinguishable from a query that matched nothing.
var ErrTopKUnsupported = errors.New("dbrpc: server does not support server-side TopK")
ErrTopKUnsupported is returned by TopK against a server too old to implement the opcode, so the caller falls back to fetching the match set and ordering it client-side.
var ErrTxnReadUnsupported = errors.New("dbrpc: server does not support transaction-scoped reads")
ErrTxnReadUnsupported is returned by the Tx read methods against a server too old to implement the transaction-scoped read opcodes (it rejects the request). A caller falls back to the connection-level Query/QueryKeys, which read the committed store and so do not see the transaction's own uncommitted writes.
var ErrUserAggregateUnsupported = ErrNamedAggregateUnsupported
ErrUserAggregateUnsupported is the former name of ErrNamedAggregateUnsupported, from when no opcode here had a field to put an aggregate's NAME in and the client refused to send one at all. The name now crosses, so the only reason a user-defined aggregate still fails for want of protocol is a server too old to know the opcode that carries it -- which is the closest thing to what this meant, and is the same error value, so errors.Is against either matches.
Deprecated: use ErrNamedAggregateUnsupported, which says which side is too old.
var ErrValuesUnsupported = errors.New("dbrpc: typed value batches unavailable for this request")
ErrValuesUnsupported reports that the typed value stream is not available for this request: a server too old to know the opcode, or a flag it does not understand. Delivered before any batch, so a caller that has already read one can treat it as a hard failure.
The fallback is the text projection, which every server and every table serves.
var ErrWatchWireUnsupported = errors.New("dbrpc: server does not support the wire-form watch")
ErrWatchWireUnsupported reports that the server does not implement the wire-form watch, so the caller should use WatchTable. It is returned before any event.
var ValueBatchRows = 2048
ValueBatchRows caps how many rows one batch frame carries. It bounds the server's buffer (it holds one batch before writing) and the client's, and it is a row count rather than a byte budget because the buffering here is per COLUMN: the server cannot know a batch's encoded size until it has decided each column's kind, which it can only do once the batch is complete. A string column makes a batch large, so the count is modest.
var WireBatchBudget = 64 << 10
WireBatchBudget caps one wire-row batch frame's payload bytes. It bounds the per-stream buffer on BOTH sides (each holds ~one frame), amortizes the per-frame syscall/wakeup cost, and stays well under the transport's 1MB message ceiling. Measured sensitivity (2000x21KB-ad scans over TCP loopback): 16KB->64KB gains ~6%, 64KB->256KB ~4%, beyond 256KB flat -- so the default stays memory-lean at 64KB (~128KB per active stream across both sides) and a deployment can raise it for the last few percent on whole-ad scans (projected streams are insensitive: their rows are small enough that every budget batches deeply).
Functions ¶
func ClientSessionAttrNames ¶ added in v0.33.3
func ClientSessionAttrNames() []string
ClientSessionAttrNames is what a dbrpc client should capture from the peer. Pass it as the cedar SecurityConfig.WantSessionAttrs of the dialer.
func DecodeWatchAd ¶ added in v0.25.1
DecodeWatchAd turns a wire-form watch ad into a ClassAd. It returns nil for an event carrying no ad (a delete or a reset).
func OpName ¶ added in v0.16.3
OpName maps a CallStats.Op byte to a short stable label for the write-path ops a collector times (begin/commit/batch); anything else returns "op<N>". Used to label the per-call timing metrics without exposing the op constants.
func RequestPanics ¶ added in v0.34.0
func RequestPanics() int64
RequestPanics reports how many requests this process has failed because serving them panicked. Each is a bug: the request failed instead of the daemon, and the log holds the panic and its stack.
func ServerSessionAttrs ¶ added in v0.33.3
ServerSessionAttrs is what a dbrpc BUILD implements. Pass it as the cedar SecurityConfig.SessionAttrs of the listener serving dbrpc.
Prefer (*Server).SessionAttrs when a Server exists: some features depend on what the server was given rather than on how it was built, and advertising one this server cannot serve is worse than advertising nothing -- a definite yes sends the client down a path that then fails, which is the situation advertisement exists to prevent.
Types ¶
type AdReject ¶ added in v0.13.0
AdReject reports an ad a batch writer could not apply (unparseable text), by its index in the items slice, with the server's error. The other ads were applied; the caller typically logs the rejects and carries on.
type AggFunc ¶
The aggregate spec/result types and the GROUP BY / reduce engine live in the db module (db/aggregate.go) so the mutable-table aggregate here and the archive aggregate share one implementation. These aliases keep dbrpc's public API and wire encoding unchanged.
type AggRow ¶
The aggregate spec/result types and the GROUP BY / reduce engine live in the db module (db/aggregate.go) so the mutable-table aggregate here and the archive aggregate share one implementation. These aliases keep dbrpc's public API and wire encoding unchanged.
type AggSpec ¶
The aggregate spec/result types and the GROUP BY / reduce engine live in the db module (db/aggregate.go) so the mutable-table aggregate here and the archive aggregate share one implementation. These aliases keep dbrpc's public API and wire encoding unchanged.
type AttrTypeReport ¶ added in v0.34.2
type AttrTypeReport struct {
Profiles []db.AttrTypeProfile `json:"profiles"`
Sampled int `json:"sampled"`
}
AttrTypeReport is one table's measured attribute type profile: what each attribute held across a sample, and how large that sample was.
Sampled is reported separately because every count in a Profile is relative to it -- Present against Sampled is an absence rate, which is a different question from which classes of value appeared, and a caller that has only the profiles cannot recover it.
type CallStats ¶ added in v0.16.3
type CallStats struct {
Op uint8 // request op byte (see OpName)
WriteWait time.Duration // blocked acquiring the write lock (another send ahead)
Send time.Duration // in conn.WriteMsg -- getting the request onto the wire
Wait time.Duration // from send complete to response received
}
CallStats is the per-call timing breakdown delivered to Client.Observer. The three durations partition a unary round-trip and pinpoint a stall: a large WriteWait means another sender held the single-writer lock; a large Send means conn.WriteMsg itself blocked (the socket's send buffer filled because the peer was not reading -- comms/server backpressure, not this client); a large Wait means the request left promptly but the server (or return path) was slow.
type Catalog ¶
type Catalog interface {
// Table returns the named table's DB.
Table(name string) (*db.DB, bool)
// CreateTable creates (or returns the existing) named table.
CreateTable(name string) (*db.DB, error)
// CreateTableInMemory creates (or returns the existing) named table as RAM-only.
CreateTableInMemory(name string) (*db.DB, error)
// ConvertTableToMemory drops an existing table's on-disk backing, keeping its data in
// RAM only. DAEMON-gated by the server (it changes a table's durability).
ConvertTableToMemory(name string) error
// DropTable removes the named table and its data.
DropTable(name string) error
// Tables lists the table names.
Tables() []string
// ArchiveTable / CreateArchiveTable / ArchiveTables manage append-only history
// tables. A single-table server does not support them.
ArchiveTable(name string) (*db.ArchiveTable, bool)
CreateArchiveTable(name string, cfg db.ArchiveConfig) (*db.ArchiveTable, error)
ArchiveTables() []string
// CreateView / DropView / Views manage materialized views; ViewBacking returns a view's
// in-memory backing so reads resolve a view name to its materialized rows.
CreateView(name string, spec db.ViewSpec) error
DropView(name string) error
Views() []string
ViewBacking(name string) (*db.DB, bool)
// Exporter registry: passive storage of external-sink definitions and their opaque
// resume state. The server never runs exporters; it only persists these for the
// out-of-process exporter that reads them.
CreateExporter(def db.ExporterDef) error
DropExporter(name string) error
Exporters() []db.ExporterDef
Exporter(name string) (db.ExporterDef, bool)
SaveExporterState(name string, state []byte) error
LoadExporterState(name string) ([]byte, bool, error)
}
Catalog is the set of named tables the server serves. *db.Catalog implements it; a single-DB server wraps its DB as a one-table catalog.
type Client ¶
type Client struct {
// Observer, if set before use, receives a timing breakdown for each completed unary
// call. It lets an embedder (e.g. a collector's Prometheus metrics) localize where a
// round-trip spends its time -- client-side write-lock contention, the socket write
// (TCP backpressure when the server stops reading), or the server+return wait --
// without this package depending on any metrics library. Must not block.
Observer func(CallStats)
// contains filtered or unexported fields
}
Client is one multiplexed connection to a dbrpc Server. It issues requests with monotonic ids and a single reader goroutine demuxes responses by id, so many calls (and independent transactions) proceed concurrently over the one connection with no head-of-line blocking. Safe for concurrent use. For many connections, see Pool.
func NewClient ¶
func NewClient(conn MsgConn, opts ...ClientOption) *Client
NewClient runs the mux over conn. Close (or a transport error) fails all in-flight and future calls.
func (*Client) Admin ¶
Admin runs a management action (index/hot-set) on the default table. Requires a DAEMON-authorized connection (see AdminTable).
func (*Client) AdminTable ¶
func (c *Client) AdminTable(ctx context.Context, table, action string, args ...string) (string, error)
AdminTable runs a management action on the named table; it returns the server's human-readable result. Every admin action retunes or restructures the store, so it requires DAEMON authorization -- refused on read-only and ordinary read-write connections alike.
func (*Client) Aggregate ¶
func (c *Client) Aggregate(ctx context.Context, constraint string, groupBy []string, aggs []AggSpec) ([]AggRow, error)
Aggregate runs a server-side GROUP BY: the server buckets the constraint match by the group-by column tuple in a hash map and returns one AggRow per group. With no group columns it returns a single row aggregating the whole match. The aggregation happens on the server, so only the (small) grouped result crosses the wire, not every matched ad.
A spec may name a USER-DEFINED aggregate (db.AggUser plus db.AggSpec.UserFunc). It is resolved in the SERVER's process, against what that server registered with db.RegisterAggregate -- registering it in this process registers it for queries this process answers, which is a different thing. A name the server does not know fails the query with db.ErrNoSuchAggregate; a server too old to carry a name at all refuses with ErrNamedAggregateUnsupported. Neither is ever answered as an undefined column.
func (*Client) AggregateBucketed ¶ added in v0.13.0
func (c *Client) AggregateBucketed(ctx context.Context, constraint string, groups []GroupCol, aggs []AggSpec) ([]AggRow, error)
AggregateBucketed is Aggregate with group columns that may be time-bucketed (see GroupCol), pushing the bucketing to the server so only the grouped rows cross the wire. Against a server that does not implement the opcode it returns an error that wraps ErrBucketedUnsupported.
func (*Client) AggregateBucketedTable ¶ added in v0.13.0
func (c *Client) AggregateBucketedTable(ctx context.Context, table, constraint string, groups []GroupCol, aggs []AggSpec) ([]AggRow, error)
AggregateBucketedTable is AggregateBucketed on the named table.
func (*Client) AggregateTable ¶
func (c *Client) AggregateTable(ctx context.Context, table, constraint string, groupBy []string, aggs []AggSpec) ([]AggRow, error)
AggregateTable is Aggregate on the named table.
func (*Client) ArchiveAggregate ¶ added in v0.16.8
func (c *Client) ArchiveAggregate(ctx context.Context, name, constraint string, groupBy []string, aggs []AggSpec) ([]AggRow, error)
ArchiveAggregate runs a server-side GROUP BY over the named history table: the server applies the constraint (with zone-map pruning), groups by groupBy, and reduces each group with the COUNT/SUM/AVG/MIN/MAX aggs, returning one AggRow per group. With no group columns it returns a single row over the whole match. Only the (small) grouped result crosses the wire, not every matched ad -- the point of pushing a COUNT over ~200k history rows to the server. Against a server that does not implement the opcode it returns an error wrapping ErrArchiveAggregateUnsupported so the caller can fall back to client-side aggregation.
A spec may name a user-defined aggregate the SERVER registered, on the same terms as (*Client).Aggregate -- including that the fallback above does not apply to one, because reducing the rows here would need the aggregate here.
func (*Client) ArchiveAggregateBucketed ¶ added in v0.23.1
func (c *Client) ArchiveAggregateBucketed(ctx context.Context, name, constraint string, groups []GroupCol, aggs []AggSpec) ([]AggRow, error)
ArchiveAggregateBucketed is ArchiveAggregate where a group column may carry a bucket width, flooring a numeric dimension into fixed-width buckets on the server -- "jobs per group per day" without the matched rows crossing the wire. Bucket widths and per-aggregate filters compose: both ride the same extended opcode.
Against a server that does not implement that opcode it returns an error wrapping ErrFilteredAggregateUnsupported when a filter was requested (which must not be retried without its filters) and ErrArchiveAggregateUnsupported otherwise, so the caller can fall back to client-side reduction.
func (*Client) ArchiveAggregateStats ¶ added in v0.29.6
func (c *Client) ArchiveAggregateStats(ctx context.Context, name, constraint string, groups []GroupCol, aggs []AggSpec, stats *collections.ScanStats) ([]AggRow, error)
ArchiveAggregateStats is ArchiveAggregateBucketed that also fills stats (may be nil) with the scan's work breakdown for EXPLAIN ANALYZE -- notably a large RecordsReassembled when the aggregate fell to the per-record scan (an attribute the columnar accelerator does not cover) -- and with stats.Tier, which NAMES the strategy that answered rather than leaving it to be inferred from the counts. A server too old to send a tier leaves it TierUnreported; the rest of the trailer is unaffected, because the tier is appended to it (see putScanStats). It always sends the extended frame under opArchiveAggregateStats; against a server too old for the opcode it returns an error wrapping ErrArchiveAggregateUnsupported (or ErrExtendedAggregateUnsupported when a filter was requested), so the caller falls back to a plain aggregate without a breakdown.
func (*Client) ArchiveAppend ¶ added in v0.7.0
ArchiveAppend appends an ad (old-ClassAd text) to the named history table.
func (*Client) ArchiveQuery ¶ added in v0.7.0
func (c *Client) ArchiveQuery(ctx context.Context, name, constraint string, limit int) ([]string, error)
ArchiveQuery returns up to limit (<= 0 = all) newest-first matches (old-ClassAd texts) from the named history table -- the condor_history "last K" pattern.
func (*Client) ArchiveRotate ¶ added in v0.7.0
ArchiveRotate enforces the named archive's retention policy now (using the server's clock for age-based rules), returning how many sealed segments were dropped.
func (*Client) ArchiveTables ¶ added in v0.7.0
ArchiveTables lists the history table names.
func (*Client) AttrTypeProfiles ¶ added in v0.34.2
func (c *Client) AttrTypeProfiles(ctx context.Context, table string, sampleMax int) (*AttrTypeReport, error)
AttrTypeProfiles returns what each of the table's attributes held across a sample of up to sampleMax records: the classes of value, and exact bounds for the numeric ones.
It is the measurement a caller makes before declaring a type per attribute. See collections.AttrTypeProfiles for what a profile does and does not answer -- in particular that it reports STORED classes, so an attribute held as an expression reads as an expression even though evaluating it would yield a scalar. AttrTypeProfilesEvaluated is the measurement that answers that one.
Private attributes are absent unless the connection is authorized to see them.
func (*Client) AttrTypeProfilesEvaluated ¶ added in v0.36.0
func (c *Client) AttrTypeProfilesEvaluated(ctx context.Context, table string, sampleMax int) (*AttrTypeReport, error)
AttrTypeProfilesEvaluated is AttrTypeProfiles with each profile's Evaluated counts filled in: the same cells measured by what they EVALUATE to.
That is the measurement a consumer declaring Arrow or SQL types needs, because such a consumer evaluates each cell before it encodes it. Reading the stored classes instead is how an attribute that is an integer in every ad ends up with no column at all, on the strength of being STORED as an expression.
Returns ErrEvalTypesUnsupported when the peer has said it cannot (FeatEvalTypes). The caller's fallback is AttrTypeProfiles, whose answer is a different one rather than a lesser one.
func (*Client) BackupKeyTable ¶ added in v0.7.0
BackupKeyTable retrieves the named table's backup key -- the escrow key that decrypts its encrypted snapshots independently of the pool keys. DAEMON-level. Errors if encryption is not enabled.
func (*Client) BeginTable ¶
BeginTable starts a new independent transaction on the named table.
func (*Client) ConvertTableToMemory ¶ added in v0.10.2
ConvertTableToMemory drops an existing table's on-disk backing, keeping its current contents in RAM only (they are gone after a server restart). Requires DAEMON authorization. Best run during low write activity: a write that races the conversion can be lost (the server takes a consistent snapshot but does not globally quiesce writers).
func (*Client) CreateArchiveTable ¶ added in v0.7.0
CreateArchiveTable creates (or no-ops if present) an append-only history table. cfg configures indexes / zone maps / retention on first creation.
func (*Client) CreateExporter ¶ added in v0.14.0
CreateExporter registers an external-sink exporter definition. DAEMON-only. The server stores the definition (and later its resume state) but never runs the exporter -- an out-of-process exporter reads these back and does the work.
func (*Client) CreateTable ¶
CreateTable creates (or no-ops if present) the named table.
func (*Client) CreateTableInMemory ¶ added in v0.10.2
CreateTableInMemory creates (or no-ops if present) the named table as RAM-only: its data lives only in memory and is not recovered across a server restart. On a persistent server this avoids the disk I/O of persistence for high-churn, reconstructible data.
func (*Client) CreateView ¶ added in v0.12.0
CreateView creates a materialized view named name from spec. The server materializes it synchronously and then maintains it live from the base table's change stream, so an error (e.g. the base table exceeds the view's cardinality limit) is reported here. A view is queried like a table (SELECT ... FROM <name>) but is read-only.
func (*Client) DeleteWhere ¶ added in v0.8.0
DeleteWhere removes every ad matching constraint from the default table and returns the number removed. The delete runs server-side as a single call -- the db.DeleteWhere pushdown (batched, optimistic-retry, self-healing) -- rather than a per-key query-then-delete round trip per ad. Refused on a read-only connection.
func (*Client) DeleteWhereTable ¶ added in v0.8.0
DeleteWhereTable is DeleteWhere against a named table. Express a time-based expiry sweep as the constraint (e.g. "<now> > LastHeardFrom + Lifetime").
func (*Client) Diagnostics ¶
func (c *Client) Diagnostics(ctx context.Context) (*Diagnostics, error)
Diagnostics fetches the default table's storage stats, hot set, indexes, and tuning suggestions.
func (*Client) DiagnosticsBasic ¶ added in v0.40.2
func (c *Client) DiagnosticsBasic(ctx context.Context) (*Diagnostics, error)
DiagnosticsBasic fetches the default table's basic diagnostics (see DiagnosticsBasicTable).
func (*Client) DiagnosticsBasicTable ¶ added in v0.40.2
DiagnosticsBasicTable fetches the named table's diagnostics WITHOUT the index suggestions, per-attribute index sizes and sidecar structural breakdown -- everything a storage stats report prints, and none of what costs it. On a large archive with a cold page cache that is the difference between a report that returns and one an operator gives up waiting on.
Diagnostics.Detail is false on the result, including when the peer predates the option and answered in full anyway: the fields are there to be trusted only when they were asked for.
func (*Client) DiagnosticsTable ¶
DiagnosticsTable fetches the named table's full diagnostics.
func (*Client) DropExporter ¶ added in v0.14.0
DropExporter removes an exporter's definition and resume state. DAEMON-only.
func (*Client) DropView ¶ added in v0.12.0
DropView removes a materialized view (its definition and its in-memory data).
func (*Client) ExplainJoin ¶ added in v0.38.0
func (c *Client) ExplainJoin(ctx context.Context, jr JoinRequest) (*db.JoinExplain, error)
ExplainJoin reports how the server would run this join, without running it.
It reads no records, so it is safe to ask about a join too expensive to execute. What it cannot tell you is how many distinct key values the streamed side turns out to hold -- counting those means reading that side -- so a step reported with a probe attribute is ELIGIBLE for an index probe rather than promised one. See db.JoinExplain.
A server that does not implement the opcode reports ErrJoinUnsupported, the same sentinel Join uses, so a caller classifies both the same way.
func (*Client) ExplainTable ¶
func (c *Client) ExplainTable(ctx context.Context, table, constraint string) (*db.QueryExplain, error)
ExplainTable reports how the named table would execute a constraint query.
func (*Client) GetExporter ¶ added in v0.14.0
GetExporter returns a single exporter's full definition (including Config). DAEMON-only. The bool is false when no exporter of that name exists.
func (*Client) GetExporterState ¶ added in v0.14.0
GetExporterState returns an exporter's last checkpointed resume-state blob. DAEMON-only. The bool is false when the exporter has never checkpointed (start from the beginning).
func (*Client) InsertRows ¶ added in v0.42.0
func (c *Client) InsertRows(ctx context.Context, target, source, constraint string, attrs, names []string) (int, error)
InsertRows copies rows from source into target, all on the server, and reports how many were written.
attrs and names are the projection: each source attribute and the target attribute it is written under, which is how a copy renames a column. Both empty means SELECT *, copying every attribute of each ad under its own name.
target must declare a primary key, and a projection must include it; the server refuses otherwise rather than inventing keys. The copy is bounded -- a statement matching more rows than the server will hold is refused, not truncated -- so a caller moving a large table narrows it with a constraint and copies in parts, which compose because each row's key comes from the row.
func (*Client) InsertSelect ¶ added in v0.41.0
func (c *Client) InsertSelect(ctx context.Context, target, source, constraint string, groups []db.GroupCol, aggs []db.AggSpec, names []string) (int, error)
InsertSelect runs a grouped aggregate over source and writes its rows into target, all on the server, and reports how many rows were written.
Only the COUNT crosses the wire. That is the point of serving it here rather than composing a SELECT and a batch of writes in the client: the reduction already happened on the server, and shipping its rows out only to ship them back differs from this by two network traversals of the whole result and a round trip per row.
names has one entry per group column followed by one per aggregate: the attribute each value is written under in target. They are explicit because a SQL column header is not an attribute name -- `COUNT(*)` is a legal header and cannot be an attribute -- so the caller aliases them and sends the aliases.
target must declare a primary key covering columns that names includes; the server refuses otherwise rather than inventing keys. See db/primarykey.go.
func (*Client) Join ¶ added in v0.35.0
Join streams the joined rows as old-ClassAd text, one per row, each a composite ad whose members are the two sides under their aliases.
It reports ErrJoinUnsupported WITHOUT sending anything when the handshake already says the server cannot join. FeatJoin was introduced after feature advertisement existed, so a peer that advertised nothing cannot have the opcode and a peer that advertised a list without it definitely lacks it -- in both cases the request would buy a guaranteed round-trip rejection, on exactly the mixed-version link where that costs most.
The rejection path is kept for the case advertisement cannot cover: a server that names the feature and then cannot honour it. That should not happen, since a server advertises FeatJoin only when it holds a catalog, but a definite yes that turns out wrong must still reach the caller as the same sentinel rather than as an opaque bad request.
An error can arrive AFTER rows have been yielded, which is the other point: a join that met a limit part way through has produced a partial answer, and reading that as a complete one is the failure this reports. A caller that must not emit a partial result should not commit to its output until Join returns nil.
func (*Client) JoinAggregate ¶ added in v0.38.1
JoinAggregate runs a join inside the store and returns one row per GROUP.
The groups come back in the order the server produced them, which for the reduce engine is first-seen order over the join's rows. A GROUP BY defines no order, so a caller that wants one sorts.
Each row's Group and GroupValues are both filled, as the typed single-table aggregate's are, and each row's Elided says per aggregate how many values a bounded aggregate left out. A caller that asks for no list-valued aggregate never sees a non-nil Elided.
It reports ErrJoinAggUnsupported WITHOUT sending anything when the handshake already says the server cannot serve this, and maps the server's own refusal to the same sentinel. A caller distinguishes with errors.Is rather than by matching prose.
func (*Client) JoinAggregateStats ¶ added in v0.38.2
func (c *Client) JoinAggregateStats(ctx context.Context, jr JoinAggRequest, rep *JoinStatsReport) ([]db.AggRow, error)
JoinAggregateStats is JoinAggregate that also fills rep with how the server ran the join under the reduction. The join's statistics are the same ones the row-streaming join reports; Groups additionally says how many groups the reduction produced.
A reduction is produced whole before any of it is written, so a failure arrives before the groups and before the trailer: an error from this call means Reported is false, where the row-streaming join can report a join that failed part way.
func (*Client) JoinRows ¶ added in v0.35.0
JoinRows is Join collected into a slice, for a caller small enough not to care.
func (*Client) JoinStats ¶ added in v0.38.2
func (c *Client) JoinStats(ctx context.Context, jr JoinRequest, yield func(row string) bool, rep *JoinStatsReport) error
JoinStats is Join that also fills rep with how the server ran the join.
rep is filled whatever happens, including when nothing crossed: a caller reads Reported before Stats, and Unsupported tells it which kind of nothing it got. Stats are asked for only when the peer advertises FeatJoinStats, so this costs a silent peer nothing at all and an advertising one one frame per query -- not per row.
Everything else is Join's contract, including that an error can arrive after rows have been yielded. When one does, rep.Complete is false and rep.Stats describes the join up to the point it stopped.
func (*Client) JoinWire ¶ added in v0.37.0
func (c *Client) JoinWire(ctx context.Context, jr JoinRequest, redact bool, yield func(row []byte) bool) error
JoinWire streams the joined rows as wire-form rows -- self-contained inline-names composite ads -- batched many rows per frame.
The row slice passed to yield ALIASES THE FRAME BUFFER and is valid only until yield returns. A caller that keeps one must copy it. That is the same contract QueryRawWireStream has, and it is what lets a steady stream allocate nothing.
redact withholds each side's private attributes. An unprivileged connection is always redacted, whatever is asked for.
It returns ErrJoinWireUnsupported, before any row, when the stream is unavailable. An empty stream means the join produced no rows; it never means the server could not answer.
func (*Client) JoinWireStats ¶ added in v0.38.2
func (c *Client) JoinWireStats(ctx context.Context, jr JoinRequest, redact bool, yield func(row []byte) bool, rep *JoinStatsReport) error
JoinWireStats is JoinWire that also fills rep. See JoinStats for what the report says and what each of its answers means.
func (*Client) ListExporters ¶ added in v0.14.0
func (c *Client) ListExporters(ctx context.Context) ([]ExporterInfo, error)
ListExporters returns the registered exporters' names and kinds (no config).
func (*Client) MatchExplain ¶
func (c *Client) MatchExplain(ctx context.Context, reqTable, jobSelector, resTable, targetWhere string) (*db.MatchExplain, error)
MatchExplain reports how matchmaking the first request in reqTable matching jobSelector against resTable would execute: the job's Requirements rewritten over the slot (job constants baked in) and which resulting probes prune via a resource index. jobSelector is a constraint (e.g. `Key == "1.0"`) identifying the request.
func (*Client) MatchSorted ¶
MatchSorted returns job's matches in the default table, ranked best-first, at most limit (<=0 = all).
func (*Client) MatchSortedTable ¶
func (c *Client) MatchSortedTable(ctx context.Context, table, jobText string, limit int) ([]string, error)
MatchSortedTable returns job's ranked matches in the named table.
func (*Client) MatchTables ¶
func (c *Client) MatchTables(ctx context.Context, reqTable, resTable, keyAttr, reqWhere, targetWhere string, limit int, significantAttrs []string) ([]MatchRow, error)
MatchTables performs cross-table matchmaking as a greedy assignment: it walks the requests in reqTable matching reqWhere (in table order) and gives each one the single best-ranked resource in resTable — bilaterally matching, passing the resource-side filter targetWhere, and not already assigned to an earlier request — then removes that resource from the pool. limit bounds the number of *requests* assigned (the first `limit` in order), not resources per request. The result is one row per assigned request (Resource is "" when it could not be placed). keyAttr is the ad attribute holding each row's key.
significantAttrs, if non-empty, enables autoclustering: requests whose significant attributes are textually identical share one ranked-candidate computation (the matchmaking runs once per distinct signature and is reused; assignment still consumes resources per request), mirroring HTCondor's significant-attribute autoclusters. It must list every attribute the match depends on (the request's Requirements and Rank, and the request attributes the resources' Requirements reference).
func (*Client) Ordered ¶
Ordered streams one partition of the index-th configured ordered index in sort order (the negotiator resource-request path). One-shot: the whole partition is returned (the server-side resume cursor is not carried over the wire).
func (*Client) OrderedTable ¶
func (c *Client) OrderedTable(ctx context.Context, table string, index int, partition string) ([]OrderedRow, error)
OrderedTable streams one partition of an ordered index in the named table.
func (*Client) PeerProtocol ¶ added in v0.33.3
PeerProtocol is the protocol version the server advertised, or 0 if it advertised none. Prefer Supports for deciding what to send.
func (*Client) PrimaryKey ¶ added in v0.41.0
PrimaryKey returns the attributes whose values derive table's storage keys, or nil if the table declares none.
func (*Client) PutExporterState ¶ added in v0.14.0
PutExporterState durably stores an exporter's opaque resume-state blob. DAEMON-only. The exporter calls this after downstream has accepted its data (the at-least-once boundary).
func (*Client) Query ¶
Query returns the committed ads (old-ClassAd texts) in the default table ("ads") matching a constraint. The server streams results; a slow scan does not block other calls.
func (*Client) QueryAsOfTable ¶ added in v0.12.0
func (c *Client) QueryAsOfTable(ctx context.Context, table, constraint string, limit int, asOf time.Time) ([]string, error)
QueryAsOfTable is QueryTable for a point-in-time ("AS OF") query: it returns the ads in the named table matching constraint as they were at asOf. The server errors if time travel is not enabled on the table or asOf is older than the retained window.
func (*Client) QueryKeys ¶ added in v0.20.4
func (c *Client) QueryKeys(ctx context.Context, constraint string, yield func(key string) bool) error
QueryKeys streams the storage keys of the rows matching constraint on the default table.
func (*Client) QueryKeysTable ¶ added in v0.20.4
QueryKeysTable collects all matching storage keys into a slice (streamed under the hood, so the server never buffers the whole set). Suitable for the bounded match sets of an UPDATE/DELETE.
func (*Client) QueryKeysTableStream ¶ added in v0.20.4
func (c *Client) QueryKeysTableStream(ctx context.Context, table, constraint string, yield func(key string) bool) error
QueryKeysTableStream streams the storage KEYS of the rows matching constraint on the named table (not their ads), so a caller can address matched rows for UPDATE/DELETE by their real db key regardless of whether the ad carries a self-reported key attribute. Read-only.
func (*Client) QueryLimit ¶
QueryLimit is Query with a row cap (<= 0 = all) on the default table.
func (*Client) QueryRaw ¶ added in v0.8.0
QueryRaw is Query but returns each ad as old-ClassAd wire text (the AST-free relay form for the caller): the server renders straight from the stored representation via the db QueryRaw pushdown, and the caller can forward the text without building a ClassAd. Private attributes are stripped unless the connection is privileged.
func (*Client) QueryRawProject ¶ added in v0.10.0
func (c *Client) QueryRawProject(ctx context.Context, table, constraint string, attrs []string, limit int) ([]string, error)
QueryRawProject is QueryRawTable with a server-side projection: each returned ad carries only the attributes in attrs (matched case-insensitively) plus MyType/TargetType, so a query for a few attributes does not pull every attribute of every ad across the wire. An empty attrs behaves like QueryRawTable (all attributes). Private attributes are stripped unless the connection is privileged.
func (*Client) QueryRawProjectRefs ¶ added in v0.24.0
func (c *Client) QueryRawProjectRefs(ctx context.Context, table, constraint string, attrs []string, limit int) ([]string, error)
QueryRawProjectRefs is QueryRawProject whose projection also carries the attributes the projected expressions reference, transitively, so each returned ad EVALUATES self-contained.
Use this whenever the caller is going to evaluate what it receives. QueryRawProject returns exactly the requested attributes -- correct for a relay reproducing HTCondor's query protocol, and wrong for an evaluator, because an attribute holding an expression over its siblings (Requirements, Rank, ...) loses those siblings and evaluates to undefined.
Against a server without the opcode it returns ErrProjectRefsUnsupported.
func (*Client) QueryRawProjectRefsStream ¶ added in v0.24.0
func (c *Client) QueryRawProjectRefsStream(ctx context.Context, table, constraint string, attrs []string, limit int, yield func(row string) bool) error
QueryRawProjectRefsStream is QueryRawProjectRefs with the streaming delivery of QueryRawTableStream.
func (*Client) QueryRawProjectRefsStreamStats ¶ added in v0.29.1
func (c *Client) QueryRawProjectRefsStreamStats(ctx context.Context, table, constraint string, attrs []string, limit int, yield func(row string) bool, stats *collections.ScanStats) error
QueryRawProjectRefsStreamStats is QueryRawProjectRefsStream that also fills stats (may be nil) with the server's per-scan work breakdown, for EXPLAIN ANALYZE. Against a server too old to implement the opcode it returns ErrProjectRefsUnsupported with stats untouched, so the caller falls back to the plain stream and simply reports no server-side breakdown.
func (*Client) QueryRawProjectStream ¶ added in v0.13.0
func (c *Client) QueryRawProjectStream(ctx context.Context, table, constraint string, attrs []string, limit int, yield func(row string) bool) error
QueryRawProjectStream is QueryRawProject (server-side projection) with the streaming delivery of QueryRawTableStream.
func (*Client) QueryRawProjectedFromSeq ¶ added in v0.29.5
func (c *Client) QueryRawProjectedFromSeq(ctx context.Context, table, constraint string, attrs []string, after SeqCursor, limit int) ([]string, *SeqPage, error)
QueryRawProjectedFromSeq is QueryRawProjectedFromSeqStream collected into a slice, for a caller that wants the page in hand.
func (*Client) QueryRawProjectedFromSeqStream ¶ added in v0.29.5
func (c *Client) QueryRawProjectedFromSeqStream(ctx context.Context, table, constraint string, attrs []string, after SeqCursor, limit int, yield func(row string) bool) (*SeqPage, error)
QueryRawProjectedFromSeqStream is QueryRawProjectedStream a page at a time: at most limit rows after the cursor, in the store's commit-sequence order, plus the cursor to continue from.
Pagination lives on the server's own ordering, so a page costs a resume rather than a re-scan, and the cursor names no physical position, so compaction moving records does not disturb it. Hand the returned page's Next back unchanged for the following page; the zero cursor starts at the beginning.
A cursor's snapshot is kept readable for the server's lease period (collections.Options.SeqCursorLease). A cursor resumed after the server has reclaimed versions its snapshot sees fails with ErrCursorExpired before any row: restart the listing from the zero cursor.
func (*Client) QueryRawTable ¶ added in v0.8.0
func (c *Client) QueryRawTable(ctx context.Context, table, constraint string, limit int) ([]string, error)
QueryRawTable is QueryRaw against a named table with an optional limit.
func (*Client) QueryRawTableStream ¶ added in v0.13.0
func (c *Client) QueryRawTableStream(ctx context.Context, table, constraint string, limit int, yield func(row string) bool) error
QueryRawTableStream is QueryRawTable that hands each matching ad's old-ClassAd wire text to yield as it arrives, instead of collecting the whole result into a slice -- so a relay (e.g. the collector) can forward each ad to its own client without buffering the entire result set. yield returns false to stop early. See streamEach for the error contract (a failure can arrive after some rows have been yielded).
func (*Client) QueryRawWireRefsStream ¶ added in v0.33.2
func (c *Client) QueryRawWireRefsStream(ctx context.Context, table, constraint string, attrs []string, limit int, redact bool, yield func(row []byte) bool) error
QueryRawWireRefsStream is QueryRawWireStream whose projection also carries the attributes the projected expressions reference, transitively, so each row EVALUATES self-contained -- the wire-row form of QueryRawProjectRefsStream, returning the same attributes it does.
Use it whenever the rows will be evaluated rather than relayed verbatim. The closure is computed at the store, per record, which is the only place it can be: whether a projected attribute holds an expression is a property of the stored ad, so a client cannot decide it for the query and cannot discover it from rows it has already been sent.
An empty attrs is a whole-ad request, which has no closure to compute and behaves exactly like QueryRawWireStream. Against a server without the opcode it returns ErrRawWireRefsUnsupported before any row; see there for the fallback.
func (*Client) QueryRawWireStream ¶ added in v0.16.4
func (c *Client) QueryRawWireStream(ctx context.Context, table, constraint string, attrs []string, limit int, redact bool, yield func(row []byte) bool) error
QueryRawWireStream streams matching ads as wire-form rows (self-contained inline-names subset ads -- render them with collections.RenderRawAdInline, or decode them with wire.DecodeInline), batched many rows per frame (WireBatchBudget on the server side). The row slice passed to yield aliases the frame buffer and is valid only until yield returns; redact requests source-side private-attribute stripping even on a privileged connection.
It returns ErrRawWireUnsupported, before any row, when the stream is unavailable -- an older server, or a table that cannot serve wire rows. Callers fall back to the text row stream. An empty stream means the query matched nothing; it never means the table could not answer.
func (*Client) QueryStats ¶ added in v0.32.0
QueryStats fetches the server's aggregated query report: every query it has served bucketed by shape and ranked by total time, each bucket carrying the slowest actual instances it saw. table filters to one table ("" = every table).
Exemplars (the real constraint text, the requesting identity) come back only on a DAEMON-authorized connection; QueryStatsReport.Exemplars says whether they did. A server that is not aggregating answers with Enabled false rather than an error.
func (*Client) QueryTable ¶
func (c *Client) QueryTable(ctx context.Context, table, constraint string, limit int) ([]string, error)
QueryTable returns the committed ads in the named table matching constraint, stopping after at most limit matches (<= 0 = all). The limit is pushed to the server, which stops the scan early.
func (*Client) QueryTableStream ¶ added in v0.13.0
func (c *Client) QueryTableStream(ctx context.Context, table, constraint string, limit int, yield func(row string) bool) error
QueryTableStream is QueryTable that hands each matching ad's text to yield as it arrives instead of collecting the whole result first (see streamEach). yield returns false to stop early.
func (*Client) QueryValues ¶ added in v0.36.0
func (c *Client) QueryValues(ctx context.Context, table, constraint string, attrs []string, limit int, redact bool) ([]ValueBatch, error)
QueryValues collects QueryValuesStream into whole batches the caller owns, for a caller that does not want to think about buffer lifetime. It copies, which is the cost of owning them.
func (*Client) QueryValuesStream ¶ added in v0.36.0
func (c *Client) QueryValuesStream(ctx context.Context, table, constraint string, attrs []string, limit int, redact bool, yield func(*ValueBatch) bool) error
QueryValuesStream streams matching records as typed, EVALUATED value batches, column-major and aligned with attrs. Each batch is handed to yield and is valid only until yield returns: its slices alias the decode buffer.
redact requests source-side redaction even on a privileged connection; an unprivileged connection is redacted whatever it asks.
Returns ErrValuesUnsupported, before any batch, when the stream is unavailable.
func (*Client) QueryValuesStreamOpts ¶ added in v0.37.0
func (c *Client) QueryValuesStreamOpts(ctx context.Context, table, constraint string, attrs []string, limit int, redact bool, opts ValueStreamOptions, yield func(*ValueBatch) bool) error
QueryValuesStreamOpts is QueryValuesStream with the opt-in parts of the stream (see ValueStreamOptions).
func (*Client) ResetQueryStats ¶ added in v0.32.0
ResetQueryStats clears the aggregate and restarts the observation window, returning the report as it stood before the reset. DAEMON-authorized.
func (*Client) RestoreTable ¶ added in v0.7.0
RestoreTable replaces the named table with the snapshot read from r. DAEMON-level and destructive: the server spools the upload and restores under the DB-wide lock. The upload streams in chunks, so the client does not buffer the whole snapshot.
func (*Client) SetEncryptedAttrs ¶ added in v0.7.0
func (c *Client) SetEncryptedAttrs(ctx context.Context, table string, attrs ...string) (string, error)
SetEncryptedAttrs sets the explicit attributes encrypted at rest on the named table (private attributes are always encrypted). It is a DAEMON-level action: the server refuses it unless the connection is privileged. Passing no attributes clears the explicit set. Returns the server's human-readable result.
func (*Client) SetPrimaryKey ¶ added in v0.41.0
SetPrimaryKey declares the attributes whose values derive table's storage keys. An empty cols clears the declaration.
It does not rewrite existing rows; see db.DB.SetPrimaryKey for why a declaration governs later writes rather than reorganising what is already stored.
func (*Client) SnapshotTable ¶ added in v0.7.0
SnapshotTable streams a consistent backup of the named table to w. DAEMON-level: a snapshot carries every attribute, including private ones. The server streams it in chunks, so neither side buffers the whole database.
func (*Client) Supports ¶ added in v0.33.3
Supports reports whether the server can serve f.
The three-valued answer is the point: SupportUnknown is not a no. Treating it as one would refuse work that an un-advertising server does perfectly well, which is every server built before this existed.
For a feature added AFTER advertisement, that reasoning inverts and SupportsNewer says so.
func (*Client) SupportsNewer ¶ added in v0.35.0
SupportsNewer reports whether to USE f, resolving the unknown case by when f was introduced.
Supports answers what the peer said. This answers what to do about it, which differs for the two kinds of feature. For one that predates advertisement, silence is uninformative and the caller should still try -- so this reports true, preserving the existing probe-and-fall-back behaviour. For one introduced after advertisement, silence is conclusive: a peer that advertises nothing was built before advertisement, which was built before the feature, so it cannot have it. Issuing the request would buy a guaranteed round-trip rejection.
Having this as its own call rather than folding the rule into Supports keeps the distinction visible at the call site, where the question is "should I send this" and the answer depends on something the feature name alone does not carry.
func (*Client) Tables ¶
Tables lists the table names: mutable tables and append-only archives, sorted, each once. Views are not included; they have their own listing (ListViews).
A server from before archives were listed here returns the mutable tables only, so a caller that wants every name against either server unions this with ArchiveTables and DEDUPLICATES -- concatenating lists each archive twice against a current server. A caller that needs the names it may WRITE must subtract ArchiveTables rather than assume every listed name is mutable.
func (*Client) TopK ¶ added in v0.29.1
func (c *Client) TopK(ctx context.Context, table, constraint string, attrs []string, orderAttr string, desc bool, k int) ([]string, error)
TopK returns the k rows matching constraint ordered by orderAttr -- the largest k when desc, the smallest k otherwise -- each projected to attrs and rendered as old-ClassAd text, in sorted order (best first). It is the wire form of db.DB/ArchiveTable.TopK: the server holds and returns only k rows, so ORDER BY <orderAttr> {DESC|ASC} LIMIT k does not ship the whole match set to be sorted at the far end. Parse each returned row with classad.ParseOld.
orderAttr must resolve to a number; a matching row whose order value is not numeric is skipped. If orderAttr is not among attrs it is used only for the ordering and is not included in the returned rows. Against a server without the opcode it returns ErrTopKUnsupported.
func (*Client) TopKRowsPerGroup ¶ added in v0.43.0
func (c *Client) TopKRowsPerGroup(ctx context.Context, constraint string, spec TopKRowsSpec) ([]TopKWireGroup, error)
TopKRowsPerGroup asks the server for the K best rows of each group over the default table.
func (*Client) TopKRowsPerGroupTable ¶ added in v0.43.0
func (c *Client) TopKRowsPerGroupTable(ctx context.Context, table, constraint string, spec TopKRowsSpec) ([]TopKWireGroup, error)
TopKRowsPerGroupTable is TopKRowsPerGroup against a named table, history included.
Against a server without FeatTopKRows it returns ErrTopKRowsUnsupported before any group, so a caller falls back to fetching and ranking itself rather than seeing an empty answer and believing nothing matched.
func (*Client) TopKStats ¶ added in v0.29.4
func (c *Client) TopKStats(ctx context.Context, table, constraint string, attrs []string, orderAttr string, desc bool, k int, stats *collections.ScanStats) ([]string, error)
TopKStats is TopK that also fills stats (may be nil) with the cutoff scan's work breakdown, for EXPLAIN ANALYZE. Against a server too old for the opcode it returns ErrTopKUnsupported with stats untouched, so the caller falls back to TopK and reports no breakdown.
func (*Client) TruncateTable ¶ added in v0.7.0
TruncateTable removes every ad from the named table. It is a DAEMON-level action (destructive, DB-wide locked): the server refuses it unless the connection is privileged. Returns the server's human-readable result.
func (*Client) Watch ¶
Watch streams changes committed after cursor (nil = from now) on the channel, which closes when the returned stop is called, the connection fails, or the server ends the stream. A full replay leads with a reset event.
func (*Client) WatchHead ¶ added in v0.7.0
WatchHead returns an opaque cursor at the current head of table's change log, so a following WatchTable(table, cursor) streams only subsequent changes (no replay of current contents) -- the "tail from now" path.
func (*Client) WatchTable ¶
func (c *Client) WatchTable(ctx context.Context, table string, cursor []byte) (<-chan WatchEvent, func(), error)
WatchTable streams changes to the named table.
func (*Client) WatchWire ¶ added in v0.25.1
func (c *Client) WatchWire(ctx context.Context, cursor []byte) (<-chan WireWatchEvent, func(), error)
WatchWire is WatchWireTable on the default table.
func (*Client) WatchWireTable ¶ added in v0.25.1
func (c *Client) WatchWireTable(ctx context.Context, table string, cursor []byte) (<-chan WireWatchEvent, func(), error)
WatchWireTable is WatchTable with each event's ad in wire form. Against a server that does not implement the opcode it returns ErrWatchWireUnsupported, before any event, and the caller falls back to WatchTable.
type ClientOption ¶ added in v0.33.3
type ClientOption func(*Client)
A ClientOption configures a Client at construction.
func WithPeerSessionAttrs ¶ added in v0.33.3
func WithPeerSessionAttrs(attrs map[string]string) ClientOption
WithPeerSessionAttrs gives the client what the peer advertised, as captured from the CEDAR handshake under ClientSessionAttrNames. Attributes that are absent or unparseable leave the peer unknown, which is the safe reading.
type Diagnostics ¶
type Diagnostics struct {
Stats db.Stats `json:"stats"`
OpStats db.OpStats `json:"opStats"`
Hot []string `json:"hot"`
CategoricalIndexes []string `json:"categoricalIndexes"`
ValueIndexes []string `json:"valueIndexes"`
IndexSizes db.IndexSizes `json:"indexSizes"`
Codec db.CodecStats `json:"codec"`
Suggestions []db.IndexSuggestion `json:"suggestions"`
DropSuggestions []db.DropSuggestion `json:"dropSuggestions"`
// EncryptionEnabled reports whether encryption at rest is active; EncryptedAttrs is
// the explicit encrypted-attribute set (private attributes are always encrypted and
// are not listed here).
EncryptionEnabled bool `json:"encryptionEnabled"`
EncryptedAttrs []string `json:"encryptedAttrs,omitempty"`
// SchemaScan reports the per-segment columnar (adschema) accelerator's state: whether it is
// enabled (a numeric COUNT(*) WHERE takes the columnar fast path), its uncompressed hot
// columns, and how many sealed segments carry a block. Reported for BOTH table kinds --
// archives carry columnar blocks too, and reporting it for only one made an archive look
// like it had no accelerator at all.
SchemaScan db.SchemaScanInfo `json:"schemaScan"`
// GroupDrift, GroupChanges and GroupAgreement are the READ-level observability for the
// secondary (group) schemas: how the sampler's candidate groups have drifted, the log of
// changes to the COMMITTED set (each a real block-rebuild event, with its diff and reason),
// and the last per-segment agreement. All read persisted sidecar state -- no sampling -- so
// they are safe to include in the READ diagnostics, unlike the fresh candidate derivation.
GroupDrift db.GroupSchemaDrift `json:"groupDrift"`
GroupChanges []db.GroupSchemaChange `json:"groupChanges,omitempty"`
GroupAgreement *db.GroupSchemaLastAgreement `json:"groupAgreement,omitempty"`
// Archive marks an append-only history table (vs. a mutable one), and Retention carries
// its rotation bounds -- the one genuinely kind-specific pair, since a mutable table has no
// rotation. SidecarSizes reports sealed-segment sidecar index bytes, for both kinds.
Archive bool `json:"archive,omitempty"`
Retention *db.Retention `json:"retention,omitempty"`
SidecarSizes db.SidecarSizes `json:"sidecarSizes"`
// ZoneAttrs are the archive attributes carrying per-segment [min,max] zone maps: a
// range query on one of these prunes whole segments, not just postings.
ZoneAttrs []string `json:"zoneAttrs,omitempty"`
// SealedSegments is how many segments are sealed to an immutable index sidecar, and
// StaleIndexSegments how many of those were sealed under an older index configuration --
// i.e. how much of the table a runtime .addindex/.dropindex has NOT reached yet (only a
// rewrite reaches them). Both kinds.
SealedSegments int `json:"sealedSegments,omitempty"`
StaleIndexSegments int `json:"staleIndexSegments,omitempty"`
// BackfillPendingSegments and BackfillPendingBytes are the sealed archive segments (and
// their stored bytes) whose index lacks postings for some indexed attribute -- typically
// ones beyond the backfill horizon when an index was added. Queries scan them in full on
// that attribute until the maintenance backfill or an index.reindex reaches them.
BackfillPendingSegments int `json:"backfillPendingSegments,omitempty"`
BackfillPendingBytes int64 `json:"backfillPendingBytes,omitempty"`
// Detail reports whether this is a FULL read. The fields a BASIC read leaves unmeasured --
// Suggestions, DropSuggestions, IndexSizes, and the MPH/bloom halves of SidecarSizes -- are
// zero in one, which is not the same as measured-and-zero. Anything printing them must check
// this first, or it reports "no index suggestions" to an operator who was never asked.
//
// They are separated because between them they cost more than the whole of the rest of the
// report: the suggestions decode a sample of records, and the sidecar breakdown reads
// scattered header words out of every sealed sidecar, which on a large archive with a cold
// page cache is tens of thousands of random faults. A stats report prints neither.
//
// A client sets this from the read it ASKED for, not from the wire: a server predating the
// option ignores the request and answers in full, and its reply carries no flag to read.
Detail bool `json:"detail,omitempty"`
}
Diagnostics is a snapshot of the store's storage, hot set, indexes, and index tuning advice -- the payload of the diagnostic ".stats"/".indexes"/".hot" commands.
type ExporterInfo ¶ added in v0.14.0
ExporterInfo is the name+kind pair returned by ListExporters. It deliberately omits the exporter's Config, which may hold credentials and is only returned by GetExporter to a DAEMON-authorized client.
type Feature ¶ added in v0.33.3
type Feature string
A Feature is one capability a client would otherwise have to probe for. The string values are protocol, not display: they go on the wire and a released name must never change meaning.
const ( // FeatProjectRefs is the reference-chasing projection (opQueryRawProjRefs). FeatProjectRefs Feature = "projrefs" // FeatRawWire is the wire-form row stream (opQueryRawWire). FeatRawWire Feature = "rawwire" // FeatRawWireRefs is the reference-chasing wire-form row stream. FeatRawWireRefs Feature = "rawwirerefs" // FeatPagination is cursor-resumed projected reads (opQueryRawProjSeq). FeatPagination Feature = "pagination" // FeatTopK is the server-side ordered top-K read (opTopK). FeatTopK Feature = "topk" // FeatArchiveAggregate is GROUP BY over an archive table. FeatArchiveAggregate Feature = "archiveagg" // FeatBucketedAggregate is GROUP BY with server-side bucketing. FeatBucketedAggregate Feature = "bucketagg" // FeatExtendedAggregate is the extended aggregate frame (bucket widths, // filtered aggregates). FeatExtendedAggregate Feature = "extagg" // FeatWatchWire is the wire-form change feed. FeatWatchWire Feature = "watchwire" // FeatTxnRead is a transaction-scoped read. FeatTxnRead Feature = "txnread" // FeatJoin is the server-side two-table join (opJoin). // // The FIRST feature added after advertisement existed, which changes what silence // means for it. Every feature above predates advertisement, so a peer that says // nothing may still implement it and SupportUnknown is not a no. Nothing that // advertises nothing can implement this one, because advertisement shipped first -- // so for this feature, and every one added from here, unknown IS no. See // SupportsNewer. FeatJoin Feature = "join" // FeatQueryValues is the typed, evaluated value-batch stream (opQueryValues). Like // FeatJoin it postdates advertisement, so a peer that says nothing does not have it. FeatQueryValues Feature = "queryvalues" // FeatJoinWire is the wire-form, batched join row stream (opJoinWire). Postdates // advertisement, so silence is a no -- and a peer that advertises FeatJoin but not this // one is a server that can join and cannot batch, which is exactly the fallback the // client takes. FeatJoinWire Feature = "joinwire" // FeatValueDict is the DICTIONARY form of a string column in the value stream // (KindStringDict, requested with valueFlagDict). Postdates advertisement, so silence is // a no -- and it must be, because an older server REFUSES a request flag it does not // know rather than ignoring it, so asking a silent peer would turn an optimization into // an error. FeatValueDict Feature = "valuedict" // FeatJoinChain is the join over a CHAIN of tables (opJoinChain), and FeatJoinChainWire // is its wire-form reply (opJoinChainWire). Both postdate advertisement, so a peer that // says nothing does not have them -- and a peer advertising FeatJoin says nothing about // either, because a two-table join is what a build without chains has. FeatJoinChain Feature = "joinchain" FeatJoinChainWire Feature = "joinchainwire" // FeatJoinExplain is the join access-path report (opJoinExplain, opJoinChainExplain). // Postdates advertisement, so silence is a no. // // One feature for both opcodes, unlike the join itself. A server gains the two // together -- they are one handler over the two body layouts -- and the thing that // made FeatJoinChain separate does not apply: a build without chains cannot answer // either, because it has no chain planner to report on. FeatJoinExplain Feature = "joinexplain" // FeatEvalTypes is the EVALUATED half of the attribute type report (opAttrTypes with // attrFlagEvaluate). Postdates advertisement, so silence is a no. // // It is a feature rather than a probe precisely because the old reply is a valid one: // a server without it answers the request, successfully, with stored classes only. A // client that inferred support from a non-error would conclude that every // expression-valued attribute is untypeable, which is the wrong answer rather than a // missing one. FeatEvalTypes Feature = "evaltypes" // FeatNamedAggregate is the aggregate frame carrying an aggregate's NAME // (opAggregateNamed, opArchiveAggregateNamed), which is what lets a request reach a // user-defined aggregate the server has registered. Postdates advertisement, so silence // is a no -- but silence is still probed rather than refused outright, because the // probe's answer is a clean stBadReq before any row. // // What the server registered is a separate question this cannot answer: the feature // says the NAME crosses, not that any particular name resolves. A name the server does // not know fails the query with db.ErrNoSuchAggregate, which is a different error for a // different cause, and deliberately not an "undefined" column. FeatNamedAggregate Feature = "namedagg" // FeatTypedGroups is the aggregate reply whose GROUP cells are tagged VALUES rather // than the group's rendered text (opAggregateTyped, opArchiveAggregateTyped). // Postdates advertisement, so silence is a no. // // A client that does not get it is not stuck: db.ParseGroupValue reads the rendered // text back, and the client fills AggRow.GroupValues with it, so a caller sees values // from every server. The feature is what saves the round trip through text, not what // makes the values available -- which is why its absence costs accuracy nowhere and // needs no fallback path in the caller. FeatTypedGroups Feature = "typedgroups" // FeatJoinAgg is the GROUP BY over a join's rows, reduced in the store (opJoinAgg). // Postdates advertisement, so silence is a no. // // One feature for the two body layouts the opcode carries, unlike the row-streaming // join, where FeatJoinChain is separate. Nothing is lost by that: a server advertising // this has the handler, and whether it can also fold in a THIRD table is a property of // its catalog that FeatJoinChain already states. A client sending a chain asks for both. // // A peer advertising FeatJoin says nothing about this one. The two share a planner and // nothing else -- a server can stream joined rows perfectly well and have no reduce // engine wired to them -- so inferring one from the other would be the definite yes // that turns out wrong. FeatJoinAgg Feature = "joinagg" // FeatJoinBand is the join carrying ordered WINDOWS over a folded side (db.JoinBand): // the band opcodes, and the two layout values that reach the reduce opcode. Postdates // advertisement, so silence is a no. // // One feature for all four funnels, unlike the chain, which has one per funnel. The // thing that made those separate does not apply: a band is a property of the CATALOG's // join -- it either honours a window or it does not -- and no funnel can have it while // another lacks it. What a funnel can lack is the funnel, so a banded request asks for // this AND for the feature its unbanded form would have needed. // // It is the feature whose absence matters most to get right. Every other one here // degrades to a slower path or a less precise answer; a band that went missing would // pair the rows its window excludes, and a superset of the right rows is not a // recognisable failure. FeatJoinBand Feature = "joinband" // FeatTopKRows is the server-side "K best rows per group" read (opTopKRows), the // pushdown for a ranking window. It postdates advertisement, so a peer that says // nothing does not have it. FeatTopKRows Feature = "topkrows" // FeatJoinStats is the join's EXECUTION statistics, returned as a trailer on the join // opcodes when the request asks for them (db.JoinStats: which access path each step // took and why, what the planner priced the orientations at, and how much work it did). // Postdates advertisement, so silence is a no. // // It must be a feature rather than a probe, and that is the whole point of it. The // request byte is trailing and optional, so a server that does not know it answers the // query perfectly well and simply sends no trailer -- which is indistinguishable, at the // client, from a server that ran the join and used no index. Those are different facts: // one is "nobody told me", the other is "it scanned". A client that could not tell them // apart would reintroduce, across the wire, exactly the ambiguity db.JoinStats.ProbeWhy // exists to remove inside the store. // // Distinct from FeatJoinExplain, which answers how a join WOULD run without running it. // A plan and the run that followed it are different questions, and a store that can // state the first need not have been taught to report the second. FeatJoinStats Feature = "joinstats" // FeatPrimaryKey is a table's declared PRIMARY KEY: reading it, and setting it. // Postdates advertisement, so silence is a no. FeatPrimaryKey Feature = "primarykey" // FeatInsertSelect is the server-side roll-up: reduce one table, write the rows into // another, reply with a count. Postdates advertisement, so silence is a no. // // Separate from FeatPrimaryKey although the roll-up is useless without one, because the // declaration is useful on its own -- it is what makes a table's row identity explicit // for anything that writes to it -- and a build could carry it without the roll-up. FeatInsertSelect Feature = "insertselect" // FeatInsertRows is the server-side row copy: INSERT ... SELECT with no GROUP BY, which // moves rows rather than summarising them. Postdates advertisement, so silence is a no. // // Separate from FeatInsertSelect because they are separate opcodes bounding separate // risks, and a client has to know which one it may send: a server carrying the roll-up // need not carry the copy, and a client that assumed otherwise would have its statement // refused as malformed rather than as unimplemented. FeatInsertRows Feature = "insertrows" )
type GroupCol ¶ added in v0.13.0
The aggregate spec/result types and the GROUP BY / reduce engine live in the db module (db/aggregate.go) so the mutable-table aggregate here and the archive aggregate share one implementation. These aliases keep dbrpc's public API and wire encoding unchanged.
type JoinAggRequest ¶ added in v0.38.1
type JoinAggRequest struct {
Spec db.JoinAggSpec
Limits db.JoinLimits
}
JoinAggRequest is a planned join together with the reduction over its rows.
Limits is the JOIN's budget, asked for and clamped exactly as JoinRequest's is. The caps on the ANSWER -- how many groups, how long a list-valued aggregate's list -- live on the spec (db.JoinAggSpec), because they describe the shape of the reply rather than the work.
type JoinRequest ¶ added in v0.35.0
type JoinRequest struct {
Spec db.JoinSpec
Limits db.JoinLimits
}
JoinRequest is a planned two-table join to execute inside the store.
Spec carries the plan the caller already made: which conjuncts went to which side, which equalities drive the hash, and what each side contributes to the joined row. Limits asks for bounds no larger than the server's own -- the server clamps, so asking for more gets the server's.
type JoinStatsReport ¶ added in v0.38.2
type JoinStatsReport struct {
// Stats is the join's own record of what it did. It means something only when Reported
// is true; until then it is a zero value that was never written to.
Stats db.JoinStats
// Reported says the server sent its statistics and they were decoded. False means the
// statistics did not cross -- NOT that the join used no index.
//
// Three things make it false: the peer does not have the feature (Unsupported), the
// stream ended before the trailer, and a consumer that stopped yielding early, which
// abandons the rest of the reply including the trailer.
Reported bool
// Complete says the statistics describe the WHOLE join: the trailer arrived and the
// stream ran to its end without an error.
//
// A join that met a limit part way through still reports -- its access path is exactly
// what a reader wants to see -- but its counts stopped where the limit did, and
// presenting those as a finished join's would overstate what was read.
Complete bool
// Unsupported says the peer does not advertise FeatJoinStats, so nothing was asked for
// and nothing came back. It is the one cause of a missing report that is a property of
// the peer rather than of this call, and it is the answer a diagnostic should print:
// "this store does not report join statistics" is a different sentence from "the join
// was cut short before they arrived".
//
// It is never set alongside Reported. A trailer that arrives from a peer that advertised
// nothing is still a report, and the arrival is the better evidence.
Unsupported bool
// Err is the text of a failure the join met WHILE streaming, empty when it met none.
//
// Text, not an error: the typed error is what the call returns, and reconstructing a
// second copy here would be one that errors.Is cannot match -- a caller comparing it
// against a sentinel would get a wrong answer rather than no answer.
Err string
}
JoinStatsReport is what a server said about how it ran a join.
The three booleans are the point of the type. A db.JoinStats on its own cannot say that it was never filled: a zero ProbeKeys with an empty ProbeWhy is what a server that did not report anything leaves behind, and it is also what a join that narrowed on nothing and gave no reason would leave behind. A caller that reads Stats without reading Reported first is back to the ambiguity this exists to remove.
type LocalSession ¶ added in v0.37.0
type LocalSession struct {
// contains filtered or unexported fields
}
LocalSession is one in-process reader, scoped by the ServeOptions its session was opened with.
PROTOTYPE. The batches it yields are the producer's own and are only valid until the callback returns -- see collections.Batch -- which is the same contract the Arrow encoder already works to, since it copies into its builders.
func (*LocalSession) ScanColumns ¶ added in v0.37.0
func (ls *LocalSession) ScanColumns(ctx context.Context, table, constraint string, attrs []string, redact bool, yield func(*collections.Batch) bool) (err error)
ScanColumns reads the table as columns, handing each batch to yield. Returning false stops it.
redact is the caller's own request; an unprivileged session redacts whatever it asks, exactly as over the wire.
type MatchRow ¶
MatchRow is one matchmaking result: a request key, the resource it matched, and the request's Rank of that resource (Rank is "" when the request has no numeric Rank for it).
type MsgConn ¶
MsgConn is a reliable, ordered, bidirectional message transport: each WriteMsg / ReadMsg carries one discrete frame. CEDAR's stream.Stream satisfies it via an adapter (see cedaradapter); tests use a length-prefixed net.Pipe. WriteMsg may be called concurrently with ReadMsg, but not concurrently with itself -- the mux serializes writers.
WRITEMSG MUST NOT RETAIN b. It may read it and copy from it, and must be finished with it by the time it returns; the caller is free to overwrite it afterwards, and the value stream does, reusing one buffer for every batch of a scan rather than allocating a fresh frame each time. This is the ordinary io.Writer rule, written down because something now depends on it: both implementations copy (pipeConn writes it through, cedarConn hands it to CEDAR's SendMessage, which copies into the stream's own frame buffer), and so does every test transport.
func NewCedarConn ¶
NewCedarConn wraps an established CEDAR stream (from a client dial, or a server HandlerFunc's Conn.Stream) as a dbrpc transport. ctx bounds the connection's I/O.
func NewStreamConn ¶
func NewStreamConn(rw io.ReadWriteCloser) MsgConn
NewStreamConn frames messages over an ordered byte stream. Exposed so a caller can run dbrpc over any io.ReadWriteCloser (e.g. a plain TCP conn) without CEDAR.
type OrderedRow ¶
OrderedRow is one ad from an ordered scan (old-ClassAd text) with its cluster signature -- run-length-fold equal signatures into a resource-request list.
type Pool ¶
type Pool struct {
// contains filtered or unexported fields
}
Pool is a set of multiplexed connections to one Server, load-balanced round-robin. It borrows the database/sql idea of a connection pool, but adapts it: an SQL driver needs a connection per in-flight query (connections do not mux), so its pool serializes checkout; here each Client already multiplexes many concurrent calls, so the pool exists for throughput across connections and fault isolation, not mutual exclusion. A transaction begun through the pool pins the connection it started on (its ops go over that one Client, in order), while that connection keeps serving everyone else.
func NewPool ¶
NewPool dials n connections with dial and multiplexes each. On any dial error it closes the connections already opened and returns the error.
func (*Pool) Begin ¶
Begin starts a transaction on a pool connection; the returned Tx's operations all run over that connection (server-side transaction state + op ordering).
func (*Pool) MatchSorted ¶
MatchSorted runs a ranked match on a pool connection.
type ProposeFunc ¶ added in v0.7.0
ProposeFunc applies a committing transaction's ops to the replicated store via consensus. table is the transaction's table; ops preserve issue order. It returns nil on a quorum-committed apply, or an error (e.g. not-the-leader) that is surfaced to the client.
type QueryExemplar ¶ added in v0.32.0
type QueryExemplar struct {
Constraint string `json:"constraint"`
Duration time.Duration `json:"duration"`
Rows int `json:"rows"`
Projection int `json:"projection"`
Scan collections.ScanStats `json:"scan"`
User string `json:"user,omitempty"`
At time.Time `json:"at"`
}
QueryExemplar is one actual observed query: the slowest instances of a shape are kept so an operator can reproduce the cost, not merely read about it.
The exemplar is the point of keeping both levels. In this engine the LITERAL decides the work: `CompletionDate >= <now-1h>` prunes to a couple of sealed segments on their zone maps while `CompletionDate >= <now-365d>` touches every one of them, and the two are the same shape. The shape bucket says which pattern costs; the exemplar says which parameterization to paste into EXPLAIN.
An exemplar carries the real constraint text and the requesting identity, so it can name a user and the job ids they asked about -- unlike a shape, which carries no data. The server returns exemplars only to a DAEMON-authorized peer.
type QueryLog ¶ added in v0.10.0
type QueryLog struct {
Op string // the query opcode: "Query", "QueryRaw", ...
Table string // the queried table
Constraint string // the constraint expression ("" / "true" = match-all)
Limit int // the client's LIMIT (0 = unlimited)
Rows int // rows streamed back
Duration time.Duration // wall-clock time to run + stream the query
// Projection is how many attributes the client asked the server to return; 0 means
// it asked for no projection at all, so every attribute of every matched ad crossed
// the wire. That is the chatty pattern this log exists to make visible, and without
// this field it was the one thing the log could not show.
Projection int
// Scan is the server-side work breakdown for the query -- notably segments scanned
// versus pruned, which separates "slow because the window is big" from "slow because
// nothing pruned". It is filled on the read paths that thread it (the projection,
// wire and top-K ops); elsewhere it is zero, which means NOT MEASURED rather than
// no work.
Scan collections.ScanStats
// Columns is the work breakdown when the read was served from the table's COLUMNS rather
// than by materializing each record -- including how many cells came from a block's
// columns and how many still had to be read from the records themselves.
//
// Zero means the read was NOT served that way, which is the ordinary case for a table
// with no columnar accelerator, for a constraint the columns cannot evaluate, and for
// any read carrying a LIMIT. A caller wondering why a read is slow wants to know which
// of those it got, and nothing else in this log says.
Columns collections.ColumnScanStats
// User is the connection's authenticated identity (ServeOptions.Peer), or empty when
// the front end did not name one.
User string
// Groups and Aggs describe an AGGREGATE query: the GROUP BY columns (each with its
// bucket width, where it has one) and the functions reduced over each group. Both are
// empty on a plain read.
//
// They are recorded because grouping decides the work rather than merely the output.
// A single raw numeric group column is answered from the columnar blocks; the same
// WHERE grouped by an expression, or by a column the accelerator does not cover,
// falls through to rebuilding every matched record. Those are the same constraint and
// orders of magnitude apart, so a report that logged only the constraint would rank
// them as one shape and name neither.
//
// Note that Rows counts GROUPS for these -- a handful, however many records were read
// to produce them. Scan carries the cost.
Groups []GroupCol
Aggs []AggSpec
}
QueryLog is one observed query, passed to ServeOptions.QueryLog.
func (QueryLog) AggregatesText ¶ added in v0.34.0
AggregatesText renders the functions an observed query reduced over each group. Empty for a plain read.
func (QueryLog) GroupingText ¶ added in v0.34.0
GroupingText renders an observed query's GROUP BY columns -- the same fingerprint the shape report keys on. It is empty for a plain read and for an aggregate over the whole match, so a caller prints the field only when it has one.
Exported because the per-query log is the other operator surface these records feed, and a front end that formatted the raw GroupCol slice itself would render the same aggregate differently from the shape report it sits beside.
type QueryShapeStat ¶ added in v0.32.0
type QueryShapeStat struct {
Op string `json:"op"`
Table string `json:"table"`
// Shape is the constraint fingerprint: the constant-folded expression with every
// literal replaced by "?" and attribute names lowercased (see queryShape).
Shape string `json:"shape"`
// Limited records whether the client sent a LIMIT. It is part of the shape key
// because the same predicate with and without one does genuinely different work.
Limited bool `json:"limited"`
// GroupBy and Aggregates are an aggregate query's grouping and its reduced functions,
// fingerprinted (see groupShape and aggShape): `owner, qdate/3600` and
// `count(*), sum(remotewallclocktime)`. Both are empty for a plain read, and GroupBy
// alone is empty for an aggregate over the whole match.
//
// They are part of the shape key. Two aggregates over one constraint that group
// differently are not the same query in any sense an operator cares about: the
// grouping decides which tier answers, and merging them would report their summed
// cost against whichever grouping happened to be rendered.
GroupBy string `json:"groupBy,omitempty"`
Aggregates string `json:"aggregates,omitempty"`
Count int64 `json:"count"`
Total time.Duration `json:"total"`
Max time.Duration `json:"max"`
Rows int64 `json:"rows"`
// Unprojected counts the instances that asked for no server-side projection, i.e.
// every attribute of every matched ad crossed the wire. ProjectionMax is the widest
// projection any instance asked for.
Unprojected int64 `json:"unprojected"`
ProjectionMax int `json:"projectionMax"`
// SegmentsScanned and SegmentsPruned total the scan work across the instances that
// reported it. Their ratio is what separates "slow because the window is big" from
// "slow because nothing pruned".
SegmentsScanned int64 `json:"segmentsScanned"`
SegmentsPruned int64 `json:"segmentsPruned"`
// ScanSamples is how many instances reported scan work; the rest ran on a path that
// does not thread it, so a zero scan total with a non-zero Count means "not
// measured", not "no work".
ScanSamples int64 `json:"scanSamples"`
// Exemplars are the slowest observed instances, slowest first. Empty for a peer not
// authorized to see them.
Exemplars []QueryExemplar `json:"exemplars,omitempty"`
}
QueryShapeStat is the aggregate for one query shape.
func (QueryShapeStat) Mean ¶ added in v0.32.0
func (q QueryShapeStat) Mean() time.Duration
Mean is the shape's average duration.
type QueryStatsOptions ¶ added in v0.32.0
type QueryStatsOptions struct {
// MaxShapes bounds how many distinct query shapes are tracked. A client that
// composes a novel shape per request must not be able to grow the map without
// limit, so past the cap the shape with the LOWEST total time is evicted to make
// room -- the one least worth an operator's attention. Default queryStatsMaxShapes.
MaxShapes int
// MaxExemplars is how many slowest instances each shape keeps. Default
// queryStatsMaxExemplars.
MaxExemplars int
// MinDuration records only queries at least this slow. The default (0) records
// every query, which is deliberate: the query worth fixing is usually a 40ms one
// run ten thousand times, and a threshold high enough to be called "slow" is
// exactly the one that hides it. Raise it on a server where recording cost
// matters (each recorded query parses its constraint once).
MinDuration time.Duration
}
QueryStatsOptions configures the server's query aggregation (Server.EnableQueryStats). The zero value is the default of every field.
type QueryStatsReport ¶ added in v0.32.0
type QueryStatsReport struct {
// Enabled is false when the server is not aggregating; every other field is then zero.
Enabled bool `json:"enabled"`
// Since is when aggregation started (or was last reset).
Since time.Time `json:"since"`
// Recorded is how many queries have been aggregated, Evicted how many shapes were
// dropped to stay inside the cap.
Recorded int64 `json:"recorded"`
Evicted int64 `json:"evicted"`
// Tracked is how many shapes are held right now.
Tracked int `json:"tracked"`
// Shapes, ordered by total time descending -- count x mean, not max. The query worth
// an operator's time is usually the cheap one run constantly, not the single slowest.
Shapes []QueryShapeStat `json:"shapes"`
// Exemplars reports whether exemplars were included. False means the peer is not
// DAEMON-authorized, not that none were recorded.
Exemplars bool `json:"exemplars"`
}
QueryStatsReport is the server's aggregated query report.
type SeqCursor ¶ added in v0.29.5
type SeqCursor = collections.SeqCursor
SeqCursor and SeqPage re-export the storage layer's pagination types, so a dbrpc caller does not import collections just to hold a cursor.
type SeqPage ¶ added in v0.29.5
type SeqPage = collections.SeqPage
SeqCursor and SeqPage re-export the storage layer's pagination types, so a dbrpc caller does not import collections just to hold a cursor.
type ServeOptions ¶
type ServeOptions struct {
// ReadOnly rejects the mutating operations (NewClassAd, DestroyClassAd,
// SetAttribute, DeleteAttribute) with an error; reads, snapshots (Begin),
// and queries still work. A read-only peer may still Begin/Abort a
// transaction to get a stable snapshot for reads.
ReadOnly bool
// IncludePrivate renders returned ads with their private (secret)
// attributes intact (classad.StringWithPrivate). When false (the default),
// private attributes are stripped from every ad this connection sees, so an
// under-privileged peer never learns claim ids and other secrets.
IncludePrivate bool
// Privileged admits the DAEMON-level administrative actions -- ALL of the admin
// table (index/hot/compact/rewrite/codec.retrain, plus the security/durability ones
// like the encryption toggle and truncate). These retune or restructure the store,
// so an ordinary WRITE-level session (which may read and write ads) is refused them;
// only a DAEMON peer sets this true. Read-only diagnostics are a separate opcode and
// are not gated here.
Privileged bool
// TableWritable, if non-nil, is consulted for every request that would modify a
// named table; returning false refuses it as if the connection were read-only for
// that table alone, with an error the client sees as ErrTableReadOnly. A server
// whose table is owned by an in-process writer (a mirror of another store) uses it
// so no remote peer -- Privileged included -- can change that table underneath
// the writer. Reads, watches, snapshots and read-only transactions on the table
// are unaffected. Nil means every table is writable (the historical behavior).
//
// Gated: ad writes in a transaction on the table and a Commit that carries them,
// CommitIdempotent (it writes a marker), DeleteWhere, ArchiveAppend, ArchiveRotate,
// Create/Drop of a table, archive or view by that name, ConvertTableToMemory,
// Restore, and the admin actions that remove data or change what is kept
// (truncate, rotate, retention.set). Not gated: layout and tuning admin actions
// that preserve every row (index.*, hot.*, compact, rewrite, codec.retrain,
// schema.*, analyze, encrypt.set, timetravel.*, backup.key), and exporter ops,
// which name no table. See tableWriteTarget.
//
// Called from request goroutines concurrently; it must be safe for that and must
// not block. It may change its answer over time: a Commit re-checks it, so writes
// buffered while a table was writable do not land after it stops being so.
TableWritable func(table string) bool
// QueryLog, if set, is called once per streamed query with a summary of what
// the client asked for and what it cost. It is an opt-in query log for
// operators: it makes visible, for example, a client that fetches every
// attribute of every ad instead of projecting server-side (a slow, chatty
// query pattern). Called on the goroutine that served the query, once its last
// frame has been written -- so it delays nothing the client is waiting for, but it
// does hold a serving goroutine and must not block.
QueryLog func(QueryLog)
// Peer names the authenticated identity on the other end of this connection, for
// diagnostics only: it is stamped into QueryLog.User, so a per-query log line and a
// slow-query exemplar can say WHO ran the expensive query. Never used for access
// control -- that is ReadOnly/IncludePrivate/Privileged.
Peer string
}
ServeOptions scopes what a single served connection may do. The zero value is full read/write access with private attributes excluded from returned ads (the historical ServeConn behavior). A privilege-scoped front end (e.g. an HTCondor daemon serving READ vs WRITE peers) sets these per connection.
type Server ¶
type Server struct {
// CloseGrace bounds how long Close waits for cancelled requests to unwind before it
// gives up on them. Zero means defaultCloseGrace. Set it before serving connections.
CloseGrace time.Duration
// Logger receives what the server reports on its own, such as a request that panicked
// (see containPanic). Nil uses slog.Default(). Set it before serving connections.
Logger *slog.Logger
// contains filtered or unexported fields
}
Server serves an embedded ClassAd log (*db.DB) to remote clients. It is embeddable: create it around a DB, optionally start its managed maintenance goroutines, and hand it each accepted connection via ServeConn. Concurrent connections and concurrent in-flight requests per connection are supported; requests are dispatched as they arrive and responses carry the request id, so a slow call never head-of-line-blocks others.
func NewServer ¶
NewServer returns a single-table server over d, served as table "ads". The caller owns d's lifetime. Table create/drop are unsupported on this server.
func NewServerCatalog ¶
NewServerCatalog returns a multi-table server over cat.
func (*Server) Close ¶
func (s *Server) Close()
Close stops the server's managed goroutines and the requests it is serving. It does not close the DB; the caller does that afterwards, which is why Close must leave nothing running that still reads the catalog.
It takes no new work, stops and waits for background maintenance, cancels every served connection (ending its watches and stopping its scans) and then waits up to CloseGrace for the in-flight requests to unwind. It closes the connections' transports, which both tells the peers the server is gone and releases a request blocked writing to a client that stopped reading -- the one case cancellation cannot reach, since no context reaches a blocked WriteMsg. A request still running when the budget runs out is abandoned: Close returns anyway rather than hang, so closing the catalog can still race it.
Close is idempotent. A request that completes on its own is unaffected.
func (*Server) DisableQueryStats ¶ added in v0.32.0
func (s *Server) DisableQueryStats()
DisableQueryStats stops aggregation and discards what was collected.
func (*Server) EnableQueryStats ¶ added in v0.32.0
func (s *Server) EnableQueryStats(opts QueryStatsOptions)
EnableQueryStats starts (or reconfigures and restarts) aggregation of the queries this server serves: each one is bucketed by its shape -- opcode, table, constraint fingerprint and whether a LIMIT was sent -- with the slowest actual instances of each shape kept as exemplars. A client reads the result with Client.QueryStats; exemplars are returned only on a DAEMON-authorized connection, since they carry real constraint text and the requesting identity.
Recording happens on the goroutine that served the query, after its last frame went out, and costs one constraint parse plus a map update under a mutex.
func (*Server) LocalSession ¶ added in v0.37.0
func (s *Server) LocalSession(opts ServeOptions) *LocalSession
LocalSession opens an in-process reader under opts, which the caller has already derived from its own authorization exactly as it would for ServeConnOpts.
func (*Server) ServeConn ¶
ServeConn runs the request loop on one connection until it errors or the peer closes. Blocking; run one per accepted connection. Watches started on the connection are cancelled when it returns. Equivalent to ServeConnOpts with the zero ServeOptions (full read/write, private attributes excluded).
func (*Server) ServeConnOpts ¶
func (s *Server) ServeConnOpts(conn MsgConn, opts ServeOptions) error
ServeConnOpts is ServeConn scoped by opts: a read-only and/or private-stripping view of the same DB. Use it to serve a privilege-scoped peer (e.g. an HTCondor READ-level client) from the same Server.
func (*Server) SessionAttrs ¶ added in v0.35.0
SessionAttrs is what THIS server can serve: the build's features, minus any this particular server cannot honour.
A join is the case that forces the distinction. It reads two tables, so it needs a catalog; a server made with NewServer holds a single database and has no second table to join against. Advertising FeatJoin from it would be a definite yes that is wrong.
func (*Server) SetProposeHook ¶ added in v0.7.0
func (s *Server) SetProposeHook(fn ProposeFunc)
SetProposeHook routes committing writes through fn (raft) instead of the local store. Call once before serving. Passing nil restores direct local commits (the default). When set, the server is otherwise read-only against its local store: the hook is the sole writer.
func (*Server) StartIdemReaper ¶ added in v0.9.0
StartIdemReaper runs reapIdemMarkers every interval, deleting idempotency markers older than maxAge, until the returned stop is called. maxAge should exceed the longest window over which a client might replay a unit of work (its retry budget), so a marker is never reaped while a late replay could still arrive. Servers that never see CommitIdempotent need not start it (there is nothing to reap).
func (*Server) StartMaintenance ¶
func (s *Server) StartMaintenance(interval time.Duration, opts db.MaintainOptions)
StartMaintenance starts server-managed background maintenance: a single goroutine that, each tick, re-enumerates the catalog's tables and runs one Maintain pass (index auto-tune, hot-set refresh, and -- if opts.Retrain -- dictionary retrain) on each. Re-enumerating each tick means tables created after startup are maintained too (the old per-table start missed them).
The cadence is adaptive: interval is a floor, but the wait before the next pass is raised to keep maintenance under maintenanceMaxDutyCycle of wall-clock time -- so on a large store where a pass (dominated by dictionary retrain's recompaction) takes minutes, passes automatically space out rather than consuming the daemon. Stopped by Close.
func (*Server) StartTxnReaper ¶ added in v0.9.0
StartTxnReaper runs reapIdleTxns every interval, aborting transactions idle longer than maxIdle, until the returned stop is called. Optional: disconnect cleanup is automatic; a server that wants to bound transactions abandoned on still-open connections starts this.
type ServerError ¶ added in v0.9.0
type ServerError struct{ Msg string }
ServerError is a logical error the server returned for a request (wire status stErr): a bad constraint, an unknown table, a malformed op. It is deterministic -- replaying the request fails the same way -- so callers should surface it rather than retry.
func (*ServerError) Error ¶ added in v0.9.0
func (e *ServerError) Error() string
func (*ServerError) Is ¶ added in v0.31.0
func (e *ServerError) Is(target error) bool
Is matches the prefix-tagged refusals -- a per-table write refusal and an op-does-not-serve-an-archive refusal -- so errors.Is tells each apart from other server errors while errors.As still yields the *ServerError.
type Support ¶ added in v0.33.3
type Support int
Support is a peer's answer about one feature. It has three values, not two, because "the peer did not say" is a real and common answer that must not be collapsed into either of the others.
const ( // SupportUnknown means the peer advertised nothing: it predates // advertisement, or it is reached over a transport that carries none. It // says nothing about whether the feature works -- every feature named here // existed before advertisement did, so a silent peer may well implement it. // Probe, or emulate, or refuse with a message that says which. SupportUnknown Support = iota // SupportNo means the peer advertised its features and this was not among // them. This is a definite no, so a caller must not issue the request to // find out. SupportNo // SupportYes means the peer named the feature. Issue the request directly; // a fallback here is unreachable code. SupportYes )
type TopKOrderTerm ¶ added in v0.43.0
type TopKOrderTerm = db.TopKOrderTerm
TopKRowsSpec, TopKOrderTerm and GroupCol re-export db's, so a caller writes one spec and hands it to either the embedded rung or this one.
type TopKRowsSpec ¶ added in v0.43.0
type TopKRowsSpec = db.TopKRowsSpec
TopKRowsSpec, TopKOrderTerm and GroupCol re-export db's, so a caller writes one spec and hands it to either the embedded rung or this one.
type TopKWireGroup ¶ added in v0.43.0
TopKWireGroup is one group's answer: the group columns' values, and its best rows in wire form, ordered best first.
Rows are self-contained wire ads, owned by the caller -- decode one with wire.DecodeInline, or relay it. They are NOT decoded here, because a caller that is going to re-render them (a REPL printing a table, a relay forwarding them) would pay for a decode it throws away.
type Tx ¶
type Tx struct {
// contains filtered or unexported fields
}
Tx is a client-side handle to a server transaction. Its operations are issued in order over the client (the transaction is server-side state); different Tx (even on one Client) proceed concurrently.
func (*Tx) Commit ¶
Commit applies the transaction, returning *db.ConflictError with the conflicted keys if any lost a write-write race (the rest committed), or nil.
func (*Tx) CommitIdempotent ¶ added in v0.9.0
CommitIdempotent is Commit with exactly-once semantics across retries: the server records a durable marker under idemKey, committed atomically with this transaction's writes, so replaying the SAME unit of work (same idemKey) after an ambiguous failure -- a reconnect where the original commit's fate is unknown -- applies it at most once. The caller must generate idemKey ONCE per unit of work and reuse it verbatim on every replay. Use this for non-idempotent transactions (e.g. relative updates, appends); an already-idempotent-by-key workload can use the cheaper plain Commit. Returns the same results as Commit (nil, or *db.ConflictError on a genuine write-write conflict).
func (*Tx) DeleteAttribute ¶
DeleteAttribute removes key's attribute name.
func (*Tx) DestroyClassAd ¶
DestroyClassAd removes key.
func (*Tx) KeysWhere ¶ added in v0.24.0
KeysWhere returns the storage keys of the rows matching the constraint as this transaction sees them. It is what lets a caller address rows -- for a subsequent SetAttribute or DestroyClassAd in the same transaction -- including rows the transaction itself created, which Client.QueryKeysTable cannot see.
Against a server without the opcode it returns ErrTxnReadUnsupported.
func (*Tx) LookupAttr ¶
LookupAttr returns key's attribute name (unparsed expression) as the transaction sees it, or ("", false).
func (*Tx) LookupClassAd ¶
LookupClassAd returns key's ad (old-ClassAd text) as the transaction sees it.
func (*Tx) NewClassAd ¶
NewClassAd stores the ad (old-ClassAd text) under key.
func (*Tx) NewClassAdBatch ¶ added in v0.13.0
NewClassAdBatch stores many ads under one transaction in a single round-trip, instead of one request/ack per ad. It returns the ads the server rejected (bad text) so the caller can log them; the rest are applied. The whole request must fit the stream's max message size, so for a large batch prefer NewClassAdBatchPipelined, which chunks and pipelines automatically.
func (*Tx) NewClassAdBatchPipelined ¶ added in v0.13.0
func (t *Tx) NewClassAdBatchPipelined(ctx context.Context, items []AdKV, chunkSize int) ([]AdReject, error)
NewClassAdBatchPipelined stores many ads under one transaction with the round-trips pipelined: it splits items into chunks of chunkSize ads and sends EVERY chunk frame back-to-back without waiting for any chunk's ack, then collects all the acks. So the chunks flow independently of the acks and the wire cost is ~one round-trip regardless of ad count -- not one per ad, nor one serialized round-trip per chunk -- while each message stays small and the server applies a whole chunk under a single transaction- lock acquisition (chunkSize goroutines/locks for the batch, not one per ad).
It returns the ads the server rejected (unparseable), indexed into items. A whole- chunk failure -- a dropped connection, or the transaction gone ("no such transaction") -- is returned as an error so the caller aborts and replays the batch. chunkSize <= 0 defaults to 64.
func (*Tx) Query ¶ added in v0.24.0
Query returns the ads matching the constraint as this transaction sees them: the committed rows with the transaction's own uncommitted writes overlaid. Client.Query reads the committed store and cannot see them, which is the whole reason this exists -- a caller that has staged writes and wants to read them back must come through here.
Reads are snapshot-isolated: the committed rows are read at the transaction's own snapshot, so a concurrent commit is invisible for the transaction's lifetime.
The transaction's table is implied (opBegin bound it), so unlike QueryTable there is no table argument. A limit <= 0 returns every match. Against a server without the opcode it returns ErrTxnReadUnsupported.
type ValueBatch ¶ added in v0.36.0
type ValueBatch struct {
Rows int
Cols []ValueColumn
}
ValueBatch is one column-major batch: Rows rows, one column per requested attribute, in the order the attributes were requested.
func (*ValueBatch) Column ¶ added in v0.37.0
Column expands one column's rows into dst in order, appending and returning the extended slice so a caller can reuse one buffer per column across batches.
This is how a batch is meant to be read. A column is a dense slice indexed by how many valid rows PRECEDE a row, so the index is cheap to carry forward and expensive to recover -- which is exactly what ValueAt does, per call, from row zero. Reading a whole batch through ValueAt is therefore quadratic in the batch size, and it is the obvious way to read one: measured on a six-column projection it was three quarters of a read's total cost, and the read looked like the op was slow rather than the accessor.
A row-wise consumer expands each column once and then indexes them together; that is still one forward pass per column, and it is what the endpoint this op was built for does.
func (*ValueBatch) ValueAt ¶ added in v0.36.0
func (b *ValueBatch) ValueAt(col, row int) classad.Value
ValueAt reassembles ONE cell, for a caller that wants a value rather than a column -- a test oracle, or a consumer reading a handful of cells at known positions.
It is linear in row, because a column's dense slice is indexed by how many valid rows precede this one and nothing records that per row. Reading a whole batch through it is therefore quadratic in the batch size; use Column, which carries the index forward in one pass. The popcount below makes this cheaper by a constant, not by an order.
type ValueColumn ¶ added in v0.36.0
type ValueColumn struct {
Kind ValueKind
Valid []byte
Ints []int64
Reals []float64
Bools []bool
Strs []string
Exceptions []ValueException
}
ValueColumn is one attribute's values across a batch.
Valid is a bitmap over the batch's rows, LSB-first within each byte: a set bit means that row holds a value of Kind, and its value is the next one in the dense slice for that kind. A clear bit means the row is in Exceptions. Exactly one of Ints/Reals/Bools/Strs is populated, according to Kind, and each holds popcount(Valid) entries -- not one per row.
type ValueException ¶ added in v0.36.0
type ValueException struct {
Row int
// Value is the cell. For a value with NO scalar form -- a list, a nested ad -- it is the
// unparser's text of that value held as a string, and NonScalar says so.
//
// The flag is not decoration. Without it a list arrives indistinguishable from a string
// that happens to look like one, and a consumer has no way back to the difference. Text is
// the right carrier -- it is the only form such a value is useful in, and it is what a
// schema's text tail column wants -- but it has to be labelled as a rendering rather than
// as the value itself.
Value classad.Value
NonScalar bool
}
ValueException is one cell that a column could not pack: its row within the batch, and its value. Exceptions are in ascending Row order.
type ValueKind ¶ added in v0.36.0
type ValueKind uint8
ValueKind is the kind a value column packs densely. A cell of any other kind, and a cell that is undefined or error, is carried in the column's exceptions list instead.
const ( // KindNone is a column with no densely-codable kind: every cell is an exception. It is the // honest answer for a column of lists, nested ads or a free mix, not a failure. KindNone ValueKind = 0 // KindInt, KindReal, KindBool and KindString are the scalar kinds a column packs. KindInt ValueKind = 1 KindReal ValueKind = 2 KindBool ValueKind = 3 KindString ValueKind = 4 // KindStringDict is a string column shipped as a DICTIONARY plus one code per cell, // for a column whose values repeat. It carries the same values KindString would and // decodes to the same per-cell strings; what changes is how many of them exist. // // A pool's Owner, a job's status text, a site name: storage already holds these as a // per-block dictionary, and KindString expanded them to one string per ROW -- 20000 // allocations on the sending side and 20000 more on the receiving side to carry 64 // distinct values. Here the receiver allocates the dictionary and the per-cell slice // holds references into it, so the count follows the column's CARDINALITY rather than // its length. // // Gated on the client asking (valueFlagDict), because an older decoder refuses a kind // it does not know -- which is the right thing for it to do and the reason this cannot // simply be switched on. KindStringDict ValueKind = 5 )
type ValueStreamOptions ¶ added in v0.37.0
type ValueStreamOptions struct {
// Dictionary asks for string columns as a DICTIONARY plus one code per cell where that
// pays (KindStringDict). The decoder expands it into the same ValueColumn.Strs slice the
// plain form fills, so a caller reading values through ValueAt or Strs needs no change --
// only one that SWITCHES ON Kind has to learn the new one, which is the whole reason this
// is opt-in rather than always on.
//
// Ignored against a server that does not advertise FeatValueDict: an older one refuses a
// flag it does not know, and getting an error back for asking would be a poor trade for
// an optimization.
Dictionary bool
}
ValueStreamOptions are the opt-in parts of the value stream.
type WatchEvent ¶
type WatchEvent struct {
Kind uint8 // 0 upsert, 1 delete, 2 reset (see db.WatchKind)
Key string
AdText string
Cursor []byte
}
WatchEvent is one change delivered over a Watch. AdText is the ad's old-ClassAd text (empty for a delete/reset). Cursor resumes the watch just after this event.
type WireWatchEvent ¶ added in v0.25.1
type WireWatchEvent struct {
Kind uint8 // 0 upsert, 1 delete, 2 reset (see db.WatchKind)
Key string
Ad []byte // wire-form ad; decode with DecodeWatchAd
Cursor []byte
}
WireWatchEvent is a WatchEvent whose ad is the wire form. Ad is nil for a delete or a reset, exactly as AdText is empty for those.
Source Files
¶
- aggregate.go
- archive.go
- attrtypes.go
- cedarconn.go
- client.go
- conn.go
- delete.go
- diag.go
- errors.go
- exporter.go
- idempotency.go
- insertrows.go
- insertselect.go
- join.go
- joinagg.go
- joinband.go
- joinstats.go
- local.go
- match.go
- panics.go
- pool.go
- private_gate.go
- propose.go
- proto.go
- protocol_version.go
- queryraw.go
- queryshape.go
- querystats.go
- resolve.go
- server.go
- snapshot.go
- table_gate.go
- topk.go
- topkrows.go
- txnquery.go
- valuecolumns.go
- values.go
- view.go
- watchwire.go