recordengine

type AppendLimits struct {
	// MaxSeries caps the number of distinct streams buffered in the head. A record that would
	// register a new stream once the head already holds MaxSeries is rejected (the whole batch,
	// since a batch is one stream); known streams are unaffected. 0 ⇒ unlimited.
	MaxSeries int64
	// MaxInFlightBytes caps the head's buffered record bytes. A record arriving while the head is
	// at or over the cap is rejected (memory backpressure) until a flush drains it. 0 ⇒ unlimited.
	MaxInFlightBytes int64
}

type AppendResult struct {
	Accepted            int
	RejectedOOO         int // older than the out-of-order window
	RejectedCardinality int // would exceed AppendLimits.MaxSeries (a new stream)
	RejectedBytes       int // head at or over AppendLimits.MaxInFlightBytes
}

type Batch struct {
	Stream   signal.SeriesID
	Identity func() signal.Series // materialized only when the stream is newly seen
	Ts       []int64
	Ints     [][]int64  // len == schema int count; Ints[k][row]
	Bytes    [][][]byte // len == schema byte count; Bytes[k][row]
	// Side is an optional encoded side-store delta (the content-addressed symbols this batch's
	// records reference) absorbed by [Config.SideStore]. nil when the engine has no side store.
	Side []byte
}

type BloomMode uint8

type CardinalityStat struct {
	TotalSeries        int64 // distinct streams
	DistinctLabelNames int
	SymbolCount        int
	Top                []LabelCard // sorted by Series desc, then Name; truncated to the requested top-N
}

type Column struct {
	Name  string
	Kind  Kind
	Codec chunk.Codec
	Bloom BloomMode
}

type ColumnCostStat struct {
	Name      string
	RawBytes  int64
	DiskBytes int64 // approximate, as [StreamCostStat.DiskBytes]

	// Distinct estimates the distinct values the group's rows hold in this column (HyperLogLog,
	// ~1.6% standard error). 0 for an int column and for a group outside the sketch budget.
	Distinct int64

	// DistinctNormalized is Distinct over values with every run of ASCII digits collapsed to '#'.
	// It is what separates a genuinely high-entropy column from a templated one carrying an embedded
	// timestamp or id: a group whose Distinct is large and whose DistinctNormalized is tiny is not a
	// storage problem but a parsing one — the same line, repeated, never turned into fields.
	DistinctNormalized int64
}

type ColumnStat struct {
	Name     string
	Kind     string // physical type: int64 / float64 / bytes / int128
	Codec    string // value codec
	Compress string // block-compression algorithm
	Level    int    // block-compression level (0 ⇒ algorithm default or uncompressed)
}

type Config struct {
	// Schema is the per-record column set this engine stores (required; the signal supplies it).
	Schema *Schema
	// OOOWindow is a per-stream lateness bound: a record older than OOOWindow (nanoseconds) behind
	// the stream's own newest admitted record is rejected. 0 disables.
	OOOWindow int64
	// WAL, when non-nil, durably logs streams and records for crash recovery. nil ⇒ ephemeral.
	WAL *wal.SegmentWriter
	// Backend stores flushed parts. Required for [Engine.Flush]; nil ⇒ head-only.
	Backend backend.Backend
	// Prefix is the backend key prefix under which this engine's parts are written.
	Prefix string
	// Term reports this writer's current ownership term for Prefix — which tenure of the shard
	// this engine is writing as. It is stamped into every bucket index written, so a reader can
	// order two indexes of the same prefix even when neither the part names nor FlushedEpoch
	// moved; see [github.com/oteldb/storage/backend/bucketindex.Generation]. nil is a writer with
	// no cluster, whose generation is then a plain local counter.
	Term func() uint64
	// WriterID is this writer's stable identity — the cluster node id — under which its WAL flush
	// watermark is kept in the bucket index. It matters because that index is *shared*: over a
	// shared object store every replica of a shard commits one index object under one prefix, and
	// the watermark is a per-node count of that node's own flushes, so one scalar in a shared
	// object is meaningless to whichever node did not write it (see
	// [github.com/oteldb/storage/backend/bucketindex.WriterEpoch]).
	//
	// Empty is the anonymous writer: a single-writer engine, which keeps the sole pre-v4 slot.
	// Leaving it empty where two engines do share a prefix makes them share one slot, which is
	// the defect the slots exist to prevent.
	WriterID string
	// SideStore, when non-nil, is a signal-supplied content-addressed auxiliary store (e.g. the
	// profiles symbol store) that the engine persists as part sidecars on flush and unions on merge.
	// nil ⇒ no side data (logs, traces).
	SideStore SideStore
	// Obs is the observability handle (spans + metrics). nil ⇒ a no-op handle.
	Obs *obs.Obs
	// Signal is the signal label for this engine's metrics ("log"/"trace"/"profile"); the facade
	// sets it per signal. Empty ⇒ "record".
	Signal string
	// MaxPartBytes bounds a flushed/merged part's approximate uncompressed size. It is what lets
	// size-tiered compaction seal large parts and bound both part count and a single merge's decoded
	// working set (see compact.go) — without it a continuously-ingesting engine's merge grows to
	// re-materialize the whole dataset every cycle. 0 ⇒ unlimited (merge everything into one part;
	// the legacy behavior, unbounded working set). The facade resolves it from the tenant policy.
	MaxPartBytes int64
	// MergeMemoryBytes is how much memory all concurrent merges together may hold. A merge holds its
	// selected sources decoded plus the output buffer it is filling, so this bounds the merge cap
	// (see mergecap.go) independently of MaxPartBytes — a tiering target sized for the disk must not
	// size a working set the process cannot hold. 0 ⇒ a share of the process memory budget
	// (GOMEMLIMIT, else the cgroup limit, else host memory); negative ⇒ unbounded.
	MergeMemoryBytes int64
	// MergeConcurrency reports how many merges may run concurrently in this process, dividing the
	// merge memory allowance so they cannot collectively exceed it. nil or ≤ 1 ⇒ no division.
	MergeConcurrency func() int
	// MergeCompression block-compresses the columns of merged (compacted) parts on top of their chunk
	// codecs — the cold, long-lived data. Flushed parts stay codec-only so ingest is cheap; the
	// background merge is where recompression is amortized. Record byte columns are dict-coded but not
	// entropy-coded, so ZSTD here is a large on-disk win (≈10× on log-shaped data). AlgorithmNone (the
	// default) keeps the legacy uncompressed behavior.
	MergeCompression compress.Algorithm
	// MergeCompressionLevel is the level for MergeCompression (ZSTD only). 0 ⇒ the algorithm default.
	MergeCompressionLevel compress.Level
	// MinFreeBytes is the headroom the engine leaves unused on a backend that reports its free
	// space: a flush is refused, and the ingest path starts rejecting, once the medium holds less
	// than the pending part plus this. It leaves a merge room for its output — a merge must write
	// before it can retire the inputs it frees. 0 ⇒ [diskguard.DefaultReserveBytes]; negative ⇒ the
	// byte axis is not checked.
	MinFreeBytes int64
	// MinFreeInodes is the same headroom on the object-count axis, for a backend that reports free
	// inodes. It is checked separately because a part is many small objects: an inode table can
	// exhaust with the disk half empty, and byte accounting cannot see it. 0 ⇒
	// [diskguard.DefaultReserveInodes]; negative ⇒ the inode axis is not checked.
	MinFreeInodes int64
}

type Engine struct {
	// contains filtered or unexported fields
}

type KeyInfo struct {
	Key   []byte
	Scope KeyScope
}

type KeyScope uint8

type Kind uint8

type LabelCard struct {
	Name           string
	Series         int64
	DistinctValues int
}

type MergeOptions struct {
	// RetainFrom drops records older than this absolute unix-nanosecond cutoff (retention);
	// ≤ 0 disables it.
	RetainFrom int64
	// Force compacts a bucket's unsealed parts whatever their tiers, instead of waiting for one tier
	// to accumulate minTierParts of them — the operator escape from a part set the tier rule will
	// never select. It bypasses the selection heuristic only: sealing, the time-bucket ladder, and
	// the cumulative-bytes cap still bound what one merge decodes and holds.
	Force bool
}

type MergeShape struct {
	// Parts is the flushed parts; Sealed those already at the cap, which no merge reconsiders;
	// Backlog the rest — the parts a merge may still take.
	Parts   int
	Sealed  int
	Backlog int
	// Candidates is how many parts the next merge would select right now. 0 with a non-zero Backlog
	// is the stuck state: parts remain mergeable but no tier of any time bucket holds minTierParts
	// of them, which is what [MergeOptions.Force] exists to break.
	Candidates int
	// CapBytes is the seal threshold in effect, in decoded bytes (0 ⇒ sealing disabled).
	CapBytes int64
	// Tiers is how many distinct size tiers the unsealed parts fall into and LargestTierParts the
	// count in the fullest of them — a LargestTierParts below minTierParts is why nothing merges.
	// Tiers span the whole engine here, not one time bucket, so LargestTierParts is an upper bound
	// on what the ladder can select.
	Tiers            int
	LargestTierParts int
	// MinTierParts is the same-tier count a merge waits for.
	MinTierParts int
}

type PartDetailStat struct {
	PartStat

	Bytes   int64        // sum of the part's backend object sizes
	Chunks  int          // sparse-index granules: ceil(RowCount / GranuleSize)
	Columns []ColumnStat // per-column physical layout
}

type PartStat struct {
	ID      string // the part's backend key prefix
	MinTime int64  // inclusive unix-ns bounds of the part's records
	MaxTime int64
	Series  int   // distinct streams in the part (len of its row-range index)
	Rows    int64 // total records (sum of the per-stream row spans)
	// SizeBytes is the part's *decoded* footprint as recorded in its manifest — what the merge cap
	// compares against, and so what explains why a part is or is not sealed. It needs no I/O. Unlike
	// the metric engine's [engine.PartStat.SizeBytes], which is a size on disk, this is the size in
	// memory: the record merge is bounded by what it holds, not by what it writes. 0 for a part
	// written before the manifest recorded it (the row estimate then stands in internally).
	SizeBytes int64
}

type PruneOptions struct {
	// Force runs the prune even when the background thresholds would skip it — an engine that has
	// merged nothing away since the last pass, or one whose dead set is too small to pay for the
	// rebuild. It is what an operator-triggered sweep wants.
	Force bool
}

type Schema struct {
	// contains filtered or unexported fields
}

type SideStore interface {
	// Absorb merges one batch's encoded side delta ([Batch.Side]) into the live accumulator.
	Absorb(delta []byte) error
	// Encode serializes the accumulated side data into named sidecar payloads (name → bytes),
	// written as {prefix}/sym-{name}.bin at flush.
	Encode() map[string][]byte
	// Reset clears the live accumulator (after a flush drains the head).
	Reset()
	// Restore merges an [SideStore.Encode] snapshot back into the live accumulator. The engine calls
	// it when a flush fails after the snapshot+Reset: the records return to the head, so their side
	// data must too. Content-addressing makes the merge a plain dedup with whatever the accumulator
	// gained meanwhile.
	Restore(snapshot map[string][]byte) error
	// Names returns the sidecar names to read back for a part on merge (the keys [SideStore.Encode]
	// may produce). A part missing a named sidecar is skipped.
	Names() []string
	// Union merges the loaded sidecars of the compacted parts (one map per part) and returns the
	// merged named payloads to write under the new part. Pure; ignores the live accumulator.
	Union(parts []map[string][]byte) (map[string][]byte, error)
}

type Stats struct {
	Streams     int64 // distinct streams ever seen (index span: head ∪ flushed)
	HeadRecords int64 // records currently buffered in the head (unflushed)
	HeadBytes   int64 // buffered record bytes, head + in-flight flush (the in-flight memory measure)
	// IdentityBytes is the resident identity state (symbols + stream index + postings + OOO
	// watermarks) — memory a flush does not drain, and which no other counter here reports.
	IdentityBytes int64
	Parts         int   // flushed immutable parts
	MinTime       int64 // oldest flushed record time (unix ns); 0 when no parts
	MaxTime       int64 // newest record time across parts and the head (unix ns); 0 when empty
	// OutOfSpace is set while the engine refuses writes because its backend is out of bytes or
	// inodes. Reads still answer from what is on disk; it clears when a flush finds room again.
	OutOfSpace bool
}

type StreamCostOptions struct {
	// GroupBy is the stream label (a resource or scope attribute name, e.g. "service.name") whose
	// value keys the report. Empty groups by raw stream id. Grouping by label is the useful form:
	// what an operator can act on is a service, and stream identity is a field policy that can
	// change under them.
	GroupBy string

	// Columns restricts the byte columns that are decoded and attributed (nil ⇒ every byte column).
	// The int columns and the timestamp are always accounted — they need no decode — so a narrowed
	// request still reports complete row counts, and RawBytes/DiskBytes cover exactly the columns
	// listed in each group's Columns.
	Columns []string

	// TopN keeps only the N costliest groups by RawBytes (≤0 ⇒ every group).
	TopN int

	// MaxSketchGroups bounds how many groups carry distinct estimates, which cost 8 KiB of sketch
	// each for the duration of one column's pass (≤0 ⇒ [DefaultStreamCostSketchGroups]). The budget
	// goes to the groups with the most rows; the rest report DistinctEstimated false.
	MaxSketchGroups int
}

type StreamCostStat struct {
	Key     string // the GroupBy label's value, or the stream id; empty ⇒ the label is absent
	Streams int    // distinct streams folded into this group
	Rows    int64

	// RawBytes is the decoded footprint of the group's rows over the accounted columns.
	RawBytes int64

	// DiskBytes is APPROXIMATE. Compression is per column per frame and a frame spans whatever
	// streams its rows fall in, so a group's compressed footprint is not directly measurable: each
	// frame's compressed size is apportioned across the groups holding its rows by their raw-byte
	// share. Rows are (stream, ts)-ordered, so most frames hold one stream and the estimate is
	// close; a group narrower than a frame is the case where it is not.
	DiskBytes int64

	// DistinctEstimated reports whether the distinct counts below were computed for this group — see
	// [StreamCostOptions.MaxSketchGroups]. False ⇒ Distinct/DistinctNormalized are 0, not zero.
	DistinctEstimated bool

	Columns []ColumnCostStat
}