synth

type CategoricalNumericCategoryFidelity struct {
	Category string `json:"category"`
	// SourceMean/SourceStd echo the captured
	// CategoricalNumericCategorySpec verbatim.
	SourceMean float64 `json:"source_mean"`
	SourceStd  float64 `json:"source_std"`
	// SyntheticMean/SyntheticStd are the realized conditional mean/std
	// over the _synthetic=true partition. MeanDelta/StdDelta are
	// abs(source-synthetic). All four are zero-valued and meaningless
	// when Error is set.
	SyntheticMean float64 `json:"synthetic_mean,omitempty"`
	SyntheticStd  float64 `json:"synthetic_std,omitempty"`
	MeanDelta     float64 `json:"mean_delta,omitempty"`
	StdDelta      float64 `json:"std_delta,omitempty"`
	// N is the number of _synthetic=true rows carrying this category
	// value with a non-null B — the realized category's own observation
	// count.
	N int `json:"n"`
	// Error is set instead of the Synthetic*/*Delta fields when the
	// synthetic partition realized no observation of this category at
	// all — mirrors PairwiseFidelity's own per-entry non-fatal contract.
	Error string `json:"error,omitempty"`
}

type CategoricalNumericCategorySpec struct {
	Category string  `json:"category"`
	Mean     float64 `json:"mean"`
	Std      float64 `json:"std"`
}

type CategoricalNumericCategoryStat struct {
	Category string  `json:"category"`
	Mean     float64 `json:"mean"`
	Std      float64 `json:"std"`
	// N is the number of non-null B observations conditioned on this
	// category — the input thinPairWarning checks against
	// MinPairObservations, exactly as NumericPairProfile.N and each
	// ContingencyCell.Count already do for the other two pair kinds.
	N int `json:"n"`
}

type CategoricalNumericPairFidelity struct {
	A          string                                `json:"a"`
	B          string                                `json:"b"`
	Categories []*CategoricalNumericCategoryFidelity `json:"categories,omitempty"`
}

type CategoricalNumericPairProfile struct {
	A          string                           `json:"a"` // categorical field name
	B          string                           `json:"b"` // numeric field name
	Categories []CategoricalNumericCategoryStat `json:"categories"`
	// N is the number of rows in the capture where both A and B were
	// simultaneously non-null, summed across every category (including
	// "other") — the pair's true co-occurrence count, mirroring
	// CategoricalPairProfile.N and NumericPairProfile.N.
	N int `json:"n"`
}

type CategoricalNumericPairSpec struct {
	A          string                           `json:"a"`
	B          string                           `json:"b"`
	Categories []CategoricalNumericCategorySpec `json:"categories"`
	// Min/Max clamp the conditional draw exactly as field B's own
	// unconditional normal reconstruction clamps to its observed range
	// (see SpecFromProfile). HasClamp distinguishes "clamp to [0,0]"
	// (rare, but a legitimate observed range for a constant field) from
	// "no clamp was declared" — a hand-authored from-schema spec that
	// omits Min/Max gets no clamping, exactly as normal's own optional
	// min/max params behave when absent.
	Min      float64 `json:"min,omitempty"`
	Max      float64 `json:"max,omitempty"`
	HasClamp bool    `json:"has_clamp,omitempty"`
	// Bernoulli is RETIRED, exactly as
	// CategoricalNumericPairSpec.Bernoulli is — accepted so an older
	// document still parses, read by nothing, derived from the target's
	// own FieldSpec instead. THIS is the slot whose silent omission
	// proved the case: SpecFromProfile never set it, so every
	// profile-derived set-numeric pair over a boolean target drew a
	// clamped normal. See CategoricalNumericPairSpec.Bernoulli for the
	// full reasoning.
	//
	// Deprecated: declare the target field's distribution as
	// DistBernoulli; this slot is ignored.
	Bernoulli bool `json:"bernoulli,omitempty"`
}

type CategoricalPairCellSpec struct {
	AValue string `json:"a_value"`
	BValue string `json:"b_value"`
	Count  int    `json:"count"`
}

type CategoricalPairFidelity struct {
	A string `json:"a"`
	B string `json:"b"`
	// Delta is the total variation distance between the source and
	// synthetic joint-frequency tables (each normalized to proportions
	// before comparison), in [0, 1] — 0 when the two tables carry
	// identical support and proportions, 1 when they share none. Chosen
	// over a chi-square-style distance because it stays well-defined and
	// boundedly comparable to a fixed tolerance even when synthesis
	// realizes a cell the capture never saw (or vice versa) — a
	// chi-square statistic diverges when an expected cell is zero. Zero
	// valued and meaningless when Error is set.
	Delta float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with both A and B
	// simultaneously non-null and dictionary-resolvable — the realized
	// pair's own co-occurrence count. CategoricalPairSpec carries no
	// source N of its own; the source total used to normalize its side
	// of the comparison is recovered by summing Cells' Count.
	N int `json:"n"`
	// Error is set instead of Delta when the synthetic partition
	// produced no co-occurring observation at all for this pair — mirrors
	// PairwiseFidelity's own per-pair non-fatal failure contract.
	Error string `json:"error,omitempty"`
}

type CategoricalPairProfile struct {
	A     string            `json:"a"`
	B     string            `json:"b"`
	Cells []ContingencyCell `json:"cells"`
	// N is the number of rows in the capture where both A and B were
	// simultaneously non-null — the pair's true co-occurrence count,
	// summed across every cell (the cells kept plus whatever is folded
	// into the "other"/"other" catch-all).
	N int `json:"n"`
}

type CategoricalPairSpec struct {
	A     string                    `json:"a"`
	B     string                    `json:"b"`
	Cells []CategoricalPairCellSpec `json:"cells"`
}

type CategoricalProfile struct {
	Cardinality int           `json:"cardinality"`
	Top         []CategoryHit `json:"top"`
}

type CategoryHit struct {
	Value  string  `json:"value"`
	Weight float64 `json:"weight"`
}

type ConditionalProfile struct {
	// NumericPairs lists the numeric-numeric pairs captured: the
	// row-aligned Pearson correlation and the count of rows where both
	// fields were simultaneously non-null (N) — the input
	// thinPairWarning checks against MinPairObservations. Capped at
	// ProfileOptions.CorrelationTopK strongest |rho| pairs, same
	// convention as Profile.Pairwise.
	NumericPairs []NumericPairProfile `json:"numeric_pairs,omitempty"`
	// CategoricalPairs lists the categorical-categorical pairs
	// captured: a bounded contingency table of observed
	// (a_value, b_value) co-occurrence counts per pair. See
	// CategoricalPairProfile for the two composed collapse points
	// (per-field top-K, then joint-cell ContingencyCellCap) that keep
	// this section bounded regardless of either field's cardinality.
	CategoricalPairs []CategoricalPairProfile `json:"categorical_pairs,omitempty"`
	// CategoricalNumericPairs lists the categorical-numeric pairs
	// captured: the numeric field's conditional mean/std broken out per
	// observed category of the categorical field. See
	// CategoricalNumericPairProfile for the single collapse point
	// (per-field top-K) that keeps this section bounded — no separate
	// joint-cardinality cap is needed here, unlike CategoricalPairs,
	// because a conditional numeric summary is one row per already-capped
	// category rather than a joint cell per pair of categories.
	CategoricalNumericPairs []CategoricalNumericPairProfile `json:"categorical_numeric_pairs,omitempty"`
	// SetCategoricalPairs lists the set-option x categorical pairs
	// captured (E5-S2): one bounded contingency table per (set field,
	// option, categorical field) combination, the set-option axis fixed
	// to the two-value {"selected","not_selected"} Bernoulli domain. See
	// SetCategoricalPairProfile.
	SetCategoricalPairs []SetCategoricalPairProfile `json:"set_categorical_pairs,omitempty"`
	// SetNumericPairs lists the set-option x numeric pairs captured
	// (E5-S2): one conditional mean/std pair (selected vs. not_selected)
	// per (set field, option, numeric field) combination. See
	// SetNumericPairProfile.
	SetNumericPairs []SetNumericPairProfile `json:"set_numeric_pairs,omitempty"`
	// SetSetPairs lists the option x option pairs captured between two
	// DIFFERENT set_* fields (E5-S2): a 2x2 contingency table per
	// (fieldA option, fieldB option) combination. Never captured between
	// two options of the same field. See SetSetPairProfile.
	SetSetPairs []SetSetPairProfile `json:"set_set_pairs,omitempty"`
}

type ConstraintSpec struct {
	Expr string `json:"expr"`
}

type ContingencyCell struct {
	AValue string `json:"a_value"`
	BValue string `json:"b_value"`
	Count  int    `json:"count"`
}

type CorrelationSpec struct {
	A           string  `json:"a"`
	B           string  `json:"b"`
	Correlation float64 `json:"correlation"`
}

type CorrelationStat struct {
	A   string  `json:"a"`
	B   string  `json:"b"`
	Rho float64 `json:"rho"`
}

type DateProfile struct {
	Start    string `json:"start"`
	End      string `json:"end"`
	Weekdays [7]int `json:"weekdays"`
}

type DiscreteLevel struct {
	Value     float64 `json:"value"`
	Count     int     `json:"count"`
	Frequency float64 `json:"frequency"`
}

type DiscreteProfile struct {
	Levels []DiscreteLevel `json:"levels"`
}

type FidelityReport struct {
	SourceRows                 int                               `json:"source_rows"`
	SyntheticRows              int                               `json:"synthetic_rows"`
	Fields                     []*FieldFidelity                  `json:"fields,omitempty"`
	Pairwise                   []*PairwiseFidelity               `json:"pairwise,omitempty"`
	CategoricalPairwise        []*CategoricalPairFidelity        `json:"categorical_pairwise,omitempty"`
	CategoricalNumericPairwise []*CategoricalNumericPairFidelity `json:"categorical_numeric_pairwise,omitempty"`
	SetFields                  []*SetFieldFidelity               `json:"set_fields,omitempty"`
	SetCategoricalPairwise     []*SetCategoricalPairFidelity     `json:"set_categorical_pairwise,omitempty"`
	SetNumericPairwise         []*SetNumericPairFidelity         `json:"set_numeric_pairwise,omitempty"`
	SetSetPairwise             []*SetSetPairFidelity             `json:"set_set_pairwise,omitempty"`
	// Models is the per-field MODEL-RECOVERY section (E5-S1): for every
	// linear model generation actually applied, the captured coefficients
	// beside the ones a refit on the generated partition recovers. It
	// answers a different question from every section above — those ask
	// whether the generated rows LOOK like the source, this asks whether
	// the captured STRUCTURE survived generation — and it is the only
	// section that can tell a faithful generation whose marginal
	// contrasts are merely compressed apart from one that silently lost
	// its conditioning. Absent (omitempty) for every spec carrying no
	// `models`, which is every spec predating `profile create
	// --fit-models`. See synth/fidelity_models.go.
	Models []*ModelFidelity `json:"models,omitempty"`
	// ModelResidualCorrelations is the residual-correlation half of the
	// same model-recovery question (E5-S2): for every residual
	// correlation generation applied, the captured rho beside the rho
	// recovered from the generated partition. It is deliberately NOT
	// folded into Pairwise, which scores the value-scale copula arm over
	// a DISJOINT set of fields — a modelled field is excluded from
	// Spec.Correlations by resolveConflicts precisely so the two arms
	// never both fire on one field. The two section names are what tell
	// a reader scanning this document which mechanism a given number
	// describes. Absent (omitempty) for every spec carrying no
	// `residual_correlations`, which is every spec predating `profile
	// create --residual-correlations`. See synth/fidelity_residual.go.
	ModelResidualCorrelations *ModelResidualCorrelationFidelity `json:"model_residual_correlations,omitempty"`
	Warnings                  []string                          `json:"warnings,omitempty"`
}

type FieldContinuation struct {
	Name string  `json:"name"`
	Rate float64 `json:"rate"`
}

type FieldFidelity struct {
	Field  string            `json:"field"`
	Test   types.TestType    `json:"test"`
	Result *types.TestResult `json:"result,omitempty"`
	Error  string            `json:"error,omitempty"`
}

type FieldModel struct {
	// Field is the numeric target this model predicts.
	Field string `json:"field"`
	// Intercept is the fitted constant term: the predicted value for a
	// row sitting at every categorical's reference level (see
	// References) with no set option selected.
	Intercept float64 `json:"intercept"`
	// Predictors lists the fitted coefficients in design-column order.
	// Empty is legal and meaningful only in the sense that such a model
	// is never retained — a target with no usable candidate columns is
	// skipped, not stored with an empty predictor list.
	Predictors []ModelPredictor `json:"predictors"`
	// References names, per categorical predictor field, the level
	// dropped from Predictors and folded into Intercept. Absent
	// (omitempty) exactly when the model carries no categorical
	// predictor at all — a set-only model has nothing to drop, since a
	// multi-select is not a partition — which a reader can confirm from
	// predictors[].kind rather than having to assume. See ModelReference
	// for why the baseline is written down instead of inferred.
	References []ModelReference `json:"references,omitempty"`
	// NObs is the number of rows that actually contributed to the fit.
	// The engine applies listwise deletion, so a row with a null target
	// or a null predictor SOURCE field contributes nothing and this is
	// below the profile's RowCount whenever the cohort has nulls.
	NObs int `json:"n_obs"`
	// R2 and ResidualStd are the fit's explanatory power and residual
	// scale. ResidualStd is the standard deviation of the residual that
	// generation adds back on top of the linear predictor.
	R2          float64 `json:"r2"`
	ResidualStd float64 `json:"residual_std"`
	// ShrinkageAlpha is the ridge penalty this model was fitted under,
	// absent (omitempty, and therefore zero) when it was fitted by plain
	// unpenalized OLS.
	//
	// It is on the wire because a shrunk coefficient and a free one are
	// not the same kind of number, and nothing else in the document says
	// which was produced. A reader comparing two models' coefficients —
	// or comparing one model against a hand-run regression — needs to
	// know that at least one of this model's levels was thin enough to
	// pull the whole design toward its baselines, and by how much: the
	// penalty is minLevelObservations pseudo-observations spread over
	// NObs rows, so alpha·NObs recovers the pseudo-count. See
	// profile_models_shrink.go for the construction.
	ShrinkageAlpha float64 `json:"shrinkage_alpha,omitempty"`
	// Residuals is the row-aligned fitted residual (observed − predicted)
	// for each row retained in the capture's residual reservoir, and
	// ResidualPresent[i] reports whether row i contributed one at all —
	// a row listwise-deleted for THIS field has no residual even though
	// another field's model may have one for the same row.
	//
	// Every FieldModel from one capture indexes the SAME retained rows,
	// which is the whole point: residual correlation between two fields
	// is only meaningful on rows where both residuals exist.
	//
	// Both are `json:"-"` — DELIBERATELY not part of the document. They
	// are a bounded row-aligned SAMPLE (modelResidualCap entries per
	// model, per field), so serialising them would add tens of thousands
	// of numbers per numeric column to a document whose every other
	// section is a handful of summary statistics, and would do it to
	// carry rows a profile exists precisely to avoid retaining. Their
	// consumer is the residual-correlation measurement, which runs in
	// the SAME process as the capture and reads them straight off
	// FittedModels(); what belongs in the document is the small derived
	// quantity that measurement produces, not its input. A Profile read
	// back from JSON therefore has coefficients and ResidualStd but nil
	// Residuals, and any consumer that needs per-row residuals must
	// capture rather than re-read.
	Residuals       []float64 `json:"-"`
	ResidualPresent []bool    `json:"-"`
}

type FieldModelSpec struct {
	// Field is the numeric field this model draws.
	Field string `json:"field"`
	// Intercept is the constant term — the prediction for a row sitting
	// at every categorical predictor's reference level with no set
	// option selected.
	Intercept float64 `json:"intercept"`
	// Predictors are the per-(field, level) contributions summed into
	// the prediction. A row whose drawn value for a predictor's Field is
	// not that predictor's Level contributes nothing for it.
	Predictors []ModelPredictorSpec `json:"predictors"`
	// ResidualStd is the scale of the residual added on top of the
	// linear prediction. Zero means a deterministic prediction, which is
	// a legal (if degenerate) model rather than an error.
	ResidualStd float64 `json:"residual_std"`
	// Min/Max are the numeric field's observed bounds, carried alongside
	// HasClamp exactly as CategoricalNumericPairSpec and
	// SetNumericPairSpec already carry them. They are the same bounds
	// the field's own reconstructed marginal would clamp to; whether and
	// where the draw applies them is the generation stage's decision,
	// and this slot only guarantees the numbers are available to it
	// without a second lookup into the profile.
	Min      float64 `json:"min,omitempty"`
	Max      float64 `json:"max,omitempty"`
	HasClamp bool    `json:"has_clamp,omitempty"`
}

type FieldProfile struct {
	Name        string              `json:"name"`
	Type        string              `json:"type"`
	Description string              `json:"description,omitempty"`
	NullRate    float64             `json:"null_rate"`
	Numeric     *NumericProfile     `json:"numeric,omitempty"`
	Categorical *CategoricalProfile `json:"categorical,omitempty"`
	Date        *DateProfile        `json:"date,omitempty"`
	// Set carries per-option marginal selection-frequency stats for a
	// set_* (multi-select bitmask) field (E5-S1). Populated instead of
	// Numeric/Categorical/Date — exactly one of the four is non-nil per
	// field, based on the field's schema type.
	Set *SetProfile `json:"set,omitempty"`
	// Precision/Scale carry decimal128 metadata so synth-from-profile can
	// reconstruct the original field shape.
	Precision uint8 `json:"precision,omitempty"`
	Scale     uint8 `json:"scale,omitempty"`
}

type FieldSpec struct {
	Name         string         `json:"name"`
	Type         string         `json:"type"`
	Nullable     bool           `json:"nullable,omitempty"`
	Description  string         `json:"description,omitempty"`
	Distribution string         `json:"distribution"`
	Params       map[string]any `json:"params,omitempty"`

	// Precision and Scale apply to decimal128.
	Precision uint8 `json:"precision,omitempty"`
	Scale     uint8 `json:"scale,omitempty"`

	// NullRate is the per-row probability that the field will be null.
	// Only meaningful when Nullable is true; ignored otherwise.
	NullRate float64 `json:"null_rate,omitempty"`
}

type ModelFidelity struct {
	Field string `json:"field"`
	// Marginal marks an INTERCEPT-ONLY model — one whose predictor
	// selection admitted nothing (see profile_models_select.go's variance
	// floor). It is a COMPLETE model, not a failed capture: the linear
	// predictor is the field's own mean and the residual carries all of
	// its spread, so generation reproduces the marginal, which is the
	// correct answer for a field nothing explains. Such an entry carries
	// no Predictors and reports only the intercept comparison — which is
	// still a real check, since a drifted intercept on a marginal model
	// means the marginal itself did not survive.
	Marginal bool `json:"marginal,omitempty"`
	// Scale names how this entry's comparison was put on the latent
	// scale. Empty — the ordinary case — means the generated value was
	// inverted through latentFor, a point inverse, and the recovered
	// coefficient is a direct estimate. RecoveryScaleProbitScore means
	// the target's Q is a step or staircase with no point inverse, so
	// the recovery ran through the CALIBRATED interval-midpoint probit
	// score (synth/fidelity_score.go): still latent-scale and still
	// unbiased, but divided by a retention factor that widens the
	// standard error, so the same numeric gap carries less evidence.
	// A score-scale entry reports no intercept comparison — see
	// RecoveredIntercept.
	Scale string `json:"scale,omitempty"`
	// LatentScale is the target's reconstructed marginal standard
	// deviation — the divisor that puts every captured coefficient below
	// on the latent scale. Multiply a latent figure by it to return to
	// the units the profile document's `models` section reports, which
	// for a `normal` target are the same units either way.
	LatentScale float64 `json:"latent_scale"`
	// CapturedIntercept / RecoveredIntercept are the latent-scale
	// intercepts: (Intercept − mean)/std as generation standardises it,
	// against the refit's own. InterceptDelta is their absolute
	// difference.
	//
	// On a Scale == RecoveryScaleProbitScore entry the recovered
	// intercept is ABSENT and InterceptDelta with it. A staircase's cut
	// points come from the captured MARGINAL, which generation holds
	// exactly by construction, so the score-scale intercept is pinned by
	// the marginal rather than by the model and carries no independent
	// information about whether the coefficients were applied. Reporting
	// a calibrated one would put a number where there is no measurement;
	// the captured figure still ships, because the reader needs to see
	// what the model declared.
	CapturedIntercept  float64 `json:"captured_intercept"`
	RecoveredIntercept float64 `json:"recovered_intercept,omitempty"`
	InterceptDelta     float64 `json:"intercept_delta,omitempty"`
	// Predictors is one entry per compiled model term, in the order
	// generation evaluates them. Absent on a Marginal model.
	Predictors []*ModelPredictorFidelity `json:"predictors,omitempty"`
	// NObs is the number of synthetic rows the refit admitted: rows
	// carrying a non-null, latent-invertible target and a non-null value
	// for every predictor SOURCE field (listwise deletion, the same rule
	// capture's fit applied). Bounded by modelRecoveryRowCap.
	NObs int `json:"n_obs"`
	// R2 is the refit's own coefficient of determination on the latent
	// scale — how much of the generated latent the recovered model
	// explains. It is NOT comparable to the captured FieldModel.R2, which
	// was measured on real source rows against real source noise;
	// synthetic residuals are Gaussian by construction, so this figure
	// reads high and is here to characterise the refit, not to score it.
	R2 float64 `json:"r2,omitempty"`
	// MaxDelta is the largest absolute latent coefficient gap across
	// Predictors (the intercept excluded — a shifted intercept moves the
	// level, not the structure). Flagged is true when any predictor is.
	MaxDelta float64 `json:"max_delta,omitempty"`
	Flagged  bool    `json:"flagged,omitempty"`
	// Error is set instead of the recovered figures when the refit could
	// not run at all for this model — too few admitted rows, a design the
	// solver refused. It never withholds the rest of the report, the same
	// non-fatal per-entry contract every other Fidelity kind follows.
	Error string `json:"error,omitempty"`
}

type ModelPredictor struct {
	// Column is the design-matrix column name the engine keyed the
	// coefficient by. Internal encoding; diagnostics and the in-process
	// residual replay only, never serialised — see the type comment.
	Column string `json:"-"`
	// Kind is one of the ModelPredictor* constants above. It is carried
	// on the wire because it is not derivable from (Field, Level): a
	// categorical level is one arm of a partition read against a dropped
	// reference (see FieldModel.References), while a set option is an
	// independent indicator with no reference at all, and a consumer
	// reconstructing effects has to know which it is holding.
	Kind string `json:"kind"`
	// Field is the real cohort field this column was expanded from.
	Field string `json:"field"`
	// Level is the categorical level text or set option text this column
	// indicates. Empty when Kind is ModelPredictorNumeric. Deliberately
	// NOT omitempty: it is half of the addressing key, and a categorical
	// dictionary may legitimately carry the empty-string level, so the
	// key is always written rather than left to be inferred from its
	// absence.
	Level string `json:"level"`
	// Coefficient is the fitted slope for this column.
	Coefficient float64 `json:"coefficient"`
}

type ModelPredictorFidelity struct {
	// Kind / Field / Level address the term exactly as
	// ModelPredictorSpec does, so an entry here zips against the profile
	// document's `models[].predictors[]` without a join key.
	Kind  string `json:"kind"`
	Field string `json:"field"`
	Level string `json:"level"`
	// CapturedCoefficient is the spec's coefficient divided by
	// LatentScale — the quantity buildModelDrawers actually adds into the
	// latent, not the raw number the profile document carries.
	CapturedCoefficient float64 `json:"captured_coefficient"`
	// RecoveredCoefficient is the refit's estimate for the same term.
	// StdError is the refit's own standard error for it, which is the
	// second half of the flagging band — see ModelRecoveryTolerance.
	RecoveredCoefficient float64 `json:"recovered_coefficient,omitempty"`
	StdError             float64 `json:"std_error,omitempty"`
	Delta                float64 `json:"delta,omitempty"`
	// ScoreRetention is the fraction of this term's captured latent
	// effect that survives onto the interval-midpoint probit score,
	// present only on a Scale == RecoveryScaleProbitScore entry and
	// absent (0) otherwise.
	//
	// It is computed, not estimated: the same score is projected onto
	// the same design once from the generated values and once from the
	// conditional mean the CAPTURED model implies, and their ratio is
	// this number. Both the recovered coefficient and its standard error
	// have been divided by it, so a small retention means a wide band
	// rather than a suspect estimate — which is the whole reason it is
	// on the wire. Two levels retain the least of any K; a long scale
	// retains the most.
	ScoreRetention float64 `json:"score_retention,omitempty"`
	// NFired is the number of admitted synthetic rows on which this
	// term's indicator was 1. It is the term's own support and it is what
	// distinguishes the two ways a recovered coefficient reaches zero: a
	// term that fired thousands of times and recovered nothing is a
	// generation fault, while a term that never fired had nothing to
	// recover — the level was captured but the field's reconstructed
	// marginal does not generate it.
	NFired int `json:"n_fired"`
	// Flagged is true when Delta exceeds both ModelRecoveryTolerance and
	// modelRecoverySEMultiple standard errors. False whenever Error is
	// set: a term that was not estimated has no gap to judge.
	Flagged bool `json:"flagged,omitempty"`
	// Error is set instead of the recovered figures when this term could
	// not be carried into the refit at all — its level is absent from the
	// output cohort's dictionary, or its column is constant over the
	// admitted rows and so carries no information to estimate from. Both
	// are informative outcomes rather than failures, and both leave every
	// OTHER term of the same model estimated.
	Error string `json:"error,omitempty"`
}

type ModelPredictorSpec struct {
	// Kind is one of the ModelPredictor* constants
	// (ModelPredictorCategoricalLevel / ModelPredictorSetOption /
	// ModelPredictorNumeric).
	Kind string `json:"kind"`
	// Field is the predictor's source field.
	Field string `json:"field"`
	// Level is the categorical level text or set option text that turns
	// this contribution on.
	Level string `json:"level"`
	// Coefficient is added to the prediction when the indicator is 1.
	Coefficient float64 `json:"coefficient"`
}

type ModelReference struct {
	// Field is the categorical predictor source field.
	Field string `json:"field"`
	// Level is the level whose effect is folded into the intercept. A
	// row carrying it contributes nothing beyond the intercept for this
	// field; every other level's coefficient is read relative to it.
	Level string `json:"level"`
}

type ModelResidualCorrelationFidelity struct {
	// Fields names the participants in COMPONENT order — the order
	// buildResidualCorrelator assigned them, which is drawer order,
	// which is schema field order. It is the generation-side
	// participant set, so a modelled field absent from it took no part
	// in the correlated draw (its model was dropped, or no surviving
	// correlation named it) and drew an independent residual.
	Fields []string `json:"fields"`
	// Compared is the number of applied residual correlations that
	// produced a recovered figure. Flagged counts those whose Delta
	// exceeds ResidualRecoveryTolerance. Both count over every compared
	// pair, not over the bounded Pairs listing.
	Compared int `json:"compared"`
	Flagged  int `json:"flagged,omitempty"`
	// MeanDelta / MaxDelta are the mean and maximum absolute gap over
	// every compared pair — the two figures that make the truncated
	// listing safe to read, since they cannot be improved by what the
	// listing left out.
	MeanDelta float64 `json:"mean_delta"`
	MaxDelta  float64 `json:"max_delta"`
	// Pairs is the worst maxResidualRecoveryPairs compared pairs by
	// absolute Delta, largest first, ties broken by (A, B) so the
	// listing is a function of the numbers alone. Omitted is how many
	// compared pairs it left out; every one of them recovered at least
	// as well as the last entry listed.
	Pairs   []*ModelResidualPairFidelity `json:"pairs,omitempty"`
	Omitted int                          `json:"omitted,omitempty"`
	// Unmeasured holds pairs generation applied that the synthetic
	// partition could not produce a correlation for, each with its
	// reason, capped the same way. It is a separate list rather than an
	// Error field on ModelResidualPairFidelity for the same structural
	// reason ResidualCorrelationProfile splits its two lists: an
	// unmeasured pair has no rho, and a struct with a rho slot in it is
	// how a zero gets written into one.
	Unmeasured        []*ModelResidualUnmeasuredPair `json:"unmeasured,omitempty"`
	UnmeasuredOmitted int                            `json:"unmeasured_omitted,omitempty"`
}

type ModelResidualPairFidelity struct {
	A string `json:"a"`
	B string `json:"b"`
	// CapturedRho is CorrelationSpec.Correlation — the figure
	// buildResidualCorrelator was asked to impose, which is the source
	// residual correlation the profile captured. RecoveredRho is the
	// Pearson correlation between the two models' REFIT residuals over
	// the synthetic partition, on the latent scale. Delta is their
	// absolute difference.
	CapturedRho  float64 `json:"captured_rho"`
	RecoveredRho float64 `json:"recovered_rho"`
	Delta        float64 `json:"delta"`
	// N is the number of synthetic rows carrying BOTH residuals — the
	// realized pair's own co-occurrence count within the refit row cap,
	// not the cohort's row count and not either model's NObs.
	N int `json:"n"`
	// Flagged is true when Delta exceeds ResidualRecoveryTolerance.
	Flagged bool `json:"flagged,omitempty"`
}

type ModelResidualUnmeasuredPair struct {
	A string `json:"a"`
	B string `json:"b"`
	// CapturedRho still rides, because the reader needs to know how much
	// structure went unverified: an unmeasured pair captured at 0.8 is a
	// far larger hole in the report than one captured at 0.02.
	CapturedRho float64 `json:"captured_rho"`
	// N is how many synthetic rows DID carry both residuals — zero or a
	// handful for ResidualUnmeasuredNoOverlap, and however many
	// overlapped for ResidualUnmeasuredNoVariance. Zero and meaningless
	// for ResidualUnmeasuredNoModelFit, where one side has no residual
	// vector at all.
	N int `json:"n"`
	// Reason is ResidualUnmeasuredNoOverlap, ResidualUnmeasuredNoVariance
	// or ResidualUnmeasuredNoModelFit.
	Reason string `json:"reason"`
}

type NumericPairProfile struct {
	A   string  `json:"a"`
	B   string  `json:"b"`
	Rho float64 `json:"rho"`
	// N is the number of rows in the capture where both A and B were
	// simultaneously non-null — the pair's true co-occurrence count,
	// unlike Profile.Pairwise's Rho (computed from two independently
	// capped reservoirs that can drift out of alignment once either
	// field has nulls).
	N int `json:"n"`
}

type NumericProfile struct {
	Min  float64 `json:"min"`
	Max  float64 `json:"max"`
	Mean float64 `json:"mean"`
	Std  float64 `json:"std"`
	// Percentiles holds {p1, p5, p25, p50, p75, p95, p99} when
	// IncludeStats was on; nil otherwise.
	Percentiles []float64 `json:"percentiles,omitempty"`
	// Shape carries a fitted 2-component Gaussian mixture, captured
	// only when ProfileOptions.FitShape was set AND the mixture is a
	// genuine improvement over the plain Mean/Std normal above (BIC
	// selection — see fitNumericShape in shape.go). Additive +
	// omitempty: absent from every profile document captured without
	// --fit-shape (including every pre-existing document), and absent
	// for any numeric field whose observed distribution is already
	// close to normal — SpecFromProfile falls back to the ordinary
	// Mean/Std/Min/Max normal reconstruction whenever this is nil,
	// exactly as it always has.
	Shape *ShapeProfile `json:"shape,omitempty"`
	// Discrete carries the EXACT per-level histogram of an
	// integer-quantized column (u4/u8/u16/u32/u64 — see
	// isIntegerQuantizedFieldType), captured unconditionally on the same
	// single scan and present whenever the column carried at most
	// maxDiscreteLevels distinct values.
	//
	// Additive + omitempty, so a document captured before this key
	// existed is unchanged and SpecFromProfile's clamped-normal fallback
	// still applies to it verbatim — the same silent-fallback shape
	// Shape and conditional.numeric_pairs already use.
	//
	// The key's ABSENCE on an integer field is itself the record that the
	// column was too wide and the histogram was ABANDONED rather than
	// truncated: presence/absence is exactly the "which reconstruction
	// will this field get" answer, so nothing is hidden by there being no
	// warning for it. (Structural rather than conventional, for the same
	// reason ResidualCorrelationProfile keeps `unmeasured` out of
	// `pairs`.)
	//
	// It sits BESIDE Min/Max/Mean/Std/Percentiles and replaces none of
	// them: they remain the field's summary for every reader, and
	// reconstructing from the histogram recomputes its own exact moments
	// rather than trusting these (see parseDiscreteLevels).
	Discrete *DiscreteProfile `json:"discrete,omitempty"`
}

type Options struct {
	// Seed makes the output deterministic. Same spec + same seed must
	// produce a byte-identical .pulse file.
	Seed int64

	// SourceCohort activates the tagged top-up path (AugmentFromProfile)
	// when non-empty: real rows from this cohort are copied into the
	// output tagged _synthetic=false, spec.RowCount new rows are
	// generated and tagged _synthetic=true, and the source is never
	// opened for write. Leaving it empty (the default) reproduces
	// today's behavior exactly — a plain synthesis with no tag column,
	// used by synth from-schema and any caller building a Spec directly.
	SourceCohort string

	// FidelityReportPath activates the post-generation fidelity report
	// (`synth from-profile --fidelity-report`) when non-empty: after
	// generation completes, AugmentFromProfile drives TEST_KS (numeric
	// fields, SplitBy=_synthetic) / TEST_CHISQ (categorical fields,
	// contingency against _synthetic) against the tagged output cohort
	// and writes the resulting FidelityReport JSON document to this
	// path. Only meaningful alongside SourceCohort — the plain Synth
	// path (SourceCohort empty) has no _synthetic partition to compare
	// against and ignores this field entirely.
	FidelityReportPath string

	// FidelityWarnings, when non-empty, are copied verbatim onto the
	// written FidelityReport's own Warnings slot — the mechanism that
	// surfaces thin-pair warnings (e.g. Profile.Warnings from a
	// `profile create --conditional` capture) beside the pairwise
	// correlation deltas they qualify in the same document (FR-18),
	// rather than requiring a caller to cross-reference the profile
	// file separately. This package has no Profile type of its own to
	// read Warnings from directly — the caller (internal/cli's `synth
	// from-profile`, or any programmatic caller building Options from a
	// Profile it already has in hand) is expected to populate this.
	// Ignored when FidelityReportPath is empty, exactly like every
	// other Fidelity* field.
	FidelityWarnings []string
}

type PairwiseFidelity struct {
	A            string  `json:"a"`
	B            string  `json:"b"`
	SourceRho    float64 `json:"source_rho"`
	SyntheticRho float64 `json:"synthetic_rho,omitempty"`
	// Delta is abs(SourceRho - SyntheticRho). Zero-valued and
	// meaningless when Error is set.
	Delta float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with both A and B
	// simultaneously non-null — the realized pair's own co-occurrence
	// count, distinct from (and typically much larger than) the source
	// capture's own N.
	N int `json:"n"`
	// Error is set instead of SyntheticRho/Delta when the synthetic
	// partition could not produce a defined correlation for this pair
	// (fewer than two co-occurring non-null observations) — mirrors
	// FieldFidelity's own per-field failure contract: one pair's
	// failure never withholds the rest of the report.
	Error string `json:"error,omitempty"`
}

type Profile struct {
	RowCount int               `json:"row_count"`
	Fields   []FieldProfile    `json:"fields"`
	Pairwise []CorrelationStat `json:"pairwise,omitempty"`
	// Conditional carries joint-reconstruction structure captured only
	// when ProfileOptions.IncludeConditional was set. Additive and
	// omitempty: absent on every profile document captured without
	// --conditional, including every pre-existing one, and
	// SpecFromProfile treats a nil Conditional exactly as it always
	// has (falling back to Pairwise).
	Conditional *ConditionalProfile `json:"conditional,omitempty"`
	// Models carries the per-numeric additive linear predictors captured
	// only when ProfileOptions.FitModels was set (`profile create
	// --fit-models`). Additive and omitempty, exactly like Conditional:
	// the key is entirely absent from every document captured without
	// the flag — including every document written before this section
	// existed — and SpecFromProfile treats an empty Models exactly as it
	// always has, falling back to Conditional's numeric-target pairs.
	//
	// The section is a REPLACEMENT for those pairs, not a supplement:
	// when it is present SpecFromProfile stops populating
	// Spec.CategoricalNumericPairs / Spec.SetNumericPairs entirely,
	// because one additive model already accounts for every predictor at
	// once and the pairs exist only to be applied one overwrite at a
	// time. See SpecFromProfile for the full rule.
	//
	// Not every part of a captured FieldModel is durable: the residual
	// reservoir and the internal design-column names are `json:"-"`, so
	// a Profile read back from JSON carries coefficients but no
	// per-row residuals. See FieldModel for why.
	Models []FieldModel `json:"models,omitempty"`
	// ResidualCorrelations carries the correlation structure among the
	// fitted model residuals, captured only when
	// ProfileOptions.FitResidualCorrelations was set (`profile create
	// --residual-correlations`) alongside --fit-models. Additive and
	// omitempty, exactly like Conditional and Models.
	//
	// It is a SEPARATE measurement from Pairwise and
	// Conditional.NumericPairs, not a refinement of either: those two
	// correlate raw values and this correlates what is left after each
	// field's model has explained what it can. All three can coexist in
	// one document, and none of them replaces another.
	//
	// SpecFromProfile translates the MEASURED pairs onto
	// Spec.ResidualCorrelations, where generation consumes them as the
	// shared source of randomness every modelled field's residual is
	// drawn from (synth/residual_draw.go). The `unmeasured` list is
	// translated into nothing at all: an absent pair is completed as
	// independent by the generator, which counts and names the
	// assumption, whereas writing it out as a zero would present it as
	// a measurement. See ResidualCorrelationProfile.
	ResidualCorrelations *ResidualCorrelationProfile `json:"residual_correlations,omitempty"`
	// RunContinuation carries the measured per-field run-continuation,
	// captured only when ProfileOptions.RunContinuation was set. Additive
	// and omitempty; SpecFromProfile never reads it. See
	// RunContinuationProfile.
	RunContinuation *RunContinuationProfile `json:"run_continuation,omitempty"`
	Warnings        []string                `json:"warnings,omitempty"`
	Meta            map[string]any          `json:"meta,omitempty"`

	// RuleCandidates carries the structural rules
	// ProfileOptions.SuggestRules proposed, in the exact shape
	// `synth from-profile --rules` consumes. It is `json:"-"` ON
	// PURPOSE: these are a PROPOSAL for a human to read, edit and feed
	// back, not a captured statistic, and writing them into the profile
	// document would put an unreviewed structural claim inside the
	// document that drives generation. It matches FieldModel.Residuals'
	// precedent: a capture product whose only consumer runs in the same
	// process, absent from a document read back from disk.
	//
	// The CLI writes them to the `--suggest-rules` path via
	// WriteRuleCandidates.
	RuleCandidates []RuleSpec `json:"-"`
}

type ProfileOptions struct {
	// TopK is the number of top categorical values to capture per
	// categorical field. Defaults to 32 when zero.
	TopK int
	// IncludeStats turns on percentile / stdev / kurtosis collection.
	// When false, only mean / min / max / null-rate are recorded for
	// numeric fields. Defaults to true.
	IncludeStats bool
	// IncludeCorrelations enables pairwise Pearson correlation capture
	// between numeric fields. Off by default to keep profile size bounded.
	IncludeCorrelations bool
	// CorrelationTopK caps the number of strongest |rho| pairs retained.
	// Defaults to 16 when IncludeCorrelations or IncludeConditional is
	// true.
	CorrelationTopK int
	// IncludeConditional enables capture of the joint reconstruction
	// structure (`profile create --conditional`): for numeric-numeric
	// field pairs, a row-aligned correlation + observation count
	// (Profile.Conditional.NumericPairs) that SpecFromProfile uses to
	// drive the exact conditional-Gaussian reconstruction in
	// synth/copula.go, in place of the plain (and, for cohorts with
	// nulls, only approximately aligned) Pairwise correlation stats.
	// Off by default; the profile document's `conditional` key is
	// entirely absent (omitempty) when this is false, and every
	// pre-existing profile document — which has no such key — remains
	// valid SpecFromProfile input either way.
	IncludeConditional bool
	// SampleLimit caps the number of records ingested for the profile.
	// Zero = unlimited.
	SampleLimit int
	// FitShape enables shape-fitting for numeric fields
	// (`profile create --fit-shape`, E4-S2): instead of always
	// summarizing a numeric field as a single normal(mean, std),
	// attempt a 2-component Gaussian mixture fit and keep it — as
	// FieldProfile.Numeric.Shape — only when it is a genuine
	// improvement over the plain normal by BIC (see fitNumericShape in
	// shape.go for the full selection rule and its documented
	// limitations). Off by default; a field whose fit is not kept (not
	// enough data, or the source is already close to normal) carries no
	// Shape, and SpecFromProfile reconstructs it exactly as it always
	// has. Forces sample retention (as if IncludeStats were also set)
	// for numeric fields so there is something to fit against, even
	// when IncludeStats itself is false.
	FitShape bool
	// FitModels enables per-numeric linear model capture
	// (`profile create --fit-models`, E1-S2): for each numeric field,
	// fit one linear predictor of that field on the cohort's
	// categorical LEVELS and set OPTIONS, driven off this same single
	// scan, and retain the fitted coefficients, the residual scale and
	// a bounded row-aligned residual vector.
	//
	// Off by default. When on, the fits are written to the document as
	// the additive `models` section (Profile.Models) and nothing else in
	// the document moves: every other section is byte-identical to the
	// same capture with the flag off, and a capture WITHOUT the flag
	// emits no `models` key at all. It draws from its own RNG stream, so
	// --conditional's captured output does not depend on whether this
	// flag was also passed.
	//
	// It does change what SpecFromProfile builds: a document carrying
	// `models` reconstructs its numeric fields from the additive linear
	// predictors instead of from --conditional's one-pair-at-a-time
	// numeric-target arms, which are then not populated at all.
	FitModels bool
	// FitResidualCorrelations enables capture of the correlation
	// structure among the fitted model RESIDUALS
	// (`profile create --residual-correlations`, E3-S1), written as the
	// additive `residual_correlations` section
	// (Profile.ResidualCorrelations).
	//
	// It requires FitModels — residuals are what the models produce, so
	// with no models there is nothing to correlate, and asking for this
	// alone warns rather than silently emitting nothing.
	//
	// It is deliberately its own flag rather than an implication of
	// FitModels: the section is QUADRATIC in modelled fields (105
	// targets is 5,460 pairs) and no existing caller asked for it, so
	// switching it on with --fit-models would grow every such document
	// without notice. It is likewise deliberately NOT wired to
	// IncludeCorrelations / CorrelationTopK, which keep their exact
	// meaning — the strongest top-K pairs of RAW values. This section is
	// a different measurement (residuals, not raw values) with a
	// different shape (a full submatrix, not a ranked list), and sharing
	// either flag would silently change what those two already mean.
	FitResidualCorrelations bool
	// SuggestRules enables the structural-rule DETECTION pass
	// (`profile create --suggest-rules <path>`): for every
	// low-cardinality field and every other field, measure
	// P(target null | field = level) off the same single scan and
	// propose a rule for each field whose levels split a target's null
	// rate into ~1 and ~0.
	//
	// Off by default, and ABSENT THE FLAG the profile document is
	// byte-identical — detection is a proposal pass, not a capture.
	//
	// WITH the flag the document gains no SECTION: the candidates reach
	// the caller on Profile.RuleCandidates, which is `json:"-"`, because
	// an unreviewed structural claim must not ride inside the document
	// that drives generation. The one thing that does move is
	// `warnings`, which gains the detector's own lines (what it could
	// not consider, what it counted rather than proposed, which
	// candidates rest on thin levels) — deliberately, so those survive
	// in a document rather than only on a terminal, which is how this
	// package's last three silent defects survived. Nothing here is read
	// back by SpecFromProfile and no generated byte moves;
	// TestSuggestRules_DocumentMovesOnlyItsWarnings pins exactly that
	// boundary. The candidates are written as a standalone rules file by
	// the CLI (see WriteRuleCandidates) for the analyst to edit and feed
	// to `synth from-profile --rules`.
	SuggestRules bool

	// RunContinuation enables the run-continuation measurement
	// (`profile create --run-continuation`): per field, the fraction of
	// adjacent row pairs whose on-wire bytes and null bit repeat — the
	// exact hit rate of the run-skip decode — written as the additive
	// `run_continuation` section (Profile.RunContinuation).
	//
	// Off by default and ABSENT THE FLAG the document is byte-identical
	// and the scan pays nothing: the whole-row read that feeds it is only
	// installed when the flag is set. With it, the measurement rides the
	// existing scan (TestRunContinuation_RidesTheExistingScan counts the
	// bytes pulled). SampleLimit bounds it like every other section.
	RunContinuation bool

	// Seed drives the reservoir-sampling RNG used by
	// IncludeConditional's categorical-categorical contingency capture
	// (E7-S2) once the source cohort exceeds conditionalJointCap rows.
	// Zero (the default) is itself a valid, deterministic seed — same
	// as synth.Options.Seed's convention — so two profile runs against
	// the same source with the same Seed (including the unset zero
	// value) produce byte-identical captured output. It does not affect
	// any other capture path: every other section in this file is
	// either an exact online accumulation or a first-N reservoir with
	// no random draws.
	Seed int64
}

type ResidualCorrelation struct {
	A string `json:"a"`
	B string `json:"b"`
	// Rho is the Pearson correlation between the two fields' fitted
	// residuals over the rows where both exist. Deliberately NOT
	// omitempty: a measured zero must appear on the wire, because
	// omitting it would make it indistinguishable from an unmeasured
	// pair on read-back and would reintroduce exactly the ambiguity
	// this section removes.
	Rho float64 `json:"rho"`
	// N is the number of retained rows carrying both residuals — the
	// pair's true co-occurrence count within the capture's residual
	// reservoir, not the cohort's row count and not either model's
	// NObs.
	N int `json:"n"`
}

type ResidualCorrelationProfile struct {
	// Fields names the model residuals this submatrix covers, sorted.
	// It is the participant set, and it is written down so a reader can
	// tell an absent pair from an absent FIELD — a field with no model
	// has no residual and was never a candidate, which is a different
	// statement from a modelled pair that could not be measured.
	Fields []string `json:"fields"`
	// Pairs holds every MEASURED pair, in (A, B) sorted order.
	Pairs []ResidualCorrelation `json:"pairs,omitempty"`
	// Unmeasured holds every pair among Fields that Pairs does not, each
	// with its reason. Present whenever there is one; a fully measured
	// submatrix omits the key entirely, which is itself the signal that
	// nothing was assumed.
	Unmeasured []ResidualUnmeasuredPair `json:"unmeasured,omitempty"`
}

type ResidualUnmeasuredPair struct {
	A string `json:"a"`
	B string `json:"b"`
	// N is how many rows DID carry both residuals — zero or a handful
	// for ResidualUnmeasuredNoOverlap, and however many overlapped for
	// ResidualUnmeasuredNoVariance. It is the diagnostic that separates
	// "these two fields never co-occur" from "they co-occur constantly
	// and one of them is fitted exactly".
	N int `json:"n"`
	// Reason is one of the ResidualUnmeasured* constants. There is no
	// rho key on this struct at all — an unmeasured pair has no
	// correlation to report, and providing a slot for one is how a zero
	// gets written into it.
	Reason string `json:"reason"`
}

type Result struct {
	RowsGenerated int      `json:"rows_generated"`
	RowsRejected  int      `json:"rows_rejected"`
	OutputPath    string   `json:"output_path"`
	Warnings      []string `json:"warnings,omitempty"`

	// FidelityReportPath echoes Options.FidelityReportPath when a
	// fidelity report was written alongside the output cohort. Empty
	// when the flag was not set.
	FidelityReportPath string `json:"fidelity_report_path,omitempty"`
}

type RuleEvidence struct {
	// Detector names the measurement that proposed the rule. One value
	// per detector kind, so a reader can tell a gating candidate apart
	// from the co-missing-block and exact-dependency candidates later
	// detectors add to the same file.
	Detector string `json:"detector"`

	// Note is the prose caveat, interpolated with this candidate's own
	// gate. It says what the measurement IS and what it is not, because
	// detection finds the STATISTICAL gate and a human knows the
	// SEMANTIC one, and those are routinely different fields that move
	// together.
	Note string `json:"note"`

	// GateField is the field whose levels split the targets' null rate.
	//
	// This and the four slots below are the GATING detector's own, and
	// carry `omitempty` so a co-missing candidate — which has no gate
	// field, no levels and no targets — does not write five keys spelled
	// like measurements with nothing measured in them. A gating
	// candidate populates every one of them by construction (a gate has
	// at least one level on each side and at least one target), so the
	// wire shape of a gating candidate is unchanged by the tags; the
	// keys that are shared with the co-missing detector deliberately do
	// NOT carry omitempty, because a real 0 there is a real measurement.
	GateField string `json:"gate_field,omitempty"`
	// GatedLevels and OpenLevels are that split, as the level spellings
	// the `when` predicate tests. The null pseudo-level is spelled
	// "(null)" here and reaches the predicate as isnull(field).
	GatedLevels []string `json:"gated_levels,omitempty"`
	OpenLevels  []string `json:"open_levels,omitempty"`

	// Levels carries the per-level support and the mean conditional null
	// rate across this candidate's targets — the shape of the split, at
	// a glance, before a reader descends into Targets.
	Levels []RuleEvidenceLevel `json:"levels,omitempty"`

	// Targets carries the per-target conditional null rates: what the
	// candidate actually measured, one row per field it proposes to
	// null.
	Targets []RuleEvidenceTarget `json:"targets,omitempty"`

	// Block carries the CO-MISSING detector's members: the fields a
	// null_together candidate names, each with the type, null count and
	// null rate that admitted it.
	//
	// It is a slot on the one evidence type rather than a second
	// evidence type because a reader of the candidate file walks one
	// shape and reads `detector` to know which half of it is populated —
	// and because the inertness contract (RuleEvidence's doc, and
	// TestRuleEvidence_IsInert) is a property of the whole struct, so a
	// parallel type would need its own copy of that guarantee and could
	// drift out of it silently.
	//
	// Agreement is each member's null-set overlap with the FIRST member
	// and is 1 for every member of an emitted block, by the admission
	// rule. It is carried anyway, and the constant is the point: it is
	// the detector's central claim — identical PATTERN, not merely an
	// identical rate — stated on the wire where a reader can check it
	// against the null counts beside it, rather than left implicit in
	// the fact that the candidate was emitted at all.
	Block []RuleEvidenceBlockMember `json:"block,omitempty"`

	// SourceField and SourceLevels are the EXACT-DEPENDENCY detector's
	// own: the single field every target of the candidate is a function
	// of, and the number of levels it was observed to take.
	//
	// It is deliberately NOT GateField reused. A gate field's levels
	// decide whether a target is ABSENT; a source field's levels decide
	// what a present target's VALUE is. They are different claims about
	// different halves of the record, and one key carrying both would
	// make `detector` the only thing standing between a reader and the
	// wrong reading.
	SourceField  string `json:"source_field,omitempty"`
	SourceLevels int    `json:"source_levels,omitempty"`

	// Dependency carries the measured lookup, one entry per target: the
	// source level to target value table the expression was READ OFF,
	// and the rendering form chosen.
	//
	// It is a slot on the one evidence type rather than a third evidence
	// type, exactly as Block is, so a reader walks one shape and reads
	// `detector` to know which part of it is populated — and so the
	// inertness contract (this type's own doc, and
	// TestRuleEvidence_IsInert) stays a property of the whole struct
	// rather than something a parallel type could drift out of.
	Dependency []RuleEvidenceDependency `json:"dependency,omitempty"`

	// RowsObserved is the cohort rows the detection saw (every row lands
	// in exactly one level, the null pseudo-level included).
	RowsObserved int `json:"rows_observed"`
	// RowsAffected is the rows the `when` would select — the row count
	// this rule would change. For a co-missing candidate, which carries
	// no `when`, it is the rows on which the block is null. For an
	// exact-dependency candidate, which also carries no `when`, it is
	// the CO-PRESENT rows: the rows the lookup was measured on, which
	// the admission rule (identical null patterns) makes exactly the
	// rows either field is present on.
	RowsAffected int `json:"rows_affected"`
	// GatedShare is RowsAffected / RowsObserved. It is the figure to
	// compare against each target's own null_rate in the profile
	// document: for a real gate the two agree, because a target null on
	// every gated row and present on every open one has exactly the
	// gated share as its marginal null rate. That agreement is how this
	// cohort's gate was first identified by hand.
	GatedShare float64 `json:"gated_share"`
	// MaxNullRateDeviation is the largest |null_rate - GatedShare| over
	// the targets: how far the agreement above falls short.
	//
	// The figure that identified the motivating cohort's gate BY HAND
	// was exactly this agreement — `1 - mean(aware)` = 0.2526093295989762
	// matching 50 fields' captured null_rate to ten digits — and the
	// obvious move is to make it the detector's lead ranking signal. It
	// is carried, and it is deliberately NOT the lead signal, because it
	// is ARITHMETICALLY IMPLIED by the split test rather than
	// independent of it: if P(null | gated) >= gateHighNullRate and
	// P(null | open) <= gateLowNullRate then the marginal null rate is
	// GatedShare x ~1 + (1 - GatedShare) x ~0, so it lies within the
	// thresholds of GatedShare for EVERY candidate the detector admits.
	// Ranking by it would be ranking by rounding error, and
	// TestSuggestRules_DeviationIsImpliedByTheSplitTest pins that.
	//
	// What it is genuinely for is CHECKING WITHOUT RE-MEASURING. A
	// reader holding the profile document compares GatedShare against
	// the targets' own null_rate, which is how the relationship was
	// found in the first place; and a deviation that is unexpectedly
	// large — several targets sitting near rather than at the open
	// boundary — is the signal that a candidate is association wearing
	// a gate's clothes.
	MaxNullRateDeviation float64 `json:"max_null_rate_deviation"`

	// MinLevelSupport is the thinnest level's row count, and ThinSupport
	// says whether it fell below minGateLevelSupport. A thin candidate
	// still SHIPS — suppressing it would hide the finding rather than
	// qualify it — so this is the qualification.
	//
	// A co-missing candidate has two arms rather than levels — the rows
	// where the block is null and the rows where it is present — and
	// this is the thinner of them, which asks the same question of the
	// same threshold: a block observed present on four rows is not a
	// measurement of a question block. An exact-dependency candidate's
	// is the thinnest SOURCE level: a mapping arm resting on four rows
	// is a coincidence, not a derivation.
	MinLevelSupport int  `json:"min_level_support"`
	ThinSupport     bool `json:"thin_support,omitempty"`
}

type RuleEvidenceBlockMember struct {
	Field string `json:"field"`
	// Type is the field's `.pulse` type. It is carried because a block
	// mixing a u4 with three packed_bool flags is a derived-partition
	// shape and a block of twenty categoricals is a question battery,
	// and a reader deciding whether to believe a proposal wants that
	// without opening the profile document.
	Type string `json:"type"`
	// NullCount and NullRate are the member's own marginal. They are
	// identical across an emitted block by the admission rule, which is
	// what makes the member order arbitrary and what makes
	// max_null_rate_deviation 0.
	NullCount int     `json:"null_count"`
	NullRate  float64 `json:"null_rate"`
	// Agreement is the overlap of this member's null set with the FIRST
	// member's: of the rows where either is null, the share where both
	// are. 1 exactly when the two are null on the same rows.
	Agreement float64 `json:"agreement"`
}

type RuleEvidenceDependency struct {
	Field string `json:"field"`
	// Type is the target's `.pulse` type, carried for Block's reason: a
	// reader deciding whether to believe a derived-flag proposal wants
	// to know whether the target is one bit or a small integer without
	// opening the profile document.
	Type string `json:"type"`
	// Form names the RENDERING judgement — which reading of the lookup
	// below the expression is. It is the one thing the expression itself
	// cannot say, and the thing most worth checking: `threshold` and
	// `threshold_chain` are TOTAL functions of the source (an
	// unobserved value lands in the nearest band), while `membership`,
	// `membership_complement` and `enumeration_chain` fall to a default
	// arm.
	Form string `json:"form"`
	// Mapping is the measured lookup itself, in the source's own level
	// order, with the rows behind each arm. Exceptions is 0 for every
	// emitted candidate BY THE ADMISSION RULE, and is carried anyway for
	// the reason Block's Agreement is: it is the detector's central
	// claim — this is a FUNCTION, not a tendency — stated on the wire
	// where a reader can check it against the supports beside it rather
	// than left implicit in the fact that the candidate was emitted.
	Mapping    []RuleEvidenceMapValue `json:"mapping"`
	Exceptions int                    `json:"exceptions"`
}

type RuleEvidenceLevel struct {
	// Level is the level's spelling, matching an entry of
	// RuleEvidence.GatedLevels or .OpenLevels.
	Level string `json:"level"`
	// Side is "gated" or "open".
	Side string `json:"side"`
	// N is the rows on which the gate held this level: the support
	// behind every conditional rate computed at it.
	N int `json:"n"`
	// MeanTargetNullRate is the mean of P(target null | this level)
	// across this candidate's targets. At a gated level it is ~1 and at
	// an open one ~0, by the admission rule.
	MeanTargetNullRate float64 `json:"mean_target_null_rate"`
}

type RuleEvidenceMapValue struct {
	Level string `json:"level"`
	// Value is the target's value in its own JSON shape — a bool for a
	// packed_bool, a number for a u4 — rather than a uniform string, so
	// a reader can compare it against the profile document's own
	// marginal for the field without decoding a convention.
	Value any `json:"value"`
	N     int `json:"n"`
}

type RuleEvidenceTarget struct {
	Field string `json:"field"`
	// NullRate is the field's own marginal null rate over RowsObserved —
	// the same figure the profile document carries for it, so a reader
	// can check the candidate against the document without re-measuring.
	NullRate float64 `json:"null_rate"`
	// GatedNullRate and OpenNullRate are P(null | gated levels) and
	// P(null | open levels), pooled over the levels on each side.
	GatedNullRate float64 `json:"gated_null_rate"`
	OpenNullRate  float64 `json:"open_null_rate"`
}

type RuleSpec struct {
	// When is an optional expr-lang predicate over the row. The rule
	// applies to rows satisfying it; ABSENT MEANS EVERY ROW, which is
	// not the same as "never" and is the form a derived-field rule
	// (`set_expr` computing a value from other columns) normally takes.
	// Compiled against the same environment Spec.Constraints uses
	// (rowExprEnv / rowExprOptions), so the two cannot diverge: every
	// scalar including packed_bool is a number, a categorical is a
	// string, a set_* is a map[string]bool, and absence is tested with
	// isnull(field). Must return a bool.
	When string `json:"when,omitempty"`

	// SetNull names fields to null on a matching row. It removes a value
	// rather than supplying one, so the field's own generation still
	// matters on every row the rule does not gate — which is why a
	// set_null target is never pre-claimed away from its model.
	SetNull []string `json:"set_null,omitempty"`

	// OwnsNulls declares that THIS RULE is the only source of absence
	// for the fields it names in SetNull: their own `null_rate` draw is
	// DISCARDED and the rule's gate becomes the single thing that can
	// null them. Scoped to SetNull and to nothing else — a rule
	// declaring it without a SetNull field is refused
	// (PULSE_SYNTH_RULE_OWNERSHIP_INVALID).
	//
	// # The arithmetic it fixes
	//
	// A captured `null_rate` is a MARGINAL: it already includes whatever
	// the gate removed. Left in place beneath a `set_null` the two
	// compose — a field is null when the gate fires OR when its own
	// draw says so — and the generated rate comes out at
	// g + (1-g)*r rather than r. Measured on the motivating 122-field
	// survey profile, `regard` declares 0.2526 and generated 0.4418
	// behind a `{"when": "aware == 0"}` gate that is exactly right about
	// WHICH rows are absent. Declaring ownership lands it on the gate's
	// own firing rate, which is what the captured number was measuring.
	//
	// # Zero, not a residual, and why
	//
	// The exactly-correct suppression is the RESIDUAL rate conditional
	// on the rule not firing, (r - g) / (1 - g). It needs g, the rule's
	// firing PROBABILITY, and a `when` is an arbitrary predicate over a
	// row: nothing in a Spec knows how often it will be true, and
	// estimating it would need a pre-pass whose own rows are drawn from
	// the rates it is trying to correct. So the declaration zeroes the
	// draw and the gap is MEASURED at generation instead of guessed —
	// ownershipWarnings reports any owned field whose realised null rate
	// misses its declared one (synth/rules_ownership.go).
	//
	// Where g IS knowable the residual is bounded rather than unknown:
	// `profile create --suggest-rules` admits a gate only when the
	// target is null on at least gateHighNullRate of gated rows and at
	// most gateLowNullRate (0.02) of open rows, and that second
	// threshold IS the residual. Every candidate detection can emit is
	// therefore within 0.02 of exact under a zeroed draw — the same
	// distance nullRateDivergenceThreshold already calls immaterial —
	// which is why detection emits this flag on every gating candidate.
	//
	// # Two rules may own one field
	//
	// Ownership is a UNION, not an exclusive claim, and deliberately not
	// routed through resolveConflicts' claim(): `set_null` REMOVES a
	// value rather than supplying one, so it never claims a field away
	// from its own generation (E2-S2), and "two gates can each account
	// for this field's absence" is a coherent statement needing no
	// arbitration. The draw is suppressed once.
	//
	// # Its relationship to NullTogether
	//
	// A block already does this for its non-gate members, by COPY rather
	// than by suppression: applyNullTogether overwrites every member's
	// decision with the gate's, so their own null_rate is discarded
	// exactly as this flag discards it. The gated-block idiom — name a
	// never-null field first so the copy clears the block's own MCAR
	// nulls, then supply the real gate with a `set_null` — is ownership
	// obtained as a SIDE EFFECT of that copy. OwnsNulls is the same
	// statement said directly, and on the rule that makes it true. The
	// block is still required for the shape it alone expresses: a
	// co-missing block with NO gating field, where one member's own draw
	// is the block's only source of absence.
	OwnsNulls bool `json:"owns_nulls,omitempty"`

	// Set assigns LITERAL values: a number or bool for a scalar target,
	// a declared category string for a categorical_*, an array of
	// declared options for a set_*. Never an expression — see SetExpr,
	// and see the type doc for why the two are separate keys.
	Set map[string]any `json:"set,omitempty"`

	// SetExpr assigns the RESULT of an expression evaluated over the row
	// at the moment this rule runs. The expressions compile here; the
	// per-target coercion of the result (bool to 1/0 for a numeric
	// target, string to a dictionary entry for a categorical) is applied
	// by the rule pass.
	SetExpr map[string]string `json:"set_expr,omitempty"`

	// NullTogether nulls a block of fields as ONE decision rather than
	// per field, the co-missingness shape a survey question block has:
	// the FIRST named field's null state is copied to the rest, so the
	// block is all null or all present and never a mixture. Every other
	// member's own null_rate is consequently IGNORED — the cost of the
	// only resolution that adds no randomness, reported by
	// nullTogetherWarnings when the discarded rate is materially
	// different. Requires at least two distinct fields — a one-field
	// block is set_null spelled less clearly. Applied LAST within its
	// rule; see applyNullTogether.
	NullTogether []string `json:"null_together,omitempty"`

	// Evidence is the ONE INERT SLOT: the measurement that proposed this
	// rule, when `profile create --suggest-rules` wrote it. Nothing in
	// generation reads it — not compileRules, not the rule pass, not
	// validateRules — and it cannot change a generated byte. It rides
	// the rule rather than a sibling document so a candidate an analyst
	// deletes takes its evidence with it, and so the file stays the bare
	// array `--rules` consumes unmodified. Read RuleEvidence's own doc
	// before touching this; the inertness is the contract, and
	// TestRuleEvidence_IsInert is the gate on it.
	Evidence *RuleEvidence `json:"_evidence,omitempty"`
}

type RunContinuationProfile struct {
	// Pairs is the number of within-shard adjacent row pairs compared.
	Pairs int `json:"pairs"`
	// Shards is the number of payloads scanned.
	Shards int `json:"shards"`
	// Overall is the mean of the per-field rates — the fraction of all
	// (pair, field) comparisons that repeated, i.e. the share of field
	// writes the run-skip decode avoids on a full-row read.
	Overall float64 `json:"overall"`
	// HighThreshold is the rate at or above which a field is listed in
	// HighFields (RunContinuationHighThreshold).
	HighThreshold float64 `json:"high_threshold"`
	// HighFields lists, in schema order, the fields at or above
	// HighThreshold: the high-continuation block a sort key holds
	// constant (on a denormalised join, the parent block).
	HighFields []string `json:"high_fields"`
	// Advice is a one-sentence reading of Overall for a user deciding
	// whether to sort the source upstream.
	Advice string `json:"advice"`
	// Fields carries every schema field's rate, in schema order.
	Fields []FieldContinuation `json:"fields"`
}

type SetCategoricalPairFidelity struct {
	Set         string  `json:"set"`
	Option      string  `json:"option"`
	Categorical string  `json:"categorical"`
	Delta       float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with both the set field
	// and the categorical field simultaneously non-null — the realized
	// pair's own co-occurrence count.
	N int `json:"n"`
	// Error is set instead of Delta when the synthetic partition
	// produced no co-occurring observation at all for this pair —
	// mirrors CategoricalPairFidelity's own per-pair contract.
	Error string `json:"error,omitempty"`
}

type SetCategoricalPairProfile struct {
	Set         string            `json:"set"`
	Option      string            `json:"option"`
	Categorical string            `json:"categorical"`
	Cells       []ContingencyCell `json:"cells"`
	// N is the number of rows where both the set field and the
	// categorical field were simultaneously non-null — this option's
	// true co-occurrence count, mirroring CategoricalPairProfile.N.
	N int `json:"n"`
}

type SetCategoricalPairSpec struct {
	Set         string                    `json:"set"`
	Option      string                    `json:"option"`
	Categorical string                    `json:"categorical"`
	Cells       []CategoricalPairCellSpec `json:"cells"`
}

type SetFieldFidelity struct {
	Field   string               `json:"field"`
	Options []*SetOptionFidelity `json:"options,omitempty"`
}

type SetNumericPairFidelity struct {
	Set        string                                `json:"set"`
	Option     string                                `json:"option"`
	Numeric    string                                `json:"numeric"`
	Categories []*CategoricalNumericCategoryFidelity `json:"categories,omitempty"`
}

type SetNumericPairProfile struct {
	Set     string `json:"set"`
	Option  string `json:"option"`
	Numeric string `json:"numeric"`
	// Categories carries at most two entries, keyed "selected" /
	// "not_selected" — reuses CategoricalNumericCategoryStat verbatim
	// rather than a bespoke two-field struct, so a caller already
	// handling CategoricalNumericPairProfile.Categories needs no second
	// shape to learn.
	Categories []CategoricalNumericCategoryStat `json:"categories"`
	// N mirrors CategoricalNumericPairProfile.N: the total non-null
	// co-occurrence count summed across both categories.
	N int `json:"n"`
}

type SetNumericPairSpec struct {
	Set        string                           `json:"set"`
	Option     string                           `json:"option"`
	Numeric    string                           `json:"numeric"`
	Categories []CategoricalNumericCategorySpec `json:"categories"`
	Min        float64                          `json:"min,omitempty"`
	Max        float64                          `json:"max,omitempty"`
	HasClamp   bool                             `json:"has_clamp,omitempty"`
	// Bernoulli is RETIRED: accepted so every document written before
	// the retirement still parses, and READ BY NOTHING.
	//
	// What it used to declare — the numeric target is a BOOLEAN
	// marginal, so the cell's captured Mean is a prevalence and the
	// draw is Bernoulli(Mean) rather than Normal(Mean, Std) — is now
	// DERIVED from the target's own FieldSpec at generate() setup
	// (buildBernoulliConditionals), exactly as the `discrete` staircase
	// always was.
	//
	// The retirement is the point, not a tidy-up. A flag can DISAGREE
	// with the field's own reconstruction, and a marginal disagreeing
	// with the draw that overwrites it is the failure class the boolean
	// and small-integer work exists to remove — invisible, because every
	// cell still renders a plausible prevalence and only the number is
	// wrong. It was not hypothetical: SpecFromProfile's SET-numeric arm
	// admitted a DistBernoulli target, said in a comment that the flag
	// was "carried the same way" as the categorical-numeric arm's, and
	// did not set it, so a profile-derived set-numeric pair over a
	// boolean target reproduced the clamped-normal defect once PER CELL.
	// Deriving makes that omission unrepresentable.
	//
	// A spec still declaring it over a target whose marginal is NOT
	// bernoulli gets a warning naming the pair (bernoulliFlagWarnings),
	// never a refusal and never silence; declare the FIELD `bernoulli`
	// instead. Do not reintroduce a read of this slot.
	//
	// Deprecated: declare the target field's distribution as
	// DistBernoulli; this slot is ignored.
	Bernoulli bool `json:"bernoulli,omitempty"`
}

type SetOptionFidelity struct {
	Value string `json:"value"`
	// SourceFrequency/SyntheticFrequency are P(bit set) over each
	// partition's non-null rows of Field. Delta is
	// abs(SourceFrequency - SyntheticFrequency). SyntheticFrequency and
	// Delta are zero-valued and meaningless when Error is set.
	SourceFrequency    float64 `json:"source_frequency"`
	SyntheticFrequency float64 `json:"synthetic_frequency,omitempty"`
	Delta              float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with a non-null Field —
	// the realized synthetic partition's own denominator for this
	// option's frequency, shared across every option of the same field.
	N int `json:"n"`
	// Error is set instead of SyntheticFrequency/Delta when the
	// synthetic partition has no non-null observation of Field at all —
	// mirrors every other Fidelity entry kind's non-fatal per-entry
	// contract.
	Error string `json:"error,omitempty"`
}

type SetOptionStat struct {
	Value string `json:"value"`
	// Count is the number of non-null rows with this option's bit set.
	Count int `json:"count"`
	// Frequency is Count / SetProfile.N — P(bit set), the per-option
	// marginal Bernoulli rate.
	Frequency float64 `json:"frequency"`
}

type SetProfile struct {
	// N is the number of non-null rows Options' frequencies are computed
	// over — the same denominator FieldProfile.NullRate already implies,
	// surfaced here so a caller reading only the Set block can interpret
	// Count as a rate without cross-referencing NullRate.
	N int `json:"n"`
	// Options lists EVERY dictionary entry in bit/dictionary order (bit i
	// first) — NOT sorted by frequency like CategoricalProfile.Top — so
	// two profiles of the same schema are directly comparable
	// position-for-position regardless of which option happens to be
	// more popular.
	Options []SetOptionStat `json:"options"`
}

type SetSetPairFidelity struct {
	SetA    string  `json:"set_a"`
	OptionA string  `json:"option_a"`
	SetB    string  `json:"set_b"`
	OptionB string  `json:"option_b"`
	Delta   float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with both set fields
	// simultaneously non-null — the realized pair's own co-occurrence
	// count.
	N int `json:"n"`
	// Error is set instead of Delta when the synthetic partition
	// produced no co-occurring observation at all for this pair.
	Error string `json:"error,omitempty"`
}

type SetSetPairProfile struct {
	SetA    string            `json:"set_a"`
	OptionA string            `json:"option_a"`
	SetB    string            `json:"set_b"`
	OptionB string            `json:"option_b"`
	Cells   []ContingencyCell `json:"cells"`
	// N is the number of rows where both fields were simultaneously
	// non-null — this option pair's true co-occurrence count.
	N int `json:"n"`
}

type SetSetPairSpec struct {
	SetA    string                    `json:"set_a"`
	OptionA string                    `json:"option_a"`
	SetB    string                    `json:"set_b"`
	OptionB string                    `json:"option_b"`
	Cells   []CategoricalPairCellSpec `json:"cells"`
}

type ShapeProfile struct {
	Means   []float64 `json:"means"`
	Stds    []float64 `json:"stds"`
	Weights []float64 `json:"weights"`
}

type Spec struct {
	// RowCount is the number of rows to generate. Required (>0).
	RowCount int `json:"row_count"`

	// Fields declares each column with its type and distribution.
	Fields []FieldSpec `json:"fields"`

	// Constraints, if non-empty, are evaluated per row by the expression
	// evaluator. Rows that fail any constraint are rejected and re-drawn.
	Constraints []ConstraintSpec `json:"constraints,omitempty"`

	// MaxRejectionRate caps the fraction of rejected rows during
	// constraint-driven rejection sampling. Defaults to 0.5 if zero.
	MaxRejectionRate float64 `json:"max_rejection_rate,omitempty"`

	// Correlations lists optional pairwise correlations to induce via
	// Gaussian copula post-processing. Each entry references two numeric
	// fields by name with a target Pearson correlation in [-1, 1]. Only
	// numeric fields can participate.
	Correlations []CorrelationSpec `json:"correlations,omitempty"`

	// CategoricalPairs lists optional categorical-categorical joint
	// structure to reproduce during generation: for each pair, field B
	// is resampled conditioned on field A's independently-drawn value
	// using the captured contingency cells, instead of being drawn from
	// its own unconditional marginal. Populated by SpecFromProfile from
	// ConditionalProfile.CategoricalPairs (`profile create --conditional`)
	// when present; absent (nil, the zero value) reproduces today's
	// independent-marginal behavior exactly — every pre-existing spec,
	// hand-authored or profile-derived, has no such key.
	CategoricalPairs []CategoricalPairSpec `json:"categorical_pairs,omitempty"`

	// CategoricalNumericPairs lists optional categorical-numeric
	// conditional structure to reproduce during generation: for each
	// pair, numeric field B is resampled from Normal(mean, std)
	// parameterized by categorical field A's drawn value, using the
	// captured per-category conditional mean/std, instead of being drawn
	// from its own unconditional normal. Populated by SpecFromProfile
	// from ConditionalProfile.CategoricalNumericPairs when present;
	// absent (nil) reproduces today's independent-marginal behavior
	// exactly.
	CategoricalNumericPairs []CategoricalNumericPairSpec `json:"categorical_numeric_pairs,omitempty"`

	// SetCategoricalPairs lists optional set-option x categorical joint
	// structure to reproduce during generation (E5-S3): for each pair,
	// the set field's declared Option (bit) is resampled from
	// Bernoulli(P(selected | categorical field's drawn value)) using the
	// captured contingency cells, instead of the option's own
	// independent marginal frequency. Populated by SpecFromProfile from
	// ConditionalProfile.SetCategoricalPairs when present; absent (nil)
	// reproduces the independent per-option marginal behavior exactly.
	SetCategoricalPairs []SetCategoricalPairSpec `json:"set_categorical_pairs,omitempty"`

	// SetNumericPairs lists optional set-option x numeric joint
	// structure to reproduce during generation (E5-S3): for each pair,
	// numeric field B is resampled from Normal(mean, std) conditioned on
	// whether the set field's declared Option (bit) was independently
	// drawn selected, using the captured per-bucket conditional
	// mean/std. Populated by SpecFromProfile from
	// ConditionalProfile.SetNumericPairs when present; absent (nil)
	// reproduces today's independent-marginal behavior exactly.
	SetNumericPairs []SetNumericPairSpec `json:"set_numeric_pairs,omitempty"`

	// SetSetPairs lists optional option x option joint structure between
	// two DIFFERENT set_* fields to reproduce during generation (E5-S3):
	// for each pair, set B's declared OptionB (bit) is resampled from
	// Bernoulli(P(selected | set A's OptionA drawn state)) using the
	// captured 2x2 contingency cells. Populated by SpecFromProfile from
	// ConditionalProfile.SetSetPairs when present; absent (nil)
	// reproduces the independent per-option marginal behavior exactly.
	SetSetPairs []SetSetPairSpec `json:"set_set_pairs,omitempty"`

	// Models lists optional per-numeric ADDITIVE linear predictors to
	// drive generation with, the Spec-facing counterpart of the profile
	// document's `models` section (`profile create --fit-models`).
	// Populated by SpecFromProfile from Profile.Models when present;
	// absent (nil, the zero value) reproduces today's behaviour exactly
	// — every pre-existing spec, hand-authored or profile-derived, has
	// no such key.
	//
	// Where CategoricalNumericPairs and SetNumericPairs each RESAMPLE a
	// numeric field from one paired field's conditional moments — so two
	// pairs naming the same numeric compete to be the last writer, and
	// resolveConflicts has to drop one — a model accounts for every
	// predictor in a single expression. SpecFromProfile therefore treats
	// the two as alternatives rather than layers: a profile carrying
	// `models` populates this slot and leaves both numeric-target pair
	// slots empty.
	Models []FieldModelSpec `json:"models,omitempty"`

	// ResidualCorrelations lists optional target correlations between
	// the RESIDUALS of two modelled numeric fields — the Spec-facing
	// counterpart of the profile document's `residual_correlations`
	// section (`profile create --residual-correlations`), populated by
	// SpecFromProfile from Profile.ResidualCorrelations when present.
	// Absent (nil, the zero value) reproduces the independent-residual
	// draw exactly: every modelled field takes its own fresh z, which
	// is what every pre-existing spec gets.
	//
	// This is a DIFFERENT quantity from Correlations above and the two
	// are not interchangeable. Correlations names a correlation between
	// two fields' VALUES and is realized by drawing both values from a
	// shared copula, which requires owning the value outright. Once a
	// field carries a model, its value is the model's to produce and
	// the only part still free to move is the residual — so this slot
	// correlates THAT, supplying each modelled field its component of
	// one shared correlated normal vector instead of overwriting what
	// the model drew. Imposing a value-scale figure on the residual
	// draw would apply every predictor the two targets share a second
	// time (see synth/residual_corr.go), which is why a Correlations
	// entry naming a modelled field is excluded rather than rerouted
	// here.
	//
	// Entries reuse CorrelationSpec verbatim: a pair of field names and
	// a rho, read on the residual scale. Both endpoints must carry a
	// model; an entry naming a field whose model did not survive is
	// dropped with a warning at generate() time, and validateSpec
	// refuses one outright for a hand-authored spec.
	ResidualCorrelations []CorrelationSpec `json:"residual_correlations,omitempty"`

	// Rules lists optional STRUCTURAL rules — statements about which
	// fields a row may carry a value for and what that value is, imposed
	// on top of whatever the distributions, conditional pairs,
	// correlations and models produced. Additive and omitempty: absent
	// (nil, the zero value) reproduces today's behaviour exactly, and
	// because the rule pass consumes no RNG, a spec declaring no rules
	// is byte-identical to one written before the slot existed.
	//
	// This is the slot for the class of fact no statistical summary can
	// express, because it is not variation at all: a question block that
	// is not ASKED of a respondent who failed a screener, a flag that is
	// a band of another column. Reconstructed from marginals those come
	// back as soft noise with a plausible rate and no gate.
	//
	// Rules apply in DECLARATION ORDER, sequentially, LAST WRITE WINS,
	// in one pass at the very end of the row — see RuleSpec for the full
	// semantics and for why they are deliberately not topologically
	// sorted. The standalone rules-file format is this array itself, so
	// an inline declaration and a `--rules` file are the same JSON.
	//
	// Validation is EAGER: validateRules refuses every malformed rule at
	// spec parse with a PULSE_SYNTH_RULE_* code naming the rule index.
	Rules []RuleSpec `json:"rules,omitempty"`
}

type TestRunner func(physicalSchema, presentedSchema *encoding.Schema, records []byte, test *types.Test) (*types.TestResult, error)

type WarningGroup struct {
	// Kind is the stable, field-free label for this group. It never
	// interpolates a field name, so the group count is a count of
	// findings of one shape.
	Kind string
	// Attention is true when this kind means a requested capture or
	// generation step did NOT happen. False marks an expected outcome:
	// a complete, correct result the caller may nonetheless want to know
	// about.
	Attention bool
	// Members holds every warning classified into this group, in
	// production order. Callers that render to a terminal are expected
	// to show a bounded prefix; the full list stays in whatever document
	// the command already writes.
	Members []string
}