synth

type CategoricalNumericCategoryFidelity struct {
	Category string `json:"category"`
	// SourceMean/SourceStd echo the captured
	// CategoricalNumericCategorySpec verbatim.
	SourceMean float64 `json:"source_mean"`
	SourceStd  float64 `json:"source_std"`
	// SyntheticMean/SyntheticStd are the realized conditional mean/std
	// over the _synthetic=true partition. MeanDelta/StdDelta are
	// abs(source-synthetic). All four are zero-valued and meaningless
	// when Error is set.
	SyntheticMean float64 `json:"synthetic_mean,omitempty"`
	SyntheticStd  float64 `json:"synthetic_std,omitempty"`
	MeanDelta     float64 `json:"mean_delta,omitempty"`
	StdDelta      float64 `json:"std_delta,omitempty"`
	// N is the number of _synthetic=true rows carrying this category
	// value with a non-null B — the realized category's own observation
	// count.
	N int `json:"n"`
	// Error is set instead of the Synthetic*/*Delta fields when the
	// synthetic partition realized no observation of this category at
	// all — mirrors PairwiseFidelity's own per-entry non-fatal contract.
	Error string `json:"error,omitempty"`
}

type CategoricalNumericCategorySpec struct {
	Category string  `json:"category"`
	Mean     float64 `json:"mean"`
	Std      float64 `json:"std"`
}

type CategoricalNumericCategoryStat struct {
	Category string  `json:"category"`
	Mean     float64 `json:"mean"`
	Std      float64 `json:"std"`
	// N is the number of non-null B observations conditioned on this
	// category — the input thinPairWarning checks against
	// MinPairObservations, exactly as NumericPairProfile.N and each
	// ContingencyCell.Count already do for the other two pair kinds.
	N int `json:"n"`
}

type CategoricalNumericPairFidelity struct {
	A          string                                `json:"a"`
	B          string                                `json:"b"`
	Categories []*CategoricalNumericCategoryFidelity `json:"categories,omitempty"`
}

type CategoricalNumericPairProfile struct {
	A          string                           `json:"a"` // categorical field name
	B          string                           `json:"b"` // numeric field name
	Categories []CategoricalNumericCategoryStat `json:"categories"`
	// N is the number of rows in the capture where both A and B were
	// simultaneously non-null, summed across every category (including
	// "other") — the pair's true co-occurrence count, mirroring
	// CategoricalPairProfile.N and NumericPairProfile.N.
	N int `json:"n"`
}

type CategoricalNumericPairSpec struct {
	A          string                           `json:"a"`
	B          string                           `json:"b"`
	Categories []CategoricalNumericCategorySpec `json:"categories"`
	// Min/Max clamp the conditional draw exactly as field B's own
	// unconditional normal reconstruction clamps to its observed range
	// (see SpecFromProfile). HasClamp distinguishes "clamp to [0,0]"
	// (rare, but a legitimate observed range for a constant field) from
	// "no clamp was declared" — a hand-authored from-schema spec that
	// omits Min/Max gets no clamping, exactly as normal's own optional
	// min/max params behave when absent.
	Min      float64 `json:"min,omitempty"`
	Max      float64 `json:"max,omitempty"`
	HasClamp bool    `json:"has_clamp,omitempty"`
}

type CategoricalPairCellSpec struct {
	AValue string `json:"a_value"`
	BValue string `json:"b_value"`
	Count  int    `json:"count"`
}

type CategoricalPairFidelity struct {
	A string `json:"a"`
	B string `json:"b"`
	// Delta is the total variation distance between the source and
	// synthetic joint-frequency tables (each normalized to proportions
	// before comparison), in [0, 1] — 0 when the two tables carry
	// identical support and proportions, 1 when they share none. Chosen
	// over a chi-square-style distance because it stays well-defined and
	// boundedly comparable to a fixed tolerance even when synthesis
	// realizes a cell the capture never saw (or vice versa) — a
	// chi-square statistic diverges when an expected cell is zero. Zero
	// valued and meaningless when Error is set.
	Delta float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with both A and B
	// simultaneously non-null and dictionary-resolvable — the realized
	// pair's own co-occurrence count. CategoricalPairSpec carries no
	// source N of its own; the source total used to normalize its side
	// of the comparison is recovered by summing Cells' Count.
	N int `json:"n"`
	// Error is set instead of Delta when the synthetic partition
	// produced no co-occurring observation at all for this pair — mirrors
	// PairwiseFidelity's own per-pair non-fatal failure contract.
	Error string `json:"error,omitempty"`
}

type CategoricalPairProfile struct {
	A     string            `json:"a"`
	B     string            `json:"b"`
	Cells []ContingencyCell `json:"cells"`
	// N is the number of rows in the capture where both A and B were
	// simultaneously non-null — the pair's true co-occurrence count,
	// summed across every cell (the cells kept plus whatever is folded
	// into the "other"/"other" catch-all).
	N int `json:"n"`
}

type CategoricalPairSpec struct {
	A     string                    `json:"a"`
	B     string                    `json:"b"`
	Cells []CategoricalPairCellSpec `json:"cells"`
}

type CategoricalProfile struct {
	Cardinality int           `json:"cardinality"`
	Top         []CategoryHit `json:"top"`
}

type CategoryHit struct {
	Value  string  `json:"value"`
	Weight float64 `json:"weight"`
}

type ConditionalProfile struct {
	// NumericPairs lists the numeric-numeric pairs captured: the
	// row-aligned Pearson correlation and the count of rows where both
	// fields were simultaneously non-null (N) — the input
	// thinPairWarning checks against MinPairObservations. Capped at
	// ProfileOptions.CorrelationTopK strongest |rho| pairs, same
	// convention as Profile.Pairwise.
	NumericPairs []NumericPairProfile `json:"numeric_pairs,omitempty"`
	// CategoricalPairs lists the categorical-categorical pairs
	// captured: a bounded contingency table of observed
	// (a_value, b_value) co-occurrence counts per pair. See
	// CategoricalPairProfile for the two composed collapse points
	// (per-field top-K, then joint-cell ContingencyCellCap) that keep
	// this section bounded regardless of either field's cardinality.
	CategoricalPairs []CategoricalPairProfile `json:"categorical_pairs,omitempty"`
	// CategoricalNumericPairs lists the categorical-numeric pairs
	// captured: the numeric field's conditional mean/std broken out per
	// observed category of the categorical field. See
	// CategoricalNumericPairProfile for the single collapse point
	// (per-field top-K) that keeps this section bounded — no separate
	// joint-cardinality cap is needed here, unlike CategoricalPairs,
	// because a conditional numeric summary is one row per already-capped
	// category rather than a joint cell per pair of categories.
	CategoricalNumericPairs []CategoricalNumericPairProfile `json:"categorical_numeric_pairs,omitempty"`
	// SetCategoricalPairs lists the set-option x categorical pairs
	// captured (E5-S2): one bounded contingency table per (set field,
	// option, categorical field) combination, the set-option axis fixed
	// to the two-value {"selected","not_selected"} Bernoulli domain. See
	// SetCategoricalPairProfile.
	SetCategoricalPairs []SetCategoricalPairProfile `json:"set_categorical_pairs,omitempty"`
	// SetNumericPairs lists the set-option x numeric pairs captured
	// (E5-S2): one conditional mean/std pair (selected vs. not_selected)
	// per (set field, option, numeric field) combination. See
	// SetNumericPairProfile.
	SetNumericPairs []SetNumericPairProfile `json:"set_numeric_pairs,omitempty"`
	// SetSetPairs lists the option x option pairs captured between two
	// DIFFERENT set_* fields (E5-S2): a 2x2 contingency table per
	// (fieldA option, fieldB option) combination. Never captured between
	// two options of the same field. See SetSetPairProfile.
	SetSetPairs []SetSetPairProfile `json:"set_set_pairs,omitempty"`
}

type ConstraintSpec struct {
	Expr string `json:"expr"`
}

type ContingencyCell struct {
	AValue string `json:"a_value"`
	BValue string `json:"b_value"`
	Count  int    `json:"count"`
}

type CorrelationSpec struct {
	A           string  `json:"a"`
	B           string  `json:"b"`
	Correlation float64 `json:"correlation"`
}

type CorrelationStat struct {
	A   string  `json:"a"`
	B   string  `json:"b"`
	Rho float64 `json:"rho"`
}

type DateProfile struct {
	Start    string `json:"start"`
	End      string `json:"end"`
	Weekdays [7]int `json:"weekdays"`
}

type FidelityReport struct {
	SourceRows                 int                               `json:"source_rows"`
	SyntheticRows              int                               `json:"synthetic_rows"`
	Fields                     []*FieldFidelity                  `json:"fields,omitempty"`
	Pairwise                   []*PairwiseFidelity               `json:"pairwise,omitempty"`
	CategoricalPairwise        []*CategoricalPairFidelity        `json:"categorical_pairwise,omitempty"`
	CategoricalNumericPairwise []*CategoricalNumericPairFidelity `json:"categorical_numeric_pairwise,omitempty"`
	SetFields                  []*SetFieldFidelity               `json:"set_fields,omitempty"`
	SetCategoricalPairwise     []*SetCategoricalPairFidelity     `json:"set_categorical_pairwise,omitempty"`
	SetNumericPairwise         []*SetNumericPairFidelity         `json:"set_numeric_pairwise,omitempty"`
	SetSetPairwise             []*SetSetPairFidelity             `json:"set_set_pairwise,omitempty"`
	// Models is the per-field MODEL-RECOVERY section (E5-S1): for every
	// linear model generation actually applied, the captured coefficients
	// beside the ones a refit on the generated partition recovers. It
	// answers a different question from every section above — those ask
	// whether the generated rows LOOK like the source, this asks whether
	// the captured STRUCTURE survived generation — and it is the only
	// section that can tell a faithful generation whose marginal
	// contrasts are merely compressed apart from one that silently lost
	// its conditioning. Absent (omitempty) for every spec carrying no
	// `models`, which is every spec predating `profile create
	// --fit-models`. See synth/fidelity_models.go.
	Models []*ModelFidelity `json:"models,omitempty"`
	// ModelResidualCorrelations is the residual-correlation half of the
	// same model-recovery question (E5-S2): for every residual
	// correlation generation applied, the captured rho beside the rho
	// recovered from the generated partition. It is deliberately NOT
	// folded into Pairwise, which scores the value-scale copula arm over
	// a DISJOINT set of fields — a modelled field is excluded from
	// Spec.Correlations by resolveConflicts precisely so the two arms
	// never both fire on one field. The two section names are what tell
	// a reader scanning this document which mechanism a given number
	// describes. Absent (omitempty) for every spec carrying no
	// `residual_correlations`, which is every spec predating `profile
	// create --residual-correlations`. See synth/fidelity_residual.go.
	ModelResidualCorrelations *ModelResidualCorrelationFidelity `json:"model_residual_correlations,omitempty"`
	Warnings                  []string                          `json:"warnings,omitempty"`
}

type FieldFidelity struct {
	Field  string            `json:"field"`
	Test   types.TestType    `json:"test"`
	Result *types.TestResult `json:"result,omitempty"`
	Error  string            `json:"error,omitempty"`
}

type FieldModel struct {
	// Field is the numeric target this model predicts.
	Field string `json:"field"`
	// Intercept is the fitted constant term: the predicted value for a
	// row sitting at every categorical's reference level (see
	// References) with no set option selected.
	Intercept float64 `json:"intercept"`
	// Predictors lists the fitted coefficients in design-column order.
	// Empty is legal and meaningful only in the sense that such a model
	// is never retained — a target with no usable candidate columns is
	// skipped, not stored with an empty predictor list.
	Predictors []ModelPredictor `json:"predictors"`
	// References names, per categorical predictor field, the level
	// dropped from Predictors and folded into Intercept. Absent
	// (omitempty) exactly when the model carries no categorical
	// predictor at all — a set-only model has nothing to drop, since a
	// multi-select is not a partition — which a reader can confirm from
	// predictors[].kind rather than having to assume. See ModelReference
	// for why the baseline is written down instead of inferred.
	References []ModelReference `json:"references,omitempty"`
	// NObs is the number of rows that actually contributed to the fit.
	// The engine applies listwise deletion, so a row with a null target
	// or a null predictor SOURCE field contributes nothing and this is
	// below the profile's RowCount whenever the cohort has nulls.
	NObs int `json:"n_obs"`
	// R2 and ResidualStd are the fit's explanatory power and residual
	// scale. ResidualStd is the standard deviation of the residual that
	// generation adds back on top of the linear predictor.
	R2          float64 `json:"r2"`
	ResidualStd float64 `json:"residual_std"`
	// ShrinkageAlpha is the ridge penalty this model was fitted under,
	// absent (omitempty, and therefore zero) when it was fitted by plain
	// unpenalized OLS.
	//
	// It is on the wire because a shrunk coefficient and a free one are
	// not the same kind of number, and nothing else in the document says
	// which was produced. A reader comparing two models' coefficients —
	// or comparing one model against a hand-run regression — needs to
	// know that at least one of this model's levels was thin enough to
	// pull the whole design toward its baselines, and by how much: the
	// penalty is minLevelObservations pseudo-observations spread over
	// NObs rows, so alpha·NObs recovers the pseudo-count. See
	// profile_models_shrink.go for the construction.
	ShrinkageAlpha float64 `json:"shrinkage_alpha,omitempty"`
	// Residuals is the row-aligned fitted residual (observed − predicted)
	// for each row retained in the capture's residual reservoir, and
	// ResidualPresent[i] reports whether row i contributed one at all —
	// a row listwise-deleted for THIS field has no residual even though
	// another field's model may have one for the same row.
	//
	// Every FieldModel from one capture indexes the SAME retained rows,
	// which is the whole point: residual correlation between two fields
	// is only meaningful on rows where both residuals exist.
	//
	// Both are `json:"-"` — DELIBERATELY not part of the document. They
	// are a bounded row-aligned SAMPLE (modelResidualCap entries per
	// model, per field), so serialising them would add tens of thousands
	// of numbers per numeric column to a document whose every other
	// section is a handful of summary statistics, and would do it to
	// carry rows a profile exists precisely to avoid retaining. Their
	// consumer is the residual-correlation measurement, which runs in
	// the SAME process as the capture and reads them straight off
	// FittedModels(); what belongs in the document is the small derived
	// quantity that measurement produces, not its input. A Profile read
	// back from JSON therefore has coefficients and ResidualStd but nil
	// Residuals, and any consumer that needs per-row residuals must
	// capture rather than re-read.
	Residuals       []float64 `json:"-"`
	ResidualPresent []bool    `json:"-"`
}

type FieldModelSpec struct {
	// Field is the numeric field this model draws.
	Field string `json:"field"`
	// Intercept is the constant term — the prediction for a row sitting
	// at every categorical predictor's reference level with no set
	// option selected.
	Intercept float64 `json:"intercept"`
	// Predictors are the per-(field, level) contributions summed into
	// the prediction. A row whose drawn value for a predictor's Field is
	// not that predictor's Level contributes nothing for it.
	Predictors []ModelPredictorSpec `json:"predictors"`
	// ResidualStd is the scale of the residual added on top of the
	// linear prediction. Zero means a deterministic prediction, which is
	// a legal (if degenerate) model rather than an error.
	ResidualStd float64 `json:"residual_std"`
	// Min/Max are the numeric field's observed bounds, carried alongside
	// HasClamp exactly as CategoricalNumericPairSpec and
	// SetNumericPairSpec already carry them. They are the same bounds
	// the field's own reconstructed marginal would clamp to; whether and
	// where the draw applies them is the generation stage's decision,
	// and this slot only guarantees the numbers are available to it
	// without a second lookup into the profile.
	Min      float64 `json:"min,omitempty"`
	Max      float64 `json:"max,omitempty"`
	HasClamp bool    `json:"has_clamp,omitempty"`
}

type FieldProfile struct {
	Name        string              `json:"name"`
	Type        string              `json:"type"`
	Description string              `json:"description,omitempty"`
	NullRate    float64             `json:"null_rate"`
	Numeric     *NumericProfile     `json:"numeric,omitempty"`
	Categorical *CategoricalProfile `json:"categorical,omitempty"`
	Date        *DateProfile        `json:"date,omitempty"`
	// Set carries per-option marginal selection-frequency stats for a
	// set_* (multi-select bitmask) field (E5-S1). Populated instead of
	// Numeric/Categorical/Date — exactly one of the four is non-nil per
	// field, based on the field's schema type.
	Set *SetProfile `json:"set,omitempty"`
	// Precision/Scale carry decimal128 metadata so synth-from-profile can
	// reconstruct the original field shape.
	Precision uint8 `json:"precision,omitempty"`
	Scale     uint8 `json:"scale,omitempty"`
}

type FieldSpec struct {
	Name         string         `json:"name"`
	Type         string         `json:"type"`
	Nullable     bool           `json:"nullable,omitempty"`
	Description  string         `json:"description,omitempty"`
	Distribution string         `json:"distribution"`
	Params       map[string]any `json:"params,omitempty"`

	// Precision and Scale apply to decimal128.
	Precision uint8 `json:"precision,omitempty"`
	Scale     uint8 `json:"scale,omitempty"`

	// NullRate is the per-row probability that the field will be null.
	// Only meaningful when Nullable is true; ignored otherwise.
	NullRate float64 `json:"null_rate,omitempty"`
}

type ModelFidelity struct {
	Field string `json:"field"`
	// Marginal marks an INTERCEPT-ONLY model — one whose predictor
	// selection admitted nothing (see profile_models_select.go's variance
	// floor). It is a COMPLETE model, not a failed capture: the linear
	// predictor is the field's own mean and the residual carries all of
	// its spread, so generation reproduces the marginal, which is the
	// correct answer for a field nothing explains. Such an entry carries
	// no Predictors and reports only the intercept comparison — which is
	// still a real check, since a drifted intercept on a marginal model
	// means the marginal itself did not survive.
	Marginal bool `json:"marginal,omitempty"`
	// LatentScale is the target's reconstructed marginal standard
	// deviation — the divisor that puts every captured coefficient below
	// on the latent scale. Multiply a latent figure by it to return to
	// the units the profile document's `models` section reports, which
	// for a `normal` target are the same units either way.
	LatentScale float64 `json:"latent_scale"`
	// CapturedIntercept / RecoveredIntercept are the latent-scale
	// intercepts: (Intercept − mean)/std as generation standardises it,
	// against the refit's own. InterceptDelta is their absolute
	// difference.
	CapturedIntercept  float64 `json:"captured_intercept"`
	RecoveredIntercept float64 `json:"recovered_intercept,omitempty"`
	InterceptDelta     float64 `json:"intercept_delta,omitempty"`
	// Predictors is one entry per compiled model term, in the order
	// generation evaluates them. Absent on a Marginal model.
	Predictors []*ModelPredictorFidelity `json:"predictors,omitempty"`
	// NObs is the number of synthetic rows the refit admitted: rows
	// carrying a non-null, latent-invertible target and a non-null value
	// for every predictor SOURCE field (listwise deletion, the same rule
	// capture's fit applied). Bounded by modelRecoveryRowCap.
	NObs int `json:"n_obs"`
	// R2 is the refit's own coefficient of determination on the latent
	// scale — how much of the generated latent the recovered model
	// explains. It is NOT comparable to the captured FieldModel.R2, which
	// was measured on real source rows against real source noise;
	// synthetic residuals are Gaussian by construction, so this figure
	// reads high and is here to characterise the refit, not to score it.
	R2 float64 `json:"r2,omitempty"`
	// MaxDelta is the largest absolute latent coefficient gap across
	// Predictors (the intercept excluded — a shifted intercept moves the
	// level, not the structure). Flagged is true when any predictor is.
	MaxDelta float64 `json:"max_delta,omitempty"`
	Flagged  bool    `json:"flagged,omitempty"`
	// Error is set instead of the recovered figures when the refit could
	// not run at all for this model — too few admitted rows, a design the
	// solver refused. It never withholds the rest of the report, the same
	// non-fatal per-entry contract every other Fidelity kind follows.
	Error string `json:"error,omitempty"`
}

type ModelPredictor struct {
	// Column is the design-matrix column name the engine keyed the
	// coefficient by. Internal encoding; diagnostics and the in-process
	// residual replay only, never serialised — see the type comment.
	Column string `json:"-"`
	// Kind is one of the ModelPredictor* constants above. It is carried
	// on the wire because it is not derivable from (Field, Level): a
	// categorical level is one arm of a partition read against a dropped
	// reference (see FieldModel.References), while a set option is an
	// independent indicator with no reference at all, and a consumer
	// reconstructing effects has to know which it is holding.
	Kind string `json:"kind"`
	// Field is the real cohort field this column was expanded from.
	Field string `json:"field"`
	// Level is the categorical level text or set option text this column
	// indicates. Empty when Kind is ModelPredictorNumeric. Deliberately
	// NOT omitempty: it is half of the addressing key, and a categorical
	// dictionary may legitimately carry the empty-string level, so the
	// key is always written rather than left to be inferred from its
	// absence.
	Level string `json:"level"`
	// Coefficient is the fitted slope for this column.
	Coefficient float64 `json:"coefficient"`
}

type ModelPredictorFidelity struct {
	// Kind / Field / Level address the term exactly as
	// ModelPredictorSpec does, so an entry here zips against the profile
	// document's `models[].predictors[]` without a join key.
	Kind  string `json:"kind"`
	Field string `json:"field"`
	Level string `json:"level"`
	// CapturedCoefficient is the spec's coefficient divided by
	// LatentScale — the quantity buildModelDrawers actually adds into the
	// latent, not the raw number the profile document carries.
	CapturedCoefficient float64 `json:"captured_coefficient"`
	// RecoveredCoefficient is the refit's estimate for the same term.
	// StdError is the refit's own standard error for it, which is the
	// second half of the flagging band — see ModelRecoveryTolerance.
	RecoveredCoefficient float64 `json:"recovered_coefficient,omitempty"`
	StdError             float64 `json:"std_error,omitempty"`
	Delta                float64 `json:"delta,omitempty"`
	// NFired is the number of admitted synthetic rows on which this
	// term's indicator was 1. It is the term's own support and it is what
	// distinguishes the two ways a recovered coefficient reaches zero: a
	// term that fired thousands of times and recovered nothing is a
	// generation fault, while a term that never fired had nothing to
	// recover — the level was captured but the field's reconstructed
	// marginal does not generate it.
	NFired int `json:"n_fired"`
	// Flagged is true when Delta exceeds both ModelRecoveryTolerance and
	// modelRecoverySEMultiple standard errors. False whenever Error is
	// set: a term that was not estimated has no gap to judge.
	Flagged bool `json:"flagged,omitempty"`
	// Error is set instead of the recovered figures when this term could
	// not be carried into the refit at all — its level is absent from the
	// output cohort's dictionary, or its column is constant over the
	// admitted rows and so carries no information to estimate from. Both
	// are informative outcomes rather than failures, and both leave every
	// OTHER term of the same model estimated.
	Error string `json:"error,omitempty"`
}

type ModelPredictorSpec struct {
	// Kind is one of the ModelPredictor* constants
	// (ModelPredictorCategoricalLevel / ModelPredictorSetOption /
	// ModelPredictorNumeric).
	Kind string `json:"kind"`
	// Field is the predictor's source field.
	Field string `json:"field"`
	// Level is the categorical level text or set option text that turns
	// this contribution on.
	Level string `json:"level"`
	// Coefficient is added to the prediction when the indicator is 1.
	Coefficient float64 `json:"coefficient"`
}

type ModelReference struct {
	// Field is the categorical predictor source field.
	Field string `json:"field"`
	// Level is the level whose effect is folded into the intercept. A
	// row carrying it contributes nothing beyond the intercept for this
	// field; every other level's coefficient is read relative to it.
	Level string `json:"level"`
}

type ModelResidualCorrelationFidelity struct {
	// Fields names the participants in COMPONENT order — the order
	// buildResidualCorrelator assigned them, which is drawer order,
	// which is schema field order. It is the generation-side
	// participant set, so a modelled field absent from it took no part
	// in the correlated draw (its model was dropped, or no surviving
	// correlation named it) and drew an independent residual.
	Fields []string `json:"fields"`
	// Compared is the number of applied residual correlations that
	// produced a recovered figure. Flagged counts those whose Delta
	// exceeds ResidualRecoveryTolerance. Both count over every compared
	// pair, not over the bounded Pairs listing.
	Compared int `json:"compared"`
	Flagged  int `json:"flagged,omitempty"`
	// MeanDelta / MaxDelta are the mean and maximum absolute gap over
	// every compared pair — the two figures that make the truncated
	// listing safe to read, since they cannot be improved by what the
	// listing left out.
	MeanDelta float64 `json:"mean_delta"`
	MaxDelta  float64 `json:"max_delta"`
	// Pairs is the worst maxResidualRecoveryPairs compared pairs by
	// absolute Delta, largest first, ties broken by (A, B) so the
	// listing is a function of the numbers alone. Omitted is how many
	// compared pairs it left out; every one of them recovered at least
	// as well as the last entry listed.
	Pairs   []*ModelResidualPairFidelity `json:"pairs,omitempty"`
	Omitted int                          `json:"omitted,omitempty"`
	// Unmeasured holds pairs generation applied that the synthetic
	// partition could not produce a correlation for, each with its
	// reason, capped the same way. It is a separate list rather than an
	// Error field on ModelResidualPairFidelity for the same structural
	// reason ResidualCorrelationProfile splits its two lists: an
	// unmeasured pair has no rho, and a struct with a rho slot in it is
	// how a zero gets written into one.
	Unmeasured        []*ModelResidualUnmeasuredPair `json:"unmeasured,omitempty"`
	UnmeasuredOmitted int                            `json:"unmeasured_omitted,omitempty"`
}

type ModelResidualPairFidelity struct {
	A string `json:"a"`
	B string `json:"b"`
	// CapturedRho is CorrelationSpec.Correlation — the figure
	// buildResidualCorrelator was asked to impose, which is the source
	// residual correlation the profile captured. RecoveredRho is the
	// Pearson correlation between the two models' REFIT residuals over
	// the synthetic partition, on the latent scale. Delta is their
	// absolute difference.
	CapturedRho  float64 `json:"captured_rho"`
	RecoveredRho float64 `json:"recovered_rho"`
	Delta        float64 `json:"delta"`
	// N is the number of synthetic rows carrying BOTH residuals — the
	// realized pair's own co-occurrence count within the refit row cap,
	// not the cohort's row count and not either model's NObs.
	N int `json:"n"`
	// Flagged is true when Delta exceeds ResidualRecoveryTolerance.
	Flagged bool `json:"flagged,omitempty"`
}

type ModelResidualUnmeasuredPair struct {
	A string `json:"a"`
	B string `json:"b"`
	// CapturedRho still rides, because the reader needs to know how much
	// structure went unverified: an unmeasured pair captured at 0.8 is a
	// far larger hole in the report than one captured at 0.02.
	CapturedRho float64 `json:"captured_rho"`
	// N is how many synthetic rows DID carry both residuals — zero or a
	// handful for ResidualUnmeasuredNoOverlap, and however many
	// overlapped for ResidualUnmeasuredNoVariance. Zero and meaningless
	// for ResidualUnmeasuredNoModelFit, where one side has no residual
	// vector at all.
	N int `json:"n"`
	// Reason is ResidualUnmeasuredNoOverlap, ResidualUnmeasuredNoVariance
	// or ResidualUnmeasuredNoModelFit.
	Reason string `json:"reason"`
}

type NumericPairProfile struct {
	A   string  `json:"a"`
	B   string  `json:"b"`
	Rho float64 `json:"rho"`
	// N is the number of rows in the capture where both A and B were
	// simultaneously non-null — the pair's true co-occurrence count,
	// unlike Profile.Pairwise's Rho (computed from two independently
	// capped reservoirs that can drift out of alignment once either
	// field has nulls).
	N int `json:"n"`
}

type NumericProfile struct {
	Min  float64 `json:"min"`
	Max  float64 `json:"max"`
	Mean float64 `json:"mean"`
	Std  float64 `json:"std"`
	// Percentiles holds {p1, p5, p25, p50, p75, p95, p99} when
	// IncludeStats was on; nil otherwise.
	Percentiles []float64 `json:"percentiles,omitempty"`
	// Shape carries a fitted 2-component Gaussian mixture, captured
	// only when ProfileOptions.FitShape was set AND the mixture is a
	// genuine improvement over the plain Mean/Std normal above (BIC
	// selection — see fitNumericShape in shape.go). Additive +
	// omitempty: absent from every profile document captured without
	// --fit-shape (including every pre-existing document), and absent
	// for any numeric field whose observed distribution is already
	// close to normal — SpecFromProfile falls back to the ordinary
	// Mean/Std/Min/Max normal reconstruction whenever this is nil,
	// exactly as it always has.
	Shape *ShapeProfile `json:"shape,omitempty"`
}

type Options struct {
	// Seed makes the output deterministic. Same spec + same seed must
	// produce a byte-identical .pulse file.
	Seed int64

	// SourceCohort activates the tagged top-up path (AugmentFromProfile)
	// when non-empty: real rows from this cohort are copied into the
	// output tagged _synthetic=false, spec.RowCount new rows are
	// generated and tagged _synthetic=true, and the source is never
	// opened for write. Leaving it empty (the default) reproduces
	// today's behavior exactly — a plain synthesis with no tag column,
	// used by synth from-schema and any caller building a Spec directly.
	SourceCohort string

	// FidelityReportPath activates the post-generation fidelity report
	// (`synth from-profile --fidelity-report`) when non-empty: after
	// generation completes, AugmentFromProfile drives TEST_KS (numeric
	// fields, SplitBy=_synthetic) / TEST_CHISQ (categorical fields,
	// contingency against _synthetic) against the tagged output cohort
	// and writes the resulting FidelityReport JSON document to this
	// path. Only meaningful alongside SourceCohort — the plain Synth
	// path (SourceCohort empty) has no _synthetic partition to compare
	// against and ignores this field entirely.
	FidelityReportPath string

	// FidelityWarnings, when non-empty, are copied verbatim onto the
	// written FidelityReport's own Warnings slot — the mechanism that
	// surfaces thin-pair warnings (e.g. Profile.Warnings from a
	// `profile create --conditional` capture) beside the pairwise
	// correlation deltas they qualify in the same document (FR-18),
	// rather than requiring a caller to cross-reference the profile
	// file separately. This package has no Profile type of its own to
	// read Warnings from directly — the caller (internal/cli's `synth
	// from-profile`, or any programmatic caller building Options from a
	// Profile it already has in hand) is expected to populate this.
	// Ignored when FidelityReportPath is empty, exactly like every
	// other Fidelity* field.
	FidelityWarnings []string
}

type PairwiseFidelity struct {
	A            string  `json:"a"`
	B            string  `json:"b"`
	SourceRho    float64 `json:"source_rho"`
	SyntheticRho float64 `json:"synthetic_rho,omitempty"`
	// Delta is abs(SourceRho - SyntheticRho). Zero-valued and
	// meaningless when Error is set.
	Delta float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with both A and B
	// simultaneously non-null — the realized pair's own co-occurrence
	// count, distinct from (and typically much larger than) the source
	// capture's own N.
	N int `json:"n"`
	// Error is set instead of SyntheticRho/Delta when the synthetic
	// partition could not produce a defined correlation for this pair
	// (fewer than two co-occurring non-null observations) — mirrors
	// FieldFidelity's own per-field failure contract: one pair's
	// failure never withholds the rest of the report.
	Error string `json:"error,omitempty"`
}

type Profile struct {
	RowCount int               `json:"row_count"`
	Fields   []FieldProfile    `json:"fields"`
	Pairwise []CorrelationStat `json:"pairwise,omitempty"`
	// Conditional carries joint-reconstruction structure captured only
	// when ProfileOptions.IncludeConditional was set. Additive and
	// omitempty: absent on every profile document captured without
	// --conditional, including every pre-existing one, and
	// SpecFromProfile treats a nil Conditional exactly as it always
	// has (falling back to Pairwise).
	Conditional *ConditionalProfile `json:"conditional,omitempty"`
	// Models carries the per-numeric additive linear predictors captured
	// only when ProfileOptions.FitModels was set (`profile create
	// --fit-models`). Additive and omitempty, exactly like Conditional:
	// the key is entirely absent from every document captured without
	// the flag — including every document written before this section
	// existed — and SpecFromProfile treats an empty Models exactly as it
	// always has, falling back to Conditional's numeric-target pairs.
	//
	// The section is a REPLACEMENT for those pairs, not a supplement:
	// when it is present SpecFromProfile stops populating
	// Spec.CategoricalNumericPairs / Spec.SetNumericPairs entirely,
	// because one additive model already accounts for every predictor at
	// once and the pairs exist only to be applied one overwrite at a
	// time. See SpecFromProfile for the full rule.
	//
	// Not every part of a captured FieldModel is durable: the residual
	// reservoir and the internal design-column names are `json:"-"`, so
	// a Profile read back from JSON carries coefficients but no
	// per-row residuals. See FieldModel for why.
	Models []FieldModel `json:"models,omitempty"`
	// ResidualCorrelations carries the correlation structure among the
	// fitted model residuals, captured only when
	// ProfileOptions.FitResidualCorrelations was set (`profile create
	// --residual-correlations`) alongside --fit-models. Additive and
	// omitempty, exactly like Conditional and Models.
	//
	// It is a SEPARATE measurement from Pairwise and
	// Conditional.NumericPairs, not a refinement of either: those two
	// correlate raw values and this correlates what is left after each
	// field's model has explained what it can. All three can coexist in
	// one document, and none of them replaces another.
	//
	// SpecFromProfile translates the MEASURED pairs onto
	// Spec.ResidualCorrelations, where generation consumes them as the
	// shared source of randomness every modelled field's residual is
	// drawn from (synth/residual_draw.go). The `unmeasured` list is
	// translated into nothing at all: an absent pair is completed as
	// independent by the generator, which counts and names the
	// assumption, whereas writing it out as a zero would present it as
	// a measurement. See ResidualCorrelationProfile.
	ResidualCorrelations *ResidualCorrelationProfile `json:"residual_correlations,omitempty"`
	Warnings             []string                    `json:"warnings,omitempty"`
	Meta                 map[string]any              `json:"meta,omitempty"`
}

type ProfileOptions struct {
	// TopK is the number of top categorical values to capture per
	// categorical field. Defaults to 32 when zero.
	TopK int
	// IncludeStats turns on percentile / stdev / kurtosis collection.
	// When false, only mean / min / max / null-rate are recorded for
	// numeric fields. Defaults to true.
	IncludeStats bool
	// IncludeCorrelations enables pairwise Pearson correlation capture
	// between numeric fields. Off by default to keep profile size bounded.
	IncludeCorrelations bool
	// CorrelationTopK caps the number of strongest |rho| pairs retained.
	// Defaults to 16 when IncludeCorrelations or IncludeConditional is
	// true.
	CorrelationTopK int
	// IncludeConditional enables capture of the joint reconstruction
	// structure (`profile create --conditional`): for numeric-numeric
	// field pairs, a row-aligned correlation + observation count
	// (Profile.Conditional.NumericPairs) that SpecFromProfile uses to
	// drive the exact conditional-Gaussian reconstruction in
	// synth/copula.go, in place of the plain (and, for cohorts with
	// nulls, only approximately aligned) Pairwise correlation stats.
	// Off by default; the profile document's `conditional` key is
	// entirely absent (omitempty) when this is false, and every
	// pre-existing profile document — which has no such key — remains
	// valid SpecFromProfile input either way.
	IncludeConditional bool
	// SampleLimit caps the number of records ingested for the profile.
	// Zero = unlimited.
	SampleLimit int
	// FitShape enables shape-fitting for numeric fields
	// (`profile create --fit-shape`, E4-S2): instead of always
	// summarizing a numeric field as a single normal(mean, std),
	// attempt a 2-component Gaussian mixture fit and keep it — as
	// FieldProfile.Numeric.Shape — only when it is a genuine
	// improvement over the plain normal by BIC (see fitNumericShape in
	// shape.go for the full selection rule and its documented
	// limitations). Off by default; a field whose fit is not kept (not
	// enough data, or the source is already close to normal) carries no
	// Shape, and SpecFromProfile reconstructs it exactly as it always
	// has. Forces sample retention (as if IncludeStats were also set)
	// for numeric fields so there is something to fit against, even
	// when IncludeStats itself is false.
	FitShape bool
	// FitModels enables per-numeric linear model capture
	// (`profile create --fit-models`, E1-S2): for each numeric field,
	// fit one linear predictor of that field on the cohort's
	// categorical LEVELS and set OPTIONS, driven off this same single
	// scan, and retain the fitted coefficients, the residual scale and
	// a bounded row-aligned residual vector.
	//
	// Off by default. When on, the fits are written to the document as
	// the additive `models` section (Profile.Models) and nothing else in
	// the document moves: every other section is byte-identical to the
	// same capture with the flag off, and a capture WITHOUT the flag
	// emits no `models` key at all. It draws from its own RNG stream, so
	// --conditional's captured output does not depend on whether this
	// flag was also passed.
	//
	// It does change what SpecFromProfile builds: a document carrying
	// `models` reconstructs its numeric fields from the additive linear
	// predictors instead of from --conditional's one-pair-at-a-time
	// numeric-target arms, which are then not populated at all.
	FitModels bool
	// FitResidualCorrelations enables capture of the correlation
	// structure among the fitted model RESIDUALS
	// (`profile create --residual-correlations`, E3-S1), written as the
	// additive `residual_correlations` section
	// (Profile.ResidualCorrelations).
	//
	// It requires FitModels — residuals are what the models produce, so
	// with no models there is nothing to correlate, and asking for this
	// alone warns rather than silently emitting nothing.
	//
	// It is deliberately its own flag rather than an implication of
	// FitModels: the section is QUADRATIC in modelled fields (105
	// targets is 5,460 pairs) and no existing caller asked for it, so
	// switching it on with --fit-models would grow every such document
	// without notice. It is likewise deliberately NOT wired to
	// IncludeCorrelations / CorrelationTopK, which keep their exact
	// meaning — the strongest top-K pairs of RAW values. This section is
	// a different measurement (residuals, not raw values) with a
	// different shape (a full submatrix, not a ranked list), and sharing
	// either flag would silently change what those two already mean.
	FitResidualCorrelations bool
	// Seed drives the reservoir-sampling RNG used by
	// IncludeConditional's categorical-categorical contingency capture
	// (E7-S2) once the source cohort exceeds conditionalJointCap rows.
	// Zero (the default) is itself a valid, deterministic seed — same
	// as synth.Options.Seed's convention — so two profile runs against
	// the same source with the same Seed (including the unset zero
	// value) produce byte-identical captured output. It does not affect
	// any other capture path: every other section in this file is
	// either an exact online accumulation or a first-N reservoir with
	// no random draws.
	Seed int64
}

type ResidualCorrelation struct {
	A string `json:"a"`
	B string `json:"b"`
	// Rho is the Pearson correlation between the two fields' fitted
	// residuals over the rows where both exist. Deliberately NOT
	// omitempty: a measured zero must appear on the wire, because
	// omitting it would make it indistinguishable from an unmeasured
	// pair on read-back and would reintroduce exactly the ambiguity
	// this section removes.
	Rho float64 `json:"rho"`
	// N is the number of retained rows carrying both residuals — the
	// pair's true co-occurrence count within the capture's residual
	// reservoir, not the cohort's row count and not either model's
	// NObs.
	N int `json:"n"`
}

type ResidualCorrelationProfile struct {
	// Fields names the model residuals this submatrix covers, sorted.
	// It is the participant set, and it is written down so a reader can
	// tell an absent pair from an absent FIELD — a field with no model
	// has no residual and was never a candidate, which is a different
	// statement from a modelled pair that could not be measured.
	Fields []string `json:"fields"`
	// Pairs holds every MEASURED pair, in (A, B) sorted order.
	Pairs []ResidualCorrelation `json:"pairs,omitempty"`
	// Unmeasured holds every pair among Fields that Pairs does not, each
	// with its reason. Present whenever there is one; a fully measured
	// submatrix omits the key entirely, which is itself the signal that
	// nothing was assumed.
	Unmeasured []ResidualUnmeasuredPair `json:"unmeasured,omitempty"`
}

type ResidualUnmeasuredPair struct {
	A string `json:"a"`
	B string `json:"b"`
	// N is how many rows DID carry both residuals — zero or a handful
	// for ResidualUnmeasuredNoOverlap, and however many overlapped for
	// ResidualUnmeasuredNoVariance. It is the diagnostic that separates
	// "these two fields never co-occur" from "they co-occur constantly
	// and one of them is fitted exactly".
	N int `json:"n"`
	// Reason is one of the ResidualUnmeasured* constants. There is no
	// rho key on this struct at all — an unmeasured pair has no
	// correlation to report, and providing a slot for one is how a zero
	// gets written into it.
	Reason string `json:"reason"`
}

type Result struct {
	RowsGenerated int      `json:"rows_generated"`
	RowsRejected  int      `json:"rows_rejected"`
	OutputPath    string   `json:"output_path"`
	Warnings      []string `json:"warnings,omitempty"`

	// FidelityReportPath echoes Options.FidelityReportPath when a
	// fidelity report was written alongside the output cohort. Empty
	// when the flag was not set.
	FidelityReportPath string `json:"fidelity_report_path,omitempty"`
}

type SetCategoricalPairFidelity struct {
	Set         string  `json:"set"`
	Option      string  `json:"option"`
	Categorical string  `json:"categorical"`
	Delta       float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with both the set field
	// and the categorical field simultaneously non-null — the realized
	// pair's own co-occurrence count.
	N int `json:"n"`
	// Error is set instead of Delta when the synthetic partition
	// produced no co-occurring observation at all for this pair —
	// mirrors CategoricalPairFidelity's own per-pair contract.
	Error string `json:"error,omitempty"`
}

type SetCategoricalPairProfile struct {
	Set         string            `json:"set"`
	Option      string            `json:"option"`
	Categorical string            `json:"categorical"`
	Cells       []ContingencyCell `json:"cells"`
	// N is the number of rows where both the set field and the
	// categorical field were simultaneously non-null — this option's
	// true co-occurrence count, mirroring CategoricalPairProfile.N.
	N int `json:"n"`
}

type SetCategoricalPairSpec struct {
	Set         string                    `json:"set"`
	Option      string                    `json:"option"`
	Categorical string                    `json:"categorical"`
	Cells       []CategoricalPairCellSpec `json:"cells"`
}

type SetFieldFidelity struct {
	Field   string               `json:"field"`
	Options []*SetOptionFidelity `json:"options,omitempty"`
}

type SetNumericPairFidelity struct {
	Set        string                                `json:"set"`
	Option     string                                `json:"option"`
	Numeric    string                                `json:"numeric"`
	Categories []*CategoricalNumericCategoryFidelity `json:"categories,omitempty"`
}

type SetNumericPairProfile struct {
	Set     string `json:"set"`
	Option  string `json:"option"`
	Numeric string `json:"numeric"`
	// Categories carries at most two entries, keyed "selected" /
	// "not_selected" — reuses CategoricalNumericCategoryStat verbatim
	// rather than a bespoke two-field struct, so a caller already
	// handling CategoricalNumericPairProfile.Categories needs no second
	// shape to learn.
	Categories []CategoricalNumericCategoryStat `json:"categories"`
	// N mirrors CategoricalNumericPairProfile.N: the total non-null
	// co-occurrence count summed across both categories.
	N int `json:"n"`
}

type SetNumericPairSpec struct {
	Set        string                           `json:"set"`
	Option     string                           `json:"option"`
	Numeric    string                           `json:"numeric"`
	Categories []CategoricalNumericCategorySpec `json:"categories"`
	Min        float64                          `json:"min,omitempty"`
	Max        float64                          `json:"max,omitempty"`
	HasClamp   bool                             `json:"has_clamp,omitempty"`
}

type SetOptionFidelity struct {
	Value string `json:"value"`
	// SourceFrequency/SyntheticFrequency are P(bit set) over each
	// partition's non-null rows of Field. Delta is
	// abs(SourceFrequency - SyntheticFrequency). SyntheticFrequency and
	// Delta are zero-valued and meaningless when Error is set.
	SourceFrequency    float64 `json:"source_frequency"`
	SyntheticFrequency float64 `json:"synthetic_frequency,omitempty"`
	Delta              float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with a non-null Field —
	// the realized synthetic partition's own denominator for this
	// option's frequency, shared across every option of the same field.
	N int `json:"n"`
	// Error is set instead of SyntheticFrequency/Delta when the
	// synthetic partition has no non-null observation of Field at all —
	// mirrors every other Fidelity entry kind's non-fatal per-entry
	// contract.
	Error string `json:"error,omitempty"`
}

type SetOptionStat struct {
	Value string `json:"value"`
	// Count is the number of non-null rows with this option's bit set.
	Count int `json:"count"`
	// Frequency is Count / SetProfile.N — P(bit set), the per-option
	// marginal Bernoulli rate.
	Frequency float64 `json:"frequency"`
}

type SetProfile struct {
	// N is the number of non-null rows Options' frequencies are computed
	// over — the same denominator FieldProfile.NullRate already implies,
	// surfaced here so a caller reading only the Set block can interpret
	// Count as a rate without cross-referencing NullRate.
	N int `json:"n"`
	// Options lists EVERY dictionary entry in bit/dictionary order (bit i
	// first) — NOT sorted by frequency like CategoricalProfile.Top — so
	// two profiles of the same schema are directly comparable
	// position-for-position regardless of which option happens to be
	// more popular.
	Options []SetOptionStat `json:"options"`
}

type SetSetPairFidelity struct {
	SetA    string  `json:"set_a"`
	OptionA string  `json:"option_a"`
	SetB    string  `json:"set_b"`
	OptionB string  `json:"option_b"`
	Delta   float64 `json:"delta,omitempty"`
	// N is the number of _synthetic=true rows with both set fields
	// simultaneously non-null — the realized pair's own co-occurrence
	// count.
	N int `json:"n"`
	// Error is set instead of Delta when the synthetic partition
	// produced no co-occurring observation at all for this pair.
	Error string `json:"error,omitempty"`
}

type SetSetPairProfile struct {
	SetA    string            `json:"set_a"`
	OptionA string            `json:"option_a"`
	SetB    string            `json:"set_b"`
	OptionB string            `json:"option_b"`
	Cells   []ContingencyCell `json:"cells"`
	// N is the number of rows where both fields were simultaneously
	// non-null — this option pair's true co-occurrence count.
	N int `json:"n"`
}

type SetSetPairSpec struct {
	SetA    string                    `json:"set_a"`
	OptionA string                    `json:"option_a"`
	SetB    string                    `json:"set_b"`
	OptionB string                    `json:"option_b"`
	Cells   []CategoricalPairCellSpec `json:"cells"`
}

type ShapeProfile struct {
	Means   []float64 `json:"means"`
	Stds    []float64 `json:"stds"`
	Weights []float64 `json:"weights"`
}

type Spec struct {
	// RowCount is the number of rows to generate. Required (>0).
	RowCount int `json:"row_count"`

	// Fields declares each column with its type and distribution.
	Fields []FieldSpec `json:"fields"`

	// Constraints, if non-empty, are evaluated per row by the expression
	// evaluator. Rows that fail any constraint are rejected and re-drawn.
	Constraints []ConstraintSpec `json:"constraints,omitempty"`

	// MaxRejectionRate caps the fraction of rejected rows during
	// constraint-driven rejection sampling. Defaults to 0.5 if zero.
	MaxRejectionRate float64 `json:"max_rejection_rate,omitempty"`

	// Correlations lists optional pairwise correlations to induce via
	// Gaussian copula post-processing. Each entry references two numeric
	// fields by name with a target Pearson correlation in [-1, 1]. Only
	// numeric fields can participate.
	Correlations []CorrelationSpec `json:"correlations,omitempty"`

	// CategoricalPairs lists optional categorical-categorical joint
	// structure to reproduce during generation: for each pair, field B
	// is resampled conditioned on field A's independently-drawn value
	// using the captured contingency cells, instead of being drawn from
	// its own unconditional marginal. Populated by SpecFromProfile from
	// ConditionalProfile.CategoricalPairs (`profile create --conditional`)
	// when present; absent (nil, the zero value) reproduces today's
	// independent-marginal behavior exactly — every pre-existing spec,
	// hand-authored or profile-derived, has no such key.
	CategoricalPairs []CategoricalPairSpec `json:"categorical_pairs,omitempty"`

	// CategoricalNumericPairs lists optional categorical-numeric
	// conditional structure to reproduce during generation: for each
	// pair, numeric field B is resampled from Normal(mean, std)
	// parameterized by categorical field A's drawn value, using the
	// captured per-category conditional mean/std, instead of being drawn
	// from its own unconditional normal. Populated by SpecFromProfile
	// from ConditionalProfile.CategoricalNumericPairs when present;
	// absent (nil) reproduces today's independent-marginal behavior
	// exactly.
	CategoricalNumericPairs []CategoricalNumericPairSpec `json:"categorical_numeric_pairs,omitempty"`

	// SetCategoricalPairs lists optional set-option x categorical joint
	// structure to reproduce during generation (E5-S3): for each pair,
	// the set field's declared Option (bit) is resampled from
	// Bernoulli(P(selected | categorical field's drawn value)) using the
	// captured contingency cells, instead of the option's own
	// independent marginal frequency. Populated by SpecFromProfile from
	// ConditionalProfile.SetCategoricalPairs when present; absent (nil)
	// reproduces the independent per-option marginal behavior exactly.
	SetCategoricalPairs []SetCategoricalPairSpec `json:"set_categorical_pairs,omitempty"`

	// SetNumericPairs lists optional set-option x numeric joint
	// structure to reproduce during generation (E5-S3): for each pair,
	// numeric field B is resampled from Normal(mean, std) conditioned on
	// whether the set field's declared Option (bit) was independently
	// drawn selected, using the captured per-bucket conditional
	// mean/std. Populated by SpecFromProfile from
	// ConditionalProfile.SetNumericPairs when present; absent (nil)
	// reproduces today's independent-marginal behavior exactly.
	SetNumericPairs []SetNumericPairSpec `json:"set_numeric_pairs,omitempty"`

	// SetSetPairs lists optional option x option joint structure between
	// two DIFFERENT set_* fields to reproduce during generation (E5-S3):
	// for each pair, set B's declared OptionB (bit) is resampled from
	// Bernoulli(P(selected | set A's OptionA drawn state)) using the
	// captured 2x2 contingency cells. Populated by SpecFromProfile from
	// ConditionalProfile.SetSetPairs when present; absent (nil)
	// reproduces the independent per-option marginal behavior exactly.
	SetSetPairs []SetSetPairSpec `json:"set_set_pairs,omitempty"`

	// Models lists optional per-numeric ADDITIVE linear predictors to
	// drive generation with, the Spec-facing counterpart of the profile
	// document's `models` section (`profile create --fit-models`).
	// Populated by SpecFromProfile from Profile.Models when present;
	// absent (nil, the zero value) reproduces today's behaviour exactly
	// — every pre-existing spec, hand-authored or profile-derived, has
	// no such key.
	//
	// Where CategoricalNumericPairs and SetNumericPairs each RESAMPLE a
	// numeric field from one paired field's conditional moments — so two
	// pairs naming the same numeric compete to be the last writer, and
	// resolveConflicts has to drop one — a model accounts for every
	// predictor in a single expression. SpecFromProfile therefore treats
	// the two as alternatives rather than layers: a profile carrying
	// `models` populates this slot and leaves both numeric-target pair
	// slots empty.
	Models []FieldModelSpec `json:"models,omitempty"`

	// ResidualCorrelations lists optional target correlations between
	// the RESIDUALS of two modelled numeric fields — the Spec-facing
	// counterpart of the profile document's `residual_correlations`
	// section (`profile create --residual-correlations`), populated by
	// SpecFromProfile from Profile.ResidualCorrelations when present.
	// Absent (nil, the zero value) reproduces the independent-residual
	// draw exactly: every modelled field takes its own fresh z, which
	// is what every pre-existing spec gets.
	//
	// This is a DIFFERENT quantity from Correlations above and the two
	// are not interchangeable. Correlations names a correlation between
	// two fields' VALUES and is realized by drawing both values from a
	// shared copula, which requires owning the value outright. Once a
	// field carries a model, its value is the model's to produce and
	// the only part still free to move is the residual — so this slot
	// correlates THAT, supplying each modelled field its component of
	// one shared correlated normal vector instead of overwriting what
	// the model drew. Imposing a value-scale figure on the residual
	// draw would apply every predictor the two targets share a second
	// time (see synth/residual_corr.go), which is why a Correlations
	// entry naming a modelled field is excluded rather than rerouted
	// here.
	//
	// Entries reuse CorrelationSpec verbatim: a pair of field names and
	// a rho, read on the residual scale. Both endpoints must carry a
	// model; an entry naming a field whose model did not survive is
	// dropped with a warning at generate() time, and validateSpec
	// refuses one outright for a hand-authored spec.
	ResidualCorrelations []CorrelationSpec `json:"residual_correlations,omitempty"`
}

type TestRunner func(physicalSchema, presentedSchema *encoding.Schema, records []byte, test *types.Test) (*types.TestResult, error)

type WarningGroup struct {
	// Kind is the stable, field-free label for this group. It never
	// interpolates a field name, so the group count is a count of
	// findings of one shape.
	Kind string
	// Attention is true when this kind means a requested capture or
	// generation step did NOT happen. False marks an expected outcome:
	// a complete, correct result the caller may nonetheless want to know
	// about.
	Attention bool
	// Members holds every warning classified into this group, in
	// production order. Callers that render to a terminal are expected
	// to show a bounded prefix; the full list stays in whatever document
	// the command already writes.
	Members []string
}