io

type CohortBuild struct {
	// contains filtered or unexported fields
}

type CohortBuildOptions struct {
	// Strict turns every PULSE_FIELD_DESCRIPTION_LOW_QUALITY finding
	// into a fatal error raised by NewCohortBuild.
	Strict bool
	// Overwrite lets the build replace an existing target. Without it
	// an existing target is refused (SERVICE_VALIDATION).
	Overwrite bool
	// Groups declares parent groups exactly as ImportJob.Groups does:
	// the PULSE_GROUP_* declaration checks and the viability gate's
	// width screen run in NewCohortBuild, the encode and the ratio
	// assessment in Close. Strict is the gate's strictness too (as
	// `--strict` is for import).
	Groups []GroupDecl
	// ElideConstants stores every field holding one value on every row
	// once, as ImportJob.ElideConstants does — decided by a full pass
	// over the spooled rows at Close.
	ElideConstants bool
	// RatioFloor is the viability gate's rows-per-tuple floor
	// (ImportJob.DedupRatioFloor); 0 selects the default.
	RatioFloor float64
	// ShardMaxRecords > 0 makes Close publish a shard archive at the
	// target instead of a single-file cohort: the spooled rows are cut
	// into consecutive shards of ShardMaxRecords rows (the last one
	// partial; one empty shard for an empty build), each a standalone
	// cohort with the ONE layout the full pass decided — so the group
	// gate and constant elision are decided once over all rows and every
	// shard carries the same schema, dictionaries and group layout —
	// staged as ShardName(k) in a directory beside the target and handed
	// to Archive in one call. Negative is refused; 0 is a single file.
	ShardMaxRecords int
	// Archive publishes the staged shard files as an archive at target,
	// atomically (the facade wires Pulse.CreateShardArchive), returning
	// its non-fatal warnings. Required when ShardMaxRecords > 0.
	Archive func(ctx context.Context, target string, shardPaths []string) ([]*errors.CodedError, error)
	// AppendShard adds one staged shard file to an EXISTING archive,
	// atomically (the facade wires Pulse.AddShard; the entry name is
	// the file's basename), returning its non-fatal warnings. Required
	// for an anchored target (`archive.pulse#name.pulse`): the build is
	// then ONE shard appended to that archive — see NewCohortBuild.
	AppendShard func(ctx context.Context, archivePath, shardPath string) ([]*errors.CodedError, error)
	// CheckAppend, when set, is the up-front cohesion check an anchored
	// build runs at NewCohortBuild against the archive's canonical
	// schema, with the build's (layout-recomputed) schema: it refuses
	// only what AddShard is certain to refuse, with AddShard's own code
	// and message (the facade wires the service's
	// PrecheckShardCohesion). AppendShard at Close stays the authority.
	CheckAppend func(ctx context.Context, archivePath string, schema *encoding.Schema) error
}

type CohortBuildReport struct {
	// Target is the cohort path written.
	Target string
	// Records is the number of rows written.
	Records int64
	// FormatVersion is the header version byte written — a function of
	// the schema's content, never a flag.
	FormatVersion byte
	// Schema is the schema written, dictionaries final.
	Schema *encoding.Schema
	// Warnings are the non-fatal findings: description quality, then
	// the viability gate's (PULSE_GROUP_TOO_NARROW from the width
	// screen, then PULSE_DEDUP_LOW_RATIO from the ratio assessment).
	Warnings []*errors.CodedError
	// Groups describes every declared group, in declaration order, as
	// ImportReport.Groups does. Nil when none was declared.
	Groups []GroupReport
	// ElidedConstants names the fields ElideConstants stored once.
	ElidedConstants []string
	// Replaced reports that a cohort already existed at Target and was
	// atomically replaced (Overwrite).
	Replaced bool
	// Shards names a sharded build's shard entries, in archive order,
	// or the one entry an anchored (append) build added. Nil for a
	// single-file build.
	Shards []string
}

type CohortSource = iocore.CohortSource

type CohortValidator = iocore.CohortValidator

type CohortWriter = iocore.CohortWriter

type ConvertJob struct {
	Source Reader
	Target Writer
	// Schema is the caller-authored schema, matched to the source's
	// columns by exact name exactly as ImportJob.Schema is.
	Schema      *encoding.Schema
	KeepPulseAt string // optional: also write intermediate .pulse
	SampleRows  int
	FS          afero.Fs
	// IgnoreExtra drops source columns (and late NDJSON / JSON-array
	// keys) the explicit Schema does not name, instead of refusing; see
	// ImportJob.IgnoreExtra.
	IgnoreExtra bool
	// FloatWidth is the width an inferred all-float column takes: "f64"
	// (default; "") or "f32"; anything else is
	// PULSE_IMPORT_FLOAT_WIDTH_INVALID. See ImportJob.FloatWidth. CLI
	// `--float`.
	FloatWidth string
	// Includes restricts the export half to the named schema fields,
	// in schema order. Nil / empty means write every field. The
	// intermediate .pulse file (when KeepPulseAt is set) always
	// carries the full schema — projection is an output-time overlay,
	// not an on-disk schema change. See ExportJob.Includes.
	Includes []string
	// Labels apply to the export phase only — the import side reads
	// raw source bytes and has no use for label translation. See
	// ExportJob.Labels.
	Labels []*types.LabelBinding
	// LabelResolver carries the runtime resolver. See
	// ExportJob.LabelResolver.
	LabelResolver LabelResolver
	// IncludeOverlays controls whether the export half embeds
	// Response.Overlays in the format-native sidecar. Identical
	// tri-state semantics to ExportJob.IncludeOverlays — nil defaults
	// to emit, *true forces emit, *false drops overlays. The
	// intermediate .pulse file (when KeepPulseAt is set) is byte-
	// identical regardless of this flag because overlays are an
	// export-side concern only; the .pulse byte format never carries
	// overlay payloads. See ExportJob.IncludeOverlays for the per-
	// format wire shape.
	IncludeOverlays *bool
	// Overlays carries the Response.Overlays layers the convert should
	// embed in the target adapter. Identical semantics to
	// ExportJob.Overlays — the export half of the convert dispatches the
	// slice via SetOverlays on writers satisfying OverlayAwareWriter
	// when IncludeOverlays does not explicitly opt out. The intermediate
	// .pulse file (when KeepPulseAt is set) never carries overlay
	// payloads.
	Overlays []*types.OverlayLayer
	// SourceTZ, ColumnSourceTZ and DSTPolicy read the source's naive
	// datetime literals in a source zone, with ImportJob's exact
	// semantics and refusals (see ImportJob.SourceTZ). Convert is a TEXT
	// pass, so a zoned naive literal reaches the target as the canonical
	// UTC literal of the instant it names (`2026-01-15T14:00:00Z`) — the
	// value a source-zone import stores — while a literal with its own
	// `Z` / offset, an unparseable cell and every non-zoned column pass
	// through verbatim. The KeepPulseAt intermediate is imported in the
	// same zone. Empty (the default) changes nothing.
	SourceTZ       string
	ColumnSourceTZ map[string]string
	DSTPolicy      DSTPolicy
}

type ConvertReport struct {
	RowsConverted   int
	Schema          *encoding.Schema
	RowErrors       []RowError
	LabelWarnings   []LabelWarning
	OverlayWarnings []*errors.CodedError
	// SourceWarnings carries the source Reader's non-fatal diagnostics
	// — see ImportReport.SourceWarnings. Distinct from OverlayWarnings,
	// which come from the TARGET Writer.
	SourceWarnings []*errors.CodedError
	// TargetWarnings carries the target Writer's non-fatal diagnostics
	// — see ExportReport.TargetWarnings.
	TargetWarnings []*errors.CodedError
	// WidthWarnings carries one PULSE_IMPORT_WIDTH_PROMOTED warning per
	// field of an INFERRED schema the row pass promoted past its
	// sample-inferred width (see ImportReport.WidthWarnings); Schema
	// carries the promoted types. Nil for a declared schema, and when
	// nothing outgrew its width.
	WidthWarnings []*errors.CodedError `json:"WidthWarnings,omitempty"`
	// ZoneWarnings carries the one PULSE_IMPORT_DST_RESOLVED warning a
	// ConvertJob.DSTPolicy of earlier / later raises when it resolved a
	// source-zone local time (see ImportReport.ZoneWarnings).
	ZoneWarnings []*errors.CodedError `json:"ZoneWarnings,omitempty"`
}

type ConvertSource = iocore.ConvertSource

type DSTPolicy string

type DedupJob struct {
	// FS is the filesystem both Source and Target live on. Required.
	FS afero.Fs
	// Source is the cohort to convert. Any supported format version.
	Source string
	// Target is where the converted cohort is written. Empty (or equal
	// to Source) rewrites Source IN PLACE. A different path must not
	// exist yet: the job never overwrites a file it was not asked to
	// convert.
	Target string
	// Groups declares the parent groups, exactly as ImportJob.Groups.
	Groups []GroupDecl
	// ElideConstants stores every field holding one value on every
	// record once, as ImportJob.ElideConstants. Declared members are
	// never elided.
	ElideConstants bool
	// DedupRatioFloor and StrictDedup are the viability gate's policy,
	// as on ImportJob.
	DedupRatioFloor float64
	StrictDedup     bool
	// SuggestGroups runs candidate detection over the source (before any
	// rewrite) and returns it on DedupReport.GroupCandidates. With no
	// Groups and no ElideConstants the job is detection-only and writes
	// nothing.
	SuggestGroups bool
}

type DedupReport struct {
	// Source and Target are the paths read and written; InPlace is true
	// when they are the same file.
	Source  string `json:"source"`
	Target  string `json:"target"`
	InPlace bool   `json:"in_place"`
	// Rewritten is false when nothing was written: a detection-only run,
	// or one whose every declared group was dropped by the width floor
	// with no elision to apply. The target is then untouched.
	Rewritten bool `json:"rewritten"`
	// Records is the record count of the source (and of the rewrite).
	Records int64 `json:"records"`
	// FormatVersionBefore/After, BytesBefore/After and
	// StrideBefore/After describe the source and the written cohort.
	// Without a rewrite the After figures repeat the Before ones.
	FormatVersionBefore int   `json:"format_version_before"`
	FormatVersionAfter  int   `json:"format_version_after"`
	BytesBefore         int64 `json:"bytes_before"`
	BytesAfter          int64 `json:"bytes_after"`
	StrideBefore        int   `json:"stride_before"`
	StrideAfter         int   `json:"stride_after"`
	// ElidedConstants names the fields ElideConstants stored once.
	ElidedConstants []string `json:"elided_constants,omitempty"`
	// Groups reports every declaration's verdict and figures, in
	// declaration order — the same GroupReport import produces.
	Groups []GroupReport `json:"groups,omitempty"`
	// GroupWarnings carries the viability gate's findings
	// (PULSE_GROUP_TOO_NARROW, PULSE_DEDUP_LOW_RATIO).
	GroupWarnings []*errors.CodedError `json:"group_warnings,omitempty"`
	// GroupCandidates is the SuggestGroups detection report, measured
	// over the source's records.
	GroupCandidates *GroupDetection `json:"group_candidates,omitempty"`
}

type DiscardableWriter = iocore.DiscardableWriter

type ExportJob struct {
	Source string // input .pulse path
	Target Writer
	FS     afero.Fs
	// Includes restricts the export to the named source-schema fields,
	// in source-schema order. Nil / empty means export every field
	// (prior behaviour). Names must match Schema.Fields[i].Name
	// exactly. Unknown names return PULSE_EXPORT_FIELD_UNKNOWN with
	// the offending name + list of known fields. Label augment
	// siblings ("<field>_label") are emitted only for included source
	// fields; replace mode applies to included fields as before.
	Includes []string
	// Labels rewrites or augments categorical column values during
	// export using embedder-registered label tables. Bindings name a
	// categorical field plus a label table; mode=replace overwrites
	// the column value, mode=augment emits a sibling "<field>_label"
	// column. See types.LabelBinding for semantics. Nil/empty means
	// the exporter writes raw resolved categorical values, matching
	// pre-label behaviour.
	Labels []*types.LabelBinding
	// LabelResolver carries the runtime resolver built from Labels +
	// the pulse Service's registered LabelTables. The pulse.Export
	// facade builds the resolver and sets this field; callers using
	// io.ExportJob directly without a Pulse instance must supply a
	// satisfying implementation (typically wrapping
	// processing.BuildLabelResolver). Nil means no label translation.
	LabelResolver LabelResolver
	// IncludeOverlays controls whether Response.Overlays land in the
	// exported artefact for adapters that can carry them. The slot is
	// a tri-state pointer:
	//
	//   - nil   ⇒ DEFAULT EMIT — when the host Response carries any
	//             overlay layers the adapter embeds them in the
	//             format-native sidecar (additive-by-default, per PRD
	//             §6 FR-L5). Output for an overlay-free Response is
	//             byte-identical to a pre-overlay ExportJob.
	//   - true  ⇒ explicit emit — same behaviour as nil for downstream
	//             adapters, but the explicit toggle distinguishes the
	//             intent in canonical-hash composition so cache keys
	//             differ between "default" and "explicit yes".
	//   - false ⇒ opt out — overlays are dropped on export even when
	//             the host Response carries layers. Output is byte-
	//             identical to a pre-overlay ExportJob against the
	//             same host result.
	//
	// Per-format semantics (research/export-embedding-shape.md):
	//
	//   - Arrow / Parquet — overlays ride as a top-level
	//     LIST<STRUCT> "overlays" field group emitted once in the
	//     first record-batch / row-group; the host record stream is
	//     byte-identical to the opt-out shape.
	//   - Excel — one sheet per layer named "__overlay_<layer_name>";
	//     the host workbook sheet is unchanged from the opt-out shape.
	//   - NDJSON — single trailing line {"_overlays": [...]} after the
	//     last host-record line; the host stream is byte-identical
	//     until the trailer.
	//   - CSV / TSV — warn-and-skip. The dispatcher emits one
	//     PULSE_OVERLAY_EXPORT_CSV_UNSUPPORTED warning and writes the
	//     host CSV verbatim; no overlay output lands. Setting
	//     IncludeOverlays=false suppresses the warning while keeping
	//     the same CSV body.
	//
	// The pointer shape mirrors the Includes-slot precedent (a nil
	// slice means "default": include every field) — nil here means
	// "default: emit overlays when present" rather than "explicit no"
	// because Go bool zero is false. Marshalling via encoding/json
	// honours `omitempty` so nil pointers do not appear in canonical
	// JSON; explicit *true and *false both serialise as booleans and
	// produce distinct canonical-hash keys via ExportJob.Hash().
	IncludeOverlays *bool
	// Overlays carries the Response.Overlays layers the export should
	// embed in the target adapter's format-native sidecar (Arrow /
	// Parquet column family, Excel sheets, NDJSON trailer) per
	// research/export-embedding-shape.md. ExportJob.Run dispatches the
	// slice to writers satisfying OverlayAwareWriter when IncludeOverlays
	// does not explicitly opt out. The CSV / TSV warn-and-skip family
	// also accepts the slice through SetOverlays but emits a
	// PULSE_OVERLAY_EXPORT_CSV_UNSUPPORTED warning (surfaced on
	// ExportReport.OverlayWarnings) instead of writing the layers into
	// the CSV body. nil / empty leaves the export byte-identical to a
	// pre-overlay job. The slot is intentionally NOT part of the
	// canonical-hash composition — two jobs sharing IncludeOverlays /
	// Source / Includes / Labels but differing in the overlay payload
	// resolve through the SAME cache key because the cache identity is
	// the export REQUEST, not the response.
	Overlays []*types.OverlayLayer
	// TimeZone renders every `datetime` cell on this IANA zone's wall
	// clock with its numeric offset — 2026-03-29T08:00:00+05:30 under
	// "Asia/Kolkata" — instead of the canonical UTC `…Z` literal. The
	// literal names the same instant, so re-importing the export (the
	// importer honours an explicit offset) round-trips every value
	// exactly. `date` cells are calendar days and never move.
	//
	// Validated by temporal.LoadZone at Run / Predict time: IANA
	// Area/Location names or "UTC" only, anything else is
	// PULSE_TIMEZONE_UNKNOWN. Empty (the default), "UTC" and every
	// UTC-equivalent zone ("Etc/UTC") leave the output byte-identical
	// to a zone-free export.
	//
	// Every row-stream target (csv, tsv, ndjson, jsonarray, excel, and
	// the UTF8 datetime columns of arrow / parquet) carries the local
	// literal. The `.sav` writer encodes from raw storage, where a
	// DATETIME is a naive wall-clock count with no offset slot, so it
	// refuses a non-UTC zone with PULSE_SPSS_EXPORT_UNSUPPORTED rather
	// than silently ignore or lossily apply it.
	TimeZone string
}

type ExportReport struct {
	RowsExported    int
	RowErrors       []RowError
	LabelWarnings   []LabelWarning
	OverlayWarnings []*errors.CodedError
	// TargetWarnings carries the non-fatal diagnostics the target
	// Writer surfaced through the optional TargetWarningEmitter
	// contract — today the PULSE_SPSS_SIDECAR_* and
	// PULSE_SPSS_NAME_SANITIZED codes the `.sav` encode raises. Nil
	// for targets that do not implement the interface and for those
	// that implement it and raised nothing, so the report shape is
	// unchanged for every pre-existing adapter.
	TargetWarnings []*errors.CodedError
}

type GroupCandidate struct {
	Label string `json:"label"`
	// Key and Members are the structured declaration — the GroupDecl
	// shape {key, members}; Declaration is the same as a ready-to-paste
	// --group value (empty when a name holds ',' or ':', which the CLI
	// form cannot carry).
	Key         []string `json:"key"`
	Members     []string `json:"members"`
	Declaration string   `json:"declaration"`
	// Suggested marks a candidate in GroupDetection.Suggested.
	// OverlapsWith names the better-ranked suggested candidate sharing
	// a field with this one (a field belongs to at most one group).
	Suggested    bool   `json:"suggested"`
	OverlapsWith string `json:"overlaps_with,omitempty"`
	// RejectedMembers were constant within the key across the
	// nomination window but varied later in the file.
	RejectedMembers []string `json:"rejected_members,omitempty"`
	// Verdict / Reason as the viability gate judges the group
	// (encoding.GroupVerdict*), or CandidateVerdictUnmeasured.
	Verdict         string  `json:"verdict"`
	Reason          string  `json:"reason,omitempty"`
	EntryCount      int     `json:"entry_count"`
	EntryWidth      int     `json:"entry_width"`
	MemberRowBytes  int     `json:"member_row_bytes"`
	IndexWidth      int     `json:"index_width"`
	DictionaryBytes int64   `json:"dictionary_bytes"`
	Ratio           float64 `json:"ratio"`
	BreakEvenRatio  float64 `json:"break_even_ratio"`
	RatioFloor      float64 `json:"ratio_floor"`
	ByteDelta       int64   `json:"byte_delta"`
	GrowsFile       bool    `json:"grows_file"`
	// ProjectedFileBytes is the exact cohort size with this group alone
	// declared (compare ImportProjection.FlatFileBytes).
	ProjectedFileBytes int64 `json:"projected_file_bytes"`
}

type GroupDecl struct {
	Key     []string `json:"key,omitempty"`
	Members []string `json:"members"`
}

type GroupDetection struct {
	// Rows is the number of rows every candidate was measured over —
	// the rows the import would write (row-error rows excluded).
	Rows int64 `json:"rows"`
	// WindowRows is how many leading rows nomination looked at;
	// WindowBound is the most it would have looked at for this schema.
	WindowRows  int `json:"window_rows"`
	WindowBound int `json:"window_bound"`
	// FieldsConsidered counts the fields eligible as key or member (not
	// declared in a group, not constant across the window).
	FieldsConsidered int `json:"fields_considered"`
	// KeysEvaluated / KeysOverBound: candidate keys whose dependencies
	// were evaluated, and those skipped past detectMaxKeys.
	KeysEvaluated int `json:"keys_evaluated"`
	KeysOverBound int `json:"keys_over_bound"`
	// CandidatesOverBound counts nominated candidates not confirmed
	// because detectMaxCandidates were already carried forward.
	CandidatesOverBound int `json:"candidates_over_bound"`
	// CandidatesDisproved counts nominated candidates every one of
	// whose members varied within the key later in the file.
	CandidatesDisproved int `json:"candidates_disproved"`
	// Candidates, best projected saving first. Every measured candidate
	// is listed with its verdict — one below the width or ratio floor
	// says so rather than disappearing.
	Candidates []GroupCandidate `json:"candidates"`
	// Suggested holds the --group values of the admitted candidates
	// that do not overlap a better one, in rank order: the declaration
	// set detection recommends. Empty when no candidate is viable.
	Suggested []string `json:"suggested"`
}

type GroupReport struct {
	// Label names the group in errors and output ("group 1 [key: id]").
	Label string `json:"label"`
	// Key and Members are as declared; Fields is every member in
	// logical (schema) order.
	Key     []string `json:"key,omitempty"`
	Members []string `json:"members"`
	Fields  []string `json:"fields"`
	// Verdict is admitted, low_ratio (written, with a
	// PULSE_DEDUP_LOW_RATIO warning) or dropped_too_narrow (not
	// written, PULSE_GROUP_TOO_NARROW). Reason is the code behind a
	// non-admitted verdict.
	Verdict string `json:"verdict"`
	Reason  string `json:"reason,omitempty"`
	// EntryCount is the number of distinct tuples stored.
	EntryCount int `json:"entry_count"`
	// EntryWidth is the bytes per dictionary entry (member bytes plus a
	// member null bitmap when any member is nullable).
	EntryWidth int `json:"entry_width"`
	// MemberRowBytes is the bytes the members occupied in each flat row.
	MemberRowBytes int `json:"member_row_bytes"`
	// IndexWidth is the bytes of the per-row index that replaces them.
	IndexWidth int `json:"index_width"`
	// DictionaryBytes is EntryCount × EntryWidth, resident in memory
	// whenever the cohort is open.
	DictionaryBytes int64 `json:"dictionary_bytes"`
	// Ratio is rows ÷ EntryCount; BreakEvenRatio the ratio below which
	// the group grows the file; RatioFloor the floor it was judged
	// against.
	Ratio          float64 `json:"ratio"`
	BreakEvenRatio float64 `json:"break_even_ratio"`
	RatioFloor     float64 `json:"ratio_floor"`
	// ByteDelta is the grouped bytes (indexes + dictionary + descriptor)
	// minus the members stored per row: negative is a saving.
	ByteDelta int64 `json:"byte_delta"`
}

type ImportJob struct {
	Source Reader
	Target string // output .pulse path
	// Schema is the caller-authored .pulse schema. When non-nil it wins
	// over everything: inference is skipped AND a SchemaAwareReader
	// source is not consulted at all. It is therefore the escape hatch
	// for overriding an authoritative source dictionary.
	Schema *encoding.Schema
	// SampleRows caps the inference sample: default 500, min 50.
	// Inert when Schema is supplied or the Source is a
	// SchemaAwareReader that yields a schema — there is nothing to
	// sample.
	SampleRows int
	FS         afero.Fs
	// SetInferenceMinPct configures the delimited-cell heuristic for
	// inferring set_* field types during the inference pass. A column
	// is classified as set_* when at least this percentage of non-
	// null sampled cells contain the inferred delimiter, the
	// post-split unique token count fits the widest set rung (≤256),
	// and the average post-split cardinality is > 1. Zero is treated
	// as 30%.
	// Ignored when Schema is supplied, and inert when the Source is a
	// SchemaAwareReader that yields a schema.
	SetInferenceMinPct int
	// FloatWidth is the type inference gives an all-float column: "f64"
	// (the default; "" means the same) or "f32", a size opt-in that
	// rounds every value to ~7 significant digits. Under "f32" an
	// out-of-sample value past f32's range still promotes the field to
	// f64 with PULSE_IMPORT_WIDTH_PROMOTED. Any other value refuses the
	// import (Run and Predict alike, with or without a Schema) with
	// PULSE_IMPORT_FLOAT_WIDTH_INVALID. Steers inference only: inert when
	// Schema is supplied or a SchemaAwareReader yields a schema, and a
	// ColumnTypeOverrides entry wins for its column. CLI `--float`.
	FloatWidth string
	// ColumnTypeOverrides bypasses inference for the named columns
	// (keyed by exact header name). Used by the managed-import
	// sidecar's force_type escape hatch. An override is applied exactly
	// or refused with PULSE_IMPORT_OVERRIDE_INVALID — never narrowed,
	// widened or ignored: an unknown column, a present value the forced
	// type cannot hold (on ANY row, not just the inference sample —
	// never a skipped row), and an override on a job with no inferred
	// schema (Schema supplied, or a SchemaAwareReader source whose
	// authoritative dictionary is not a guess to override) are all
	// refused. Re-type an authoritative column by supplying Schema.
	// See SchemaAwareReader and import_override.go.
	ColumnTypeOverrides map[string]encoding.FieldType
	// SourceTZ is the zone naive datetime literals are read in: an
	// IANA Area/Location name, "UTC", or a fixed offset "+HH:MM" /
	// "-HH:MM" (accepted here, never in a request). It applies to every
	// `datetime` field of the resolved schema and skips `date` fields
	// (a calendar day has no zone). A literal carrying its own `Z` or
	// offset names that instant and ignores the zone. Storage is
	// unchanged: the stored value is always UTC epoch seconds. Empty
	// (the default) reads naive literals as UTC, byte-identically to an
	// import without the field. An unknown zone is PULSE_TIMEZONE_UNKNOWN.
	SourceTZ string
	// ColumnSourceTZ sets the source zone per column (keyed by exact
	// field name, same spelling as SourceTZ) and wins over SourceTZ for
	// that column. A key naming no field, or a field that is not
	// `datetime` (a `date` column included), is refused with
	// SERVICE_VALIDATION before the row pass.
	ColumnSourceTZ map[string]string
	// DSTPolicy decides a naive literal a source zone's DST transition
	// makes ambiguous (shown twice) or nonexistent (skipped): DSTPolicyError
	// (the default; "" means it) refuses the import at the first such row
	// with PULSE_IMPORT_DST_AMBIGUOUS / PULSE_IMPORT_DST_NONEXISTENT;
	// DSTPolicyEarlier / DSTPolicyLater resolve with the pre- / post-
	// transition offset and report the counts in one
	// PULSE_IMPORT_DST_RESOLVED warning (ImportReport.ZoneWarnings).
	// Inert without a source zone and under a fixed-offset zone.
	DSTPolicy DSTPolicy
	// SetDelimiters maps set-typed column name to the delimiter the
	// importer should use when splitting cell strings into tokens
	// for per-row mask packing. Populated by inference; absent
	// entries (explicit-schema imports or non-set columns) fall back
	// to DefaultSetDelimiter ("|"). Inert when the Source is a
	// SchemaAwareReader that yields a schema: such a source builds
	// set_* masks from its own set definitions, not by splitting
	// delimited cell strings.
	SetDelimiters map[string]string
	// InferredSchema marks a supplied Schema as inference-originated so
	// the row pass promotes non-nullable fields to nullable on an
	// out-of-sample null instead of failing the row (see Run). When
	// Schema is nil this is implied — Run infers the schema and always
	// promotes. Set it explicitly only when handing Run a pre-built
	// schema that came from inference (e.g. ConvertJob's KeepPulseAt
	// re-import) so it inherits the same tolerance. Leave false for a
	// user-authored explicit schema, where a null in a declared
	// non-nullable field must stay a PULSE_IMPORT_ROW_ERROR.
	//
	// Inert when the Source is a SchemaAwareReader that yields a
	// schema. Such a schema is authoritative, not inference-originated,
	// so its declared nullability is a contract: an unexpected null is
	// a row error, never a silent widening. See SchemaAwareReader.
	InferredSchema bool
	// IgnoreExtra relaxes by-name matching of a user-authored Schema:
	// a source column no schema field names is DROPPED instead of
	// refusing the import with PULSE_IMPORT_SCHEMA_COLUMN_UNMAPPED, and a
	// keyed source's (NDJSON, JSON array) key first seen after the first
	// object is dropped instead of refused with
	// PULSE_IMPORT_SCHEMA_KEY_LATE. A schema field with no column is
	// still PULSE_IMPORT_SCHEMA_FIELD_MISSING. An explicit Schema is
	// always matched to the source header by exact name, never by
	// position. Inert without a user-authored Schema. CLI
	// `--ignore-extra`. Default false.
	IgnoreExtra bool
	// ElideConstants stores every field that holds exactly one value
	// (or is null) on EVERY imported row once, in the schema block, and
	// drops it from each record — a format 0x02 cohort, which binaries
	// older than 0x02 support cannot open. Constancy is decided over the
	// full row pass, never the inference sample. Nothing is elided (and
	// the output stays a byte-identical 0x01 cohort) for fewer than two
	// rows, when no field is constant, or when declaring the constants
	// would cost more schema bytes than it saves. When every field is
	// constant the lowest-index one stays in the row. Default false.
	// See encoding.PlanConstantElision.
	ElideConstants bool
	// Groups declares parent groups (see GroupDecl): each stores its
	// distinct member tuples ONCE in the schema block and every record a
	// u32 index into them — a format 0x02 cohort, which binaries older
	// than 0x02 support cannot open. Groups are independent; a field
	// belongs to at most one. Names are checked against the resolved
	// schema (inferred, authoritative or explicit alike) before the row
	// pass; the dictionaries are built in one pass over the imported
	// rows. Composes with ElideConstants: declared members are never
	// elided. Empty (the default) writes the 0x01 cohort unchanged.
	//
	// Every declared group passes the per-group viability gate
	// (encoding.DedupGate): one no wider than its u32 index is dropped
	// with PULSE_GROUP_TOO_NARROW; one below DedupRatioFloor rows per
	// distinct tuple, or that makes the file no smaller, is written with
	// PULSE_DEDUP_LOW_RATIO. Both land in ImportReport.GroupWarnings.
	Groups []GroupDecl
	// Vectors folds sibling source columns into native vec_f32 /
	// vec_f64 fields (see VectorDecl) — a format 0x02 cohort. Each
	// vector takes its first member's source position; its members
	// vanish from the cohort. Per row: every element cell empty is a
	// null vector (an inferred field is promoted to nullable), some
	// empty are NaN elements, a non-numeric cell fails the row
	// (PULSE_IMPORT_ROW_ERROR). Refused before any row is read with
	// PULSE_IMPORT_VECTOR_INVALID: a dim outside 2..1024, a pattern
	// matching no column, a column claimed twice (two vectors, or a
	// vector and a Groups entry), an unknown column, a bad kind or
	// scale, or an authoritative-schema (SPSS) source. With an explicit
	// Schema each vec field must be a declared vector of the schema's
	// dim. Empty (the default) imports unchanged. CLI `--vector` /
	// `--vector-kind`.
	Vectors []VectorDecl
	// DedupRatioFloor is the rows-per-distinct-tuple floor below which
	// a declared group draws PULSE_DEDUP_LOW_RATIO. Zero (the default)
	// selects encoding.DefaultDedupRatioFloor (2); a value in (0, 1]
	// disables the floor, leaving only the grows-the-file check.
	DedupRatioFloor float64
	// StrictDedup turns the viability gate's warnings into errors: Run
	// fails with the finding's own code (PULSE_GROUP_TOO_NARROW or
	// PULSE_DEDUP_LOW_RATIO) and writes nothing. It governs the gate
	// only — other import warnings are unaffected.
	StrictDedup bool
	// SuggestGroups makes Predict detect candidate parent groups —
	// single-field keys and the fields they determine — and measure
	// each over the full row pass (PredictReport.GroupCandidates). It
	// SUGGESTS only: Run ignores it, and a candidate is formed only when
	// declared through Groups. Costs Predict a full conversion pass (as
	// Groups and ElideConstants do) plus the bounded detection work
	// documented on GroupDetection.
	SuggestGroups bool
}

type ImportProjection struct {
	// RowsImported / RowErrors: rows that would import, and rows that
	// would be skipped with a row error.
	RowsImported int `json:"rows_imported"`
	RowErrors    int `json:"row_errors"`
	// FlatFileBytes is the exact size of the 0x01 cohort (no groups,
	// no elision).
	FlatFileBytes int64 `json:"flat_file_bytes"`
	// ProjectedFileBytes is the exact size of the cohort this job would
	// write: its admitted Groups plus, with ElideConstants, the
	// constant group. Equal to FlatFileBytes when neither applies.
	ProjectedFileBytes int64 `json:"projected_file_bytes"`
	// ElisionBytesSaved is what ElideConstants would save (0 when off
	// or nothing is elided).
	ElisionBytesSaved int64 `json:"elision_bytes_saved,omitempty"`
}

type ImportReport struct {
	RowsImported   int
	Schema         *encoding.Schema
	RowErrors      []RowError
	PromotedFields []string
	// WidthWarnings carries one PULSE_IMPORT_WIDTH_PROMOTED warning per
	// field the row pass promoted past its sample-inferred width
	// (categorical_* rung, integer width, or integer → f64), with
	// field / from / to / source_row in Details. Empty (and omitted from
	// JSON) when nothing outgrew its inferred width — always, for an
	// explicit or authoritative schema. See import_widen.go.
	WidthWarnings []*errors.CodedError `json:"WidthWarnings,omitempty"`
	// ZoneWarnings carries the one PULSE_IMPORT_DST_RESOLVED warning a
	// non-default ImportJob.DSTPolicy raises when it resolved at least
	// one ambiguous or nonexistent source-zone wall clock (details:
	// policy, ambiguous_n, nonexistent_n). Empty (and omitted from
	// JSON) otherwise — always, without a source zone.
	ZoneWarnings []*errors.CodedError `json:"ZoneWarnings,omitempty"`
	// SourceWarnings carries the non-fatal diagnostics the source
	// Reader surfaced through the optional SourceWarningEmitter
	// contract — today the PULSE_SPSS_* family raised by the `.sav`
	// dictionary walk, schema mapping and data pass. Nil for sources
	// that do not implement the interface and for sources that
	// implement it and raised nothing, so the report shape is
	// unchanged for every pre-existing adapter.
	SourceWarnings []*errors.CodedError
	// ElidedConstants names the fields ImportJob.ElideConstants stored
	// once in the schema block instead of per row, in schema order.
	// Empty (and omitted from JSON) when elision was off or elided
	// nothing, in which case the cohort is 0x01.
	ElidedConstants []string `json:"ElidedConstants,omitempty"`
	// Groups describes each ImportJob.Groups declaration as written, in
	// declaration order: its label, fields, distinct-tuple count and
	// the entry / member / index widths a viability check weighs. Empty
	// (and omitted from JSON) when no group was declared.
	Groups []GroupReport `json:"Groups,omitempty"`
	// GroupWarnings carries the viability gate's findings, one per
	// flagged group: PULSE_GROUP_TOO_NARROW (the group was dropped) and
	// PULSE_DEDUP_LOW_RATIO (the group was written anyway), each with
	// its numbers in Details. Empty (and omitted from JSON) when every
	// declared group passed. Under ImportJob.StrictDedup a finding is
	// returned as Run's error instead and there is no report.
	GroupWarnings []*errors.CodedError `json:"GroupWarnings,omitempty"`
}

type InferOptions struct {
	// SampleRows caps how many rows InferSchema reads before
	// finalizing column types. Defaults to defaultSampleRows when
	// <= 0; min-clamped to minSampleRows.
	SampleRows int
	// SetInferenceMinPct is the minimum percentage of non-null
	// sampled cells that must contain the inferred delimiter for the
	// column to be classified as set_*. Defaults to
	// defaultSetInferenceMinPct (30) when 0.
	SetInferenceMinPct int
	// FloatWidth is the type an all-float column infers as: "f64" (the
	// default; "" means the same) or "f32". f64 keeps every decimal the
	// source wrote exact to ~15-17 significant digits; f32 is a size
	// opt-in that rounds to ~7. Any other value is refused with
	// PULSE_IMPORT_FLOAT_WIDTH_INVALID. An integer column, and a column
	// carrying a negative integer (always f64), is unaffected.
	FloatWidth string
	// ColumnTypeOverrides maps column name to forced FieldType,
	// bypassing the inference heuristics entirely for those columns.
	// For set_* / categorical_* types the dictionary is still built
	// from observed sample values during the inference pass; the
	// override only fixes the column's storage type.
	ColumnTypeOverrides map[string]encoding.FieldType
	// contains filtered or unexported fields
}

type InferenceResult struct {
	Schema     *encoding.Schema
	Warnings   []InferenceWarning
	Delimiters map[string]string
}

type InferenceWarning struct {
	Column  string
	Message string
}

type LabelResolver interface {
	// Has reports whether the resolver carries any binding for the
	// named field.
	Has(field string) bool
	// Mode returns the binding mode for the named field. Empty when
	// no binding exists.
	Mode(field string) types.LabelMode
	// Apply translates a raw value through the binding for the named
	// field. See processing.LabelResolver.Apply for semantics.
	Apply(field, raw string) (out string, sibling string, ok bool)
	// FieldsWithAugment returns the field names that should emit a
	// "<field>_label" sibling column.
	FieldsWithAugment() []string
	// Warnings flushes the per-binding collision and miss summaries
	// once the job finishes.
	Warnings() []LabelWarning
}

type LabelWarning struct {
	Code    string
	Message string
	Details map[string]any
}

type LateKeyRefuser = iocore.LateKeyRefuser

type ListAwareReader = iocore.ListAwareReader

type ListAwareWriter = iocore.ListAwareWriter

type ListCell = iocore.ListCell

type ListColumn = iocore.ListColumn

type NullAwareReader = iocore.NullAwareReader

type NullAwareWriter = iocore.NullAwareWriter

type OverlayAwareWriter = iocore.OverlayAwareWriter

type OverlayWarningEmitter = iocore.OverlayWarningEmitter

type PredictReport struct {
	Schema        *encoding.Schema
	EstimatedRows int
	Warnings      []InferenceWarning
	// ZoneWarnings is what ImportReport.ZoneWarnings would carry: the
	// predict converts every source-zone datetime cell exactly as Run
	// does, so a DST refusal is Predict's error too.
	ZoneWarnings []*errors.CodedError `json:"ZoneWarnings,omitempty"`
	// SourceWarnings carries the source Reader's non-fatal coded
	// diagnostics — see ImportReport.SourceWarnings. Held apart from
	// Warnings, which is the inference pass's own untyped channel: a
	// predict against an authoritative source runs no inference, so
	// Warnings is empty there and SourceWarnings is the only signal.
	SourceWarnings []*errors.CodedError
	// TargetWarnings carries the non-fatal diagnostics the TARGET
	// Writer would raise if the export ran, lifted off the optional
	// CohortValidator contract — today the PULSE_SPSS_SIDECAR_ABSENT /
	// _IGNORED and PULSE_SPSS_NAME_SANITIZED codes a `.sav` export
	// raises before it reads a single record. Symmetric with
	// ExportReport.TargetWarnings, and held apart from SourceWarnings
	// for the same reason ConvertReport holds the two apart: one is
	// about what was read, the other about what would be written.
	//
	// Nil for a predict whose target does not implement
	// CohortValidator, which is every format but `.sav` today, and nil
	// for a validating target that raised nothing.
	TargetWarnings []*errors.CodedError
	// Columns is the header an ExportJob.Predict with a Target says the
	// export of a cohort WITH A VECTOR FIELD would write: one column per
	// projected field, a `_label` sibling per augment binding, and each
	// vector as ONE column for a ListAwareWriter or as its dim expanded
	// `<field>_<label>` / `<field>_1..N` columns for every other row
	// writer. Nil for a vector-free cohort (its header is its field
	// list), for a predict with no Target, for a CohortWriter target
	// (which lays out its own variables) and for every ImportJob.Predict.
	Columns []string `json:"Columns,omitempty"`
	// The fields below are filled only by an ImportJob.Predict that runs
	// the MEASURED pass — the job declares Groups, sets ElideConstants
	// or sets SuggestGroups — and are nil (omitted from JSON) otherwise,
	// so a plain predict's report is unchanged. The measured pass
	// converts every row exactly as Run does, so each figure is what the
	// import would produce, not a sample estimate.
	//
	// WidthWarnings are the PULSE_IMPORT_WIDTH_PROMOTED warnings Run
	// would raise (see ImportReport.WidthWarnings), and Schema carries
	// the promoted types. Only the measured pass converts values, so a
	// plain predict never reports a width promotion: its Schema is the
	// sample-inferred one.
	WidthWarnings []*errors.CodedError `json:"WidthWarnings,omitempty"`
	// Projection sizes the file the import would write.
	Projection *ImportProjection `json:"Projection,omitempty"`
	// Groups / GroupWarnings are what ImportReport.Groups /
	// GroupWarnings would carry for the declared Groups. A declaration
	// Run would refuse — an unknown field, or a member that varies
	// within its key (PULSE_GROUP_MEMBER_NOT_CONSTANT) — is Predict's
	// error too, and StrictDedup fails Predict as it fails Run.
	Groups        []GroupReport        `json:"Groups,omitempty"`
	GroupWarnings []*errors.CodedError `json:"GroupWarnings,omitempty"`
	// ElidedConstants names what ElideConstants would elide.
	ElidedConstants []string `json:"ElidedConstants,omitempty"`
	// GroupCandidates is the SuggestGroups detection report.
	GroupCandidates *GroupDetection `json:"GroupCandidates,omitempty"`
}

type Reader = iocore.Reader

type ResetReader = iocore.ResetReader

type RowError struct {
	Row int
	Err error
}

type SchemaAwareReader = iocore.SchemaAwareReader

type SchemaAwareWriter = iocore.SchemaAwareWriter

type SchemaFileField struct {
	Name        string   `json:"name"`
	Type        string   `json:"type"`
	Description string   `json:"description"`
	Nullable    bool     `json:"nullable"`
	Precision   uint8    `json:"precision,omitempty"`
	Scale       uint8    `json:"scale,omitempty"`
	Dim         int      `json:"dim,omitempty"`
	Labels      []string `json:"labels,omitempty"`
	Kind        string   `json:"kind,omitempty"`
	ScaleMin    *float64 `json:"scale_min,omitempty"`
	ScaleMax    *float64 `json:"scale_max,omitempty"`
}

type SidecarEmitter = iocore.SidecarEmitter

type SourceAwareWriter = iocore.SourceAwareWriter

type SourceWarningEmitter = iocore.SourceWarningEmitter

type TargetWarningEmitter = iocore.TargetWarningEmitter

type TransferExportJob struct {
	// FS is the filesystem both paths resolve against. Required.
	FS afero.Fs
	// Source is the cohort path.
	Source string
	// Output is the artifact path, conventionally Source + ".zst".
	Output string
	// Level is the zstd level, MinTransferLevel..MaxTransferLevel. Zero
	// selects DefaultTransferLevel.
	Level int
}

type TransferImportJob struct {
	// FS is the filesystem both paths resolve against. Required.
	FS afero.Fs
	// Source is the `.pulse.zst` artifact path.
	Source string
	// Output is the cohort path to write.
	Output string
	// Overwrite permits replacing an existing Output. Default false:
	// silently replacing a working cohort is refused.
	Overwrite bool
}

type TransferReport struct {
	Source string `json:"source"`
	Output string `json:"output"`
	Codec  string `json:"codec"`
	// Level is the zstd level used; set on compress only.
	Level int `json:"level,omitempty"`
	// Layout is single_file or shard_archive.
	Layout string `json:"layout"`
	// CohortBytes is the uncompressed cohort size.
	CohortBytes int64 `json:"cohort_bytes"`
	// CompressedBytes is the artifact size.
	CompressedBytes int64 `json:"compressed_bytes"`
	// Ratio is CohortBytes / CompressedBytes (0 when the artifact is
	// empty, which cannot happen for a real stream).
	Ratio float64 `json:"ratio"`
	// SHA256 is the lowercase hex digest of the uncompressed cohort.
	SHA256 string `json:"sha256"`
}

type VectorDecl struct {
	Name     string   `json:"name"`
	Element  string   `json:"element,omitempty"`
	Columns  []string `json:"columns,omitempty"`
	Pattern  string   `json:"pattern,omitempty"`
	Kind     string   `json:"kind,omitempty"`
	ScaleMin *float64 `json:"scale_min,omitempty"`
	ScaleMax *float64 `json:"scale_max,omitempty"`
}

type Writer = iocore.Writer