dataset

type ExportOptions struct {
	// RowGroupRows is the Parquet row-group size in rows. A row group is the unit a
	// reader seeks to and the unit column statistics summarize, so it trades scan
	// granularity against footer size. Zero uses a sane default.
	RowGroupRows int
	// FileRows caps the rows per output file in repo mode, so a 100M export becomes a
	// folder of evenly sized shards a git-LFS push and the Hugging Face viewer both
	// handle better than one multi-gigabyte blob. Zero uses a default; single-file
	// export ignores it.
	FileRows int
	// Codec is the column compression: "zstd" (default, best ratio), "snappy" (faster
	// decode), or "none".
	Codec string
	// SinceHours filters to rows with activity (first_seen, last_crawled, or
	// last_changed) at or after this epoch-hour, the incremental cursor. Zero exports
	// every row (a full dump). A caller sets it to a prior dump's watermark to export
	// only what is new or changed since.
	SinceHours uint32
}

type ExportStats struct {
	Rows       int64  `json:"rows"`
	Skipped    int64  `json:"skipped"` // rows filtered out by SinceHours
	Hosts      int    `json:"hosts"`
	Files      int    `json:"files"`
	Bytes      int64  `json:"bytes"`
	LossyETags int64  `json:"lossy_etags"`
	Watermark  uint32 `json:"watermark_hours"` // max activity epoch-hour seen, the next incremental cursor
}

type FileMeta struct {
	Name  string `json:"name"`  // path relative to the dataset root
	Rows  int64  `json:"rows"`  // row count
	Bytes int64  `json:"bytes"` // file size on disk
}

type ImportOptions struct {
	// TmpDir is the scratch directory for the build's arena and record temps. Empty
	// uses the system default.
	TmpDir string
	// Codec is the output .meguri body codec: format.CodecZstd (default) or
	// format.CodecNone. It is the meguri codec, not the Parquet one.
	Codec uint8
	// PageRows caps the output column page size. Zero uses a scale-friendly default so
	// a dedup confirm decodes one page, not the whole column.
	PageRows int
	// FPRate is the resident seen-set filter false-positive budget. Zero uses the
	// build default.
	FPRate float64
	// ExpectedKeys sizes the filter. Zero lets the build guess; a caller that knows the
	// row count (the manifest does) should set it so the filter is not oversized.
	ExpectedKeys uint64
	// PartitionID stamps the output partition.
	PartitionID uint32
}

type Manifest struct {
	SchemaVersion int    `json:"schema_version"`
	Codec         string `json:"codec"`
	RowGroupRows  int    `json:"row_group_rows"`
	FileRows      int    `json:"file_rows"`

	Rows       int64 `json:"rows"`
	Hosts      int   `json:"hosts"`
	LossyETags int64 `json:"lossy_etags,omitempty"`

	// Watermark is the max last_changed epoch-hour across all rows, the cursor the next
	// incremental dump exports past. Zero means nothing has been observed to change yet.
	Watermark uint32 `json:"watermark_hours"`

	Files []FileMeta `json:"files"`
}

type Row struct {
	// Identity: the 128-bit URLKey split into its two halves, and the strings.
	HostKey uint64 `parquet:"host_key"`
	PathKey uint64 `parquet:"path_key"`
	URL     string `parquet:"url,zstd"`
	Host    string `parquet:"host,zstd,dict"`

	// State machine and discovery.
	Status     string  `parquet:"status,dict"`
	StatusCode uint8   `parquet:"status_code"`
	Priority   float32 `parquet:"priority"`
	Depth      uint16  `parquet:"depth"`
	Source     string  `parquet:"source,dict"`
	SourceCode uint8   `parquet:"source_code"`

	// Lifecycle timestamps, all nullable. A freshly seeded frontier carries none of
	// them (a URL is discovered but not yet dated or scheduled), so every one is nil
	// until its event happens: the zero hour is a null column, not a boundary date.
	// This also keeps the columns inside the representable timestamp range; a non-null
	// value is a real epoch-hour a scheduler set.
	FirstSeen    *time.Time `parquet:"first_seen,timestamp(millisecond),optional"`
	NextDue      *time.Time `parquet:"next_due,timestamp(millisecond),optional"`
	LastCrawled  *time.Time `parquet:"last_crawled,timestamp(millisecond),optional"`
	LastChanged  *time.Time `parquet:"last_changed,timestamp(millisecond),optional"`
	LastModified *time.Time `parquet:"last_modified,timestamp(millisecond),optional"`

	// Change model and counters.
	Lambda         float32 `parquet:"lambda"`
	CrawlCount     uint32  `parquet:"crawl_count"`
	ChangeCount    uint32  `parquet:"change_count"`
	NoChangeStreak uint16  `parquet:"no_change_streak"`

	// Validators and content fingerprints.
	ETag      string `parquet:"etag,zstd,optional"`
	ContentFP uint64 `parquet:"content_fp"`
	Simhash   uint64 `parquet:"simhash"`

	// Last-crawl outcome.
	HTTPStatus  uint16 `parquet:"http_status"`
	RedirectRef uint64 `parquet:"redirect_ref"`
	RetryCount  uint8  `parquet:"retry_count"`
	ErrorCount  uint16 `parquet:"error_count"`
}