ccrawl

type APIServer struct {
	// contains filtered or unexported fields
}

type BM25Params struct {
	K1 float64 // term frequency saturation (default 1.2)
	B  float64 // length normalization (default 0.75)
}

type CDXBatchOptions struct {
	// BatchSize is the number of prefix letters per DuckDB query (default 1).
	// Higher values reduce total S3 reads at the cost of a larger hash table.
	// Rule of thumb: each extra prefix adds ~600 MB to the DuckDB hash table.
	BatchSize int
	// Parallel is how many DuckDB queries run concurrently (default 1).
	// Memory budget: Parallel × (BatchSize × 600 MB + 1.5 GB DuckDB overhead).
	Parallel int
	// DuckDBThreads is passed as SET threads=N inside each query (default 2).
	// Higher values speed up CPU-bound aggregation; each thread also opens one
	// extra S3 connection, so keep ≤4 to avoid CC rate limiting.
	DuckDBThreads int
	// MemoryLimit is passed as SET memory_limit= per DuckDB instance (default "2GB").
	// When BatchSize > 1 increase this to BatchSize × 600 MB + 1 GB headroom.
	MemoryLimit string
}

type CDXQuery struct {
	URL    string // URL or pattern
	Match  string // exact|prefix|host|domain (empty -> inferred from URL)
	From   string // 14-digit (or loose) lower time bound
	To     string // 14-digit (or loose) upper time bound
	Status string // HTTP status filter (e.g. "200")
	MIME   string // mime-detected filter
	Lang   string // languages filter (ISO-639-3)
	Filter []string
	Limit  int
}

type CDXRawOutputRow struct {
	Host     string `json:"host"`
	RD       string `json:"rd,omitempty"`
	TLD      string `json:"tld,omitempty"`
	Proto    string `json:"proto,omitempty"`
	URL      string `json:"url,omitempty"`
	Surt     string `json:"surt,omitempty"`
	ST       int32  `json:"st"`
	Redir    string `json:"redir,omitempty"`
	Digest   string `json:"digest,omitempty"`
	MIME     string `json:"mime,omitempty"`
	MIMEDecl string `json:"mime_d,omitempty"`
	Charset  string `json:"charset,omitempty"`
	Lang     string `json:"lang,omitempty"`
	Trunc    string `json:"trunc,omitempty"`
	TS       string `json:"ts,omitempty"`
	Bytes    int64  `json:"bytes,omitempty"`
	WARCFile string `json:"warc_f,omitempty"`
	WARCOff  int64  `json:"warc_o,omitempty"`
	RobotsOK bool   `json:"robots_ok,omitempty"`
	Crawl    string `json:"crawl,omitempty"`
}

type CDXRawRow struct {
	URL                     string `parquet:"url"`
	URLSurtKey              string `parquet:"url_surtkey"`
	URLHostName             string `parquet:"url_host_name"`
	URLHostRegisteredDomain string `parquet:"url_host_registered_domain"`
	URLHostTLD              string `parquet:"url_host_tld"`
	URLProtocol             string `parquet:"url_protocol"`
	FetchStatus             int32  `parquet:"fetch_status"`
	FetchRedirect           string `parquet:"fetch_redirect"`
	FetchTime               string `parquet:"fetch_time"`
	Digest                  string `parquet:"digest"`
	ContentMIMEType         string `parquet:"content_mime_type"`
	ContentMIMEDetected     string `parquet:"content_mime_detected"`
	ContentCharset          string `parquet:"content_charset"`
	ContentLanguages        string `parquet:"content_languages"`
	ContentTruncated        string `parquet:"content_truncated"`
	WARCFilename            string `parquet:"warc_filename"`
	WARCRecordOffset        int64  `parquet:"warc_record_offset"`
	WARCRecordLength        int64  `parquet:"warc_record_length"`
	RobotsTXTForceGet       bool   `parquet:"robotstxt_forceget"`
	Crawl                   string `parquet:"crawl"`
}

type CDXRecord struct {
	CrawlID      string `json:"crawl,omitempty" table:"crawl"`
	URLKey       string `json:"urlkey" table:"-"`
	Timestamp    string `json:"timestamp" table:"timestamp"` // 14-digit YYYYMMDDHHmmss
	URL          string `json:"url" table:"url,url"`
	MIME         string `json:"mime" table:"-"`
	MIMEDetected string `json:"mime-detected" table:"mime"`
	Status       string `json:"status" table:"status"`
	Digest       string `json:"digest" table:"digest"`
	Length       string `json:"length" table:"length"`
	Offset       string `json:"offset" table:"offset"`
	Filename     string `json:"filename" table:"filename"`
	Charset      string `json:"charset,omitempty" table:"-"`
	Languages    string `json:"languages,omitempty" table:"languages"`
	Truncated    string `json:"truncated,omitempty" table:"-"`
	Redirect     string `json:"redirect,omitempty" table:"-"`
}

type Cache struct {
	// contains filtered or unexported fields
}

type ColumnarQuery struct {
	Crawl      string
	Subset     string // warc (default) | crawldiagnostics | robotstxt
	Domain     string // url_host_registered_domain
	Host       string // url_host_name
	TLD        string // url_host_tld
	MIME       string // content_mime_detected
	Lang       string // content_languages (substring match)
	PathPrefix string // url_path prefix
	Status     int    // fetch_status (0 = any)
	Select     []string
	Limit      int
}

type Config struct {
	DataDir   string
	CacheDir  string
	DBPath    string
	Source    Source
	Workers   int
	Timeout   time.Duration
	Delay     time.Duration
	Retries   int
	UserAgent string
	CrawlID   string
}

type Crawl struct {
	ID     string `json:"id" kit:"id" table:"id"`
	Name   string `json:"name" table:"name"`
	CDXAPI string `json:"cdx-api" table:"-"`
	From   string `json:"from,omitempty" table:"from"`
	To     string `json:"to,omitempty" table:"to"`
}

type CrawlConfig struct {
	UserAgent   string
	MaxRedirect int
	Timeout     time.Duration
}

type CrawlResult struct {
	URL         string
	FinalURL    string // after redirects
	Status      int
	ContentType string
	Body        []byte
	Digest      string // SHA-1 of body
	FetchedAt   time.Time
	// Links extracted from HTML (relative links resolved to FinalURL)
	Links []string
}

type DatasetStats struct {
	CrawlID         string
	CommittedShards int
	TotalShards     int
	TotalURLs       int64
	TotalBytes      int64 // sum of WARC record bytes across all URL rows
	// Batched-pipeline fields (non-zero activates batch-aware progress line).
	TotalBatches     int
	CommittedBatches int
}

type DownloadResult struct {
	Path      string
	LocalPath string
	Bytes     int64
	Skipped   bool
	Err       error
}

type EdgeDegrees struct {
	NodeID    int64 `json:"node_id"`
	InDegree  int32 `json:"in_degree"`
	OutDegree int32 `json:"out_degree"`
}

type ForwardDoc struct {
	DocID       uint64  `json:"doc_id" table:"doc_id"`
	URL         string  `json:"url" table:"url"`
	CanonURL    string  `json:"canon_url" table:"canon_url"`
	Host        string  `json:"host" table:"host"`
	Title       string  `json:"title" table:"title"`
	Description string  `json:"description" table:"description"`
	Language    string  `json:"language" table:"language"`
	WordCount   int     `json:"word_count" table:"word_count"`
	LinkScore   float32 `json:"link_score" table:"link_score"`
	Snippet     string  `json:"snippet" table:"snippet"`
}

type ForwardIndexWriter struct {
	// contains filtered or unexported fields
}

type Frontier struct {
	// contains filtered or unexported fields
}

type FrontierEntry struct {
	URL      string  // normalized URL
	Host     string  // hostname for politeness grouping
	Priority float32 // harmonic centrality (higher = crawl sooner)
	NextAt   int64   // earliest Unix timestamp to fetch
	Depth    uint8   // BFS depth from seed
	Retries  uint8
}

type HFClient struct {
	// contains filtered or unexported fields
}

type HFOperation struct {
	LocalPath  string
	PathInRepo string
}

type HTTPClient struct {
	// contains filtered or unexported fields
}

type HostCDXStats struct {
	Host             string `json:"url_host_name"`
	RegisteredDomain string `json:"registered_domain"`
	URLCount         int64  `json:"url_count"`
	Status2xx        int64  `json:"status_2xx"`
	Status3xx        int64  `json:"status_3xx"`
	Status4xx        int64  `json:"status_4xx"`
	Status5xx        int64  `json:"status_5xx"`
	TopMIME          string `json:"top_mime"`
	Language         string `json:"language"`
	FirstSeen        string `json:"first_seen"`
	LastSeen         string `json:"last_seen"`
	TotalBytes       int64  `json:"total_bytes"`
}

type HostDatasetRow struct {
	Host             string  `parquet:"host"`
	HostRev          string  `parquet:"host_rev,dict"`
	TLD              string  `parquet:"tld,dict"`
	RegisteredDomain string  `parquet:"registered_domain,dict"`
	CrawlID          string  `parquet:"crawl_id,dict"`
	GraphID          string  `parquet:"graph_id,dict"`
	HarmonicPos      int64   `parquet:"harmonic_pos"`
	HarmonicVal      float64 `parquet:"harmonic_val"`
	PageRankPos      int64   `parquet:"pagerank_pos"`
	PageRankVal      float64 `parquet:"pagerank_val"`
	InDegree         int32   `parquet:"in_degree"`
	OutDegree        int32   `parquet:"out_degree"`
	URLCount         int64   `parquet:"url_count"`
	Status2xx        int64   `parquet:"status_2xx"`
	Status3xx        int64   `parquet:"status_3xx"`
	Status4xx        int64   `parquet:"status_4xx"`
	Status5xx        int64   `parquet:"status_5xx"`
	TopMIME          string  `parquet:"top_mime,dict"`
	Language         string  `parquet:"language,dict"`
	FirstSeen        string  `parquet:"first_seen,dict"`
	LastSeen         string  `parquet:"last_seen,dict"`
	TotalBytes       int64   `parquet:"total_bytes"`
}

type HostDiffEntry struct {
	Host        string  `json:"host" table:"host"`
	TotalURLs   int64   `json:"total_urls" table:"total_urls"`
	ChangedURLs int64   `json:"changed_urls" table:"changed_urls"`
	ChangeRate  float64 `json:"change_rate" table:"change_rate"`
	Tier        int     `json:"tier" table:"tier"`
}

type HostRecord struct {
	// Identity
	Host             string `json:"host" kit:"id" table:"host"`
	HostRev          string `json:"host_rev,omitempty" table:"-"`
	TLD              string `json:"tld" table:"tld"`
	RegisteredDomain string `json:"registered_domain,omitempty" table:"registered_domain"`

	// Rank signals (from rank table)
	HarmonicPos int64   `json:"harmonic_pos" table:"harmonic_pos"`
	HarmonicVal float64 `json:"harmonic_val" table:"harmonic_val"`
	PageRankPos int64   `json:"pagerank_pos" table:"pagerank_pos"`
	PageRankVal float64 `json:"pagerank_val" table:"pagerank_val"`

	// Graph topology (from edge files)
	InDegree  int64 `json:"in_degree" table:"in_degree"`
	OutDegree int64 `json:"out_degree" table:"out_degree"`

	// CDX statistics (from columnar Parquet index)
	URLCount   int64  `json:"url_count" table:"url_count"`
	Status2xx  int64  `json:"status_2xx" table:"status_2xx"`
	Status3xx  int64  `json:"status_3xx" table:"status_3xx"`
	Status4xx  int64  `json:"status_4xx" table:"status_4xx"`
	Status5xx  int64  `json:"status_5xx" table:"status_5xx"`
	TopMIME    string `json:"top_mime,omitempty" table:"top_mime"`
	Language   string `json:"language,omitempty" table:"language"`
	FirstSeen  string `json:"first_seen,omitempty" table:"first_seen"`
	LastSeen   string `json:"last_seen,omitempty" table:"last_seen"`
	TotalBytes int64  `json:"total_bytes" table:"total_bytes"`
}

type HostSchedule struct {
	Host        string  `json:"host" table:"host"`
	HarmonicPos int64   `json:"harmonic_pos" table:"harmonic_pos"`
	HarmonicVal float64 `json:"harmonic_val" table:"harmonic_val"`
	ChangeRate  float64 `json:"change_rate" table:"change_rate"`
	Tier        int     `json:"tier" table:"tier"`
	IntervalH   int     `json:"interval_h" table:"interval_h"`
	Score       float64 `json:"score" table:"score"`
}

type HostStore interface {
	GetHost(ctx context.Context, host string) (*HostRecord, error)
	TopHosts(ctx context.Context, n int, tld string) ([]HostRecord, error)
}

type IndexReader struct {
	N     int
	AvgDL float64
	// contains filtered or unexported fields
}

type IndexSearchStore struct {
	// contains filtered or unexported fields
}

type InvertedIndexBuilder struct {
	N int // total docs indexed

	TermCount int // unique term count (set by Flush)
	// contains filtered or unexported fields
}

type Ledger struct {
	// contains filtered or unexported fields
}

type Library struct {
	Root  string
	Crawl string
}

type Location struct {
	Filename string `json:"filename" table:"filename"`
	Offset   int64  `json:"offset" table:"offset"`
	Length   int64  `json:"length" table:"length"`
	URL      string `json:"url,omitempty" table:"url,url"`
}

type MarkdownDatasetStats struct {
	CrawlID         string
	CommittedShards int
	TotalShards     int
	Rows            int64
	WARCBytes       int64 // compressed .warc.gz bytes downloaded
	HTMLBytes       int64
	MDBytes         int64
	ParquetBytes    int64
	// Cumulative pipeline timings across committed shards, in seconds. Zero
	// values drop the Processing Times chart from the card.
	DownloadS int64
	ConvertS  int64
	ExportS   int64
	PublishS  int64
}

type MarkdownExportConfig struct {
	CrawlID   string
	Indices   []int    // shard indices to process, in order
	WARCPaths []string // full manifest, indexed by shard index
	OutDir    string
	Repo      string
	Push      bool

	// ShardParallel is the number of shards processed at once (P). More than a
	// couple only helps when downloads are slow relative to conversion. 0 picks
	// a small default.
	ShardParallel int
	// ConvertWorkers caps total concurrent conversions across all shards (C).
	// 0 selects runtime.NumCPU().
	ConvertWorkers int
	// CommitBatch is how many finished parquets go into one HF commit (K). The
	// committer runs off the critical path, so batching keeps commit round
	// trips from dominating wall-clock on long runs. 0 means 1.
	CommitBatch int
	// KeepParquet leaves local parquet files in place after they are committed.
	// The default deletes them, which is required on small disks.
	KeepParquet bool
	// MinFreeBytes pauses new downloads while free disk is below this. 0 selects
	// 2 GiB.
	MinFreeBytes int64
	// Ledger, when set, skips already-committed shards and records new ones.
	Ledger *Ledger

	// Progress is called once per committed batch with a snapshot of the run.
	// It may be nil.
	Progress func(MarkdownRunStats)
}

type MarkdownPackConfig struct {
	// CrawlID is the CC crawl identifier (e.g. CC-MAIN-2026-25).
	CrawlID string
	// ShardIdx is the 0-based index of this WARC file in the crawl manifest.
	ShardIdx int
	// WARCPath is the Common Crawl relative path (crawl-data/.../warc.gz).
	WARCPath string
	// OutPath is the local parquet file to write.
	OutPath string
	// Workers is the number of goroutines for HTML→Markdown conversion.
	// 0 selects runtime.NumCPU().
	Workers int
	// MaxRecords, when > 0, stops the shard after this many HTML response
	// records have been queued for conversion. 0 means convert the whole shard.
	// It exists for the throughput benchmark, which needs a fixed, fast slice of
	// a shard; production export leaves it at 0.
	MaxRecords int
	// ConvertSem, when non-nil, caps the number of conversions running at once
	// across every shard that shares it. The orchestrator passes one semaphore
	// sized to the CPU count so several shards can be in flight (hiding download
	// latency) without oversubscribing the cores. nil means no global cap.
	ConvertSem chan struct{}
	// Progress is called after each row is written. It may be nil.
	Progress func(MarkdownStats)
}

type MarkdownRow struct {
	DocID          string `parquet:"doc_id"`
	URL            string `parquet:"url"`
	Host           string `parquet:"host"`
	CrawlDate      string `parquet:"crawl_date"`
	WARCRecordID   string `parquet:"warc_record_id"`
	HTMLLength     int64  `parquet:"html_length"`
	MarkdownLength int64  `parquet:"markdown_length"`
	Markdown       string `parquet:"markdown"`
}

type MarkdownRunStats struct {
	Total         int   // shards requested this run
	Skipped       int   // shards skipped via the ledger
	Committed     int   // shards committed so far
	Failed        int   // shards that errored
	Rows          int64 // cumulative rows across committed shards
	WARCBytes     int64
	HTMLBytes     int64
	MDBytes       int64
	ParquetBytes  int64
	ConvertS      int64 // cumulative per-shard conversion wall-clock, seconds
	PublishS      int64 // cumulative HF commit wall-clock, seconds
	Elapsed       time.Duration
	ShardsPerHour float64
	ETA           time.Duration // estimated time to finish the remaining shards
	FreeDiskBytes int64
}

type MarkdownStats struct {
	ShardIdx     int
	Rows         int64
	WARCBytes    int64 // compressed .warc.gz bytes downloaded (Content-Length)
	HTMLBytes    int64
	MDBytes      int64
	ParquetBytes int64
	DurDownload  time.Duration
	DurConvert   time.Duration
	DurExport    time.Duration
	DurPublish   time.Duration
}

type MemHostStore struct {
	// contains filtered or unexported fields
}

type NewWARCRecord struct {
	TargetURI string
	Date      string
	RecordID  string
	Block     []byte // raw HTTP response bytes
}

type NewsFile struct {
	Path string `json:"path" table:"path"`
	Year int    `json:"year" table:"year"`
	Mon  int    `json:"month" table:"month"`
}

type ParquetWriter[T any] struct {
	// contains filtered or unexported fields
}

type PostingEntry struct {
	DocID uint64
	TF    uint32
	DL    uint32 // document token length at index time
}

type QualityResult struct {
	WordCount      int
	TitleLength    int
	HasMainContent bool
	SpamScore      float64 // 0–1
	IsParked       bool
	IsShortContent bool // word_count < 50
}

type Rank struct {
	Key         string  `json:"key" kit:"id" table:"key"` // host or domain (forward form)
	HarmonicPos int64   `json:"harmonic_pos" table:"harmonic_pos"`
	HarmonicVal float64 `json:"harmonic_val" table:"harmonic_val"`
	PageRankPos int64   `json:"pagerank_pos" table:"pagerank_pos"`
	PageRankVal float64 `json:"pagerank_val" table:"pagerank_val"`
}

type RateLimitError struct {
	RetryAfter time.Duration
	Msg        string
}

type RefetchDatasetStats struct {
	CrawlID         string
	CommittedShards int
	TotalShards     int
	Rows            int64
	URLsFound       int64
	WARCBytes       int64
	FetchBytes      int64
	HTMLBytes       int64
	MDBytes         int64
	ParquetBytes    int64
	ConvertS        int64
	PublishS        int64
}

type RefetchExportConfig struct {
	CrawlID   string
	Indices   []int    // shard indices to process, in order
	WARCPaths []string // full manifest, indexed by shard index
	OutDir    string
	Repo      string
	Push      bool

	// FetchCfg is the ami config passed to each shard's FetchBatch call.
	// Workers, MaxRedirects, and PerHostDelay are the knobs most callers set.
	FetchCfg config.Config

	// ShardParallel is the number of shards processed concurrently. 0 picks 2.
	ShardParallel int
	// ConvertWorkers caps total concurrent HTML-to-Markdown conversions. 0 picks NumCPU.
	ConvertWorkers int
	// CommitBatch is how many finished parquets go into one HF commit. 0 means 1.
	CommitBatch int
	// KeepParquet leaves local parquet files in place after they are committed.
	KeepParquet bool
	// MinFreeBytes pauses new downloads while free disk is below this. 0 selects 2 GiB.
	MinFreeBytes int64
	// CacheDir caches downloaded WARC shards so a re-run skips the download.
	// Empty disables caching.
	CacheDir string
	// FetchOnly stores raw HTML and skips the HTML-to-Markdown convert, so the
	// fetch step runs at its true ceiling and Markdown is produced by a separate
	// offline pass over the stored html column.
	FetchOnly bool
	// Ledger, when set, skips already-committed shards and records new ones.
	Ledger *Ledger

	// Progress is called once per committed batch with a snapshot of the run.
	Progress func(RefetchRunStats)
}

type RefetchPackConfig struct {
	CrawlID    string
	ShardIdx   int
	WARCPath   string
	OutPath    string
	FetchCfg   config.Config
	ConvertSem chan struct{}
	Progress   func(RefetchStats)

	// FetchOnly stores the raw HTML body in the html column and skips the
	// HTML-to-Markdown convert entirely. Convert is the CPU-bound phase and the
	// end-to-end wall on a multi-core box, so skipping it lets the fetch step run
	// at its true ceiling; the Markdown is produced later by an offline convert
	// pass over the stored html column, which can run on a bigger box or a
	// cluster without holding back the crawl.
	FetchOnly bool

	// CacheDir, when set, is where the downloaded WARC is cached so a re-run of
	// the same shard skips the multi-second download. The download streams to a
	// .part file beside the final name and is renamed into place only once it is
	// complete, so an interrupted run never leaves a truncated file that a later
	// run would mistake for a good cache. Empty disables caching.
	CacheDir string
}

type RefetchRow struct {
	DocID          string `parquet:"doc_id"`
	URL            string `parquet:"url"`
	FinalURL       string `parquet:"final_url"`
	Host           string `parquet:"host"`
	IPAddress      string `parquet:"ip_address"`
	CrawlID        string `parquet:"crawl_id"`
	CrawlDate      string `parquet:"crawl_date"`
	WARCRecordID   string `parquet:"warc_record_id"`
	FetchedAt      int64  `parquet:"fetched_at"`
	Status         int32  `parquet:"status"`
	ContentType    string `parquet:"content_type"`
	FetchDurMS     int64  `parquet:"fetch_duration_ms"`
	TTFBMS         int64  `parquet:"ttfb_ms"`
	ETag           string `parquet:"etag"`
	LastModified   string `parquet:"last_modified"`
	RespHeaders    string `parquet:"resp_headers"`
	BodyLength     int64  `parquet:"body_length"`
	Digest         string `parquet:"digest"`
	HTMLLength     int64  `parquet:"html_length"`
	HTML           string `parquet:"html"`
	MarkdownLength int64  `parquet:"markdown_length"`
	Markdown       string `parquet:"markdown"`
	Error          string `parquet:"error"`
}

type RefetchRunStats struct {
	Total        int
	Skipped      int
	Committed    int
	Failed       int
	Rows         int64
	URLsFound    int64
	WARCBytes    int64
	FetchBytes   int64
	HTMLBytes    int64
	MDBytes      int64
	ParquetBytes int64
	// Per-phase wall-time sums (in seconds, across all completed shards).
	ExtractS int64 // phase 1: WARC download + URL extraction
	FetchS   int64 // phase 2: ami live refetch
	ConvertS int64 // phase 3: HTML to Markdown
	ExportS  int64 // phase 4: Parquet write
	PublishS int64 // HF commit (off critical path)
	// Failure breakdown summed across shards, for throughput diagnosis.
	Failures      int64
	ErrDNS        int64
	ErrTimeout    int64
	ErrRefused    int64
	ErrSkip       int64
	ErrOther      int64
	Elapsed       time.Duration
	ShardsPerHour float64
	ETA           time.Duration
	FreeDiskBytes int64
}

type RefetchStats struct {
	ShardIdx int

	// Phase 1: WARC URL extraction.
	URLsFound  int64
	DurExtract time.Duration
	WARCBytes  int64

	// Phase 2: ami refetch.
	Fetched    int64
	Failed     int64
	Redirected int64
	FetchBytes int64
	DurFetch   time.Duration

	// Failure breakdown, so a low-yield shard can be diagnosed without a second
	// run. These count Result.Err by class and sum to Failed.
	ErrDNS     int64 // name does not resolve (no such host / NXDOMAIN)
	ErrTimeout int64 // deadline exceeded before a response
	ErrRefused int64 // connection refused / reset by peer
	ErrSkip    int64 // engine skipped the URL (dead-domain breaker, non-HTTP)
	ErrOther   int64 // TLS, protocol, and everything else

	// Phase 3: HTML to Markdown convert.
	Rows       int64
	HTMLBytes  int64
	MDBytes    int64
	DurConvert time.Duration

	// Phase 4: parquet write.
	ParquetBytes int64
	DurExport    time.Duration

	MemRSSBytes int64
}

type RobotsCache struct {
	// contains filtered or unexported fields
}

type RobotsEntry struct {
	Rules      []RobotsRule
	CrawlDelay time.Duration
	ExpiresAt  int64 // Unix timestamp
}

type RobotsRule struct {
	Allow   bool
	Pattern string
}

type ScoredDoc struct {
	DocID uint64
	Score float64
}

type SearchResult struct {
	DocID    uint64  `json:"doc_id"`
	URL      string  `json:"url"`
	Host     string  `json:"host"`
	Title    string  `json:"title"`
	Snippet  string  `json:"snippet"`
	Score    float64 `json:"score"`
	Language string  `json:"language,omitempty"`
}

type SearchStore interface {
	Search(ctx context.Context, query string, k int) ([]SearchResult, error)
}

type SeedExportOptions struct {
	// Status keeps only captures with this fetch_status. Zero means any status.
	Status int32
	// MIME, when set, keeps only captures whose detected MIME contains it.
	MIME string
	// Lang, when set, keeps only captures whose content languages contain it.
	Lang string
	// Limit stops after this many written rows. Zero means no limit.
	Limit int64
}

type SeedExportStats struct {
	Scanned int64
	Written int64
}

type SeedRow struct {
	URL    string `parquet:"url" json:"url"`
	Digest string `parquet:"digest" json:"digest,omitempty"`
	Host   string `parquet:"host" json:"host,omitempty"`
	MIME   string `parquet:"mime" json:"mime,omitempty"`
	Lang   string `parquet:"lang" json:"lang,omitempty"`
	Status int32  `parquet:"status" json:"status,omitempty"`
	Time   string `parquet:"time" json:"time,omitempty"`
}

type ServeConfig struct {
	Addr     string // e.g. ":8080"
	DBPath   string // path to DuckDB/SQLite host database (optional)
	IndexDir string // path to the inverted index directory (optional)
}

type Source string

type StreamingParquetWriter struct {
	// contains filtered or unexported fields
}

type TextResult struct {
	Title       string
	Description string // <meta name="description">
	CanonURL    string // canonical URL from meta or link[rel=canonical]
	Body        string // extracted clean body text
	WordCount   int
	Language    string // BCP-47 code inferred from lang attribute
}

type URLDatasetRow struct {
	// CDX identity
	Host  string `parquet:"host,dict"`
	RD    string `parquet:"rd,dict"`
	TLD   string `parquet:"tld,dict"`
	Proto string `parquet:"proto,dict"`
	URL   string `parquet:"url"`
	Surt  string `parquet:"surt"`
	// CDX fetch result
	ST     int32  `parquet:"st"`
	Redir  string `parquet:"redir"`
	Digest string `parquet:"digest,dict"`
	// CDX content
	MIME     string `parquet:"mime,dict"`
	MIMEDecl string `parquet:"mime_d,dict"`
	Charset  string `parquet:"charset,dict"`
	Lang     string `parquet:"lang,dict"`
	Trunc    string `parquet:"trunc,dict"`
	// CDX timing and size
	TS    string `parquet:"ts,dict"`
	Bytes int64  `parquet:"bytes"`
	// CDX WARC location (for byte-range content fetch)
	WARCFile string `parquet:"warc_f,dict"`
	WARCOff  int64  `parquet:"warc_o"`
	RobotsOK bool   `parquet:"robots_ok"`
	Crawl    string `parquet:"crawl,dict"`
	// Rank signals (joined from rank table by host)
	GraphID     string  `parquet:"graph_id,dict"`
	HarmonicPos int64   `parquet:"harmonic_pos"`
	HarmonicVal float64 `parquet:"harmonic_val"`
	PageRankPos int64   `parquet:"pagerank_pos"`
	PageRankVal float64 `parquet:"pagerank_val"`
}

type VertexRecord struct {
	ID      int64  `json:"id" kit:"id" table:"id"`
	HostRev string `json:"host_rev" table:"host_rev"`
	Host    string `json:"host" table:"host"`
}

type WARCExporter struct {
	Files []string // paths written, in order
	// contains filtered or unexported fields
}

type WARCHeader = warc.Header

type WARCInfo struct {
	Software    string // tool and version, e.g. "ccrawl/0.4.0"
	IsPartOf    string // prefix, with the subprefix appended when set
	Description string // the exact command line used
	Format      string // e.g. "WARC file version 1.0"
	Creator     string // optional: a person, organization, or service
	Operator    string // optional: a person, when the creator is an organization
}

type WARCParquetRow struct {
	RecordID      string    `parquet:"record_id,dict"`
	CrawlID       string    `parquet:"crawl_id,dict"`
	WARCType      string    `parquet:"warc_type,dict"`
	TargetURI     string    `parquet:"target_uri"`
	Date          time.Time `parquet:"date,timestamp(microsecond)"`
	IPAddress     string    `parquet:"ip_address,dict"`
	PayloadDigest string    `parquet:"payload_digest"`
	ContentType   string    `parquet:"content_type,dict"`
	ContentLength int64     `parquet:"content_length"`
	Truncated     string    `parquet:"truncated,dict"`
	HTTPStatus    int32     `parquet:"http_status"`
	HTTPMIME      string    `parquet:"http_mime,dict"`
	WARCFilename  string    `parquet:"warc_filename,dict"`
	WARCOffset    int64     `parquet:"warc_offset"`
	WARCLength    int64     `parquet:"warc_length"`
	Title         string    `parquet:"title"`
	Language      string    `parquet:"language,dict"`
	Markdown      string    `parquet:"markdown"`
	Text          string    `parquet:"text"`
}

type WARCRecord = warc.Record

type WARCURLRecord struct {
	URL      string
	Date     string // YYYY-MM-DD
	RecordID string
}

type WATLink = wat.Link

type WATMeta = wat.Meta

type WATParquetRow struct {
	RecordID    string    `parquet:"record_id,dict"`
	CrawlID     string    `parquet:"crawl_id,dict"`
	URL         string    `parquet:"url"`
	Date        time.Time `parquet:"date,timestamp(microsecond)"`
	HTTPStatus  int32     `parquet:"http_status"`
	ContentType string    `parquet:"content_type,dict"`
	Title       string    `parquet:"title"`
	LinksCount  int32     `parquet:"links_count"`
	Links       string    `parquet:"links"` // JSON
	Metas       string    `parquet:"metas"` // JSON
}

type WATRecord = wat.Record

type WETParquetRow struct {
	RecordID        string    `parquet:"record_id,dict"`
	CrawlID         string    `parquet:"crawl_id,dict"`
	URL             string    `parquet:"url"`
	Date            time.Time `parquet:"date,timestamp(microsecond)"`
	ContentLanguage string    `parquet:"content_language,dict"`
	TextLength      int32     `parquet:"text_length"`
	Text            string    `parquet:"text"`
}

type WETRecord = wet.Record

type WebGraph struct {
	ID        string // e.g. cc-main-2026-mar-apr-may
	BaseURL   string // https://data.commoncrawl.org/projects/hyperlinkgraph/{ID}/
	HostNodes int64
	HostArcs  int64
	Published string
}