ccrawl

type APIServer struct {
	// contains filtered or unexported fields
}

type BM25Params struct {
	K1 float64 // term frequency saturation (default 1.2)
	B  float64 // length normalization (default 0.75)
}

type CDXBatchOptions struct {
	// BatchSize is the number of prefix letters per DuckDB query (default 1).
	// Higher values reduce total S3 reads at the cost of a larger hash table.
	// Rule of thumb: each extra prefix adds ~600 MB to the DuckDB hash table.
	BatchSize int
	// Parallel is how many DuckDB queries run concurrently (default 1).
	// Memory budget: Parallel × (BatchSize × 600 MB + 1.5 GB DuckDB overhead).
	Parallel int
	// DuckDBThreads is passed as SET threads=N inside each query (default 2).
	// Higher values speed up CPU-bound aggregation; each thread also opens one
	// extra S3 connection, so keep ≤4 to avoid CC rate limiting.
	DuckDBThreads int
	// MemoryLimit is passed as SET memory_limit= per DuckDB instance (default "2GB").
	// When BatchSize > 1 increase this to BatchSize × 600 MB + 1 GB headroom.
	MemoryLimit string
}

type CDXQuery struct {
	URL    string // URL or pattern
	Match  string // exact|prefix|host|domain (empty -> inferred from URL)
	From   string // 14-digit (or loose) lower time bound
	To     string // 14-digit (or loose) upper time bound
	Status string // HTTP status filter (e.g. "200")
	MIME   string // mime-detected filter
	Lang   string // languages filter (ISO-639-3)
	Filter []string
	Limit  int
}

type CDXRawOutputRow struct {
	Host     string `json:"host"`
	RD       string `json:"rd,omitempty"`
	TLD      string `json:"tld,omitempty"`
	Proto    string `json:"proto,omitempty"`
	URL      string `json:"url,omitempty"`
	Surt     string `json:"surt,omitempty"`
	ST       int32  `json:"st"`
	Redir    string `json:"redir,omitempty"`
	Digest   string `json:"digest,omitempty"`
	MIME     string `json:"mime,omitempty"`
	MIMEDecl string `json:"mime_d,omitempty"`
	Charset  string `json:"charset,omitempty"`
	Lang     string `json:"lang,omitempty"`
	Trunc    string `json:"trunc,omitempty"`
	TS       string `json:"ts,omitempty"`
	Bytes    int64  `json:"bytes,omitempty"`
	WARCFile string `json:"warc_f,omitempty"`
	WARCOff  int64  `json:"warc_o,omitempty"`
	RobotsOK bool   `json:"robots_ok,omitempty"`
	Crawl    string `json:"crawl,omitempty"`
}

type CDXRawRow struct {
	URL                     string `parquet:"url"`
	URLSurtKey              string `parquet:"url_surtkey"`
	URLHostName             string `parquet:"url_host_name"`
	URLHostRegisteredDomain string `parquet:"url_host_registered_domain"`
	URLHostTLD              string `parquet:"url_host_tld"`
	URLProtocol             string `parquet:"url_protocol"`
	FetchStatus             int32  `parquet:"fetch_status"`
	FetchRedirect           string `parquet:"fetch_redirect"`
	FetchTime               string `parquet:"fetch_time"`
	Digest                  string `parquet:"digest"`
	ContentMIMEType         string `parquet:"content_mime_type"`
	ContentMIMEDetected     string `parquet:"content_mime_detected"`
	ContentCharset          string `parquet:"content_charset"`
	ContentLanguages        string `parquet:"content_languages"`
	ContentTruncated        string `parquet:"content_truncated"`
	WARCFilename            string `parquet:"warc_filename"`
	WARCRecordOffset        int64  `parquet:"warc_record_offset"`
	WARCRecordLength        int64  `parquet:"warc_record_length"`
	RobotsTXTForceGet       bool   `parquet:"robotstxt_forceget"`
	Crawl                   string `parquet:"crawl"`
}

type CDXRecord struct {
	CrawlID      string `json:"crawl,omitempty" table:"crawl"`
	URLKey       string `json:"urlkey" table:"-"`
	Timestamp    string `json:"timestamp" table:"timestamp"` // 14-digit YYYYMMDDHHmmss
	URL          string `json:"url" table:"url,url"`
	MIME         string `json:"mime" table:"-"`
	MIMEDetected string `json:"mime-detected" table:"mime"`
	Status       string `json:"status" table:"status"`
	Digest       string `json:"digest" table:"digest"`
	Length       string `json:"length" table:"length"`
	Offset       string `json:"offset" table:"offset"`
	Filename     string `json:"filename" table:"filename"`
	Charset      string `json:"charset,omitempty" table:"-"`
	Languages    string `json:"languages,omitempty" table:"languages"`
	Truncated    string `json:"truncated,omitempty" table:"-"`
	Redirect     string `json:"redirect,omitempty" table:"-"`
}

type Cache struct {
	// contains filtered or unexported fields
}

type ColumnarQuery struct {
	Crawl      string
	Subset     string // warc (default) | crawldiagnostics | robotstxt
	Domain     string // url_host_registered_domain
	Host       string // url_host_name
	TLD        string // url_host_tld
	MIME       string // content_mime_detected
	Lang       string // content_languages (substring match)
	PathPrefix string // url_path prefix
	Status     int    // fetch_status (0 = any)
	Select     []string
	Limit      int
}

type Config struct {
	DataDir   string
	CacheDir  string
	DBPath    string
	Source    Source
	Workers   int
	Timeout   time.Duration
	Delay     time.Duration
	Retries   int
	UserAgent string
	CrawlID   string
}

type Crawl struct {
	ID     string `json:"id" kit:"id" table:"id"`
	Name   string `json:"name" table:"name"`
	CDXAPI string `json:"cdx-api" table:"-"`
	From   string `json:"from,omitempty" table:"from"`
	To     string `json:"to,omitempty" table:"to"`
}

type CrawlConfig struct {
	UserAgent   string
	MaxRedirect int
	Timeout     time.Duration
}

type CrawlResult struct {
	URL         string
	FinalURL    string // after redirects
	Status      int
	ContentType string
	Body        []byte
	Digest      string // SHA-1 of body
	FetchedAt   time.Time
	// Links extracted from HTML (relative links resolved to FinalURL)
	Links []string
}

type DatasetStats struct {
	CrawlID         string
	CommittedShards int
	TotalShards     int
	TotalURLs       int64
	TotalBytes      int64 // sum of WARC record bytes across all URL rows
}

type DownloadResult struct {
	Path      string
	LocalPath string
	Bytes     int64
	Skipped   bool
	Err       error
}

type EdgeDegrees struct {
	NodeID    int64 `json:"node_id"`
	InDegree  int32 `json:"in_degree"`
	OutDegree int32 `json:"out_degree"`
}

type ForwardDoc struct {
	DocID       uint64  `json:"doc_id" table:"doc_id"`
	URL         string  `json:"url" table:"url"`
	CanonURL    string  `json:"canon_url" table:"canon_url"`
	Host        string  `json:"host" table:"host"`
	Title       string  `json:"title" table:"title"`
	Description string  `json:"description" table:"description"`
	Language    string  `json:"language" table:"language"`
	WordCount   int     `json:"word_count" table:"word_count"`
	LinkScore   float32 `json:"link_score" table:"link_score"`
	Snippet     string  `json:"snippet" table:"snippet"`
}

type ForwardIndexWriter struct {
	// contains filtered or unexported fields
}

type Frontier struct {
	// contains filtered or unexported fields
}

type FrontierEntry struct {
	URL      string  // normalized URL
	Host     string  // hostname for politeness grouping
	Priority float32 // harmonic centrality (higher = crawl sooner)
	NextAt   int64   // earliest Unix timestamp to fetch
	Depth    uint8   // BFS depth from seed
	Retries  uint8
}

type HFClient struct {
	// contains filtered or unexported fields
}

type HFOperation struct {
	LocalPath  string
	PathInRepo string
}

type HTTPClient struct {
	// contains filtered or unexported fields
}

type HostCDXStats struct {
	Host             string `json:"url_host_name"`
	RegisteredDomain string `json:"registered_domain"`
	URLCount         int64  `json:"url_count"`
	Status2xx        int64  `json:"status_2xx"`
	Status3xx        int64  `json:"status_3xx"`
	Status4xx        int64  `json:"status_4xx"`
	Status5xx        int64  `json:"status_5xx"`
	TopMIME          string `json:"top_mime"`
	Language         string `json:"language"`
	FirstSeen        string `json:"first_seen"`
	LastSeen         string `json:"last_seen"`
	TotalBytes       int64  `json:"total_bytes"`
}

type HostDatasetRow struct {
	Host             string  `parquet:"host"`
	HostRev          string  `parquet:"host_rev,dict"`
	TLD              string  `parquet:"tld,dict"`
	RegisteredDomain string  `parquet:"registered_domain,dict"`
	CrawlID          string  `parquet:"crawl_id,dict"`
	GraphID          string  `parquet:"graph_id,dict"`
	HarmonicPos      int64   `parquet:"harmonic_pos"`
	HarmonicVal      float64 `parquet:"harmonic_val"`
	PageRankPos      int64   `parquet:"pagerank_pos"`
	PageRankVal      float64 `parquet:"pagerank_val"`
	InDegree         int32   `parquet:"in_degree"`
	OutDegree        int32   `parquet:"out_degree"`
	URLCount         int64   `parquet:"url_count"`
	Status2xx        int64   `parquet:"status_2xx"`
	Status3xx        int64   `parquet:"status_3xx"`
	Status4xx        int64   `parquet:"status_4xx"`
	Status5xx        int64   `parquet:"status_5xx"`
	TopMIME          string  `parquet:"top_mime,dict"`
	Language         string  `parquet:"language,dict"`
	FirstSeen        string  `parquet:"first_seen,dict"`
	LastSeen         string  `parquet:"last_seen,dict"`
	TotalBytes       int64   `parquet:"total_bytes"`
}

type HostDiffEntry struct {
	Host        string  `json:"host" table:"host"`
	TotalURLs   int64   `json:"total_urls" table:"total_urls"`
	ChangedURLs int64   `json:"changed_urls" table:"changed_urls"`
	ChangeRate  float64 `json:"change_rate" table:"change_rate"`
	Tier        int     `json:"tier" table:"tier"`
}

type HostRecord struct {
	// Identity
	Host             string `json:"host" kit:"id" table:"host"`
	HostRev          string `json:"host_rev,omitempty" table:"-"`
	TLD              string `json:"tld" table:"tld"`
	RegisteredDomain string `json:"registered_domain,omitempty" table:"registered_domain"`

	// Rank signals (from rank table)
	HarmonicPos int64   `json:"harmonic_pos" table:"harmonic_pos"`
	HarmonicVal float64 `json:"harmonic_val" table:"harmonic_val"`
	PageRankPos int64   `json:"pagerank_pos" table:"pagerank_pos"`
	PageRankVal float64 `json:"pagerank_val" table:"pagerank_val"`

	// Graph topology (from edge files)
	InDegree  int64 `json:"in_degree" table:"in_degree"`
	OutDegree int64 `json:"out_degree" table:"out_degree"`

	// CDX statistics (from columnar Parquet index)
	URLCount   int64  `json:"url_count" table:"url_count"`
	Status2xx  int64  `json:"status_2xx" table:"status_2xx"`
	Status3xx  int64  `json:"status_3xx" table:"status_3xx"`
	Status4xx  int64  `json:"status_4xx" table:"status_4xx"`
	Status5xx  int64  `json:"status_5xx" table:"status_5xx"`
	TopMIME    string `json:"top_mime,omitempty" table:"top_mime"`
	Language   string `json:"language,omitempty" table:"language"`
	FirstSeen  string `json:"first_seen,omitempty" table:"first_seen"`
	LastSeen   string `json:"last_seen,omitempty" table:"last_seen"`
	TotalBytes int64  `json:"total_bytes" table:"total_bytes"`
}

type HostSchedule struct {
	Host        string  `json:"host" table:"host"`
	HarmonicPos int64   `json:"harmonic_pos" table:"harmonic_pos"`
	HarmonicVal float64 `json:"harmonic_val" table:"harmonic_val"`
	ChangeRate  float64 `json:"change_rate" table:"change_rate"`
	Tier        int     `json:"tier" table:"tier"`
	IntervalH   int     `json:"interval_h" table:"interval_h"`
	Score       float64 `json:"score" table:"score"`
}

type HostStore interface {
	GetHost(ctx context.Context, host string) (*HostRecord, error)
	TopHosts(ctx context.Context, n int, tld string) ([]HostRecord, error)
}

type IndexReader struct {
	N     int
	AvgDL float64
	// contains filtered or unexported fields
}

type IndexSearchStore struct {
	// contains filtered or unexported fields
}

type InvertedIndexBuilder struct {
	N int // total docs indexed

	TermCount int // unique term count (set by Flush)
	// contains filtered or unexported fields
}

type Library struct {
	Root  string
	Crawl string
}

type Location struct {
	Filename string `json:"filename" table:"filename"`
	Offset   int64  `json:"offset" table:"offset"`
	Length   int64  `json:"length" table:"length"`
	URL      string `json:"url,omitempty" table:"url,url"`
}

type MemHostStore struct {
	// contains filtered or unexported fields
}

type NewWARCRecord struct {
	TargetURI string
	Date      string
	RecordID  string
	Block     []byte // raw HTTP response bytes
}

type NewsFile struct {
	Path string `json:"path" table:"path"`
	Year int    `json:"year" table:"year"`
	Mon  int    `json:"month" table:"month"`
}

type ParquetWriter[T any] struct {
	// contains filtered or unexported fields
}

type PostingEntry struct {
	DocID uint64
	TF    uint32
	DL    uint32 // document token length at index time
}

type QualityResult struct {
	WordCount      int
	TitleLength    int
	HasMainContent bool
	SpamScore      float64 // 0–1
	IsParked       bool
	IsShortContent bool // word_count < 50
}

type Rank struct {
	Key         string  `json:"key" kit:"id" table:"key"` // host or domain (forward form)
	HarmonicPos int64   `json:"harmonic_pos" table:"harmonic_pos"`
	HarmonicVal float64 `json:"harmonic_val" table:"harmonic_val"`
	PageRankPos int64   `json:"pagerank_pos" table:"pagerank_pos"`
	PageRankVal float64 `json:"pagerank_val" table:"pagerank_val"`
}

type RateLimitError struct {
	RetryAfter time.Duration
	Msg        string
}

type RobotsCache struct {
	// contains filtered or unexported fields
}

type RobotsEntry struct {
	Rules      []RobotsRule
	CrawlDelay time.Duration
	ExpiresAt  int64 // Unix timestamp
}

type RobotsRule struct {
	Allow   bool
	Pattern string
}

type ScoredDoc struct {
	DocID uint64
	Score float64
}

type SearchResult struct {
	DocID    uint64  `json:"doc_id"`
	URL      string  `json:"url"`
	Host     string  `json:"host"`
	Title    string  `json:"title"`
	Snippet  string  `json:"snippet"`
	Score    float64 `json:"score"`
	Language string  `json:"language,omitempty"`
}

type SearchStore interface {
	Search(ctx context.Context, query string, k int) ([]SearchResult, error)
}

type ServeConfig struct {
	Addr     string // e.g. ":8080"
	DBPath   string // path to DuckDB/SQLite host database (optional)
	IndexDir string // path to the inverted index directory (optional)
}

type Source string

type TextResult struct {
	Title       string
	Description string // <meta name="description">
	CanonURL    string // canonical URL from meta or link[rel=canonical]
	Body        string // extracted clean body text
	WordCount   int
	Language    string // BCP-47 code inferred from lang attribute
}

type URLDatasetRow struct {
	// CDX identity
	Host  string `parquet:"host,dict"`
	RD    string `parquet:"rd,dict"`
	TLD   string `parquet:"tld,dict"`
	Proto string `parquet:"proto,dict"`
	URL   string `parquet:"url"`
	Surt  string `parquet:"surt"`
	// CDX fetch result
	ST     int32  `parquet:"st"`
	Redir  string `parquet:"redir"`
	Digest string `parquet:"digest,dict"`
	// CDX content
	MIME     string `parquet:"mime,dict"`
	MIMEDecl string `parquet:"mime_d,dict"`
	Charset  string `parquet:"charset,dict"`
	Lang     string `parquet:"lang,dict"`
	Trunc    string `parquet:"trunc,dict"`
	// CDX timing and size
	TS    string `parquet:"ts,dict"`
	Bytes int64  `parquet:"bytes"`
	// CDX WARC location (for byte-range content fetch)
	WARCFile string `parquet:"warc_f,dict"`
	WARCOff  int64  `parquet:"warc_o"`
	RobotsOK bool   `parquet:"robots_ok"`
	Crawl    string `parquet:"crawl,dict"`
	// Rank signals (joined from rank table by host)
	GraphID     string  `parquet:"graph_id,dict"`
	HarmonicPos int64   `parquet:"harmonic_pos"`
	HarmonicVal float64 `parquet:"harmonic_val"`
	PageRankPos int64   `parquet:"pagerank_pos"`
	PageRankVal float64 `parquet:"pagerank_val"`
}

type VertexRecord struct {
	ID      int64  `json:"id" kit:"id" table:"id"`
	HostRev string `json:"host_rev" table:"host_rev"`
	Host    string `json:"host" table:"host"`
}

type WARCHeader = warc.Header

type WARCParquetRow struct {
	RecordID      string    `parquet:"record_id,dict"`
	CrawlID       string    `parquet:"crawl_id,dict"`
	WARCType      string    `parquet:"warc_type,dict"`
	TargetURI     string    `parquet:"target_uri"`
	Date          time.Time `parquet:"date,timestamp(microsecond)"`
	IPAddress     string    `parquet:"ip_address,dict"`
	PayloadDigest string    `parquet:"payload_digest"`
	ContentType   string    `parquet:"content_type,dict"`
	ContentLength int64     `parquet:"content_length"`
	Truncated     string    `parquet:"truncated,dict"`
	HTTPStatus    int32     `parquet:"http_status"`
	HTTPMIME      string    `parquet:"http_mime,dict"`
	WARCFilename  string    `parquet:"warc_filename,dict"`
	WARCOffset    int64     `parquet:"warc_offset"`
	WARCLength    int64     `parquet:"warc_length"`
	Title         string    `parquet:"title"`
	Language      string    `parquet:"language,dict"`
	Markdown      string    `parquet:"markdown"`
	Text          string    `parquet:"text"`
}

type WARCRecord = warc.Record

type WATLink = wat.Link

type WATMeta = wat.Meta

type WATParquetRow struct {
	RecordID    string    `parquet:"record_id,dict"`
	CrawlID     string    `parquet:"crawl_id,dict"`
	URL         string    `parquet:"url"`
	Date        time.Time `parquet:"date,timestamp(microsecond)"`
	HTTPStatus  int32     `parquet:"http_status"`
	ContentType string    `parquet:"content_type,dict"`
	Title       string    `parquet:"title"`
	LinksCount  int32     `parquet:"links_count"`
	Links       string    `parquet:"links"` // JSON
	Metas       string    `parquet:"metas"` // JSON
}

type WATRecord = wat.Record

type WETParquetRow struct {
	RecordID        string    `parquet:"record_id,dict"`
	CrawlID         string    `parquet:"crawl_id,dict"`
	URL             string    `parquet:"url"`
	Date            time.Time `parquet:"date,timestamp(microsecond)"`
	ContentLanguage string    `parquet:"content_language,dict"`
	TextLength      int32     `parquet:"text_length"`
	Text            string    `parquet:"text"`
}

type WETRecord = wet.Record

type WebGraph struct {
	ID        string // e.g. cc-main-2026-mar-apr-may
	BaseURL   string // https://data.commoncrawl.org/projects/hyperlinkgraph/{ID}/
	HostNodes int64
	HostArcs  int64
	Published string
}