meguri

type AnchorHint uint8

type Discovery struct {
	URLKey URLKey // identity and routing key

	CanonicalURL string // the canonical URL text, inline (crosses partitions)

	Depth           uint16          // source depth + 1, candidate depth
	DiscoverySource DiscoverySource // seed | link | sitemap | redirect | manual

	SrcHostKey uint64     // HostKey of the page the link was found on
	LinkWeight float32    // OPIC cash this link carries from the source
	AnchorHint AnchorHint // coarse anchor-text quality bucket

	ObservedAt uint32 // epoch-hours, when discovered
}

type DiscoverySource uint8

type HostGrouping uint8

type HostRecord struct {
	HostKey uint64 // the host group identity

	HostRef        uint64       // offset into string region, the host group key
	Grouping       HostGrouping // registrable-domain or full-host
	RegistrableRef uint64       // offset into string region, the registrable domain

	ResolvedIP [16]byte // cached DNS result, IPv4-mapped into 16 bytes
	IPExpiry   uint32   // epoch-hours, DNS cache expiry

	RobotsFetched uint32 // epoch-hours, robots.txt last fetched
	RobotsExpiry  uint32 // epoch-hours, robots.txt cache expiry
	RobotsRef     uint64 // offset into blob region, parsed robots rules
	CrawlDelay    uint16 // deciseconds, effective crawl delay

	HostNextEligible uint32 // epoch-seconds, per-host token bucket
	IPNextEligible   uint32 // epoch-seconds, per-IP token bucket

	URLBudget uint32 // per-host URL cap
	URLCount  uint32 // current URLs of this host in the frontier
	DepthCap  uint16 // max crawl depth for this host

	HostScore float32 // imported host quality / PageRank, 0 if none

	CrawlTotal uint32 // lifetime successful fetches
	ErrorTotal uint32 // lifetime failed fetches
	AvgLatency uint16 // milliseconds, smoothed fetch latency

	Flags uint16 // trap-suspect | robots-missing | dead-host | grouping-override
}

type HostSignal struct {
	HostKey   uint64
	HostScore float32
}

type Outcome struct {
	URLKey URLKey // which URL this outcome is for

	HTTPStatus uint16 // the status returned
	FetchedAt  uint32 // epoch-hours, fetch completion
	LatencyMS  uint16 // round-trip latency, milliseconds

	NotModified bool   // 304 from the conditional GET
	ContentFP   uint64 // fingerprint of the body, 0 if none
	Simhash     uint64 // near-dup signature, 0 if none
	ContentLen  uint32 // body length in bytes

	ETag           string // ETag header, empty if none
	LastModified   uint32 // epoch-hours, 0 if none
	RedirectTarget string // resolved redirect URL, empty if none

	Retryable        bool   // transient failure, retry warranted
	RetryAfter       uint16 // deciseconds from a Retry-After header, 0 if none
	RobotsCrawlDelay uint16 // deciseconds, fresh Crawl-delay if robots re-read

	// RobotsBody is the raw robots.txt bytes, populated only for a robots fetch
	// (fetch.Request.Robots). Parsing is policy and stays in meguri (doc 07): the
	// fetcher hands back the bytes, the frontier turns them into rules.
	RobotsBody []byte

	Links []Discovery // extracted out-links, canonicalized and ready to route
}

type Signal struct {
	Epoch uint64       // the import's version, higher supersedes lower
	Hosts []HostSignal // dense host reputation
	URLs  []URLSignal  // sparse per-page rank
}

type URLKey struct {
	HostKey uint64
	PathKey uint64
}

type URLRecord struct {
	URLKey  URLKey // 128 bits, HostKey || PathKey
	HostKey uint64 // high half of URLKey, kept for host joins

	Status          URLStatus       // state-machine state
	Priority        float32         // OPIC + optional imported PageRank
	Depth           uint16          // link distance from nearest seed
	DiscoverySource DiscoverySource // seed | link | sitemap | redirect | manual

	URLRef uint64 // offset into the string region for the canonical URL

	FirstSeen   uint32 // epoch-hours, first discovery
	LastCrawled uint32 // epoch-hours, last successful fetch
	LastChanged uint32 // epoch-hours, last observed content change
	NextDue     uint32 // epoch-hours, next scheduled crawl

	Lambda         float32 // Poisson change rate, changes/hour
	CrawlCount     uint32  // successful fetches
	ChangeCount    uint32  // fetches that saw a change
	NoChangeStreak uint16  // consecutive no-change fetches

	ETagRef      uint64 // offset into string region, 0 if no ETag
	LastModified uint32 // epoch-hours, 0 if no Last-Modified

	ContentFP uint64 // content fingerprint of last body
	Simhash   uint64 // near-dup signature of last body

	HTTPStatus  uint16 // status of last crawl
	RedirectRef uint64 // reference to redirect-target record, 0 if none

	RetryCount uint8  // consecutive transient failures
	ErrorCount uint16 // lifetime failed fetches
}

type URLSignal struct {
	URLKey   URLKey
	PageRank float32
}

type URLStatus uint8