gate

type Checker struct {
	// contains filtered or unexported fields
}

type Fetched struct {
	// Title and Markdown are the fetched title and body.
	Title    string
	Markdown string
	// Build returns the would-be document (path, frontmatter with
	// provenance, body) for a title and body. The gate calls it again after
	// a refetch, so the judged document is always the one that is written.
	Build func(title, markdown string) (*corpus.Document, error)
	// RequestedURL, FinalURL, StatusCode and SiteName are what the fetcher
	// reported (SiteName feeds the "title equals the site name" rule).
	RequestedURL string
	FinalURL     string
	StatusCode   int
	SiteName     string
	// Refetch performs the one fresh stage-4 scrape (URL sources only; nil
	// disables stage 4).
	Refetch func(ctx context.Context) (*firecrawl.ScrapeResult, error)
	// PlatformID is the `youtube:<id>` / `instagram:<shortcode>` identity
	// when the source URL does not carry it.
	PlatformID string
	// Transcript adds the `no_speech_content` question and disables the thin
	// rule.
	Transcript bool
	// PrefetchReview marks an item stage 2 put in the review band; Shadow
	// names a would-be pre-fetch skip that a shadow gate did not apply.
	PrefetchReview bool
	PrefetchShadow string
	// PrefetchPOffTopic is the pre-fetch P(off_topic) (review evidence).
	PrefetchPOffTopic float64
	// PrefetchReceipt is the pre-fetch receipt key.
	PrefetchReceipt string
}

type Index struct {
	// contains filtered or unexported fields
}

type Item struct {
	// ID is the stable skipped id; derived from URL with SkippedID when
	// empty.
	ID          string
	URL         string
	Title       string
	Description string
	// Channel is the channel, uploader or site name.
	Channel string
	// Date is the publication date as text.
	Date string
	// Path is the would-be topic-relative path (for collected_on_purpose_paths).
	Path string
	// SourceKind is the would-be source kind (provenance).
	SourceKind string
}

type Match struct {
	// Path is the topic-relative path of the existing source (the quarantine
	// path for a quarantined one).
	Path string
	// Quarantined reports that the existing source is in raw/_quarantine/.
	Quarantined bool
	// By names what matched: url, platform_id or body_hash.
	By string
}

type NearCandidate struct {
	Doc *corpus.Document
	// Origin is "bm25" or "title".
	Origin string
}

type Outcome struct {
	// Triage is kept, review, quarantined, skipped or duplicate-skipped.
	Triage string `json:"triage"`
	// Reason is the triage_reason (off_topic, thin, error_page, paywall,
	// not_an_article, no_speech, duplicate); empty when kept.
	Reason string `json:"triage_reason,omitempty"`
	// Stage is the stage that set Triage.
	Stage string `json:"stage,omitempty"`
	// Quality is the value for the `quality` key: the quality reason when a
	// quality check or question fired at or above the review threshold.
	Quality string `json:"quality,omitempty"`
	// Purpose, Question, Probability and ReceiptKey describe the decision
	// behind Triage (for review items and labels).
	Purpose     decisions.Purpose `json:"purpose,omitempty"`
	Question    string            `json:"question,omitempty"`
	Probability float64           `json:"probability,omitempty"`
	ReceiptKey  string            `json:"receipt_key,omitempty"`
	// DuplicateOf is the existing source a duplicate matched.
	DuplicateOf string `json:"duplicate_of,omitempty"`
	// DuplicateQuarantined reports that DuplicateOf is quarantined.
	DuplicateQuarantined bool `json:"duplicate_quarantined,omitempty"`
	// Shadow lists would-be outcomes a shadow gate did not apply, such as
	// "quarantined:off_topic (relevance shadow)".
	Shadow []string `json:"shadow,omitempty"`
	// Supersedes lists `[[target]]` wikilinks of older versions (stage 6).
	Supersedes []string `json:"supersedes,omitempty"`
	// Flags are the code quality flags of the kept body.
	Flags []quality.Flag `json:"flags,omitempty"`
	// Refetched reports that the stage-4 refetch body was kept.
	Refetched bool `json:"refetched,omitempty"`
	// Undecided lists "<question>:<reason>" answers that could not be used.
	Undecided []string `json:"undecided,omitempty"`
	// Excluded reports that the source matches topic.yaml
	// `decisions.exclude`: no decision call was made about it (code checks
	// still ran) and it is kept without a judgment.
	Excluded bool `json:"excluded,omitempty"`
	// Evidence is one human-readable line for review items and logs.
	Evidence string `json:"evidence,omitempty"`
}

type PrefetchDecision struct {
	Item Item `json:"-"`
	// Action is fetch, review (fetch and write with triage: review) or skip.
	Action string `json:"action"`
	// Reason explains a fetch without a role answer (collected path,
	// relevance off, undecided:<reason>).
	Reason     string  `json:"reason,omitempty"`
	Role       string  `json:"role,omitempty"`
	POffTopic  float64 `json:"p_off_topic"`
	PKept      float64 `json:"p_kept"`
	ReceiptKey string  `json:"receipt_key,omitempty"`
	// Shadow is the would-be action a shadow gate did not apply ("skip").
	Shadow string `json:"shadow,omitempty"`
}

type PrefetchOptions struct {
	Batch string
	Query string
}

type Ref struct {
	// URL is the source URL (normalized with NormalizeURL).
	URL string
	// PlatformID is a `youtube:<id>` / `instagram:<shortcode>` identity; when
	// empty it is derived from URL.
	PlatformID string
	// BodyHash is corpus.BodyHash of the body about to be written.
	BodyHash string
}

type SkippedRow struct {
	ID          string  `json:"id"`
	Time        string  `json:"time"`
	Batch       string  `json:"batch,omitempty"`
	Query       string  `json:"query,omitempty"`
	URL         string  `json:"url"`
	Title       string  `json:"title,omitempty"`
	Description string  `json:"description,omitempty"`
	Channel     string  `json:"channel,omitempty"`
	Date        string  `json:"date,omitempty"`
	POffTopic   float64 `json:"p_off_topic"`
	ReceiptKey  string  `json:"receipt_key,omitempty"`
	Rescued     bool    `json:"rescued"`
}

type Tally struct {
	Kept        int `json:"kept"`
	Review      int `json:"review"`
	Quarantined int `json:"quarantined"`
	Skipped     int `json:"skipped"`
	Duplicates  int `json:"duplicates"`
	// Undecided counts review sources whose gate judgment failed (a subset
	// of Review).
	Undecided int `json:"undecided"`
	// Excluded counts written sources matching decisions.exclude (never
	// judged).
	Excluded int `json:"excluded,omitempty"`
	// Shadow counts sources with at least one would-be outcome not applied.
	Shadow int `json:"shadow"`
}