graph

type Action struct {
	ID     string  `json:"id"`
	Type   string  `json:"type"` // link | form | button
	Label  string  `json:"label"`
	Href   string  `json:"href,omitempty"`
	Method string  `json:"method,omitempty"`
	Fields []Field `json:"fields,omitempty"`
	Region Region  `json:"region,omitempty"`
	// Internal marks links that stay on the same registrable domain.
	Internal bool `json:"internal,omitempty"`
	Disabled bool `json:"disabled,omitempty"`
}

type Audit struct {
	// GraphRetention is the share of the text the capture observed that
	// survived into the emitted graph.
	//
	// The name is narrow on purpose. It measures the graph stage, not the
	// capture stage: if the sweep missed half the page, this number cheerfully
	// reports that all of the remaining half was retained. That is true and it
	// is not what "coverage" would be read to mean. True coverage against
	// hand-written ground truth is a benchmark number, measured where the right
	// answer is actually known, and it lives in the benchmark report rather
	// than here.
	GraphRetention float64 `json:"graph_retention"`
	// ObservedChars and EmittedChars are the raw terms, so the ratio can be
	// checked rather than trusted.
	ObservedChars int `json:"observed_chars,omitempty"`
	EmittedChars  int `json:"emitted_chars,omitempty"`

	// OrderConfidence is how much to trust the reading order.
	OrderConfidence Confidence `json:"order_confidence"`
	OrderScore      float64    `json:"order_score,omitempty"`
	// OrderBasis is how the order was computed: "geometry" when document
	// coordinates were meaningful, "checkpoint" when the page scrolled by
	// transform and coordinates were not.
	OrderBasis string `json:"order_basis,omitempty"`
	// OrderAgreement is how much the geometric ordering and the
	// first-appearance ordering agree. Both are computed anyway, so comparing
	// them is nearly free, and divergence is exactly where reading order goes
	// wrong.
	OrderAgreement float64 `json:"order_agreement,omitempty"`

	// HeadingSeparation measures how cleanly the type scale separated into
	// levels. A clean gap between size clusters means the heading inference had
	// something real to work with; an overlapping continuum means it guessed.
	HeadingSeparation float64    `json:"heading_separation,omitempty"`
	HeadingConfidence Confidence `json:"heading_confidence"`

	// Dropped counts the runs that were captured but did not become blocks,
	// grouped by why.
	//
	// This is the difference between an artifact that is empty and an artifact
	// that explains itself. A page that comes back with nothing is the single
	// most alarming output this tool can produce, and without this the user has
	// no way to tell "the page really is a canvas" from "every run was excluded
	// by a threshold". Both have happened; only one is a bug.
	Dropped []DropCount `json:"dropped,omitempty"`

	// ReachedBottom is false when a budget stopped the sweep early.
	ReachedBottom bool `json:"reached_bottom"`
	// FramesBlocked counts cross-origin iframes whose content could not be read.
	FramesBlocked int `json:"frames_blocked,omitempty"`
	// CanvasesUnrecovered counts canvases that passed the size gate but yielded
	// nothing.
	CanvasesUnrecovered int `json:"canvases_unrecovered,omitempty"`
	// Notes are human-readable limitations discovered during the render.
	Notes []string `json:"notes,omitempty"`
}

type Block struct {
	ID   string    `json:"id"`
	Type BlockType `json:"type"`
	// Level is the heading level, 1 to 6. Zero for non-headings.
	Level int    `json:"level,omitempty"`
	Text  string `json:"text"`
	Order int    `json:"order"`

	Source Source `json:"source"`
	// Confidence is the reported bucket. Score is the underlying number, kept
	// for calibration work and deliberately not the headline.
	Confidence Confidence `json:"confidence"`
	Score      float64    `json:"score,omitempty"`
	// Verified is set only on blocks recovered from pixels.
	Verified Verification `json:"verified,omitempty"`

	Checkpoint int        `json:"checkpoint"`
	BBox       [4]float64 `json:"bbox"`

	Region Region `json:"region"`
	// SectionID is the id of the section this block belongs to, which is what
	// lets an agent pull one section instead of the whole artifact.
	SectionID string `json:"section_id,omitempty"`

	// Href is set when the whole block is a link.
	Href string `json:"href,omitempty"`
	// MediaID links an image or media block to its media entry.
	MediaID string `json:"media_id,omitempty"`
	// ListOrdered distinguishes numbered from bulleted lists.
	ListOrdered bool `json:"list_ordered,omitempty"`

	// Style is the typographic evidence the heading inference used. Kept
	// because a wrong heading level is the most common extraction complaint,
	// and without this there is no way to see why the level was chosen.
	Style StyleInfo `json:"style,omitempty"`

	// Flags records anything unusual the extraction noticed about this run:
	// text hidden by colour, metadata that hit its cap, control characters
	// stripped. A consumer that wants to be suspicious has something to be
	// suspicious about.
	Flags []string `json:"flags,omitempty"`
}

type BlockType string

type Confidence string

type DropCount struct {
	Reason string `json:"reason"`
	Runs   int    `json:"runs"`
	Chars  int    `json:"chars"`
}

type Field struct {
	Name     string   `json:"name"`
	Type     string   `json:"type"`
	Label    string   `json:"label,omitempty"`
	Required bool     `json:"required,omitempty"`
	Options  []string `json:"options,omitempty"`
	Pattern  string   `json:"pattern,omitempty"`
}

type Gap struct {
	Label string `json:"label"`
	Kind  string `json:"kind"`
	// Reason says why it was not opened.
	Reason string `json:"reason"`
	// LatentIDs are the latent blocks this control would reveal, when they are
	// already captured.
	LatentIDs []string `json:"latent_ids,omitempty"`
}

type Graph struct {
	SchemaVersion string    `json:"schema_version"`
	URL           string    `json:"url"`
	FinalURL      string    `json:"final_url,omitempty"`
	DistilledAt   time.Time `json:"distilled_at"`
	// ContentHash covers the normalised semantic graph -- block text, type,
	// level, order -- and not the serialised output. Trivial variation in
	// whitespace or asset URLs then does not churn the hash, and re-distilling
	// an unchanged site is genuinely a no-op.
	ContentHash string `json:"content_hash"`
	Generator   string `json:"generator"`

	// Outcome is the first thing a caller should read: whether this artifact
	// describes the page that was asked for, or something that stood in front
	// of it. Placed above the content for the same reason.
	Outcome Outcome `json:"outcome"`

	Title       string `json:"title"`
	Description string `json:"description,omitempty"`
	Summary     string `json:"summary"`
	Lang        string `json:"lang,omitempty"`

	Sections []Section `json:"sections"`
	Blocks   []Block   `json:"blocks"`
	Actions  []Action  `json:"actions"`
	MediaAll []Media   `json:"media"`
	Links    []Link    `json:"links"`

	// Structured holds whitelisted facts lifted from JSON-LD. Never raw.
	Structured []StructuredFact `json:"structured,omitempty"`
	// FAQ holds question-and-answer pairs published as FAQPage structured data.
	//
	// They are kept apart from Blocks because their provenance is different:
	// nobody saw these words rendered, they were declared. A consumer that
	// wants only what a visitor could read can ignore this array; one answering
	// a question about the site would be foolish to.
	FAQ []QAPair `json:"faq,omitempty"`

	// Latent is the quarantine. It is a separate top-level key by design and
	// must never be merged into Blocks. See LatentBlock.
	Latent []LatentBlock `json:"latent,omitempty"`
	// Gaps are disclosure controls whose content was not opened.
	Gaps []Gap `json:"gaps,omitempty"`

	Stats      Stats      `json:"stats"`
	Audit      Audit      `json:"audit"`
	Provenance Provenance `json:"provenance"`
	// contains filtered or unexported fields
}

type HydrationLink struct {
	Text string
	Href string
}

type Input struct {
	RequestedURL string
	FinalURL     string
	Merged       *capture.Merged
	Notes        []string
	// OriginalBytes is the transfer size of the undistilled page, used for the
	// stats that justify the project.
	OriginalBytes int64
	// OriginalText is the raw page text an unaided agent would have had to
	// read, used to estimate the token cost that was avoided.
	OriginalText string
	// ReachedBottom reports whether the sweep saw the end of the document.
	ReachedBottom bool
	// EntryGate names an interstitial standing between the visitor and the
	// site, when one was detected.
	EntryGate string
	// Now is injectable so golden tests are not time-dependent.
	Now time.Time
	// Generator identifies the build, e.g. "sieve/0.1.0".
	Generator string
	// Provenance carries the render-level facts through unchanged.
	Provenance Provenance
	// Outcome describes what happened when the page was read: whether it was
	// refused, challenged, unhydrated, empty or fine. Every path that builds a
	// graph fills it in, because the one thing a caller must never have to
	// infer is whether reading worked.
	Outcome OutcomeInput
}

type LatentBlock struct {
	ID   string    `json:"id"`
	Type BlockType `json:"type"`
	Text string    `json:"text"`

	// Reason is why this text was never rendered.
	Reason string `json:"reason"`
	// ControlLabel is the accessible name of the widget that would reveal it --
	// the tab, the accordion header, the summary. This is the whole point of
	// keeping the tier: an artifact can say "there is a section behind a tab
	// labelled Pricing" instead of silently omitting it.
	ControlLabel string `json:"control_label,omitempty"`
	ControlKind  string `json:"control_kind,omitempty"`

	Region Region `json:"region,omitempty"`
	Href   string `json:"href,omitempty"`
	Level  int    `json:"level,omitempty"`

	// Trust is the marker that survives into every format. It is a constant,
	// not a computed field, because a computed field can compute to empty.
	Trust string `json:"trust"`

	Flags []string `json:"flags,omitempty"`
}

type Link struct {
	Href     string `json:"href"`
	Text     string `json:"text"`
	Internal bool   `json:"internal"`
	Region   Region `json:"region,omitempty"`
}

type Media struct {
	ID      string `json:"id"`
	Type    string `json:"type"` // image | video | model
	Src     string `json:"src"`
	Alt     string `json:"alt,omitempty"`
	Caption string `json:"caption,omitempty"`
	// Source says where the description came from: the page's own alt text, a
	// figure caption, or a vision model.
	Source     string     `json:"source"`
	Confidence Confidence `json:"confidence,omitempty"`
	Width      float64    `json:"width,omitempty"`
	Height     float64    `json:"height,omitempty"`
	// Local is the path inside the artifact's media/ folder, when the asset
	// was downloaded.
	Local string `json:"local,omitempty"`
	// Flags records metadata that hit its cap.
	Flags []string `json:"flags,omitempty"`
}

type Outcome struct {
	Status Status `json:"status"`
	// Evidence lists what led to the status, shortest first. It is never empty
	// for a status other than ok: a verdict a caller cannot check is a verdict
	// they have to trust.
	Evidence []string `json:"evidence,omitempty"`

	// HTTPStatus is the code the server answered with, recorded whatever it
	// was. An agent that knows a read failed because of a 429 stops retrying;
	// one that only knows the page was empty keeps going.
	HTTPStatus int `json:"http_status,omitempty"`
	// BodyExcerpt is the beginning of the response body on an error, which is
	// where a proxy or policy filter says who blocked the request and why.
	BodyExcerpt string `json:"body_excerpt,omitempty"`
}

type OutcomeInput struct {
	HTTPStatus    int
	Body          string
	Blocked       bool
	BlockedReason string
	// RobotsRefused records a robots.txt disallow, which is a refusal by policy
	// rather than by the server.
	RobotsRefused bool
	// EntryGate is an interstitial that was found and not passed.
	EntryGate string
	// ShellHTML reports that the served document was judged a shell.
	ShellHTML bool
	// Rendered reports that a browser ran the page. Without it, an empty result
	// cannot be called empty-after-render.
	Rendered bool
	// TierFellBack reports that a higher tier was chosen and did not deliver.
	TierFellBack bool
	// TierReason explains that fallback, and is used as the evidence for it.
	TierReason string
	// SweepTruncated reports that the sweep did not finish the document.
	//
	// It is evidence rather than a verdict. Long pages routinely end a sweep
	// before the bottom, so a status driven by it would mark most of the web
	// partial and mean nothing; it is recorded when the artifact is partial for
	// a reason that does distinguish it.
	SweepTruncated bool
}

type Provenance struct {
	// Tier is which rung of the escalation ladder answered: fetch, render,
	// sweep or recover.
	Tier string `json:"tier"`
	// TierReason is the scoring that led to that decision, so a caller can see
	// why a page was judged cheap or expensive rather than having to guess.
	TierReason string `json:"tier_reason,omitempty"`
	// TierScore is the escalation score itself.
	TierScore float64 `json:"tier_score,omitempty"`
	// TierFellBack reports that a higher tier was chosen and did not deliver,
	// so the artifact rests on a cheaper one than the escalator asked for. It
	// is the difference between "this page only needed a fetch" and "this page
	// needed a browser and did not get a usable one", which read identically in
	// the tier field alone.
	TierFellBack bool `json:"tier_fell_back,omitempty"`
	// TierPinned reports that the tier came from a per-domain memory rather
	// than from a fresh judgement. Once a domain escalates it stays escalated,
	// so a page near the threshold does not oscillate between runs.
	TierPinned bool `json:"tier_pinned,omitempty"`

	// Libraries are the animation, scroll and 3D libraries detected.
	Libraries []string `json:"libraries,omitempty"`
	// NormalizerVersion is part of the content hash. Changing normalization
	// invalidates every cached artifact, so it is recorded rather than implied.
	NormalizerVersion int `json:"normalizer_version"`
	// SchemaVersion of the trace payload.
	Trace any `json:"trace,omitempty"`

	// Blocked records that the site refused this client.
	Blocked       bool   `json:"blocked,omitempty"`
	BlockedReason string `json:"blocked_reason,omitempty"`

	// Private marks an artifact produced against an authenticated or
	// user-profile session. It is never eligible for a shared cache and never
	// included in a diagnostic bundle.
	Private bool `json:"private,omitempty"`

	// Incomplete marks a partial artifact from progressive emission. It is
	// checked at the cache boundary: a partial result must never be stored as
	// if it were final.
	Incomplete bool `json:"incomplete,omitempty"`

	// CorpusBytes and CorpusSaturated describe the confirm-only index used for
	// canvas cross-checking. A saturated index makes a negative result weaker
	// evidence, and saying so is more useful than pretending otherwise.
	CorpusBytes     int  `json:"corpus_bytes,omitempty"`
	CorpusSaturated bool `json:"corpus_saturated,omitempty"`
}

type QAPair struct {
	Question string `json:"question"`
	Answer   string `json:"answer"`
}

type Region string

type Section struct {
	ID    string `json:"id"`
	Title string `json:"title"`
	Level int    `json:"level"`
	// FirstBlock and LastBlock are inclusive block ids bounding the section.
	FirstBlock string `json:"first_block"`
	LastBlock  string `json:"last_block"`
	BlockCount int    `json:"block_count"`
	// Chars is the section's text length, which is what an agent needs to
	// decide whether pulling it will fit in its budget.
	Chars int `json:"chars"`
	// Tokens is an estimate, not a measurement. See the tokens package.
	Tokens int `json:"tokens"`
}

type Source string

type Stats struct {
	OriginalBytes  int64 `json:"original_bytes"`
	ArtifactBytes  int64 `json:"artifact_bytes"`
	OriginalTokens int   `json:"original_tokens"`
	// ArtifactTokens counts the default payload only. Latent content is
	// excluded: it is not delivered by default, and counting it would make the
	// headline number worse for every caller who never asks for it.
	ArtifactTokens int `json:"artifact_tokens"`
	// LatentTokens is reported separately so the exclusion is visible rather
	// than convenient.
	LatentTokens int `json:"latent_tokens,omitempty"`

	Checkpoints  int `json:"checkpoints"`
	RawNodes     int `json:"raw_nodes"`
	ContentNodes int `json:"content_blocks"`
	ChromeNodes  int `json:"chrome_blocks"`
	LatentNodes  int `json:"latent_blocks"`
	DroppedNodes int `json:"dropped_nodes"`
}

type Status string

type StructuredFact struct {
	// Type is the schema.org type it came from, e.g. "Organization".
	Type string `json:"type"`
	// Field is the whitelisted property name.
	Field string `json:"field"`
	// Value is the normalised, length-capped value.
	Value string `json:"value"`
}

type StyleInfo struct {
	FontSize   float64 `json:"font_size,omitempty"`
	Weight     int     `json:"weight,omitempty"`
	Tracking   float64 `json:"tracking,omitempty"`
	Uppercase  bool    `json:"uppercase,omitempty"`
	Italic     bool    `json:"italic,omitempty"`
	Family     string  `json:"family,omitempty"`
	MaxOpacity float64 `json:"max_opacity,omitempty"`
}

type Verification string