crawler

type Coverage struct {
	// Complete is true when no in-scope, robots-allowed URL was left un-fetched because of a
	// depth or page-count limit.
	Complete bool `json:"complete"`
	// DiscoveredNotCrawled is the number of distinct in-scope URLs that were discovered but
	// never fetched because a limit was reached.
	DiscoveredNotCrawled int `json:"discovered_not_crawled,omitempty"`
	// PageLimitReached / DepthLimitReached record which configured limit cut the crawl short.
	PageLimitReached  bool `json:"page_limit_reached,omitempty"`
	DepthLimitReached bool `json:"depth_limit_reached,omitempty"`
	// MaxPages / MaxDepth echo the limits in effect (0 = unlimited), for the report message.
	MaxPages int `json:"max_pages,omitempty"`
	MaxDepth int `json:"max_depth,omitempty"`
}

type Engine struct {
	// contains filtered or unexported fields
}

type Fetcher interface {
	Fetch(ctx context.Context, rawURL string) (*Page, error)
}

type HTTPFetcher struct {
	// contains filtered or unexported fields
}

type Link struct {
	URL      string `json:"url"`                // absolute, normalized for dedup (trailing slash stripped)
	Resolved string `json:"resolved,omitempty"` // absolute, normalized but trailing slash preserved
	Anchor   string `json:"anchor"`             // visible text
	Rel      string `json:"rel"`
	Nofollow bool   `json:"nofollow"`
	External bool   `json:"external"` // points outside the seed host
}

type Options struct {
	MaxDepth      int
	MaxPages      int
	Concurrency   int
	RatePerSecond float64
	UserAgent     string
	// UserAgents is an optional pool of User-Agent strings to rotate across. When non-empty it
	// supersedes UserAgent; UserAgentRotation picks one per request.
	UserAgents        []string
	UserAgentRotation RotationStrategy
	// Proxies is an optional pool of proxy URLs (http, https, or socks5) to route requests
	// through. When non-empty, ProxyRotation picks one per request. Empty leaves Go's default
	// proxy behavior (honoring HTTP_PROXY/HTTPS_PROXY/NO_PROXY) in place.
	Proxies         []*url.URL
	ProxyRotation   RotationStrategy
	Include         []*regexp.Regexp
	Exclude         []*regexp.Regexp
	RespectRobots   bool
	AllowSubdomains bool
	FollowExternal  bool
	FollowNofollow  bool
	// StripQuery drops the query string when normalizing URLs, so URLs that differ only by
	// their query collapse to one and are crawled once.
	StripQuery   bool
	Timeout      time.Duration
	MaxBodyBytes int64
	MaxRedirects int
	// Verbose logs each fetch and every rate-limit change to stderr while crawling.
	Verbose bool
	// AdaptiveDelay automatically slows the crawl (halving requests-per-second, honoring any
	// Retry-After header) when the server responds with HTTP 429 or 503.
	AdaptiveDelay bool
}

type Page struct {
	RequestedURL string            `json:"requested_url"`
	FinalURL     string            `json:"final_url"`
	StatusCode   int               `json:"status_code"`
	Header       http.Header       `json:"-"`
	ContentType  string            `json:"content_type"`
	Body         []byte            `json:"-"`
	RawBody      []byte            `json:"-"` // pre-JS HTML captured during headless render; nil in raw mode
	Doc          *goquery.Document `json:"-"` // nil if not HTML or parse failed
	Redirects    []Redirect        `json:"redirects,omitempty"`
	Links        []Link            `json:"-"`
	Depth        int               `json:"depth"`
	Referrer     string            `json:"referrer,omitempty"`
	Duration     time.Duration     `json:"duration_ms"`
	FetchedAt    time.Time         `json:"fetched_at"`
	Err          string            `json:"error,omitempty"`
	Render       *RenderResult     `json:"render,omitempty"`
}

type Redirect struct {
	From   string `json:"from"`
	To     string `json:"to"`
	Status int    `json:"status"`
}

type RenderResult struct {
	Implemented bool   `json:"implemented"`
	Note        string `json:"note,omitempty"`
	// RawFallback is set when the rendered DOM came back substantially thinner than the raw
	// HTML — a sign the page had not finished rendering when it was snapshotted. The analyzed
	// Body/Doc are then taken from the raw fetch (so structural checks like the H1 and meta
	// tags aren't false-negatives), while the Core Web Vitals below stay from the render but
	// should be treated as unreliable for this page. RenderedBytes/RawBytes record the sizes
	// that triggered it.
	RawFallback   bool    `json:"raw_fallback,omitempty"`
	RenderedBytes int     `json:"rendered_bytes,omitempty"`
	RawBytes      int     `json:"raw_bytes,omitempty"`
	LCP           float64 `json:"lcp_ms,omitempty"`
	FCP           float64 `json:"fcp_ms,omitempty"`
	CLS           float64 `json:"cls,omitempty"`
	TBT           float64 `json:"tbt_ms,omitempty"`
	TTFB          float64 `json:"ttfb_ms,omitempty"`

	// DataLayerPresent reports whether window.dataLayer was an array after the page rendered.
	DataLayerPresent bool `json:"data_layer_present,omitempty"`
	// DataLayer is the post-render snapshot of window.dataLayer, each element the raw JSON of
	// one entry (a GTM event push or a gtag() arguments object). It is deliberately not
	// serialized into reports: it can carry PII, so the datalayer analyzer emits sanitized
	// findings from it instead. Nil in raw mode or when the page has no dataLayer.
	DataLayer []json.RawMessage `json:"-"`
	// Requests holds outbound request URLs observed during render, bounded to avoid unbounded
	// growth. The datalayer analyzer uses it to confirm analytics/marketing tags actually
	// fired a network beacon. Not serialized.
	Requests []string `json:"-"`
}

type Result struct {
	Seed      string                 `json:"seed"`
	Pages     []*Page                `json:"pages"`
	Robots    map[string]*RobotsData `json:"robots,omitempty"`
	StartedAt time.Time              `json:"started_at"`
	Finished  time.Time              `json:"finished_at"`
	Opts      Options                `json:"-"`

	// ThrottleEvents counts how many times adaptive delay reduced the crawl rate after HTTP
	// 429/503 responses. FinalRate is the requests-per-second in effect when the crawl ended;
	// it is meaningful only when ThrottleEvents > 0.
	ThrottleEvents int     `json:"throttle_events,omitempty"`
	FinalRate      float64 `json:"final_rate,omitempty"`

	// Coverage reports whether the crawl visited every in-scope URL it discovered, or stopped
	// at a configured limit. When not Complete, findings that depend on fetching a page —
	// broken links above all — may be incomplete.
	Coverage Coverage `json:"coverage"`
	// contains filtered or unexported fields
}

type RobotsData struct {
	Host     string   `json:"host"`
	Found    bool     `json:"found"`
	Status   int      `json:"status"`
	Sitemaps []string `json:"sitemaps,omitempty"`
	// contains filtered or unexported fields
}

type RotationStrategy int

type UAPool struct {
	// contains filtered or unexported fields
}