crawler

type AdaptiveFetcher struct {
	Fetcher Fetcher
	Limiter *AdaptiveLimiter
}

type AdaptiveLimiter struct {
	// contains filtered or unexported fields
}

type CertInfo struct {
	Subject            string
	Issuer             string
	NotBefore          time.Time
	NotAfter           time.Time
	SignatureAlgorithm x509.SignatureAlgorithm
	// KeyType and KeyBits describe the public key ("RSA"/2048, "ECDSA"/256, "Ed25519"/256).
	// KeyBits is 0 for a key type we don't measure.
	KeyType string
	KeyBits int
	IsCA    bool
	// SelfSigned is true when issuer and subject are the same distinguished name — either a
	// root CA at the top of the chain, or an untrusted self-signed server certificate.
	SelfSigned bool
}

type Cookie struct {
	Name   string
	Domain string
	Path   string
	// Session is true for a cookie with no expiry, which the browser drops when it closes.
	Session bool
	// Expires is the cookie's expiry, zero for a session cookie.
	Expires  time.Time
	Secure   bool
	HTTPOnly bool
	SameSite string
}

type Coverage struct {
	// Complete is true when no in-scope, robots-allowed URL was left un-fetched because of a
	// depth or page-count limit.
	Complete bool `json:"complete"`
	// DiscoveredNotCrawled is the number of distinct in-scope URLs that were discovered but
	// never fetched because a limit was reached.
	DiscoveredNotCrawled int `json:"discovered_not_crawled,omitempty"`
	// PageLimitReached / DepthLimitReached record which configured limit cut the crawl short.
	PageLimitReached  bool `json:"page_limit_reached,omitempty"`
	DepthLimitReached bool `json:"depth_limit_reached,omitempty"`
	// Interrupted is true when the crawl's context was canceled before it finished on its own
	// (e.g. an operator's Ctrl-C) rather than a configured limit being reached. The site may
	// be far less covered than DiscoveredNotCrawled reflects, since much of it may never have
	// been discovered at all.
	Interrupted bool `json:"interrupted,omitempty"`
	// DurationLimitReached is true when the crawl stopped because it hit its --max-duration
	// wall-clock budget rather than being interrupted externally. Also implies Interrupted,
	// since the same context-cancellation path stops the crawl either way.
	DurationLimitReached bool `json:"duration_limit_reached,omitempty"`
	// MaxPages / MaxDepth echo the limits in effect (0 = unlimited), for the report message.
	MaxPages int `json:"max_pages,omitempty"`
	MaxDepth int `json:"max_depth,omitempty"`
}

type Engine struct {
	// contains filtered or unexported fields
}

type Fetcher interface {
	Fetch(ctx context.Context, rawURL string) (*Page, error)
}

type HTTPFetcher struct {
	// contains filtered or unexported fields
}

type Link struct {
	URL      string `json:"url"`                // absolute, normalized for dedup (trailing slash stripped)
	Resolved string `json:"resolved,omitempty"` // absolute, normalized but trailing slash preserved
	Anchor   string `json:"anchor"`             // visible text
	Rel      string `json:"rel"`
	Nofollow bool   `json:"nofollow"`
	External bool   `json:"external"` // points outside the seed host
}

type Options struct {
	MaxDepth      int
	MaxPages      int
	Concurrency   int
	RatePerSecond float64
	UserAgent     string
	// UserAgents is an optional pool of User-Agent strings to rotate across. When non-empty it
	// supersedes UserAgent; UserAgentRotation picks one per request.
	UserAgents        []string
	UserAgentRotation RotationStrategy
	// Proxies is an optional pool of proxy URLs (http, https, or socks5) to route requests
	// through. When non-empty, ProxyRotation picks one per request. Empty leaves Go's default
	// proxy behavior (honoring HTTP_PROXY/HTTPS_PROXY/NO_PROXY) in place.
	Proxies       []*url.URL
	ProxyRotation RotationStrategy
	// BasicAuthUser and BasicAuthPass, when BasicAuthUser is non-empty, are sent as an HTTP
	// Basic Authorization header on every request. This targets server-level Basic Auth (the
	// realm challenge common on staging/acceptance environments), sent preemptively rather
	// than in response to a 401 challenge.
	BasicAuthUser   string
	BasicAuthPass   string
	Include         []*regexp.Regexp
	Exclude         []*regexp.Regexp
	RespectRobots   bool
	AllowSubdomains bool
	FollowExternal  bool
	FollowNofollow  bool
	// StripQuery drops the query string when normalizing URLs, so URLs that differ only by
	// their query collapse to one and are crawled once.
	StripQuery   bool
	Timeout      time.Duration
	MaxBodyBytes int64
	MaxRedirects int
	// Verbose logs each fetch and every rate-limit change to stderr while crawling.
	Verbose bool
	// AdaptiveDelay automatically slows the crawl (halving requests-per-second, honoring any
	// Retry-After header) when the server responds with HTTP 429 or 503.
	AdaptiveDelay bool
	// MaxDuration bounds the crawl's total wall-clock time. Zero means unlimited. When it
	// elapses, the crawl stops early and still returns everything fetched so far as a
	// partial result (see Coverage.DurationLimitReached), the same way a canceled context
	// (e.g. an operator's Ctrl-C) does.
	MaxDuration time.Duration
}

type Page struct {
	RequestedURL string      `json:"requested_url"`
	FinalURL     string      `json:"final_url"`
	StatusCode   int         `json:"status_code"`
	Header       http.Header `json:"-"`
	// TLS records the handshake behind the final response: protocol version, cipher suite, and
	// the certificate chain the server presented. Nil for plain HTTP, and in headless render
	// mode, where the response never passes through Go's TLS stack.
	TLS         *TLSInfo          `json:"-"`
	ContentType string            `json:"content_type"`
	Body        []byte            `json:"-"`
	RawBody     []byte            `json:"-"` // pre-JS HTML captured during headless render; nil in raw mode
	Doc         *goquery.Document `json:"-"` // nil if not HTML or parse failed
	Redirects   []Redirect        `json:"redirects,omitempty"`
	Links       []Link            `json:"-"`
	Depth       int               `json:"depth"`
	Referrer    string            `json:"referrer,omitempty"`
	Duration    time.Duration     `json:"duration_ms"`
	FetchedAt   time.Time         `json:"fetched_at"`
	Err         string            `json:"error,omitempty"`
	Render      *RenderResult     `json:"render,omitempty"`
	// Truncated reports whether Body was cut short of the real response — either because it
	// hit the fetcher's body-size cap or because the connection failed partway through the
	// read. A truncated body may be missing elements (e.g. </head>, the closing tag of a
	// sitemap) that a downstream analyzer would otherwise expect to find, so treat findings
	// like "missing title" or "invalid sitemap" on a truncated page with that in mind.
	Truncated bool `json:"truncated,omitempty"`
}

type Redirect struct {
	From   string `json:"from"`
	To     string `json:"to"`
	Status int    `json:"status"`
}

type RenderResult struct {
	Implemented bool   `json:"implemented"`
	Note        string `json:"note,omitempty"`
	// RawFallback is set when the rendered DOM came back substantially thinner than the raw
	// HTML — a sign the page had not finished rendering when it was snapshotted. The analyzed
	// Body/Doc are then taken from the raw fetch (so structural checks like the H1 and meta
	// tags aren't false-negatives), while the Core Web Vitals below stay from the render but
	// should be treated as unreliable for this page. RenderedBytes/RawBytes record the sizes
	// that triggered it.
	RawFallback   bool    `json:"raw_fallback,omitempty"`
	RenderedBytes int     `json:"rendered_bytes,omitempty"`
	RawBytes      int     `json:"raw_bytes,omitempty"`
	LCP           float64 `json:"lcp_ms,omitempty"`
	FCP           float64 `json:"fcp_ms,omitempty"`
	CLS           float64 `json:"cls,omitempty"`
	TBT           float64 `json:"tbt_ms,omitempty"`
	TTFB          float64 `json:"ttfb_ms,omitempty"`

	// DataLayerPresent reports whether window.dataLayer was an array after the page rendered.
	DataLayerPresent bool `json:"data_layer_present,omitempty"`
	// DataLayer is the post-render snapshot of window.dataLayer, each element the raw JSON of
	// one entry (a GTM event push or a gtag() arguments object). It is deliberately not
	// serialized into reports: it can carry PII, so the datalayer analyzer emits sanitized
	// findings from it instead. Nil in raw mode or when the page has no dataLayer.
	DataLayer []json.RawMessage `json:"-"`
	// Requests holds outbound request URLs observed during render, bounded to avoid unbounded
	// growth. The datalayer analyzer uses it to confirm analytics/marketing tags actually
	// fired a network beacon. Not serialized.
	Requests []string `json:"-"`
	// Cookies is the browser's cookie jar after the page settled — every cookie actually
	// stored, first- and third-party, however it was set (Set-Cookie header, document.cookie,
	// or a third-party script). Response headers alone cannot see the last two, which is why
	// this is captured separately.
	//
	// gocrawl never interacts with a consent banner, so this jar is the site's *pre-consent*
	// state: the consent analyzer reads it to find tracking cookies dropped before a visitor
	// has agreed to anything. The jar belongs to the browser, not the tab, so it accumulates
	// across the pages of one crawl — treat it as a site-level observation rather than proof
	// that a particular page set a particular cookie. Nil in raw mode.
	Cookies []Cookie `json:"-"`
}

type Result struct {
	Seed      string                 `json:"seed"`
	Pages     []*Page                `json:"pages"`
	Robots    map[string]*RobotsData `json:"robots,omitempty"`
	StartedAt time.Time              `json:"started_at"`
	Finished  time.Time              `json:"finished_at"`
	Opts      Options                `json:"-"`

	// ThrottleEvents counts how many times adaptive delay reduced the crawl rate after HTTP
	// 429/503 responses. FinalRate is the requests-per-second in effect when the crawl ended;
	// it is meaningful only when ThrottleEvents > 0.
	ThrottleEvents int     `json:"throttle_events,omitempty"`
	FinalRate      float64 `json:"final_rate,omitempty"`

	// Coverage reports whether the crawl visited every in-scope URL it discovered, or stopped
	// at a configured limit. When not Complete, findings that depend on fetching a page —
	// broken links above all — may be incomplete.
	Coverage Coverage `json:"coverage"`
	// contains filtered or unexported fields
}

type RobotsData struct {
	Host     string   `json:"host"`
	Found    bool     `json:"found"`
	Status   int      `json:"status"`
	Sitemaps []string `json:"sitemaps,omitempty"`
	// contains filtered or unexported fields
}

type RotationStrategy int

type TLSInfo struct {
	Version     uint16
	CipherSuite uint16
	// ALPN is the application protocol negotiated during the handshake ("h2", "http/1.1"), or
	// empty when the server offered none.
	ALPN string
	// Chain is the certificate chain the server presented, leaf first. It is what the server
	// actually sent, not the verified chain, so a server that omits its intermediates shows up
	// here as a one-element chain even though the fetch succeeded.
	Chain []CertInfo
}

type UAPool struct {
	// contains filtered or unexported fields
}