crawler

type Engine struct {
	// contains filtered or unexported fields
}

type Fetcher interface {
	Fetch(ctx context.Context, rawURL string) (*Page, error)
}

type HTTPFetcher struct {
	// contains filtered or unexported fields
}

type Link struct {
	URL      string `json:"url"`    // absolute, normalized
	Anchor   string `json:"anchor"` // visible text
	Rel      string `json:"rel"`
	Nofollow bool   `json:"nofollow"`
	External bool   `json:"external"` // points outside the seed host
}

type Options struct {
	MaxDepth      int
	MaxPages      int
	Concurrency   int
	RatePerSecond float64
	UserAgent     string
	// UserAgents is an optional pool of User-Agent strings to rotate across. When non-empty it
	// supersedes UserAgent; UserAgentRotation picks one per request.
	UserAgents        []string
	UserAgentRotation RotationStrategy
	// Proxies is an optional pool of proxy URLs (http, https, or socks5) to route requests
	// through. When non-empty, ProxyRotation picks one per request. Empty leaves Go's default
	// proxy behavior (honoring HTTP_PROXY/HTTPS_PROXY/NO_PROXY) in place.
	Proxies         []*url.URL
	ProxyRotation   RotationStrategy
	Include         []*regexp.Regexp
	Exclude         []*regexp.Regexp
	RespectRobots   bool
	AllowSubdomains bool
	FollowExternal  bool
	FollowNofollow  bool
	// StripQuery drops the query string when normalizing URLs, so URLs that differ only by
	// their query collapse to one and are crawled once.
	StripQuery   bool
	Timeout      time.Duration
	MaxBodyBytes int64
	MaxRedirects int
	// Verbose logs each fetch and every rate-limit change to stderr while crawling.
	Verbose bool
	// AdaptiveDelay automatically slows the crawl (halving requests-per-second, honoring any
	// Retry-After header) when the server responds with HTTP 429 or 503.
	AdaptiveDelay bool
}

type Page struct {
	RequestedURL string            `json:"requested_url"`
	FinalURL     string            `json:"final_url"`
	StatusCode   int               `json:"status_code"`
	Header       http.Header       `json:"-"`
	ContentType  string            `json:"content_type"`
	Body         []byte            `json:"-"`
	RawBody      []byte            `json:"-"` // pre-JS HTML captured during headless render; nil in raw mode
	Doc          *goquery.Document `json:"-"` // nil if not HTML or parse failed
	Redirects    []Redirect        `json:"redirects,omitempty"`
	Links        []Link            `json:"-"`
	Depth        int               `json:"depth"`
	Referrer     string            `json:"referrer,omitempty"`
	Duration     time.Duration     `json:"duration_ms"`
	FetchedAt    time.Time         `json:"fetched_at"`
	Err          string            `json:"error,omitempty"`
	Render       *RenderResult     `json:"render,omitempty"`
}

type Redirect struct {
	From   string `json:"from"`
	To     string `json:"to"`
	Status int    `json:"status"`
}

type RenderResult struct {
	Implemented bool    `json:"implemented"`
	Note        string  `json:"note,omitempty"`
	LCP         float64 `json:"lcp_ms,omitempty"`
	FCP         float64 `json:"fcp_ms,omitempty"`
	CLS         float64 `json:"cls,omitempty"`
	TBT         float64 `json:"tbt_ms,omitempty"`
	TTFB        float64 `json:"ttfb_ms,omitempty"`

	// DataLayerPresent reports whether window.dataLayer was an array after the page rendered.
	DataLayerPresent bool `json:"data_layer_present,omitempty"`
	// DataLayer is the post-render snapshot of window.dataLayer, each element the raw JSON of
	// one entry (a GTM event push or a gtag() arguments object). It is deliberately not
	// serialized into reports: it can carry PII, so the datalayer analyzer emits sanitized
	// findings from it instead. Nil in raw mode or when the page has no dataLayer.
	DataLayer []json.RawMessage `json:"-"`
	// Requests holds outbound request URLs observed during render, bounded to avoid unbounded
	// growth. The datalayer analyzer uses it to confirm analytics/marketing tags actually
	// fired a network beacon. Not serialized.
	Requests []string `json:"-"`
}

type Result struct {
	Seed      string                 `json:"seed"`
	Pages     []*Page                `json:"pages"`
	Robots    map[string]*RobotsData `json:"robots,omitempty"`
	StartedAt time.Time              `json:"started_at"`
	Finished  time.Time              `json:"finished_at"`
	Opts      Options                `json:"-"`

	// ThrottleEvents counts how many times adaptive delay reduced the crawl rate after HTTP
	// 429/503 responses. FinalRate is the requests-per-second in effect when the crawl ended;
	// it is meaningful only when ThrottleEvents > 0.
	ThrottleEvents int     `json:"throttle_events,omitempty"`
	FinalRate      float64 `json:"final_rate,omitempty"`
	// contains filtered or unexported fields
}

type RobotsData struct {
	Host     string   `json:"host"`
	Found    bool     `json:"found"`
	Status   int      `json:"status"`
	Sitemaps []string `json:"sitemaps,omitempty"`
	// contains filtered or unexported fields
}

type RotationStrategy int

type UAPool struct {
	// contains filtered or unexported fields
}