crawler

type CircuitBreakerRegistry struct {
	// contains filtered or unexported fields
}

type CircuitState int

type Config struct {
	MaxDepth        int
	Concurrency     int
	Timeout         time.Duration
	PageTimeout     time.Duration
	RateLimit       int
	RetryAttempts   int
	RetryDelay      time.Duration
	UserAgent       string
	FollowRedirects int
	RespectRobots   bool
	Exclude         []string
	AuthHeader      string
	AuthValue       string
	CookieJar       http.CookieJar
	AllowPrivateIPs bool // When true, skip SSRF protection for private IPs
	Logger          *slog.Logger
	MaxPages        int // Maximum number of pages to crawl; 0 means no limit

	// CircuitBreaker, when non-nil, gates requests per host. If a host has
	// accumulated too many consecutive failures the breaker opens and the
	// crawler skips further requests to that host until the cooldown expires.
	CircuitBreaker *CircuitBreakerRegistry

	// Fetcher, when non-nil, overrides the default HTTP page retrieval. It is
	// used to plug in a headless-browser fetcher that renders JavaScript. SSRF
	// validation, rate limiting, retries, and the circuit breaker still run in
	// front of the fetcher.
	Fetcher Fetcher
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Fetcher interface {
	Fetch(ctx context.Context, page *Page, targetURL string) error
}

type Form struct {
	Action  string
	Method  string
	ID      string
	Inputs  []FormInput
	HasCSRF bool
}

type FormInput struct {
	Name     string
	Type     string
	Required bool
	Value    string
}

type HostCircuitBreaker struct {
	// contains filtered or unexported fields
}

type Link struct {
	Href     string
	Text     string
	Rel      string
	External bool
	Anchor   bool
	Resource bool   // true for non-anchor resource URLs (img, script, iframe, etc.)
	Tag      string // source element tag (e.g., "img", "script", "iframe")
}

type Page struct {
	URL          string
	StatusCode   int
	Headers      http.Header
	Body         []byte
	Links        []Link
	Forms        []Form
	Depth        int
	ParentURL    string
	Duration     time.Duration
	Error        error
	AuthRequired bool       // true when server returned 401/403
	Doc          *html.Node // parsed HTML tree, populated once during fetch
}

type ParseResult struct {
	Links    []Link
	Forms    []Form
	ParseErr error // non-nil if HTML parsing encountered an error (partial results still returned)
}

type RobotsCache struct {
	// contains filtered or unexported fields
}

type SitemapURL struct {
	Loc        string `xml:"loc"`
	Lastmod    string `xml:"lastmod,omitempty"`
	Changefreq string `xml:"changefreq,omitempty"`
	Priority   string `xml:"priority,omitempty"`
}