crawler

type Config struct {
	MaxDepth        int
	Concurrency     int
	Timeout         time.Duration
	PageTimeout     time.Duration
	RateLimit       int
	RetryAttempts   int
	RetryDelay      time.Duration
	UserAgent       string
	FollowRedirects int
	RespectRobots   bool
	Exclude         []string
	AuthHeader      string
	AuthValue       string
	CookieJar       http.CookieJar
	AllowPrivateIPs bool // When true, skip SSRF protection for private IPs
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Form struct {
	Action  string
	Method  string
	ID      string
	Inputs  []FormInput
	HasCSRF bool
}

type FormInput struct {
	Name     string
	Type     string
	Required bool
	Value    string
}

type Link struct {
	Href     string
	Text     string
	Rel      string
	External bool
	Anchor   bool
	Resource bool   // true for non-anchor resource URLs (img, script, iframe, etc.)
	Tag      string // source element tag (e.g., "img", "script", "iframe")
}

type Page struct {
	URL          string
	StatusCode   int
	Headers      http.Header
	Body         []byte
	Links        []Link
	Forms        []Form
	Depth        int
	ParentURL    string
	Duration     time.Duration
	Error        error
	AuthRequired bool // true when server returned 401/403
}

type ParseResult struct {
	Links    []Link
	Forms    []Form
	ParseErr error // non-nil if HTML parsing encountered an error (partial results still returned)
}

type RobotsCache struct {
	// contains filtered or unexported fields
}

type SitemapURL struct {
	Loc        string `xml:"loc"`
	Lastmod    string `xml:"lastmod,omitempty"`
	Changefreq string `xml:"changefreq,omitempty"`
	Priority   string `xml:"priority,omitempty"`
}