crawler

type Config struct {
	MaxDepth        int
	Concurrency     int
	Timeout         time.Duration
	PageTimeout     time.Duration
	RateLimit       int
	RetryAttempts   int
	RetryDelay      time.Duration
	UserAgent       string
	FollowRedirects int
	RespectRobots   bool
	Exclude         []string
	AuthHeader      string
	AuthValue       string
	CookieJar       http.CookieJar
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Form struct {
	Action  string
	Method  string
	ID      string
	Inputs  []FormInput
	HasCSRF bool
}

type FormInput struct {
	Name     string
	Type     string
	Required bool
	Value    string
}

type Link struct {
	Href     string
	Text     string
	Rel      string
	External bool
	Anchor   bool
	Resource bool   // true for non-anchor resource URLs (img, script, iframe, etc.)
	Tag      string // source element tag (e.g., "img", "script", "iframe")
}

type Page struct {
	URL        string
	StatusCode int
	Headers    http.Header
	Body       []byte
	Links      []Link
	Forms      []Form
	Depth      int
	ParentURL  string
	Duration   time.Duration
	Error      error
}

type RobotsCache struct {
	// contains filtered or unexported fields
}

type SitemapURL struct {
	Loc        string `xml:"loc"`
	Lastmod    string `xml:"lastmod,omitempty"`
	Changefreq string `xml:"changefreq,omitempty"`
	Priority   string `xml:"priority,omitempty"`
}