fetcher

type DialOptions struct {
	SourceIP        string
	ForceIPv4       bool
	AllowPrivateIPs bool
}

type FetchResult struct {
	URL           string
	FinalURL      string
	StatusCode    int
	ContentType   string
	Headers       map[string]string
	Body          []byte
	BodySize      int64
	BodyTruncated bool
	RedirectChain []RedirectHop
	Duration      time.Duration
	Error         string
	Depth         int
	FoundOn       string
	Attempt       int // retry attempt number (0 = first try)
}

type Fetcher struct {
	// contains filtered or unexported fields
}

type RedirectHop struct {
	URL        string
	StatusCode int
}

type RobotsCache struct {
	// contains filtered or unexported fields
}

type RobotsCacheEntry struct {
	Content    string
	StatusCode int
	FetchedAt  time.Time
	// contains filtered or unexported fields
}

type SitemapEntry struct {
	URL        string
	Type       string // "index" or "urlset"
	StatusCode int
	URLs       []SitemapURL
	Sitemaps   []string // child sitemap URLs if index
}

type SitemapURL struct {
	Loc        string `xml:"loc"`
	LastMod    string `xml:"lastmod"`
	ChangeFreq string `xml:"changefreq"`
	Priority   string `xml:"priority"`
}

type TLSProfile string