crawler

type Channels map[int]chan Page

type Config struct {
	StartURL           string
	AllowedDomains     []string // Domains to stay within
	UserAgents         []string
	CrawlDelay         time.Duration // Delay between requests to the same domain
	MaxDepth           int           // Maximum crawl depth
	MaxRetries         int           // Max retries for a failed request
	RequestTimeout     time.Duration
	QueueIdleTimeout   time.Duration
	ProxyURL           string // e.g., "http://user:pass@host:port"
	RobotsUserAgent    string // User agent to use for robots.txt checks
	ConcurrentRequests int    // Number of concurrent fetch workers
	Channels           Channels
	Headers            map[string]string
	LanguageCode       string
	Filters            []func(Page, *Config) bool
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Page struct {
	URL  *url.URL      // Page url
	Resp http.Response // Page response as returned from the GET request
	Body string        // Response body string
}