crawler

type ConcurrentCrawler struct {
	*Crawler // Embed the original crawler
	// contains filtered or unexported fields
}

type Config struct {
	MaxDepth       int              // Maximum depth to crawl (-1 = no limit, 0 = root only)
	SameDomain     bool             // Whether to limit crawling to same domain
	UserAgent      string           // User agent to use for requests
	Timeout        time.Duration    // Request timeout
	Logger         *slog.Logger     // Logger instance
	Workers        int              // Number of concurrent workers
	ShowProgress   bool             // Whether to show progress indicators
	ProgressConfig *progress.Config // Progress reporting configuration
}

type CrawlJob struct {
	URL   string // URL to crawl
	Depth int    // Depth of this URL in the crawl tree
}

type CrawlResult struct {
	URL          string        // The URL that was crawled
	Depth        int           // The depth at which this URL was found
	Links        []string      // Links found on this page
	Error        error         // Error if crawling failed
	FetchTime    time.Time     // When this URL was crawled
	ResponseTime time.Duration // Time taken to fetch this URL
	StatusCode   int           // HTTP status code
}

type CrawlStats struct {
	TotalURLs       int           // Total URLs discovered
	CrawledURLs     int           // URLs successfully crawled
	FailedURLs      int           // URLs that failed to crawl
	SkippedURLs     int           // URLs skipped (duplicates, depth limit)
	MaxDepthReached int           // Maximum depth reached
	TotalTime       time.Duration // Total crawling time
	StartTime       time.Time     // When crawling started
}

type Crawler struct {
	// contains filtered or unexported fields
}