Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
Types ¶
type Fetcher ¶
type Fetcher struct {
// contains filtered or unexported fields
}
Fetcher handles making HTTP requests with configured retry logic, using an underlying http.Client
func NewFetcher ¶
NewFetcher creates a new Fetcher instance
func (*Fetcher) FetchWithRetry ¶
FetchWithRetry performs an HTTP request associated with the provided context It implements a retry mechanism with exponential backoff and jitter for transient network errors and specific HTTP status codes (5xx, 429)
type HostSemaphorePool ¶
type HostSemaphorePool struct {
// contains filtered or unexported fields
}
HostSemaphorePool manages per-host semaphores for rate limiting concurrent requests to each host. A single pool should be shared across all components (crawler, image processor) so the per-host limit is enforced globally.
func NewHostSemaphorePool ¶
func NewHostSemaphorePool(maxPerHost int, log *logrus.Entry) *HostSemaphorePool
NewHostSemaphorePool creates a new pool with the given per-host concurrency limit.
type RateLimiter ¶
type RateLimiter struct {
// contains filtered or unexported fields
}
RateLimiter manages request timing per host for politeness
func NewRateLimiter ¶
func NewRateLimiter(defaultDelay time.Duration, log *logrus.Logger) *RateLimiter
NewRateLimiter creates a RateLimiter
func (*RateLimiter) ApplyDelay ¶
func (rl *RateLimiter) ApplyDelay(host string, minDelay time.Duration)
ApplyDelay sleeps if the time since the last request to the host is less than minDelay Includes jitter (+/- 10%) to desynchronize requests
func (*RateLimiter) UpdateLastRequestTime ¶
func (rl *RateLimiter) UpdateLastRequestTime(host string)
UpdateLastRequestTime records the current time as the last request attempt time for the host Call this *after* an HTTP request attempt to the host
type RobotsHandler ¶
type RobotsHandler struct {
// contains filtered or unexported fields
}
RobotsHandler manages fetching, parsing, caching, and checking robots.txt data
func NewRobotsHandler ¶
func NewRobotsHandler( fetcher *Fetcher, rateLimiter *RateLimiter, globalSemaphore *semaphore.Weighted, sitemapNotifier SitemapDiscoverer, cfg *config.AppConfig, log *logrus.Entry, ) *RobotsHandler
NewRobotsHandler creates a RobotsHandler
func (*RobotsHandler) GetRobotsData ¶
func (rh *RobotsHandler) GetRobotsData(targetURL *url.URL, signalChan chan<- bool, ctx context.Context) *robotstxt.RobotsData
GetRobotsData retrieves robots.txt data for the targetURL's host, using cache or fetching Returns parsed data or nil on any error/4xx/missing file signalChan is only for coordinating the initial crawler startup fetch
type SitemapDiscoverer ¶
type SitemapDiscoverer interface {
FoundSitemap(sitemapURL string)
}
SitemapDiscoverer defines the callback interface for handling discovered sitemap URLs
Source Files