Versions in this module Expand all Collapse all v1 v1.0.0 Feb 1, 2026 Changes in this version + type AdaptiveRateLimiter struct + func NewAdaptiveRateLimiter(minRate, maxRate float64, burst int) *AdaptiveRateLimiter + func (a *AdaptiveRateLimiter) CurrentRate() float64 + func (a *AdaptiveRateLimiter) RecordError() + func (a *AdaptiveRateLimiter) RecordSuccess() + type Limiter struct + func NewLimiter(requestsPerSecond float64, burst int) *Limiter + func (l *Limiter) Allow() bool + func (l *Limiter) AllowDomain(domain string) bool + func (l *Limiter) GetCrawlDelay(domain, userAgent string) time.Duration + func (l *Limiter) GetRobots() *RobotsManager + func (l *Limiter) IsAllowed(ctx context.Context, domain, path, userAgent string, respectRobots bool) bool + func (l *Limiter) Reserve() *rate.Reservation + func (l *Limiter) SetDomainDelay(delay time.Duration) + func (l *Limiter) SetDomainRate(domain string, requestsPerSecond float64, burst int) + func (l *Limiter) SetRate(requestsPerSecond float64, burst int) + func (l *Limiter) Stats() LimiterStats + func (l *Limiter) Wait(ctx context.Context) error + func (l *Limiter) WaitDomain(ctx context.Context, domain string) error + type LimiterStats struct + DefaultBurst int + DefaultRate float64 + DomainCount int + DomainDelay time.Duration + type RobotsManager struct + func NewRobotsManager() *RobotsManager + func (m *RobotsManager) Fetch(ctx context.Context, domain string) error + func (m *RobotsManager) GetCrawlDelay(domain, userAgent string) time.Duration + func (m *RobotsManager) GetSitemaps(domain string) []string + func (m *RobotsManager) IsAllowed(domain, path, userAgent string) bool + type RobotsRules struct + Allow []*regexp.Regexp + CrawlDelay time.Duration + Disallow []*regexp.Regexp + FetchedAt time.Time + Sitemaps []string + func ParseRobots(r io.Reader, userAgent string) (*RobotsRules, error) + func (r *RobotsRules) IsAllowed(path string) bool
Go to main page