crawler

Mar 9, 2026

Changes in this version

type CrawlRequest

+ ExcludePatterns []string

type Engine

+ func (e *Engine) Done() <-chan struct{}

Mar 7, 2026

Changes in this version

type CrawlRequest

+ IgnoreRobots bool

Mar 7, 2026

Changes in this version

type Engine

+ func (e *Engine) MarkSeen(url string)

type HostHealth

+ func NewHostHealthWithWindow(windowSize int) *HostHealth

+ func (hh *HostHealth) RateLimitRate() float64

+ func (hh *HostHealth) RecordRateLimit(host string)

Mar 4, 2026

Changes in this version

type CrawlRequest

+ FetchSitemaps *bool

type Engine

+ func (e *Engine) Phase() string

type Manager

+ func (m *Manager) Phase(sessionID string) string

Mar 3, 2026

Changes in this version

type CrawlRequest

+ ExtractorSetID string

+ type ExtractorSetLoader interface

+ GetExtractorSet func(id string) (*extraction.ExtractorSet, error)

Mar 2, 2026

Changes in this version

+ type CrawlRequest struct

+ CheckExternalLinks *bool

+ CheckPageResources *bool

+ CrawlScope string

+ CrawlSitemapOnly bool

+ Delay string

+ ExternalLinkWorkers int

+ FollowJSLinks bool

+ ForceIPv4 bool

+ JSRenderMaxPages int

+ JSRenderMode string

+ JSRenderTimeout string

+ MaxDepth int

+ MaxPages int

+ ProjectID *string

+ ResourceWorkers int

+ RetryStatusCode int

+ Seeds []string

+ SourceIP string

+ StoreHTML bool

+ TLSProfile string

+ UserAgent string

+ Workers int

+ type Engine struct

+ func NewEngine(cfg *config.Config, store *storage.Store) *Engine

+ func (e *Engine) BufferState() storage.BufferErrorState

+ func (e *Engine) PagesCrawled() int64

+ func (e *Engine) PreSeedDedup(urls []string)

+ func (e *Engine) QueueLen() int

+ func (e *Engine) ResumeSession(id string, originalSeeds []string)

+ func (e *Engine) Run(seeds []string) error

+ func (e *Engine) SessionID(seeds []string) string

+ func (e *Engine) SetSessionID(id string)

+ func (e *Engine) Stop()

+ type HostHealth struct

+ func NewHostHealth() *HostHealth

+ func (hh *HostHealth) GlobalErrorRate() float64

+ func (hh *HostHealth) RecordFailure(host string)

+ func (hh *HostHealth) RecordSuccess(host string)

+ func (hh *HostHealth) ShouldRetry(host string, maxConsecutiveFails int) bool

+ type Manager struct

+ func NewManager(cfg *config.Config, store *storage.Store) *Manager

+ func (m *Manager) ActiveSessions() []string

+ func (m *Manager) BufferState(sessionID string) storage.BufferErrorState

+ func (m *Manager) IsQueued(sessionID string) bool

+ func (m *Manager) IsRunning(sessionID string) bool

+ func (m *Manager) LastError(sessionID string) string

+ func (m *Manager) Progress(sessionID string) (int64, int, bool)

+ func (m *Manager) QueuedSessions() []string

+ func (m *Manager) RecoverOrphanedSessions(ctx context.Context)

+ func (m *Manager) ResumeCrawl(sessionID string, overrides *CrawlRequest) (string, error)

+ func (m *Manager) RetryFailed(sessionID string, overrides *CrawlRequest) (int, error)

+ func (m *Manager) Shutdown(timeout time.Duration)

+ func (m *Manager) StartCrawl(req CrawlRequest) (string, error)

+ func (m *Manager) StopCrawl(sessionID string) error

+ type RetryItem struct

+ Attempt int

+ Depth int

+ FoundOn string

+ Host string

+ LastCode int

+ LastErr string

+ ReadyAt time.Time

+ URL string

+ type RetryPolicy struct

+ BaseDelay time.Duration

+ MaxDelay time.Duration

+ MaxRetries int

+ func (p *RetryPolicy) ComputeDelay(attempt int, retryAfterHeader string) time.Duration

+ func (p *RetryPolicy) ShouldRetry(statusCode int, errString string, attempt int) bool

+ type RetryQueue struct

+ func NewRetryQueue() *RetryQueue

+ func (rq *RetryQueue) Len() int

+ func (rq *RetryQueue) PopReady() *RetryItem

+ func (rq *RetryQueue) Push(item *RetryItem)

+ type Session struct

+ Config *config.Config

+ ID string

+ Pages uint64

+ ProjectID *string

+ SeedURLs []string

+ StartedAt time.Time

+ Status string

+ func NewSession(seeds []string, cfg *config.Config) *Session

+ func (s *Session) ToStorageRow() *storage.CrawlSession