crawler

package

v0.1.6 Latest Latest Go to latest Published: Aug 27, 2023 License: MIT Imports: 19 Imported by: 0

Details

This section is empty.

This section is empty.

func NewRobotManager() *robortManager

type CrawlState int32

const (
	CrawlQuit CrawlState = iota
	CrawlFinished
	CrawlInProgress
	CrawlPaused
)

type Crawler struct {
	// contains filtered or unexported fields
}

func New(opts ...Option) *Crawler

func (c *Crawler) OnError(fn func(err error))

func (c *Crawler) OnReponse(fn func(*wbot.Response))

func (c *Crawler) Start(links ...string)

func (c *Crawler) Stats() map[string]any

func (c *Crawler) Stop()

type Option func(c *Crawler)

func WithFetcher(fetcher wbot.Fetcher) Option

func WithFilter(rules ...*wbot.FilterRule) Option

func WithLogger(logger wbot.Logger) Option

func WithMaxDepth(maxDepth int32) Option

func WithParallel(parallel int) Option

func WithProxies(proxies []string) Option

func WithRateLimit(rates ...*wbot.RateLimit) Option

func WithStore(store wbot.Store) Option

func WithUserAgents(userAgents []string) Option