crawler

package

v0.2.0 Latest Latest Go to latest Published: Feb 18, 2024 License: MIT Imports: 21 Imported by: 0

Details

This section is empty.

This section is empty.

This section is empty.

type Crawler struct {
	// contains filtered or unexported fields
}

func New(opts ...Option) *Crawler

func (c *Crawler) Metrics() map[string]int64

func (c *Crawler) OnReponse(fn func(*wbot.Response))

func (c *Crawler) Run(links ...string) error

func (c *Crawler) Shutdown()

type Option func(c *Crawler)

func WithFetcher(fetcher wbot.Fetcher) Option

func WithFilter(rules ...*wbot.FilterRule) Option

func WithLogLevel(level zerolog.Level) Option

func WithMaxDepth(maxDepth int32) Option

func WithParallel(parallel int) Option

func WithProxies(proxies []string) Option

func WithQueue(queue wbot.Queue) Option

func WithRateLimit(rates ...*wbot.RateLimit) Option

func WithStore(store wbot.Store) Option

func WithUserAgents(userAgents []string) Option