wbot

Feb 18, 2024

Changes in this version

+ type Crawler struct

+ func New(opts ...Option) *Crawler

+ func (c *Crawler) Metrics() map[string]int64

+ func (c *Crawler) OnReponse(fn func(*api.Response))

+ func (c *Crawler) Run(links ...string) error

+ func (c *Crawler) Shutdown()

type Option

+ func WithFetcher(fetcher api.Fetcher) Option

+ func WithFilter(rules ...*api.FilterRule) Option

+ func WithLogLevel(level zerolog.Level) Option

+ func WithMaxDepth(maxDepth int32) Option

+ func WithParallel(parallel int) Option

+ func WithProxies(proxies []string) Option

+ func WithQueue(queue api.Queue) Option

+ func WithRateLimit(rates ...*api.RateLimit) Option

+ func WithStore(store api.Store) Option

+ func WithUserAgents(userAgents []string) Option

Feb 18, 2024

Changes in this version

+ func Hostname(link string) (string, error)

type MetricsMonitor

+ IncCrawledLink func()

+ IncDuplicatedLink func()

+ IncSkippedLink func()

+ IncTotalLink func()

+ Metrics func() map[string]int64

type Param

+ Timeout time.Duration

type ParsedURL

+ func (u *ParsedURL) String() string

Aug 27, 2023

Changes in this version

+ func FindLinks(body []byte) (hrefs []string)

+ type FilterRule struct

+ Allow []*regexp.Regexp

+ Disallow []*regexp.Regexp

+ Hostname string

+ type Log struct

+ ContentSize int64

+ Depth int32

+ Err error

+ RedirectURL string

+ RequestURL string

+ ResponseTime time.Duration

+ Status int

+ Timestamp time.Time

+ UserAgent string

type Logger

+ Write func(ctx context.Context, log *Log) error

+ type MetricsMonitor interface

+ IncClientErrors func()

+ IncCrawledPages func()

+ IncFailedRequests func()

+ IncParsedLinks func()

+ IncRedirects func()

+ IncRetries func()

+ IncServerErrors func()

+ IncSkippedPages func()

+ IncSuccessfulRequests func()

+ IncTotalPages func()

+ IncTotalRequests func()

+ type ParsedURL struct

+ Hash string

+ Root string

+ URL *url.URL

+ func NewURL(raw string) (*ParsedURL, error)

type Queue

+ Cancel func()

+ IsDone func() bool

+ Len func() int32

+ Push func(ctx context.Context, req *Request) error

+ type RateLimit struct

+ Hostname string

+ Rate string

type Request

+ Target *ParsedURL

+ func (r *Request) ResolveURL(u string) (*url.URL, error)

type Response

+ ElapsedTime time.Duration

+ Err error

type Store

+ HasVisited func(ctx context.Context, u *ParsedURL) (bool, error)

Jul 3, 2022

Changes in this version

+ type Param struct

+ MaxBodySize int64

+ Proxy string

+ Referer string

+ UserAgent string

type Queue

+ Dequeue func() (Request, error)

+ Enqueue func(req Request) error

type Request

+ Param Param

Jun 20, 2022

Changes in this version

type Fetcher

+ Close func() error

type Logger

+ Close func() error

May 14, 2022

Changes in this version

+ type Fetcher interface

+ Fetch func(req *Request) (*Response, error)

+ type Logger interface

+ Send func(rep *Report)

+ type Option func(*WBot)

+ func SetFetcher(f Fetcher) Option

+ func SetFilter(allowed, disallowed []string) Option

+ func SetLogger(l Logger) Option

+ func SetMaxBodySize(size int64) Option

+ func SetMaxDepth(depth int32) Option

+ func SetParallel(parallel int) Option

+ func SetProxies(proxies []string) Option

+ func SetQueue(q Queue) Option

+ func SetRateLimit(rate int, interval time.Duration) Option

+ func SetStore(s Store) Option

+ func SetUserAgents(agents []string) Option

+ type Queue interface

+ Add func(req *Request)

+ Close func()

+ Next func() bool

+ Pop func() *Request

+ type Report struct

+ Depth int32

+ Err error

+ RequestURL string

+ Status int

+ type Request struct

+ BaseDomain string

+ Depth int32

+ URL *url.URL

+ func (r *Request) AbsURL(u string) (*url.URL, error)

+ type Response struct

+ Body []byte

+ Depth int32

+ NextURLs []string

+ Status int

+ URL *url.URL

+ type Store interface

+ Close func()

+ Visited func(link string) bool

+ type WBot struct

+ func NewWBot(opts ...Option) *WBot

+ func (wb *WBot) Close()

+ func (wb *WBot) Crawl(link string) error

+ func (wb *WBot) SetOptions(opts ...Option)

+ func (wb *WBot) Stream() <-chan *Response