Versions in this module Expand all Collapse all v0 v0.2.2 Feb 18, 2024 Changes in this version + func FindLinks(body []byte) (hrefs []string) + func Hostname(link string) (string, error) + func TestHostname(t *testing.T) + type Fetcher interface + Close func() error + Fetch func(ctx context.Context, req *Request) (*Response, error) + type FilterRule struct + Allow []*regexp.Regexp + Disallow []*regexp.Regexp + Hostname string + type MetricsMonitor interface + IncCrawledLink func() + IncDuplicatedLink func() + IncFailedRequests func() + IncSkippedLink func() + IncSuccessfulRequests func() + IncTotalLink func() + IncTotalRequests func() + Metrics func() map[string]int64 + type Param struct + MaxBodySize int64 + Proxy string + Referer string + Timeout time.Duration + UserAgent string + type ParsedURL struct + Hash string + Root string + URL *url.URL + func NewURL(raw string) (*ParsedURL, error) + func (u *ParsedURL) String() string + type Queue interface + Close func() error + Len func() int32 + Pop func(ctx context.Context) (*Request, error) + Push func(ctx context.Context, req *Request) error + type RateLimit struct + Hostname string + Rate string + type Request struct + Depth int32 + Param *Param + Target *ParsedURL + func (r *Request) ResolveURL(u string) (*url.URL, error) + type Response struct + Body []byte + Depth int32 + ElapsedTime time.Duration + Err error + NextURLs []*ParsedURL + Status int + URL *ParsedURL + type Store interface + Close func() error + HasVisited func(ctx context.Context, u *ParsedURL) (bool, error)
Go to main page