gocrawl

package module

Go to main page

Versions in this module

v0

v0.4.0

Apr 3, 2013

Changes in this version

+ var ErrEnqueueRedirect = errors.New("redirection not followed")

+ var ErrMaxVisits = errors.New("the maximum number of visits is reached")

type CrawlError

+ Ctx *URLContext

type CrawlErrorKind

+ const CekParseRedirectURL

+ const CekParseURL

+ func (this CrawlErrorKind) String() string

type FetchInfo

+ Ctx *URLContext

+ IsHeadRequest bool

+ type S map[string]interface

+ type U map[*url.URL]interface

+ type URLContext struct

+ HeadBeforeGet bool

+ State interface{}

+ func (this *URLContext) IsRobotsURL() bool

+ func (this *URLContext) NormalizedSourceURL() *url.URL

+ func (this *URLContext) NormalizedURL() *url.URL

+ func (this *URLContext) SourceURL() *url.URL

+ func (this *URLContext) URL() *url.URL

v0.3.2

Mar 26, 2013

v0.3.1

Jan 23, 2013

Changes in this version

+ const DefaultCrawlDelay

+ const DefaultEnqueueChanBuffer

+ const DefaultHostBufferFactor

+ const DefaultIdleTTL

+ const DefaultNormalizationFlags

+ const DefaultRobotUserAgent

+ const DefaultUserAgent

+ var HttpClient = &http.Client

+ type CrawlError struct

+ Err error

+ Kind CrawlErrorKind

+ URL *url.URL

+ func (this CrawlError) Error() string

+ type CrawlErrorKind uint8

+ const CekFetch

+ const CekHttpStatusCode

+ const CekParseBody

+ const CekParseNormalizedSeed

+ const CekParseRedirectUrl

+ const CekParseRobots

+ const CekParseSeed

+ const CekProcessLinks

+ const CekReadBody

+ type Crawler struct

+ Options *Options

+ func NewCrawler(ext Extender) *Crawler

+ func NewCrawlerWithOptions(opts *Options) *Crawler

+ func (this *Crawler) Run(seeds ...string) EndReason

+ type CrawlerCommand struct

+ Origin EnqueueOrigin

+ URL *url.URL

+ type DefaultExtender struct

+ EnqueueChan chan<- *CrawlerCommand

+ func (this *DefaultExtender) ComputeDelay(host string, di *DelayInfo, lastFetch *FetchInfo) time.Duration

+ func (this *DefaultExtender) Disallowed(u *url.URL)

+ func (this *DefaultExtender) End(reason EndReason)

+ func (this *DefaultExtender) Enqueued(u *url.URL, from *url.URL)

+ func (this *DefaultExtender) Error(err *CrawlError)

+ func (this *DefaultExtender) Fetch(u *url.URL, userAgent string, headRequest bool) (*http.Response, error)

+ func (this *DefaultExtender) FetchedRobots(res *http.Response)

+ func (this *DefaultExtender) Filter(u *url.URL, from *url.URL, isVisited bool, origin EnqueueOrigin) (enqueue bool, priority int, headRequest HeadRequestMode)

+ func (this *DefaultExtender) Log(logFlags LogFlags, msgLevel LogFlags, msg string)

+ func (this *DefaultExtender) RequestGet(headRes *http.Response) bool

+ func (this *DefaultExtender) RequestRobots(u *url.URL, robotAgent string) (request bool, data []byte)

+ func (this *DefaultExtender) Start(seeds []string) []string

+ func (this *DefaultExtender) Visit(res *http.Response, doc *goquery.Document) (harvested []*url.URL, findLinks bool)

+ func (this *DefaultExtender) Visited(u *url.URL, harvested []*url.URL)

+ type DelayInfo struct

+ LastDelay time.Duration

+ OptsDelay time.Duration

+ RobotsDelay time.Duration

+ type EndReason uint8

+ const ErDone

+ const ErError

+ const ErMaxVisits

+ type EnqueueOrigin int

+ const EoCustomStart

+ const EoError

+ const EoHarvest

+ const EoRedirect

+ const EoSeed

+ type EnqueueRedirectError struct

+ func (this *EnqueueRedirectError) Error() string

+ type Extender interface

+ ComputeDelay func(string, *DelayInfo, *FetchInfo) time.Duration

+ Disallowed func(*url.URL)

+ End func(EndReason)

+ Enqueued func(*url.URL, *url.URL)

+ Error func(*CrawlError)

+ Fetch func(*url.URL, string, bool) (*http.Response, error)

+ FetchedRobots func(*http.Response)

+ Filter func(*url.URL, *url.URL, bool, EnqueueOrigin) (bool, int, HeadRequestMode)

+ Log func(LogFlags, LogFlags, string)

+ RequestGet func(*http.Response) bool

+ RequestRobots func(*url.URL, string) (bool, []byte)

+ Start func([]string) []string

+ Visit func(*http.Response, *goquery.Document) ([]*url.URL, bool)

+ Visited func(*url.URL, []*url.URL)

+ type FetchInfo struct

+ Duration time.Duration

+ HeadRequest bool

+ RobotsRequest bool

+ StatusCode int

+ type HeadRequestMode uint8

+ const HrmDefault

+ const HrmIgnore

+ const HrmRequest

+ type LogFlags uint

+ const LogAll

+ const LogEnqueued

+ const LogError

+ const LogIgnored

+ const LogInfo

+ const LogNone

+ const LogTrace

+ type Options struct

+ CrawlDelay time.Duration

+ EnqueueChanBuffer int

+ Extender Extender

+ HeadBeforeGet bool

+ HostBufferFactor int

+ LogFlags LogFlags

+ MaxVisits int

+ RobotUserAgent string

+ SameHostOnly bool

+ URLNormalizationFlags purell.NormalizationFlags

+ UserAgent string

+ WorkerIdleTTL time.Duration

+ func NewOptions(ext Extender) *Options

v0.1.0

Oct 8, 2012

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL