Versions in this module Expand all Collapse all v1 v1.1.2 Sep 4, 2017 v1.1.1 Sep 4, 2017 v1.1.0 Jul 6, 2017 Changes in this version type Queue + func (q *Queue) Done() <-chan struct{} v1.0.0 Mar 24, 2017 Changes in this version + const DefaultCrawlDelay + const DefaultUserAgent + const DefaultWorkerIdleTTL + var ErrDisallowed = errors.New("fetchbot: disallowed by robots.txt") + var ErrEmptyHost = errors.New("fetchbot: invalid empty host") + var ErrQueueClosed = errors.New("fetchbot: send on a closed queue") + type BasicAuthProvider interface + BasicAuth func() (user string, pwd string) + type Cmd struct + M string + U *url.URL + func (c *Cmd) Method() string + func (c *Cmd) URL() *url.URL + type Command interface + Method func() string + URL func() *url.URL + type Context struct + Cmd Command + Q *Queue + type CookiesProvider interface + Cookies func() []*http.Cookie + type DebugInfo struct + NumHosts int + type Doer interface + Do func(*http.Request) (*http.Response, error) + type Fetcher struct + AutoClose bool + CrawlDelay time.Duration + DisablePoliteness bool + Handler Handler + HttpClient Doer + UserAgent string + WorkerIdleTTL time.Duration + func New(h Handler) *Fetcher + func (f *Fetcher) Debug() <-chan *DebugInfo + func (f *Fetcher) Start() *Queue + type Handler interface + Handle func(*Context, *http.Response, error) + type HandlerCmd struct + func NewHandlerCmd(method, rawURL string, fn func(*Context, *http.Response, error)) (*HandlerCmd, error) + type HandlerFunc func(*Context, *http.Response, error) + func (h HandlerFunc) Handle(ctx *Context, res *http.Response, err error) + type HeaderProvider interface + Header func() http.Header + type Mux struct + DefaultHandler Handler + func NewMux() *Mux + func (mux *Mux) Handle(ctx *Context, res *http.Response, err error) + func (mux *Mux) HandleError(err error, h Handler) + func (mux *Mux) HandleErrors(h Handler) + func (mux *Mux) Response() *ResponseMatcher + type Queue struct + func (q *Queue) Block() + func (q *Queue) Cancel() error + func (q *Queue) Close() error + func (q *Queue) Send(c Command) error + func (q *Queue) SendString(method string, rawurl ...string) (int, error) + func (q *Queue) SendStringGet(rawurl ...string) (int, error) + func (q *Queue) SendStringHead(rawurl ...string) (int, error) + type ReaderProvider interface + Reader func() io.Reader + type ResponseMatcher struct + func (r *ResponseMatcher) ContentType(ct string) *ResponseMatcher + func (r *ResponseMatcher) Custom(predicate func(*http.Response) bool) *ResponseMatcher + func (r *ResponseMatcher) Handler(h Handler) *ResponseMatcher + func (r *ResponseMatcher) Host(host string) *ResponseMatcher + func (r *ResponseMatcher) Method(m string) *ResponseMatcher + func (r *ResponseMatcher) Path(p string) *ResponseMatcher + func (r *ResponseMatcher) Scheme(scheme string) *ResponseMatcher + func (r *ResponseMatcher) Status(code int) *ResponseMatcher + func (r *ResponseMatcher) StatusRange(min, max int) *ResponseMatcher + type ValuesProvider interface + Values func() url.Values
Go to main page