spy

type Closer interface {
	Close(spider ISpider)
}

type Config struct {
	*viper.Viper

	ConcurrentRequests          int
	ConcurrentRequestsPerDomain int
	ConcurrentRequestsPerIp     int
	RandomizeFetchDelay         bool
	FetchDelay                  float64
}

type CrawlSpider struct {
	*Spider
}

type Crawler struct {
	*Config
	*logrus.Logger
	*Stats

	Concurrency int

	Spider    ISpider
	Scheduler IScheduler
	Fetcher   IFetcher
	*SpiderMiddlewareManager
	*ItemPipelineManager

	*tunny.WorkPool
	*concurrency.Worker
	// contains filtered or unexported fields
}

type DupeFilter interface {
	Opener
	Closer
	SeenRequest(request *Request) bool
}

type Event string

type Fetcher struct {
	TotalConcurrency  int
	DomainConcurrency int
	IpConcurrency     int
	Delay             time.Duration
	RandomizeDelay    bool
	// contains filtered or unexported fields
}

type FetcherHandler interface {
	Fetch(request *Request, spider ISpider) (*Response, error)
	Close()
}

type FetcherMiddleware interface{}

type FetcherMiddlewareManager struct {
	// contains filtered or unexported fields
}

type FetchingErrorProcessor interface {
	/* ProcessError is called when a fetcher handler or a ProcessRequest (from a fetcher middleware) returns an error.
	It should return: either nils, a Response object, or a Request object.

	If it returns nils, will continue processing this error, executing any other ProcessError methods of installed
	middleware, until no middleware is left.

	If it returns a Response object, the ProcessResponse methods chain of installed middleware is started,
	and won’t bother calling any other ProcessError methods of middleware.

	If it returns a Request object, the returned request is rescheduled to be performed in the future.
	This stops the execution of ProcessError methods of the middleware the same as returning a response would.
	*/
	ProcessError(err error, request *Request, spider ISpider) (*Response, *Request)
}

type FetchingRequestProcessor interface {
	/* ProcessRequest is called for each request that goes through the fetcher middleware.
	It should either: return nils, return a Response object, return a Request object, or return an IgnoreRequest error.

	If it returns nils, will continue processing this request, executing all other middlewares,
	until, finally, the appropriate fetcher handler performs the request performed (and its response downloaded).

	If it returns a Response object, won’t bother calling any other ProcessRequest or ProcessError methods,
	or the appropriate fetcher handler; it’ll return that response. The ProcessResponse methods of installed middleware
	is always called on every response.

	If it returns a Request object, will stop calling ProcessRequest methods and reschedule the returned request.
	Once the newly returned request is performed, the appropriate middleware chain will again be called on the downloaded response.

	If it returns an IgnoreRequest error, the ProcessError methods of installed downloader middleware will be called.
	If none of them handle the error, the callback function of the request (Request.Callback) is called.
	If no code handles the returned error, it is ignored and not logged (unlike other error).
	*/
	ProcessRequest(request *Request, spider ISpider) (*Response, *Request, error)
}

type FetchingResponseProcessor interface {
	/* ProcessResponse should either: return a Response object, return a Request object or return an IgnoreRequest error.

	If it returns a Response (it could be the same given response, or a brand-new one), that response will continue to be
	processed with the ProcessResponse method of the next middleware in the chain.

	If it returns a Request object, the middleware chain is halted and the returned request is rescheduled to be performed
	in the future. This is the same behavior as if a request is returned from ProcessRequest.

	If it returns an IgnoreRequest error, the callback function of the request (Request.Callback) is called.
	If no code handles the returned error, it is ignored and not logged (unlike other error).
	*/
	ProcessResponse(response *Response, request *Request, spider ISpider) (*Response, *Request, error)
}

type FingerprintDupeFilter struct {
	*logrus.Logger
	// contains filtered or unexported fields
}

type FromConfiger interface {
	FromConfig(config Config)
}

type FromCrawlerer interface {
	FromCrawler(crawler Crawler)
}

type GoquerySelector struct {
	*goquery.Selection
}

type HTMLLinkExtractor struct {
	// Regular expressions that the (absolute) urls must match in order to be extracted.
	// If empty, it will match all links.
	Allows []string

	// regular expressions that the (absolute) urls must match in order to be excluded.
	// It has precedence over the Allows parameter.
	// If empty, it won't exclude any links.
	Denies []string

	// Domains which will be considered for extracting the links.
	AllowDomains []string

	// Domains which won't be considered for extracting the links.
	DenyDomains []string

	// File extensions that should be ignored when extracting links.
	// If empty, it will default to the IgnoredExtensions.
	DenyExtensions []string

	// Selectors which define regions inside the response where links should be extracted from.
	// If given, only the text selected by those selectors will be scanned for links.
	RestrictSelectors []string

	// Whether duplicate filtering should be applied to extracted links.
	// Defaults to false.
	Unique bool

	// Function which receives each url value extracted from the tag and attributes scanned
	// and can modify the value and return a new one, or return nil to ignore the link altogether.
	// If not given, defaults to the untouched link.
	ProcessValue func(value *url.URL) *url.URL

	// a list of tags to consider when extracting links.
	// Defaults to {"a", "area"}.
	Tags []string

	// Attributes which should be considered when looking for links to extract.
	// Only for those tags specified in the tags parameter.
	// Defaults to {"href"}.
	Attrs []string
	// contains filtered or unexported fields
}

type IFetcher interface {
	NeedsBackout() bool
	Fetch(req *Request, spider ISpider) (*Response, *Request, error)
	Opener
	Closer
}

type IScheduler interface {
	Opener
	Closer
	EnqueueRequest(request *Request) bool
	NextRequest() *Request
}

type ISpider interface {
	StartRequests() []*Request
	Parse(response *Response) (*SpiderResult, error)
	FetchDelay() time.Duration
	ConcurrentRequests() int
	String() string
	Crawler() *Crawler
}

type Item map[string]interface{}

type ItemPipelineManager struct {
	// contains filtered or unexported fields
}

type ItemPipelineMiddleware interface{}

type ItemProcessor interface {
	ProcessItem(item *Item, spider ISpider) (*Item, error)
}

type Link struct {
	// contains filtered or unexported fields
}

type LinkExtractor interface {
	ExtractLinks(response *Response) ([]Link, error)
}

type Middleware interface {
	OnSpiderOpeneder
	OnSpiderCloseder
}

type MiddlewareManager struct {
	// contains filtered or unexported fields
}

type MiddlewareManagerIterator struct {
	// contains filtered or unexported fields
}

type OnSpiderCloseder interface {
	OnSpiderClosed(spider ISpider)
}

type OnSpiderOpeneder interface {
	OnSpiderOpened(spider ISpider)
}

type Opener interface {
	Open(spider ISpider)
}

type Request struct {
	*http.Request
	Error     error
	Meta      map[string]interface{}
	NotFilter bool
	Callback  func(response *Response, err error) (*SpiderResult, error)
}

type Response struct {
	*http.Response

	MediaType string
	HTMLDoc   *goquery.Document

	/* Request which generated this response.
		This attribute is assigned in the `Crawler`, after the response and the request have passed
	    through all `Fetcher Middlewares`. In particular, this means that:

	    - HTTP redirections will cause the original request (to the URL before
	      redirection) to be assigned to the redirected response (with the final
	      URL after redirection).

	    - Response.Request.URL doesn't always equal Response.Response.URL

	    - This attribute is only available in the spider code, and in the `Spider Middlewares`,
	      but not in `Downloader Middlewares` (although you have the Request available there by
	      other means) and handlers of the `response_downloaded` signal.
	*/
	*Request
	// contains filtered or unexported fields
}

type Rule struct {
}

type Scheduler struct {
	// contains filtered or unexported fields
}

type Selector interface {
	Select(query string) Selectors
	Regex(regex interface{}) []string
	Extract() string
	Attr(attrName string) (val string, exists bool)
}

type Selectors []Selector

type Spider struct {
	Name      string
	StartURLs []string
}

type SpiderErrorProcessor interface {
	ProcessSpiderError(err error, response *Response, spider ISpider) (*SpiderResult, error)
}

type SpiderInputProcessor interface {
	ProcessSpiderInput(response *Response, spider ISpider) error
}

type SpiderMiddleware interface{}

type SpiderMiddlewareManager struct {
	// contains filtered or unexported fields
}

type SpiderOutputProcessor interface {
	ProcessSpiderOutput(result *SpiderResult, response *Response, spider ISpider) (*SpiderResult, error)
}

type SpiderResult struct {
	Requests []*Request
	Items    []*Item
}

type StartRequestsProcessor interface {
	ProcessStartRequests(startRequests []*Request, spider ISpider) ([]*Request, error)
}

type Stats struct {
	Name string
	// contains filtered or unexported fields
}