goatscrape

type LinkStore interface {
	// GetLinks should return a string slice of links to crawl, in the
	// amount defined in the amount paramter.
	GetLinks(amount int) []string
	// AddToCrawl should add the link parameter to the to crawl list.
	AddToCrawl(link string)
	// MoveToCrawled should delete the link in the to crawl list and
	// place it in the crawled list.
	MoveToCrawled(link string)
	// MoreToCrawl returns a boolean value if there are still links
	// in the to crawl list.
	MoreToCrawl() bool
}

type ParseFunc func(*http.Response) []string

type PreRequestFunc func(*http.Request)

type RequestFunc func(*http.Request) (*http.Response, error)

type Spider struct {
	// Name is the unique scan name. It is currently use for logging purposes.
	Name string
	// StartingURLs is a string slice of all the URLs that will be loaded into
	// the spider first. These should be used to seed the scanner.
	StartingURLs []string
	// AllowedDomains is a string slice with all the allowed domains. An empty
	// slice will cause the spider to assume that there are no domains that are not allowed.
	AllowedDomains []string
	// DisallowedPages is a slice of regular expressions. Each expression is evaluated on all links
	// returned from the Parse() function. If the expression matches then the link is not added to the
	// to crawl list.
	DisallowedPages []regexp.Regexp
	// MaxPages is the maximum amount of pages to crawl before the scanner returns. A setting of zero or less
	// causes the spider to assume there are no maximum pages.
	MaxPages int
	// MaxConcurrentRequests is the maximum amount of requests to run in parallel.
	MaxConcurrentRequests int

	// The Parse function should emit a list of urls that should be added to the crawl.
	Parse ParseFunc
	// PreRequestMiddleware is a slice of functions that implement PreRequestFunc. Each of these functions
	// is called on the http.Request object before it is execute by the http.Client.
	PreRequestMiddleware []PreRequestFunc
	// The function that gets a web page. Should take a http.Request and return a http.Response
	Getter RequestFunc

	// Verbose will cause more diagnostic information to be outputted if it's set to true.
	Verbose bool
	// Quiet will repress all output to stdout or stderr
	Quiet bool

	// Links is the LinkStore object that is used by this spider. LinkStores are responsible for storing,
	// and managing the crawled and the to crawl lists used by the spider during its operation.
	Links LinkStore
	// contains filtered or unexported fields
}