gowebcrawler

type Crawler ¶

type Crawler interface {
	Crawl(string, parser Parser) ([]byte, error)
}

type Page ¶

type Page struct {
	Url      string
	Assets   []string
	Links    []string
	Children map[string]*Page
	// contains filtered or unexported fields
}

A Page represents a web page's relation to other pages and the data needed to make a site map showing assets it depends on

type PageMessage ¶

type PageMessage struct {
	Page  *Page
	Error error
	Url   string
}

type Parser ¶

type Parser interface {
	Parse(string) (links []string, assets []string, err error)
}

type UrlParser ¶

type UrlParser struct{}

UrlParser implements Parser to extract relevant data from a page at a given URL

func (u UrlParser) Parse(url string) (links []string, assets []string, err error)

type WebCrawler ¶

type WebCrawler struct {
	Parser     *UrlParser
	RootUrl    string
	FetchLimit int
}

WebCrawler implements Crawler and generates a JSON site map from a starting domain and path. It takes care to not crawl other domains or get the same page more than once. Also supports a FetchLimit to limit total fetches made.

func (w WebCrawler) Crawl(url string) ([]byte, error)