discover

type DiscoverStats struct {
	ErrorCount int
	FetchCount int
}

type Discoverer struct {
	Name               string
	StartURL           url.URL
	ArtPats            []*regexp.Regexp
	XArtPats           []*regexp.Regexp
	NavLinkSel         cascadia.Selector
	XNavPats           []*regexp.Regexp
	CruftSel           cascadia.Selector
	BaseErrorThreshold int
	StripFragments     bool
	StripQuery         bool
	HostPat            *regexp.Regexp
	UserAgent          string

	ErrorLog Logger
	InfoLog  Logger
	Stats    DiscoverStats
}

type DiscovererDef struct {
	Name string
	URL  string
	// article urls to include - regexes
	ArtPat []string
	// article urls to exclude - regexes
	XArtPat []string

	// article url forms to include (eg "/YYYY/MM/SLUG.html")
	ArtForm []string
	// article url forms to exclude
	XArtForm []string

	// CSS selector to identify navigation links
	NavSel string
	// regexp patterns of pages to skip during link discovery
	XNavPat []string

	// css selector for elements to cull during article discovery
	CruftSel string

	// BaseErrorThreshold is starting number of http errors to accept before
	// bailing out. default is 5   (and 0 is considered as unset, so default is applied)
	// error threshold formula: base + 10% of successful request count
	BaseErrorThreshold int

	// Hostpat is a regex matching accepted domains
	// if empty, reject everything on a different domain
	HostPat string

	// If NoStripQuery is set then article URLs won't have the query part zapped
	NoStripQuery bool

	// UserAgent string to use in HTTP requests
	UserAgent string
}

type LinkSet map[url.URL]bool

type Logger interface {
	Printf(format string, v ...interface{})
}

type NullLogger struct{}