discover

type DiscoverStats struct {
	ErrorCount int
	FetchCount int
}

type Discoverer struct {
	Name               string
	StartURL           url.URL
	ArtPats            []*regexp.Regexp
	NavLinkSel         cascadia.Selector
	BaseErrorThreshold int
	StripFragments     bool
	StripQuery         bool
	HostPat            *regexp.Regexp

	ErrorLog Logger
	InfoLog  Logger
	Stats    DiscoverStats
}

type DiscovererDef struct {
	Name   string
	URL    string
	ArtPat []string
	NavSel string
	// BaseErrorThreshold is starting number of http errors to accept before
	// bailing out.
	// error threshold formula: base + 10% of successful request count
	BaseErrorThreshold int

	// Hostpat is a regex matching accepted domains
	// if empty, reject everything on a different domain
	HostPat string

	// If NoStripQuery is set then article URLs won't have the query part zapped
	NoStripQuery bool
}

type LinkSet map[url.URL]bool

type Logger interface {
	Printf(format string, v ...interface{})
}

type NullLogger struct{}