scraper

type Scraper struct {
	URL *urlpkg.URL // contains the main URL to parse, will be modified in case of a redirect

	Client download.HttpClient
	Fs     afero.Fs // filesystem

	// ETagsDB stores ETags (hashes of file state) for each URL
	ETagsDB *db.DB
	// contains filtered or unexported fields
}