Versions in this module Expand all Collapse all v0 v0.1.0 Mar 21, 2021 Changes in this version + type LinkProcessor struct + func NewLinkProcessor(storage *linkstorage.Storage, batchSize int, queue *linkqueue.LinkQueue, ...) (*LinkProcessor, error) + func (lp *LinkProcessor) CheckURLExists(u *url.URL) (bool, error) + func (lp *LinkProcessor) Close() + func (lp *LinkProcessor) GracefulShutdown() <-chan bool + func (lp *LinkProcessor) MarkURLVisited(u *url.URL) + func (lp *LinkProcessor) ProcessURL(u *url.URL) error + func (lp *LinkProcessor) ScrapeLinksFromURL(u *url.URL) ([]*linkstorage.Link, error) + func (lp *LinkProcessor) SpawnWorkers(n int) chan *url.URL
Go to main page