scraper

Sep 20, 2022

Changes in this version

type CollyScraper

+ Silently bool

type RodScraper

+ Silently bool

Sep 13, 2022

Changes in this version

+ type CollyScraper struct

+ Collector *colly.Collector

+ LoadingTimeoutSeconds int

+ Response *http.Response

+ TimeoutSeconds int

+ Transport *http.Transport

+ UserAgent string

+ func (s *CollyScraper) CanRenderPage() bool

+ func (s *CollyScraper) EvalJS(jsProp string) (*string, error)

+ func (s *CollyScraper) Init() error

+ func (s *CollyScraper) Scrape(paramURL string) (*ScrapedData, error)

+ func (s *CollyScraper) SetDepth(depth int)

+ type GoWapTransport struct

+ func NewGoWapTransport(t *http.Transport, f func(resp *http.Response)) *GoWapTransport

+ func (gt *GoWapTransport) RoundTrip(req *http.Request) (*http.Response, error)

+ type RodScraper struct

+ Browser *rod.Browser

+ LoadingTimeoutSeconds int

+ Page *rod.Page

+ TimeoutSeconds int

+ UserAgent string

+ func (s *RodScraper) CanRenderPage() bool

+ func (s *RodScraper) EvalJS(jsProp string) (*string, error)

+ func (s *RodScraper) Init() error

+ func (s *RodScraper) Scrape(paramURL string) (*ScrapedData, error)

+ func (s *RodScraper) SetDepth(depth int)

+ type ScrapedData struct

+ CertIssuer []string

+ Cookies map[string]string

+ DNS map[string][]string

+ HTML string

+ Headers map[string][]string

+ Meta map[string][]string

+ Scripts []string

+ URLs ScrapedURL

+ type ScrapedURL struct

+ Status int

+ URL string

+ type Scraper interface

+ CanRenderPage func() bool

+ EvalJS func(jsProp string) (*string, error)

+ Init func() error

+ Scrape func(paramURL string) (*ScrapedData, error)

+ SetDepth func(depth int)