Versions in this module Expand all Collapse all v0 v0.9.0 Nov 24, 2024 v0.8.1 Feb 26, 2024 v0.8.0 Feb 24, 2024 v0.7.2 Feb 17, 2024 v0.7.1 Feb 9, 2024 v0.7.0 Feb 7, 2024 v0.6.1 Jan 8, 2024 Changes in this version + var Version string v0.6.0 Dec 4, 2023 v0.5.0 Nov 18, 2023 Changes in this version + const HeaderBypassCache v0.4.1 Nov 13, 2023 v0.4.0 Nov 11, 2023 Changes in this version + func Dev(file string) error + func Document(sel *goquery.Selection) map[string]any + func DocumentFromString(s string) (map[string]any, error) + func Run(file string) error + type Exports map[string]any + func (e Exports) Config() []byte + func (e Exports) Scrape(p ScrapeParams) (any, error) + func (e Exports) Setup() + type Imports map[string]map[string]any + func NewJSLibrary(client *http.Client) Imports type Scraper + Client *http.Client + Modules []Module + SetupFunc func() v0.3.0 Oct 19, 2023 v0.2.1 Oct 15, 2023 v0.2.0 Oct 12, 2023 Changes in this version + var ScriptTemplate []byte + func LoadModules(s *Scraper, cfg Config) + func MockResponse(statusCode int, html string) (*http.Response, error) + func Prettify(v any, prefix string) string + func RegisterModule(mod Module) + type Config []byte + type Context interface + DisableModule func(id string) + MarkUnvisited func(url string) + MarkVisited func(url string) + ScriptName func() string + Visit func(url string) + type Finalizer interface + Finalize func() + type Module interface + ModuleInfo func() ModuleInfo + type ModuleInfo struct + ID string + New func() Module + type Provisioner interface + Provision func(Context) + type Request struct + Cookies http.CookieJar + Depth int + Headers http.Header + Method string + URL string + type RequestBuilder interface + BuildRequest func(*Request) + type RequestValidator interface + ValidateRequest func(*Request) bool + type Response struct + Body []byte + Data any + Error error + Headers http.Header + Request *Request + StatusCode int + Visit func(url string) + type ResponseReceiver interface + ReceiveResponse func(*Response) + type RoundTripFunc func(*http.Request) (*http.Response, error) + func MockTransport(statusCode int, html string) RoundTripFunc + func (f RoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) type Scraper + Script string + func NewScraper() *Scraper + func (s *Scraper) DisableModule(id string) + func (s *Scraper) LoadModule(mod Module) + func (s *Scraper) MarkUnvisited(url string) + func (s *Scraper) MarkVisited(url string) + func (s *Scraper) Run() + func (s *Scraper) ScriptName() string + func (s *Scraper) Visit(url string) + type TransportAdapter interface + AdaptTransport func(http.RoundTripper) http.RoundTripper v0.1.0 Aug 29, 2023 Changes in this version + var StopWatch = errors.New("stop watch") + func Compile(src string) (ScrapeOptions, ScrapeFunc, error) + func PrettyPrint(v any, prefix string) string + func Print(v any, prefix string) string + func Watch(path string, fn func(string) error) error + type FetchFunc func(url string) (string, error) + func CachedFetch(fetch FetchFunc) FetchFunc + func Fetch() FetchFunc + func ProxiedFetch(proxyURL string) FetchFunc + type ScrapeFunc func(ScrapeParams) (any, error) + type ScrapeOptions struct + AllowedDomains []string + AllowedURLs []string + BlockedDomains []string + BlockedURLs []string + Depth int + Proxy string + Rate float64 + URL string + type ScrapeParams struct + HTML string + URL string + type ScrapeResult struct + Data any + Error error + Links []string + Timestamp time.Time + URL string + type Scraper struct + FetchFunc FetchFunc + ScrapeFunc ScrapeFunc + ScrapeOptions ScrapeOptions + func (s *Scraper) Scrape() <-chan ScrapeResult + type TransformError struct + Column int + Line int + Text string + func (err TransformError) Error() string
Go to main page