crawler

type Crawler struct {
	// contains filtered or unexported fields
}

type Hooks struct {
	// BeforeAction is invoked just before each action is dispatched, including
	// actions that will subsequently fail. A non-nil error aborts the action.
	BeforeAction func(page *browser.BrowserPage, action *types.Action) error
	// AfterAction is invoked only after an action completes successfully.
	// It is not called when the action returns an error. A non-nil error from
	// AfterAction is returned in place of the (otherwise nil) action error.
	AfterAction func(page *browser.BrowserPage, action *types.Action) error
	// BeforeNavigateBack is invoked once per browser-history back step during
	// state restoration, immediately before page.NavigateBack(). A non-nil
	// error aborts the navigation.
	BeforeNavigateBack func(page *browser.BrowserPage) error
}

type Options struct {
	Context             context.Context
	ChromiumPath        string
	MaxBrowsers         int
	MaxDepth            int
	PageMaxTimeout      time.Duration
	NoSandbox           bool
	NoIncognito         bool
	ShowBrowser         bool
	SlowMotion          bool
	MaxCrawlDuration    time.Duration
	MaxFailureCount     int
	Trace               bool
	CookieConsentBypass bool
	AutomaticFormFill   bool
	PageLoadStrategy    string
	ChromeWSUrl         string
	DOMWaitTime         int
	UserDataDir         string

	// EnableDiagnostics enables the diagnostics mode
	// which writes diagnostic information to a directory
	// specified by the DiagnosticsDir optionally.
	EnableDiagnostics bool
	DiagnosticsDir    string

	Proxy           string
	Logger          *slog.Logger
	ScopeValidator  browser.ScopeValidator
	RequestCallback func(*output.Result)
	ChromeUser      *user.User
	CaptchaHandler  *captcha.Handler
	UserArguments   map[string]string

	AuthUsername  string
	AuthPassword  string
	DitClassifier *dit.Classifier

	// Hooks installs optional lifecycle callbacks. See Hooks for semantics.
	// The zero value disables all callbacks.
	Hooks Hooks
}