crawlrequest

type Params struct {
	URL            string   `json:"url" jsonschema:"Seed URL to crawl (e.g. https://example.com)"`
	Depth          *int     `` /* 126-byte string literal not displayed */
	MaxPages       *int     `json:"max_pages,omitempty" jsonschema:"Hard cap on the number of pages crawled (default 500)"`
	Concurrency    *int     `json:"concurrency,omitempty" jsonschema:"Number of parallel fetch workers (default 4)"`
	RatePerSecond  *float64 `json:"rate,omitempty" jsonschema:"Max requests per second (default 0 = unlimited)"`
	MaxDuration    string   `` /* 204-byte string literal not displayed */
	Render         string   `json:"render,omitempty" jsonschema:"Rendering mode: 'raw' (default) or 'headless'"`
	Analyzers      []string `json:"analyzers,omitempty" jsonschema:"Subset of analyzer names to run; empty runs all"`
	Specialized    *bool    `` /* 140-byte string literal not displayed */
	SecurityAudit  *bool    `` /* 188-byte string literal not displayed */
	RespectRobots  *bool    `json:"respect_robots,omitempty" jsonschema:"Obey robots.txt while crawling (default true)"`
	Subdomains     *bool    `json:"subdomains,omitempty" jsonschema:"Follow links to subdomains of the seed host"`
	FollowExternal *bool    `json:"follow_external,omitempty" jsonschema:"Follow links that leave the seed host entirely (default false)"`
	Include        []string `json:"include,omitempty" jsonschema:"Only crawl URLs matching at least one of these regexes"`
	Exclude        []string `json:"exclude,omitempty" jsonschema:"Skip URLs matching any of these regexes"`

	UserAgent         string   `json:"user_agent,omitempty" jsonschema:"User-Agent header sent on every request"`
	UserAgents        []string `json:"user_agents,omitempty" jsonschema:"Pool of User-Agent strings to rotate across (supersedes user_agent)"`
	UserAgentRotation string   `json:"user_agent_rotation,omitempty" jsonschema:"Rotation across user_agents: off, round-robin, or random"`
	Proxy             string   `json:"proxy,omitempty" jsonschema:"Route requests through this proxy URL (http(s):// or socks5://; supports user:pass@host)"`
	Proxies           []string `json:"proxies,omitempty" jsonschema:"Pool of proxy URLs to rotate across"`
	ProxyRotation     string   `json:"proxy_rotation,omitempty" jsonschema:"Rotation across proxies: off, round-robin, random, or sticky-host"`
	BasicAuth         string   `` /* 165-byte string literal not displayed */
}