bluesnake

type AlreadyVisitedError struct {
	// Destination is the URL that was attempted to be visited.
	// It might not match the URL passed to Visit if redirect
	// was followed.
	Destination *url.URL
}

type Collector struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// Custom headers for the request
	Headers *http.Header
	// AllowURLRevisit allows multiple downloads of the same URL
	AllowURLRevisit bool
	// MaxBodySize is the limit of the retrieved response body in bytes.
	// 0 means unlimited.
	// The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
	MaxBodySize int
	// CacheDir specifies a location where GET requests are cached as files.
	// When it's not defined, caching is disabled.
	CacheDir string
	// IgnoreRobotsTxt allows the Collector to ignore any restrictions set by
	// the target host's robots.txt file.  See http://www.robotstxt.org/ for more
	// information.
	IgnoreRobotsTxt bool
	// ParseHTTPErrorResponse allows parsing HTTP responses with non 2xx status codes.
	// By default, BlueSnake parses only successful HTTP responses. Set ParseHTTPErrorResponse
	// to true to enable it.
	ParseHTTPErrorResponse bool
	// ID is the unique identifier of a collector
	ID uint32
	// DetectCharset can enable character encoding detection for non-utf8 response bodies
	// without explicit charset declaration. This feature uses https://github.com/saintfish/chardet
	DetectCharset bool

	// CheckHead performs a HEAD request before every GET to pre-validate the response
	CheckHead bool
	// TraceHTTP enables capturing and reporting request performance for crawler tuning.
	// When set to true, the Response.Trace will be filled in with an HTTPTrace object.
	TraceHTTP bool
	// MaxRequests limit the number of requests done by the instance.
	// Set it to 0 for infinite requests (default).
	MaxRequests uint32
	// EnableRendering enables JavaScript rendering using headless Chrome.
	// When set to true, pages will be rendered with chromedp before parsing.
	EnableRendering bool
	// RenderingConfig contains configuration for JavaScript rendering wait times
	// Only applies when EnableRendering is true
	RenderingConfig *RenderingConfig
	// EnableContentHash enables content-based duplicate detection
	EnableContentHash bool
	// ContentHashAlgorithm specifies the hash algorithm to use ("xxhash", "md5", "sha256")
	ContentHashAlgorithm string
	// ContentHashConfig contains detailed configuration for content hashing
	ContentHashConfig *ContentHashConfig

	// CacheExpiration sets the maximum age for cache files.
	// If a cached file is older than this duration, it will be ignored and refreshed.
	CacheExpiration time.Duration
	// contains filtered or unexported fields
}

type ContentFilter interface {
	// Filter applies the filter to the document and returns the modified document
	Filter(doc *goquery.Document) *goquery.Document
	// Name returns the filter name for debugging
	Name() string
}

type ContentHashConfig struct {
	// ExcludeTags specifies HTML tags to exclude from content hashing
	// Default: ["script", "style", "nav", "footer"]
	ExcludeTags []string
	// IncludeOnlyTags specifies to only include specific tags in content hashing
	// If empty, all content (minus ExcludeTags) is included
	// Example: ["article", "main"] to focus only on main content
	IncludeOnlyTags []string
	// StripTimestamps removes timestamp patterns from content before hashing
	StripTimestamps bool
	// StripAnalytics removes analytics and tracking code from content
	StripAnalytics bool
	// StripComments removes HTML comments from content
	StripComments bool
	// CollapseWhitespace normalizes whitespace (multiple spaces/newlines to single)
	CollapseWhitespace bool
}

type Context struct {
	// contains filtered or unexported fields
}

type CrawlCompletionReason string

type CrawlResult struct {
	// Reason indicates why the crawl completed
	Reason CrawlCompletionReason
	// TotalPages is the total number of HTML pages successfully crawled
	TotalPages int
	// TotalDiscovered is the total number of unique URLs discovered
	TotalDiscovered int
	// URLsVisited is the number of URLs visited in this session
	URLsVisited int
	// PendingURLs contains URLs that were queued but not visited (for resume)
	PendingURLs []URLDiscoveryRequest
}

type Crawler struct {
	// Collector is the underlying low-level collector (exported for advanced configuration)
	Collector *Collector
	// contains filtered or unexported fields
}

type CrawlerConfig struct {
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains []string
	// DisallowedDomains is a domain blacklist.
	DisallowedDomains []string
	// DisallowedURLFilters is a list of regular expressions which restricts
	// visiting URLs. If any of the rules matches to a URL the
	// request will be stopped. DisallowedURLFilters will
	// be evaluated before URLFilters
	DisallowedURLFilters []*regexp.Regexp
	// URLFilters is a list of regular expressions which restricts
	// visiting URLs. If any of the rules matches to a URL the
	// request won't be stopped. DisallowedURLFilters will
	// be evaluated before URLFilters
	URLFilters []*regexp.Regexp
	// DiscoveryMechanisms specifies which mechanisms to use for URL discovery.
	// Can be any combination: ["spider"], ["sitemap"], or ["spider", "sitemap"].
	// Default is ["spider"].
	DiscoveryMechanisms []DiscoveryMechanism
	// SitemapURLs specifies custom sitemap URLs to fetch (optional).
	// If nil/empty when sitemap discovery is enabled, tries default locations
	// (/sitemap.xml, /sitemap_index.xml).
	SitemapURLs []string
	// ResourceValidation configures checking of non-HTML resources for broken links
	ResourceValidation *ResourceValidationConfig
	// RobotsTxtMode controls how robots.txt is handled
	// Options: "respect", "ignore", or "ignore-report"
	// Default: "respect"
	RobotsTxtMode string
	// FollowInternalNofollow allows following links with rel="nofollow" on same domain
	// Default: false
	FollowInternalNofollow bool
	// FollowExternalNofollow allows following links with rel="nofollow" on external domains
	// Default: false
	FollowExternalNofollow bool
	// RespectMetaRobotsNoindex respects <meta name="robots" content="noindex">
	// Default: true
	RespectMetaRobotsNoindex bool
	// RespectNoindex respects X-Robots-Tag: noindex headers
	// Default: true
	RespectNoindex bool

	// DiscoveryChannelSize is the buffer size for the URL discovery channel
	// Larger values reduce blocking but use more memory
	// Default: 50000
	DiscoveryChannelSize int

	// WorkQueueSize is the buffer size for the worker pool work queue
	// Should be smaller than DiscoveryChannelSize
	// Default: 1000
	WorkQueueSize int

	// Parallelism is the number of concurrent HTTP requests (worker pool size)
	// This replaces/complements the existing async goroutine model
	// Default: 10
	Parallelism int

	// DebugURLs contains exact URLs to enable detailed logging for (scheme and trailing slash ignored)
	// Used for debugging race conditions by filtering logs to specific problematic URLs
	// Uses exact matching to avoid logging all subpaths
	// Example: []string{"handbook.agentberlin.ai/intro", "handbook.agentberlin.ai"}
	DebugURLs []string

	// HTTP contains HTTP client configuration for the underlying Collector
	HTTP *HTTPConfig

	// Incremental crawling support
	// MaxURLsToVisit limits the number of URLs to visit before pausing.
	// Set to 0 for unlimited (default). When the limit is reached, the crawl
	// pauses and OnCrawlPaused callback is called with pending URLs.
	MaxURLsToVisit int
	// PreVisitedHashes contains URL hashes that should be considered already visited.
	// Used when resuming a paused crawl to avoid re-crawling URLs.
	PreVisitedHashes []uint64
	// SeedURLs contains URLs to queue at the start of the crawl (in addition to the initial URL).
	// Used when resuming a paused crawl with pending URLs from the previous session.
	SeedURLs []URLDiscoveryRequest
}

type DiscoveryMechanism string

type ErrorCallback func(*Response, error)

type FilterChain struct {
	// contains filtered or unexported fields
}

type HTMLCallback func(*HTMLElement)

type HTMLElement struct {
	// Name is the name of the tag
	Name string
	Text string

	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
	// Index stores the position of the current element within all the elements matched by an OnHTML callback
	Index int
	// contains filtered or unexported fields
}

type HTTPConfig struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// Headers contains custom headers for HTTP requests
	Headers map[string]string
	// AllowURLRevisit allows multiple downloads of the same URL
	AllowURLRevisit bool
	// MaxBodySize is the limit of the retrieved response body in bytes.
	// 0 means unlimited.
	// The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
	MaxBodySize int
	// CacheDir specifies a location where GET requests are cached as files.
	// When it's not defined, caching is disabled.
	CacheDir string
	// CacheExpiration sets the maximum age for cache files.
	CacheExpiration time.Duration
	// IgnoreRobotsTxt allows the Collector to ignore any restrictions set by
	// the target host's robots.txt file.
	IgnoreRobotsTxt bool
	// ParseHTTPErrorResponse allows parsing HTTP responses with non 2xx status codes.
	ParseHTTPErrorResponse bool
	// ID is the unique identifier of a collector (auto-assigned if 0)
	ID uint32
	// DetectCharset can enable character encoding detection for non-utf8 response bodies
	DetectCharset bool
	// CheckHead performs a HEAD request before every GET to pre-validate the response
	CheckHead bool
	// TraceHTTP enables capturing and reporting request performance.
	TraceHTTP bool
	// MaxRequests limit the number of requests done by the instance.
	// Set it to 0 for infinite requests (default).
	MaxRequests uint32
	// EnableRendering enables JavaScript rendering using headless Chrome.
	EnableRendering bool
	// RenderingConfig contains configuration for JavaScript rendering wait times
	// Only applies when EnableRendering is true
	RenderingConfig *RenderingConfig
	// EnableContentHash enables content-based duplicate detection
	// When true, pages with identical content will be detected even if URLs differ
	EnableContentHash bool
	// ContentHashAlgorithm specifies the hash algorithm to use
	// Options: "xxhash" (fastest, default), "md5", "sha256"
	ContentHashAlgorithm string
	// ContentHashConfig contains detailed configuration for content hashing
	ContentHashConfig *ContentHashConfig
	// Debugger is the debugger instance to use
	Debugger debug.Debugger
}

type HTTPTrace struct {
	ConnectDuration   time.Duration
	FirstByteDuration time.Duration
	// contains filtered or unexported fields
}

type Index struct {
	XMLName xml.Name `xml:"sitemapindex"`
	Sitemap []parts  `xml:"sitemap"`
}

type LimitRule struct {
	// DomainRegexp is a regular expression to match against domains
	DomainRegexp string
	// DomainGlob is a glob pattern to match against domains
	DomainGlob string
	// Delay is the duration to wait before creating a new request to the matching domains
	Delay time.Duration
	// RandomDelay is the extra randomized duration to wait added to Delay before creating a new request
	RandomDelay time.Duration
	// Parallelism is the number of the maximum allowed concurrent requests of the matching domains
	Parallelism int
	// contains filtered or unexported fields
}

type Link struct {
	// URL is the target URL
	URL string `json:"url"`
	// Type is the link type: "anchor", "image", "script", "stylesheet", "iframe", "canonical", "video", "audio"
	Type string `json:"type"`
	// Text is the anchor text, alt text, or empty for other link types
	Text string `json:"text"`
	// Context is the surrounding text context where the link appears
	Context string `json:"context,omitempty"`
	// IsInternal indicates if this link points to the same domain/subdomain
	IsInternal bool `json:"isInternal"`
	// Status is the HTTP status code if this URL has been crawled (200, 404, 301, etc.)
	Status *int `json:"status,omitempty"`
	// Title is the page title if this URL has been crawled
	Title string `json:"title,omitempty"`
	// ContentType is the MIME type if this URL has been crawled
	ContentType string `json:"contentType,omitempty"`
	// Position indicates the semantic location of the link on the page
	// Values: "content", "navigation", "header", "footer", "sidebar", "breadcrumbs", "pagination", "unknown"
	Position string `json:"position,omitempty"`
	// DOMPath is a simplified DOM path showing the link's location in the HTML structure
	// Example: "body > main > article > p > a"
	DOMPath string `json:"domPath,omitempty"`
	// Action indicates how this URL should be handled
	// Values: "crawl" (normal), "record" (framework-specific, don't crawl), "skip" (ignored)
	Action URLAction `json:"action,omitempty"`
}

type LinkDensityFilter struct {
	// MaxLinkRatio is the maximum ratio of link words to total words (default 0.5)
	MaxLinkRatio float64
	// MinLinks is the minimum number of links before considering link density (default 3)
	MinLinks int
}

type Links struct {
	// Internal links point to same domain/subdomain
	Internal []Link `json:"internal"`
	// External links point to different domains
	External []Link `json:"external"`
}

type MockResponse struct {
	// StatusCode is the HTTP status code to return (default: 200)
	StatusCode int
	// Body is the response body content (used if BodyFunc is nil)
	Body string
	// BodyFunc is a function that generates the body dynamically based on the request
	// If set, this takes precedence over Body
	BodyFunc func(*http.Request) string
	// Headers are the HTTP headers to include in the response
	Headers http.Header
	// Delay simulates network latency before returning the response
	Delay time.Duration
	// Error simulates a network error
	Error error
}

type MockTransport struct {
	// contains filtered or unexported fields
}

type NavigationTextFilter struct {
	// contains filtered or unexported fields
}

type NoisePatternFilter struct{}

type OnCrawlCompleteFunc func(result *CrawlResult)

type OnPageCrawledFunc func(*PageResult)

type OnResourceVisitFunc func(*ResourceResult)

type OnURLDiscoveredFunc func(url string) URLAction

type PageMetadata struct {
	Status      int
	Title       string
	ContentType string
}

type PageResult struct {
	// URL is the URL that was crawled
	URL string
	// Status is the HTTP status code (e.g., 200, 404, 500)
	Status int
	// Title is the page title extracted from the <title> tag (for HTML pages)
	Title string
	// MetaDescription is the content of the <meta name="description"> tag
	MetaDescription string
	// H1 is the text of the first <h1> tag on the page
	H1 string
	// H2 is the text of the first <h2> tag on the page
	H2 string
	// CanonicalURL is the canonical URL specified in <link rel="canonical">
	CanonicalURL string
	// WordCount is the approximate word count of visible text on the page
	WordCount int
	// Indexable indicates if search engines can index this page
	// Values: "Yes", "No", or "-" for non-HTML resources
	Indexable string
	// ContentType is the Content-Type header value (e.g., "text/html", "application/json")
	ContentType string
	// Error contains any error message if the crawl failed, empty otherwise
	Error string
	// Links contains all outbound links from this page (internal and external)
	Links *Links
	// ContentHash is the hash of the normalized page content (empty if content hashing is disabled)
	ContentHash string
	// IsDuplicateContent indicates if this content hash has been seen before on a different URL
	IsDuplicateContent bool
	// contains filtered or unexported fields
}

type RedirectCallback func(req *http.Request, via []*http.Request) error

type RedirectResponse struct {
	// URL is the URL that issued the redirect
	URL string
	// StatusCode is the HTTP redirect status code (301, 302, 307, 308, etc.)
	StatusCode int
	// Headers contains the redirect response headers
	Headers *http.Header
	// Location is the value of the Location header (where the redirect points to)
	Location string
}

type RenderingConfig struct {
	// InitialWaitMs is the initial wait time after page load (in milliseconds)
	// This allows time for JavaScript frameworks (React/Next.js) to hydrate
	// Default: 1500ms (matching ScreamingFrog's 5s AJAX timeout approach)
	InitialWaitMs int
	// ScrollWaitMs is the wait time after scrolling to bottom (in milliseconds)
	// This triggers lazy-loaded images and content
	// Default: 2000ms
	ScrollWaitMs int
	// FinalWaitMs is the final wait time before capturing HTML (in milliseconds)
	// This allows remaining network requests and DOM updates to complete
	// Default: 1000ms
	FinalWaitMs int
}

type Request struct {
	// URL is the parsed URL of the HTTP request
	URL *url.URL
	// Headers contains the Request's HTTP headers
	Headers *http.Header
	// the Host header
	Host string
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Depth is the number of the parents of the request
	Depth int
	// Method is the HTTP method of the request
	Method string
	// Body is the request body which is used on POST/PUT requests
	Body io.Reader
	// ResponseCharacterencoding is the character encoding of the response body.
	// Leave it blank to allow automatic character encoding of the response body.
	// It is empty by default and it can be set in OnRequest callback.
	ResponseCharacterEncoding string
	// ID is the Unique identifier of the request
	ID uint32
	// contains filtered or unexported fields
}

type RequestCallback func(*Request)

type ResourceResult struct {
	// URL is the resource URL that was visited
	URL string
	// Status is the HTTP status code (e.g., 200, 404, 500)
	Status int
	// ContentType is the MIME type (e.g., "image/png", "text/css", "application/javascript")
	ContentType string
	// Error contains any error message if the visit failed, empty otherwise
	Error string
}

type ResourceValidationConfig struct {
	// Enabled turns on resource validation
	// Default: true
	Enabled bool
	// ResourceTypes specifies which resource types to check
	// Options: "image", "script", "stylesheet", "video", "audio", "iframe"
	// Default: ["image", "script", "stylesheet"]
	// Empty array = check all types
	ResourceTypes []string
	// CheckExternal controls whether to check external resources
	// Default: true
	CheckExternal bool
}

type Response struct {
	// StatusCode is the status code of the Response
	StatusCode int
	// Body is the content of the Response
	Body []byte
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Request is the Request object of the response
	Request *Request
	// Headers contains the Response's HTTP headers
	Headers *http.Header
	// Trace contains the HTTPTrace for the request. Will only be set by the
	// collector if Collector.TraceHTTP is set to true.
	Trace *HTTPTrace
	// RedirectChain contains all intermediate redirect responses for this request.
	// For a redirect chain A→B→C, this contains responses for A and B (not C, which is the main Response).
	// Each entry has the actual redirect status code (301, 302, 307, 308) and headers.
	RedirectChain []*RedirectResponse
}

type ResponseCallback func(*Response)

type ResponseHeadersCallback func(*Response)

type ScrapedCallback func(*Response)

type Sitemap struct {
	XMLName xml.Name `xml:"urlset"`
	URL     []URL    `xml:"url"`
}

type StopwordsScorer struct {
	// contains filtered or unexported fields
}

type URL struct {
	Loc        string  `xml:"loc"`
	LastMod    string  `xml:"lastmod"`
	ChangeFreq string  `xml:"changefreq"`
	Priority   float32 `xml:"priority"`
}

type URLAction string

type URLDiscoveryRequest struct {
	URL       string   // The discovered URL
	Source    string   // Discovery source: "initial", "sitemap", "spider", "network", "resource"
	ParentURL string   // URL where this was discovered (for spider/network)
	Depth     int      // Crawl depth
	Context   *Context // Request context for passing metadata
}

type WorkerPool struct {
	// contains filtered or unexported fields
}

type XMLCallback func(*XMLElement)

type XMLElement struct {
	// Name is the name of the tag
	Name string
	Text string

	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the DOM object of the page. DOM is relative
	// to the current XMLElement and is either a html.Node or xmlquery.Node
	// based on how the XMLElement was created.
	DOM interface{}

	// Index stores the position of the current element within all the elements matched by an OnXML callback
	Index int
	// contains filtered or unexported fields
}