crawler

type CacheCheckAttempt struct {
	Attempt     int    `json:"attempt"`
	CacheStatus string `json:"cache_status"`
	Delay       int    `json:"delay_ms"`
	// Diagnostics duplicates attempt metadata for backward-compatible probe history.
	Diagnostics *ProbeDiagnostics `json:"diagnostics,omitempty"`
}

type CacheMetadata struct {
	HeaderSource     string `json:"header_source,omitempty"`
	RawValue         string `json:"raw_value,omitempty"`
	NormalisedStatus string `json:"normalised_status,omitempty"`
	Age              string `json:"age,omitempty"`
	CacheControl     string `json:"cache_control,omitempty"`
	Vary             string `json:"vary,omitempty"`
	CacheStatus      string `json:"cache_status,omitempty"`
	CFCacheStatus    string `json:"cf_cache_status,omitempty"`
	XCache           string `json:"x_cache,omitempty"`
	XCacheRemote     string `json:"x_cache_remote,omitempty"`
	XVercelCache     string `json:"x_vercel_cache,omitempty"`
	XVarnish         string `json:"x_varnish,omitempty"`
}

type Config struct {
	DefaultTimeout time.Duration // Default timeout for requests
	MaxConcurrency int           // Maximum number of concurrent requests
	RateLimit      int           // Determines request delay range: base=1s/RateLimit, range=base to 1s
	UserAgent      string        // User agent string for requests
	RetryAttempts  int           // Number of retry attempts for failed requests
	RetryDelay     time.Duration // Delay between retry attempts
	SkipCachedURLs bool          // Whether to skip URLs that are already cached (HIT)
	Port           string        // Server port
	Env            string        // Environment (development/production)
	LogLevel       string        // Logging level
	DatabaseURL    string        // Database connection URL
	AuthToken      string        // Database authentication token
	SentryDSN      string        // Sentry DSN for error tracking
	FindLinks      bool          // Whether to extract links (e.g. PDFs/docs) from pages
	SkipSSRFCheck  bool          // Skip SSRF protection (for tests only, never enable in production)
}

type CrawlOptions struct {
	MaxPages    int  // Maximum pages to crawl
	Concurrency int  // Number of concurrent crawlers
	RateLimit   int  // Maximum requests per second
	Timeout     int  // Request timeout in seconds
	FollowLinks bool // Whether to follow links on crawled pages
}

type CrawlResult struct {
	URL                 string              `json:"url"`
	ResponseTime        int64               `json:"response_time"`
	StatusCode          int                 `json:"status_code"`
	Error               string              `json:"error,omitempty"`
	Warning             string              `json:"warning,omitempty"`
	CacheStatus         string              `json:"cache_status"`
	ContentType         string              `json:"content_type"`
	ContentLength       int64               `json:"content_length"`
	Headers             http.Header         `json:"headers"`
	RedirectURL         string              `json:"redirect_url"`
	Performance         PerformanceMetrics  `json:"performance"`
	Timestamp           int64               `json:"timestamp"`
	RetryCount          int                 `json:"retry_count"`
	SkippedCrawl        bool                `json:"skipped_crawl,omitempty"`
	Links               map[string][]string `json:"links,omitempty"`
	SecondResponseTime  int64               `json:"second_response_time,omitempty"`
	SecondCacheStatus   string              `json:"second_cache_status,omitempty"`
	SecondContentLength int64               `json:"second_content_length,omitempty"`
	SecondHeaders       http.Header         `json:"second_headers,omitempty"`
	SecondPerformance   *PerformanceMetrics `json:"second_performance,omitempty"`
	CacheCheckAttempts  []CacheCheckAttempt `json:"cache_check_attempts,omitempty"`
	RequestDiagnostics  *RequestDiagnostics `json:"request_diagnostics,omitempty"`
	BodySample          []byte              `json:"-"` // Truncated body for tech detection (not serialised)
	Body                []byte              `json:"-"` // Full body for storage upload (not serialised)
}

type Crawler struct {
	// contains filtered or unexported fields
}

type PerformanceMetrics struct {
	DNSLookupTime       int64 `json:"dns_lookup_time"`
	TCPConnectionTime   int64 `json:"tcp_connection_time"`
	TLSHandshakeTime    int64 `json:"tls_handshake_time"`
	TTFB                int64 `json:"ttfb"`
	ContentTransferTime int64 `json:"content_transfer_time"`
}

type ProbeDiagnostics struct {
	Attempt  int               `json:"attempt,omitempty"`
	Request  *RequestMetadata  `json:"request,omitempty"`
	Response *ResponseMetadata `json:"response,omitempty"`
	Cache    *CacheMetadata    `json:"cache,omitempty"`
	DelayMS  int               `json:"delay_ms,omitempty"`
}

type RequestAttemptDiagnostics struct {
	Request         *RequestMetadata    `json:"request,omitempty"`
	Response        *ResponseMetadata   `json:"response,omitempty"`
	RequestHeaders  http.Header         `json:"request_headers,omitempty"`
	ResponseHeaders http.Header         `json:"response_headers,omitempty"`
	Timing          *PerformanceMetrics `json:"timing,omitempty"`
	Cache           *CacheMetadata      `json:"cache,omitempty"`
}

type RequestDiagnostics struct {
	Primary   *RequestAttemptDiagnostics `json:"primary,omitempty"`
	Probes    []ProbeDiagnostics         `json:"probes,omitempty"`
	Secondary *RequestAttemptDiagnostics `json:"secondary,omitempty"`
}

type RequestMetadata struct {
	Method     string `json:"method,omitempty"`
	URL        string `json:"url,omitempty"`
	FinalURL   string `json:"final_url,omitempty"`
	Scheme     string `json:"scheme,omitempty"`
	Host       string `json:"host,omitempty"`
	Path       string `json:"path,omitempty"`
	Query      string `json:"query,omitempty"`
	Timestamp  int64  `json:"timestamp,omitempty"`
	Provenance string `json:"provenance,omitempty"`
}

type ResponseMetadata struct {
	StatusCode    int    `json:"status_code,omitempty"`
	ContentType   string `json:"content_type,omitempty"`
	ContentLength int64  `json:"content_length,omitempty"`
	RedirectURL   string `json:"redirect_url,omitempty"`
	Warning       string `json:"warning,omitempty"`
	Error         string `json:"error,omitempty"`
}

type RobotsRules struct {
	// CrawlDelay in seconds (0 means no delay specified)
	CrawlDelay int
	// Sitemaps found in robots.txt
	Sitemaps []string
	// DisallowPatterns are URL patterns that should not be crawled
	DisallowPatterns []string
	// AllowPatterns override DisallowPatterns (more specific)
	AllowPatterns []string
}

type Sitemap struct {
	XMLName xml.Name `xml:"sitemap"`
	Loc     string   `xml:"loc"`
}

type SitemapDiscoveryResult struct {
	Sitemaps    []string
	RobotsRules *RobotsRules
}

type SitemapIndex struct {
	XMLName  xml.Name  `xml:"sitemapindex"`
	Sitemaps []Sitemap `xml:"sitemap"`
}

type URL struct {
	XMLName xml.Name `xml:"url"`
	Loc     string   `xml:"loc"`
}

type URLSet struct {
	XMLName xml.Name `xml:"urlset"`
	URLs    []URL    `xml:"url"`
}