crawler

type Config struct {
	// AllowedDomains restricts crawling to these hosts.
	AllowedDomains []string

	// Parallelism controls concurrent requests. Required, must be > 0.
	Parallelism int

	// RetryCount sets additional retry attempts per page. 0 = no retries.
	RetryCount int

	// HTTPTimeout caps each HTTP request. 0 = no timeout.
	HTTPTimeout time.Duration

	// RateLimit sets minimum delay between requests to the same domain.
	RateLimit time.Duration

	// MaxDepth limits link-following depth. 0 = no link following.
	MaxDepth int

	// Evaluator processes fetched documents. Required.
	Evaluator Evaluator

	// CookieDomains lists domains for which CookieProvider is called.
	CookieDomains []string

	// CookieProvider returns cookies per domain. Optional.
	CookieProvider CookieProvider

	// Headers customizes outbound requests. Optional.
	Headers HeaderProvider

	// Hook runs before each request. Optional.
	Hook RequestHook

	// Logger receives diagnostic messages. Optional (no-op if nil).
	Logger Logger
}

type CookieProvider func(domain string) []*http.Cookie

type Evaluation struct {
	Findings []Finding
}

type Evaluator interface {
	Evaluate(pageID string, document *goquery.Document) (Evaluation, error)
}

type Finding struct {
	ID          string `json:"id,omitempty"`
	Description string `json:"description"`
	Passed      bool   `json:"passed"`
	Message     string `json:"message"`
	Data        string `json:"data,omitempty"` // arbitrary payload (e.g., JSON-encoded extracted data)
}

type HeaderProvider interface {
	Apply(request *colly.Request)
}

type Logger interface {
	Debug(format string, args ...interface{})
	Info(format string, args ...interface{})
	Warning(format string, args ...interface{})
	Error(format string, args ...interface{})
}

type Page struct {
	ID       string // Unique identifier for this page
	Category string // Grouping label (e.g., platform, source)
	URL      string // Full URL to fetch
}

type RequestHook interface {
	BeforeRequest(ctx context.Context, page Page) error
}

type Result struct {
	PageID         string            `json:"pageId"`
	PageURL        string            `json:"pageUrl"`
	FinalURL       string            `json:"finalUrl,omitempty"`
	Category       string            `json:"category"`
	Title          string            `json:"title,omitempty"`
	Success        bool              `json:"success"`
	ErrorMessage   string            `json:"errorMessage,omitempty"`
	HTTPStatusCode int               `json:"httpStatusCode,omitempty"`
	Findings       []Finding         `json:"findings,omitempty"`
	Document       *goquery.Document `json:"-"` // parsed HTML, not serialized
}

type Service struct {
	// contains filtered or unexported fields
}