crawler

type Config struct {
	// Category identifies the target category (e.g., "AMZN", "camps").
	Category string

	// Scraper controls concurrency, retries, and network behaviour.
	Scraper ScraperConfig

	// Platform holds domain-specific settings.
	Platform PlatformConfig

	// ResponseHandler processes HTTP responses. Optional; when nil, a default
	// handler is created using the Evaluator.
	ResponseHandler ResponseHandler

	// Evaluator produces findings for a fetched document. Required when
	// ResponseHandler is nil.
	Evaluator Evaluator

	// PlatformHooks customise title normalisation and retry decisions. Optional.
	PlatformHooks PlatformHooks

	// CookieProvider returns cookies per domain. Optional.
	CookieProvider CookieProvider

	// CookieDomains lists domains for which CookieProvider is called.
	CookieDomains []string

	// FilePersister handles artifact persistence. Optional; a default
	// implementation is created when OutputDirectory is set.
	FilePersister FilePersister

	// OutputDirectory is optional; when set and FilePersister is nil the
	// crawler will persist artifacts under this path.
	OutputDirectory string

	// RunFolder scopes persisted artifacts for a single execution.
	RunFolder string

	// Headers customises outbound requests. Optional.
	Headers HeaderProvider

	// Hook runs before each request. Optional.
	Hook RequestHook

	// Logger receives diagnostic messages. Optional (no-op if nil).
	Logger Logger
}

type CookieProvider func(domain string) []*http.Cookie

type DefaultResponseHandler struct {
	// contains filtered or unexported fields
}

type Evaluation struct {
	Findings []Finding
}

type Evaluator interface {
	Evaluate(targetID string, document *goquery.Document) (Evaluation, error)
}

type FilePersister interface {
	Save(targetID, fileName string, content []byte) error
	Close() error
}

type Finding struct {
	ID          string `json:"id,omitempty"`
	Description string `json:"description"`
	Passed      bool   `json:"passed"`
	Message     string `json:"message"`
	Data        string `json:"data,omitempty"`
}

type HeaderProvider interface {
	Apply(category string, request *colly.Request)
}

type Logger interface {
	Debug(format string, args ...interface{})
	Info(format string, args ...interface{})
	Warning(format string, args ...interface{})
	Error(format string, args ...interface{})
}

type PlatformConfig struct {
	AllowedDomains []string
	CookieDomains  []string
}

type PlatformHooks interface {
	NormalizeTitle(title string) string
	ShouldRetry(title string, document *goquery.Document) RetryDecision
}

type ProxyHealth interface {
	IsAvailable(proxy string) bool
	RecordSuccess(proxy string)
	RecordFailure(proxy string)
	RecordCriticalFailure(proxy string)
}

type RequestConfigurator interface {
	Configure(collector *colly.Collector)
}

type RequestHook interface {
	BeforeRequest(ctx context.Context, target Target) error
}

type ResponseHandler interface {
	Setup(collector *colly.Collector)
	SendResult(resp *colly.Response, success bool, errorMessage string)
	SetSlotReleaser(releaser func(*colly.Response))
}

type Result struct {
	TargetID       string            `json:"targetId"`
	TargetURL      string            `json:"targetUrl"`
	FinalURL       string            `json:"finalUrl,omitempty"`
	CanonicalURL   string            `json:"canonicalUrl,omitempty"`
	Category       string            `json:"category"`
	Title          string            `json:"title,omitempty"`
	Success        bool              `json:"success"`
	ErrorMessage   string            `json:"errorMessage,omitempty"`
	HTTPStatusCode int               `json:"httpStatusCode,omitempty"`
	Findings       []Finding         `json:"findings,omitempty"`
	Document       *goquery.Document `json:"-"`
	ProxyURL       string            `json:"proxyUrl,omitempty"`
}

type RetryDecision struct {
	ShouldRetry        bool
	Message            string
	LogMessage         string
	Policy             RetryPolicy
	ExhaustionBehavior RetryExhaustionBehavior
}

type RetryExhaustionBehavior uint8

type RetryHandler interface {
	Retry(response *colly.Response, options RetryOptions) bool
}

type RetryOptions struct {
	SkipDelay    bool
	LimitRetries bool
	MaxRetries   int
}

type RetryPolicy uint8

type ScraperConfig struct {
	MaxDepth                   int
	Parallelism                int
	RetryCount                 int
	HTTPTimeout                time.Duration
	InsecureSkipVerify         bool
	RateLimit                  time.Duration
	ProxyList                  []string
	ProxyCircuitBreakerEnabled bool
	SaveFiles                  bool
}

type Service struct {
	// contains filtered or unexported fields
}

type Target struct {
	ID       string
	Category string
	URL      string
	Metadata map[string]string
}

type TargetOption func(*Target)