crawler

type BrowserConfig = browsertransport.Config

type BrowserLaunchOptions = browsertransport.LaunchOptions

type BrowserMode = browsertransport.BrowserMode

type BrowserPageRequest = browsertransport.PageRequest

type BrowserProfile = browsertransport.BrowserProfile

type BrowserResult = browsertransport.Result

type BrowserSession = browsertransport.Session

type BrowserTabOptions = browsertransport.TabOptions

type Config struct {
	// PlatformID identifies the target platform (for example "AMZN").
	PlatformID string

	// Scraper controls concurrency, retries, and network behaviour.
	Scraper ScraperConfig

	// Platform holds domain-specific settings such as allowed hosts.
	Platform PlatformConfig

	// OutputDirectory is optional; when supplied and FilePersister is nil the
	// crawler will persist downloaded artifacts under this path.
	OutputDirectory string

	// RunFolder scopes persisted artifacts for a single execution.
	RunFolder string

	// RuleEvaluator produces rule findings for a fetched document. Mandatory.
	RuleEvaluator RuleEvaluator

	// CookieGenerator returns cookies for a given domain. Optional.
	CookieGenerator CookieGenerator

	// FilePersister handles file persistence. Optional; a default implementation
	// is created when OutputDirectory is set.
	FilePersister FilePersister

	// PlatformHooks customise platform-specific behaviour. Optional.
	PlatformHooks PlatformHooks

	// RequestHeaders applies custom headers before each outbound request.
	RequestHeaders RequestHeaderProvider

	// RequestHook runs before each outbound request. Optional.
	RequestHook RequestHook

	// Logger receives debug/info/warning/error logs. Optional; a no-op logger is
	// used when nil.
	Logger Logger
}

type CookieGenerator func(domain string) []*http.Cookie

type FilePersister interface {
	Save(productID, fileName string, content []byte) error
	Close() error
}

type HTTPProfile = httptransport.Profile

type Logger interface {
	Debug(format string, args ...interface{})
	Info(format string, args ...interface{})
	Warning(format string, args ...interface{})
	Error(format string, args ...interface{})
}

type NoopResponseHandler struct{}

type PlatformConfig struct {
	AllowedDomains      []string
	CookieDomains       []string
	SkipRulesOnRedirect bool
}

type PlatformHooks interface {
	NormalizeTitle(title string) string
	ShouldRetry(title string, document *goquery.Document) RetryDecision
	ExtractDOMTitle(document *goquery.Document) string
	IsContentComplete(document *goquery.Document) bool
	InferRedirect(productID, originalURL, finalURL, canonicalURL string) (redirected bool, redirectedProductID string)
}

type Product struct {
	ID          string
	Platform    string
	URL         string
	OriginalID  string
	OriginalURL string
}

type ProductOption func(*Product)

type ProxyFailureDiagnostic struct {
	Kind       ProxyFailureKind
	Reason     string
	StatusCode int
}

type ProxyFailureKind string

type ProxyLease struct {
	ProviderName string
	UserName     string
	ProxyURL     string
	Generation   uint64
}

type ProxyLeaseAttemptScope struct {
	// contains filtered or unexported fields
}

type ProxyLeaseSelector struct {
	// contains filtered or unexported fields
}

type ProxyLeaseSelectorOption func(*proxyLeaseSelectorOptions)

type ProxyRotationProviderConfig struct {
	Name  string
	Users []ProxyRotationUserConfig
}

type ProxyRotationSelector struct {
	// contains filtered or unexported fields
}

type ProxyRotationUserConfig struct {
	Name string
	URL  string
}

type ProxySelection = ProxyLease

type RequestConfigurator interface {
	Configure(collector *colly.Collector)
}

type RequestHeaderProvider interface {
	Apply(platformID string, request *colly.Request)
}

type RequestHook interface {
	BeforeRequest(ctx context.Context, product Product) error
}

type ResponseHandler interface {
	// HandleBinaryResponse processes non-HTML responses (e.g. images).
	// Return true to indicate the response was handled and stop further processing.
	HandleBinaryResponse(resp *colly.Response, productID string, fileExtension string) bool

	// BeforeEvaluation is called after HTML parsing and content validation but
	// before rule evaluation. Use for tasks like image retrieval.
	BeforeEvaluation(resp *colly.Response, document *goquery.Document)

	// AfterEvaluation is called once the processor has enough context to build
	// the final result, before that result is emitted. Use for tasks like
	// discoverability probing or file persistence.
	AfterEvaluation(resp *colly.Response, document *goquery.Document, result *Result)
}

type ResponseHandlerRuntimeBinder interface {
	BindRuntime(collector *colly.Collector, filePersister FilePersister, retryHandler RetryHandler)
}

type ResponseProcessor interface {
	Setup(collector *colly.Collector)
	SendFinalResult(resp *colly.Response, success bool, errorText string)
	SetResultCallback(callback func(*colly.Response))
	SetResponseHandlers(handlers []ResponseHandler)
}

type Result struct {
	ProductID               string       `json:"product_id" csv:"ID"`
	OriginalProductID       string       `json:"original_product_id,omitempty" csv:"OriginalID"`
	OriginalURL             string       `json:"original_url,omitempty" csv:""`
	FinalURL                string       `json:"final_url,omitempty" csv:""`
	CanonicalURL            string       `json:"canonical_url,omitempty" csv:""`
	ProxyURL                string       `json:"proxy_url,omitempty" csv:"ProxyURL"`
	ProductURL              string       `json:"product_url" csv:"URL"`
	ProductTitle            string       `json:"product_title,omitempty" csv:"Title"`
	ProductPlatform         string       `json:"product_platform"`
	Success                 bool         `json:"success"`
	ErrorMessage            string       `json:"error_message,omitempty" csv:"ErrorMessage"`
	HTTPStatusCode          int          `json:"http_status_code,omitempty" csv:"HTTPStatusCode"`
	Progress                int          `json:"progress,omitempty"`
	RuleResults             []RuleResult `json:"results,omitempty"`
	ConfiguredVerifierCount int          `json:"-" csv:"-"`
	ScoreOverride           *int         `json:"-" csv:"-"`
}

type RetryDecision struct {
	ShouldRetry          bool
	Message              string
	LogMessage           string
	Policy               RetryPolicy
	ExhaustionBehavior   RetryExhaustionBehavior
	ProxyFailureSeverity RetryProxyFailureSeverity
	ProxyFailureKind     ProxyFailureKind
	ProxyFailureReason   string
}

type RetryExhaustionBehavior uint8

type RetryHandler interface {
	Retry(response *colly.Response, options RetryOptions) bool
}

type RetryOptions struct {
	SkipDelay    bool
	LimitRetries bool
	MaxRetries   int
}

type RetryPolicy uint8

type RetryProxyFailureSeverity uint8

type RuleEvaluation struct {
	Passed             bool
	ConfiguredVerifier int
	RuleResults        []RuleResult
}

type RuleEvaluator interface {
	Evaluate(productID string, document *goquery.Document) (RuleEvaluation, error)
	ConfiguredVerifierCount() int
}

type RuleResult struct {
	ID                  string               `json:"id,omitempty" csv:"-"`
	Description         string               `json:"description" csv:"Description,keyValue"`
	Passed              bool                 `json:"passed" csv:"passed"`
	ReportingOrder      int                  `json:"reporting_order" csv:"-"`
	Message             string               `json:"message" csv:"message"`
	VerificationResults []VerificationResult `json:"verification_results"`
}

type ScraperConfig struct {
	MaxDepth                   int
	Parallelism                int
	RetryCount                 int
	HTTPTimeout                time.Duration
	InsecureSkipVerify         bool
	RateLimit                  time.Duration
	ProxyList                  []string
	ProxyLeaseSelector         *ProxyLeaseSelector
	SaveFiles                  bool
	ProxyCircuitBreakerEnabled bool
}

type Service struct {
	// contains filtered or unexported fields
}

type ServiceHook interface {
	// AfterInit is called after the collector, transport, and response processor
	// are fully wired. Use for binding domain-specific network configuration.
	AfterInit(collector *colly.Collector, transport http.RoundTripper)

	// BeforeRun is called before the product visit loop starts.
	BeforeRun(ctx context.Context)

	// AfterRun is called after all products have been visited and the collector
	// has finished. Use for cleanup (e.g. stopping image converter workers).
	AfterRun()
}

type ServiceOption func(*Service)

type VerificationResult struct {
	ID             string `json:"id,omitempty" csv:"-"`
	Description    string `json:"description" csv:"Description,keyValue"`
	Passed         bool   `json:"passed" csv:"passed"`
	Message        string `json:"message" csv:"message"`
	Value          string `json:"value" csv:"value"`
	ReportingOrder int    `json:"reporting_order"`
	IncludeValue   bool   `json:"-"`
}