crawler

May 30, 2026

Changes in this version

+ func ProxyLeaseCandidatesExhaustedErrorWithDiagnostics(candidateCount int, diagnostics ...ProxyFailureDiagnostic) error

+ type ProxyFailureDiagnostic struct

+ Kind ProxyFailureKind

+ Reason string

+ StatusCode int

+ type ProxyFailureKind string

+ const ProxyFailureKindChallenge

+ const ProxyFailureKindProviderAccount

+ const ProxyFailureKindProviderAuth

+ const ProxyFailureKindStatus

+ const ProxyFailureKindTransport

+ const ProxyFailureKindUnknown

type ProxyLeaseAttemptScope

+ func (scope *ProxyLeaseAttemptScope) FailureDiagnostic(lease ProxyLease) (ProxyFailureDiagnostic, bool)

+ func (scope *ProxyLeaseAttemptScope) ReportFailureWithDiagnostic(lease ProxyLease, diagnostic ProxyFailureDiagnostic)

type ProxyLeaseSelector

+ func (selector *ProxyLeaseSelector) ReportCriticalFailureWithDiagnostic(lease ProxyLease, diagnostic ProxyFailureDiagnostic)

+ func (selector *ProxyLeaseSelector) ReportFailureWithDiagnostic(lease ProxyLease, diagnostic ProxyFailureDiagnostic)

+ func (selector *ProxyLeaseSelector) ReportProxyRetryWithDiagnostic(lease ProxyLease, diagnostic ProxyFailureDiagnostic)

type RetryDecision

+ ProxyFailureKind ProxyFailureKind

+ ProxyFailureReason string

May 19, 2026

Changes in this version

type ProxyLeaseSelector

+ func (selector *ProxyLeaseSelector) ReportProxyRetry(lease ProxyLease)

type ProxyRotationSelector

+ func (selector *ProxyRotationSelector) RecordProxyRetry(selection ProxySelection)

type RetryDecision

+ ProxyFailureSeverity RetryProxyFailureSeverity

+ type RetryProxyFailureSeverity uint8

+ const RetryProxyFailureSeverityCritical

+ const RetryProxyFailureSeverityNormal

May 12, 2026

Changes in this version

type ProxyLeaseSelector

+ func NewProxyLeaseSelectorWithOptions(configs []ProxyRotationProviderConfig, optionList ...ProxyLeaseSelectorOption) (*ProxyLeaseSelector, error)

+ func (selector *ProxyLeaseSelector) IsAvailable(proxyURL string) bool

+ func (selector *ProxyLeaseSelector) RecordCriticalFailure(proxyURL string)

+ func (selector *ProxyLeaseSelector) ReportCriticalFailure(lease ProxyLease)

+ func (selector *ProxyLeaseSelector) Select(request *http.Request) (*url.URL, error)

+ type ProxyLeaseSelectorOption func(*proxyLeaseSelectorOptions)

+ func ProxyLeaseSelectorCircuitBreaker(enabled bool) ProxyLeaseSelectorOption

+ func ProxyLeaseSelectorClock(now func() time.Time) ProxyLeaseSelectorOption

+ func ProxyLeaseSelectorLogger(logger Logger) ProxyLeaseSelectorOption

+ func ProxyLeaseSelectorStartProvider(index int) ProxyLeaseSelectorOption

type ProxyRotationSelector

+ func (selector *ProxyRotationSelector) RecordProxyCriticalFailure(selection ProxySelection)

type ScraperConfig

+ ProxyLeaseSelector *ProxyLeaseSelector

+ func (cfg ScraperConfig) ProxyCandidateCount() int

May 1, 2026

Changes in this version

+ var ErrProxyLeaseCandidatesExhausted = errors.New("crawler: proxy lease candidates exhausted")

+ func ProxyLeaseCandidatesExhaustedError(candidateCount int) error

+ type ProxyLeaseAttemptScope struct

+ func NewProxyLeaseAttemptScope() *ProxyLeaseAttemptScope

+ func (scope *ProxyLeaseAttemptScope) AcquireRequired(selector *ProxyLeaseSelector) (ProxyLease, error)

+ func (scope *ProxyLeaseAttemptScope) Exhausted(candidateCount int) bool

+ func (scope *ProxyLeaseAttemptScope) Failed(lease ProxyLease) bool

+ func (scope *ProxyLeaseAttemptScope) ReportFailure(lease ProxyLease)

type ProxyLeaseSelector

+ func (selector *ProxyLeaseSelector) CandidateCount() int

Apr 30, 2026

Changes in this version

+ var ErrProxyLeaseUnavailable = errors.New("crawler: proxy lease unavailable")

+ type ProxyLease struct

+ Generation uint64

+ ProviderName string

+ ProxyURL string

+ UserName string

+ func (lease ProxyLease) Valid() bool

+ type ProxyLeaseSelector struct

+ func NewProxyLeaseSelector(configs []ProxyRotationProviderConfig) (*ProxyLeaseSelector, error)

+ func (selector *ProxyLeaseSelector) Acquire() ProxyLease

+ func (selector *ProxyLeaseSelector) AcquireForRequest(request *http.Request) (ProxyLease, error)

+ func (selector *ProxyLeaseSelector) AcquireRequired() (ProxyLease, error)

+ func (selector *ProxyLeaseSelector) RecordFailure(proxyURL string)

+ func (selector *ProxyLeaseSelector) RecordSuccess(proxyURL string)

+ func (selector *ProxyLeaseSelector) Release(lease ProxyLease)

+ func (selector *ProxyLeaseSelector) ReportFailure(lease ProxyLease)

+ func (selector *ProxyLeaseSelector) ReportSuccess(lease ProxyLease)

+ func (selector *ProxyLeaseSelector) SelectionForProxyURL(proxyURL string) (ProxySelection, bool)

Apr 28, 2026

Changes in this version

+ const BrowserModeDirect

+ const BrowserModeHTTPFetchAuth

+ const BrowserModeSOCKSForwarder

+ const DefaultBrowserStealthScript

+ const DefaultHTTPTimeout

+ func AttachProxySelection(request *http.Request, selection ProxySelection)

+ func DefaultBrowserUserAgent(execPath string) string

+ func DetectBrowserChromeVersion(execPath string) string

+ func HTTPTimeoutOrDefault(timeout time.Duration) time.Duration

+ func IsSOCKSProxy(rawProxyURL string) bool

+ func NewHTTPClient(httpProfile HTTPProfile, timeout time.Duration) (*http.Client, error)

+ type BrowserConfig = browsertransport.Config

+ type BrowserLaunchOptions = browsertransport.LaunchOptions

+ type BrowserMode = browsertransport.BrowserMode

+ type BrowserPageRequest = browsertransport.PageRequest

+ type BrowserProfile = browsertransport.BrowserProfile

+ func InferBrowserProfile(rawProxyURL string, ignoreCertErrors bool) (BrowserProfile, error)

+ type BrowserResult = browsertransport.Result

+ func RenderBrowserPage(ctx context.Context, targetURL string, config BrowserConfig) (*BrowserResult, error)

+ func RenderBrowserPages(ctx context.Context, targetURLs []string, config BrowserConfig) ([]*BrowserResult, []error)

+ type BrowserSession = browsertransport.Session

+ func NewBrowserSession(ctx context.Context, browserProfile BrowserProfile, ...) (*BrowserSession, error)

+ type BrowserTabOptions = browsertransport.TabOptions

+ type HTTPProfile = httptransport.Profile

+ func InferHTTPProfile(rawProxyURL string, ignoreCertErrors bool) (HTTPProfile, error)

+ func NormalizeHTTPProfile(httpProfile HTTPProfile) (HTTPProfile, error)

+ type ProxyRotationProviderConfig struct

+ Name string

+ Users []ProxyRotationUserConfig

+ type ProxyRotationSelector struct

+ func NewProxyRotationSelector(configs []ProxyRotationProviderConfig) (*ProxyRotationSelector, error)

+ func (selector *ProxyRotationSelector) RecordFailure(proxyURL string)

+ func (selector *ProxyRotationSelector) RecordProxyFailure(selection ProxySelection)

+ func (selector *ProxyRotationSelector) RecordProxySuccess(selection ProxySelection)

+ func (selector *ProxyRotationSelector) RecordSuccess(proxyURL string)

+ func (selector *ProxyRotationSelector) Select(request *http.Request) (*url.URL, error)

+ func (selector *ProxyRotationSelector) SelectionForProxyURL(proxyURL string) (ProxySelection, bool)

+ type ProxyRotationUserConfig struct

+ Name string

+ URL string

+ type ProxySelection struct

+ Generation uint64

+ ProviderName string

+ ProxyURL string

+ UserName string

+ func SelectedProxySelection(request *http.Request) (ProxySelection, bool)

Mar 23, 2026

Changes in this version

+ type ResponseHandlerRuntimeBinder interface

+ BindRuntime func(collector *colly.Collector, filePersister FilePersister, ...)

Mar 23, 2026

Changes in this version

type Logger

+ func EnsureLogger(logger Logger) Logger

Mar 23, 2026

Changes in this version

+ func SetPackageLogger(logger Logger)

type Config

+ CookieGenerator CookieGenerator

+ PlatformID string

+ RequestHeaders RequestHeaderProvider

+ RequestHook RequestHook

+ RuleEvaluator RuleEvaluator

+ type CookieGenerator func(domain string) []*http.Cookie

+ type NoopResponseHandler struct

+ func (NoopResponseHandler) AfterEvaluation(*colly.Response, *goquery.Document, *Result)

+ func (NoopResponseHandler) BeforeEvaluation(*colly.Response, *goquery.Document)

+ func (NoopResponseHandler) HandleBinaryResponse(*colly.Response, string, string) bool

type PlatformConfig

+ SkipRulesOnRedirect bool

type PlatformHooks

+ ExtractDOMTitle func(document *goquery.Document) string

+ InferRedirect func(productID, originalURL, finalURL, canonicalURL string) (redirected bool, redirectedProductID string)

+ IsContentComplete func(document *goquery.Document) bool

+ type Product struct

+ ID string

+ OriginalID string

+ OriginalURL string

+ Platform string

+ URL string

+ func NewProduct(id, platform, url string, opts ...ProductOption) (Product, error)

+ type ProductOption func(*Product)

+ func WithOriginalID(originalID string) ProductOption

+ func WithOriginalURL(originalURL string) ProductOption

+ type RequestHeaderProvider interface

+ Apply func(platformID string, request *colly.Request)

type ResponseHandler

+ AfterEvaluation func(resp *colly.Response, document *goquery.Document, result *Result)

+ BeforeEvaluation func(resp *colly.Response, document *goquery.Document)

+ HandleBinaryResponse func(resp *colly.Response, productID string, fileExtension string) bool

+ type ResponseProcessor interface

+ SendFinalResult func(resp *colly.Response, success bool, errorText string)

+ SetResponseHandlers func(handlers []ResponseHandler)

+ SetResultCallback func(callback func(*colly.Response))

+ Setup func(collector *colly.Collector)

type Result

+ ConfiguredVerifierCount int

+ OriginalProductID string

+ OriginalURL string

+ ProductID string

+ ProductPlatform string

+ ProductTitle string

+ ProductURL string

+ Progress int

+ RuleResults []RuleResult

+ ScoreOverride *int

+ func (result Result) CalculateScore(configuredVerifierCount int) int

+ func (result Result) IsNotFound() bool

+ func (result Result) IsNotRetryable() bool

+ type RuleEvaluation struct

+ ConfiguredVerifier int

+ Passed bool

+ RuleResults []RuleResult

+ type RuleEvaluator interface

+ ConfiguredVerifierCount func() int

+ Evaluate func(productID string, document *goquery.Document) (RuleEvaluation, error)

+ type RuleResult struct

+ Description string

+ ID string

+ Message string

+ Passed bool

+ ReportingOrder int

+ VerificationResults []VerificationResult

+ type ServiceHook interface

+ AfterInit func(collector *colly.Collector, transport http.RoundTripper)

+ AfterRun func()

+ BeforeRun func(ctx context.Context)

+ type ServiceOption func(*Service)

+ func WithResponseHandlers(handlers ...ResponseHandler) ServiceOption

+ func WithServiceHook(hook ServiceHook) ServiceOption

+ type VerificationResult struct

+ Description string

+ ID string

+ IncludeValue bool

+ Message string

+ Passed bool

+ ReportingOrder int

+ Value string

Mar 23, 2026

Changes in this version

+ const CtxCanonicalURLKey

+ const CtxErrorKey

+ const CtxEvaluationKey

+ const CtxFinalURLKey

+ const CtxHTTPStatusCodeKey

+ const CtxInitialURLKey

+ const CtxNotFoundKey

+ const CtxRedirectedKey

+ const CtxRunContextKey

+ const CtxTargetCategoryKey

+ const CtxTargetIDKey

+ const CtxTargetURLKey

+ const CtxTitleKey

+ const HTMLExtension

+ const RetriedFlagKey

+ const RetryCountKey

+ const TitleNotFound

+ const UnknownURL

+ func DescribeProxyForLog(rawProxyURL string) string

+ func ExtractCanonicalURL(doc *goquery.Document) string

+ func ExtractTitle(doc *goquery.Document) string

+ func GetContextValue(ctx *colly.Context, key, fallback string) string

+ func GetRetryAttempt(response *colly.Response) int

+ func GetTargetCategoryFromContext(resp *colly.Response) string

+ func GetTargetIDFromContext(resp *colly.Response) string

+ func GetTargetURLFromContext(resp *colly.Response) string

+ func NewContextAwareTransport(base http.RoundTripper, ctxProvider func() context.Context) http.RoundTripper

+ func NewHTTPTransport(insecureSkipVerify bool, requestTimeout time.Duration) *http.Transport

+ func NewPanicSafeTransport(base http.RoundTripper, logger Logger) http.RoundTripper

+ func NewProxyRotator(rawProxies []string, tracker ProxyHealth, logger Logger) (colly.ProxyFunc, error)

+ func ParseHTMLResponse(body []byte) (*goquery.Document, error)

+ func RecordProxyFailure(tracker ProxyHealth, resp *colly.Response)

+ func SanitizeProxyURL(rawProxyURL string) string

+ func SetupErrorHandling(collector *colly.Collector, handler ResponseHandler, retryHandler RetryHandler, ...)

type Config

+ Category string

+ FilePersister FilePersister

+ OutputDirectory string

+ Platform PlatformConfig

+ PlatformHooks PlatformHooks

+ ResponseHandler ResponseHandler

+ RunFolder string

+ Scraper ScraperConfig

+ type DefaultResponseHandler struct

+ func NewDefaultResponseHandler(cfg Config, retryHandler RetryHandler, proxyTracker ProxyHealth, ...) *DefaultResponseHandler

+ func (h *DefaultResponseHandler) SendResult(resp *colly.Response, success bool, errorMessage string)

+ func (h *DefaultResponseHandler) SetSlotReleaser(releaser func(*colly.Response))

+ func (h *DefaultResponseHandler) Setup(collector *colly.Collector)

+ type FilePersister interface

+ Close func() error

+ Save func(targetID, fileName string, content []byte) error

+ func NewBackgroundFilePersister(delegate FilePersister, workerCount, bufferSize int, logger Logger) FilePersister

+ func NewDirectoryFilePersister(rootDirectory, category, runFolder string) FilePersister

+ type PlatformConfig struct

+ AllowedDomains []string

+ CookieDomains []string

+ func (cfg PlatformConfig) Validate() error

+ type PlatformHooks interface

+ NormalizeTitle func(title string) string

+ ShouldRetry func(title string, document *goquery.Document) RetryDecision

+ type ProxyHealth interface

+ IsAvailable func(proxy string) bool

+ RecordCriticalFailure func(proxy string)

+ RecordFailure func(proxy string)

+ RecordSuccess func(proxy string)

+ func NewProxyHealthTracker(values []string, logger Logger) ProxyHealth

+ type RequestConfigurator interface

+ Configure func(collector *colly.Collector)

+ func NewRequestConfigurator(cfg Config, logger Logger) RequestConfigurator

+ type ResponseHandler interface

+ SendResult func(resp *colly.Response, success bool, errorMessage string)

+ SetSlotReleaser func(releaser func(*colly.Response))

+ Setup func(collector *colly.Collector)

type Result

+ CanonicalURL string

+ ProxyURL string

+ TargetID string

+ TargetURL string

+ type RetryDecision struct

+ ExhaustionBehavior RetryExhaustionBehavior

+ LogMessage string

+ Message string

+ Policy RetryPolicy

+ ShouldRetry bool

+ func (d RetryDecision) ResolvedLogMessage() string

+ type RetryExhaustionBehavior uint8

+ const RetryExhaustionBehaviorContinue

+ const RetryExhaustionBehaviorFail

+ type RetryHandler interface

+ Retry func(response *colly.Response, options RetryOptions) bool

+ func NewRetryHandler(scraper ScraperConfig, logger Logger) RetryHandler

+ type RetryOptions struct

+ LimitRetries bool

+ MaxRetries int

+ SkipDelay bool

+ type RetryPolicy uint8

+ const RetryPolicyDefault

+ const RetryPolicyRotateProxy

+ type ScraperConfig struct

+ HTTPTimeout time.Duration

+ InsecureSkipVerify bool

+ MaxDepth int

+ Parallelism int

+ ProxyCircuitBreakerEnabled bool

+ ProxyList []string

+ RateLimit time.Duration

+ RetryCount int

+ SaveFiles bool

+ func (cfg ScraperConfig) Validate() error

type Service

+ func (svc *Service) RetryHandler() RetryHandler

+ type Target struct

+ Category string

+ ID string

+ Metadata map[string]string

+ URL string

+ func NewTarget(id, category, url string, opts ...TargetOption) (Target, error)

+ func (t Target) MetadataValue(key string) string

+ type TargetOption func(*Target)

+ func WithMetadata(key, value string) TargetOption

Mar 22, 2026

Changes in this version

+ type Config struct

+ AllowedDomains []string

+ CookieDomains []string

+ CookieProvider CookieProvider

+ Evaluator Evaluator

+ HTTPTimeout time.Duration

+ Headers HeaderProvider

+ Hook RequestHook

+ Logger Logger

+ MaxDepth int

+ Parallelism int

+ RateLimit time.Duration

+ RetryCount int

+ func (c Config) Validate() error

+ type CookieProvider func(domain string) []*http.Cookie

+ type Evaluation struct

+ Findings []Finding

+ type Evaluator interface

+ Evaluate func(pageID string, document *goquery.Document) (Evaluation, error)

+ type Finding struct

+ Data string

+ Description string

+ ID string

+ Message string

+ Passed bool

+ type HeaderProvider interface

+ Apply func(request *colly.Request)

+ type Logger interface

+ Debug func(format string, args ...interface{})

+ Error func(format string, args ...interface{})

+ Info func(format string, args ...interface{})

+ Warning func(format string, args ...interface{})

+ type Page struct

+ Category string

+ ID string

+ URL string

+ type RequestHook interface

+ BeforeRequest func(ctx context.Context, page Page) error

+ type Result struct

+ Category string

+ Document *goquery.Document

+ ErrorMessage string

+ FinalURL string

+ Findings []Finding

+ HTTPStatusCode int

+ PageID string

+ PageURL string

+ Success bool

+ Title string

+ type Service struct

+ func NewService(cfg Config, results chan<- *Result) (*Service, error)

+ func (svc *Service) Run(ctx context.Context, pages []Page) error