Documentation
¶
Index ¶
- type CaptchaDetectionMiddleware
- type CloudflareDetectionMiddleware
- type HeaderRotationMiddleware
- type MiddlewareInfo
- type ProxyRotationMiddleware
- type RateLimitMiddleware
- type RequestFingerprintMiddleware
- type RequestMiddleware
- type RequestPipeline
- func (rp *RequestPipeline) Describe() string
- func (rp *RequestPipeline) Len() int
- func (rp *RequestPipeline) List() []MiddlewareInfo
- func (rp *RequestPipeline) ProcessRequest(req *types.Request) *types.Request
- func (rp *RequestPipeline) ProcessResponse(resp *types.Response) *types.Response
- func (rp *RequestPipeline) Use(mw RequestMiddleware)
- type RetryMiddleware
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type CaptchaDetectionMiddleware ¶
type CaptchaDetectionMiddleware struct {
// contains filtered or unexported fields
}
CaptchaDetectionMiddleware detects CAPTCHA pages in responses.
func NewCaptchaDetectionMiddleware ¶
func NewCaptchaDetectionMiddleware(logger *slog.Logger) *CaptchaDetectionMiddleware
NewCaptchaDetectionMiddleware creates a CAPTCHA detection middleware.
func (*CaptchaDetectionMiddleware) Name ¶
func (m *CaptchaDetectionMiddleware) Name() string
func (*CaptchaDetectionMiddleware) Priority ¶
func (m *CaptchaDetectionMiddleware) Priority() int
func (*CaptchaDetectionMiddleware) ProcessRequest ¶
func (m *CaptchaDetectionMiddleware) ProcessRequest(req *types.Request) *types.Request
func (*CaptchaDetectionMiddleware) ProcessResponse ¶
func (m *CaptchaDetectionMiddleware) ProcessResponse(resp *types.Response) *types.Response
type CloudflareDetectionMiddleware ¶
type CloudflareDetectionMiddleware struct {
// contains filtered or unexported fields
}
CloudflareDetectionMiddleware detects Cloudflare challenges.
func NewCloudflareDetectionMiddleware ¶
func NewCloudflareDetectionMiddleware(logger *slog.Logger) *CloudflareDetectionMiddleware
NewCloudflareDetectionMiddleware creates a Cloudflare detection middleware.
func (*CloudflareDetectionMiddleware) Name ¶
func (m *CloudflareDetectionMiddleware) Name() string
func (*CloudflareDetectionMiddleware) Priority ¶
func (m *CloudflareDetectionMiddleware) Priority() int
func (*CloudflareDetectionMiddleware) ProcessRequest ¶
func (m *CloudflareDetectionMiddleware) ProcessRequest(req *types.Request) *types.Request
func (*CloudflareDetectionMiddleware) ProcessResponse ¶
func (m *CloudflareDetectionMiddleware) ProcessResponse(resp *types.Response) *types.Response
type HeaderRotationMiddleware ¶
type HeaderRotationMiddleware struct {
// contains filtered or unexported fields
}
HeaderRotationMiddleware rotates request headers to mimic different browsers.
func NewHeaderRotationMiddleware ¶
func NewHeaderRotationMiddleware() *HeaderRotationMiddleware
NewHeaderRotationMiddleware creates a header rotation middleware.
func (*HeaderRotationMiddleware) Name ¶
func (m *HeaderRotationMiddleware) Name() string
func (*HeaderRotationMiddleware) Priority ¶
func (m *HeaderRotationMiddleware) Priority() int
func (*HeaderRotationMiddleware) ProcessRequest ¶
func (m *HeaderRotationMiddleware) ProcessRequest(req *types.Request) *types.Request
func (*HeaderRotationMiddleware) ProcessResponse ¶
func (m *HeaderRotationMiddleware) ProcessResponse(resp *types.Response) *types.Response
type MiddlewareInfo ¶
MiddlewareInfo holds summary information about a middleware.
type ProxyRotationMiddleware ¶
type ProxyRotationMiddleware struct {
// contains filtered or unexported fields
}
ProxyRotationMiddleware integrates with proxy manager for per-request proxy selection.
func NewProxyRotationMiddleware ¶
func NewProxyRotationMiddleware(proxies []string, rotation string) *ProxyRotationMiddleware
NewProxyRotationMiddleware creates a proxy rotation middleware.
func (*ProxyRotationMiddleware) Name ¶
func (m *ProxyRotationMiddleware) Name() string
func (*ProxyRotationMiddleware) Priority ¶
func (m *ProxyRotationMiddleware) Priority() int
func (*ProxyRotationMiddleware) ProcessRequest ¶
func (m *ProxyRotationMiddleware) ProcessRequest(req *types.Request) *types.Request
func (*ProxyRotationMiddleware) ProcessResponse ¶
func (m *ProxyRotationMiddleware) ProcessResponse(resp *types.Response) *types.Response
type RateLimitMiddleware ¶
type RateLimitMiddleware struct {
// contains filtered or unexported fields
}
RateLimitMiddleware enforces global rate limiting across all requests.
func NewRateLimitMiddleware ¶
func NewRateLimitMiddleware(requestsPerSecond float64) *RateLimitMiddleware
NewRateLimitMiddleware creates a rate limiter with the given interval between requests.
func (*RateLimitMiddleware) Name ¶
func (m *RateLimitMiddleware) Name() string
func (*RateLimitMiddleware) Priority ¶
func (m *RateLimitMiddleware) Priority() int
func (*RateLimitMiddleware) ProcessRequest ¶
func (m *RateLimitMiddleware) ProcessRequest(req *types.Request) *types.Request
func (*RateLimitMiddleware) ProcessResponse ¶
func (m *RateLimitMiddleware) ProcessResponse(resp *types.Response) *types.Response
type RequestFingerprintMiddleware ¶
type RequestFingerprintMiddleware struct {
// contains filtered or unexported fields
}
RequestFingerprintMiddleware diversifies request fingerprints to avoid detection.
func NewRequestFingerprintMiddleware ¶
func NewRequestFingerprintMiddleware() *RequestFingerprintMiddleware
NewRequestFingerprintMiddleware creates a fingerprint diversification middleware.
func (*RequestFingerprintMiddleware) Name ¶
func (m *RequestFingerprintMiddleware) Name() string
func (*RequestFingerprintMiddleware) Priority ¶
func (m *RequestFingerprintMiddleware) Priority() int
func (*RequestFingerprintMiddleware) ProcessRequest ¶
func (m *RequestFingerprintMiddleware) ProcessRequest(req *types.Request) *types.Request
func (*RequestFingerprintMiddleware) ProcessResponse ¶
func (m *RequestFingerprintMiddleware) ProcessResponse(resp *types.Response) *types.Response
type RequestMiddleware ¶
type RequestMiddleware interface {
// Name returns a human-readable identifier.
Name() string
// Priority controls execution order (lower = runs first).
Priority() int
// ProcessRequest is called before the fetcher. Modify the request or return nil to drop.
ProcessRequest(req *types.Request) *types.Request
// ProcessResponse is called after the fetcher. Modify or inspect the response.
ProcessResponse(resp *types.Response) *types.Response
}
RequestMiddleware intercepts requests before/after fetching. This is the core framework extension point — users plug into the request lifecycle to add proxy rotation, retries, anti-bot measures, etc.
Example:
type MyMiddleware struct{}
func (m *MyMiddleware) Name() string { return "my_middleware" }
func (m *MyMiddleware) Priority() int { return 100 }
func (m *MyMiddleware) ProcessRequest(req *types.Request) *types.Request {
req.Headers.Set("X-Custom", "value")
return req
}
func (m *MyMiddleware) ProcessResponse(resp *types.Response) *types.Response {
return resp
}
type RequestPipeline ¶
type RequestPipeline struct {
// contains filtered or unexported fields
}
RequestPipeline chains request middlewares in priority order.
func DefaultRequestPipeline ¶
func DefaultRequestPipeline(logger *slog.Logger) *RequestPipeline
DefaultRequestPipeline creates a pipeline with all built-in middlewares.
func NewRequestPipeline ¶
func NewRequestPipeline(logger *slog.Logger) *RequestPipeline
NewRequestPipeline creates a new request middleware pipeline.
func (*RequestPipeline) Describe ¶
func (rp *RequestPipeline) Describe() string
Describe returns a human-readable description of the pipeline.
func (*RequestPipeline) Len ¶
func (rp *RequestPipeline) Len() int
Len returns the number of middlewares in the pipeline.
func (*RequestPipeline) List ¶
func (rp *RequestPipeline) List() []MiddlewareInfo
List returns info about all registered middlewares.
func (*RequestPipeline) ProcessRequest ¶
func (rp *RequestPipeline) ProcessRequest(req *types.Request) *types.Request
ProcessRequest runs all middlewares on a request (before fetch).
func (*RequestPipeline) ProcessResponse ¶
func (rp *RequestPipeline) ProcessResponse(resp *types.Response) *types.Response
ProcessResponse runs all middlewares on a response (after fetch), in reverse order.
func (*RequestPipeline) Use ¶
func (rp *RequestPipeline) Use(mw RequestMiddleware)
Use adds a middleware to the pipeline (sorted by priority).
type RetryMiddleware ¶
RetryMiddleware adds intelligent retry logic with exponential backoff.
func NewRetryMiddleware ¶
func NewRetryMiddleware(maxRetries int, baseDelay time.Duration) *RetryMiddleware
func (*RetryMiddleware) Name ¶
func (m *RetryMiddleware) Name() string
func (*RetryMiddleware) Priority ¶
func (m *RetryMiddleware) Priority() int
func (*RetryMiddleware) ProcessRequest ¶
func (m *RetryMiddleware) ProcessRequest(req *types.Request) *types.Request
func (*RetryMiddleware) ProcessResponse ¶
func (m *RetryMiddleware) ProcessResponse(resp *types.Response) *types.Response
Source Files