Documentation
¶
Overview ¶
Package http provides a fast HTTP client for crawling non-JS pages.
Index ¶
- type FastClient
- func (fc *FastClient) Close()
- func (fc *FastClient) Get(ctx context.Context, targetURL string) (*FastResult, error)
- func (fc *FastClient) GetBatch(ctx context.Context, urls []string, concurrency int) []*FastResult
- func (fc *FastClient) GetWithRetry(ctx context.Context, targetURL string) (*FastResult, error)
- func (fc *FastClient) Head(ctx context.Context, targetURL string) (int, string, error)
- func (fc *FastClient) IsRetryableError(err error) bool
- func (fc *FastClient) SetCookies(cookies []*http.Cookie)
- func (fc *FastClient) SetHeaders(headers map[string]string)
- func (fc *FastClient) SetRetryConfig(config errors.RetryConfig)
- type FastClientConfig
- type FastForm
- type FastResult
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type FastClient ¶
type FastClient struct {
// contains filtered or unexported fields
}
FastClient is a high-performance HTTP client optimized for crawling.
func NewFastClient ¶
func NewFastClient(config FastClientConfig) *FastClient
NewFastClient creates a new high-performance HTTP client.
func (*FastClient) Get ¶
func (fc *FastClient) Get(ctx context.Context, targetURL string) (*FastResult, error)
Get performs a fast HTTP GET request and extracts links.
func (*FastClient) GetBatch ¶
func (fc *FastClient) GetBatch(ctx context.Context, urls []string, concurrency int) []*FastResult
GetBatch performs multiple GET requests concurrently.
func (*FastClient) GetWithRetry ¶
func (fc *FastClient) GetWithRetry(ctx context.Context, targetURL string) (*FastResult, error)
GetWithRetry performs a GET request with automatic retries for transient errors.
func (*FastClient) IsRetryableError ¶
func (fc *FastClient) IsRetryableError(err error) bool
IsRetryableError checks if an error should trigger a retry.
func (*FastClient) SetCookies ¶
func (fc *FastClient) SetCookies(cookies []*http.Cookie)
SetCookies sets cookies for all requests.
func (*FastClient) SetHeaders ¶
func (fc *FastClient) SetHeaders(headers map[string]string)
SetHeaders sets custom headers for all requests.
func (*FastClient) SetRetryConfig ¶
func (fc *FastClient) SetRetryConfig(config errors.RetryConfig)
SetRetryConfig sets custom retry configuration.
type FastClientConfig ¶
type FastClientConfig struct {
Timeout time.Duration
MaxIdleConns int
MaxIdleConnsPerHost int
MaxConnsPerHost int
UserAgent string
Headers map[string]string
SkipTLSVerify bool
}
FastClientConfig holds configuration for the fast HTTP client.
func DefaultFastClientConfig ¶
func DefaultFastClientConfig() FastClientConfig
DefaultFastClientConfig returns optimized defaults.