Documentation
¶
Index ¶
- func NewHTTPClient(proxyProfile *config.ProxyProfile, timeout time.Duration) (*http.Client, error)
- func NewRestyClient(proxyProfile *config.ProxyProfile, timeout time.Duration, retries int) (*resty.Client, error)
- func NewRestyClientNoProxy(timeout time.Duration, retries int) *resty.Client
- func NewTransport(proxyProfile *config.ProxyProfile) (*http.Transport, error)
- func SanitizeProxyURL(proxyURL string) string
- type FlareSolverr
- func GetFlareSolverrFromClient(client *resty.Client) (*FlareSolverr, bool)
- func NewFlareSolverr(cfg *config.FlareSolverrConfig) (*FlareSolverr, error)
- func NewRestyClientWithFlareSolverr(proxyProfile *config.ProxyProfile, flaresolverrCfg config.FlareSolverrConfig, ...) (*resty.Client, *FlareSolverr, error)
- func (fs *FlareSolverr) Close() error
- func (fs *FlareSolverr) CreateSession() (string, error)
- func (fs *FlareSolverr) DestroySession(sessionID string) error
- func (fs *FlareSolverr) ResolveURL(targetURL string) (string, []http.Cookie, error)
- func (fs *FlareSolverr) ResolveURLWithSession(targetURL, sessionID string) (string, []http.Cookie, error)
- type FlareSolverrProxy
- type FlareSolverrRequest
- type FlareSolverrResponse
- type FlareSolverrSession
- type HTTPClient
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func NewHTTPClient ¶
NewHTTPClient creates a standard http.Client with proxy support
func NewRestyClient ¶
func NewRestyClient(proxyProfile *config.ProxyProfile, timeout time.Duration, retries int) (*resty.Client, error)
NewRestyClient creates a resty.Client with proxy support
func NewRestyClientNoProxy ¶
NewRestyClientNoProxy creates a resty.Client that explicitly bypasses environment proxy variables by using a no-proxy transport.
func NewTransport ¶
func NewTransport(proxyProfile *config.ProxyProfile) (*http.Transport, error)
NewTransport creates an http.Transport with optional proxy support
func SanitizeProxyURL ¶
SanitizeProxyURL removes credentials from proxy URL for safe logging
Types ¶
type FlareSolverr ¶
type FlareSolverr struct {
// contains filtered or unexported fields
}
FlareSolverr represents the FlareSolverr client
func GetFlareSolverrFromClient ¶
func GetFlareSolverrFromClient(client *resty.Client) (*FlareSolverr, bool)
GetFlareSolverrFromClient extracts FlareSolverr instance from resty client context Note: This is a helper for scrapers that need to access FlareSolverr The FlareSolverr instance is typically stored separately and passed to scrapers
func NewFlareSolverr ¶
func NewFlareSolverr(cfg *config.FlareSolverrConfig) (*FlareSolverr, error)
NewFlareSolverr creates a new FlareSolverr client
func NewRestyClientWithFlareSolverr ¶
func NewRestyClientWithFlareSolverr(proxyProfile *config.ProxyProfile, flaresolverrCfg config.FlareSolverrConfig, timeout time.Duration, retries int) (*resty.Client, *FlareSolverr, error)
NewRestyClientWithFlareSolverr creates a resty.Client with optional FlareSolverr support Note: FlareSolverr config is passed separately since it's at ScrapersConfig.FlareSolverr (top-level), not inside ProxyConfig (which only holds proxy settings).
func (*FlareSolverr) Close ¶
func (fs *FlareSolverr) Close() error
Close cleans up resources held by the FlareSolverr. Currently a no-op since FlareSolverr uses stateless HTTP requests, but provides a cleanup hook for future resource management.
func (*FlareSolverr) CreateSession ¶
func (fs *FlareSolverr) CreateSession() (string, error)
CreateSession creates a new FlareSolverr session for cookie persistence
func (*FlareSolverr) DestroySession ¶
func (fs *FlareSolverr) DestroySession(sessionID string) error
DestroySession destroys a FlareSolverr session via HTTP and removes it from local cache.
func (*FlareSolverr) ResolveURL ¶
ResolveURL resolves a URL through FlareSolverr, returning HTML content and cookies. The mutex is held for the entire operation to ensure session reuse is safe from concurrent reset calls.
func (*FlareSolverr) ResolveURLWithSession ¶
func (fs *FlareSolverr) ResolveURLWithSession(targetURL, sessionID string) (string, []http.Cookie, error)
ResolveURLWithSession resolves a URL using a specific session
type FlareSolverrProxy ¶
type FlareSolverrProxy struct {
URL string `json:"url"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
}
FlareSolverrProxy represents a per-request proxy configuration passed to FlareSolverr. This is used for the target URL request made by FlareSolverr, not for calls to FlareSolverr itself.
type FlareSolverrRequest ¶
type FlareSolverrRequest struct {
Cmd string `json:"cmd"` // "request.get" or "sessions.create"
URL string `json:"url"` // Target URL
MaxTimeout int `json:"maxTimeout"` // Timeout in milliseconds (FlareSolverr expects ms)
Session string `json:"session"` // Optional: reuse existing session
SessionTTLMinutes int `json:"session_ttl_minutes,omitempty"` // Optional: rotate existing session when older than TTL
Proxy *FlareSolverrProxy `json:"proxy,omitempty"` // Optional: proxy for target URL request
}
FlareSolverrRequest represents a request to FlareSolverr
type FlareSolverrResponse ¶
type FlareSolverrResponse struct {
Status string `json:"status"`
Message string `json:"message"`
Solution struct {
Response string `json:"response"`
Cookies []struct {
Name string `json:"name"`
Value string `json:"value"`
} `json:"cookies"`
UserAgent string `json:"userAgent"`
} `json:"solution"`
Session string `json:"session"`
}
FlareSolverrResponse represents a FlareSolverr response