Documentation
¶
Index ¶
- Variables
- func BrowserEval(expression string) (string, error)
- func BrowserFetch(targetURL string) (string, error)
- func CleanupStaleProxies()
- func FetchAllSources(sources []Source)
- func GetDocFromURL(url string, customHeaders ...http.Header) (string, error)
- func GetURLThroughProxyWithRetry(u string, timeout time.Duration, proxyAddr string, retry int, ...) ([]byte, error)
- func Init()
- func ReleaseBrowser()
- func StartObscura() error
- func StopObscura()
- func TestFetchSource(src Source) ([]proxyinabox.Proxy, error)
- func UpdateSourceAvailableCounts(proxies []proxyinabox.Proxy)
- func ValidateProxy(p proxyinabox.Proxy) (country string, delay int64, err error)
- func Verify()
- type BrowserSession
- type RuntimeFailStats
- type RuntimeSkipStats
- type RuntimeStats
- type Source
- type SourceCandidateStats
- type SourceStatus
- type VerificationStats
Constants ¶
This section is empty.
Variables ¶
var ValidateJobs chan proxyinabox.Proxy
Functions ¶
func BrowserEval ¶
BrowserEval 在当前 session 的页面上执行 JS 表达式
func BrowserFetch ¶
BrowserFetch 启动临时 Obscura 实例 → 导航到 URL → 等待 JS 渲染 → 返回 HTML 优先通过代理池中的随机 proxy 启动浏览器(obscura --proxy),若导航失败则销毁 session 并用直连重试。
func CleanupStaleProxies ¶ added in v0.8.0
func CleanupStaleProxies()
func FetchAllSources ¶
func FetchAllSources(sources []Source)
FetchAllSources starts a goroutine per source to continuously fetch proxies
func GetDocFromURL ¶
GetDocFromURL fetches source documents directly with a bounded body. An optional single proxy fallback is available for sources that cannot be reached directly; a bad free proxy is never allowed to delay every source refresh by several retries.
func GetURLThroughProxyWithRetry ¶
func GetURLThroughProxyWithRetry(u string, timeout time.Duration, proxyAddr string, retry int, customHeaders ...http.Header) ([]byte, error)
GetURLThroughProxyWithRetry fetches a URL through the given proxy with retry logic
func StartObscura ¶ added in v0.9.21
func StartObscura() error
StartObscura 兼容 test-source 子命令的预启动接口
func TestFetchSource ¶
func TestFetchSource(src Source) ([]proxyinabox.Proxy, error)
TestFetchSource performs a single fetch for testing purposes (does not send to ValidateJobs)
func UpdateSourceAvailableCounts ¶ added in v0.8.0
func UpdateSourceAvailableCounts(proxies []proxyinabox.Proxy)
UpdateSourceAvailableCounts 根据代理池快照更新各源的可用代理计数
func ValidateProxy ¶ added in v0.8.0
func ValidateProxy(p proxyinabox.Proxy) (country string, delay int64, err error)
ValidateProxy 通过代理访问 Cloudflare trace 端点验证代理可用性,返回验证结果 不依赖 DB/Cache,仅做网络验证,供 test-source 命令使用
Types ¶
type BrowserSession ¶ added in v0.4.1
type BrowserSession struct {
// contains filtered or unexported fields
}
BrowserSession 管理单次浏览器抓取的完整生命周期(Obscura 进程 + CDP 连接) 每个 runScript 调用创建独立 session,用完即销毁,避免资源泄漏
type RuntimeFailStats ¶ added in v0.9.31
type RuntimeSkipStats ¶ added in v0.9.31
type RuntimeStats ¶ added in v0.9.31
type RuntimeStats struct {
UptimeSeconds int64 `json:"uptime_seconds"`
CandidateWorkers int `json:"candidate_workers"`
RoutineWorkers int `json:"routine_workers"`
MaxConcurrent int `json:"max_concurrent"`
CandidateMaxConcurrent int `json:"candidate_max_concurrent"`
RoutineReserved int `json:"routine_reserved"`
MaxAttemptsPerSecond int `json:"max_attempts_per_second"`
AttemptBurst int `json:"attempt_burst"`
CandidateActive int64 `json:"candidate_active"`
RoutineActive int64 `json:"routine_active"`
CandidateWaiting int64 `json:"candidate_waiting_for_slot"`
RoutineWaiting int64 `json:"routine_waiting_for_slot"`
CandidateQueue int `json:"candidate_queue"`
CandidateQueueCap int `json:"candidate_queue_capacity"`
RoutineQueue int `json:"routine_queue"`
RoutineQueueCap int `json:"routine_queue_capacity"`
NegativeCacheEntries int `json:"negative_cache_entries"`
QueuedDedupEntries int64 `json:"queued_dedup_entries"`
NetworkAttempts uint64 `json:"network_attempts"`
NetworkResponseBytes uint64 `json:"network_response_bytes"`
AttemptsLastSecond uint64 `json:"attempts_last_second"`
AttemptsAverage60s float64 `json:"attempts_average_60s"`
AttemptsPeak60s uint64 `json:"attempts_peak_60s"`
ResponseBytesAverage60s float64 `json:"response_bytes_average_60s"`
RateLimited uint64 `json:"rate_limited"`
RateLimitWaitSeconds float64 `json:"rate_limit_wait_seconds"`
Skipped RuntimeSkipStats `json:"skipped"`
Failures RuntimeFailStats `json:"failures"`
}
RuntimeStats exposes the bounded worker pools and the crawler's own network cost. Counters are process-lifetime values, just like request statistics. Queue sizes are point-in-time snapshots and do not include quarantined rows waiting in SQLite.
func GetRuntimeStats ¶ added in v0.9.31
func GetRuntimeStats() RuntimeStats
type Source ¶
type Source struct {
Name string `yaml:"name"`
Type string `yaml:"type"` // text, json, script
URL string `yaml:"url"`
Protocol string `yaml:"protocol"`
Headers map[string]string `yaml:"headers"`
Interval string `yaml:"interval"`
IPField string `yaml:"ip_field"`
PortField string `yaml:"port_field"`
ProtocolField string `yaml:"protocol_field"`
Script string `yaml:"script"`
}
Source represents a YAML-driven proxy source configuration
func LoadSources ¶
LoadSources reads all .yaml files from the given directory and returns parsed sources
type SourceCandidateStats ¶ added in v0.9.32
type SourceCandidateStats struct {
Checks uint64 `json:"candidate_checks"`
Success uint64 `json:"candidate_success"`
Failure uint64 `json:"candidate_failure"`
Yield float64 `json:"candidate_yield"`
RecentChecks uint64 `json:"candidate_recent_checks"`
RecentSuccess uint64 `json:"candidate_recent_success"`
RecentYield float64 `json:"candidate_recent_yield"`
CurrentLimit int `json:"candidate_limit"`
Admitted uint64 `json:"candidate_admitted"`
Capped uint64 `json:"candidate_capped"`
Filtered uint64 `json:"candidate_filtered"`
Duplicate uint64 `json:"candidate_duplicate"`
}
type SourceStatus ¶ added in v0.2.0
type SourceStatus struct {
Name string `json:"name"`
Type string `json:"type"`
LastFetch time.Time `json:"last_fetch"`
ProxyCount int `json:"proxy_count"`
Error string `json:"error"`
Interval string `json:"interval"`
// AvailableCount 该源当前在代理池中验证通过的可用代理数(实时从缓存统计)
AvailableCount int `json:"available_count"`
Candidate SourceCandidateStats `json:"candidate"`
}
SourceStatus 记录每个 proxy 源的最近抓取状态,用于 dashboard 展示
func GetSourceStatuses ¶ added in v0.2.0
func GetSourceStatuses() []SourceStatus
GetSourceStatuses 返回所有源状态的快照副本(线程安全)
type VerificationStats ¶ added in v0.9.27
type VerificationStats struct {
RoutineSuccess uint64 `json:"routine_success"`
RoutineFailure uint64 `json:"routine_failure"`
CandidateSuccess uint64 `json:"candidate_success"`
CandidateFailure uint64 `json:"candidate_failure"`
}
VerificationStats counts completed health checks since this process started. A check can make more than one HTTP attempt when retries are configured.
func GetRecentVerificationStats ¶ added in v0.9.29
func GetRecentVerificationStats() VerificationStats
GetRecentVerificationStats returns checks in the current and previous 59 calendar minutes. Like the cumulative counters, it counts checks, not retries.
func GetVerificationStats ¶ added in v0.9.27
func GetVerificationStats() VerificationStats