crawler

package
v0.9.33 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 2, 2026 License: MIT Imports: 27 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

View Source
var ValidateJobs chan proxyinabox.Proxy

Functions

func BrowserEval

func BrowserEval(expression string) (string, error)

BrowserEval 在当前 session 的页面上执行 JS 表达式

func BrowserFetch

func BrowserFetch(targetURL string) (string, error)

BrowserFetch 启动临时 Obscura 实例 → 导航到 URL → 等待 JS 渲染 → 返回 HTML 优先通过代理池中的随机 proxy 启动浏览器(obscura --proxy),若导航失败则销毁 session 并用直连重试。

func CleanupStaleProxies added in v0.8.0

func CleanupStaleProxies()

func FetchAllSources

func FetchAllSources(sources []Source)

FetchAllSources starts a goroutine per source to continuously fetch proxies

func GetDocFromURL

func GetDocFromURL(url string, customHeaders ...http.Header) (string, error)

GetDocFromURL fetches source documents directly with a bounded body. An optional single proxy fallback is available for sources that cannot be reached directly; a bad free proxy is never allowed to delay every source refresh by several retries.

func GetURLThroughProxyWithRetry

func GetURLThroughProxyWithRetry(u string, timeout time.Duration, proxyAddr string, retry int, customHeaders ...http.Header) ([]byte, error)

GetURLThroughProxyWithRetry fetches a URL through the given proxy with retry logic

func Init

func Init()

func ReleaseBrowser added in v0.4.1

func ReleaseBrowser()

ReleaseBrowser 停止当前 Obscura 实例,释放所有资源

func StartObscura added in v0.9.21

func StartObscura() error

StartObscura 兼容 test-source 子命令的预启动接口

func StopObscura added in v0.9.21

func StopObscura()

StopObscura 兼容 test-source 子命令和信号处理的停止接口

func TestFetchSource

func TestFetchSource(src Source) ([]proxyinabox.Proxy, error)

TestFetchSource performs a single fetch for testing purposes (does not send to ValidateJobs)

func UpdateSourceAvailableCounts added in v0.8.0

func UpdateSourceAvailableCounts(proxies []proxyinabox.Proxy)

UpdateSourceAvailableCounts 根据代理池快照更新各源的可用代理计数

func ValidateProxy added in v0.8.0

func ValidateProxy(p proxyinabox.Proxy) (country string, delay int64, err error)

ValidateProxy 通过代理访问 Cloudflare trace 端点验证代理可用性,返回验证结果 不依赖 DB/Cache,仅做网络验证,供 test-source 命令使用

func Verify

func Verify()

Types

type BrowserSession added in v0.4.1

type BrowserSession struct {
	// contains filtered or unexported fields
}

BrowserSession 管理单次浏览器抓取的完整生命周期(Obscura 进程 + CDP 连接) 每个 runScript 调用创建独立 session,用完即销毁,避免资源泄漏

type RuntimeFailStats added in v0.9.31

type RuntimeFailStats struct {
	Timeout     uint64 `json:"timeout"`
	Network     uint64 `json:"network"`
	Response    uint64 `json:"response"`
	IPMismatch  uint64 `json:"ip_mismatch"`
	TLSProbe    uint64 `json:"tls_probe"`
	Persistence uint64 `json:"persistence"`
}

type RuntimeSkipStats added in v0.9.31

type RuntimeSkipStats struct {
	Duplicate uint64 `json:"duplicate"`
	IPLocked  uint64 `json:"ip_locked"`
	Available uint64 `json:"already_available"`
	Backoff   uint64 `json:"backoff"`
}

type RuntimeStats added in v0.9.31

type RuntimeStats struct {
	UptimeSeconds           int64            `json:"uptime_seconds"`
	CandidateWorkers        int              `json:"candidate_workers"`
	RoutineWorkers          int              `json:"routine_workers"`
	MaxConcurrent           int              `json:"max_concurrent"`
	CandidateMaxConcurrent  int              `json:"candidate_max_concurrent"`
	RoutineReserved         int              `json:"routine_reserved"`
	MaxAttemptsPerSecond    int              `json:"max_attempts_per_second"`
	AttemptBurst            int              `json:"attempt_burst"`
	CandidateActive         int64            `json:"candidate_active"`
	RoutineActive           int64            `json:"routine_active"`
	CandidateWaiting        int64            `json:"candidate_waiting_for_slot"`
	RoutineWaiting          int64            `json:"routine_waiting_for_slot"`
	CandidateQueue          int              `json:"candidate_queue"`
	CandidateQueueCap       int              `json:"candidate_queue_capacity"`
	RoutineQueue            int              `json:"routine_queue"`
	RoutineQueueCap         int              `json:"routine_queue_capacity"`
	NegativeCacheEntries    int              `json:"negative_cache_entries"`
	QueuedDedupEntries      int64            `json:"queued_dedup_entries"`
	NetworkAttempts         uint64           `json:"network_attempts"`
	NetworkResponseBytes    uint64           `json:"network_response_bytes"`
	AttemptsLastSecond      uint64           `json:"attempts_last_second"`
	AttemptsAverage60s      float64          `json:"attempts_average_60s"`
	AttemptsPeak60s         uint64           `json:"attempts_peak_60s"`
	ResponseBytesAverage60s float64          `json:"response_bytes_average_60s"`
	RateLimited             uint64           `json:"rate_limited"`
	RateLimitWaitSeconds    float64          `json:"rate_limit_wait_seconds"`
	Skipped                 RuntimeSkipStats `json:"skipped"`
	Failures                RuntimeFailStats `json:"failures"`
}

RuntimeStats exposes the bounded worker pools and the crawler's own network cost. Counters are process-lifetime values, just like request statistics. Queue sizes are point-in-time snapshots and do not include quarantined rows waiting in SQLite.

func GetRuntimeStats added in v0.9.31

func GetRuntimeStats() RuntimeStats

type Source

type Source struct {
	Name          string            `yaml:"name"`
	Type          string            `yaml:"type"` // text, json, script
	URL           string            `yaml:"url"`
	Protocol      string            `yaml:"protocol"`
	Headers       map[string]string `yaml:"headers"`
	Interval      string            `yaml:"interval"`
	IPField       string            `yaml:"ip_field"`
	PortField     string            `yaml:"port_field"`
	ProtocolField string            `yaml:"protocol_field"`
	Script        string            `yaml:"script"`
}

Source represents a YAML-driven proxy source configuration

func LoadSources

func LoadSources(dir string) ([]Source, error)

LoadSources reads all .yaml files from the given directory and returns parsed sources

type SourceCandidateStats added in v0.9.32

type SourceCandidateStats struct {
	Checks        uint64  `json:"candidate_checks"`
	Success       uint64  `json:"candidate_success"`
	Failure       uint64  `json:"candidate_failure"`
	Yield         float64 `json:"candidate_yield"`
	RecentChecks  uint64  `json:"candidate_recent_checks"`
	RecentSuccess uint64  `json:"candidate_recent_success"`
	RecentYield   float64 `json:"candidate_recent_yield"`
	CurrentLimit  int     `json:"candidate_limit"`
	Admitted      uint64  `json:"candidate_admitted"`
	Capped        uint64  `json:"candidate_capped"`
	Filtered      uint64  `json:"candidate_filtered"`
	Duplicate     uint64  `json:"candidate_duplicate"`
}

type SourceStatus added in v0.2.0

type SourceStatus struct {
	Name       string    `json:"name"`
	Type       string    `json:"type"`
	LastFetch  time.Time `json:"last_fetch"`
	ProxyCount int       `json:"proxy_count"`
	Error      string    `json:"error"`
	Interval   string    `json:"interval"`
	// AvailableCount 该源当前在代理池中验证通过的可用代理数(实时从缓存统计)
	AvailableCount int                  `json:"available_count"`
	Candidate      SourceCandidateStats `json:"candidate"`
}

SourceStatus 记录每个 proxy 源的最近抓取状态,用于 dashboard 展示

func GetSourceStatuses added in v0.2.0

func GetSourceStatuses() []SourceStatus

GetSourceStatuses 返回所有源状态的快照副本(线程安全)

type VerificationStats added in v0.9.27

type VerificationStats struct {
	RoutineSuccess   uint64 `json:"routine_success"`
	RoutineFailure   uint64 `json:"routine_failure"`
	CandidateSuccess uint64 `json:"candidate_success"`
	CandidateFailure uint64 `json:"candidate_failure"`
}

VerificationStats counts completed health checks since this process started. A check can make more than one HTTP attempt when retries are configured.

func GetRecentVerificationStats added in v0.9.29

func GetRecentVerificationStats() VerificationStats

GetRecentVerificationStats returns checks in the current and previous 59 calendar minutes. Like the cumulative counters, it counts checks, not retries.

func GetVerificationStats added in v0.9.27

func GetVerificationStats() VerificationStats

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL