Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var DB *gorm.DB
DB instance
View Source
var DataDir string
Functions ¶
Types ¶
type Cache ¶
type Cache interface {
// --- 代理池读取 ---
RandomProxy() (string, bool)
GetProxy() (string, bool)
ProxyLength() int
PickProxy(req *http.Request) (string, error)
HasProxy(p string) bool
// IsProxyValidationDue prevents source refreshes from bypassing the
// persisted retry delay of a quarantined endpoint.
IsProxyValidationDue(proxyURI string) bool
GetAllProxies() []Proxy
// UpsertProxy 新代理首次验证成功时调用。
// 原子完成:清除 blocked_ips 旧记录 → 写入/更新 DB → 替换内存 entry。
UpsertProxy(p Proxy) error
// MarkVerifySuccess 已有代理定期验证成功时调用。
// 原子完成:清除 blocked_ips → 更新 DB delay/last_verify → 同步内存。
MarkVerifySuccess(p Proxy, delay int64, verifyTime time.Time, deepVerified bool)
// MarkVerifyFailed 已有代理定期验证失败但未达锁定阈值时调用。
// 原子完成:在 DB 中持久化不可用状态 → 从内存移除。
// 返回该端点更新后的连续失败次数。
MarkVerifyFailed(p Proxy) int
// 按 URI 隔离单个端点,等待定期健康检查恢复,不累计为整 IP 封禁。
MarkProxyUnavailable(proxyURI string)
// MarkProxyTargetFailure 对某个代理与目标的组合做短期熔断。它不会把
// 仍可服务其他目标的代理从全局池中删除。
MarkProxyTargetFailure(proxyURI string, target string)
// RecordFailure 代理请求或验证失败时调用,累计失败次数。
// 达到阈值时锁定 IP,并将该 IP 的代理持久化为不可用;记录不会删除,
// 锁定到期后仍可由健康检查或数据源重新验证恢复。
// 返回 true 表示触发了锁定。
RecordFailure(ip string) bool
// IsIPLocked 检查 IP 是否在锁定期内。
IsIPLocked(ip string) bool
// LoadLockedIPs 启动时从 DB 加载锁定状态到内存。
LoadLockedIPs()
// CleanupStaleProxies 删除 last_verify 超过阈值的陈旧代理,同步清理内存。
CleanupStaleProxies(threshold time.Duration)
}
var CI Cache
CI cache instance
type Conf ¶
type Conf struct {
Debug bool
Redis struct {
Host string
Port string
Pass string
Db int
}
Sys struct {
Name string
ProxyVerifyWorker int `mapstructure:"proxy_verify_worker"`
}
Obscura struct {
// obscura 二进制路径,留空则使用 PATH 中的默认命令
Bin string `mapstructure:"bin"`
} `mapstructure:"obscura"`
// Upstream controls how requests are attempted through proxies selected from
// the pool. Zero values use the safe defaults defined by the MITM package.
Upstream struct {
MaxAttempts int `mapstructure:"max_attempts"`
ConnectTimeout time.Duration `mapstructure:"connect_timeout"`
HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"`
ResponseHeaderTimeout time.Duration `mapstructure:"response_header_timeout"`
RequestTimeout time.Duration `mapstructure:"request_timeout"`
TargetFailureTTL time.Duration `mapstructure:"target_failure_ttl"`
} `mapstructure:"upstream"`
// Verification controls recurring health checks. Zero values use the
// conservative defaults defined by the crawler and service packages.
Verification struct {
Interval time.Duration `mapstructure:"interval"`
DeepCheckInterval time.Duration `mapstructure:"deep_check_interval"`
Retries int `mapstructure:"retries"`
ResponseBodyLimit int64 `mapstructure:"response_body_limit"`
MaxAttemptsPerSecond int `mapstructure:"max_attempts_per_second"`
AttemptBurst int `mapstructure:"attempt_burst"`
RoutineReservedWorkers int `mapstructure:"routine_reserved_workers"`
} `mapstructure:"verification"`
// SourceFetch controls downloads of proxy lists. Source documents are
// fetched directly by default: routing them through an untrusted free proxy
// makes a healthy source wait behind a failing proxy and can corrupt the
// returned list. Proxy fallback remains opt-in for installations that need
// it for reachability.
SourceFetch struct {
ProxyFallback bool `mapstructure:"proxy_fallback"`
Retries int `mapstructure:"retries"`
Timeout time.Duration `mapstructure:"timeout"`
ResponseBodyLimit int64 `mapstructure:"response_body_limit"`
MaxCandidatesPerFetch int `mapstructure:"max_candidates_per_fetch"`
} `mapstructure:"source_fetch"`
// EnableMITM 是否启用 HTTPS 中间人解密,默认 false(关闭时走 TCP 隧道透传,客户端无需关闭 TLS 验证)
EnableMITM bool `mapstructure:"enable_mitm"`
}
Conf config struct
var Config Conf
Config system config
type Proxy ¶
type Proxy struct {
gorm.Model
// BUG-FIX: GORM v2 不识别 unique_index(v1 语法),导致 IP 唯一约束从未生效,
// 同 IP 不同端口/协议的代理在 DB 中可以重复插入。改用 uniqueIndex 复合索引,
// 以 (IP, Port, Protocol) 为唯一粒度,允许同 IP 不同端口合法共存。
IP string `gorm:"type:varchar(15);uniqueIndex:idx_proxy_endpoint"`
Port string `gorm:"type:varchar(5);uniqueIndex:idx_proxy_endpoint"`
Country string `gorm:"type:varchar(15)"`
Provence string `gorm:"type:varchar(15)"`
Source string
Protocol string `gorm:"uniqueIndex:idx_proxy_endpoint"`
Delay int64
// Available is persisted so a proxy removed after a failed health check is
// not silently restored to the live pool by a process restart.
Available bool `gorm:"not null;default:true;index:idx_proxy_reverify,priority:1"`
LastVerify time.Time `gorm:"index:idx_proxy_reverify,priority:2"`
LastDeepVerify time.Time
ConsecutiveFailures int `gorm:"not null;default:0"`
NextVerifyAt time.Time `gorm:"index"`
}
Proxy proxy model
Click to show internal directories.
Click to hide internal directories.