crawler

package

v0.0.0-...-fa848ac Latest Latest Go to latest Published: Apr 12, 2020 License: Apache-2.0 Imports: 16 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/vicanso/proxy-pool

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
func NewIP66Proxy(interval time.Duration) *ip66Proxy
func NewKuaiProxy(interval time.Duration) *kuaiProxy
func NewProxyClient(p *Proxy) *http.Client
func NewXiciProxy(interval time.Duration) *xiciProxy
type Crawler
type FetchListener
type Proxy
type ProxyCrawler
type ProxyList

Constants ¶

View Source

const (
	StatusRunning = iota
	StatusStop
)

View Source

const (
	ProxyIP66 = "ip66"
)

View Source

const (
	ProxyKuai = "kuai"
)

View Source

const (
	ProxyXiCi = "xici"
)

Variables ¶

This section is empty.

Functions ¶

func NewIP66Proxy ¶

func NewIP66Proxy(interval time.Duration) *ip66Proxy

NewIP66Proxy create a new ip66 proxy crawler

func NewKuaiProxy ¶

func NewKuaiProxy(interval time.Duration) *kuaiProxy

NewKuaiProxy create a new kuai proxy crawler

func NewProxyClient ¶

func NewProxyClient(p *Proxy) *http.Client

NewProxyClient create a new http client with proxy

func NewXiciProxy ¶

func NewXiciProxy(interval time.Duration) *xiciProxy

NewXiciProxy create a new xici proxy crawler

Types ¶

type Crawler ¶

type Crawler struct {
	sync.Mutex
	HTTPDetectURL  string
	HTTPSDetectURL string
	// contains filtered or unexported fields
}

Crawler crawler

func (*Crawler) GetAvailableProxy ¶

func (c *Crawler) GetAvailableProxy(category string, speed int32) *Proxy

GetAvailableProxy get available proxy

func (*Crawler) GetAvailableProxyList ¶

func (c *Crawler) GetAvailableProxyList() []*Proxy

GetAvailableProxyList get available proxy list

func (*Crawler) RedetectAvailableProxy ¶

func (c *Crawler) RedetectAvailableProxy()

RedetectAvailableProxy redetect available proxy

func (*Crawler) Start ¶

func (c *Crawler) Start(crawlers ...ProxyCrawler)

Start start fetch proxy

type FetchListener ¶

type FetchListener func(*Proxy)

FetchListener fetch listener

type Proxy ¶

type Proxy struct {
	DetectedAt int64  `json:"detectedAt,omitempty"`
	IP         string `json:"ip,omitempty"`
	Port       string `json:"port,omitempty"`
	Category   string `json:"category,omitempty"`
	Anonymous  bool   `json:"anonymous,omitempty"`
	Speed      int32  `json:"speed,omitempty"`
	Fails      int32  `json:"fails,omitempty"`
}

Proxy proxy server

type ProxyCrawler ¶

type ProxyCrawler interface {
	// OnFetch set fetch listener
	OnFetch(FetchListener)
	// Start start the crawler
	Start()
	// Stop stop the crawler
	Stop()
}

ProxyCrawler proxy crawler

type ProxyList ¶

type ProxyList struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

ProxyList proxy list

func (*ProxyList) Add ¶

func (pl *ProxyList) Add(list ...*Proxy)

Add add proxy to list

func (*ProxyList) Exists ¶

func (pl *ProxyList) Exists(p *Proxy) bool

Exists test whether or not the proxy exists

func (*ProxyList) FindOne ¶

func (pl *ProxyList) FindOne(category string, speed int32) (p *Proxy)

FindOne find one proxy

func (*ProxyList) List ¶

func (pl *ProxyList) List() []*Proxy

List get proxy list

func (*ProxyList) Remove ¶

func (pl *ProxyList) Remove(list ...*Proxy)

Remove remove proxy from list

func (*ProxyList) Replace ¶

func (pl *ProxyList) Replace(list []*Proxy)

Replace replace the proxy list

func (*ProxyList) Reset ¶

func (pl *ProxyList) Reset() []*Proxy

Reset reset proxy list

func (*ProxyList) Size ¶

func (pl *ProxyList) Size() int

Size get the size of proxy list

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL