crawler

package
v0.0.0-...-4370831 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 23, 2021 License: MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// Errors
	ErrBadUrl         = errors.New("bad url format")
	ErrUnsupportedTLD = errors.New("unsupported TLD")
	//
	OnTimeParse     = "OnTimeParse"
	OnProxyConnect  = "OnProxyConnect"
	OnDomainExtract = "OnDomainExtract"
	OnScrap         = "OnScrapping"
	OnWhois         = "OnWhois"
	OnDomain        = "OnDomain"
	OnMoz           = "OnMoz"
	OnTextExtract   = "OnTextExtract"
	//
	OnProcessor = "OnProcessor"
	OnStorage   = "OnStorage"
	OnPool      = "OnPool"
	OnBodyRead  = "OnBodyRead"
)

Functions

This section is empty.

Types

type Engine 

type Engine struct {
	Worker      int      `yaml:"worker"`
	Parallel    int      `yaml:"parallel"`
	Depth       int      `yaml:"depth"`
	URLPath     string   `yaml:"urls"`
	ProxyList   []string `yaml:"proxies,flow"`
	TLDs        []string `yaml:"tlds,flow"`
	Level       string   `yaml:"level"`
	RandomDelay string   `yaml:"random_delay"`
	Timeout     string   `yaml:"timeout"`
}

Engine struct

type Error 

type Error struct {
	// contains filtered or unexported fields
}

Error struct

func (*Error) Error 

func (e *Error) Error() string

Error method

type Setting 

type Setting struct {
	Engine *Engine `yaml:"Engine"`
}

Setting struct

func SettingFromFile 

func SettingFromFile(path string) (*Setting, error)

SettingFromFile func

type Spider 

type Spider struct {
	// contains filtered or unexported fields
}

Spider struct

func NewSpider 

func NewSpider(setting *Setting) (*Spider, error)

NewSpider with setting

func (*Spider) Logger 

func (s *Spider) Logger() error

Logger func

func (*Spider) Run 

func (s *Spider) Run()

Run func

type Store 

type Store struct {
	// contains filtered or unexported fields
}

Store struct

func NewStore 

func NewStore() *Store

NewStore func

func (*Store) Close 

func (s *Store) Close()

Close

func (*Store) Delete 

func (s *Store) Delete(key string)

Delete a value from store.

func (*Store) Found 

func (s *Store) Found(key string) bool

Found checks if key exists on store.

func (*Store) Save 

func (s *Store) Save(key string)

Save add value to store.

type Writer 

type Writer struct {
	// contains filtered or unexported fields
}

Writer struct

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.