juemei

package

v0.0.1 Latest Latest Go to latest Published: Jan 3, 2018 License: GPL-3.0 Imports: 6 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/ckeyer/attack

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func LoadDoc(cli *httpclient.Client, Url string) (*goquery.Document, error)
type RedisStore
- func NewRedisStore(cli *redis.Client) *RedisStore
type ResolveResult
- func Resolve(cli *httpclient.Client, doc *goquery.Document) (*ResolveResult, error)
type Store
type StoreResult

Constants ¶

View Source

const (
	RootURL  = "http://www.juemei.com"
	MaxWorks = 1
)

View Source

const (
	RedisKeyPrefix = "juemei_" // redis key 前缀

	KeySetToDo     = RedisKeyPrefix + "todo"      // 未爬取的页面链接
	KeySetDoing    = RedisKeyPrefix + "doing"     // 正在处理的页面
	KeySetDone     = RedisKeyPrefix + "done"      // 已爬取完成的页面
	KeySetImgs     = RedisKeyPrefix + "imgs"      // 爬取的图片链接
	KeySetOutLinks = RedisKeyPrefix + "out_links" // 外链
)

Variables ¶

View Source

var (
	BlackPrefix = []string{
		RootURL,
	}
	IgnorePrefix = []string{
		"#",
	}
)

Functions ¶

func LoadDoc ¶

func LoadDoc(cli *httpclient.Client, Url string) (*goquery.Document, error)

Types ¶

type RedisStore ¶

type RedisStore struct {
	*redis.Client
}

func NewRedisStore ¶

func NewRedisStore(cli *redis.Client) *RedisStore

func (*RedisStore) Done ¶

func (r *RedisStore) Done(rets ...*ResolveResult) (*StoreResult, error)

func (*RedisStore) Init ¶

func (r *RedisStore) Init() error

func (*RedisStore) Jobs ¶

func (r *RedisStore) Jobs(count int) ([]string, error)

type ResolveResult ¶

type ResolveResult struct {
	URL      string
	Imgs     []string
	Links    []string
	OutLinks []string
}

func Resolve ¶

func Resolve(cli *httpclient.Client, doc *goquery.Document) (*ResolveResult, error)

type Store ¶

type Store interface {
}

type StoreResult ¶

type StoreResult struct {
	URLs                  []string
	Imgs, Links, OutLinks int64
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL