crawl

package

v0.2.0 Latest Latest Go to latest Published: Jan 21, 2026 License: MIT Imports: 10 Imported by: 0

Details

This section is empty.

View Source

var (
	Jina  = "Jina"
	Colly = "Colly"
)

View Source

var Crawlers = make(map[string]Crawler)

func Register(name string, c Crawler)

type CollyCrawler struct {
}

CollyCrawler 使用jina搜索

func (s *CollyCrawler) Crawl(ctx context.Context, url string) (*Result, error)

Crawl ...

func (s *CollyCrawler) CrawlMulti(ctx context.Context, urls []string) (map[string]*Result, error)

CrawlMulti 爬取网页内容

type Crawler interface {
	Crawl(ctx context.Context, url string) (*Result, error)
}

Crawler 爬虫接口

func NewCollyCrawler() Crawler

NewCollyCrawler 创建jina爬虫

func NewJinaCrawler() Crawler

NewJinaCrawler 创建jina爬虫

type JinaCrawler struct {
}

JinaCrawler 使用jina搜索

func (c *JinaCrawler) Crawl(ctx context.Context, link string) (*Result, error)

Crawl 爬取

type Result struct {
	Link    string
	Title   string
	Content string
}

Result 爬取结果