crawler

type Crawler interface {
	Init(*spider.Spider) Crawler // Init initializes the crawler engine
	Run()                        // Run executes the task
	Stop()                       // Stop terminates the crawler
	CanStop() bool               // CanStop reports whether the crawler can be stopped
	GetID() int                  // GetID returns the engine ID
}

type CrawlerPool interface {
	Reset(spiderNum int) int
	SetPipelineConfig(outType string, batchCap int)
	Use() Crawler
	UseOpt() option.Option[Crawler]
	Free(Crawler)
	Stop()
}

type SpiderQueue interface {
	Reset() // Reset clears the queue
	Add(*spider.Spider)
	AddAll([]*spider.Spider)
	AddKeyins(string) // AddKeyins assigns Keyin to queue members that have not been assigned yet
	GetByIndex(int) *spider.Spider
	GetByIndexOpt(int) option.Option[*spider.Spider]
	GetByName(string) *spider.Spider
	GetByNameOpt(string) option.Option[*spider.Spider]
	GetAll() []*spider.Spider
	Len() int // Len returns the queue length
}