exec

package

v1.4.2 Latest Latest Go to latest Published: Nov 3, 2023 License: AGPL-3.0 Imports: 36 Imported by: 0

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/nging-plugins/collector

Links

Open Source Insights

Documentation ¶

Constants ¶

This section is empty.

Variables ¶

View Source

var ErrForcedExit = errors.New(`Forced exit`)

View Source

var RegexpTitle = regexp.MustCompile(`(?i)<title[\s]*>([^<]+)</title[\s]*>`)

Functions ¶

This section is empty.

Types ¶

type Fether ¶

type Fether func(pageURL string, charset string) (body []byte, transcoded bool, err error)

type Recv ¶

type Recv struct {
	Index      int
	IsEmpty    bool        //是否为空结果
	LevelIndex int         //层级索引
	URLIndex   int         //网址列表索引
	Result     interface{} //采集结果数据
	//rule       *Rule       //页面规则
	Title string //页面标题
	URL   string //网址
	// contains filtered or unexported fields
}

func (*Recv) Parent ¶

func (c *Recv) Parent() *Recv

func (*Recv) ParentItem ¶ added in v1.4.2

func (c *Recv) ParentItem() interface{}

func (*Recv) ParentResult ¶

func (c *Recv) ParentResult() interface{}

func (*Recv) Parents ¶

func (c *Recv) Parents(lasts ...int) *Recv

func (*Recv) ParentsItem ¶ added in v1.4.2

func (c *Recv) ParentsItem(lasts ...int) interface{}

func (*Recv) ParentsResult ¶

func (c *Recv) ParentsResult(lasts ...int) interface{}

func (*Recv) ResultByIndex ¶ added in v1.4.2

func (c *Recv) ResultByIndex(index int) interface{}

func (*Recv) String ¶ added in v1.4.2

func (c *Recv) String() string

func (*Recv) UniqueID ¶

func (_ *Recv) UniqueID() (string, error)

type Result ¶

type Result struct {
	Title     string
	URL       string
	Result    interface{}
	Type      string //map/slice
	StartTime time.Time
	EndTime   time.Time
	Elapsed   time.Duration
}

func (*Result) ElapsedString ¶

func (c *Result) ElapsedString(lang string) string

func (*Result) IsMap ¶

func (c *Result) IsMap() bool

func (*Result) IsSlice ¶

func (c *Result) IsSlice() bool

type Rule ¶

type Rule struct {
	*dbschema.NgingCollectorPage                                // 页面配置
	RuleList                     []*dbschema.NgingCollectorRule // 采集规则列表
	// contains filtered or unexported fields
}

Rule 页面规则

func (*Rule) Collect ¶

func (c *Rule) Collect(parentID uint64, parentURL string, recv *Recv,
	fetch Fether, extra []*Rule,
	noticeSender sender.Notice,
	progress *notice.Progress) ([]Result, error)

func (*Rule) CollectOne ¶ added in v1.4.2

func (c *Rule) CollectOne(recv *Recv, urlIndex int,
	parentID uint64, parentURL string, pageURL string,
	fetch Fether, extra []*Rule,
	noticeSender sender.Notice,
	progress *notice.Progress, historyMdl *dbschema.NgingCollectorHistory) (collection interface{}, result []Result, ignore bool, err error)

func (*Rule) IsExited ¶

func (c *Rule) IsExited() bool

func (*Rule) ParseTmplContent ¶

func (c *Rule) ParseTmplContent(tmplContent string, result *Recv) (string, error)

type Rules ¶

type Rules struct {
	*Rule         //主页面配置以及采集规则列表
	Extra []*Rule //扩展页面配置以及采集规则列表
	// contains filtered or unexported fields
}

Rules 完整规则

func NewRules ¶

func NewRules() *Rules

func (*Rules) Collect ¶

func (c *Rules) Collect(debug bool, noticeSender sender.Notice, progress *notice.Progress) (rs []Result, err error)

func (*Rules) SetExitedFn ¶

func (c *Rules) SetExitedFn(exitedFn func() bool) *Rules

func (*Rules) SetExportFn ¶

func (c *Rules) SetExportFn(exportFn func(pageID uint, lastResult *Recv, collected echo.Store, noticeSender sender.Notice) error) *Rules

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL