services

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 1, 2020 License: BSD-3-Clause Imports: 48 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var LockList sync.Map

任务执行锁 Added by cloud: 2019/09/04,solve data race

View Source 
var SystemInfoChanMap = utils.NewChanMap()

系统信息 chan 映射

View Source 
var TaskLogChanMap = utils.NewChanMap()

任务日志频道映射

Functions

func AddGitCronJob 

func AddGitCronJob(s model.Spider) func()

添加Git定时任务

func AddScheduleTask 

func AddScheduleTask(s model.Schedule) func()

func AddTask 

func AddTask(t model.Task) (string, error)

func AssignTask 

func AssignTask(task model.Task) error

派发任务

func CancelSpider 

func CancelSpider(id string) error

func CancelTask 

func CancelTask(id string) (err error)

func CheckToken 

func CheckToken(tokenStr string) (user model.User, err error)

func CopySpider 

func CopySpider(spider model.Spider, newName string) error

func CreateNewUser 

func CreateNewUser(username string, password string, role string, email string, uid bson.ObjectId) error

func CreateResultsIndexes 

func CreateResultsIndexes(col string)

func CreateScrapyProject 

func CreateScrapyProject(s model.Spider) (err error)

func CreateScrapySpider 

func CreateScrapySpider(s model.Spider, name string, domain string, template string) (err error)

func DeleteLogPeriodically 

func DeleteLogPeriodically()

定时删除日志

func DownloadRepo 

func DownloadRepo(fullName string, userId bson.ObjectId) (err error)

func ExecuteShellCmd 

func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u model.User) (err error)

执行shell命令

func ExecuteTask 

func ExecuteTask(id int)

执行任务

func FetchPythonDepInfo 

func FetchPythonDepInfo(depName string) (entity.Dependency, error)

func FetchPythonDepList 

func FetchPythonDepList() ([]string, error)

从Python依赖源获取依赖列表并返回

func FinishOrCancelTask 

func FinishOrCancelTask(ch chan string, cmd *exec.Cmd, s model.Spider, t model.Task)

func FinishUpTask 

func FinishUpTask(s model.Spider, t model.Task)

func GenerateConfigSpiderFiles 

func GenerateConfigSpiderFiles(spider model.Spider, configData entity.ConfigSpiderData) error

func GenerateSpiderfileFromConfigData 

func GenerateSpiderfileFromConfigData(spider model.Spider, configData entity.ConfigSpiderData) error

func GetAdminUser 

func GetAdminUser() (user *model.User, err error)

func GetAuthQuery 

func GetAuthQuery(query bson.M, c *gin.Context) bson.M

func GetCurrentUser 

func GetCurrentUser(c *gin.Context) *model.User

func GetCurrentUserId 

func GetCurrentUserId(c *gin.Context) bson.ObjectId

func GetDocs 

func GetDocs() (data string, err error)

func GetExecuteTaskFunc 

func GetExecuteTaskFunc(id int) func()

生成执行任务方法

func GetFileNodeTree 

func GetFileNodeTree(dstPath string, level int) (f model.File, err error)

func GetGitHeadHash 

func GetGitHeadHash(repo *git.Repository) string

获取Git Head Hash

func GetGitRemoteBranchesPlain 

func GetGitRemoteBranchesPlain(gitUrl string, username string, password string) (branches []string, err error)

获得Git分支

func GetGitSshPublicKey 

func GetGitSshPublicKey() string

获取SSH公钥

func GetLangInstallStatus 

func GetLangInstallStatus(nodeId string, lang entity.Lang) (string, error)

获取语言安装状态

func GetLangList 

func GetLangList(nodeId string) []entity.Lang

获取语言列表

func GetLatestRelease 

func GetLatestRelease() (release entity.Release, err error)

func GetNodejsDepList 

func GetNodejsDepList(nodeId string, searchDepName string) (depList []entity.Dependency, err error)

获取Nodejs本地依赖列表

func GetPythonDepList 

func GetPythonDepList(nodeId string, searchDepName string) ([]entity.Dependency, error)

获取Python本地依赖列表

func GetPythonDepListFromRedis 

func GetPythonDepListFromRedis() ([]string, error)

从Redis获取Python依赖列表

func GetPythonDepListWithInfo 

func GetPythonDepListWithInfo(depList []entity.Dependency) ([]entity.Dependency, error)

获取Python依赖的源数据信息

func GetScrapyItems 

func GetScrapyItems(s model.Spider) (res []map[string]interface{}, err error)

func GetScrapyPipelines 

func GetScrapyPipelines(s model.Spider) (res []string, err error)

func GetScrapySettings 

func GetScrapySettings(s model.Spider) (res []map[string]interface{}, err error)

func GetScrapySpiderFilepath 

func GetScrapySpiderFilepath(s model.Spider, spiderName string) (res string, err error)

func GetScrapySpiderNames 

func GetScrapySpiderNames(s model.Spider) ([]string, error)

func GetSystemInfo 

func GetSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error)

获取系统信息

func GetTaskEmailMarkdownContent 

func GetTaskEmailMarkdownContent(t model.Task, s model.Spider) string

func GetTaskErrorLog 

func GetTaskErrorLog(id string, n int) (errLogItems []model.ErrorLogItem, err error)

func GetTaskLog 

func GetTaskLog(id string, keyword string, page int, pageSize int) (logItems []model.LogItem, logTotal int, err error)

func GetTaskMarkdownContent 

func GetTaskMarkdownContent(t model.Task, s model.Spider) string

func GetWorkerPrefix 

func GetWorkerPrefix(id int) string

func GitCheckout 

func GitCheckout(s model.Spider, hash string) (err error)

func InitCleanService 

func InitCleanService() error

func InitDeleteLogPeriodically 

func InitDeleteLogPeriodically() error

初始化定时删除日志

func InitDemoSpiders 

func InitDemoSpiders()

func InitDepsFetcher 

func InitDepsFetcher() error

初始化函数

func InitLogIndexes 

func InitLogIndexes() error

func InitLogService 

func InitLogService() error

func InitNodeService 

func InitNodeService() error

初始化节点服务

func InitProjectCleanUserIds 

func InitProjectCleanUserIds()

func InitScheduleCleanUserIds 

func InitScheduleCleanUserIds()

func InitScheduler 

func InitScheduler() error

func InitSpiderCleanUserIds 

func InitSpiderCleanUserIds()

func InitSpiderService 

func InitSpiderService() error

启动爬虫服务

func InitTaskCleanUserIds 

func InitTaskCleanUserIds()

func InitTaskExecutor 

func InitTaskExecutor() error

func InitUserService 

func InitUserService() error

func IsInstalledDep 

func IsInstalledDep(installedDepList []entity.Dependency, dep entity.Dependency) bool

是否已安装该依赖

func IsMasterNode 

func IsMasterNode(id string) bool

所有调用IsMasterNode的方法，都永远会在master节点执行，所以GetCurrentNode方法返回永远是master节点 该ID的节点是否为主节点

func IsUniqueConfigSpiderFields 

func IsUniqueConfigSpiderFields(fields []entity.Field) bool

func MakeToken 

func MakeToken(user *model.User) (tokenStr string, err error)

func ParserCron 

func ParserCron(spec string) error

验证cron表达式是否正确

func ProcessSpiderFilesFromConfigData 

func ProcessSpiderFilesFromConfigData(spider model.Spider, configData entity.ConfigSpiderData) error

func PublishAllSpiders 

func PublishAllSpiders()

发布所有爬虫

func PublishSpider 

func PublishSpider(spider model.Spider)

发布爬虫

func ReadFileByStep 

func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCreate *mgo.GridFile) error

分片读取大文件

func RemoveSpider 

func RemoveSpider(id string) error

func ResetSpiderGit 

func ResetSpiderGit(s model.Spider) (err error)

重置爬虫Git

func RestartTask 

func RestartTask(id string, uid bson.ObjectId) (err error)

func RetryUploadToGridFs 

func RetryUploadToGridFs(fileName string, filePath string) (fid bson.ObjectId, err error)

带重试功能的上传至 GridFS

func SaveScrapyItems 

func SaveScrapyItems(s model.Spider, itemsData []entity.ScrapyItem) (err error)

func SaveScrapySettings 

func SaveScrapySettings(s model.Spider, settingsData []entity.ScrapySettingParam) (err error)

func SaveSpiderGitSyncError 

func SaveSpiderGitSyncError(s model.Spider, errMsg string)

保存爬虫Git同步错误

func SaveTaskResultCount 

func SaveTaskResultCount(id string) func()

统计任务结果数

func ScanErrorLogs 

func ScanErrorLogs(t model.Task) func()

Scan Error Logs

func SecretFunc 

func SecretFunc() jwt.Keyfunc

func SendHeartBeat 

func SendHeartBeat()

发送心跳信息到Redis，每5秒发送一次

func SendNotifications 

func SendNotifications(u model.User, t model.Task, s model.Spider)

func SendTaskDingTalk 

func SendTaskDingTalk(u model.User, t model.Task, s model.Spider)

func SendTaskEmail 

func SendTaskEmail(u model.User, t model.Task, s model.Spider)

func SendTaskWechat 

func SendTaskWechat(u model.User, t model.Task, s model.Spider)

func SendWebHookRequest 

func SendWebHookRequest(u model.User, t model.Task, s model.Spider)

func SetEnv 

func SetEnv(cmd *exec.Cmd, envs []model.Env, task model.Task, spider model.Spider) *exec.Cmd

设置环境变量

func SetLogConfig 

func SetLogConfig(wg *sync.WaitGroup, cmd *exec.Cmd, t model.Task, u model.User) error

func SpiderFileCheck 

func SpiderFileCheck(t model.Task, spider model.Spider) error

func StartTaskProcess 

func StartTaskProcess(cmd *exec.Cmd, t model.Task) error

func SyncSpiderGit 

func SyncSpiderGit(s model.Spider) (err error)

同步爬虫Git

func UpdateGitCron 

func UpdateGitCron()

更新Git定时任务

func UpdateNodeData 

func UpdateNodeData()

更新节点数据

func UpdateNodeInfo 

func UpdateNodeInfo(data *Data) (err error)

处理节点信息

func UpdateNodeStatus 

func UpdateNodeStatus()

更新所有节点状态

func UpdateNodeStatusPeriodically 

func UpdateNodeStatusPeriodically()

每10秒刷新一次节点信息

func UpdateOfflineNodeTaskToAbnormalPeriodically 

func UpdateOfflineNodeTaskToAbnormalPeriodically()

每60秒更新异常节点信息

func UpdatePythonDepList 

func UpdatePythonDepList()

更新Python依赖列表到Redis

func UpdateSchedules 

func UpdateSchedules()

func UpdateSpiderDedup 

func UpdateSpiderDedup(spider model.Spider) error

func UploadSpiderToGridFsFromMaster 

func UploadSpiderToGridFsFromMaster(spider model.Spider) error

从主节点上传爬虫到GridFS

func UploadToGridFs 

func UploadToGridFs(fileName string, filePath string) (fid bson.ObjectId, err error)

上传zip文件到GridFS

func ValidateSpiderfile 

func ValidateSpiderfile(configData entity.ConfigSpiderData) error

验证Spiderfile

func WaitTaskProcess 

func WaitTaskProcess(cmd *exec.Cmd, t model.Task, s model.Spider) error

func WriteToGridFS 

func WriteToGridFS(content []byte, f *mgo.GridFile)

写入grid fs

Types

type Data 

type Data struct {
	Key      string `json:"key"`
	Mac      string `json:"mac"`
	Ip       string `json:"ip"`
	Hostname string `json:"hostname"`
	Name     string `json:"name"`
	NameType string `json:"name_type"`

	Master       bool      `json:"master"`
	UpdateTs     time.Time `json:"update_ts"`
	UpdateTsUnix int64     `json:"update_ts_unix"`
}

func GetNodeData 

func GetNodeData() (Data, error)

获取节点数据

func GetRedisNode 

func GetRedisNode(key string) (*Data, error)

type Executor 

type Executor struct {
	Cron *cron.Cron
}

任务执行器 

var Exec *Executor

func (*Executor) Start 

func (ex *Executor) Start() error

启动任务执行器

type GitBranch 

type GitBranch struct {
	Hash  string `json:"hash"`
	Name  string `json:"name"`
	Label string `json:"label"`
}

func GetGitBranches 

func GetGitBranches(s model.Spider) (branches []GitBranch, err error)

获取Git分支

func GetGitRemoteBranches 

func GetGitRemoteBranches(s model.Spider) (branches []GitBranch, err error)

获取Git远端分支

type GitCommit 

type GitCommit struct {
	Hash           string      `json:"hash"`
	TreeHash       string      `json:"tree_hash"`
	Author         string      `json:"author"`
	Email          string      `json:"email"`
	Message        string      `json:"message"`
	IsHead         bool        `json:"is_head"`
	Ts             time.Time   `json:"ts"`
	Branches       []GitBranch `json:"branches"`
	RemoteBranches []GitBranch `json:"remote_branches"`
	Tags           []GitTag    `json:"tags"`
}

func GetGitCommits 

func GetGitCommits(s model.Spider) (commits []GitCommit, err error)

获取Git Commits

type GitCronScheduler 

type GitCronScheduler struct {
	// contains filtered or unexported fields
}
var GitCron *GitCronScheduler

func (*GitCronScheduler) AddJob 

func (g *GitCronScheduler) AddJob(s model.Spider) error

func (*GitCronScheduler) RemoveAll 

func (g *GitCronScheduler) RemoveAll()

func (*GitCronScheduler) Start 

func (g *GitCronScheduler) Start() error

func (*GitCronScheduler) Update 

func (g *GitCronScheduler) Update() error

type GitTag 

type GitTag struct {
	Hash  string `json:"hash"`
	Name  string `json:"name"`
	Label string `json:"label"`
}

func GetGitTags 

func GetGitTags(s model.Spider) (tags []GitTag, err error)

获取Git Tags

type PythonDepJsonData 

type PythonDepJsonData struct {
	Info PythonDepJsonDataInfo `json:"info"`
}

type PythonDepJsonDataInfo 

type PythonDepJsonDataInfo struct {
	Name    string `json:"name"`
	Summary string `json:"summary"`
	Version string `json:"version"`
}

type PythonDepNameDict 

type PythonDepNameDict struct {
	Name   string `json:"name"`
	Weight int    `json:"weight"`
}

type PythonDepNameDictSlice 

type PythonDepNameDictSlice []PythonDepNameDict

func (PythonDepNameDictSlice) Len 

func (s PythonDepNameDictSlice) Len() int

func (PythonDepNameDictSlice) Less 

func (s PythonDepNameDictSlice) Less(i, j int) bool

func (PythonDepNameDictSlice) Swap 

func (s PythonDepNameDictSlice) Swap(i, j int)

type Scheduler 

type Scheduler struct {
	// contains filtered or unexported fields
}
var Sched *Scheduler

func (*Scheduler) AddJob 

func (s *Scheduler) AddJob(job model.Schedule) error

func (*Scheduler) Disable 

func (s *Scheduler) Disable(id bson.ObjectId) error

禁用定时任务

func (*Scheduler) Enable 

func (s *Scheduler) Enable(id bson.ObjectId) error

启用定时任务

func (*Scheduler) RemoveAll 

func (s *Scheduler) RemoveAll()

func (*Scheduler) Start 

func (s *Scheduler) Start() error

func (*Scheduler) Update 

func (s *Scheduler) Update() error

type SpiderFileData 

type SpiderFileData struct {
	FileName string
	File     []byte
}

type SpiderUploadMessage 

type SpiderUploadMessage struct {
	FileId   string
	FileName string
	SpiderId string
}

type TaskMessage 

type TaskMessage struct {
	Id  string
	Cmd string
}

任务消息

func (*TaskMessage) ToString 

func (m *TaskMessage) ToString() (string, error)

序列化任务消息

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.