Documentation
¶
Overview ¶
internal/task/fetch_goods_details_headless.go
internal/task/process_goods_details_headless.go
internal/task/remedy_video_details_headless.go
Index ¶
- Constants
- Variables
- func NewLoadCookiesAction(cookiePath string, logger *log.Helper) chromedp.Action
- func ProvideSchedulerWithTasks(fm *fetcher.FetcherManager, httpUC *fetcher.HttpUsecase, ...) *scheduler.Scheduler
- func WrapContextForDebug(ctx context.Context, logger *log.Helper, name string) context.Context
- type FetchGoodsDetailsHeadlessTask
- type FetchVideoDetailsHeadlessTask
- type FetchVideoRankTask
- type FetchVideoSummaryTask
- type FetchVideoTrendTask
- type ProcessGoodsDetailsHeadlessTask
- type ProcessVideoDetailHeadlessTask
- type ProcessVideoRankTask
- type ProcessVideoSummaryTask
- type ProcessVideoTrendTask
- type RemedyVideoDetailsHeadlessTask
Constants ¶
View Source
const ( FetchVideoRank = "fetch:video_rank" ProcessVideoRank = "process:video_rank" FetchVideoTrend = "fetch:video_trend" ProcessVideoTrend = "process:video_trend" FetchVideoSummary = "fetch:video_summary" ProcessVideoSummary = "process:video_summary" FetchVideoDetailsHeadless = "fetch:video_details_headless" ProcessVideoDetailsHeadless = "process:video_details_headless" // <-- 新增此行 RemedyVideoDetailsHeadless = "remedy:video_details_headless" FetchGoodsDetailsHeadless = "fetch:goods_details_headless" // <--- 新增常量 ProcessGoodsDetailsHeadless = "process:goods_details_headless" // <--- 新增常量 )
Variables ¶
View Source
var SchedulerProviderSet = wire.NewSet(ProvideSchedulerWithTasks)
SchedulerProviderSet 暴露 ProvideSchedulerWithTasks 用于依赖注入。
Functions ¶
func NewLoadCookiesAction ¶
NewLoadCookiesAction 【最终版】完全复刻您旧代码中成功的逻辑。 它接收一个文件路径，并返回一个Action。
func ProvideSchedulerWithTasks ¶
func ProvideSchedulerWithTasks( fm *fetcher.FetcherManager, httpUC *fetcher.HttpUsecase, headlessUC *fetcher.HeadlessUsecase, etlUC *etl.ETLUsecase, logger log.Logger, ) *scheduler.Scheduler
ProvideSchedulerWithTasks 创建一个 Scheduler， 注册所有任务定义 与它们的策略，并返回未运行的调度器实例。
Types ¶
type FetchGoodsDetailsHeadlessTask ¶
type FetchGoodsDetailsHeadlessTask struct {
// contains filtered or unexported fields
}
func NewFetchGoodsDetailsHeadlessTask ¶
func NewFetchGoodsDetailsHeadlessTask( logger log.Logger, huc *fetcher.HeadlessUsecase, ) *FetchGoodsDetailsHeadlessTask
type FetchVideoDetailsHeadlessTask ¶
type FetchVideoDetailsHeadlessTask struct {
// contains filtered or unexported fields
}
FetchVideoDetailsHeadlessTask 重新定义的任务结构体，只包含必要字段
func NewFetchVideoDetailsHeadlessTask ¶
func NewFetchVideoDetailsHeadlessTask( logger log.Logger, huc *fetcher.HeadlessUsecase, ) *FetchVideoDetailsHeadlessTask
NewFetchVideoDetailsHeadlessTask 重写构造函数，初始化所有字段
type FetchVideoRankTask ¶
type FetchVideoRankTask struct {
// contains filtered or unexported fields
}
func NewFetchVideoRankTask ¶
func NewFetchVideoRankTask(logger log.Logger, httpUC *fetcher.HttpUsecase) *FetchVideoRankTask
func (*FetchVideoRankTask) Run ¶
func (t *FetchVideoRankTask) Run(ctx context.Context, dataSourceName string, params ...interface{}) error
todo 当前不能执行完一次就休息，理论上，应该可以每次启动任务时控制一些参数，决定采取多少的，让整个任务停下来。 INFO ts=2025-07-30T11:42:36+08:00 caller=task/fetch_video_rank.go:70 module=task.fetch_video_rank msg=Successfully completed video rank fetch task on datasource 'feigua_http_1' INFO ts=2025-07-30T11:42:36+08:00 caller=scheduler/scheduler.go:174 module=scheduler msg=在数据源 [feigua_http_1] 上成功完成一轮任务 'fetch:video_rank'。 INFO ts=2025-07-30T11:42:36+08:00 caller=scheduler/scheduler.go:194 module=scheduler msg=战役间休整：根据调度策略，长休眠 2m39s... 这里应该停下来了。
type FetchVideoSummaryTask ¶
type FetchVideoSummaryTask struct {
// contains filtered or unexported fields
}
func NewFetchVideoSummaryTask ¶
func NewFetchVideoSummaryTask(fetcherUC *fetcher.HttpUsecase, videoRepo data.VideoRepo, logger log.Logger) *FetchVideoSummaryTask
func (*FetchVideoSummaryTask) Name ¶
func (t *FetchVideoSummaryTask) Name() string
type FetchVideoTrendTask ¶
type FetchVideoTrendTask struct {
// contains filtered or unexported fields
}
FetchVideoTrendTask 负责每日拉取视频趋势数据的任务
func NewFetchVideoTrendTask ¶
func NewFetchVideoTrendTask(fetcherUC *fetcher.HttpUsecase, videoRepo data.VideoRepo, logger log.Logger) *FetchVideoTrendTask
NewFetchVideoTrendTask 构造任务实例
func (*FetchVideoTrendTask) Name ¶
func (t *FetchVideoTrendTask) Name() string
type ProcessGoodsDetailsHeadlessTask ¶
type ProcessGoodsDetailsHeadlessTask struct {
// contains filtered or unexported fields
}
ProcessGoodsDetailsHeadlessTask 负责处理所有由无头浏览器采集的商品详情数据
func NewProcessGoodsDetailsHeadlessTask ¶
func NewProcessGoodsDetailsHeadlessTask(etl *etl.ETLUsecase, logger log.Logger) *ProcessGoodsDetailsHeadlessTask
type ProcessVideoDetailHeadlessTask ¶
type ProcessVideoDetailHeadlessTask struct {
// contains filtered or unexported fields
}
ProcessVideoDetailHeadlessTask 负责处理所有由无头浏览器采集的详情数据
func NewProcessVideoDetailHeadlessTask ¶
func NewProcessVideoDetailHeadlessTask(etl *etl.ETLUsecase, logger log.Logger) *ProcessVideoDetailHeadlessTask
NewProcessVideoDetailHeadlessTask .
type ProcessVideoRankTask ¶
type ProcessVideoRankTask struct {
// contains filtered or unexported fields
}
func NewProcessVideoRankTask ¶
func NewProcessVideoRankTask(etl *etl.ETLUsecase) *ProcessVideoRankTask
type ProcessVideoSummaryTask ¶
type ProcessVideoSummaryTask struct {
// contains filtered or unexported fields
}
func NewProcessVideoSummaryTask ¶
func NewProcessVideoSummaryTask(etl *etl.ETLUsecase) *ProcessVideoSummaryTask
type ProcessVideoTrendTask ¶
type ProcessVideoTrendTask struct {
// contains filtered or unexported fields
}
func NewProcessVideoTrendTask ¶
func NewProcessVideoTrendTask(etl *etl.ETLUsecase) *ProcessVideoTrendTask
func (*ProcessVideoTrendTask) Name ¶
func (t *ProcessVideoTrendTask) Name() string
type RemedyVideoDetailsHeadlessTask ¶
type RemedyVideoDetailsHeadlessTask struct {
// contains filtered or unexported fields
}
RemedyVideoDetailsHeadlessTask 负责修复部分采集失败的视频详情
func NewRemedyVideoDetailsHeadlessTask ¶
func NewRemedyVideoDetailsHeadlessTask( logger log.Logger, huc *fetcher.HeadlessUsecase, ) *RemedyVideoDetailsHeadlessTask
NewRemedyVideoDetailsHeadlessTask 创建一个新的修复任务实例
Source Files
¶
- debug_context.go
- fetch_goods_details_headless.go
- fetch_video_details_headless.go
- fetch_video_rank.go
- headless_utils.go
- old_fetch_video_summary.go
- old_fetch_video_trend.go
- process_goods_details_headless.go
- process_video_detail_headless.go
- process_video_rank.go
- process_video_summary.go
- process_video_trend.go
- registration.go
- remedy_video_details_headless.go
- scheduler_provider.go
Click to show internal directories.
Click to hide internal directories.