Documentation
¶
Overview ¶
internal/data/goods_aweme.go
internal/data/goods_blogger.go
internal/data/goods_portrayal.go
internal/data/goods_trend.go
Index ¶
- Constants
- Variables
- func CopyBloggerAwemeStatToDTO(stat *BloggerAwemeStat) *v1.BloggerAwemeStatDTO
- func CopyBloggerGoodsCategoryDistributionToDTO(d *BloggerGoodsCategoryDistribution) *v1.BloggerGoodsCategoryDistributionDTO
- func CopyBloggerGoodsSalesTrendToDTO(trend *BloggerGoodsSalesTrend) *v1.BloggerGoodsSalesTrendDTO
- func CopyBloggerGoodsStatToDTO(stat *BloggerGoodsStat) *v1.BloggerGoodsStatDTO
- func CopyBloggerPortrayalToDTO(p *BloggerPortrayal) *v1.BloggerPortrayalDTO
- func CopyBloggerToDTO(b *Blogger) *v1.BloggerDTO
- func CopyGoodsBloggerToDTO(gb *GoodsBlogger) *v1.GoodsBloggerDTO
- func CopyProductToDTO(p *Product) *v1.ProductDTO
- func CopySourceDataToDTO(s *SourceData) *v1.SourceData
- func CopyVideoRankToDTO(do *VideoRank) *v1.VideoRankDTO
- func CopyVideoToDTO(v *Video) *v1.VideoDTO
- func CopyVideoTrendToDTO(vt *VideoTrend) *v1.VideoTrendDTO
- func NewRedisClient(conf *conf.Data) redis.Cmdable
- type Blogger
- type BloggerAwemeStat
- type BloggerAwemeStatRepo
- type BloggerForCollection
- type BloggerGoodsCategoryDistribution
- type BloggerGoodsCategoryDistributionRepo
- type BloggerGoodsSalesTrend
- type BloggerGoodsSalesTrendRepo
- type BloggerGoodsStat
- type BloggerGoodsStatRepo
- type BloggerPortrayal
- type BloggerPortrayalRepo
- type BloggerRepo
- type Data
- type GoodsAweme
- type GoodsAwemeRepo
- type GoodsBlogger
- type GoodsBloggerRepo
- type GoodsPortrayal
- type GoodsPortrayalRepo
- type GoodsTrend
- type GoodsTrendRepo
- type JSONB
- type Product
- type ProductForCollection
- type ProductRepo
- type SourceData
- type SourceDataRepo
- type Video
- type VideoForCollection
- type VideoForSummary
- type VideoRank
- type VideoRankRepo
- type VideoRepo
- type VideoTrend
- type VideoTrendRepo
- type WorkOrder
- type WorkOrderRepo
- type WorkOrderStatus
Constants ¶
const ( DataTypeVideoRank = "video_rank" DataTypeVideoDetail = "video_detail" DataTypeVideoSummaryHeadless = "video_summary_headless" DataTypeVideoTrendHeadless = "video_trend_headless" DataTypeVideoSummaryHttp = "video_summary_http" DataTypeVideoTrendHttp = "video_trend_http" // DataTypeGoodsAwemeAnalysisHttp HTTP采集的商品分析数据 DataTypeGoodsAwemeAnalysisHttp = "goods_aweme_analysis_http" // DataTypeGoodsChartDataHttp HTTP采集的商品图表数据 DataTypeGoodsChartDataHttp = "goods_chart_data_http" // DataTypeGoodsPortrayalHttp HTTP采集的商品画像数据 DataTypeGoodsPortrayalHttp = "goods_portrayal_http" DataTypeGoodsAwemeAnalysisHeadless = "goods_aweme_analysis_headless" DataTypeGoodsChartDataHeadless = "goods_chart_data_headless" DataTypeGoodsPortrayalHeadless = "goods_portrayal_headless" // --- 【新增】Blogger详情相关的DataType常量 --- DataTypeBloggerMainPartHeadless = "blogger_mainpart_headless" DataTypeBloggerOtherPartHeadless = "blogger_otherpart_headless" DataTypeBloggerAwemeOverviewHeadless = "blogger_aweme_overview_headless" DataTypeBloggerAwemePortrayHeadless = "blogger_aweme_portray_headless" DataTypeBloggerCommerceSummaryHeadless = "blogger_commerce_summary_headless" DataTypeBloggerGoodsOverviewHeadless = "blogger_goods_overview_headless" DataTypeBloggerGoodsTrendsHeadless = "blogger_goods_trends_headless" DataTypeBloggerGoodsCateDistHeadless = "blogger_goods_cate_dist_headless" // DataType for blogger details fetched via HTTP DataTypeBloggerMainPartHttp = "blogger_mainpart_http" DataTypeBloggerOtherPartHttp = "blogger_otherpart_http" DataTypeBloggerAwemeOverviewHttp = "blogger_aweme_overview_http" DataTypeBloggerAwemePortrayalHttp = "blogger_aweme_portrayal_http" DataTypeBloggerCommerceSummaryHttp = "blogger_commerce_summary_http" DataTypeBloggerGoodsOverviewHttp = "blogger_goods_overview_http" DataTypeBloggerGoodsTrendsHttp = "blogger_goods_trends_http" DataTypeBloggerGoodsCateDistHttp = "blogger_goods_cate_dist_http" )
在这里定义data_type,全部地方都适用这里的定义
Variables ¶
var DataProviderSet = wire.NewSet( NewData, NewRedisClient, NewSourceDataRepo, NewVideoRankRepo, NewVideoRepo, NewVideoTrendRepo, NewProductRepo, NewBloggerRepo, NewGoodsAwemeRepo, NewGoodsTrendRepo, NewGoodsPortrayalRepo, NewGoodsBloggerRepo, NewBloggerPortrayalRepo, NewBloggerGoodsSalesTrendRepo, NewBloggerGoodsCategoryDistributionRepo, NewBloggerAwemeStatRepo, NewBloggerGoodsStatRepo, NewWorkOrderRepo, )
DataProviderSet is data providers.
Functions ¶
func CopyBloggerAwemeStatToDTO ¶
func CopyBloggerAwemeStatToDTO(stat *BloggerAwemeStat) *v1.BloggerAwemeStatDTO
func CopyBloggerGoodsCategoryDistributionToDTO ¶
func CopyBloggerGoodsCategoryDistributionToDTO(d *BloggerGoodsCategoryDistribution) *v1.BloggerGoodsCategoryDistributionDTO
CopyBloggerGoodsCategoryDistributionToDTO converts a BloggerGoodsCategoryDistribution model to a DTO.
func CopyBloggerGoodsSalesTrendToDTO ¶
func CopyBloggerGoodsSalesTrendToDTO(trend *BloggerGoodsSalesTrend) *v1.BloggerGoodsSalesTrendDTO
CopyBloggerGoodsSalesTrendToDTO converts a BloggerGoodsSalesTrend model to a DTO.
func CopyBloggerGoodsStatToDTO ¶
func CopyBloggerGoodsStatToDTO(stat *BloggerGoodsStat) *v1.BloggerGoodsStatDTO
func CopyBloggerPortrayalToDTO ¶
func CopyBloggerPortrayalToDTO(p *BloggerPortrayal) *v1.BloggerPortrayalDTO
CopyBloggerPortrayalToDTO 将 BloggerPortrayal 模型转换为 DTO
func CopyBloggerToDTO ¶
func CopyBloggerToDTO(b *Blogger) *v1.BloggerDTO
CopyBloggerToDTO 将 data.Blogger 模型转换为 v1.BloggerDTO
func CopyGoodsBloggerToDTO ¶
func CopyGoodsBloggerToDTO(gb *GoodsBlogger) *v1.GoodsBloggerDTO
func CopyProductToDTO ¶
func CopyProductToDTO(p *Product) *v1.ProductDTO
CopyProductToDTO 将 data.Product 模型转换为 v1.ProductDTO
func CopySourceDataToDTO ¶
func CopySourceDataToDTO(s *SourceData) *v1.SourceData
func CopyVideoRankToDTO ¶
func CopyVideoRankToDTO(do *VideoRank) *v1.VideoRankDTO
func CopyVideoToDTO ¶
CopyVideoToDTO 将 data.Video 模型转换为 v1.VideoDTO
func CopyVideoTrendToDTO ¶
func CopyVideoTrendToDTO(vt *VideoTrend) *v1.VideoTrendDTO
CopyVideoTrendToDTO 将 data.VideoTrend 模型转换为 v1.VideoTrendDTO
Types ¶
type Blogger ¶
type Blogger struct {
// --- Rank ETL 维护的核心字段 (保持不变) ---
BloggerId int64 `gorm:"primaryKey;column:blogger_id"`
CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:timestamp"`
UpdatedAt time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
BloggerUid string `gorm:"size:255;index;column:blogger_uid"`
BloggerName string `gorm:"size:255;column:blogger_name"`
BloggerAvatar string `gorm:"size:1024;column:blogger_avatar"`
BloggerFansNum int64 `gorm:"column:blogger_fans_num"`
BloggerTag string `gorm:"size:255;column:blogger_tag"`
BloggerDetailUrl string `gorm:"column:blogger_detail_url;size:1024;not null;default:''"`
RankTime time.Time `gorm:"column:rank_time;comment:上榜时间"`
// --- Detail ETL 维护的补充字段 (全面修订) ---
// 身份补充 (来自 mainpart)
UniqueId string `gorm:"size:255;index;column:unique_id;comment:平台UniqueId"`
Gender string `gorm:"size:16;column:gender;comment:性别"`
Location string `gorm:"size:64;column:location;comment:地点"`
Signature string `gorm:"type:text;column:signature;comment:签名"`
FollowerCount int64 `gorm:"column:follower_count;comment:精确粉丝数 (Fans字段)"`
FollowerCountStr string `gorm:"size:64;column:follower_count_str;comment:平台显示粉丝数 (Platform_Fans字段)"`
LastPublishAt time.Time `gorm:"column:last_publish_at;comment:最后发布时间 (LastPubTime)"`
BloggerAddTime time.Time `gorm:"column:blogger_add_time;comment:达人入库时间"`
FansMilestone JSONB `gorm:"column:fans_milestone;comment:粉丝里程碑"`
OpenBloggerId string `gorm:"size:255;index;column:open_blogger_id;comment:开放平台ID"`
// 统计指标 (来自 mainpart)
Score string `gorm:"size:32;column:score;comment:平台综合评分"`
AwemesCount int64 `gorm:"column:awemes_count;comment:总作品数"`
LikeCountStr string `gorm:"size:64;column:like_count_str;comment:总获赞数"`
AvgLikeCountStr string `gorm:"size:64;column:avg_like_count_str;comment:集均获赞"`
CommentCountStr string `gorm:"size:64;column:comment_count_str;comment:总评论数"`
AvgCommentCountStr string `gorm:"size:64;column:avg_comment_count_str;comment:集均评论"`
FansClubCount string `gorm:"size:64;column:fans_club_count;comment:粉丝团人数"`
// 状态与榜单 (来自 otherpart)
BloggerTypeName string `gorm:"size:64;column:blogger_type_name;comment:达人类型, 如'腰部达人'"`
TagRankNum int64 `gorm:"column:tag_rank_num;comment:分类榜单排名"`
TagRankDesc string `gorm:"size:255;column:tag_rank_desc;comment:分类榜单描述"`
PromotionRankNum int64 `gorm:"column:promotion_rank_num;comment:带货榜单排名"`
PromotionRankDesc string `gorm:"size:255;column:promotion_rank_desc;comment:带货榜单描述"`
BetaNearlyAWeek string `gorm:"size:32;column:beta_nearly_a_week;comment:近7日Beta系数"`
SalesGMV30Day string `gorm:"size:64;column:sales_gmv_30_day;comment:近30日GMV"`
TopicsCount int64 `gorm:"column:topics_count;comment:话题数 (Topics)"`
// 口碑与电商 (来自 otherpart)
KBScore string `gorm:"size:32;column:kb_score;comment:口碑分"`
KBPercentage string `gorm:"size:32;column:kb_percentage;comment:口碑分超越百分比"`
KBSales string `gorm:"size:64;column:kb_sales;comment:口碑销量"`
KBSales30Days string `gorm:"size:64;column:kb_sales_30_days;comment:近30日口碑销量"`
// 标签与关联 (来自 otherpart)
AwemeTagList JSONB `gorm:"column:aweme_tag_list;comment:视频内容标签"`
LiveTagList JSONB `gorm:"column:live_tag_list;comment:直播内容标签"`
IsBindBrand bool `gorm:"column:is_bind_brand;comment:是否绑定品牌"`
IsBindShop bool `gorm:"column:is_bind_shop;comment:是否绑定店铺"`
IsHaveMcn bool `gorm:"column:is_have_mcn;comment:是否有MCN"`
}
Blogger 博主维度表 Blogger 博主维度表,存储达人全面、丰富的档案信息
type BloggerAwemeStat ¶
type BloggerAwemeStat struct {
BloggerId int64 `gorm:"primaryKey;column:blogger_id;comment:达人ID"`
UpdatedAt time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
AwemeCount string `gorm:"size:64;column:aweme_count;comment:总视频数"`
HasAwemeCount string `gorm:"size:64;column:has_aweme_count;comment:有播放视频数"`
SaleGmv string `gorm:"size:64;column:sale_gmv;comment:视频带货GMV"`
AvgSaleGmv string `gorm:"size:64;column:avg_sale_gmv;comment:场均视频带货GMV"`
SaleCount string `gorm:"size:64;column:sale_count;comment:视频带货销量"`
AvgSaleCount string `gorm:"size:64;column:avg_sale_count;comment:场均视频带货销量"`
HasPromotionCount string `gorm:"size:64;column:has_promotion_count;comment:关联商品视频数"`
LikeCount string `gorm:"size:64;column:like_count;comment:获赞数"`
AvgLikeCount string `gorm:"size:64;column:avg_like_count;comment:集均获赞"`
CommentCount string `gorm:"size:64;column:comment_count;comment:评论数"`
AvgCommentCount string `gorm:"size:64;column:avg_comment_count;comment:集均评论"`
CollectCount string `gorm:"size:64;column:collect_count;comment:收藏数"`
AvgCollectCount string `gorm:"size:64;column:avg_collect_count;comment:集均收藏"`
AvgPrice string `gorm:"size:64;column:avg_price;comment:视频带货客单价"`
}
BloggerAwemeStat 存储达人视频创作的聚合统计数据
type BloggerAwemeStatRepo ¶
type BloggerAwemeStatRepo interface {
Upsert(ctx context.Context, stat *BloggerAwemeStat) error
Get(ctx context.Context, bloggerId int64) (*BloggerAwemeStat, error)
}
func NewBloggerAwemeStatRepo ¶
func NewBloggerAwemeStatRepo(data *Data) BloggerAwemeStatRepo
type BloggerForCollection ¶
type BloggerForCollection struct {
BloggerId int64
BloggerUid string
BloggerDetailUrl string
WorkOrderID string `gorm:"-" json:"work_order_id"`
}
BloggerForCollection 是为采集任务准备的轻量级结构体
type BloggerGoodsCategoryDistribution ¶
type BloggerGoodsCategoryDistribution struct {
Id int64 `gorm:"primaryKey;column:id;autoIncrement:true"`
BloggerId int64 `gorm:"index;not null;column:blogger_id;comment:达人ID"`
CategoryName string `gorm:"type:varchar(255);column:category_name;comment:品类名称"`
ShortCategoryName string `gorm:"type:varchar(255);column:short_category_name;comment:简短品类名称"`
CategoryId string `gorm:"type:varchar(64);column:category_id;comment:品类ID (Cate)"`
GoodsCount int64 `gorm:"column:goods_count;comment:商品数"`
SaleCount int64 `gorm:"column:sale_count;comment:销量"`
SaleGmv int64 `gorm:"column:sale_gmv;comment:销售额 (分)"`
AwemeCount int64 `gorm:"column:aweme_count;comment:视频数"`
LiveCount int64 `gorm:"column:live_count;comment:直播数"`
BloggerCount int64 `gorm:"column:blogger_count;comment:关联达人数"`
AvgKdj float64 `gorm:"column:avg_kdj;comment:平均客单价 (KDJ)"`
AvgKdjStr string `gorm:"type:varchar(64);column:avg_kdj_str;comment:客单价范围字符串"`
CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:timestamp"`
UpdatedAt time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
}
BloggerGoodsCategoryDistribution 存储达人带货的品类分布数据
func (BloggerGoodsCategoryDistribution) TableName ¶
func (BloggerGoodsCategoryDistribution) TableName() string
tablename
type BloggerGoodsCategoryDistributionRepo ¶
type BloggerGoodsCategoryDistributionRepo interface {
RefreshForBlogger(ctx context.Context, bloggerId int64, distributions []*BloggerGoodsCategoryDistribution) error
ListPage(ctx context.Context, page, size int, bloggerId int64, sortBy string, sortOrder v1.SortOrder) ([]*BloggerGoodsCategoryDistribution, int64, error)
}
func NewBloggerGoodsCategoryDistributionRepo ¶
func NewBloggerGoodsCategoryDistributionRepo(data *Data) BloggerGoodsCategoryDistributionRepo
type BloggerGoodsSalesTrend ¶
type BloggerGoodsSalesTrend struct {
Id int64 `gorm:"column:id;primaryKey;autoIncrement:true"`
BloggerId int64 `gorm:"column:blogger_id;index:idx_blogger_datecode,unique;not null"`
DateCode int `gorm:"column:date_code;index:idx_blogger_datecode,unique;not null;comment:日期 YYYYMMDD"`
SaleCount int64 `gorm:"column:sale_count;comment:总销量"`
SaleCountStr string `gorm:"size:64;column:sale_count_str;comment:总销量字符串"`
SaleGmv int64 `gorm:"column:sale_gmv;comment:总GMV"`
SaleGmvStr string `gorm:"size:64;column:sale_gmv_str;comment:总GMV字符串"`
AwemeSaleCount int64 `gorm:"column:aweme_sale_count;comment:视频带货销量"`
AwemeSaleCountStr string `gorm:"size:64;column:aweme_sale_count_str;comment:视频带货销量字符串"`
AwemeSaleGmv int64 `gorm:"column:aweme_sale_gmv;comment:视频带货GMV"`
AwemeSaleGmvStr string `gorm:"size:64;column:aweme_sale_gmv_str;comment:视频带货GMV字符串"`
LiveSaleCount int64 `gorm:"column:live_sale_count;comment:直播带货销量"`
LiveSaleCountStr string `gorm:"size:64;column:live_sale_count_str;comment:直播带货销量字符串"`
LiveSaleGmv int64 `gorm:"column:live_sale_gmv;comment:直播带货GMV"`
LiveSaleGmvStr string `gorm:"size:64;column:live_sale_gmv_str;comment:直播带货GMV字符串"`
PicTimeStr string `gorm:"size:64;column:pic_time_str;comment:图片时间字符串"`
ListTimeStr string `gorm:"size:64;column:list_time_str;comment:列表时间字符串"`
TimeStamp int64 `gorm:"column:timestamp;comment:时间戳"`
DateCodeStr string `gorm:"size:64;column:date_code_str;comment:日期字符串"`
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime"`
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime"`
}
BloggerGoodsSalesTrend 存储达人商品销售的每日趋势数据
func (BloggerGoodsSalesTrend) TableName ¶
func (BloggerGoodsSalesTrend) TableName() string
type BloggerGoodsSalesTrendRepo ¶
type BloggerGoodsSalesTrendRepo interface {
BatchOverwrite(ctx context.Context, trends []*BloggerGoodsSalesTrend) error
ListPage(ctx context.Context, bloggerId int64, page, size int, sortBy string, sortOrder v1.SortOrder) ([]*BloggerGoodsSalesTrend, int64, error)
}
func NewBloggerGoodsSalesTrendRepo ¶
func NewBloggerGoodsSalesTrendRepo(data *Data) BloggerGoodsSalesTrendRepo
type BloggerGoodsStat ¶
type BloggerGoodsStat struct {
BloggerId int64 `gorm:"primaryKey;column:blogger_id;comment:达人ID"`
UpdatedAt time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
SaleCount string `gorm:"size:64;column:sale_count;comment:总销量"`
SaleGmv string `gorm:"size:64;column:sale_gmv;comment:总销售额"`
LiveSaleCount string `gorm:"size:64;column:live_sale_count;comment:直播销量"`
LiveSaleGmv string `gorm:"size:64;column:live_sale_gmv;comment:直播销售额"`
AwemeSaleCount string `gorm:"size:64;column:aweme_sale_count;comment:视频销量"`
AwemeSaleGmv string `gorm:"size:64;column:aweme_sale_gmv;comment:视频销售额"`
SaleGoodsCount string `gorm:"size:64;column:sale_goods_count;comment:在售商品数"`
AvgUserPrice string `gorm:"size:64;column:avg_user_price;comment:客单价"`
MainCategoryName string `gorm:"size:128;column:main_category_name;comment:主营品类"`
MainCategoryRatio string `gorm:"size:32;column:main_category_ratio;comment:主营品类占比"`
CategoryCount string `gorm:"size:32;column:category_count;comment:品类总数"`
TopBrands string `gorm:"type:text;column:top_brands;comment:合作品牌TOP,逗号分隔"`
TopShops string `gorm:"type:text;column:top_shops;comment:合作店铺TOP,逗号分隔"`
CommerceChannel string `gorm:"size:32;column:commerce_channel;comment:主要带货渠道"`
CommerceChannelRatio string `gorm:"size:32;column:commerce_channel_ratio;comment:主要渠道占比"`
}
BloggerGoodsStat 存储达人电商表现的聚合统计数据
type BloggerGoodsStatRepo ¶
type BloggerGoodsStatRepo interface {
Upsert(ctx context.Context, stat *BloggerGoodsStat) error
Get(ctx context.Context, bloggerID int64) (*BloggerGoodsStat, error)
}
func NewBloggerGoodsStatRepo ¶
func NewBloggerGoodsStatRepo(data *Data) BloggerGoodsStatRepo
type BloggerPortrayal ¶
type BloggerPortrayal struct {
Id int64 `gorm:"primaryKey;column:id;autoIncrement:true"`
BloggerId int64 `gorm:"index;not null;column:blogger_id;comment:达人ID"`
PortrayalData JSONB `gorm:"type:jsonb;column:portrayal_data;comment:完整的画像JSON数据"`
CreatedAt time.Time `gorm:"autoCreateTime;column:created_at;type:timestamp"`
UpdatedAt time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
}
BloggerPortrayal 存储达人的粉丝画像数据
type BloggerPortrayalRepo ¶
type BloggerPortrayalRepo interface {
Save(ctx context.Context, portrayal *BloggerPortrayal) error
Upsert(ctx context.Context, portrayal *BloggerPortrayal) error
Get(ctx context.Context, bloggerId int64) (*v1.BloggerPortrayalDTO, error)
}
func NewBloggerPortrayalRepo ¶
func NewBloggerPortrayalRepo(data *Data) BloggerPortrayalRepo
type BloggerRepo ¶
type BloggerRepo interface {
Upsert(ctx context.Context, blogger *Blogger) error
UpdateDetails(ctx context.Context, blogger *Blogger) error
ListPage(ctx context.Context, page, size int, query, sortBy string, sortOrder v1.SortOrder) ([]*Blogger, int64, error)
Get(ctx context.Context, bloggerId int64) (*v1.BloggerDTO, error)
FindBloggersForCollection(ctx context.Context, limit int) ([]*BloggerForCollection, error)
FindBloggersExcludingIDs(ctx context.Context, ids []string, limit int, t time.Time) ([]*BloggerForCollection, error)
GetByBloggerUid(ctx context.Context, bloggerId int64) (*Blogger, error)
}
func NewBloggerRepo ¶
func NewBloggerRepo(data *Data) BloggerRepo
type GoodsAweme ¶
type GoodsAweme struct {
Id int64 `gorm:"primarykey;autoIncrement"`
GoodsId string `gorm:"column:goods_id;size:255;not null;index"`
AwemeId string `gorm:"column:aweme_id;size:255;not null;uniqueIndex:idx_goods_aweme"`
BloggerId int64 `gorm:"column:blogger_id;not null"`
AwemePubTime time.Time `gorm:"column:aweme_pub_time"`
AwemeDesc string `gorm:"column:aweme_desc;type:text"`
AwemeCoverUrl string `gorm:"column:aweme_cover_url;size:1024"`
AwemeDetailUrl string `gorm:"column:aweme_detail_url;size:1024"`
AwemeSaleCountStr string `gorm:"column:aweme_sale_count_str;size:255"`
AwemeSaleGmvStr string `gorm:"column:aweme_sale_gmv_str;size:255"`
LikeCountStr string `gorm:"column:like_count_str;size:255"`
CommentCountStr string `gorm:"column:comment_count_str;size:255"`
PlayCountStr string `gorm:"column:play_count_str;size:255"`
CollectCountStr string `gorm:"column:collect_count_str;size:255"`
GPM string `gorm:"column:gpm;size:255"`
CreatedAt time.Time `gorm:"column:created_at"`
UpdatedAt time.Time `gorm:"column:updated_at"`
}
GoodsAweme 对应 /api/v3/goods/aweme/loadAwemeAnalysis 接口返回的 Items
func (*GoodsAweme) TableName ¶
func (ga *GoodsAweme) TableName() string
type GoodsAwemeRepo ¶
type GoodsAwemeRepo interface {
BatchCreate(ctx context.Context, awemes []*GoodsAweme) error
List(ctx context.Context, req *v1.ListGoodsAwemesRequest) ([]*GoodsAweme, int64, error)
}
func NewGoodsAwemeRepo ¶
func NewGoodsAwemeRepo(data *Data, logger log.Logger) GoodsAwemeRepo
type GoodsBlogger ¶
type GoodsBlogger struct {
Id int64 `gorm:"primarykey;autoIncrement"`
// 复合唯一索引,确保一个商品只关联一个达人一次
GoodsId string `gorm:"column:goods_id;size:255;not null;uniqueIndex:idx_goods_blogger"`
BloggerId int64 `gorm:"column:blogger_id;not null;uniqueIndex:idx_goods_blogger"`
// --- 达人冗余信息 ---
BloggerUid string `gorm:"size:255"`
BloggerName string `gorm:"size:255"`
BloggerAvatar string `gorm:"size:1024"`
FansNum int64
Tag string `gorm:"size:255"`
// --- 视频带货聚合数据 ---
AwemeCountStr string `gorm:"column:aweme_count_str;size:255"` // 视频数量
AwemeSaleCountStr string `gorm:"column:aweme_sale_count_str;size:255"`
AwemeSaleGmvStr string `gorm:"column:aweme_sale_gmv_str;size:255"`
CreatedAt time.Time `gorm:"column:created_at"`
UpdatedAt time.Time `gorm:"column:updated_at"`
}
GoodsBlogger 存储商品和达人的关联关系,并冗余达人核心信息
func (*GoodsBlogger) TableName ¶
func (gb *GoodsBlogger) TableName() string
type GoodsBloggerRepo ¶
type GoodsBloggerRepo interface {
BatchUpsert(ctx context.Context, goodsBloggers []*GoodsBlogger) error
List(ctx context.Context, req *v1.ListGoodsBloggersRequest) ([]*GoodsBlogger, int64, error)
}
func NewGoodsBloggerRepo ¶
func NewGoodsBloggerRepo(data *Data, logger log.Logger) GoodsBloggerRepo
type GoodsPortrayal ¶
type GoodsPortrayal struct {
Id int64 `gorm:"primaryKey"`
CreatedAt time.Time `gorm:"autoCreateTime"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
GoodsId string `gorm:"size:255;not null;uniqueIndex"`
PortrayalData JSONB `gorm:"type:jsonb"` // 使用 JSONB 类型
}
GoodsPortrayal 存储完整的用户画像数据
func (GoodsPortrayal) TableName ¶
func (GoodsPortrayal) TableName() string
type GoodsPortrayalRepo ¶
type GoodsPortrayalRepo interface {
CreateOrUpdate(ctx context.Context, portrayal *GoodsPortrayal) error
GetByGID(ctx context.Context, gid string) (*GoodsPortrayal, error) // <--- 新增方法
}
func NewGoodsPortrayalRepo ¶
func NewGoodsPortrayalRepo(data *Data) GoodsPortrayalRepo
type GoodsTrend ¶
type GoodsTrend struct {
Id int64 `gorm:"primaryKey"`
CreatedAt time.Time `gorm:"autoCreateTime;type:timestamp"`
UpdatedAt time.Time `gorm:"autoUpdateTime;type:timestamp"`
GoodsId string `gorm:"size:255;not null;index:idx_goods_date,unique"`
DateCode int `gorm:"not null;index:idx_goods_date,unique"`
AwemeCount int64
AwemeCountStr string `gorm:"size:255"`
BloggerCount int64
BloggerCountStr string `gorm:"size:255"`
AwemeSaleGmv float64
AwemeSaleGmvStr string `gorm:"size:255"`
AwemeSaleCount int64
AwemeSaleCountStr string `gorm:"size:255"`
ListTimeStr string `gorm:"size:255"`
TimeStamp int64
}
GoodsTrend 存储商品每日的带货趋势数据
func (GoodsTrend) TableName ¶
func (GoodsTrend) TableName() string
type GoodsTrendRepo ¶
type GoodsTrendRepo interface {
BatchOverwrite(ctx context.Context, trends []*GoodsTrend) error
List(ctx context.Context, req *v1.ListGoodsTrendsRequest) ([]*GoodsTrend, int64, error)
}
func NewGoodsTrendRepo ¶
func NewGoodsTrendRepo(data *Data) GoodsTrendRepo
type Product ¶
type Product struct {
GoodsId string `gorm:"primaryKey;size:1024"`
CreatedAt time.Time `gorm:"autoCreateTime;type:timestamp"`
UpdatedAt time.Time `gorm:"autoUpdateTime;type:timestamp"`
GoodsTitle string `gorm:"type:text"`
GoodsCoverUrl string `gorm:"size:1024"`
GoodsPriceRange string `gorm:"size:255"`
GoodsPrice float64 `gorm:"type:numeric"`
CosRatio string `gorm:"size:255"`
CommissionPrice string `gorm:"size:255"`
ShopName string `gorm:"size:255"`
BrandName string `gorm:"size:255"`
CategoryNames string `gorm:"size:1024"`
GoodsDetailUrl string `gorm:"column:goods_detail_url;size:1024;not null;default:''"`
RankTime time.Time `gorm:"column:rank_time;comment:上榜时间"`
// Summary from /api/v3/goods/aweme/loadAwemeAnalysis
AwemeCount int64 `gorm:"column:aweme_count;not null;default:0"`
BloggerCount int64 `gorm:"column:blogger_count;not null;default:0"`
AwemeSaleCountStr string `gorm:"column:aweme_sale_count_str;size:255;not null;default:''"`
AwemeSaleGmvStr string `gorm:"column:aweme_sale_gmv_str;size:255;not null;default:''"`
// Summary from /api/v3/goods/blogger/list
RelatedBloggerCountStr string `gorm:"column:related_blogger_count_str;size:255;not null;default:''"`
BloggerSaleCountStr string `gorm:"column:blogger_sale_count_str;size:255;not null;default:''"`
BloggerSaleGmvStr string `gorm:"column:blogger_sale_gmv_str;size:255;not null;default:''"`
}
Product 商品维度表
type ProductForCollection ¶
type ProductRepo ¶
type ProductRepo interface {
Upsert(ctx context.Context, product *Product) error
ListPage(ctx context.Context, page, size int, query, sortBy string, sortOrder v1.SortOrder) ([]*Product, int64, error)
Get(ctx context.Context, goodsId string) (*Product, error)
UpdateSummary(ctx context.Context, gid string, summary *Product) error
FindProductsForCollection(ctx context.Context, limit int) ([]*ProductForCollection, error)
FindProductsExcludingGIDs(ctx context.Context, excludeGIDs []string, limit int, t time.Time) ([]*ProductForCollection, error)
}
func NewProductRepo ¶
func NewProductRepo(data *Data) ProductRepo
type SourceData ¶
type SourceData struct {
ID int64 `gorm:"primaryKey"`
ProviderName string `gorm:"type:varchar(255);not null;index"`
DataType string `gorm:"type:varchar(255);not null;index"`
EntityId string `gorm:"type:varchar(255);index"` // 可选,关联的主要实体ID,不同的数据类型可能有不同的ID
Status int32 `gorm:"not null;default:0;index"` // 0: unprocessed, 1: processed, -1: error
FetchedAt time.Time `gorm:"autoCreateTime;type:timestamp"`
Date string `gorm:"type:varchar(10);not null;index"`
RawContent string `gorm:"type:text"`
ProcessingLog string `gorm:"type:text"` // 存储ETL处理过程中的错误信息
Retries int `gorm:"not null;default:0"` // 重试次数
// --- 新增的请求上下文元数据 ---
RequestMethod string `gorm:"type:varchar(10)"` // "GET", "POST", etc.
RequestUrl string `gorm:"type:text"`
RequestParams string `gorm:"type:text"` // 存储 Query 或 Body 的 JSON 字符串
RequestHeaders string `gorm:"type:text"` // 存储请求头的 JSON 字符串
}
SourceData is the GORM model for storing raw data from various providers. It is the Data Object (DO). SourceData is the GORM model for storing raw data from various providers.
func CopySourceDataToDO ¶
func CopySourceDataToDO(s *v1.SourceData) *SourceData
Save 实现了biz层的接口,负责将数据写入数据库
func (SourceData) TableName ¶
func (SourceData) TableName() string
type SourceDataRepo ¶
type SourceDataRepo interface {
Save(context.Context, *v1.SourceData) (*v1.SourceData, error)
UpdateStatus(ctx context.Context, id int64, status int32) error
FindUnprocessed(ctx context.Context, dataType string) ([]*v1.SourceData, error)
FindUnprocessedByDate(ctx context.Context, dataType, date string) ([]*v1.SourceData, error)
// UpdateStatusAndLog 原子性地更新状态和处理日志
UpdateStatusAndLog(ctx context.Context, id int64, status int32, log string) error
// FindPartiallyCollectedEntityIDs 查找在指定时间后,只采集了部分数据类型的实体ID列表。
FindPartiallyCollectedEntityIDs(ctx context.Context, since time.Time, dataTypes []string) ([]string, error)
ListAllCollectedEntityIDs(ctx context.Context, dataTypes []string) ([]string, error)
// FindLatestByTypeAndEntityID 根据类型和实体ID查找最新的源数据记录
FindLatestByTypeAndEntityID(ctx context.Context, dataType string, entityId string) (*SourceData, error)
// ListAvailableDates 返回指定类型的所有日期
ListAvailableDates(ctx context.Context, dataType string) ([]string, error)
}
SourceDataRepo 是Biz层依赖的Data层接口,由 data/source_data.go 实现
func NewSourceDataRepo ¶
func NewSourceDataRepo(data *Data, logger log.Logger) SourceDataRepo
NewSourceDataRepo .
type Video ¶
type Video struct {
AwemeId string `gorm:"primaryKey;size:1024"`
CreatedAt time.Time `gorm:"autoCreateTime;type:timestamp"`
UpdatedAt time.Time `gorm:"autoUpdateTime;type:timestamp"`
// --- 基础信息 ---
AwemeDesc string `gorm:"type:text"`
AwemeCoverUrl string `gorm:"size:1024"`
AwemePubTime time.Time `gorm:"type:timestamp"`
AwemeDetailUrl string `gorm:"size:1024"`
BloggerId int64 `gorm:"index"`
PlayCountStr string `gorm:"size:255"`
LikeCountStr string `gorm:"size:255"`
CommentCountStr string `gorm:"size:255"`
CollectCountStr string `gorm:"size:255"`
InteractionRateStr string `gorm:"size:255"`
ScoreStr string `gorm:"size:255;comment:视频分数"`
LikeCommentRateStr string `gorm:"size:255"`
SalesGmvStr string `gorm:"size:255"`
SalesCountStr string `gorm:"size:255"`
// 新增解析后的值
SalesGmv int64 `gorm:"default:0"`
SalesCount int64 `gorm:"default:0"`
GoodsCountStr string `gorm:"size:255"`
GpmStr string `gorm:"size:255;column:gpm_str"` // 明确指定列名
AwemeType int32 `gorm:"type:integer"`
GoodsId string `gorm:"size:255;comment:商品ID"`
RankTime time.Time `gorm:"column:rank_time;comment:上榜时间"`
// --- 详情信息 (来自下钻采集) 暂时用不上 ---
DyTagsJSON string `gorm:"type:text"`
HotSearchWordsJSON string `gorm:"type:text"`
TopicsJSON string `gorm:"type:text"`
CommentSegmentsJSON string `gorm:"type:text"`
InteractionJSON string `gorm:"type:text"`
AudienceProfileJSON string `gorm:"type:text"`
//
SummaryUpdatedAt *time.Time `gorm:"index;comment:总览数据更新时间;type:timestamp"`
TrendUpdatedAt *time.Time `gorm:"index;comment:趋势数据更新时间;type:timestamp"`
CollectionStatus int32 `gorm:"comment:采集状态"`
}
Video 视频维度表
type VideoForCollection ¶
type VideoForCollection struct {
AwemeId string
AwemePubTime time.Time
AwemeDetailUrl string
WorkOrderID string `gorm:"-" json:"work_order_id"`
}
VideoForCollection 是为采集任务准备的结构体
type VideoForSummary ¶
VideoForSummary 定义了用于下钻采集的视频基础信息
type VideoRank ¶
type VideoRank struct {
ID uint `gorm:"primaryKey"`
CreatedAt time.Time `gorm:"autoCreateTime;type:timestamp"`
PeriodType string `gorm:"column:period_type;size:1024;not null"`
RankDate string `gorm:"column:rank_date;size:1024;not null;uniqueIndex:uq_aweme_rank_date"`
StartDate string `gorm:"column:start_date;size:1024;not null;default:''"`
EndDate string `gorm:"column:end_date;size:1024;not null;default:''"`
// 视频信息
AwemeId string `gorm:"column:aweme_id;size:1024;not null;uniqueIndex:uq_aweme_rank_date"`
AwemeCoverUrl string `gorm:"column:aweme_cover_url;size:1024;not null;default:''"`
AwemeDesc string `gorm:"column:aweme_desc;type:text;not null;default:''"`
AwemePubTime time.Time `gorm:"column:aweme_pub_time;type:timestamp"`
DurationStr string `gorm:"column:duration_str;size:1024;not null;default:''"`
AwemeScoreStr string `gorm:"column:aweme_score_str;size:1024;not null;default:''"`
AwemeDetailUrl string `gorm:"column:aweme_detail_url;size:1024;not null;default:''"`
// 商品信息
GoodsId string `gorm:"column:goods_id;size:1024;not null"`
GoodsTitle string `gorm:"column:goods_title;type:text;not null;default:''"`
GoodsCoverUrl string `gorm:"column:goods_cover_url;size:1024;not null;default:''"`
GoodsPriceRange string `gorm:"column:goods_price_range;size:1024;not null;default:''"`
GoodsPrice float64 `gorm:"column:goods_price"`
CosRatio string `gorm:"column:cos_ratio;size:1024;not null;default:''"`
CommissionPrice string `gorm:"column:commission_price;size:1024;not null;default:''"`
ShopName string `gorm:"column:shop_name;size:1024;not null;default:''"`
BrandName string `gorm:"column:brand_name;size:1024;not null;default:''"`
CategoryNames string `gorm:"column:category_names;size:1024;not null;default:''"`
// 博主信息
BloggerId int `gorm:"column:blogger_id;not null"`
BloggerUid string `gorm:"column:blogger_uid;size:1024;not null;default:''"`
BloggerName string `gorm:"column:blogger_name;size:1024;not null;default:''"`
BloggerAvatar string `gorm:"column:blogger_avatar;size:1024;not null;default:''"`
BloggerFansNum int `gorm:"column:blogger_fans_num;not null;default:0"`
BloggerTag string `gorm:"column:blogger_tag;size:1024;not null;default:''"`
// 榜单统计
SalesCountStr string `gorm:"column:sales_count_str;size:1024;not null;default:''"`
TotalSalesStr string `gorm:"column:total_sales_str;size:1024;not null;default:''"`
LikeCountIncStr string `gorm:"column:like_count_inc_str;size:1024;not null;default:''"`
PlayCountIncStr string `gorm:"column:play_count_inc_str;size:1024;not null;default:''"`
SalesCountLow int64 `gorm:"column:sales_count_low;comment:销量范围低值"`
SalesCountHigh int64 `gorm:"column:sales_count_high;comment:销量范围高值"`
TotalSalesLow int64 `gorm:"column:total_sales_low;comment:销售额范围低值(分)"`
TotalSalesHigh int64 `gorm:"column:total_sales_high;comment:销售额范围高值(分)"`
}
VideoRank is the GORM model for storing video ranking data.
type VideoRankRepo ¶
type VideoRankRepo interface {
// 批量创建视频榜单记录
BatchCreate(ctx context.Context, ranks []*v1.VideoRankDTO) error
// 查询单个视频榜单
GetByAwemeID(ctx context.Context, awemeID, rankType, rankDate string) (*v1.VideoRankDTO, error)
// 分页查询视频榜单
ListPage(ctx context.Context, page, size int, rankType, rankDate, sortBy string, sortOrder v1.SortOrder) ([]*VideoRank, int64, error)
// GetDistinctAwemeIDsByDate 获取指定日期之后上过榜的、不重复的视频ID
GetDistinctAwemeIDsByDate(ctx context.Context, sinceDate string) ([]string, error)
// BatchUpsert 批量更新或创建视频榜单记录
BatchUpsert(ctx context.Context, ranks []*v1.VideoRankDTO) error
}
VideoRankRepo defines the interface for batch creation of VideoRank records. VideoRankRepo 定义了视频榜单数据的持久化接口。
func NewVideoRankRepo ¶
func NewVideoRankRepo(db *Data) VideoRankRepo
NewVideoRankRepo creates a new VideoRankRepo.
type VideoRepo ¶
type VideoRepo interface {
// FindVideosExcludingIDs 查找排除指定ID列表之外且上榜时间大于等于 rankTimeAfter 的视频
SaveSourceData(context.Context, *v1.SourceData) (*v1.SourceData, error)
FindVideosByIDs(ctx context.Context, awemeIDs []string, limit int) ([]*VideoForCollection, error)
UpsertFromRank(ctx context.Context, video *Video) error
UpdateFromSummary(ctx context.Context, video *Video) error
FindVideosNeedingSummaryUpdate(ctx context.Context, limit int) ([]*VideoForSummary, error)
ListPage(ctx context.Context, page, size int, query, sortBy string, sortOrder v1.SortOrder) ([]*Video, int64, error)
Get(ctx context.Context, awemeId string) (*Video, error)
FindRecentActiveAwemeIds(ctx context.Context, days int) ([]string, error)
//FindVideosNeedingTrendUpdate(ctx context.Context, limit int) ([]*VideoForTrend, error)
FindVideosForDetailsCollection(ctx context.Context, limit int) ([]*VideoForCollection, error)
UpdateTrendTimestamp(ctx context.Context, awemeId string) error
FindVideosExcludingIDs(ctx context.Context, ids []string, limit int, t time.Time) ([]*VideoForCollection, error)
} // End of VideoRepo interface
func NewVideoRepo ¶
type VideoTrend ¶
type VideoTrend struct {
Id int64 `gorm:"primaryKey"`
CreatedAt time.Time `gorm:"autoCreateTime;type:timestamp"`
UpdatedAt time.Time `gorm:"autoUpdateTime;type:timestamp"`
AwemeId string `gorm:"size:255;not null;"` // 视频ID
DateCode int `gorm:"not null;"` // 数据日期
LikeCount int64
LikeCountStr string `gorm:"size:20"`
CommentCount int64
CommentCountStr string `gorm:"size:20"`
CollectCount int64
CollectCountStr string `gorm:"size:20"`
InteractionRate float64
InteractionRateStr string `gorm:"size:20"`
IncLikeCount int64
IncLikeCountStr string `gorm:"size:20"`
IncCommentCount int64
IncCommentCountStr string `gorm:"size:20"`
IncCollectCount int64
IncCollectCountStr string `gorm:"size:20"`
SalesCount int64
SalesCountStr string `gorm:"size:20"`
SalesGmv float64
SalesGmvStr string `gorm:"size:20"`
Fans int64
FansStr string `gorm:"size:20"`
IncSalesCount int64
IncSalesCountStr string `gorm:"size:20"`
IncSalesGmv float64
IncSalesGmvStr string `gorm:"size:20"`
IncFans int64
IncFansStr string `gorm:"size:20"`
Gpm float64
GpmStr string `gorm:"size:20"`
ListTimeStr string `gorm:"size:20"`
TimeStamp int64
}
VideoTrend 视频每日趋势数据模型
type VideoTrendRepo ¶
type VideoTrendRepo interface {
BatchUpsert(ctx context.Context, trends []*VideoTrend) error
ListPage(ctx context.Context, page, size int, awemeId, startDate, endDate string) ([]*VideoTrend, int64, error)
BatchOverwrite(ctx context.Context, trends []*VideoTrend) error
}
VideoTrendRepo 定义视频趋势数据仓库接口
type WorkOrder ¶
type WorkOrder struct {
ID uint `gorm:"primaryKey"`
WorkOrderID string `gorm:"uniqueIndex;size:64"`
TaskType string `gorm:"size:64;index"`
TaskPayload []byte `gorm:"type:jsonb"`
UniqueKey string `gorm:"size:128;index"`
Status WorkOrderStatus `gorm:"size:32;index"`
CreatedAt time.Time `gorm:"autoCreateTime"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
}
type WorkOrderRepo ¶
type WorkOrderRepo interface {
Create(ctx context.Context, order *WorkOrder) error
FindRecentByUniqueKey(ctx context.Context, uniqueKey string, within time.Duration) (*WorkOrder, error)
FindByUniqueKey(ctx context.Context, uniqueKey string) (*WorkOrder, error)
UpdateStatusByWorkOrderID(ctx context.Context, workOrderID string, status WorkOrderStatus) error
ListByDate(ctx context.Context, date string) ([]*WorkOrder, error)
}
func NewWorkOrderRepo ¶
func NewWorkOrderRepo(data *Data) WorkOrderRepo
type WorkOrderStatus ¶
type WorkOrderStatus string
const ( WorkOrderStatusCreated WorkOrderStatus = "created" WorkOrderStatusDispatched WorkOrderStatus = "dispatched" WorkOrderStatusProcessing WorkOrderStatus = "processing" WorkOrderStatusCompleted WorkOrderStatus = "completed" WorkOrderStatusFailed WorkOrderStatus = "failed" WorkOrderStatusRetrying WorkOrderStatus = "retrying" )
Source Files
¶
- blogger.go
- blogger_aweme_stat.go
- blogger_goods_category_distribution.go
- blogger_goods_sales_trend.go
- blogger_goods_stat.go
- blogger_portrayal.go
- data.go
- goods_aweme.go
- goods_blogger.go
- goods_portrayal.go
- goods_trend.go
- product.go
- source_data.go
- utils.go
- video.go
- video_for_detail.go
- video_rank.go
- video_trend.go
- work_order.go