data

package
v0.0.0-...-34846eb Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 14, 2025 License: MIT Imports: 18 Imported by: 0

Documentation

Overview

internal/data/goods_aweme.go

internal/data/goods_blogger.go

internal/data/goods_portrayal.go

internal/data/goods_trend.go

Index

Constants

View Source
const (
	DataTypeVideoRank            = "video_rank"
	DataTypeVideoDetail          = "video_detail"
	DataTypeVideoSummaryHeadless = "video_summary_headless"
	DataTypeVideoTrendHeadless   = "video_trend_headless"
	DataTypeVideoSummaryHttp     = "video_summary_http"
	DataTypeVideoTrendHttp       = "video_trend_http"

	// DataTypeGoodsAwemeAnalysisHttp HTTP采集的商品分析数据
	DataTypeGoodsAwemeAnalysisHttp = "goods_aweme_analysis_http"
	// DataTypeGoodsChartDataHttp HTTP采集的商品图表数据
	DataTypeGoodsChartDataHttp = "goods_chart_data_http"
	// DataTypeGoodsPortrayalHttp HTTP采集的商品画像数据
	DataTypeGoodsPortrayalHttp         = "goods_portrayal_http"
	DataTypeGoodsAwemeAnalysisHeadless = "goods_aweme_analysis_headless"
	DataTypeGoodsChartDataHeadless     = "goods_chart_data_headless"
	DataTypeGoodsPortrayalHeadless     = "goods_portrayal_headless"

	// --- 【新增】Blogger详情相关的DataType常量 ---
	DataTypeBloggerMainPartHeadless        = "blogger_mainpart_headless"
	DataTypeBloggerOtherPartHeadless       = "blogger_otherpart_headless"
	DataTypeBloggerAwemeOverviewHeadless   = "blogger_aweme_overview_headless"
	DataTypeBloggerAwemePortrayHeadless    = "blogger_aweme_portray_headless"
	DataTypeBloggerCommerceSummaryHeadless = "blogger_commerce_summary_headless"
	DataTypeBloggerGoodsOverviewHeadless   = "blogger_goods_overview_headless"
	DataTypeBloggerGoodsTrendsHeadless     = "blogger_goods_trends_headless"
	DataTypeBloggerGoodsCateDistHeadless   = "blogger_goods_cate_dist_headless"

	// DataType for blogger details fetched via HTTP
	DataTypeBloggerMainPartHttp        = "blogger_mainpart_http"
	DataTypeBloggerOtherPartHttp       = "blogger_otherpart_http"
	DataTypeBloggerAwemeOverviewHttp   = "blogger_aweme_overview_http"
	DataTypeBloggerAwemePortrayalHttp  = "blogger_aweme_portrayal_http"
	DataTypeBloggerCommerceSummaryHttp = "blogger_commerce_summary_http"
	DataTypeBloggerGoodsOverviewHttp   = "blogger_goods_overview_http"
	DataTypeBloggerGoodsTrendsHttp     = "blogger_goods_trends_http"
	DataTypeBloggerGoodsCateDistHttp   = "blogger_goods_cate_dist_http"
)

在这里定义data_type,全部地方都适用这里的定义

Variables

DataProviderSet is data providers.

Functions

func CopyBloggerAwemeStatToDTO

func CopyBloggerAwemeStatToDTO(stat *BloggerAwemeStat) *v1.BloggerAwemeStatDTO

func CopyBloggerGoodsCategoryDistributionToDTO

func CopyBloggerGoodsCategoryDistributionToDTO(d *BloggerGoodsCategoryDistribution) *v1.BloggerGoodsCategoryDistributionDTO

CopyBloggerGoodsCategoryDistributionToDTO converts a BloggerGoodsCategoryDistribution model to a DTO.

func CopyBloggerGoodsSalesTrendToDTO

func CopyBloggerGoodsSalesTrendToDTO(trend *BloggerGoodsSalesTrend) *v1.BloggerGoodsSalesTrendDTO

CopyBloggerGoodsSalesTrendToDTO converts a BloggerGoodsSalesTrend model to a DTO.

func CopyBloggerGoodsStatToDTO

func CopyBloggerGoodsStatToDTO(stat *BloggerGoodsStat) *v1.BloggerGoodsStatDTO

func CopyBloggerPortrayalToDTO

func CopyBloggerPortrayalToDTO(p *BloggerPortrayal) *v1.BloggerPortrayalDTO

CopyBloggerPortrayalToDTO 将 BloggerPortrayal 模型转换为 DTO

func CopyBloggerToDTO

func CopyBloggerToDTO(b *Blogger) *v1.BloggerDTO

CopyBloggerToDTO 将 data.Blogger 模型转换为 v1.BloggerDTO

func CopyGoodsBloggerToDTO

func CopyGoodsBloggerToDTO(gb *GoodsBlogger) *v1.GoodsBloggerDTO

func CopyProductToDTO

func CopyProductToDTO(p *Product) *v1.ProductDTO

CopyProductToDTO 将 data.Product 模型转换为 v1.ProductDTO

func CopySourceDataToDTO

func CopySourceDataToDTO(s *SourceData) *v1.SourceData

func CopyVideoRankToDTO

func CopyVideoRankToDTO(do *VideoRank) *v1.VideoRankDTO

func CopyVideoToDTO

func CopyVideoToDTO(v *Video) *v1.VideoDTO

CopyVideoToDTO 将 data.Video 模型转换为 v1.VideoDTO

func CopyVideoTrendToDTO

func CopyVideoTrendToDTO(vt *VideoTrend) *v1.VideoTrendDTO

CopyVideoTrendToDTO 将 data.VideoTrend 模型转换为 v1.VideoTrendDTO

func NewRedisClient

func NewRedisClient(conf *conf.Data) redis.Cmdable

NewRedisClient 初始化 Redis 客户端

Types

type Blogger

type Blogger struct {
	// --- Rank ETL 维护的核心字段 (保持不变) ---
	BloggerId        int64     `gorm:"primaryKey;column:blogger_id"`
	CreatedAt        time.Time `gorm:"autoCreateTime;column:created_at;type:timestamp"`
	UpdatedAt        time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
	BloggerUid       string    `gorm:"size:255;index;column:blogger_uid"`
	BloggerName      string    `gorm:"size:255;column:blogger_name"`
	BloggerAvatar    string    `gorm:"size:1024;column:blogger_avatar"`
	BloggerFansNum   int64     `gorm:"column:blogger_fans_num"`
	BloggerTag       string    `gorm:"size:255;column:blogger_tag"`
	BloggerDetailUrl string    `gorm:"column:blogger_detail_url;size:1024;not null;default:''"`
	RankTime         time.Time `gorm:"column:rank_time;comment:上榜时间"`

	// --- Detail ETL 维护的补充字段 (全面修订) ---
	// 身份补充 (来自 mainpart)
	UniqueId         string    `gorm:"size:255;index;column:unique_id;comment:平台UniqueId"`
	Gender           string    `gorm:"size:16;column:gender;comment:性别"`
	Location         string    `gorm:"size:64;column:location;comment:地点"`
	Signature        string    `gorm:"type:text;column:signature;comment:签名"`
	FollowerCount    int64     `gorm:"column:follower_count;comment:精确粉丝数 (Fans字段)"`
	FollowerCountStr string    `gorm:"size:64;column:follower_count_str;comment:平台显示粉丝数 (Platform_Fans字段)"`
	LastPublishAt    time.Time `gorm:"column:last_publish_at;comment:最后发布时间 (LastPubTime)"`
	BloggerAddTime   time.Time `gorm:"column:blogger_add_time;comment:达人入库时间"`
	FansMilestone    JSONB     `gorm:"column:fans_milestone;comment:粉丝里程碑"`
	OpenBloggerId    string    `gorm:"size:255;index;column:open_blogger_id;comment:开放平台ID"`

	// 统计指标 (来自 mainpart)
	Score              string `gorm:"size:32;column:score;comment:平台综合评分"`
	AwemesCount        int64  `gorm:"column:awemes_count;comment:总作品数"`
	LikeCountStr       string `gorm:"size:64;column:like_count_str;comment:总获赞数"`
	AvgLikeCountStr    string `gorm:"size:64;column:avg_like_count_str;comment:集均获赞"`
	CommentCountStr    string `gorm:"size:64;column:comment_count_str;comment:总评论数"`
	AvgCommentCountStr string `gorm:"size:64;column:avg_comment_count_str;comment:集均评论"`
	ShareCountStr      string `gorm:"size:64;column:share_count_str;comment:总分享数"`
	AvgShareCountStr   string `gorm:"size:64;column:avg_share_count_str;comment:集均分享"`
	FansClubCount      string `gorm:"size:64;column:fans_club_count;comment:粉丝团人数"`

	// 状态与榜单 (来自 otherpart)
	BloggerTypeName   string `gorm:"size:64;column:blogger_type_name;comment:达人类型, 如'腰部达人'"`
	TagRankNum        int64  `gorm:"column:tag_rank_num;comment:分类榜单排名"`
	TagRankDesc       string `gorm:"size:255;column:tag_rank_desc;comment:分类榜单描述"`
	PromotionRankNum  int64  `gorm:"column:promotion_rank_num;comment:带货榜单排名"`
	PromotionRankDesc string `gorm:"size:255;column:promotion_rank_desc;comment:带货榜单描述"`
	BetaNearlyAWeek   string `gorm:"size:32;column:beta_nearly_a_week;comment:近7日Beta系数"`
	SalesGMV30Day     string `gorm:"size:64;column:sales_gmv_30_day;comment:近30日GMV"`
	TopicsCount       int64  `gorm:"column:topics_count;comment:话题数 (Topics)"`

	// 口碑与电商 (来自 otherpart)
	KBScore       string `gorm:"size:32;column:kb_score;comment:口碑分"`
	KBPercentage  string `gorm:"size:32;column:kb_percentage;comment:口碑分超越百分比"`
	KBSales       string `gorm:"size:64;column:kb_sales;comment:口碑销量"`
	KBSales30Days string `gorm:"size:64;column:kb_sales_30_days;comment:近30日口碑销量"`

	// 标签与关联 (来自 otherpart)
	AwemeTagList JSONB `gorm:"column:aweme_tag_list;comment:视频内容标签"`
	LiveTagList  JSONB `gorm:"column:live_tag_list;comment:直播内容标签"`
	IsBindBrand  bool  `gorm:"column:is_bind_brand;comment:是否绑定品牌"`
	IsBindShop   bool  `gorm:"column:is_bind_shop;comment:是否绑定店铺"`
	IsHaveMcn    bool  `gorm:"column:is_have_mcn;comment:是否有MCN"`
}

Blogger 博主维度表 Blogger 博主维度表,存储达人全面、丰富的档案信息

func (Blogger) TableName

func (Blogger) TableName() string

type BloggerAwemeStat

type BloggerAwemeStat struct {
	BloggerId         int64     `gorm:"primaryKey;column:blogger_id;comment:达人ID"`
	UpdatedAt         time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
	AwemeCount        string    `gorm:"size:64;column:aweme_count;comment:总视频数"`
	HasAwemeCount     string    `gorm:"size:64;column:has_aweme_count;comment:有播放视频数"`
	SaleGmv           string    `gorm:"size:64;column:sale_gmv;comment:视频带货GMV"`
	AvgSaleGmv        string    `gorm:"size:64;column:avg_sale_gmv;comment:场均视频带货GMV"`
	SaleCount         string    `gorm:"size:64;column:sale_count;comment:视频带货销量"`
	AvgSaleCount      string    `gorm:"size:64;column:avg_sale_count;comment:场均视频带货销量"`
	HasPromotionCount string    `gorm:"size:64;column:has_promotion_count;comment:关联商品视频数"`
	LikeCount         string    `gorm:"size:64;column:like_count;comment:获赞数"`
	AvgLikeCount      string    `gorm:"size:64;column:avg_like_count;comment:集均获赞"`
	CommentCount      string    `gorm:"size:64;column:comment_count;comment:评论数"`
	AvgCommentCount   string    `gorm:"size:64;column:avg_comment_count;comment:集均评论"`
	ShareCount        string    `gorm:"size:64;column:share_count;comment:分享数"`
	AvgShareCount     string    `gorm:"size:64;column:avg_share_count;comment:集均分享"`
	CollectCount      string    `gorm:"size:64;column:collect_count;comment:收藏数"`
	AvgCollectCount   string    `gorm:"size:64;column:avg_collect_count;comment:集均收藏"`
	AvgPrice          string    `gorm:"size:64;column:avg_price;comment:视频带货客单价"`
}

BloggerAwemeStat 存储达人视频创作的聚合统计数据

type BloggerAwemeStatRepo

type BloggerAwemeStatRepo interface {
	Upsert(ctx context.Context, stat *BloggerAwemeStat) error
	Get(ctx context.Context, bloggerId int64) (*BloggerAwemeStat, error)
}

func NewBloggerAwemeStatRepo

func NewBloggerAwemeStatRepo(data *Data) BloggerAwemeStatRepo

type BloggerForCollection

type BloggerForCollection struct {
	BloggerId        int64
	BloggerUid       string
	BloggerDetailUrl string
	WorkOrderID      string `gorm:"-" json:"work_order_id"`
}

BloggerForCollection 是为采集任务准备的轻量级结构体

type BloggerGoodsCategoryDistribution

type BloggerGoodsCategoryDistribution struct {
	Id                int64     `gorm:"primaryKey;column:id;autoIncrement:true"`
	BloggerId         int64     `gorm:"index;not null;column:blogger_id;comment:达人ID"`
	CategoryName      string    `gorm:"type:varchar(255);column:category_name;comment:品类名称"`
	ShortCategoryName string    `gorm:"type:varchar(255);column:short_category_name;comment:简短品类名称"`
	CategoryId        string    `gorm:"type:varchar(64);column:category_id;comment:品类ID (Cate)"`
	GoodsCount        int64     `gorm:"column:goods_count;comment:商品数"`
	SaleCount         int64     `gorm:"column:sale_count;comment:销量"`
	SaleGmv           int64     `gorm:"column:sale_gmv;comment:销售额 (分)"`
	AwemeCount        int64     `gorm:"column:aweme_count;comment:视频数"`
	LiveCount         int64     `gorm:"column:live_count;comment:直播数"`
	BloggerCount      int64     `gorm:"column:blogger_count;comment:关联达人数"`
	AvgKdj            float64   `gorm:"column:avg_kdj;comment:平均客单价 (KDJ)"`
	AvgKdjStr         string    `gorm:"type:varchar(64);column:avg_kdj_str;comment:客单价范围字符串"`
	CreatedAt         time.Time `gorm:"autoCreateTime;column:created_at;type:timestamp"`
	UpdatedAt         time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
}

BloggerGoodsCategoryDistribution 存储达人带货的品类分布数据

func (BloggerGoodsCategoryDistribution) TableName

tablename

type BloggerGoodsCategoryDistributionRepo

type BloggerGoodsCategoryDistributionRepo interface {
	RefreshForBlogger(ctx context.Context, bloggerId int64, distributions []*BloggerGoodsCategoryDistribution) error
	ListPage(ctx context.Context, page, size int, bloggerId int64, sortBy string, sortOrder v1.SortOrder) ([]*BloggerGoodsCategoryDistribution, int64, error)
}

func NewBloggerGoodsCategoryDistributionRepo

func NewBloggerGoodsCategoryDistributionRepo(data *Data) BloggerGoodsCategoryDistributionRepo

type BloggerGoodsSalesTrend

type BloggerGoodsSalesTrend struct {
	Id                int64     `gorm:"column:id;primaryKey;autoIncrement:true"`
	BloggerId         int64     `gorm:"column:blogger_id;index:idx_blogger_datecode,unique;not null"`
	DateCode          int       `gorm:"column:date_code;index:idx_blogger_datecode,unique;not null;comment:日期 YYYYMMDD"`
	SaleCount         int64     `gorm:"column:sale_count;comment:总销量"`
	SaleCountStr      string    `gorm:"size:64;column:sale_count_str;comment:总销量字符串"`
	SaleGmv           int64     `gorm:"column:sale_gmv;comment:总GMV"`
	SaleGmvStr        string    `gorm:"size:64;column:sale_gmv_str;comment:总GMV字符串"`
	AwemeSaleCount    int64     `gorm:"column:aweme_sale_count;comment:视频带货销量"`
	AwemeSaleCountStr string    `gorm:"size:64;column:aweme_sale_count_str;comment:视频带货销量字符串"`
	AwemeSaleGmv      int64     `gorm:"column:aweme_sale_gmv;comment:视频带货GMV"`
	AwemeSaleGmvStr   string    `gorm:"size:64;column:aweme_sale_gmv_str;comment:视频带货GMV字符串"`
	LiveSaleCount     int64     `gorm:"column:live_sale_count;comment:直播带货销量"`
	LiveSaleCountStr  string    `gorm:"size:64;column:live_sale_count_str;comment:直播带货销量字符串"`
	LiveSaleGmv       int64     `gorm:"column:live_sale_gmv;comment:直播带货GMV"`
	LiveSaleGmvStr    string    `gorm:"size:64;column:live_sale_gmv_str;comment:直播带货GMV字符串"`
	PicTimeStr        string    `gorm:"size:64;column:pic_time_str;comment:图片时间字符串"`
	ListTimeStr       string    `gorm:"size:64;column:list_time_str;comment:列表时间字符串"`
	TimeStamp         int64     `gorm:"column:timestamp;comment:时间戳"`
	DateCodeStr       string    `gorm:"size:64;column:date_code_str;comment:日期字符串"`
	CreatedAt         time.Time `gorm:"column:created_at;autoCreateTime"`
	UpdatedAt         time.Time `gorm:"column:updated_at;autoUpdateTime"`
}

BloggerGoodsSalesTrend 存储达人商品销售的每日趋势数据

func (BloggerGoodsSalesTrend) TableName

func (BloggerGoodsSalesTrend) TableName() string

type BloggerGoodsSalesTrendRepo

type BloggerGoodsSalesTrendRepo interface {
	BatchOverwrite(ctx context.Context, trends []*BloggerGoodsSalesTrend) error
	ListPage(ctx context.Context, bloggerId int64, page, size int, sortBy string, sortOrder v1.SortOrder) ([]*BloggerGoodsSalesTrend, int64, error)
}

func NewBloggerGoodsSalesTrendRepo

func NewBloggerGoodsSalesTrendRepo(data *Data) BloggerGoodsSalesTrendRepo

type BloggerGoodsStat

type BloggerGoodsStat struct {
	BloggerId            int64     `gorm:"primaryKey;column:blogger_id;comment:达人ID"`
	UpdatedAt            time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
	SaleCount            string    `gorm:"size:64;column:sale_count;comment:总销量"`
	SaleGmv              string    `gorm:"size:64;column:sale_gmv;comment:总销售额"`
	LiveSaleCount        string    `gorm:"size:64;column:live_sale_count;comment:直播销量"`
	LiveSaleGmv          string    `gorm:"size:64;column:live_sale_gmv;comment:直播销售额"`
	AwemeSaleCount       string    `gorm:"size:64;column:aweme_sale_count;comment:视频销量"`
	AwemeSaleGmv         string    `gorm:"size:64;column:aweme_sale_gmv;comment:视频销售额"`
	SaleGoodsCount       string    `gorm:"size:64;column:sale_goods_count;comment:在售商品数"`
	AvgUserPrice         string    `gorm:"size:64;column:avg_user_price;comment:客单价"`
	MainCategoryName     string    `gorm:"size:128;column:main_category_name;comment:主营品类"`
	MainCategoryRatio    string    `gorm:"size:32;column:main_category_ratio;comment:主营品类占比"`
	CategoryCount        string    `gorm:"size:32;column:category_count;comment:品类总数"`
	TopBrands            string    `gorm:"type:text;column:top_brands;comment:合作品牌TOP,逗号分隔"`
	TopShops             string    `gorm:"type:text;column:top_shops;comment:合作店铺TOP,逗号分隔"`
	CommerceChannel      string    `gorm:"size:32;column:commerce_channel;comment:主要带货渠道"`
	CommerceChannelRatio string    `gorm:"size:32;column:commerce_channel_ratio;comment:主要渠道占比"`
}

BloggerGoodsStat 存储达人电商表现的聚合统计数据

type BloggerGoodsStatRepo

type BloggerGoodsStatRepo interface {
	Upsert(ctx context.Context, stat *BloggerGoodsStat) error
	Get(ctx context.Context, bloggerID int64) (*BloggerGoodsStat, error)
}

func NewBloggerGoodsStatRepo

func NewBloggerGoodsStatRepo(data *Data) BloggerGoodsStatRepo

type BloggerPortrayal

type BloggerPortrayal struct {
	Id            int64     `gorm:"primaryKey;column:id;autoIncrement:true"`
	BloggerId     int64     `gorm:"index;not null;column:blogger_id;comment:达人ID"`
	PortrayalData JSONB     `gorm:"type:jsonb;column:portrayal_data;comment:完整的画像JSON数据"`
	CreatedAt     time.Time `gorm:"autoCreateTime;column:created_at;type:timestamp"`
	UpdatedAt     time.Time `gorm:"autoUpdateTime;column:updated_at;type:timestamp"`
}

BloggerPortrayal 存储达人的粉丝画像数据

type BloggerPortrayalRepo

type BloggerPortrayalRepo interface {
	Save(ctx context.Context, portrayal *BloggerPortrayal) error
	Upsert(ctx context.Context, portrayal *BloggerPortrayal) error
	Get(ctx context.Context, bloggerId int64) (*v1.BloggerPortrayalDTO, error)
}

func NewBloggerPortrayalRepo

func NewBloggerPortrayalRepo(data *Data) BloggerPortrayalRepo

type BloggerRepo

type BloggerRepo interface {
	Upsert(ctx context.Context, blogger *Blogger) error
	UpdateDetails(ctx context.Context, blogger *Blogger) error
	ListPage(ctx context.Context, page, size int, query, sortBy string, sortOrder v1.SortOrder) ([]*Blogger, int64, error)
	Get(ctx context.Context, bloggerId int64) (*v1.BloggerDTO, error)
	FindBloggersForCollection(ctx context.Context, limit int) ([]*BloggerForCollection, error)
	FindBloggersExcludingIDs(ctx context.Context, ids []string, limit int, t time.Time) ([]*BloggerForCollection, error)
	GetByBloggerUid(ctx context.Context, bloggerId int64) (*Blogger, error)
}

func NewBloggerRepo

func NewBloggerRepo(data *Data) BloggerRepo

type Data

type Data struct {
	Logger *log.Helper
	// contains filtered or unexported fields
}

Data .

func NewData

func NewData(c *conf.Data, redisClient redis.Cmdable, logger log.Logger) (*Data, func(), error)

NewData .

type GoodsAweme

type GoodsAweme struct {
	Id                int64     `gorm:"primarykey;autoIncrement"`
	GoodsId           string    `gorm:"column:goods_id;size:255;not null;index"`
	AwemeId           string    `gorm:"column:aweme_id;size:255;not null;uniqueIndex:idx_goods_aweme"`
	BloggerId         int64     `gorm:"column:blogger_id;not null"`
	AwemePubTime      time.Time `gorm:"column:aweme_pub_time"`
	AwemeDesc         string    `gorm:"column:aweme_desc;type:text"`
	AwemeCoverUrl     string    `gorm:"column:aweme_cover_url;size:1024"`
	AwemeDetailUrl    string    `gorm:"column:aweme_detail_url;size:1024"`
	AwemeSaleCountStr string    `gorm:"column:aweme_sale_count_str;size:255"`
	AwemeSaleGmvStr   string    `gorm:"column:aweme_sale_gmv_str;size:255"`
	LikeCountStr      string    `gorm:"column:like_count_str;size:255"`
	CommentCountStr   string    `gorm:"column:comment_count_str;size:255"`
	ShareCountStr     string    `gorm:"column:share_count_str;size:255"`
	PlayCountStr      string    `gorm:"column:play_count_str;size:255"`
	CollectCountStr   string    `gorm:"column:collect_count_str;size:255"`
	GPM               string    `gorm:"column:gpm;size:255"`
	CreatedAt         time.Time `gorm:"column:created_at"`
	UpdatedAt         time.Time `gorm:"column:updated_at"`
}

GoodsAweme 对应 /api/v3/goods/aweme/loadAwemeAnalysis 接口返回的 Items

func (*GoodsAweme) TableName

func (ga *GoodsAweme) TableName() string

type GoodsAwemeRepo

type GoodsAwemeRepo interface {
	BatchCreate(ctx context.Context, awemes []*GoodsAweme) error
	List(ctx context.Context, req *v1.ListGoodsAwemesRequest) ([]*GoodsAweme, int64, error)
}

func NewGoodsAwemeRepo

func NewGoodsAwemeRepo(data *Data, logger log.Logger) GoodsAwemeRepo

type GoodsBlogger

type GoodsBlogger struct {
	Id int64 `gorm:"primarykey;autoIncrement"`
	// 复合唯一索引,确保一个商品只关联一个达人一次
	GoodsId   string `gorm:"column:goods_id;size:255;not null;uniqueIndex:idx_goods_blogger"`
	BloggerId int64  `gorm:"column:blogger_id;not null;uniqueIndex:idx_goods_blogger"`

	// --- 达人冗余信息 ---
	BloggerUid    string `gorm:"size:255"`
	BloggerName   string `gorm:"size:255"`
	BloggerAvatar string `gorm:"size:1024"`
	FansNum       int64
	Tag           string `gorm:"size:255"`

	// --- 视频带货聚合数据 ---
	AwemeCountStr     string `gorm:"column:aweme_count_str;size:255"` // 视频数量
	AwemeSaleCountStr string `gorm:"column:aweme_sale_count_str;size:255"`
	AwemeSaleGmvStr   string `gorm:"column:aweme_sale_gmv_str;size:255"`

	CreatedAt time.Time `gorm:"column:created_at"`
	UpdatedAt time.Time `gorm:"column:updated_at"`
}

GoodsBlogger 存储商品和达人的关联关系,并冗余达人核心信息

func (*GoodsBlogger) TableName

func (gb *GoodsBlogger) TableName() string

type GoodsBloggerRepo

type GoodsBloggerRepo interface {
	BatchUpsert(ctx context.Context, goodsBloggers []*GoodsBlogger) error
	List(ctx context.Context, req *v1.ListGoodsBloggersRequest) ([]*GoodsBlogger, int64, error)
}

func NewGoodsBloggerRepo

func NewGoodsBloggerRepo(data *Data, logger log.Logger) GoodsBloggerRepo

type GoodsPortrayal

type GoodsPortrayal struct {
	Id            int64     `gorm:"primaryKey"`
	CreatedAt     time.Time `gorm:"autoCreateTime"`
	UpdatedAt     time.Time `gorm:"autoUpdateTime"`
	GoodsId       string    `gorm:"size:255;not null;uniqueIndex"`
	PortrayalData JSONB     `gorm:"type:jsonb"` // 使用 JSONB 类型
}

GoodsPortrayal 存储完整的用户画像数据

func (GoodsPortrayal) TableName

func (GoodsPortrayal) TableName() string

type GoodsPortrayalRepo

type GoodsPortrayalRepo interface {
	CreateOrUpdate(ctx context.Context, portrayal *GoodsPortrayal) error
	GetByGID(ctx context.Context, gid string) (*GoodsPortrayal, error) // <--- 新增方法
}

func NewGoodsPortrayalRepo

func NewGoodsPortrayalRepo(data *Data) GoodsPortrayalRepo

type GoodsTrend

type GoodsTrend struct {
	Id                int64     `gorm:"primaryKey"`
	CreatedAt         time.Time `gorm:"autoCreateTime;type:timestamp"`
	UpdatedAt         time.Time `gorm:"autoUpdateTime;type:timestamp"`
	GoodsId           string    `gorm:"size:255;not null;index:idx_goods_date,unique"`
	DateCode          int       `gorm:"not null;index:idx_goods_date,unique"`
	AwemeCount        int64
	AwemeCountStr     string `gorm:"size:255"`
	BloggerCount      int64
	BloggerCountStr   string `gorm:"size:255"`
	AwemeSaleGmv      float64
	AwemeSaleGmvStr   string `gorm:"size:255"`
	AwemeSaleCount    int64
	AwemeSaleCountStr string `gorm:"size:255"`
	ListTimeStr       string `gorm:"size:255"`
	TimeStamp         int64
}

GoodsTrend 存储商品每日的带货趋势数据

func (GoodsTrend) TableName

func (GoodsTrend) TableName() string

type GoodsTrendRepo

type GoodsTrendRepo interface {
	BatchOverwrite(ctx context.Context, trends []*GoodsTrend) error
	List(ctx context.Context, req *v1.ListGoodsTrendsRequest) ([]*GoodsTrend, int64, error)
}

func NewGoodsTrendRepo

func NewGoodsTrendRepo(data *Data) GoodsTrendRepo

type JSONB

type JSONB json.RawMessage

JSONB a custom type for handling JSONB data

func (*JSONB) Scan

func (j *JSONB) Scan(value interface{}) error

func (JSONB) Value

func (j JSONB) Value() (driver.Value, error)

type Product

type Product struct {
	GoodsId   string    `gorm:"primaryKey;size:1024"`
	CreatedAt time.Time `gorm:"autoCreateTime;type:timestamp"`
	UpdatedAt time.Time `gorm:"autoUpdateTime;type:timestamp"`

	GoodsTitle      string  `gorm:"type:text"`
	GoodsCoverUrl   string  `gorm:"size:1024"`
	GoodsPriceRange string  `gorm:"size:255"`
	GoodsPrice      float64 `gorm:"type:numeric"`
	CosRatio        string  `gorm:"size:255"`
	CommissionPrice string  `gorm:"size:255"`
	ShopName        string  `gorm:"size:255"`
	BrandName       string  `gorm:"size:255"`
	CategoryNames   string  `gorm:"size:1024"`

	GoodsDetailUrl string    `gorm:"column:goods_detail_url;size:1024;not null;default:''"`
	RankTime       time.Time `gorm:"column:rank_time;comment:上榜时间"`

	// Summary from /api/v3/goods/aweme/loadAwemeAnalysis
	AwemeCount        int64  `gorm:"column:aweme_count;not null;default:0"`
	BloggerCount      int64  `gorm:"column:blogger_count;not null;default:0"`
	AwemeSaleCountStr string `gorm:"column:aweme_sale_count_str;size:255;not null;default:''"`
	AwemeSaleGmvStr   string `gorm:"column:aweme_sale_gmv_str;size:255;not null;default:''"`

	// Summary from /api/v3/goods/blogger/list
	RelatedBloggerCountStr string `gorm:"column:related_blogger_count_str;size:255;not null;default:''"`
	BloggerSaleCountStr    string `gorm:"column:blogger_sale_count_str;size:255;not null;default:''"`
	BloggerSaleGmvStr      string `gorm:"column:blogger_sale_gmv_str;size:255;not null;default:''"`
}

Product 商品维度表

func (Product) TableName

func (Product) TableName() string

type ProductForCollection

type ProductForCollection struct {
	GoodsId        string
	GoodsDetailUrl string
	WorkOrderID    string `gorm:"-" json:"work_order_id"`
}

type ProductRepo

type ProductRepo interface {
	Upsert(ctx context.Context, product *Product) error
	ListPage(ctx context.Context, page, size int, query, sortBy string, sortOrder v1.SortOrder) ([]*Product, int64, error)
	Get(ctx context.Context, goodsId string) (*Product, error)
	UpdateSummary(ctx context.Context, gid string, summary *Product) error
	FindProductsForCollection(ctx context.Context, limit int) ([]*ProductForCollection, error)
	FindProductsExcludingGIDs(ctx context.Context, excludeGIDs []string, limit int, t time.Time) ([]*ProductForCollection, error)
}

func NewProductRepo

func NewProductRepo(data *Data) ProductRepo

type SourceData

type SourceData struct {
	ID            int64     `gorm:"primaryKey"`
	ProviderName  string    `gorm:"type:varchar(255);not null;index"`
	DataType      string    `gorm:"type:varchar(255);not null;index"`
	EntityId      string    `gorm:"type:varchar(255);index"`  // 可选,关联的主要实体ID,不同的数据类型可能有不同的ID
	Status        int32     `gorm:"not null;default:0;index"` // 0: unprocessed, 1: processed, -1: error
	FetchedAt     time.Time `gorm:"autoCreateTime;type:timestamp"`
	Date          string    `gorm:"type:varchar(10);not null;index"`
	RawContent    string    `gorm:"type:text"`
	ProcessingLog string    `gorm:"type:text"`          // 存储ETL处理过程中的错误信息
	Retries       int       `gorm:"not null;default:0"` // 重试次数

	// --- 新增的请求上下文元数据 ---
	RequestMethod  string `gorm:"type:varchar(10)"` // "GET", "POST", etc.
	RequestUrl     string `gorm:"type:text"`
	RequestParams  string `gorm:"type:text"` // 存储 Query 或 Body 的 JSON 字符串
	RequestHeaders string `gorm:"type:text"` // 存储请求头的 JSON 字符串
}

SourceData is the GORM model for storing raw data from various providers. It is the Data Object (DO). SourceData is the GORM model for storing raw data from various providers.

func CopySourceDataToDO

func CopySourceDataToDO(s *v1.SourceData) *SourceData

Save 实现了biz层的接口,负责将数据写入数据库

func (SourceData) TableName

func (SourceData) TableName() string

type SourceDataRepo

type SourceDataRepo interface {
	Save(context.Context, *v1.SourceData) (*v1.SourceData, error)
	UpdateStatus(ctx context.Context, id int64, status int32) error

	FindUnprocessed(ctx context.Context, dataType string) ([]*v1.SourceData, error)
	FindUnprocessedByDate(ctx context.Context, dataType, date string) ([]*v1.SourceData, error)
	// UpdateStatusAndLog 原子性地更新状态和处理日志
	UpdateStatusAndLog(ctx context.Context, id int64, status int32, log string) error
	// FindPartiallyCollectedEntityIDs 查找在指定时间后,只采集了部分数据类型的实体ID列表。
	FindPartiallyCollectedEntityIDs(ctx context.Context, since time.Time, dataTypes []string) ([]string, error)
	ListAllCollectedEntityIDs(ctx context.Context, dataTypes []string) ([]string, error)
	// FindLatestByTypeAndEntityID 根据类型和实体ID查找最新的源数据记录
	FindLatestByTypeAndEntityID(ctx context.Context, dataType string, entityId string) (*SourceData, error)
	// ListAvailableDates 返回指定类型的所有日期
	ListAvailableDates(ctx context.Context, dataType string) ([]string, error)
}

SourceDataRepo 是Biz层依赖的Data层接口,由 data/source_data.go 实现

func NewSourceDataRepo

func NewSourceDataRepo(data *Data, logger log.Logger) SourceDataRepo

NewSourceDataRepo .

type Video

type Video struct {
	AwemeId   string    `gorm:"primaryKey;size:1024"`
	CreatedAt time.Time `gorm:"autoCreateTime;type:timestamp"`
	UpdatedAt time.Time `gorm:"autoUpdateTime;type:timestamp"`

	// --- 基础信息 ---
	AwemeDesc      string    `gorm:"type:text"`
	AwemeCoverUrl  string    `gorm:"size:1024"`
	AwemePubTime   time.Time `gorm:"type:timestamp"`
	AwemeShareUrl  string    `gorm:"size:1024"`
	AwemeDetailUrl string    `gorm:"size:1024"`
	BloggerId      int64     `gorm:"index"`

	PlayCountStr       string `gorm:"size:255"`
	LikeCountStr       string `gorm:"size:255"`
	CommentCountStr    string `gorm:"size:255"`
	ShareCountStr      string `gorm:"size:255"`
	CollectCountStr    string `gorm:"size:255"`
	InteractionRateStr string `gorm:"size:255"`
	ScoreStr           string `gorm:"size:255;comment:视频分数"`
	LikeCommentRateStr string `gorm:"size:255"`
	SalesGmvStr        string `gorm:"size:255"`
	SalesCountStr      string `gorm:"size:255"`
	// 新增解析后的值
	SalesGmv      int64     `gorm:"default:0"`
	SalesCount    int64     `gorm:"default:0"`
	GoodsCountStr string    `gorm:"size:255"`
	GpmStr        string    `gorm:"size:255;column:gpm_str"` // 明确指定列名
	AwemeType     int32     `gorm:"type:integer"`
	GoodsId       string    `gorm:"size:255;comment:商品ID"`
	RankTime      time.Time `gorm:"column:rank_time;comment:上榜时间"`

	// --- 详情信息 (来自下钻采集) 暂时用不上 ---
	DyTagsJSON          string `gorm:"type:text"`
	HotSearchWordsJSON  string `gorm:"type:text"`
	TopicsJSON          string `gorm:"type:text"`
	CommentSegmentsJSON string `gorm:"type:text"`
	InteractionJSON     string `gorm:"type:text"`
	AudienceProfileJSON string `gorm:"type:text"`

	//
	SummaryUpdatedAt *time.Time `gorm:"index;comment:总览数据更新时间;type:timestamp"`
	TrendUpdatedAt   *time.Time `gorm:"index;comment:趋势数据更新时间;type:timestamp"`
	CollectionStatus int32      `gorm:"comment:采集状态"`
}

Video 视频维度表

func (Video) TableName

func (Video) TableName() string

type VideoForCollection

type VideoForCollection struct {
	AwemeId        string
	AwemePubTime   time.Time
	AwemeDetailUrl string
	WorkOrderID    string `gorm:"-" json:"work_order_id"`
}

VideoForCollection 是为采集任务准备的结构体

type VideoForSummary

type VideoForSummary struct {
	AwemeId      string
	AwemePubTime time.Time `gorm:"type:timestamp"`
}

VideoForSummary 定义了用于下钻采集的视频基础信息

type VideoRank

type VideoRank struct {
	ID        uint      `gorm:"primaryKey"`
	CreatedAt time.Time `gorm:"autoCreateTime;type:timestamp"`

	PeriodType string `gorm:"column:period_type;size:1024;not null"`
	RankDate   string `gorm:"column:rank_date;size:1024;not null;uniqueIndex:uq_aweme_rank_date"`
	StartDate  string `gorm:"column:start_date;size:1024;not null;default:''"`
	EndDate    string `gorm:"column:end_date;size:1024;not null;default:''"`

	// 视频信息
	AwemeId        string    `gorm:"column:aweme_id;size:1024;not null;uniqueIndex:uq_aweme_rank_date"`
	AwemeCoverUrl  string    `gorm:"column:aweme_cover_url;size:1024;not null;default:''"`
	AwemeDesc      string    `gorm:"column:aweme_desc;type:text;not null;default:''"`
	AwemePubTime   time.Time `gorm:"column:aweme_pub_time;type:timestamp"`
	AwemeShareUrl  string    `gorm:"column:aweme_share_url;size:1024;not null;default:''"`
	DurationStr    string    `gorm:"column:duration_str;size:1024;not null;default:''"`
	AwemeScoreStr  string    `gorm:"column:aweme_score_str;size:1024;not null;default:''"`
	AwemeDetailUrl string    `gorm:"column:aweme_detail_url;size:1024;not null;default:''"`

	// 商品信息
	GoodsId         string  `gorm:"column:goods_id;size:1024;not null"`
	GoodsTitle      string  `gorm:"column:goods_title;type:text;not null;default:''"`
	GoodsCoverUrl   string  `gorm:"column:goods_cover_url;size:1024;not null;default:''"`
	GoodsPriceRange string  `gorm:"column:goods_price_range;size:1024;not null;default:''"`
	GoodsPrice      float64 `gorm:"column:goods_price"`
	CosRatio        string  `gorm:"column:cos_ratio;size:1024;not null;default:''"`
	CommissionPrice string  `gorm:"column:commission_price;size:1024;not null;default:''"`
	ShopName        string  `gorm:"column:shop_name;size:1024;not null;default:''"`
	BrandName       string  `gorm:"column:brand_name;size:1024;not null;default:''"`
	CategoryNames   string  `gorm:"column:category_names;size:1024;not null;default:''"`

	// 博主信息
	BloggerId      int    `gorm:"column:blogger_id;not null"`
	BloggerUid     string `gorm:"column:blogger_uid;size:1024;not null;default:''"`
	BloggerName    string `gorm:"column:blogger_name;size:1024;not null;default:''"`
	BloggerAvatar  string `gorm:"column:blogger_avatar;size:1024;not null;default:''"`
	BloggerFansNum int    `gorm:"column:blogger_fans_num;not null;default:0"`
	BloggerTag     string `gorm:"column:blogger_tag;size:1024;not null;default:''"`

	// 榜单统计
	SalesCountStr   string `gorm:"column:sales_count_str;size:1024;not null;default:''"`
	TotalSalesStr   string `gorm:"column:total_sales_str;size:1024;not null;default:''"`
	LikeCountIncStr string `gorm:"column:like_count_inc_str;size:1024;not null;default:''"`
	PlayCountIncStr string `gorm:"column:play_count_inc_str;size:1024;not null;default:''"`

	SalesCountLow  int64 `gorm:"column:sales_count_low;comment:销量范围低值"`
	SalesCountHigh int64 `gorm:"column:sales_count_high;comment:销量范围高值"`
	TotalSalesLow  int64 `gorm:"column:total_sales_low;comment:销售额范围低值(分)"`
	TotalSalesHigh int64 `gorm:"column:total_sales_high;comment:销售额范围高值(分)"`
}

VideoRank is the GORM model for storing video ranking data.

type VideoRankRepo

type VideoRankRepo interface {
	// 批量创建视频榜单记录
	BatchCreate(ctx context.Context, ranks []*v1.VideoRankDTO) error
	// 查询单个视频榜单
	GetByAwemeID(ctx context.Context, awemeID, rankType, rankDate string) (*v1.VideoRankDTO, error)
	// 分页查询视频榜单
	ListPage(ctx context.Context, page, size int, rankType, rankDate, sortBy string, sortOrder v1.SortOrder) ([]*VideoRank, int64, error)
	// GetDistinctAwemeIDsByDate 获取指定日期之后上过榜的、不重复的视频ID
	GetDistinctAwemeIDsByDate(ctx context.Context, sinceDate string) ([]string, error)
	// BatchUpsert 批量更新或创建视频榜单记录
	BatchUpsert(ctx context.Context, ranks []*v1.VideoRankDTO) error
}

VideoRankRepo defines the interface for batch creation of VideoRank records. VideoRankRepo 定义了视频榜单数据的持久化接口。

func NewVideoRankRepo

func NewVideoRankRepo(db *Data) VideoRankRepo

NewVideoRankRepo creates a new VideoRankRepo.

type VideoRepo

type VideoRepo interface {
	// FindVideosExcludingIDs 查找排除指定ID列表之外且上榜时间大于等于 rankTimeAfter 的视频
	SaveSourceData(context.Context, *v1.SourceData) (*v1.SourceData, error)
	FindVideosByIDs(ctx context.Context, awemeIDs []string, limit int) ([]*VideoForCollection, error)
	UpsertFromRank(ctx context.Context, video *Video) error
	UpdateFromSummary(ctx context.Context, video *Video) error
	FindVideosNeedingSummaryUpdate(ctx context.Context, limit int) ([]*VideoForSummary, error)
	ListPage(ctx context.Context, page, size int, query, sortBy string, sortOrder v1.SortOrder) ([]*Video, int64, error)
	Get(ctx context.Context, awemeId string) (*Video, error)
	FindRecentActiveAwemeIds(ctx context.Context, days int) ([]string, error)
	//FindVideosNeedingTrendUpdate(ctx context.Context, limit int) ([]*VideoForTrend, error)
	FindVideosForDetailsCollection(ctx context.Context, limit int) ([]*VideoForCollection, error)
	UpdateTrendTimestamp(ctx context.Context, awemeId string) error
	FindVideosExcludingIDs(ctx context.Context, ids []string, limit int, t time.Time) ([]*VideoForCollection, error)

} // End of VideoRepo interface

func NewVideoRepo

func NewVideoRepo(data *Data) VideoRepo

type VideoTrend

type VideoTrend struct {
	Id                 int64     `gorm:"primaryKey"`
	CreatedAt          time.Time `gorm:"autoCreateTime;type:timestamp"`
	UpdatedAt          time.Time `gorm:"autoUpdateTime;type:timestamp"`
	AwemeId            string    `gorm:"size:255;not null;"` // 视频ID
	DateCode           int       `gorm:"not null;"`          // 数据日期
	LikeCount          int64
	LikeCountStr       string `gorm:"size:20"`
	ShareCount         int64
	ShareCountStr      string `gorm:"size:20"`
	CommentCount       int64
	CommentCountStr    string `gorm:"size:20"`
	CollectCount       int64
	CollectCountStr    string `gorm:"size:20"`
	InteractionRate    float64
	InteractionRateStr string `gorm:"size:20"`
	IncLikeCount       int64
	IncLikeCountStr    string `gorm:"size:20"`
	IncShareCount      int64
	IncShareCountStr   string `gorm:"size:20"`
	IncCommentCount    int64
	IncCommentCountStr string `gorm:"size:20"`
	IncCollectCount    int64
	IncCollectCountStr string `gorm:"size:20"`
	SalesCount         int64
	SalesCountStr      string `gorm:"size:20"`
	SalesGmv           float64
	SalesGmvStr        string `gorm:"size:20"`
	Fans               int64
	FansStr            string `gorm:"size:20"`
	IncSalesCount      int64
	IncSalesCountStr   string `gorm:"size:20"`
	IncSalesGmv        float64
	IncSalesGmvStr     string `gorm:"size:20"`
	IncFans            int64
	IncFansStr         string `gorm:"size:20"`
	Gpm                float64
	GpmStr             string `gorm:"size:20"`
	ListTimeStr        string `gorm:"size:20"`
	TimeStamp          int64
}

VideoTrend 视频每日趋势数据模型

type VideoTrendRepo

type VideoTrendRepo interface {
	BatchUpsert(ctx context.Context, trends []*VideoTrend) error
	ListPage(ctx context.Context, page, size int, awemeId, startDate, endDate string) ([]*VideoTrend, int64, error)
	BatchOverwrite(ctx context.Context, trends []*VideoTrend) error
}

VideoTrendRepo 定义视频趋势数据仓库接口

func NewVideoTrendRepo

func NewVideoTrendRepo(data *Data) VideoTrendRepo

NewVideoTrendRepo .

type WorkOrder

type WorkOrder struct {
	ID          uint            `gorm:"primaryKey"`
	WorkOrderID string          `gorm:"uniqueIndex;size:64"`
	TaskType    string          `gorm:"size:64;index"`
	TaskPayload []byte          `gorm:"type:jsonb"`
	UniqueKey   string          `gorm:"size:128;index"`
	Status      WorkOrderStatus `gorm:"size:32;index"`
	CreatedAt   time.Time       `gorm:"autoCreateTime"`
	UpdatedAt   time.Time       `gorm:"autoUpdateTime"`
}

func (WorkOrder) TableName

func (WorkOrder) TableName() string

type WorkOrderRepo

type WorkOrderRepo interface {
	Create(ctx context.Context, order *WorkOrder) error
	FindRecentByUniqueKey(ctx context.Context, uniqueKey string, within time.Duration) (*WorkOrder, error)
	FindByUniqueKey(ctx context.Context, uniqueKey string) (*WorkOrder, error)
	UpdateStatusByWorkOrderID(ctx context.Context, workOrderID string, status WorkOrderStatus) error
	ListByDate(ctx context.Context, date string) ([]*WorkOrder, error)
}

func NewWorkOrderRepo

func NewWorkOrderRepo(data *Data) WorkOrderRepo

type WorkOrderStatus

type WorkOrderStatus string
const (
	WorkOrderStatusCreated    WorkOrderStatus = "created"
	WorkOrderStatusDispatched WorkOrderStatus = "dispatched"
	WorkOrderStatusProcessing WorkOrderStatus = "processing"
	WorkOrderStatusCompleted  WorkOrderStatus = "completed"
	WorkOrderStatusFailed     WorkOrderStatus = "failed"
	WorkOrderStatusRetrying   WorkOrderStatus = "retrying"
)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL