Documentation
¶
Index ¶
- Constants
- func Migrate(db *gorm.DB, forced bool) error
- func RunMigrations(cfg *config.Config, forced bool) error
- type AnalyzedArticle
- type AnalyzedItem
- type Base
- type CustomGormLogger
- func (l *CustomGormLogger) Error(ctx context.Context, msg string, data ...interface{})
- func (l *CustomGormLogger) Info(ctx context.Context, msg string, data ...interface{})
- func (l *CustomGormLogger) LogMode(lev logger.LogLevel) logger.Interface
- func (l *CustomGormLogger) Trace(ctx context.Context, begin time.Time, ...)
- func (l *CustomGormLogger) Warn(ctx context.Context, msg string, data ...interface{})
- type DomainComparison
- type Feed
- type FeedSchedule
- type Item
- type Lifecycle
- type MediaContent
- type MediaThumbnail
- type Repository
- type Sentiment
- type SentimentItem
- type SentimentScores
- type StringArray
- type ThinkResult
- type Trend
- type TrendContext
- type TrendMetric
Constants ¶
const DomainComparisonLimit = 10
const DomainComparisonOutlierRatioThreshold = 1.5
const DomainComparisonUtilityThreshold = 1.0
const UserUIDKey = "user_uid"
Variables ¶
This section is empty.
Functions ¶
Types ¶
type AnalyzedArticle ¶ added in v0.0.7
type AnalyzedArticle struct {
URL string `gorm:"column:url" json:"url"`
Title *string `gorm:"column:title" json:"title,omitempty"`
Rating *float64 `gorm:"column:rating" json:"rating,omitempty"`
Authors StringArray `gorm:"column:authors;type:text[]" json:"authors,omitempty"`
PubDate time.Time `gorm:"column:pub_date" json:"pub_date"`
}
type AnalyzedItem ¶ added in v0.0.9
type AnalyzedItem struct {
Hash string `json:"hash"`
URL string `json:"url"`
ThinkResult *ThinkResult `gorm:"type:jsonb"`
Sentiments *SentimentScores `gorm:"type:jsonb" json:"sentiments,omitempty"`
SentimentsDeframed *SentimentScores `gorm:"type:jsonb" json:"sentiments_deframed,omitempty"`
MediaContent *MediaContent `gorm:"type:jsonb" json:"media,omitempty"`
ThinkRating float64 `json:"rating"`
Authors StringArray `gorm:"type:text[]" json:"authors,omitempty"`
PubDate time.Time `json:"pubDate"`
}
func (AnalyzedItem) MarshalJSON ¶ added in v0.0.9
func (a AnalyzedItem) MarshalJSON() ([]byte, error)
MarshalJSON implements json.Marshaler to conditionally include ThinkResult fields based on ThinkRating. When ThinkRating is 0, none of the ThinkResult fields are included. When ThinkRating is not 0, the ThinkResult fields are flattened into the AnalyzedItem object.
type Base ¶
type Base struct {
ID uuid.UUID `gorm:"primary_key;type:uuid;default:uuid_generate_v4()"`
CreatedAt time.Time `gorm:"not null;default:now()"`
UpdatedAt time.Time `gorm:"not null;default:now()"`
DeletedAt gorm.DeletedAt `gorm:"index"`
}
Base is a gorm.Model but with a UUID id
type CustomGormLogger ¶ added in v0.0.13
type CustomGormLogger struct {
// contains filtered or unexported fields
}
CustomGormLogger defines custom gorm logger
func NewCustomGormLogger ¶ added in v0.0.13
func NewCustomGormLogger(ctx context.Context) *CustomGormLogger
NewCustomGormLogger new a gorm custom logger instance
func (*CustomGormLogger) Error ¶ added in v0.0.13
func (l *CustomGormLogger) Error(ctx context.Context, msg string, data ...interface{})
Error print error messages
func (*CustomGormLogger) Info ¶ added in v0.0.13
func (l *CustomGormLogger) Info(ctx context.Context, msg string, data ...interface{})
Info print info
func (*CustomGormLogger) LogMode ¶ added in v0.0.13
func (l *CustomGormLogger) LogMode(lev logger.LogLevel) logger.Interface
LogMode log mode
type DomainComparison ¶ added in v0.0.5
type DomainComparison struct {
Classification string `gorm:"column:classification" json:"classification"`
RankGroup int `gorm:"column:rank_group" json:"rank_group"`
TrendTopic string `gorm:"column:trend_topic" json:"trend_topic"`
ScoreA float64 `gorm:"column:score_a" json:"score_a"`
ScoreB float64 `gorm:"column:score_b" json:"score_b"`
}
type Feed ¶
type Feed struct {
Base
URL string `gorm:"index"` // we can't enforce uniqueness here (because of the soft deletes)
RootDomain *string `gorm:"index"` // example.com
PortalUrl *string `gorm:"type:text"` // web.foo.bar/whatever
Language *string `gorm:"type:char(2)"` // ISO 639-1 language code
EnforceFeedDomain bool `gorm:"not null;default:true"` // item url must be from our URL
Enabled bool `gorm:"not null;default:false;index"`
Polling bool `gorm:"not null;default:false"`
Mining bool `gorm:"not null;default:false"`
ResolveItemUrl bool `gorm:"not null;default:false"`
LastSyncedAt *time.Time `gorm:"index"`
Categories StringArray `gorm:"type:text[];not null;default:'{}'"`
FeedSchedule *FeedSchedule `gorm:"foreignKey:ID;references:ID"`
}
type FeedSchedule ¶
type FeedSchedule struct {
ID uuid.UUID `gorm:"primaryKey;type:uuid"` // this is a FK to Feed.ID
CreatedAt time.Time `gorm:"not null;default:now()"`
UpdatedAt time.Time `gorm:"not null;default:now()"`
NextThinkerAt *time.Time `gorm:"index"`
ThinkerLockedUntil *time.Time
NextMiningAt *time.Time `gorm:"index"`
MiningLockedUntil *time.Time
}
type Item ¶
type Item struct {
ID uuid.UUID `gorm:"primaryKey;type:uuid"`
CreatedAt time.Time `gorm:"not null;default:now()"`
UpdatedAt time.Time `gorm:"not null;default:now()"`
Hash string `gorm:"type:char(64);uniqueIndex:idx_hash_feed_url;uniqueIndex:idx_hash_feed;not null"`
FeedID uuid.UUID `gorm:"type:uuid;index;uniqueIndex:idx_feed_url;uniqueIndex:idx_hash_feed_url;uniqueIndex:idx_hash_feed;not null"`
Feed Feed `gorm:"foreignKey:FeedID;references:ID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"`
URL string `gorm:"index;uniqueIndex:idx_feed_url;uniqueIndex:idx_hash_feed_url;not null"`
Language *string `gorm:"type:char(2)"` // ISO 639-1 language code
Content string `gorm:"type:text;not null"`
PubDate time.Time `gorm:"not null;index;default:now()"`
MediaContent *MediaContent `gorm:"type:jsonb"`
ThinkResult *ThinkResult `gorm:"type:jsonb"`
ThinkError *string `gorm:"type:text;null"`
ThinkErrorCount int `gorm:"not null;default:0"`
ThinkRating float64 `gorm:"not null;default:0.0"`
Categories StringArray `gorm:"type:text[];not null;default:'{}'"`
Authors StringArray `gorm:"type:text[];not null;default:'{}'"`
}
type MediaContent ¶
type MediaContent struct {
// Technical attributes required for <img src> or <video src>
URL string `xml:"url,attr" json:"url,omitempty"`
Type string `xml:"type,attr" json:"type,omitempty"` // e.g., "image/jpeg"
Medium string `xml:"medium,attr" json:"medium,omitempty"` // e.g., "image" or "video"
// Dimensions are essential for preventing HTML Layout Shift (CLS)
Height int `xml:"height,attr" json:"height,omitempty"`
Width int `xml:"width,attr" json:"width,omitempty"`
// Descriptive text used for HTML 'alt' attributes and captions
Title string `xml:"title" json:"title,omitempty"`
Description string `xml:"description" json:"description,omitempty"`
// Essential for Video: Used for the <video poster="..."> attribute
// We use a pointer so it is nil if no thumbnail exists
Thumbnail *MediaThumbnail `xml:"thumbnail" json:"thumbnail,omitempty"`
// Optional Copyright for the content
Credit string `xml:"credit" json:"credit,omitempty"`
}
func (*MediaContent) Scan ¶
func (j *MediaContent) Scan(value interface{}) error
type MediaThumbnail ¶
type Repository ¶
type Repository interface {
FindFeedByUrl(u *url.URL) (*Feed, error)
FindFeedByUrlAndAvailability(u *url.URL, onlyEnabled bool) (*Feed, error)
FindFeedById(feedID uuid.UUID) (*Feed, error)
UpsertFeed(feed *Feed) error
// FindItemsByUrl retrieves all items associated with a specific URL.
// As per the specification (Syndication), a single URL can legitimately appear in multiple feeds.
// Therefore, the system allows multiple Item records for the same URL, distinguished by their FeedID.
// This means the same content (probably different Hashes) can exist multiple times if it is syndicated across different feeds.
// Feed.EnforceFeedDomain = will enforce only items with the same base domain as the Feed URL
FindItemsByUrl(u *url.URL) ([]Item, error)
FindItemsByRootDomain(rootDomain string, limit int) ([]Item, error)
GetAllFeeds(deleted bool) ([]Feed, error)
DeleteFeedById(id uuid.UUID) error
PurgeFeedById(id uuid.UUID) error
EnqueueSync(id uuid.UUID, pollingInterval time.Duration) error
EnqueueMine(id uuid.UUID, miningInterval time.Duration) error
RemoveMine(id uuid.UUID) error
RemoveSync(id uuid.UUID) error
BeginFeedUpdate(lockDuration time.Duration) (*Feed, error)
EndFeedUpdate(id uuid.UUID, jobErr error, pollingInterval time.Duration) error
GetPendingItems(feedID uuid.UUID, hashes []string, maxRetries int) (map[string]int, error)
GetItemsByHashes(feedID uuid.UUID, hashes []string) ([]Item, error)
BeginThinkerBatch(limit int, lockDuration time.Duration) ([]Item, error)
BeginThinkerFixerBatch(limit int, since time.Time, minErrorCount int, maxErrorCount int, lockDuration time.Duration) ([]Item, error)
UpsertItem(item *Item) error
UpsertItemWithTrendInvalidation(item *Item) error
FindFeedScheduleById(feedID uuid.UUID) (*FeedSchedule, error)
CreateFeedSchedule(feedID uuid.UUID) error
GetTopTrendByDomain(domain string, language string, date *time.Time, days int) ([]TrendMetric, error)
GetContextByDomain(term string, domain string, language string, date *time.Time, days int) ([]TrendContext, error)
GetLifecycleByDomain(term string, domain string, language string, date *time.Time, days int) ([]Lifecycle, error)
GetDomainComparison(domainA string, domainB string, language string, date *time.Time, days int, utilityThreshold float64, outlierRatioThreshold float64, limit int) ([]DomainComparison, error)
GetArticlesByTrend(term string, domain string, date *time.Time, days int, offset int, limit int) ([]AnalyzedArticle, error)
GetSentimentsByTrend(term string, domain string, date *time.Time, days int) (*SentimentItem, error)
FindAnalyzedItemsByRootDomain(rootDomain string, limit int) ([]AnalyzedItem, error)
FindFirstAnalyzedItemByUrl(u *url.URL) (*AnalyzedItem, error)
}
func NewFromDB ¶
func NewFromDB(db *gorm.DB) Repository
NewFromDB creates a repository from an existing GORM connection.
func NewRepository ¶
NewRepository initializes a new repository with a database connection from config.
type Sentiment ¶ added in v0.0.9
type Sentiment struct {
Valence float64 `json:"v,omitempty"` // VAD: 1.0..9.0, 5.0 neutral
Arousal float64 `json:"a,omitempty"` // VAD: 1.0..9.0, 5.0 neutral
Dominance float64 `json:"d,omitempty"` // VAD: 1.0..9.0, 5.0 neutral
Joy float64 `json:"j,omitempty"` // BE5: 1.0..5.0, 1.0 absent/neutral
Anger float64 `json:"a_n,omitempty"` // BE5: 1.0..5.0, 1.0 absent/neutral
Sadness float64 `json:"s,omitempty"` // BE5: 1.0..5.0, 1.0 absent/neutral
Fear float64 `json:"f,omitempty"` // BE5: 1.0..5.0, 1.0 absent/neutral
Disgust float64 `json:"d_g,omitempty"` // BE5: 1.0..5.0, 1.0 absent/neutral
}
Sentiment represents psychological language analysis scores.
It combines two complementary emotion models:
1. Dimensional approach (VAD), scale 1.0 to 9.0 where 5.0 is neutral:
- Valence: polarity/pleasantness (1 = max negative, 9 = max positive)
- Arousal: activation/excitement (1 = sleepy/sluggish, 9 = excited/frantic)
- Dominance: perceived control (1 = submissive/weak, 9 = dominant/powerful)
2. Discrete approach (BE5), scale 1.0 to 5.0 where 1.0 is neutral/absence:
- Joy
- Anger
- Sadness
- Fear
- Disgust
For BE5, 1 indicates a total absence of the emotion and 5 indicates maximum intensity. Because these scores are generated by a regression-based neural network via cross-lingual distant supervision, floating-point values may occasionally dip slightly below 1.0 or slightly exceed the nominal maximum. This struct is used to map the content of the Sentiments jsonb column.
type SentimentItem ¶ added in v0.0.9
type SentimentItem struct {
Sentiments *SentimentScores `json:"sentiments,omitempty"`
SentimentsDeframed *SentimentScores `json:"sentiments_deframed,omitempty"`
}
type SentimentScores ¶ added in v0.0.9
type SentimentScores struct {
Valence float64 `gorm:"column:valence" json:"valence,omitempty"`
Arousal float64 `gorm:"column:arousal" json:"arousal,omitempty"`
Dominance float64 `gorm:"column:dominance" json:"dominance,omitempty"`
Joy float64 `gorm:"column:joy" json:"joy,omitempty"`
Anger float64 `gorm:"column:anger" json:"anger,omitempty"`
Sadness float64 `gorm:"column:sadness" json:"sadness,omitempty"`
Fear float64 `gorm:"column:fear" json:"fear,omitempty"`
Disgust float64 `gorm:"column:disgust" json:"disgust,omitempty"`
}
func (*SentimentScores) Scan ¶ added in v0.0.9
func (j *SentimentScores) Scan(value interface{}) error
Scan implements the sql.Scanner interface for SentimentScores.
type StringArray ¶
type StringArray []string
StringArray aliases []string to implement sql.Scanner and driver.Valuer for PostgreSQL text[]
func (*StringArray) Scan ¶
func (a *StringArray) Scan(src interface{}) error
type ThinkResult ¶
type ThinkResult struct {
TitleOriginal string `json:"title_original,omitempty"`
DescriptionOriginal string `json:"description_original,omitempty"`
TitleCorrected string `json:"title_corrected,omitempty"`
TitleCorrectionReason string `json:"title_correction_reason,omitempty"`
DescriptionCorrected string `json:"description_corrected,omitempty"`
DescriptionCorrectionReason string `json:"description_correction_reason,omitempty"`
Framing float64 `json:"framing,omitempty"`
FramingReason string `json:"framing_reason,omitempty"`
Clickbait float64 `json:"clickbait,omitempty"`
ClickbaitReason string `json:"clickbait_reason,omitempty"`
Persuasive float64 `json:"persuasive,omitempty"`
PersuasiveReason string `json:"persuasive_reason,omitempty"`
HyperStimulus float64 `json:"hyper_stimulus,omitempty"`
HyperStimulusReason string `json:"hyper_stimulus_reason,omitempty"`
Speculative float64 `json:"speculative,omitempty"`
SpeculativeReason string `json:"speculative_reason,omitempty"`
Overall float64 `json:"overall,omitempty"`
OverallReason string `json:"overall_reason,omitempty"`
Category string `json:"category,omitempty"`
}
ThinkResult we make omitempty to not serialize default e.g. 0.0 or ""
func (*ThinkResult) Scan ¶
func (j *ThinkResult) Scan(value interface{}) error
type Trend ¶ added in v0.0.5
type Trend struct {
ItemID uuid.UUID `gorm:"primaryKey;type:uuid"` // FK to ItemID
FeedID uuid.UUID `gorm:"type:uuid;not null"` // FK to FeedID
Language string `gorm:"not null"`
PubDate time.Time `gorm:"not null"`
CategoryStems StringArray `gorm:"type:text[]"`
NounStems StringArray `gorm:"type:text[]"`
VerbStems StringArray `gorm:"type:text[]"`
AdjectiveStems StringArray `gorm:"type:text[]"`
RootDomain string `gorm:"not null"`
Sentiments *Sentiment `gorm:"type:jsonb;not null;default:'{}'"`
SentimentsDeframed *Sentiment `gorm:"type:jsonb;not null;default:'{}'"`
}
type TrendContext ¶ added in v0.0.5
type TrendMetric ¶ added in v0.0.5
type TrendMetric struct {
TrendTopic string `gorm:"column:trend_topic" json:"trend_topic"`
Frequency int64 `gorm:"column:frequency" json:"frequency"`
Utility int64 `gorm:"column:utility" json:"utility"`
OutlierRatio float64 `gorm:"column:outlier_ratio" json:"outlier_ratio"`
TimeSlice time.Time `gorm:"column:time_slice" json:"time_slice"`
}