database

package
v0.0.14 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 15, 2026 License: MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source
const DomainComparisonLimit = 10
View Source
const DomainComparisonOutlierRatioThreshold = 1.5
View Source
const DomainComparisonUtilityThreshold = 1.0
View Source
const UserUIDKey = "user_uid"

Variables

This section is empty.

Functions

func Migrate

func Migrate(db *gorm.DB, forced bool) error

Migrate updates the database schema and seeds initial data.

func RunMigrations

func RunMigrations(cfg *config.Config, forced bool) error

Connects to the database and runs the migration

Types

type AnalyzedArticle added in v0.0.7

type AnalyzedArticle struct {
	URL     string      `gorm:"column:url" json:"url"`
	Title   *string     `gorm:"column:title" json:"title,omitempty"`
	Rating  *float64    `gorm:"column:rating" json:"rating,omitempty"`
	Authors StringArray `gorm:"column:authors;type:text[]" json:"authors,omitempty"`
	PubDate time.Time   `gorm:"column:pub_date" json:"pub_date"`
}

type AnalyzedItem added in v0.0.9

type AnalyzedItem struct {
	Hash               string           `json:"hash"`
	URL                string           `json:"url"`
	ThinkResult        *ThinkResult     `gorm:"type:jsonb"`
	Sentiments         *SentimentScores `gorm:"type:jsonb" json:"sentiments,omitempty"`
	SentimentsDeframed *SentimentScores `gorm:"type:jsonb" json:"sentiments_deframed,omitempty"`
	MediaContent       *MediaContent    `gorm:"type:jsonb" json:"media,omitempty"`
	ThinkRating        float64          `json:"rating"`
	Authors            StringArray      `gorm:"type:text[]" json:"authors,omitempty"`
	PubDate            time.Time        `json:"pubDate"`
}

func (AnalyzedItem) MarshalJSON added in v0.0.9

func (a AnalyzedItem) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler to conditionally include ThinkResult fields based on ThinkRating. When ThinkRating is 0, none of the ThinkResult fields are included. When ThinkRating is not 0, the ThinkResult fields are flattened into the AnalyzedItem object.

type Base

type Base struct {
	ID        uuid.UUID      `gorm:"primary_key;type:uuid;default:uuid_generate_v4()"`
	CreatedAt time.Time      `gorm:"not null;default:now()"`
	UpdatedAt time.Time      `gorm:"not null;default:now()"`
	DeletedAt gorm.DeletedAt `gorm:"index"`
}

Base is a gorm.Model but with a UUID id

func (*Base) BeforeCreate

func (base *Base) BeforeCreate(tx *gorm.DB) error

BeforeCreate will set a UUID rather than numeric ID.

type CustomGormLogger added in v0.0.13

type CustomGormLogger struct {
	// contains filtered or unexported fields
}

CustomGormLogger defines custom gorm logger

func NewCustomGormLogger added in v0.0.13

func NewCustomGormLogger(ctx context.Context) *CustomGormLogger

NewCustomGormLogger new a gorm custom logger instance

func (*CustomGormLogger) Error added in v0.0.13

func (l *CustomGormLogger) Error(ctx context.Context, msg string, data ...interface{})

Error print error messages

func (*CustomGormLogger) Info added in v0.0.13

func (l *CustomGormLogger) Info(ctx context.Context, msg string, data ...interface{})

Info print info

func (*CustomGormLogger) LogMode added in v0.0.13

LogMode log mode

func (*CustomGormLogger) Trace added in v0.0.13

func (l *CustomGormLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error)

Trace print custom sql message with uid

func (*CustomGormLogger) Warn added in v0.0.13

func (l *CustomGormLogger) Warn(ctx context.Context, msg string, data ...interface{})

Warn print warn messages

type DomainComparison added in v0.0.5

type DomainComparison struct {
	Classification string  `gorm:"column:classification" json:"classification"`
	RankGroup      int     `gorm:"column:rank_group" json:"rank_group"`
	TrendTopic     string  `gorm:"column:trend_topic" json:"trend_topic"`
	ScoreA         float64 `gorm:"column:score_a" json:"score_a"`
	ScoreB         float64 `gorm:"column:score_b" json:"score_b"`
}

type Feed

type Feed struct {
	Base
	URL               string        `gorm:"index"`                 // we can't enforce uniqueness here (because of the soft deletes)
	RootDomain        *string       `gorm:"index"`                 // example.com
	PortalUrl         *string       `gorm:"type:text"`             // web.foo.bar/whatever
	Language          *string       `gorm:"type:char(2)"`          // ISO 639-1 language code
	EnforceFeedDomain bool          `gorm:"not null;default:true"` // item url must be from our URL
	Enabled           bool          `gorm:"not null;default:false;index"`
	Polling           bool          `gorm:"not null;default:false"`
	Mining            bool          `gorm:"not null;default:false"`
	ResolveItemUrl    bool          `gorm:"not null;default:false"`
	LastSyncedAt      *time.Time    `gorm:"index"`
	Categories        StringArray   `gorm:"type:text[];not null;default:'{}'"`
	FeedSchedule      *FeedSchedule `gorm:"foreignKey:ID;references:ID"`
}

type FeedSchedule

type FeedSchedule struct {
	ID                 uuid.UUID  `gorm:"primaryKey;type:uuid"` // this is a FK to Feed.ID
	CreatedAt          time.Time  `gorm:"not null;default:now()"`
	UpdatedAt          time.Time  `gorm:"not null;default:now()"`
	NextThinkerAt      *time.Time `gorm:"index"`
	ThinkerLockedUntil *time.Time
	NextMiningAt       *time.Time `gorm:"index"`
	MiningLockedUntil  *time.Time
}

type Item

type Item struct {
	ID              uuid.UUID     `gorm:"primaryKey;type:uuid"`
	CreatedAt       time.Time     `gorm:"not null;default:now()"`
	UpdatedAt       time.Time     `gorm:"not null;default:now()"`
	Hash            string        `gorm:"type:char(64);uniqueIndex:idx_hash_feed_url;uniqueIndex:idx_hash_feed;not null"`
	FeedID          uuid.UUID     `gorm:"type:uuid;index;uniqueIndex:idx_feed_url;uniqueIndex:idx_hash_feed_url;uniqueIndex:idx_hash_feed;not null"`
	Feed            Feed          `gorm:"foreignKey:FeedID;references:ID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"`
	URL             string        `gorm:"index;uniqueIndex:idx_feed_url;uniqueIndex:idx_hash_feed_url;not null"`
	Language        *string       `gorm:"type:char(2)"` // ISO 639-1 language code
	Content         string        `gorm:"type:text;not null"`
	PubDate         time.Time     `gorm:"not null;index;default:now()"`
	MediaContent    *MediaContent `gorm:"type:jsonb"`
	ThinkResult     *ThinkResult  `gorm:"type:jsonb"`
	ThinkError      *string       `gorm:"type:text;null"`
	ThinkErrorCount int           `gorm:"not null;default:0"`
	ThinkRating     float64       `gorm:"not null;default:0.0"`
	Categories      StringArray   `gorm:"type:text[];not null;default:'{}'"`
	Authors         StringArray   `gorm:"type:text[];not null;default:'{}'"`
}

func (*Item) BeforeCreate

func (item *Item) BeforeCreate(tx *gorm.DB) error

BeforeCreate will set a UUID rather than numeric ID.

type Lifecycle added in v0.0.5

type Lifecycle struct {
	TimeSlice time.Time `gorm:"column:time_slice" json:"time_slice"`
	Frequency int64     `gorm:"column:frequency" json:"frequency"`
	Velocity  int64     `gorm:"column:velocity" json:"velocity"`
}

type MediaContent

type MediaContent struct {
	// Technical attributes required for <img src> or <video src>
	URL    string `xml:"url,attr" json:"url,omitempty"`
	Type   string `xml:"type,attr" json:"type,omitempty"`     // e.g., "image/jpeg"
	Medium string `xml:"medium,attr" json:"medium,omitempty"` // e.g., "image" or "video"

	// Dimensions are essential for preventing HTML Layout Shift (CLS)
	Height int `xml:"height,attr" json:"height,omitempty"`
	Width  int `xml:"width,attr" json:"width,omitempty"`

	// Descriptive text used for HTML 'alt' attributes and captions
	Title       string `xml:"title" json:"title,omitempty"`
	Description string `xml:"description" json:"description,omitempty"`

	// Essential for Video: Used for the <video poster="..."> attribute
	// We use a pointer so it is nil if no thumbnail exists
	Thumbnail *MediaThumbnail `xml:"thumbnail" json:"thumbnail,omitempty"`

	// Optional Copyright for the content
	Credit string `xml:"credit" json:"credit,omitempty"`
}

func (*MediaContent) Scan

func (j *MediaContent) Scan(value interface{}) error

func (MediaContent) Value

func (j MediaContent) Value() (driver.Value, error)

type MediaThumbnail

type MediaThumbnail struct {
	URL    string `xml:"url,attr" json:"url,omitempty"`
	Height int    `xml:"height,attr" json:"height,omitempty"`
	Width  int    `xml:"width,attr" json:"width,omitempty"`
}

type Repository

type Repository interface {
	FindFeedByUrl(u *url.URL) (*Feed, error)
	FindFeedByUrlAndAvailability(u *url.URL, onlyEnabled bool) (*Feed, error)
	FindFeedById(feedID uuid.UUID) (*Feed, error)
	UpsertFeed(feed *Feed) error
	// FindItemsByUrl retrieves all items associated with a specific URL.
	// As per the specification (Syndication), a single URL can legitimately appear in multiple feeds.
	// Therefore, the system allows multiple Item records for the same URL, distinguished by their FeedID.
	// This means the same content (probably different Hashes) can exist multiple times if it is syndicated across different feeds.
	// Feed.EnforceFeedDomain = will enforce only items with the same base domain as the Feed URL
	FindItemsByUrl(u *url.URL) ([]Item, error)
	FindItemsByRootDomain(rootDomain string, limit int) ([]Item, error)
	GetAllFeeds(deleted bool) ([]Feed, error)
	DeleteFeedById(id uuid.UUID) error
	PurgeFeedById(id uuid.UUID) error
	EnqueueSync(id uuid.UUID, pollingInterval time.Duration) error
	EnqueueMine(id uuid.UUID, miningInterval time.Duration) error
	RemoveMine(id uuid.UUID) error
	RemoveSync(id uuid.UUID) error
	BeginFeedUpdate(lockDuration time.Duration) (*Feed, error)
	EndFeedUpdate(id uuid.UUID, jobErr error, pollingInterval time.Duration) error
	GetPendingItems(feedID uuid.UUID, hashes []string, maxRetries int) (map[string]int, error)
	GetItemsByHashes(feedID uuid.UUID, hashes []string) ([]Item, error)
	BeginThinkerBatch(limit int, lockDuration time.Duration) ([]Item, error)
	BeginThinkerFixerBatch(limit int, since time.Time, minErrorCount int, maxErrorCount int, lockDuration time.Duration) ([]Item, error)
	UpsertItem(item *Item) error
	UpsertItemWithTrendInvalidation(item *Item) error
	FindFeedScheduleById(feedID uuid.UUID) (*FeedSchedule, error)
	CreateFeedSchedule(feedID uuid.UUID) error
	GetTopTrendByDomain(domain string, language string, date *time.Time, days int) ([]TrendMetric, error)
	GetContextByDomain(term string, domain string, language string, date *time.Time, days int) ([]TrendContext, error)
	GetLifecycleByDomain(term string, domain string, language string, date *time.Time, days int) ([]Lifecycle, error)
	GetDomainComparison(domainA string, domainB string, language string, date *time.Time, days int, utilityThreshold float64, outlierRatioThreshold float64, limit int) ([]DomainComparison, error)
	GetArticlesByTrend(term string, domain string, date *time.Time, days int, offset int, limit int) ([]AnalyzedArticle, error)
	GetSentimentsByTrend(term string, domain string, date *time.Time, days int) (*SentimentItem, error)
	FindAnalyzedItemsByRootDomain(rootDomain string, limit int) ([]AnalyzedItem, error)
	FindFirstAnalyzedItemByUrl(u *url.URL) (*AnalyzedItem, error)
}

func NewFromDB

func NewFromDB(db *gorm.DB) Repository

NewFromDB creates a repository from an existing GORM connection.

func NewRepository

func NewRepository(ctx context.Context, cfg *config.Config) (Repository, error)

NewRepository initializes a new repository with a database connection from config.

type Sentiment added in v0.0.9

type Sentiment struct {
	Valence   float64 `json:"v,omitempty"`   // VAD: 1.0..9.0, 5.0 neutral
	Arousal   float64 `json:"a,omitempty"`   // VAD: 1.0..9.0, 5.0 neutral
	Dominance float64 `json:"d,omitempty"`   // VAD: 1.0..9.0, 5.0 neutral
	Joy       float64 `json:"j,omitempty"`   // BE5: 1.0..5.0, 1.0 absent/neutral
	Anger     float64 `json:"a_n,omitempty"` // BE5: 1.0..5.0, 1.0 absent/neutral
	Sadness   float64 `json:"s,omitempty"`   // BE5: 1.0..5.0, 1.0 absent/neutral
	Fear      float64 `json:"f,omitempty"`   // BE5: 1.0..5.0, 1.0 absent/neutral
	Disgust   float64 `json:"d_g,omitempty"` // BE5: 1.0..5.0, 1.0 absent/neutral
}

Sentiment represents psychological language analysis scores.

It combines two complementary emotion models:

1. Dimensional approach (VAD), scale 1.0 to 9.0 where 5.0 is neutral:

  • Valence: polarity/pleasantness (1 = max negative, 9 = max positive)
  • Arousal: activation/excitement (1 = sleepy/sluggish, 9 = excited/frantic)
  • Dominance: perceived control (1 = submissive/weak, 9 = dominant/powerful)

2. Discrete approach (BE5), scale 1.0 to 5.0 where 1.0 is neutral/absence:

  • Joy
  • Anger
  • Sadness
  • Fear
  • Disgust

For BE5, 1 indicates a total absence of the emotion and 5 indicates maximum intensity. Because these scores are generated by a regression-based neural network via cross-lingual distant supervision, floating-point values may occasionally dip slightly below 1.0 or slightly exceed the nominal maximum. This struct is used to map the content of the Sentiments jsonb column.

func (*Sentiment) Scan added in v0.0.9

func (j *Sentiment) Scan(value interface{}) error

Scan implements the sql.Scanner interface for Sentiment.

func (Sentiment) Value added in v0.0.9

func (j Sentiment) Value() (driver.Value, error)

Value implements the driver.Valuer interface for Sentiment.

type SentimentItem added in v0.0.9

type SentimentItem struct {
	Sentiments         *SentimentScores `json:"sentiments,omitempty"`
	SentimentsDeframed *SentimentScores `json:"sentiments_deframed,omitempty"`
}

type SentimentScores added in v0.0.9

type SentimentScores struct {
	Valence   float64 `gorm:"column:valence" json:"valence,omitempty"`
	Arousal   float64 `gorm:"column:arousal" json:"arousal,omitempty"`
	Dominance float64 `gorm:"column:dominance" json:"dominance,omitempty"`
	Joy       float64 `gorm:"column:joy" json:"joy,omitempty"`
	Anger     float64 `gorm:"column:anger" json:"anger,omitempty"`
	Sadness   float64 `gorm:"column:sadness" json:"sadness,omitempty"`
	Fear      float64 `gorm:"column:fear" json:"fear,omitempty"`
	Disgust   float64 `gorm:"column:disgust" json:"disgust,omitempty"`
}

func (*SentimentScores) Scan added in v0.0.9

func (j *SentimentScores) Scan(value interface{}) error

Scan implements the sql.Scanner interface for SentimentScores.

func (SentimentScores) Value added in v0.0.9

func (j SentimentScores) Value() (driver.Value, error)

Value implements the driver.Valuer interface for SentimentScores.

type StringArray

type StringArray []string

StringArray aliases []string to implement sql.Scanner and driver.Valuer for PostgreSQL text[]

func (*StringArray) Scan

func (a *StringArray) Scan(src interface{}) error

func (StringArray) Value

func (a StringArray) Value() (driver.Value, error)

type ThinkResult

type ThinkResult struct {
	TitleOriginal               string  `json:"title_original,omitempty"`
	DescriptionOriginal         string  `json:"description_original,omitempty"`
	TitleCorrected              string  `json:"title_corrected,omitempty"`
	TitleCorrectionReason       string  `json:"title_correction_reason,omitempty"`
	DescriptionCorrected        string  `json:"description_corrected,omitempty"`
	DescriptionCorrectionReason string  `json:"description_correction_reason,omitempty"`
	Framing                     float64 `json:"framing,omitempty"`
	FramingReason               string  `json:"framing_reason,omitempty"`
	Clickbait                   float64 `json:"clickbait,omitempty"`
	ClickbaitReason             string  `json:"clickbait_reason,omitempty"`
	Persuasive                  float64 `json:"persuasive,omitempty"`
	PersuasiveReason            string  `json:"persuasive_reason,omitempty"`
	HyperStimulus               float64 `json:"hyper_stimulus,omitempty"`
	HyperStimulusReason         string  `json:"hyper_stimulus_reason,omitempty"`
	Speculative                 float64 `json:"speculative,omitempty"`
	SpeculativeReason           string  `json:"speculative_reason,omitempty"`
	Overall                     float64 `json:"overall,omitempty"`
	OverallReason               string  `json:"overall_reason,omitempty"`
	Category                    string  `json:"category,omitempty"`
}

ThinkResult we make omitempty to not serialize default e.g. 0.0 or ""

func (*ThinkResult) Scan

func (j *ThinkResult) Scan(value interface{}) error

func (ThinkResult) Value

func (j ThinkResult) Value() (driver.Value, error)

type Trend added in v0.0.5

type Trend struct {
	ItemID             uuid.UUID   `gorm:"primaryKey;type:uuid"` // FK to ItemID
	FeedID             uuid.UUID   `gorm:"type:uuid;not null"`   // FK to FeedID
	Language           string      `gorm:"not null"`
	PubDate            time.Time   `gorm:"not null"`
	CategoryStems      StringArray `gorm:"type:text[]"`
	NounStems          StringArray `gorm:"type:text[]"`
	VerbStems          StringArray `gorm:"type:text[]"`
	AdjectiveStems     StringArray `gorm:"type:text[]"`
	RootDomain         string      `gorm:"not null"`
	Sentiments         *Sentiment  `gorm:"type:jsonb;not null;default:'{}'"`
	SentimentsDeframed *Sentiment  `gorm:"type:jsonb;not null;default:'{}'"`
}

type TrendContext added in v0.0.5

type TrendContext struct {
	Context   string `gorm:"column:context_word" json:"context"`
	Frequency int64  `gorm:"column:frequency" json:"frequency"`
}

type TrendMetric added in v0.0.5

type TrendMetric struct {
	TrendTopic   string    `gorm:"column:trend_topic" json:"trend_topic"`
	Frequency    int64     `gorm:"column:frequency" json:"frequency"`
	Utility      int64     `gorm:"column:utility" json:"utility"`
	OutlierRatio float64   `gorm:"column:outlier_ratio" json:"outlier_ratio"`
	TimeSlice    time.Time `gorm:"column:time_slice" json:"time_slice"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL