firehose

type BodyScope string

type Config struct {
	Settings Settings     `toml:"settings"`
	Fetch    FetchConfig  `toml:"fetch"`
	Fonts    FontConfig   `toml:"fonts"`
	Outputs  []OutputConf `toml:"output"`
	Feeds    []FeedConf   `toml:"feed"`

	// Location is the resolved display *time.Location, threaded through parsing
	// (ParseInLocation) and rendering. A TZ slip reorders the river, not just
	// misdisplays it
	Location *time.Location `toml:"-"`

	// Warnings collects non-fatal problems found while loading (e.g. unknown
	// keys, likely typos).
	Warnings []string `toml:"-"`
}

type Duration time.Duration

type Error struct {
	Code    string
	Message string
}

type ExcerptImage string

type Feed struct {
	ID  int64  // cache primary key
	URL string // canonical URL; updated in place when a 301 is followed

	// Title is the feed's self-reported title. A config Title override wins
	// when the feed's own title is garbage ("Untitled Feed", a bare domain).
	Title string

	// Categories tag the feed into sections. A feed may belong to several; an
	// item inherits its feed's categories. Empty means it appears only in the
	// ALL river.
	Categories []string

	// Config-derived per-feed overrides. Nil/zero means "inherit from output
	// then settings" per the three-tier resolution.
	Body           string            // "", title, excerpt, full, excerpt+expand
	ExcerptImage   string            // "", lead, none
	Exclude        []string          // simple keyword filters (config, not hook)
	RewriteHost    map[string]string // wrong host -> right host (config-only, overlaid per run)
	IncludeURL     []string          // URL-scoped keep filter (config-only, overlaid per run)
	Timezone       string            // zone for zoneless rescued dates (config-only, overlaid per run)
	ExcludeURL     []string          // URL-scoped drop filter (config-only, overlaid per run)
	Include        []string          // simple keyword filters (config, not hook)
	StripSelectors []string          // per-feed cruft removal, applied post-parse

	// Per-feed fetch overrides for CDN-hostile endpoints. UserAgent replaces
	// the global one; Headers are set verbatim on the request. Identifying
	// honestly is the default; impersonation is the operator's per-feed call.
	UserAgent string
	Headers   map[string]string

	// Conditional-GET validators, persisted between runs.
	ETag         string
	LastModified string

	// Health / backoff state. LastStatus is an application error code
	// (ENOTFOUND, EPARSE, ETIMEOUT, ...) reported in firehose.html.
	FailCount    int
	LastStatus   string
	LastSuccess  time.Time // last fetch that produced items
	LastFetched  time.Time
	NextEarliest time.Time // backoff; do not fetch before this
}

type FeedConf struct {
	URL            string            `toml:"url"`
	Title          string            `toml:"title"` // override garbage self-reported titles
	Categories     []string          `toml:"categories"`
	Body           BodyScope         `toml:"body"`
	ExcerptImage   ExcerptImage      `toml:"excerpt_image"`
	Exclude        []string          `toml:"exclude"`
	Include        []string          `toml:"include"`
	StripSelectors []string          `toml:"strip_selectors"`
	DisplayWindow  Duration          `toml:"display_window"` // per-feed override; zero inherits settings
	RewriteHost    map[string]string `toml:"rewrite_host"`   // wrong host -> right host in item links/GUIDs
	IncludeURL     []string          `toml:"include_url"`    // keep only items whose LINK contains one of these
	Timezone       string            `toml:"timezone"`       // zone of zoneless dates in this feed (rescue parser)
	ExcludeURL     []string          `toml:"exclude_url"`    // drop items whose LINK contains any of these

	// Per-feed fetch overrides (CDN-hostile endpoints).
	UserAgent string            `toml:"user_agent"`
	Headers   map[string]string `toml:"headers"`
}

type FeedFilter struct {
	ID       *int64
	URL      *string
	Category *string

	// DueOnly, when true, returns only feeds whose NextEarliest has passed
	// (i.e. not currently in backoff).
	DueOnly bool

	Offset int
	Limit  int
}

type FeedService interface {
	// FindFeeds returns feeds matching filter, and the total count.
	FindFeeds(ctx context.Context, filter FeedFilter) ([]*Feed, int, error)

	// FindFeedByURL returns the feed with the given URL, or an ENOTFOUND error.
	FindFeedByURL(ctx context.Context, url string) (*Feed, error)

	// SyncFeeds reconciles the configured feed set into the cache: inserts new
	// feeds, updates categories/config on existing ones, and removes feeds no
	// longer in config (cascading their items).
	SyncFeeds(ctx context.Context, feeds []*Feed) error

	// UpdateFeed applies fetch/health state to a feed.
	UpdateFeed(ctx context.Context, id int64, upd FeedUpdate) (*Feed, error)
}

type FeedUpdate struct {
	URL          *string // set when a 301 redirect is persisted
	Title        *string
	ETag         *string
	LastModified *string
	FailCount    *int
	LastStatus   *string
	LastSuccess  *time.Time
	LastFetched  *time.Time
	NextEarliest *time.Time
}

type FetchConfig struct {
	Concurrency   int      `toml:"concurrency"`
	PerHostSerial bool     `toml:"per_host_serial"`
	Timeout       Duration `toml:"timeout"`
	UserAgent     string   `toml:"user_agent"`

	// AcceptLanguage is sent on every request when non-empty. Browsers always
	// send it; its absence is a bot tell for CDN filtering. We are honest
	// except when we cannot be, but this is a personal use tool after all.
	AcceptLanguage string `toml:"accept_language"`
}

type FontConfig struct {
	ContentFamily string `toml:"content_family"`
	ChromeFamily  string `toml:"chrome_family"`

	// CSSURL is a remote font stylesheet imported by style.css. Defaulted to
	// Google Fonts for the default families when no self-hosted sources are
	// configured. If you change the families, change this too (or self-host).
	CSSURL string `toml:"css_url"`

	// Self-hosted overrides: when set, @font-face rules are emitted and no
	// remote stylesheet is defaulted.
	ContentSrc string `toml:"content_src"`
	ChromeSrc  string `toml:"chrome_src"`
}

type Item struct {
	ID     kid.ID // first-class value type; id.Time() etc.
	FeedID int64
	GUID   string // feed-provided; dedupe key with FeedID

	Title  string
	URL    string // source link; MAY be empty (see linkless-title rule)
	Author string

	Published time.Time // sort key for the river (TZ-correct, ParseInLocation)

	// BodyHTML is the sanitized item body. When FullContent is true this is the
	// whole item (content:encoded); otherwise it is a teaser (description).
	BodyHTML string

	// SummaryHTML is the sanitized feed-provided summary when it is distinct
	// from the body (feed shipped both content:encoded and description).
	// Excerpt source-of-truth prefers this over truncating BodyHTML.
	SummaryHTML string

	// LeadImage is the first image URL for the item — the first inline <img>
	// in the sanitized body, else the first image enclosure/media thumbnail.
	// Used when excerpt_image = "lead" restores an image to a text excerpt.
	LeadImage string

	// FullContent gates excerpt+expand: the expand affordance is only offered
	// when the feed actually shipped a full body. Inferred at fetch time.
	FullContent bool

	// WordCount of the sanitized text, for the reading-time hint.
	WordCount int

	FetchedAt time.Time

	// Categories are inherited from the item's feed at render time (not stored
	// on the row). Populated when rendering the ALL river's section label.
	Categories []string
}

type ItemFilter struct {
	// Categories selects items in any of these categories. Empty selects all
	// (the ALL river).
	Categories []string

	// Since bounds the display window (e.g. now - 14d). Zero means unbounded.
	Since time.Time

	Limit  int
	Offset int
}

type ItemService interface {
	// FindItems returns items matching filter, newest-first (Published desc,
	// then ID as a stable tiebreaker for deterministic output), and the total
	// count.
	FindItems(ctx context.Context, filter ItemFilter) ([]*Item, int, error)

	// UpsertItems inserts new items and updates changed ones, keyed by
	// (FeedID, GUID). Existing IDs are preserved so kid.ID stays stable.
	UpsertItems(ctx context.Context, items []*Item) error

	// ItemStats returns (FeedID, Published) for every cached item — enough
	// for the health page to compute per-feed shown/cached counts without
	// loading bodies.
	ItemStats(ctx context.Context) ([]ItemStat, error)

	// PurgeExpired deletes items older than the cache retention cutoff. Returns
	// the number removed. (Distinct from the display window: retention keeps
	// GUID history longer to prevent re-published old items reappearing.)
	PurgeExpired(ctx context.Context, olderThan time.Time) (int, error)
}

type ItemStat struct {
	FeedID    int64
	Published time.Time
}

type OPML struct {
	XMLName xml.Name `xml:"opml"`
	Version string   `xml:"version,attr"`
	Head    OPMLHead `xml:"head"`
	Body    OPMLBody `xml:"body"`
}

type OPMLBody struct {
	Outlines []Outline `xml:"outline"`
}

type OPMLHead struct {
	Title string `xml:"title"`
}

type Outline struct {
	Text     string    `xml:"text,attr"`
	Title    string    `xml:"title,attr,omitempty"`
	Type     string    `xml:"type,attr,omitempty"`
	XMLURL   string    `xml:"xmlUrl,attr,omitempty"`
	Children []Outline `xml:"outline,omitempty"`
}

type Output struct {
	Name  string // stable identifier ("gov", "all")
	File  string // output filename ("gov.html", "index.html")
	Title string // page <h1> ("Government & EM")

	// Categories selects which items appear. ["*"] (or empty) is the ALL river.
	Categories []string

	// Per-output overrides (win over settings, lose to per-feed).
	Body         BodyScope
	ExcerptImage ExcerptImage
	ReadingTime  *bool

	// InNav controls whether this output appears in the cross-page nav strip.
	// The firehose.html health page sets this false — generated every run, but
	// unlinked to the "public" nav. There is no security in this scheme.
	InNav bool

	// Health marks the special firehose.html output, rendered from feed
	// health state rather than the item river.
	Health bool
}

type OutputConf struct {
	Name         string       `toml:"name"`
	File         string       `toml:"file"`
	Title        string       `toml:"title"`
	Categories   []string     `toml:"categories"`
	Body         BodyScope    `toml:"body"`
	ExcerptImage ExcerptImage `toml:"excerpt_image"`
	ReadingTime  *bool        `toml:"reading_time"`
}

type Settings struct {
	OutputDir          string       `toml:"output_dir"`
	CacheDB            string       `toml:"cache_db"`
	DisplayWindow      Duration     `toml:"display_window"`  // what renders (e.g. 14d)
	CacheRetention     Duration     `toml:"cache_retention"` // GUID history (longer)
	Timezone           string       `toml:"timezone"`        // IANA name; resolved into Location
	Body               BodyScope    `toml:"body"`
	ExcerptWords       int          `toml:"excerpt_words"`
	ExcerptImage       ExcerptImage `toml:"excerpt_image"`
	Typography         bool         `toml:"typography"`
	ReadingTime        bool         `toml:"reading_time"`
	Highlight          bool         `toml:"highlight"`       // declared-language-only; never guess
	Dedupe             bool         `toml:"dedupe"`          // collapse the same story arriving via multiple feeds
	Theme              string       `toml:"theme"`           // auto | light | dark (page default; toggle overrides)
	HighlightTheme     string       `toml:"highlight_theme"` // chroma style, light mode
	HighlightThemeDark string       `toml:"highlight_theme_dark"`
	LinksNewTab        bool         `toml:"links_new_tab"` // story links open in a new tab (render-time <base>)

	// Integration with the Author's CMS; NoteTemplate, when set, renders a per-item "note" link with {title} and
	// {url} substituted (query-escaped) No backend: it is just a URL and could be used to support other CMS.
	NoteTemplate string `toml:"note_template"`
}