database

package
v0.4.7-1-testing-2025-... Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 3, 2025 License: GPL-2.0 Imports: 32 Imported by: 0

Documentation

Overview

Package database provides database abstraction and management for go-pugleaf

Package database provides fetching progress tracking

Package database provides query helpers for go-pugleaf models

Index

Constants

View Source 
const (
	ShutdownStateRunning    = "running"
	ShutdownStateInProgress = "shutting_down"
	ShutdownStateClean      = "clean_shutdown"
	ShutdownStateCrashed    = "crashed"
)

Shutdown state constants

View Source 
const InitialShutDownCounter = 10
View Source 
const LockLimitBlocking = true
View Source 
const MaxOpenDatabases = 256
View Source 
const (
	SessionIDLength = 64
)

Session security constants

Variables

View Source 
var (
	SessionTimeout   = 1 * time.Hour    // 1 hour sliding timeout
	LoginLockoutTime = 15 * time.Minute // Lockout time after max attempts
	MaxLoginAttempts = 5                // Max failed login attempts
)
View Source 
var BatchDividerChan = make(chan *models.Article, 100)
View Source 
var BatchInterval = 3 * time.Second

SQLite safety limits: split large batches to avoid parameter/length limits

View Source 
var CronGroupSectionsCache = 15 * time.Second
View Source 
var DBidleTimeOut = 1 * time.Hour // HARDCODED cleanupIdleGroups
View Source 
var DefaultCronGroupHash = 15 * time.Minute // TODO expose to config
View Source 
var DefaultGroupHashExpiry = 12 * time.Hour // TODO expose to config
View Source 
var DefaultGroupSectionsCacheExpiry = 60 * time.Second
View Source 
var ENABLE_ARTICLE_CACHE = true
View Source 
var EmbeddedMigrationsFS embed.FS
View Source 
var FETCH_MODE = false // set true in fetcher/main.go
View Source 
var GlobalDBMutex sync.Mutex // Mutex to protect database operations
View Source 
var INIT bool
View Source 
var InitialBatchChannelSize = MaxBatchSize // @AI: DO NOT CHANGE THIS!!!! per group cache channel size. should be less or equal to MaxBatch in processor aka MaxReadLinesXover in nntp-client-commands
View Source 
var LPending = make(chan struct{}, 1)
View Source 
var LimitChan = make(chan struct{}, MaxBatchThreads)
View Source 
var MaxBatchSize int = 100
View Source 
var MaxBatchThreads = 16

don't process more than N groups in parallel: better have some cpu & mem when importing hard!

View Source 
var MemCacheThreadsExpiry = 5 * time.Minute // Default expiry for thread cache entries TODO should match cron cycle
View Source 
var NNTP_AUTH_CACHE_TIME = 15 * time.Minute
View Source 
var PragmaMutex sync.RWMutex // Mutex to protect pragma execution
View Source 
var PragmasGroupDB []string
View Source 
var QueryChan = make(chan struct{}, MaxBatchThreads)
View Source 
var SQLITE_BACKUP_ENABLED = false
View Source 
var SQLITE_BACKUP_INTERVAL = 24 * 7 * time.Hour // weekly backup
View Source 
var SQLITE_busy_timeout = 30000
View Source 
var SQLITE_cache_size = 2000 // 2000 pages or -2000 = 2 MB
View Source 
var SQLITE_foreign_keys = "ON"
View Source 
var SQLITE_sync_mode = "NORMAL" // FULL, OFF
View Source 
var SQLITE_temp_store = "MEMORY"

Functions

func ExtractHierarchyFromGroupName 

func ExtractHierarchyFromGroupName(groupName string) string

ExtractHierarchyFromGroupName extracts the top-level hierarchy from a newsgroup name

func FileExists 

func FileExists(path string) bool

fileExists checks if a file exists

func GenerateAPIToken 

func GenerateAPIToken() (string, error)

GenerateAPIToken creates a new cryptographically secure API token

func GenerateSecureSessionID 

func GenerateSecureSessionID() (string, error)

GenerateSecureSessionID creates a cryptographically secure session ID

func HashToken 

func HashToken(token string) string

HashToken creates a SHA-256 hash of the token for database storage

func LockLimitChan 

func LockLimitChan() bool

func LockPending 

func LockPending() bool

func MD5Hash 

func MD5Hash(input string) string

MD5Hash generates an MD5 hash for a given input string

func MoveFile 

func MoveFile(oldpath, newpath string) error

MoveFile moves/renames a file

func RetryableExec 

func RetryableExec(db *sql.DB, query string, args ...interface{}) (sql.Result, error)

RetryableExec executes a SQL statement with retry logic for lock conflicts

func RetryableQuery 

func RetryableQuery(db *sql.DB, query string, args ...interface{}) (*sql.Rows, error)

RetryableQuery executes a SQL query with retry logic for lock conflicts

func RetryableQueryRow 

func RetryableQueryRow(db *sql.DB, query string, args ...interface{}) *sql.Row

RetryableQueryRow executes a SQL query and returns a single row with retry logic

func RetryableQueryRowScan 

func RetryableQueryRowScan(db *sql.DB, query string, args []interface{}, dest ...interface{}) error

RetryableQueryRowScan executes a SQL query and scans the result with retry logic

func RetryableStmtExec 

func RetryableStmtExec(stmt *sql.Stmt, args ...interface{}) (sql.Result, error)

RetryableStmtExec executes a prepared statement with retry logic for lock conflicts

func RetryableStmtQueryRowScan 

func RetryableStmtQueryRowScan(stmt *sql.Stmt, args []interface{}, dest ...interface{}) error

RetryableStmtQueryRowScan executes a prepared statement QueryRow and scans with retry logic

func RetryableTransactionExec 

func RetryableTransactionExec(db *sql.DB, txFunc func(*sql.Tx) error) error

RetryableTransactionExec executes a transaction with retry logic for lock conflicts

func ReturnLimitChan 

func ReturnLimitChan()

func ReturnPending 

func ReturnPending()

func RsyncDIR 

func RsyncDIR(oldpath, newpath string, removesource bool) error

func SanitizeGroupName 

func SanitizeGroupName(groupName string) string

sanitizeGroupName converts a newsgroup name to a safe filename

func SetEmbeddedMigrations 

func SetEmbeddedMigrations(fs embed.FS)

SetEmbeddedMigrations sets the embedded filesystem for migrations

Types

type APIToken 

type APIToken struct {
	ID         int        `db:"id"`
	APIToken   string     `db:"apitoken"`
	OwnerName  string     `db:"ownername"`
	OwnerID    int        `db:"ownerid"`
	CreatedAt  time.Time  `db:"created_at"`
	LastUsedAt *time.Time `db:"last_used_at"`
	ExpiresAt  *time.Time `db:"expires_at"`
	IsEnabled  bool       `db:"is_enabled"`
	UsageCount int        `db:"usage_count"`
}

APIToken represents an API token record

type ArticleCache 

type ArticleCache struct {
	DB *Database
	// contains filtered or unexported fields
}

ArticleCache provides LRU caching for articles

func NewArticleCache 

func NewArticleCache(maxSize int, ttl time.Duration, db *Database) *ArticleCache

NewArticleCache creates a new article cache

func (*ArticleCache) Cleanup 

func (ac *ArticleCache) Cleanup()

Cleanup removes expired entries (call periodically)

func (*ArticleCache) Clear 

func (ac *ArticleCache) Clear()

Clear removes all cached articles

func (*ArticleCache) ClearGroup 

func (ac *ArticleCache) ClearGroup(groupName string)

ClearGroup removes all cached articles for a specific group

func (*ArticleCache) Get 

func (ac *ArticleCache) Get(groupName string, articleNum int64) (*models.Article, bool)

Get retrieves an article from cache

func (*ArticleCache) Put 

func (ac *ArticleCache) Put(groupName string, articleNum int64, article *models.Article)

Put adds an article to cache

func (*ArticleCache) Remove 

func (ac *ArticleCache) Remove(groupName string, articleNum int64)

Remove explicitly removes an article from cache

func (*ArticleCache) Stats 

func (ac *ArticleCache) Stats() map[string]interface{}

Stats returns cache statistics

type ArticleCacheEntry 

type ArticleCacheEntry struct {
	Article     *models.Article `json:"article"`
	CachedAt    time.Time       `json:"cached_at"`
	AccessCount int64           `json:"access_count"`
	LastAccess  time.Time       `json:"last_access"`
	GroupName   string          `json:"group_name"`
}

ArticleCacheEntry represents a cached article with metadata

type AuthCacheEntry 

type AuthCacheEntry struct {
	UserID       int
	Username     string
	PasswordHash string // hash of the provided password for verification
	ExpiresAt    time.Time
}

AuthCacheEntry represents a cached authentication result

type BatchOrchestrator 

type BatchOrchestrator struct {
	// Configuration
	BatchInterval time.Duration // Timer interval for fallback processing
	// contains filtered or unexported fields
}

func NewBatchOrchestrator 

func NewBatchOrchestrator(batch *SQ3batch) *BatchOrchestrator

func (*BatchOrchestrator) StartOrch 

func (o *BatchOrchestrator) StartOrch()

Start is a convenience method that calls StartOrchestrator

func (*BatchOrchestrator) StartOrchestrator 

func (o *BatchOrchestrator) StartOrchestrator()

StartOrchestrator runs the main orchestrator loop that monitors channels and sends notifications

type BatchTasks 

type BatchTasks struct {
	Newsgroup *string

	// Single unified batch processing
	Mux             sync.RWMutex
	BATCHchan       chan *models.Article // Single channel for all batch items
	BATCHprocessing bool                 // Flag to indicate if batch processing is ongoing
	Expires         time.Time
}

type ConsistencyReport 

type ConsistencyReport struct {
	Newsgroup           string
	MainDBLastArticle   int64
	ArticlesMaxNum      int64
	OverviewMaxNum      int64
	ThreadsMaxNum       int64
	ArticleCount        int64
	OverviewCount       int64
	ThreadCount         int64
	MissingArticles     []int64
	MissingOverviews    []int64
	OrphanedOverviews   []int64 // New: overview entries without articles
	OrphanedThreads     []int64
	MessageIDMismatches []string
	Errors              []string
	HasInconsistencies  bool
}

ConsistencyReport represents the results of a database consistency check

func (*ConsistencyReport) PrintReport 

func (report *ConsistencyReport) PrintReport()

PrintConsistencyReport prints a human-readable consistency report

type DBConfig 

type DBConfig struct {
	// Directory to store database files
	DataDir string

	// Connection pool settings
	MaxOpenConns    int
	MaxIdleConns    int
	ConnMaxLifetime time.Duration

	// Performance settings
	WALMode   bool   // Write-Ahead Logging
	SyncMode  string // OFF, NORMAL, FULL
	CacheSize int    // KB
	TempStore string // MEMORY, FILE

	// Backup settings
	BackupEnabled  bool
	BackupInterval time.Duration
	BackupDir      string

	// Cache settings
	ArticleCacheSize   int           // Maximum number of cached articles
	ArticleCacheExpiry time.Duration // Cache expiry duration
}

Config represents database configuration

func DefaultDBConfig 

func DefaultDBConfig() (dbconfig *DBConfig)

DefaultDBConfig returns default database configuration

type Database 

type Database struct {
	MainMutex sync.RWMutex

	// Caches
	SectionsCache  *GroupSectionDBCache
	MemThreadCache *MemCachedThreads
	ArticleCache   *ArticleCache   // LRU cache for individual articles
	NNTPAuthCache  *NNTPAuthCache  // Authentication cache for NNTP users
	HierarchyCache *HierarchyCache // Fast hierarchy and group browsing cache
	Batch          *SQ3batch       // sqlite3 Batch operations

	WG       *sync.WaitGroup
	StopChan chan struct{} // Channel to signal shutdown
	// contains filtered or unexported fields
}

Database represents the main database connection and per-group database pool

func OpenDatabase 

func OpenDatabase(dbconfig *DBConfig) (*Database, error)

New creates a new Database instance

func (*Database) ActivateNNTPUser 

func (db *Database) ActivateNNTPUser(userID int) error

ActivateNNTPUser activates an NNTP user (reverses soft delete)

func (*Database) AddProvider 

func (db *Database) AddProvider(provider *models.Provider) error

func (*Database) AuthenticateNNTPUser 

func (db *Database) AuthenticateNNTPUser(username, password string) (*models.NNTPUser, error)

AuthenticateNNTPUser authenticates an NNTP user with caching support This function first checks the authentication cache before doing expensive bcrypt verification

func (*Database) BuildThreadTree 

func (db *Database) BuildThreadTree(groupDBs *GroupDBs, threadRoot int64) (*ThreadTree, error)

BuildThreadTree constructs a hierarchical tree for a given thread root

func (*Database) BulkDeleteNewsgroups 

func (db *Database) BulkDeleteNewsgroups(names []string) (int, error)

BulkDeleteNewsgroups deletes multiple inactive newsgroups

func (*Database) BulkUpdateNewsgroupActive 

func (db *Database) BulkUpdateNewsgroupActive(names []string, active bool) (int, error)

BulkUpdateNewsgroupActive updates the active status for multiple newsgroups

func (*Database) CacheTreeStructure 

func (db *Database) CacheTreeStructure(groupDBs *GroupDBs, tree *ThreadTree) error

CacheTreeStructure saves a computed tree to the cache

func (*Database) CheckDatabaseConsistency 

func (db *Database) CheckDatabaseConsistency(newsgroup string) (*ConsistencyReport, error)

CheckDatabaseConsistency performs a comprehensive consistency check for a newsgroup

func (*Database) CheckPreviousShutdown 

func (db *Database) CheckPreviousShutdown() (bool, error)

CheckPreviousShutdown checks if the previous shutdown was clean

func (*Database) CleanupExpiredSessions 

func (db *Database) CleanupExpiredSessions() error

CleanupExpiredSessions removes expired sessions from the database

func (*Database) CleanupExpiredTokens 

func (db *Database) CleanupExpiredTokens() (int, error)

CleanupExpiredTokens removes expired tokens from the database

func (*Database) CleanupOldNNTPSessions 

func (db *Database) CleanupOldNNTPSessions(olderThan time.Duration) error

CleanupOldNNTPSessions removes inactive sessions older than specified duration

func (*Database) CloseNNTPSession 

func (db *Database) CloseNNTPSession(connectionID string) error

CloseNNTPSession marks a session as inactive

func (*Database) CountSearchNewsgroups 

func (db *Database) CountSearchNewsgroups(searchTerm string) (int, error)

func (*Database) CreateAIModel 

func (db *Database) CreateAIModel(postKey, ollamaModelName, displayName, description string, isActive, isDefault bool, sortOrder int) (*models.AIModel, error)

CreateAIModel creates a new AI model

func (*Database) CreateAPIToken 

func (db *Database) CreateAPIToken(ownerName string, ownerID int, expiresAt *time.Time) (*APIToken, string, error)

CreateAPIToken generates and stores a new API token

func (*Database) CreateNNTPSession 

func (db *Database) CreateNNTPSession(userID int, connectionID, remoteAddr string) error

CreateNNTPSession creates a new NNTP session

func (*Database) CreateNNTPUserForWebUser 

func (db *Database) CreateNNTPUserForWebUser(webUserID int64) error

CreateNNTPUserForWebUser automatically creates an NNTP user for a web user

func (*Database) CreateSection 

func (db *Database) CreateSection(section *models.Section) error

CreateSection creates a new section

func (*Database) CreateSectionGroup 

func (db *Database) CreateSectionGroup(sg *models.SectionGroup) error

CreateSectionGroup creates a new section group assignment

func (*Database) CreateSiteNews 

func (db *Database) CreateSiteNews(news *models.SiteNews) error

func (*Database) CreateUserSession 

func (db *Database) CreateUserSession(userID int64, remoteIP string) (string, error)

CreateUserSession creates a new session for the user and invalidates any existing session

func (*Database) CronDB 

func (db *Database) CronDB()

func (*Database) DeactivateNNTPUser 

func (db *Database) DeactivateNNTPUser(userID int) error

DeactivateNNTPUser deactivates an NNTP user (soft delete)

func (*Database) DecrementArticleSpam 

func (db *Database) DecrementArticleSpam(groupName string, articleNum int64) error

DecrementArticleSpam decrements the spam counter for a specific article (admin only)

func (*Database) DeleteAIModel 

func (db *Database) DeleteAIModel(id int) error

DeleteAIModel deletes an AI model (if it's not the last active one)

func (*Database) DeleteAPIToken 

func (db *Database) DeleteAPIToken(tokenID int) error

DeleteAPIToken permanently removes a token

func (*Database) DeleteNNTPUser 

func (db *Database) DeleteNNTPUser(userID int) error

DeleteNNTPUser permanently deletes an NNTP user

func (*Database) DeleteNewsgroup 

func (db *Database) DeleteNewsgroup(name string) error

func (*Database) DeleteProvider 

func (db *Database) DeleteProvider(id int) error

DeleteProvider deletes a provider from the main database

func (*Database) DeleteSection 

func (db *Database) DeleteSection(id int) error

DeleteSection deletes a section and all its group assignments

func (*Database) DeleteSectionGroup 

func (db *Database) DeleteSectionGroup(id int) error

DeleteSectionGroup deletes a section group assignment

func (*Database) DeleteSession 

func (db *Database) DeleteSession(id string) error

func (*Database) DeleteSiteNews 

func (db *Database) DeleteSiteNews(id int) error

func (*Database) DeleteUser 

func (db *Database) DeleteUser(userID int64) error

DeleteUser deletes a user and all associated data

func (*Database) DisableAPIToken 

func (db *Database) DisableAPIToken(tokenID int) error

DisableAPIToken deactivates a token

func (*Database) EnableAPIToken 

func (db *Database) EnableAPIToken(tokenID int) error

EnableAPIToken reactivates a token

func (*Database) ForceCloseGroupDBs 

func (db *Database) ForceCloseGroupDBs(groupsDB *GroupDBs) error

func (*Database) GetAIModelByPostKey 

func (db *Database) GetAIModelByPostKey(postKey string) (*models.AIModel, error)

GetAIModelByPostKey returns an AI model by its post_key

func (*Database) GetActiveAIModels 

func (db *Database) GetActiveAIModels() ([]*models.AIModel, error)

GetActiveAIModels returns all active AI models ordered by sort_order

func (*Database) GetActiveNNTPSessionsForUser 

func (db *Database) GetActiveNNTPSessionsForUser(userID int) (int, error)

GetActiveNNTPSessionsForUser counts active sessions for a user

func (*Database) GetActiveNewsgroupByName 

func (db *Database) GetActiveNewsgroupByName(name string) (*models.Newsgroup, error)

func (*Database) GetActiveNewsgroups 

func (db *Database) GetActiveNewsgroups() ([]*models.Newsgroup, error)

func (*Database) GetActiveNewsgroupsWithMessages 

func (db *Database) GetActiveNewsgroupsWithMessages() ([]*models.Newsgroup, error)

func (*Database) GetAllAIModels 

func (db *Database) GetAllAIModels() ([]*models.AIModel, error)

GetAllAIModels returns all AI models (for admin interface)

func (*Database) GetAllArticles 

func (db *Database) GetAllArticles(groupDBs *GroupDBs) ([]*models.Article, error)

func (*Database) GetAllHierarchies 

func (db *Database) GetAllHierarchies() ([]*models.Hierarchy, error)

func (*Database) GetAllNNTPUsers 

func (db *Database) GetAllNNTPUsers() ([]*models.NNTPUser, error)

GetAllNNTPUsers retrieves all NNTP users

func (*Database) GetAllSectionGroups 

func (db *Database) GetAllSectionGroups() ([]*models.SectionGroup, error)

GetAllSectionGroups retrieves all section group assignments

func (*Database) GetAllSections 

func (db *Database) GetAllSections() ([]*models.Section, error)

GetAllSections retrieves all sections ordered by sort_order

func (*Database) GetAllSectionsWithCounts 

func (db *Database) GetAllSectionsWithCounts() ([]*models.Section, error)

GetAllSectionsWithCounts retrieves all sections with their newsgroup counts

func (*Database) GetAllSiteNews 

func (db *Database) GetAllSiteNews() ([]*models.SiteNews, error)

func (*Database) GetAllUsers 

func (db *Database) GetAllUsers() ([]*models.User, error)

func (*Database) GetArticleByMessageID 

func (db *Database) GetArticleByMessageID(groupDBs *GroupDBs, messageID string) (*models.Article, error)

func (*Database) GetArticleByNum 

func (db *Database) GetArticleByNum(groupDBs *GroupDBs, articleNum int64) (*models.Article, error)

func (*Database) GetArticleCountFromMainDB 

func (db *Database) GetArticleCountFromMainDB(groupName string) (int64, error)

GetArticleCountFromMainDB gets the article count from the main database without opening the group database. This is much more efficient than GetArticlesCount. The message_count field is kept up-to-date by db_batch.go during article processing.

func (*Database) GetArticlesCount 

func (db *Database) GetArticlesCount(groupDBs *GroupDBs) (int64, error)

func (*Database) GetCachedThreadReplies 

func (db *Database) GetCachedThreadReplies(groupDBs *GroupDBs, threadRoot int64, page int, pageSize int) ([]*models.Overview, int, error)

GetCachedThreadReplies retrieves paginated replies for a specific thread

func (*Database) GetCachedThreads 

func (db *Database) GetCachedThreads(groupDBs *GroupDBs, page int64, pageSize int64) ([]*models.ForumThread, int64, error)

GetCachedThreads retrieves cached thread data with pagination (thread list only - no children) First tries memory cache, falls back to database if cache miss

func (*Database) GetCachedTree 

func (db *Database) GetCachedTree(groupDBs *GroupDBs, threadRoot int64) (*ThreadTree, error)

GetCachedTree retrieves a pre-computed tree from the cache

func (*Database) GetConfigBool 

func (db *Database) GetConfigBool(key string) (bool, error)

GetConfigBool retrieves a boolean configuration value

func (*Database) GetConfigValue 

func (db *Database) GetConfigValue(key string) (string, error)

GetConfigValue retrieves a configuration value from the config table

func (*Database) GetDataDir 

func (db *Database) GetDataDir() string

GetDataDir returns the data directory path

func (*Database) GetDefaultAIModel 

func (db *Database) GetDefaultAIModel() (*models.AIModel, error)

GetDefaultAIModel returns the default AI model for new chats

func (*Database) GetDirectGroupsAtLevel 

func (db *Database) GetDirectGroupsAtLevel(prefix string, sortBy string, page int, pageSize int) ([]*models.Newsgroup, int, error)

func (*Database) GetFirstActiveAIModel 

func (db *Database) GetFirstActiveAIModel() (*models.AIModel, error)

GetFirstActiveAIModel returns the first active AI model as fallback

func (*Database) GetGroupDBs 

func (db *Database) GetGroupDBs(groupName string) (*GroupDBs, error)

GetGroupDBs returns groupDB for a specific newsgroup

func (*Database) GetHeaderFieldRange 

func (db *Database) GetHeaderFieldRange(groupDBs *GroupDBs, field string, startNum, endNum int64) (map[int64]string, error)

func (*Database) GetHeaderSections 

func (db *Database) GetHeaderSections() ([]*models.Section, error)

func (*Database) GetHierarchiesPaginated 

func (db *Database) GetHierarchiesPaginated(page, pageSize int, sortBy string) ([]*models.Hierarchy, int, error)

GetHierarchiesPaginated returns hierarchies with pagination and optional sorting

func (*Database) GetHierarchySubLevels 

func (db *Database) GetHierarchySubLevels(prefix string, page int, pageSize int) (map[string]int, int, error)

GetHierarchySubLevels gets immediate sub-hierarchy names and their group counts efficiently with pagination

func (*Database) GetHistoryUseShortHashLen 

func (db *Database) GetHistoryUseShortHashLen(defaultValue int) (int, bool, error)

GetHistoryUseShortHashLen retrieves the UseShortHashLen setting from the database Returns the stored value, or the provided default if not found

func (*Database) GetLastArticleDate 

func (db *Database) GetLastArticleDate(groupDBs *GroupDBs) (*time.Time, error)

func (*Database) GetLatestArticleNumberFromOverview 

func (db *Database) GetLatestArticleNumberFromOverview(newsgroup string) (int64, error)

func (*Database) GetLatestArticleNumbers 

func (db *Database) GetLatestArticleNumbers(newsgroup string) (map[string]int64, error)

func (*Database) GetMainDB 

func (db *Database) GetMainDB() *sql.DB

GetMainDB returns the main database connection for direct access This should only be used by specialized tools like importers

func (*Database) GetNNTPAuthCacheStats 

func (db *Database) GetNNTPAuthCacheStats() map[string]interface{}

GetNNTPAuthCacheStats returns authentication cache statistics

func (*Database) GetNNTPUserByID 

func (db *Database) GetNNTPUserByID(id int) (*models.NNTPUser, error)

GetNNTPUserByID retrieves an NNTP user by ID

func (*Database) GetNNTPUserByUsername 

func (db *Database) GetNNTPUserByUsername(username string) (*models.NNTPUser, error)

GetNNTPUserByUsername retrieves an NNTP user by username

func (*Database) GetNewsgroupByName 

func (db *Database) GetNewsgroupByName(name string) (*models.Newsgroup, error)

GetNewsgroupByName retrieves a newsgroup by name (for getting description)

func (*Database) GetNewsgroupID 

func (db *Database) GetNewsgroupID(groupName string) (int, error)

GetNewsgroupID returns the ID of a newsgroup by name

func (*Database) GetNewsgroupsByExactPrefix 

func (db *Database) GetNewsgroupsByExactPrefix(prefix string) ([]*models.Newsgroup, error)

func (*Database) GetNewsgroupsByHierarchy 

func (db *Database) GetNewsgroupsByHierarchy(hierarchy string, page, pageSize int, sortBy string) ([]*models.Newsgroup, int, error)

func (*Database) GetNewsgroupsByPattern 

func (db *Database) GetNewsgroupsByPattern(pattern string) ([]*models.Newsgroup, error)

func (*Database) GetNewsgroupsByPrefix 

func (db *Database) GetNewsgroupsByPrefix(prefix string) ([]*models.Newsgroup, error)

GetNewsgroupsByPrefix gets newsgroups with names starting with the given prefix

func (*Database) GetNewsgroupsPaginated 

func (db *Database) GetNewsgroupsPaginated(page, pageSize int) ([]*models.Newsgroup, int, error)

func (*Database) GetNewsgroupsPaginatedAdmin 

func (db *Database) GetNewsgroupsPaginatedAdmin(page, pageSize int) ([]*models.Newsgroup, int, error)

func (*Database) GetOverviewByArticleNum 

func (db *Database) GetOverviewByArticleNum(groupDBs *GroupDBs, articleNum int64) (*models.Overview, error)

GetOverviewByArticleNum gets a single overview from articles table by article number

func (*Database) GetOverviewByMessageID 

func (db *Database) GetOverviewByMessageID(groupDBs *GroupDBs, messageID string) (*models.Overview, error)

func (*Database) GetOverviews 

func (db *Database) GetOverviews(groupDBs *GroupDBs) ([]*models.Overview, error)

func (*Database) GetOverviewsPaginated 

func (db *Database) GetOverviewsPaginated(groupDBs *GroupDBs, lastArticleNum int64, pageSize int) ([]*models.Overview, int, bool, error)

func (*Database) GetOverviewsRange 

func (db *Database) GetOverviewsRange(groupDBs *GroupDBs, startNum, endNum int64) ([]*models.Overview, error)

func (*Database) GetProviderByID 

func (db *Database) GetProviderByID(id int) (*models.Provider, error)

func (*Database) GetProviderByName 

func (db *Database) GetProviderByName(name string) (*models.Provider, error)

func (*Database) GetProviders 

func (db *Database) GetProviders() ([]*models.Provider, error)

func (*Database) GetReplyCount 

func (db *Database) GetReplyCount(groupDBs *GroupDBs, messageID string) (int, error)

func (*Database) GetSectionByID 

func (db *Database) GetSectionByID(id int) (*models.Section, error)

GetSectionByID retrieves a section by its ID

func (*Database) GetSectionByName 

func (db *Database) GetSectionByName(name string) (*models.Section, error)

func (*Database) GetSectionGroupByID 

func (db *Database) GetSectionGroupByID(id int) (*models.SectionGroup, error)

GetSectionGroupByID retrieves a section group by its ID

func (*Database) GetSectionGroups 

func (db *Database) GetSectionGroups(sectionID int) ([]*models.SectionGroup, error)

func (*Database) GetSectionGroupsByName 

func (db *Database) GetSectionGroupsByName(newsgroupName string) ([]*models.SectionGroup, error)

func (*Database) GetSections 

func (db *Database) GetSections() ([]*models.Section, error)

func (*Database) GetSession 

func (db *Database) GetSession(id string) (*models.Session, error)

func (*Database) GetShutdownState 

func (db *Database) GetShutdownState() (string, error)

GetShutdownState retrieves the current shutdown state from the database

func (*Database) GetSiteNewsByID 

func (db *Database) GetSiteNewsByID(id int) (*models.SiteNews, error)

func (*Database) GetSpamArticles 

func (db *Database) GetSpamArticles(offset, limit int) ([]*models.Overview, []string, int, error)

func (*Database) GetStats 

func (db *Database) GetStats() *Stats

GetStats returns database connection statistics

func (*Database) GetThreadTreeView 

func (db *Database) GetThreadTreeView(groupDBs *GroupDBs, threadRoot int64, options TreeViewOptions) (*TreeViewResponse, error)

GetThreadTreeView returns a hierarchical tree view for a thread

func (*Database) GetThreads 

func (db *Database) GetThreads(groupDBs *GroupDBs) ([]*models.Thread, error)

func (*Database) GetThreadsCount 

func (db *Database) GetThreadsCount(groupDBs *GroupDBs) (int64, error)

func (*Database) GetTopGroupsByMessageCount 

func (db *Database) GetTopGroupsByMessageCount(limit int) ([]*models.Newsgroup, error)

func (*Database) GetTotalThreadsCount 

func (db *Database) GetTotalThreadsCount() (int64, error)

GetTotalThreadsCount returns the total number of threads across all groups

func (*Database) GetUndownloadedOverviews 

func (db *Database) GetUndownloadedOverviews(groupDBs *GroupDBs, fetchMax int) ([]*models.Overview, error)

func (*Database) GetUserByEmail 

func (db *Database) GetUserByEmail(email string) (*models.User, error)

func (*Database) GetUserByID 

func (db *Database) GetUserByID(id int64) (*models.User, error)

func (*Database) GetUserByUsername 

func (db *Database) GetUserByUsername(username string) (*models.User, error)

func (*Database) GetUserPermissions 

func (db *Database) GetUserPermissions(userID int64) ([]*models.UserPermission, error)

func (*Database) GetVisibleSiteNews 

func (db *Database) GetVisibleSiteNews() ([]*models.SiteNews, error)

func (*Database) HandleThreadTreeAPI 

func (db *Database) HandleThreadTreeAPI(w http.ResponseWriter, r *http.Request)

Example HTTP handler for testing tree view API

func (*Database) HasUserFlaggedSpam 

func (db *Database) HasUserFlaggedSpam(userID int64, groupName string, articleNum int64) (bool, error)

HasUserFlaggedSpam checks if a user has already flagged a specific article as spam

func (*Database) IncrementArticleHide 

func (db *Database) IncrementArticleHide(groupName string, articleNum int64) error

IncrementArticleHide increments the hide counter for a specific article

func (*Database) IncrementArticleSpam 

func (db *Database) IncrementArticleSpam(groupName string, articleNum int64) error

IncrementArticleSpam increments the spam counter for a specific article

func (*Database) IncrementLoginAttempts 

func (db *Database) IncrementLoginAttempts(username string) error

IncrementLoginAttempts increases the failed login counter

func (*Database) IncrementOverviewReplyCount 

func (db *Database) IncrementOverviewReplyCount(groupDBs *GroupDBs, messageID string) error

func (*Database) IncrementReplyCount 

func (db *Database) IncrementReplyCount(groupDBs *GroupDBs, messageID string) error

func (*Database) InitializeSystemStatus 

func (db *Database) InitializeSystemStatus(appVersion string, pid int, hostname string) error

InitializeSystemStatus sets up the system status on startup

func (*Database) InitializeThreadCache 

func (db *Database) InitializeThreadCache(groupDBs *GroupDBs, threadRoot int64, rootArticle *models.Article) error

InitializeThreadCache creates a new cache entry for a thread root

func (*Database) InsertNNTPUser 

func (db *Database) InsertNNTPUser(u *models.NNTPUser) error

InsertNNTPUser creates a new NNTP user with bcrypt password hashing

func (*Database) InsertNewsgroup 

func (db *Database) InsertNewsgroup(g *models.Newsgroup) error

func (*Database) InsertOverview 

func (db *Database) InsertOverview(groupDBs *GroupDBs, o *models.Overview) (int64, error)

func (*Database) InsertSection 

func (db *Database) InsertSection(s *models.Section) error

func (*Database) InsertSectionGroup 

func (db *Database) InsertSectionGroup(sg *models.SectionGroup) error

func (*Database) InsertSession 

func (db *Database) InsertSession(s *models.Session) error

func (*Database) InsertThread 

func (db *Database) InsertThread(groupDBs *GroupDBs, t *models.Thread, a *models.Article) error

func (*Database) InsertUser 

func (db *Database) InsertUser(u *models.User) error

func (*Database) InsertUserPermission 

func (db *Database) InsertUserPermission(up *models.UserPermission) error

func (*Database) InvalidateNNTPUserAuth 

func (db *Database) InvalidateNNTPUserAuth(username string)

InvalidateNNTPUserAuth removes a user from the authentication cache Call this when changing passwords or deactivating users

func (*Database) InvalidateTreeCache 

func (db *Database) InvalidateTreeCache(groupDBs *GroupDBs, threadRoot int64) error

InvalidateTreeCache removes cached tree data when thread structure changes

func (*Database) InvalidateUserSession 

func (db *Database) InvalidateUserSession(userID int64) error

InvalidateUserSession clears the user's session

func (*Database) InvalidateUserSessionBySessionID 

func (db *Database) InvalidateUserSessionBySessionID(sessionID string) error

InvalidateUserSessionBySessionID clears session by session ID

func (*Database) IsDBshutdown 

func (db *Database) IsDBshutdown() bool

func (*Database) IsNewsGroupInSections 

func (db *Database) IsNewsGroupInSections(name string) bool

func (*Database) IsRegistrationEnabled 

func (db *Database) IsRegistrationEnabled() (bool, error)

IsRegistrationEnabled checks if user registration is enabled

func (*Database) IsShuttingDown 

func (db *Database) IsShuttingDown() bool

IsShuttingDown returns true if the database is in the process of shutting down

func (*Database) IsUserLockedOut 

func (db *Database) IsUserLockedOut(username string) (bool, error)

IsUserLockedOut checks if user is temporarily locked out due to failed attempts

func (*Database) ListAPITokens 

func (db *Database) ListAPITokens() ([]*APIToken, error)

ListAPITokens returns all API tokens (for admin purposes)

func (*Database) LoadDefaultProviders 

func (db *Database) LoadDefaultProviders() error

sync default config providers to database

func (*Database) MainDBGetAllNewsgroups 

func (db *Database) MainDBGetAllNewsgroups() ([]*models.Newsgroup, error)

func (*Database) MainDBGetAllNewsgroupsCount 

func (db *Database) MainDBGetAllNewsgroupsCount() int64

func (*Database) MainDBGetNewsgroup 

func (db *Database) MainDBGetNewsgroup(newsgroup string) (*models.Newsgroup, error)

func (*Database) MainDBGetNewsgroupsActiveCount 

func (db *Database) MainDBGetNewsgroupsActiveCount() int64

func (*Database) Migrate 

func (db *Database) Migrate() error

Migrate applies database migrations to all database types

func (*Database) MigrateGroup 

func (db *Database) MigrateGroup(groupName string) error

MigrateGroup applies migrations for a specific newsgroup database

func (*Database) RecordUserSpamFlag 

func (db *Database) RecordUserSpamFlag(userID int64, groupName string, articleNum int64) error

RecordUserSpamFlag records that a user has flagged an article as spam

func (*Database) Rescan 

func (db *Database) Rescan(newsgroup string) error

func (*Database) ResetAllNewsgroupData 

func (db *Database) ResetAllNewsgroupData() error

ResetAllNewsgroupData resets all newsgroup counters and flushes all articles, threads, overview, and cache tables WARNING: This will permanently delete ALL articles, threads, and overview data from ALL newsgroups!

func (*Database) ResetLoginAttempts 

func (db *Database) ResetLoginAttempts(userID int64) error

ResetLoginAttempts clears the failed login counter

func (*Database) ResetNewsgroupCounters 

func (db *Database) ResetNewsgroupCounters(newsgroupName string) error

func (*Database) ResetNewsgroupData 

func (db *Database) ResetNewsgroupData(newsgroupName string) error

ResetNewsgroupData resets articles, threads, overview, and cache tables for a specific newsgroup

func (*Database) SearchNewsgroups 

func (db *Database) SearchNewsgroups(searchTerm string, limit, offset int, admin bool) ([]*models.Newsgroup, error)

func (*Database) SectionGroupExists 

func (db *Database) SectionGroupExists(sectionID int, newsgroupName string) (bool, error)

SectionGroupExists checks if a newsgroup is already assigned to a section

func (*Database) SectionNameExists 

func (db *Database) SectionNameExists(name string) (bool, error)

SectionNameExists checks if a section name already exists

func (*Database) SectionNameExistsExcluding 

func (db *Database) SectionNameExistsExcluding(name string, excludeID int) (bool, error)

SectionNameExistsExcluding checks if a section name exists excluding a specific ID

func (*Database) SetConfigBool 

func (db *Database) SetConfigBool(key string, value bool) error

SetConfigBool sets a boolean configuration value

func (*Database) SetConfigValue 

func (db *Database) SetConfigValue(key, value string) error

SetConfigValue sets or updates a configuration value in the config table

func (*Database) SetDefaultAIModel 

func (db *Database) SetDefaultAIModel(id int) error

SetDefaultAIModel sets a model as default (and unsets others)

func (*Database) SetHistoryUseShortHashLen 

func (db *Database) SetHistoryUseShortHashLen(value int) error

SetHistoryUseShortHashLen stores the UseShortHashLen setting in the database This should only be called on first initialization

func (*Database) SetProvider 

func (db *Database) SetProvider(provider *models.Provider) error

func (*Database) SetShutdownState 

func (db *Database) SetShutdownState(state string) error

SetShutdownState updates the shutdown state in the database

func (*Database) Shutdown 

func (db *Database) Shutdown() error

Close closes all database connections

func (*Database) ToggleSiteNewsVisibility 

func (db *Database) ToggleSiteNewsVisibility(id int) error

func (*Database) UnHideArticle 

func (db *Database) UnHideArticle(groupName string, articleNum int64) error

UnHideArticle sets the hide counter to zero for a specific article

func (*Database) UpdateAIModel 

func (db *Database) UpdateAIModel(id int, ollamaModelName, displayName, description string, isActive, isDefault bool, sortOrder int) error

UpdateAIModel updates an existing AI model

func (*Database) UpdateHeartbeat 

func (db *Database) UpdateHeartbeat()

UpdateHeartbeat updates the last heartbeat timestamp

func (*Database) UpdateHierarchiesLastUpdated 

func (db *Database) UpdateHierarchiesLastUpdated() error

func (*Database) UpdateHierarchyCounts 

func (db *Database) UpdateHierarchyCounts() error

func (*Database) UpdateNNTPSessionActivity 

func (db *Database) UpdateNNTPSessionActivity(connectionID string) error

UpdateNNTPSessionActivity updates the last activity timestamp

func (*Database) UpdateNNTPUserLastLogin 

func (db *Database) UpdateNNTPUserLastLogin(userID int) error

UpdateNNTPUserLastLogin updates the last login timestamp

func (*Database) UpdateNNTPUserPassword 

func (db *Database) UpdateNNTPUserPassword(userID int, password string) error

UpdateNNTPUserPassword updates an NNTP user's password with bcrypt hashing

func (*Database) UpdateNNTPUserPermissions 

func (db *Database) UpdateNNTPUserPermissions(userID int, maxConns int, posting bool) error

UpdateNNTPUserPermissions updates maxconns and posting permissions

func (*Database) UpdateNewsgroup 

func (db *Database) UpdateNewsgroup(g *models.Newsgroup) error

func (*Database) UpdateNewsgroupActive 

func (db *Database) UpdateNewsgroupActive(name string, active bool) error

UpdateNewsgroupActive updates the active status for a newsgroup

func (*Database) UpdateNewsgroupDescription 

func (db *Database) UpdateNewsgroupDescription(name string, description string) error

func (*Database) UpdateNewsgroupExpiry 

func (db *Database) UpdateNewsgroupExpiry(name string, expiryDays int) error

func (*Database) UpdateNewsgroupExpiryPrefix 

func (db *Database) UpdateNewsgroupExpiryPrefix(name string, expiryDays int) error

func (*Database) UpdateNewsgroupMaxArtSize 

func (db *Database) UpdateNewsgroupMaxArtSize(name string, maxArtSize int) error

func (*Database) UpdateNewsgroupMaxArticles 

func (db *Database) UpdateNewsgroupMaxArticles(name string, maxArticles int) error

func (*Database) UpdateNewsgroupMaxArticlesPrefix 

func (db *Database) UpdateNewsgroupMaxArticlesPrefix(name string, maxArticles int) error

func (*Database) UpdateNewsgroupStatus 

func (db *Database) UpdateNewsgroupStatus(name string, status string) error

func (*Database) UpdateNewsgroupWatermarks 

func (db *Database) UpdateNewsgroupWatermarks(name string, highWater, lowWater int) error

func (*Database) UpdateOverviewReplyCount 

func (db *Database) UpdateOverviewReplyCount(groupDBs *GroupDBs, messageID string, replyCount int) error

func (*Database) UpdateReplyCount 

func (db *Database) UpdateReplyCount(groupDBs *GroupDBs, messageID string, replyCount int) error

func (*Database) UpdateSection 

func (db *Database) UpdateSection(section *models.Section) error

UpdateSection updates an existing section

func (*Database) UpdateSiteNews 

func (db *Database) UpdateSiteNews(news *models.SiteNews) error

func (*Database) UpdateThreadCache 

func (db *Database) UpdateThreadCache(groupDBs *GroupDBs, threadRoot int64, childArticleNum int64, childDate time.Time) error

UpdateThreadCache updates an existing cache entry when a reply is added

func (*Database) UpdateTokenUsage 

func (db *Database) UpdateTokenUsage(tokenID int) error

UpdateTokenUsage updates the last_used_at timestamp and increments usage_count

func (*Database) UpdateUserEmail 

func (db *Database) UpdateUserEmail(userID int64, email string) error

func (*Database) UpdateUserPassword 

func (db *Database) UpdateUserPassword(userID int64, passwordHash string) error

func (*Database) UpsertNewsgroupDescription 

func (db *Database) UpsertNewsgroupDescription(name, description string) error

func (*Database) ValidateAPIToken 

func (db *Database) ValidateAPIToken(plainToken string) (*APIToken, error)

ValidateAPIToken checks if a token exists, is enabled, and not expired

func (*Database) ValidateUserSession 

func (db *Database) ValidateUserSession(sessionID string) (*models.User, error)

ValidateUserSession checks if the session is valid and extends expiration

func (*Database) VerifyNNTPUserPassword 

func (db *Database) VerifyNNTPUserPassword(username, password string) (*models.NNTPUser, error)

VerifyNNTPUserPassword verifies a user's password against the stored bcrypt hash

type DateParseAdapter 

type DateParseAdapter func(string) time.Time

DateParseAdapter is a function type for parsing date strings 

var GlobalDateParser DateParseAdapter

Global date parser adapter - can be set by the calling package to use their date parser

type GHmap 

type GHmap struct {

	// map group name to hashed value
	GHmap  map[string]*HashedEntry // key=group value=hashed entry
	Groups map[string]*GroupEntry  // key=hash value=group
	Hhits  uint64                  // Number of times a hash was found in the map
	Hnegs  uint64                  // Number of times a hash was not found in the map
	Ghits  uint64                  // Number of times a group was found in the map
	Gnegs  uint64                  // Number of times a group was not found in the map
	// contains filtered or unexported fields
}
var GroupHashMap *GHmap // Global variable for group hash map

func NewGHmap 

func NewGHmap() *GHmap

func (*GHmap) GHmapCron 

func (h *GHmap) GHmapCron()

func (*GHmap) GetGroupFromHash 

func (h *GHmap) GetGroupFromHash(hash string) (string, bool)

GetGroup retrieves the group name for a given hash

func (*GHmap) Ghit 

func (h *GHmap) Ghit()

func (*GHmap) Gneg 

func (h *GHmap) Gneg()

func (*GHmap) GroupToHash 

func (h *GHmap) GroupToHash(group string) string

GetHash retrieves the hash for a given group name, generating it if not found

func (*GHmap) Hhit 

func (h *GHmap) Hhit()

func (*GHmap) Hneg 

func (h *GHmap) Hneg()

type GroupDBs 

type GroupDBs struct {
	Newsgroup    string    // Name of the newsgroup TODO: remove and use ptr below
	NewsgroupPtr *string   // pointer to the newsgroup
	Idle         time.Time // Last time this group was used
	Workers      int64     // how many are working with this DB
	DB           *sql.DB   // Single database containing articles, overview, threads, etc.
	// contains filtered or unexported fields
}

GroupDBs holds a single database connection for a group

func (*GroupDBs) Close 

func (dbs *GroupDBs) Close(who string) error

func (*GroupDBs) ExistsMsgIdInArticlesDB 

func (db *GroupDBs) ExistsMsgIdInArticlesDB(messageID string) bool

func (*GroupDBs) IncrementWorkers 

func (dbs *GroupDBs) IncrementWorkers()

func (*GroupDBs) Return 

func (dbs *GroupDBs) Return(db *Database)

type GroupEntry 

type GroupEntry struct {
	Group  string    // The hashed value of the group name
	Expiry time.Time // When this hash will expire
}

type GroupSectionDBCache 

type GroupSectionDBCache struct {
	SectionsCache map[string]time.Time // (key: group name, value: expiry time)
	// contains filtered or unexported fields
}

GroupSectionDBCache only caches if a group is in legacy sections

func NewGroupSectionDBCache 

func NewGroupSectionDBCache() *GroupSectionDBCache

func (*GroupSectionDBCache) AddGroupToSectionsCache 

func (g *GroupSectionDBCache) AddGroupToSectionsCache(group string)

func (*GroupSectionDBCache) CronClean 

func (g *GroupSectionDBCache) CronClean()

func (*GroupSectionDBCache) IsInSections 

func (g *GroupSectionDBCache) IsInSections(group string) bool

type HashedEntry 

type HashedEntry struct {
	Hash   string    // The hashed value of the group name
	Expiry time.Time // When this hash will expire
}

type HierarchyCache 

type HierarchyCache struct {
	// contains filtered or unexported fields
}

HierarchyCache provides fast access to hierarchy data

func NewHierarchyCache 

func NewHierarchyCache() *HierarchyCache

NewHierarchyCache creates a new hierarchy cache

func (*HierarchyCache) ForceInvalidateHierarchy 

func (hc *HierarchyCache) ForceInvalidateHierarchy(hierarchyName string)

ForceInvalidateHierarchy forces invalidation bypassing the throttle (use sparingly)

func (*HierarchyCache) GetDirectGroupsAtLevel 

func (hc *HierarchyCache) GetDirectGroupsAtLevel(db *Database, prefix, sortBy string, page, pageSize int) ([]*models.Newsgroup, int, error)

GetDirectGroupsAtLevel returns cached direct groups data

func (*HierarchyCache) GetHierarchiesPaginated 

func (hc *HierarchyCache) GetHierarchiesPaginated(db *Database, page, pageSize int, sortBy string) ([]*models.Hierarchy, int, error)

GetHierarchiesPaginated returns cached hierarchy data with pagination

func (*HierarchyCache) GetHierarchySubLevels 

func (hc *HierarchyCache) GetHierarchySubLevels(db *Database, prefix string, page, pageSize int) (map[string]int, int, error)

GetHierarchySubLevels returns cached sub-level data

func (*HierarchyCache) GetInvalidationStatus 

func (hc *HierarchyCache) GetInvalidationStatus() (time.Time, time.Duration, time.Duration)

GetInvalidationStatus returns information about the throttling status

func (*HierarchyCache) InvalidateAll 

func (hc *HierarchyCache) InvalidateAll()

InvalidateAll clears all cached data

func (*HierarchyCache) InvalidateHierarchy 

func (hc *HierarchyCache) InvalidateHierarchy(hierarchyName string)

InvalidateHierarchy invalidates cache entries for a specific hierarchy with throttling

func (*HierarchyCache) UpdateHierarchyLastUpdated 

func (hc *HierarchyCache) UpdateHierarchyLastUpdated(db *Database) error

UpdateHierarchyLastUpdated updates the cached hierarchy last_updated values from the database

func (*HierarchyCache) UpdateNewsgroupActiveStatus 

func (hc *HierarchyCache) UpdateNewsgroupActiveStatus(newsgroupName string, active bool)

UpdateNewsgroupActiveStatus updates the active status of a newsgroup in cache

func (*HierarchyCache) UpdateNewsgroupStats 

func (hc *HierarchyCache) UpdateNewsgroupStats(newsgroupName string, messageCountIncrement int, newLastArticle int64)

UpdateNewsgroupStats updates cached newsgroup statistics incrementally

func (*HierarchyCache) WaitForWarmup 

func (hc *HierarchyCache) WaitForWarmup(timeout time.Duration) error

WaitForWarmup waits for cache warming to complete

func (*HierarchyCache) WarmCache 

func (hc *HierarchyCache) WarmCache(db *Database)

type MemCachedThreads 

type MemCachedThreads struct {
	Groups map[string]*MemGroupThreadCache // [group] -> all cache data
	// contains filtered or unexported fields
}

func NewMemCachedThreads 

func NewMemCachedThreads() *MemCachedThreads

func (*MemCachedThreads) CleanCron 

func (mem *MemCachedThreads) CleanCron()

func (*MemCachedThreads) GetCachedThreadsFromMemory 

func (mem *MemCachedThreads) GetCachedThreadsFromMemory(db *Database, groupDBs *GroupDBs, group string, page int64, pageSize int64) ([]*models.ForumThread, int64, bool)

GetCachedThreadsFromMemory retrieves threads using the two-level memory cache

func (*MemCachedThreads) GetMemCachedTreadsCount 

func (mem *MemCachedThreads) GetMemCachedTreadsCount(group string) int64

func (*MemCachedThreads) InvalidateGroup 

func (mem *MemCachedThreads) InvalidateGroup(group string)

InvalidateGroup clears all cache for a group

func (*MemCachedThreads) InvalidateThreadRoot 

func (mem *MemCachedThreads) InvalidateThreadRoot(group string, threadRoot int64)

InvalidateThreadRoot removes a specific thread from cache (when thread deleted)

func (*MemCachedThreads) RefreshThreadCache 

func (mem *MemCachedThreads) RefreshThreadCache(db *Database, groupDBs *GroupDBs, group string, requestedPage int64, pageSize int64) error

RefreshThreadCache loads thread data from database and updates memory cache Uses hybrid cursor+page pagination like articles for ultra-fast performance

func (*MemCachedThreads) UpdateThreadMetadata 

func (mem *MemCachedThreads) UpdateThreadMetadata(group string, threadRoot int64, messageCount int, lastActivity time.Time, childArticles string)

UpdateThreadMetadata updates metadata for a specific thread (when new reply added)

type MemGroupThreadCache 

type MemGroupThreadCache struct {
	Expiry            time.Time                   // When this group cache expires
	CountThreads      int64                       // Total thread count
	ThreadRoots       []int64                     // Ordered thread roots by last_activity DESC
	ThreadRootsTS     time.Time                   // When ThreadRoots was last updated
	ThreadMeta        map[int64]*ThreadCacheEntry // [thread_root] -> metadata
	ThreadMetaTS      map[int64]time.Time         // [thread_root] -> last updated
	CacheWindowOffset int64                       // The database offset where this cache window starts
}

MemGroupThreadCache holds all thread cache data for a single newsgroup

type MigrationFile 

type MigrationFile struct {
	FileName    string
	Version     int
	Type        MigrationType
	Description string
	FilePath    string
	IsEmbedded  bool // True if migration is from embedded filesystem
}

MigrationFile represents a migration file with its metadata

type MigrationType 

type MigrationType string

MigrationType represents the type of database that migrations apply to 

const (
	MigrationTypeMain   MigrationType = "main"
	MigrationTypeActive MigrationType = "active"
	MigrationTypeGroup  MigrationType = "group" // Single migration type for group databases
)

type MsgIdTmpCacheItem 

type MsgIdTmpCacheItem struct {
	MessageId    string
	ArtNum       int64
	RootArticle  int64 // Thread root article number (0 if this IS the root)
	IsThreadRoot bool  // True if this article is a thread root
}

MsgIdTmpCacheItem represents a cached message ID item - matches processor definition

type NNTPAuthCache 

type NNTPAuthCache struct {
	// contains filtered or unexported fields
}

NNTPAuthCache provides in-memory caching of successful NNTP authentications

func NewNNTPAuthCache 

func NewNNTPAuthCache(ttl time.Duration) *NNTPAuthCache

NewNNTPAuthCache creates a new authentication cache with specified TTL

func (*NNTPAuthCache) Clear 

func (c *NNTPAuthCache) Clear()

Clear removes all entries from the cache

func (*NNTPAuthCache) Get 

func (c *NNTPAuthCache) Get(username, password string) (int, bool)

Get checks if authentication is cached and still valid

func (*NNTPAuthCache) Remove 

func (c *NNTPAuthCache) Remove(username string)

Remove removes a user from the cache (useful for password changes)

func (*NNTPAuthCache) Set 

func (c *NNTPAuthCache) Set(userID int, username, password string)

Set caches a successful authentication

func (*NNTPAuthCache) Stats 

func (c *NNTPAuthCache) Stats() map[string]interface{}

Stats returns cache statistics

type ProgressDB 

type ProgressDB struct {
	// contains filtered or unexported fields
}

ProgressDB tracks fetching progress for newsgroups per backend

func NewProgressDB 

func NewProgressDB(dataDir string) (*ProgressDB, error)

NewProgressDB creates a new progress tracking database

func (*ProgressDB) Close 

func (p *ProgressDB) Close() error

Close closes the progress database

func (*ProgressDB) GetAllProgress 

func (p *ProgressDB) GetAllProgress() ([]*ProgressEntry, error)

GetAllProgress returns all progress entries

func (*ProgressDB) GetLastArticle 

func (p *ProgressDB) GetLastArticle(backendName, newsgroupName string) (int64, error)

GetLastArticle returns the last fetched article number for a newsgroup on a backend

func (*ProgressDB) GetProgressForBackend 

func (p *ProgressDB) GetProgressForBackend(backendName string) ([]*ProgressEntry, error)

GetProgressForBackend returns progress entries for a specific backend

func (*ProgressDB) UpdateProgress 

func (p *ProgressDB) UpdateProgress(backendName, newsgroupName string, lastArticle int64) error

UpdateProgress updates the fetching progress for a newsgroup on a backend

type ProgressEntry 

type ProgressEntry struct {
	ID            int       `db:"id"`
	BackendName   string    `db:"backend_name"`
	NewsgroupName string    `db:"newsgroup_name"`
	LastArticle   int64     `db:"last_article"`
	LastFetched   time.Time `db:"last_fetched"`
	CreatedAt     time.Time `db:"created_at"`
	UpdatedAt     time.Time `db:"updated_at"`
}

ProgressEntry represents the fetching progress for a newsgroup on a backend

type SQ3batch 

type SQ3batch struct {
	GMux               sync.RWMutex           // Mutex for TasksMap to ensure thread safety
	TasksMap           map[string]*BatchTasks // Map which holds newsgroup cron taskspointers
	TmpTasksChans      chan chan *BatchTasks  // holds temporary batchTasks channels
	TmpArticleSlices   chan []*models.Article // holds temporary article slices
	TmpStringSlices    chan []string          // holds temporary string slices for valuesClauses, messageIDs etc
	TmpStringPtrSlices chan []*string         // holds temporary string slices for valuesClauses, messageIDs etc
	TmpInterfaceSlices chan []interface{}     // holds temporary interface slices for args in threading operations
	// contains filtered or unexported fields
}

func NewSQ3batch 

func NewSQ3batch(db *Database) *SQ3batch

func (*SQ3batch) BatchCaptureOverviewForLater 

func (sq *SQ3batch) BatchCaptureOverviewForLater(newsgroupPtr *string, article *models.Article)

#1 Entry Point for capturing articles / overview data for batched processing

func (*SQ3batch) BatchDivider 

func (sq *SQ3batch) BatchDivider()

func (*SQ3batch) CheckNoMoreWorkInMaps 

func (c *SQ3batch) CheckNoMoreWorkInMaps() bool

CheckNoMoreWorkInMaps checks if all batch channels are empty and not processing

func (*SQ3batch) ExpireCache 

func (sq *SQ3batch) ExpireCache()

func (*SQ3batch) GetChan 

func (sq *SQ3batch) GetChan(newsgroup *string) chan *models.Article

GetChan returns the channel for a specific newsgroup

func (*SQ3batch) GetNewsgroupPointer 

func (sq *SQ3batch) GetNewsgroupPointer(newsgroup string) *string

GetNewsgroupPointer returns a pointer to the newsgroup name in TasksMap

func (*SQ3batch) GetOrCreateTasksMapKey 

func (sq *SQ3batch) GetOrCreateTasksMapKey(newsgroup string) *BatchTasks

GetOrCreateTasksMapKey returns a pointer to the BatchTasks for a specific newsgroup

func (*SQ3batch) SetProcessor 

func (c *SQ3batch) SetProcessor(proc ThreadingProcessor)

SetProcessor sets the threading processor callback interface

type SanitizedArticleCache 

type SanitizedArticleCache struct {
	//mux   sync.RWMutex
	Cache map[string]*SanitizedFields // (key: group name, value: expiry time
}

type SanitizedFields 

type SanitizedFields struct {
	GroupName string
}

type Stats 

type Stats struct {
	MainDB struct {
		OpenConnections int
		IdleConnections int
		WaitCount       int64
		WaitDuration    time.Duration
	}
	GroupDBs map[string]struct {
		OpenConnections int
		IdleConnections int
		WaitCount       int64
		WaitDuration    time.Duration
	}
}

Stats returns database statistics

type ThreadCacheBatch 

type ThreadCacheBatch struct {
	Newsgroup  string
	ThreadRoot int64
	Article    *models.Article
}

ThreadCacheBatch represents a staged thread cache initialization waiting for batch processing

type ThreadCacheEntry 

type ThreadCacheEntry struct {
	ThreadRoot      int64
	RootDate        time.Time
	MessageCount    int
	ChildArticles   string
	LastChildNumber int64
	LastActivity    time.Time
	CreatedAt       time.Time
}

ThreadCacheEntry represents a cached thread

type ThreadTree 

type ThreadTree struct {
	ThreadRoot  int64               `json:"thread_root"`
	MaxDepth    int                 `json:"max_depth"`
	TotalNodes  int                 `json:"total_nodes"`
	LeafCount   int                 `json:"leaf_count"`
	RootNode    *TreeNode           `json:"root_node"`
	NodeMap     map[int64]*TreeNode `json:"-"` // For quick lookup by article_num
	LastUpdated time.Time           `json:"last_updated"`
}

ThreadTree represents a complete thread tree structure

func (*ThreadTree) GetThreadTreeHTML 

func (tree *ThreadTree) GetThreadTreeHTML(groupName string) template.HTML

GetThreadTreeHTML generates HTML representation of the tree (for web display)

func (*ThreadTree) GetTreeStats 

func (tree *ThreadTree) GetTreeStats() TreeStats

GetTreeStats returns a TreeStats struct with tree statistics and MessageCount

func (*ThreadTree) GetTreeStructureJSON 

func (tree *ThreadTree) GetTreeStructureJSON() (string, error)

GetTreeStructureJSON returns a JSON representation of the tree structure

func (*ThreadTree) PrintTreeASCII 

func (tree *ThreadTree) PrintTreeASCII()

PrintThreadTreeASCII prints a simple ASCII representation of the tree for debugging

type ThreadingProcessor 

type ThreadingProcessor interface {
	MsgIdExists(group *string, messageID string) bool
	// Add methods for history and cache operations
	AddProcessedArticleToHistory(msgIdItem *history.MessageIdItem, newsgroup *string, articleNumber int64)
	// Add method for finding thread roots - matches proc_MsgIDtmpCache.go signature (updated to use pointer)
	FindThreadRootInCache(groupName *string, refs []string) *MsgIdTmpCacheItem
	CheckNoMoreWorkInHistory() bool
}

type TreeNode 

type TreeNode struct {
	ArticleNum      int64            `json:"article_num"`
	ParentArticle   *int64           `json:"parent_article"` // nil for root
	Depth           int              `json:"depth"`
	ChildCount      int              `json:"child_count"`
	DescendantCount int              `json:"descendant_count"`
	TreePath        string           `json:"tree_path"` // "0.1.3.7"
	SortOrder       int              `json:"sort_order"`
	Children        []*TreeNode      `json:"children,omitempty"` // Loaded on demand
	Overview        *models.Overview `json:"overview,omitempty"` // Article data
}

TreeNode represents a single node in the hierarchical thread tree

type TreeStats 

type TreeStats struct {
	ThreadRoot    int64     `json:"thread_root"`
	MaxDepth      int       `json:"max_depth"`
	TotalNodes    int       `json:"total_nodes"`
	LeafCount     int       `json:"leaf_count"`
	MessageCount  int       `json:"message_count"`  // Same as TotalNodes for compatibility
	TreeStructure string    `json:"tree_structure"` // JSON representation
	LastUpdated   time.Time `json:"last_updated"`
}

TreeStats holds aggregated statistics for a thread tree

type TreeViewOptions 

type TreeViewOptions struct {
	MaxDepth        int    `json:"max_depth"`        // Limit tree depth (0 = no limit)
	CollapseDepth   int    `json:"collapse_depth"`   // Auto-collapse nodes deeper than this
	IncludeOverview bool   `json:"include_overview"` // Include full overview data for each node
	PageSize        int    `json:"page_size"`        // For paginated tree loading
	SortBy          string `json:"sort_by"`          // "date", "author", "subject"
}

TreeViewOptions configures how the tree should be displayed

type TreeViewResponse 

type TreeViewResponse struct {
	ThreadRoot int64           `json:"thread_root"`
	Tree       *ThreadTree     `json:"tree"`
	Options    TreeViewOptions `json:"options"`
	Error      string          `json:"error,omitempty"`
	CacheHit   bool            `json:"cache_hit"`
	BuildTime  string          `json:"build_time,omitempty"`
}

TreeViewResponse is the JSON response for tree view API calls

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.