database

package
v1.37.6 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 13, 2026 License: MIT Imports: 19 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var Logger *slog.Logger

Logger is global since we will need it everywhere

Functions

func CalculateUUID

func CalculateUUID(time time.Time) (ulid.ULID, error)

CalculateUUID for the incoming file

func DeleteDocument

func DeleteDocument(docULIDSt string, db Repository) error

DeleteDocument fetches the requested document by ULID

func FetchAllDocuments

func FetchAllDocuments(db Repository) (*[]Document, error)

FetchAllDocuments fetches all the documents in the database

func FetchConfigFromDB

func FetchConfigFromDB(db Repository) (config.ServerConfig, error)

FetchConfigFromDB pulls the server config from the database

func UpdateDocumentField

func UpdateDocumentField(docULIDSt string, field string, newValue interface{}, db Repository) (int, error)

UpdateDocumentField updates a single field in a document

func WriteConfigToDB

func WriteConfigToDB(serverConfig config.ServerConfig, db Repository)

WriteConfigToDB writes the serverconfig to the database for later retrieval

Types

type Dimension

type Dimension struct {
	ID            int       `json:"id"`
	Name          string    `json:"name"`
	DisplayName   string    `json:"display_name"`
	Description   string    `json:"description,omitempty"`
	DimensionType string    `json:"dimension_type"` // 'single' or 'multiple'
	IsRequired    bool      `json:"is_required"`
	CreatedAt     time.Time `json:"created_at"`
	UpdatedAt     time.Time `json:"updated_at"`
}

Dimension represents a structured metadata category (e.g., Person, Location)

type DimensionValue

type DimensionValue struct {
	ID          int       `json:"id"`
	DimensionID int       `json:"dimension_id"`
	Value       string    `json:"value"`
	DisplayName string    `json:"display_name"`
	Description string    `json:"description,omitempty"`
	Color       string    `json:"color"`
	SortOrder   int       `json:"sort_order"`
	CreatedAt   time.Time `json:"created_at"`
	UpdatedAt   time.Time `json:"updated_at"`
}

DimensionValue represents an allowed value for a dimension

type DimensionWithValues

type DimensionWithValues struct {
	Dimension
	Values []DimensionValue `json:"values"`
}

DimensionWithValues includes the dimension and its possible values

type Document

type Document struct {
	ID           int       `json:"id"`
	Name         string    `json:"name"`
	Path         string    `json:"path"` // full path to the file
	IngressTime  time.Time `json:"ingress_time"`
	Folder       string    `json:"folder"`
	Hash         string    `json:"hash"`
	ULID         ulid.ULID `json:"ulid"`          // Have a smaller (than hash) id that can be used in URL's, hopefully speed things up
	DocumentType string    `json:"document_type"` // type of document (pdf, txt, etc)
	FullText     string    `json:"full_text"`
	URL          string    `json:"url"`
}

Document is all of the document information stored in the database

func AddNewDocument

func AddNewDocument(filePath string, fullText string, db Repository) (*Document, error)

AddNewDocument adds a new document to the database

func FetchDocument

func FetchDocument(docULIDSt string, db Repository) (Document, int, error)

FetchDocument fetches the requested document by ULID

func FetchDocumentFromPath

func FetchDocumentFromPath(path string, db Repository) (Document, error)

FetchDocumentFromPath fetches the document by document path

func FetchDocuments

func FetchDocuments(docULIDSt []string, db Repository) ([]Document, int, error)

FetchDocuments fetches an array of documents // TODO: Not fucking needed?

func FetchFolder

func FetchFolder(folderName string, db Repository) ([]Document, error)

FetchFolder grabs all of the documents contained in a folder

func FetchNewestDocuments

func FetchNewestDocuments(numberOf int, db Repository) ([]Document, error)

FetchNewestDocuments fetches the documents that were added last

type DocumentDimension

type DocumentDimension struct {
	ID               int       `json:"id"`
	DocumentID       int       `json:"document_id"`
	DimensionID      int       `json:"dimension_id"`
	DimensionValueID int       `json:"dimension_value_id"`
	CreatedAt        time.Time `json:"created_at"`
	UpdatedAt        time.Time `json:"updated_at"`
}

DocumentDimension represents a dimension value assigned to a document

type DocumentTag

type DocumentTag struct {
	DocumentID int       `json:"document_id"`
	TagID      int       `json:"tag_id"`
	CreatedAt  time.Time `json:"created_at"`
}

DocumentTag represents the many-to-many relationship between documents and tags

type DocumentTagsAndDimensions

type DocumentTagsAndDimensions struct {
	Tags      []string          `json:"tags"`                 // Free tags (no group)
	TagGroups map[string]string `json:"tag_groups,omitempty"` // group_name -> tag_name (one per group)
}

DocumentTagsAndDimensions is a helper struct for JSON sidecar files

type DocumentWithTagsAndDimensions

type DocumentWithTagsAndDimensions struct {
	Document
	Tags       []Tag                     `json:"tags"`
	Dimensions map[string]DimensionValue `json:"dimensions"` // dimension_name -> value
}

DocumentWithTagsAndDimensions extends Document with its tags and dimensions

type EphemeralPostgresDB

type EphemeralPostgresDB struct {
	*PGDB
	// contains filtered or unexported fields
}

EphemeralPostgresDB wraps PGDB with an ephemeral PostgreSQL server for testing

type Job

type Job struct {
	ID          ulid.ULID  `json:"id"`
	Type        JobType    `json:"type"`
	Status      JobStatus  `json:"status"`
	Progress    int        `json:"progress"`         // 0-100
	CurrentStep string     `json:"currentStep"`      // Human-readable current step
	TotalSteps  int        `json:"totalSteps"`       // Total number of steps
	Message     string     `json:"message"`          // Status message
	Error       string     `json:"error,omitempty"`  // Error message if failed
	Result      string     `json:"result,omitempty"` // JSON result data
	CreatedAt   time.Time  `json:"createdAt"`
	UpdatedAt   time.Time  `json:"updatedAt"`
	StartedAt   *time.Time `json:"startedAt,omitempty"`
	CompletedAt *time.Time `json:"completedAt,omitempty"`
}

Job represents a background job or operation

type JobStatus

type JobStatus string

JobStatus represents the status of a job

const (
	JobStatusPending   JobStatus = "pending"
	JobStatusRunning   JobStatus = "running"
	JobStatusCompleted JobStatus = "completed"
	JobStatusFailed    JobStatus = "failed"
	JobStatusCancelled JobStatus = "cancelled"
)

type JobSummary

type JobSummary struct {
	FilesProcessed int    `json:"filesProcessed"`
	FilesTotal     int    `json:"filesTotal"`
	BytesProcessed int64  `json:"bytesProcessed"`
	Errors         int    `json:"errors"`
	Details        string `json:"details,omitempty"`
}

JobSummary provides summary statistics for a job

type JobType

type JobType string

JobType represents the type of job

const (
	JobTypeIngestion     JobType = "ingestion"
	JobTypeCleanup       JobType = "cleanup"
	JobTypeWordCloud     JobType = "wordcloud"
	JobTypeSearchReindex JobType = "search_reindex"
)

type PGDB

type PGDB struct {
	// contains filtered or unexported fields
}

PGDB implements Repository using database/sql with raw PostgreSQL SQL. For production it connects via lib/pq; for dev/testing via go-postgres ("pglike") which transparently translates PostgreSQL SQL to SQLite.

func NewRepository

func NewRepository(cfg config.ServerConfig) *PGDB

NewRepository initializes the database based on configuration

func SetupEphemeralPostgresDatabase

func SetupEphemeralPostgresDatabase() (*PGDB, error)

SetupEphemeralPostgresDatabase creates an ephemeral PostgreSQL instance

func (*PGDB) AddTagToDocument

func (p *PGDB) AddTagToDocument(documentID int, tagID int) error

AddTagToDocument associates a tag with a document

func (*PGDB) Close

func (p *PGDB) Close() error

Close closes the database connection

func (*PGDB) CompleteJob

func (p *PGDB) CompleteJob(jobID ulid.ULID, result string) error

CompleteJob marks a job as completed with optional result data

func (*PGDB) CreateJob

func (p *PGDB) CreateJob(jobType JobType, message string) (*Job, error)

CreateJob creates a new job in the database

func (*PGDB) CreateSavedSearch

func (p *PGDB) CreateSavedSearch(search *SavedSearch) error

CreateSavedSearch creates a new saved search

func (*PGDB) CreateTag

func (p *PGDB) CreateTag(tag *Tag) error

CreateTag creates a new tag

func (*PGDB) DeleteDocument

func (p *PGDB) DeleteDocument(ulidStr string) error

DeleteDocument deletes a document by ULID

func (*PGDB) DeleteOldJobs

func (p *PGDB) DeleteOldJobs(olderThan time.Duration) (int, error)

DeleteOldJobs deletes completed jobs older than the specified duration

func (*PGDB) DeleteSavedSearch

func (p *PGDB) DeleteSavedSearch(id int) error

DeleteSavedSearch deletes a saved search by ID

func (*PGDB) DeleteTag

func (p *PGDB) DeleteTag(id int) error

DeleteTag deletes a tag (cascade removes document associations)

func (*PGDB) ExecuteSearch

func (p *PGDB) ExecuteSearch(parsed *ParsedSearch, page, pageSize int) ([]Document, int, error)

ExecuteSearch executes a parsed search query and returns paginated results

func (*PGDB) GetActiveJobs

func (p *PGDB) GetActiveJobs() ([]Job, error)

GetActiveJobs retrieves all running or pending jobs

func (*PGDB) GetAllDimensions

func (p *PGDB) GetAllDimensions() ([]Dimension, error)

GetAllDimensions returns all dimension definitions

func (*PGDB) GetAllDocuments

func (p *PGDB) GetAllDocuments() ([]Document, error)

GetAllDocuments retrieves all documents

func (*PGDB) GetAllSavedSearches

func (p *PGDB) GetAllSavedSearches() ([]SavedSearch, error)

GetAllSavedSearches returns all saved searches sorted by sort_order

func (*PGDB) GetAllTags

func (p *PGDB) GetAllTags() ([]Tag, error)

GetAllTags returns all tags sorted by group and sort_order Uses CASE WHEN workaround for NULLS FIRST (not yet supported by go-postgres)

func (*PGDB) GetConfig

func (p *PGDB) GetConfig() (*config.ServerConfig, error)

GetConfig retrieves server configuration

func (*PGDB) GetDimensionByID

func (p *PGDB) GetDimensionByID(id int) (*Dimension, error)

GetDimensionByID returns a dimension by its ID

func (*PGDB) GetDimensionByName

func (p *PGDB) GetDimensionByName(name string) (*Dimension, error)

GetDimensionByName returns a dimension by its name

func (*PGDB) GetDimensionValueByValue

func (p *PGDB) GetDimensionValueByValue(dimensionID int, value string) (*DimensionValue, error)

GetDimensionValueByValue returns a dimension value by dimension ID and value string

func (*PGDB) GetDimensionValues

func (p *PGDB) GetDimensionValues(dimensionID int) ([]DimensionValue, error)

GetDimensionValues returns all possible values for a dimension

func (*PGDB) GetDocumentByHash

func (p *PGDB) GetDocumentByHash(hash string) (*Document, error)

GetDocumentByHash retrieves a document by hash

func (*PGDB) GetDocumentByID

func (p *PGDB) GetDocumentByID(id int) (*Document, error)

GetDocumentByID retrieves a document by ID

func (*PGDB) GetDocumentByPath

func (p *PGDB) GetDocumentByPath(path string) (*Document, error)

GetDocumentByPath retrieves a document by file path

func (*PGDB) GetDocumentByULID

func (p *PGDB) GetDocumentByULID(ulidStr string) (*Document, error)

GetDocumentByULID retrieves a document by ULID

func (*PGDB) GetDocumentDimensions

func (p *PGDB) GetDocumentDimensions(documentID int) (map[string]DimensionValue, error)

GetDocumentDimensions returns all dimension values assigned to a document

func (*PGDB) GetDocumentsByFolder

func (p *PGDB) GetDocumentsByFolder(folder string) ([]Document, error)

GetDocumentsByFolder retrieves documents in a specific folder

func (*PGDB) GetDocumentsByTag

func (p *PGDB) GetDocumentsByTag(tagID int, page, pageSize int) ([]Document, int, error)

GetDocumentsByTag returns paginated documents that have a specific tag

func (*PGDB) GetJob

func (p *PGDB) GetJob(jobID ulid.ULID) (*Job, error)

GetJob retrieves a job by ID

func (*PGDB) GetNewestDocuments

func (p *PGDB) GetNewestDocuments(limit int) ([]Document, error)

GetNewestDocuments retrieves the newest documents

func (*PGDB) GetNewestDocumentsWithPagination

func (p *PGDB) GetNewestDocumentsWithPagination(page int, pageSize int) ([]Document, int, error)

GetNewestDocumentsWithPagination retrieves documents with pagination support

func (*PGDB) GetRecentJobs

func (p *PGDB) GetRecentJobs(limit, offset int) ([]Job, error)

GetRecentJobs retrieves the most recent jobs with pagination

func (*PGDB) GetSavedSearchByID

func (p *PGDB) GetSavedSearchByID(id int) (*SavedSearch, error)

GetSavedSearchByID returns a saved search by its ID

func (*PGDB) GetSchemaVersion

func (p *PGDB) GetSchemaVersion() (string, error)

GetSchemaVersion returns the latest applied migration version

func (*PGDB) GetTagByID

func (p *PGDB) GetTagByID(id int) (*Tag, error)

GetTagByID returns a tag by its ID

func (*PGDB) GetTagByName

func (p *PGDB) GetTagByName(name string) (*Tag, error)

GetTagByName returns a tag by its name

func (*PGDB) GetTagGroups

func (p *PGDB) GetTagGroups() ([]string, error)

GetTagGroups returns all distinct tag group names

func (*PGDB) GetTagUsageCount

func (p *PGDB) GetTagUsageCount(tagID int) (int, error)

GetTagUsageCount returns the number of documents using a specific tag

func (*PGDB) GetTagsForDocument

func (p *PGDB) GetTagsForDocument(documentID int) ([]Tag, error)

GetTagsForDocument returns all tags associated with a document

func (*PGDB) GetTopWords

func (p *PGDB) GetTopWords(limit int) ([]WordFrequency, error)

GetTopWords retrieves the top N most frequent words

func (*PGDB) GetUntaggedDocuments

func (p *PGDB) GetUntaggedDocuments(page, pageSize int) ([]Document, int, error)

GetUntaggedDocuments returns paginated documents that have no tags

func (*PGDB) GetWordCloudMetadata

func (p *PGDB) GetWordCloudMetadata() (*WordCloudMetadata, error)

GetWordCloudMetadata retrieves metadata about the word cloud

func (*PGDB) RecalculateAllWordFrequencies

func (p *PGDB) RecalculateAllWordFrequencies() error

RecalculateAllWordFrequencies performs a full recalculation of word frequencies

func (*PGDB) ReindexSearchDocuments

func (p *PGDB) ReindexSearchDocuments() (int, error)

ReindexSearchDocuments reindexes all documents for full-text search

func (*PGDB) RemoveDocumentDimension

func (p *PGDB) RemoveDocumentDimension(documentID int, dimensionID int) error

RemoveDocumentDimension removes a dimension value from a document

func (*PGDB) RemoveTagFromDocument

func (p *PGDB) RemoveTagFromDocument(documentID int, tagID int) error

RemoveTagFromDocument removes a tag association from a document

func (*PGDB) SaveConfig

func (p *PGDB) SaveConfig(cfg *config.ServerConfig) error

SaveConfig saves server configuration

func (*PGDB) SaveDocument

func (p *PGDB) SaveDocument(doc *Document) error

SaveDocument saves or updates a document

func (*PGDB) SearchDocuments

func (p *PGDB) SearchDocuments(searchTerm string) ([]Document, error)

SearchDocuments performs full-text search

func (*PGDB) SetDocumentDimension

func (p *PGDB) SetDocumentDimension(documentID int, dimensionID int, dimensionValueID int) error

SetDocumentDimension sets a dimension value for a document (replaces existing)

func (*PGDB) UpdateDocumentFolder

func (p *PGDB) UpdateDocumentFolder(ulidStr string, folder string) error

UpdateDocumentFolder updates the Folder field of a document

func (*PGDB) UpdateDocumentURL

func (p *PGDB) UpdateDocumentURL(ulidStr string, url string) error

UpdateDocumentURL updates the URL field of a document

func (*PGDB) UpdateJobError

func (p *PGDB) UpdateJobError(jobID ulid.ULID, errorMsg string) error

UpdateJobError updates a job with an error

func (*PGDB) UpdateJobProgress

func (p *PGDB) UpdateJobProgress(jobID ulid.ULID, progress int, currentStep string) error

UpdateJobProgress updates the progress of a job

func (*PGDB) UpdateJobStatus

func (p *PGDB) UpdateJobStatus(jobID ulid.ULID, status JobStatus, message string) error

UpdateJobStatus updates the status of a job

func (*PGDB) UpdateSavedSearch

func (p *PGDB) UpdateSavedSearch(search *SavedSearch) error

UpdateSavedSearch updates an existing saved search

func (*PGDB) UpdateTag

func (p *PGDB) UpdateTag(tag *Tag) error

UpdateTag updates an existing tag

func (*PGDB) UpdateWordFrequencies

func (p *PGDB) UpdateWordFrequencies(docID string) error

UpdateWordFrequencies updates word frequencies after document ingestion

type ParsedSearch

type ParsedSearch struct {
	TextTerms   string   // Free text to search for
	IncludeTags []string // Tags to include (prefixed with #)
	ExcludeTags []string // Tags to exclude (prefixed with ~)
	IsUntagged  bool     // Special: show only untagged documents
	IsAllDocs   bool     // Special: show all documents
}

ParsedSearch represents a parsed search query

func ParseSearchQuery

func ParseSearchQuery(query string) *ParsedSearch

ParseSearchQuery parses a unified search query string Syntax:

  • text: full-text search terms
  • #tagname: include documents with this tag
  • ~tagname: exclude documents with this tag
  • Special: "*" means all documents
  • Special: "!untagged" means only untagged documents

Examples:

  • "cancer" -> text search for "cancer"
  • "#Invoice" -> documents with Invoice tag
  • "~2025" -> documents without 2025 tag
  • "cancer #Invoice ~Draft" -> text "cancer", has Invoice, no Draft

type Repository

type Repository interface {
	Close() error
	SaveDocument(doc *Document) error
	GetDocumentByID(id int) (*Document, error)
	GetDocumentByULID(ulid string) (*Document, error)
	GetDocumentByPath(path string) (*Document, error)
	GetDocumentByHash(hash string) (*Document, error)
	GetNewestDocuments(limit int) ([]Document, error)
	GetNewestDocumentsWithPagination(page int, pageSize int) ([]Document, int, error)
	GetAllDocuments() ([]Document, error)
	GetDocumentsByFolder(folder string) ([]Document, error)
	DeleteDocument(ulid string) error
	UpdateDocumentURL(ulid string, url string) error
	UpdateDocumentFolder(ulid string, folder string) error
	SaveConfig(config *config.ServerConfig) error
	GetConfig() (*config.ServerConfig, error)
	SearchDocuments(searchTerm string) ([]Document, error)
	ReindexSearchDocuments() (int, error)
	// Word cloud methods
	GetTopWords(limit int) ([]WordFrequency, error)
	GetWordCloudMetadata() (*WordCloudMetadata, error)
	RecalculateAllWordFrequencies() error
	UpdateWordFrequencies(docID string) error
	// Job tracking methods
	CreateJob(jobType JobType, message string) (*Job, error)
	UpdateJobProgress(jobID ulid.ULID, progress int, currentStep string) error
	UpdateJobStatus(jobID ulid.ULID, status JobStatus, message string) error
	UpdateJobError(jobID ulid.ULID, errorMsg string) error
	CompleteJob(jobID ulid.ULID, result string) error
	GetJob(jobID ulid.ULID) (*Job, error)
	GetRecentJobs(limit, offset int) ([]Job, error)
	GetActiveJobs() ([]Job, error)
	DeleteOldJobs(olderThan time.Duration) (int, error)
	// Tag methods
	CreateTag(tag *Tag) error
	GetAllTags() ([]Tag, error)
	GetTagByID(id int) (*Tag, error)
	GetTagByName(name string) (*Tag, error)
	UpdateTag(tag *Tag) error
	DeleteTag(id int) error
	GetTagsForDocument(documentID int) ([]Tag, error)
	AddTagToDocument(documentID int, tagID int) error
	RemoveTagFromDocument(documentID int, tagID int) error
	GetTagUsageCount(tagID int) (int, error)
	GetTagGroups() ([]string, error)
	// Dimension methods
	GetAllDimensions() ([]Dimension, error)
	GetDimensionByID(id int) (*Dimension, error)
	GetDimensionByName(name string) (*Dimension, error)
	GetDimensionValues(dimensionID int) ([]DimensionValue, error)
	GetDimensionValueByValue(dimensionID int, value string) (*DimensionValue, error)
	GetDocumentDimensions(documentID int) (map[string]DimensionValue, error)
	SetDocumentDimension(documentID int, dimensionID int, dimensionValueID int) error
	RemoveDocumentDimension(documentID int, dimensionID int) error
	// Schema version
	GetSchemaVersion() (string, error)
	// Saved search methods
	GetAllSavedSearches() ([]SavedSearch, error)
	GetSavedSearchByID(id int) (*SavedSearch, error)
	CreateSavedSearch(search *SavedSearch) error
	UpdateSavedSearch(search *SavedSearch) error
	DeleteSavedSearch(id int) error
	// Search execution methods
	GetDocumentsByTag(tagID int, page, pageSize int) ([]Document, int, error)
	GetUntaggedDocuments(page, pageSize int) ([]Document, int, error)
	ExecuteSearch(parsed *ParsedSearch, page, pageSize int) ([]Document, int, error)
}

Repository defines database operations

type SavedSearch

type SavedSearch struct {
	ID          int       `json:"id"`
	Name        string    `json:"name"`
	Description string    `json:"description"`
	Query       string    `json:"query"`      // Unified search query (text #tag ~exclude)
	Icon        string    `json:"icon"`       // Emoji icon
	SortOrder   int       `json:"sort_order"` // Display order
	IsSystem    bool      `json:"is_system"`  // True for built-in searches
	CreatedAt   time.Time `json:"created_at"`
	UpdatedAt   time.Time `json:"updated_at"`
}

SavedSearch represents a saved search query

type SavedSearchWithCount

type SavedSearchWithCount struct {
	SavedSearch
	DocumentCount int `json:"document_count"`
}

SavedSearchWithCount includes the document count for display

type SearchResult

type SearchResult struct {
	Document
	MatchScore float64 `json:"match_score,omitempty"`
}

SearchResult extends Document with search-related info

type Tag

type Tag struct {
	ID          int       `json:"id"`
	Name        string    `json:"name"`
	Color       string    `json:"color"`
	Description string    `json:"description,omitempty"`
	TagGroup    *string   `json:"tag_group,omitempty"`
	SortOrder   int       `json:"sort_order"`
	CreatedAt   time.Time `json:"created_at"`
	UpdatedAt   time.Time `json:"updated_at"`
}

Tag represents a tag that can be applied to documents If TagGroup is nil/empty, it's a free tag (multiple allowed per document) If TagGroup has a value, only one tag from that group is allowed per document

type TagWithCount

type TagWithCount struct {
	Tag
	DocumentCount int `json:"document_count" db:"document_count"`
}

TagWithCount includes the count of documents using this tag

type WordCloudMetadata

type WordCloudMetadata struct {
	LastCalculation    time.Time `json:"lastCalculation"`
	TotalDocsProcessed int       `json:"totalDocsProcessed"`
	TotalWordsIndexed  int       `json:"totalWordsIndexed"`
	Version            int       `json:"version"`
}

WordCloudMetadata tracks word cloud calculation status

type WordFrequency

type WordFrequency struct {
	Word      string    `json:"word"`
	Frequency int       `json:"frequency"`
	Updated   time.Time `json:"updated"`
}

WordFrequency represents a word and its frequency count

type WordTokenizer

type WordTokenizer struct {
	// contains filtered or unexported fields
}

WordTokenizer handles text processing for word cloud

func NewWordTokenizer

func NewWordTokenizer() *WordTokenizer

NewWordTokenizer creates a new word tokenizer

func (*WordTokenizer) TokenizeAndCount

func (wt *WordTokenizer) TokenizeAndCount(text string) map[string]int

TokenizeAndCount extracts words from text and counts frequencies

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL