store

type Config struct {
	ID                     uint   `gorm:"primaryKey"`
	ProjectID              uint   `gorm:"uniqueIndex;not null"`
	Domain                 string `gorm:"not null"`
	JSRenderingEnabled     bool   `gorm:"default:false"`
	InitialWaitMs          int    `gorm:"default:1500"` // Initial wait after page load for JS frameworks to hydrate (in milliseconds)
	ScrollWaitMs           int    `gorm:"default:2000"` // Wait after scrolling to bottom for lazy-loaded content (in milliseconds)
	FinalWaitMs            int    `gorm:"default:1000"` // Final wait before capturing HTML (in milliseconds)
	Parallelism            int    `gorm:"default:5"`
	UserAgent              string `gorm:"type:text;default:'bluesnake/1.0 (+https://snake.blue)'"`
	IncludeSubdomains      bool   `gorm:"default:false"`                                // When true, crawl all subdomains of the project domain
	DiscoveryMechanisms    string `gorm:"type:text;default:'[\"spider\",\"sitemap\"]'"` // JSON array
	SitemapURLs            string `gorm:"type:text"`                                    // JSON array, nullable
	CheckExternalResources bool   `gorm:"default:true"`                                 // When true, validate external resources for broken links
	// Crawler directive configuration (robots.txt, nofollow, noindex, etc.)
	RobotsTxtMode            string `gorm:"default:'respect'"` // "respect", "ignore", or "ignore-report"
	FollowInternalNofollow   bool   `gorm:"default:false"`     // When true, follow links with rel="nofollow" on same domain
	FollowExternalNofollow   bool   `gorm:"default:false"`     // When true, follow links with rel="nofollow" on external domains
	RespectMetaRobotsNoindex bool   `gorm:"default:true"`      // When true, respect <meta name="robots" content="noindex">
	RespectNoindex           bool   `gorm:"default:true"`      // When true, respect X-Robots-Tag: noindex headers
	// Incremental crawling configuration
	IncrementalCrawlingEnabled bool     `gorm:"default:false"` // When true, crawl in chunks and allow resume
	CrawlBudget                int      `gorm:"default:0"`     // Max URLs to crawl per session (0 = unlimited)
	Project                    *Project `gorm:"foreignKey:ProjectID;constraint:OnDelete:CASCADE"`
	CreatedAt                  int64    `gorm:"autoCreateTime"`
	UpdatedAt                  int64    `gorm:"autoUpdateTime"`
}

type Crawl struct {
	ID             uint                 `gorm:"primaryKey"`
	ProjectID      uint                 `gorm:"not null;index"`
	RunID          *uint                `gorm:"index"` // nullable FK to IncrementalCrawlRun (null for non-incremental crawls)
	CrawlDateTime  int64                `gorm:"not null"`
	CrawlDuration  int64                `gorm:"not null"`
	PagesCrawled   int                  `gorm:"not null"`
	State          string               `gorm:"not null;default:'completed'"` // in_progress, completed, failed
	DiscoveredUrls []DiscoveredUrl      `gorm:"foreignKey:CrawlID;constraint:OnDelete:CASCADE"`
	Run            *IncrementalCrawlRun `gorm:"foreignKey:RunID"`
	CreatedAt      int64                `gorm:"autoCreateTime"`
	UpdatedAt      int64                `gorm:"autoUpdateTime"`
}

type CrawlHistoryEntry struct {
	ID            uint // The crawl ID to use for fetching results (latest crawl in run, or the crawl itself)
	ProjectID     uint
	CrawlDateTime int64  // When the crawl/run started
	CrawlDuration int64  // Total duration across all sessions
	PagesCrawled  int    // Total pages across all sessions
	State         string // Current state (from run if incremental, from crawl if standalone)
}

type CrawlQueueItem struct {
	ID        uint     `gorm:"primaryKey"`
	ProjectID uint     `gorm:"not null;index:idx_queue_project_visited"`
	URL       string   `gorm:"not null"`
	URLHash   int64    `gorm:"not null;index:idx_queue_hash"` // Stored as int64 for SQLite compatibility
	Source    string   `gorm:"not null"`                      // initial, spider, sitemap, network, resource
	Depth     int      `gorm:"not null;default:0"`
	Visited   bool     `gorm:"not null;default:false;index:idx_queue_project_visited"`
	Project   *Project `gorm:"foreignKey:ProjectID;constraint:OnDelete:CASCADE"`
	CreatedAt int64    `gorm:"autoCreateTime"`
	UpdatedAt int64    `gorm:"autoUpdateTime"`
}

type DiscoveredUrl struct {
	ID              uint   `gorm:"primaryKey"`
	CrawlID         uint   `gorm:"not null;index"`
	URL             string `gorm:"not null"`
	Visited         bool   `gorm:"not null;default:false;index"` // true = URL was crawled/visited, false = discovered but not visited
	Status          int    `gorm:"not null"`
	Title           string `gorm:"type:text"`
	MetaDescription string `gorm:"type:text"`
	H1              string `gorm:"type:text"`
	H2              string `gorm:"type:text"`
	CanonicalURL    string `gorm:"type:text"`
	WordCount       int    `gorm:"default:0"`
	ContentHash     string `gorm:"type:text;index"`
	Indexable       string `gorm:"not null"`
	ContentType     string `gorm:"type:text"` // MIME type: text/html, image/jpeg, text/css, application/javascript, etc.
	Error           string `gorm:"type:text"`
	CreatedAt       int64  `gorm:"autoCreateTime"`
}

type DomainFramework struct {
	ID          uint     `gorm:"primaryKey"`
	ProjectID   uint     `gorm:"not null;index:idx_project_domain"`
	Domain      string   `gorm:"not null;index:idx_project_domain;index:idx_domain"`
	Framework   string   `gorm:"not null"`
	ManuallySet bool     `gorm:"default:false"`
	Project     *Project `gorm:"foreignKey:ProjectID;constraint:OnDelete:CASCADE"`
	CreatedAt   int64    `gorm:"autoCreateTime"`
	UpdatedAt   int64    `gorm:"autoUpdateTime"`
}

type IncrementalCrawlRun struct {
	ID        uint     `gorm:"primaryKey"`
	ProjectID uint     `gorm:"not null;index"`
	State     string   `gorm:"not null;default:'in_progress'"` // in_progress, paused, completed
	Project   *Project `gorm:"foreignKey:ProjectID;constraint:OnDelete:CASCADE"`
	Crawls    []Crawl  `gorm:"foreignKey:RunID"`
	CreatedAt int64    `gorm:"autoCreateTime"`
	UpdatedAt int64    `gorm:"autoUpdateTime"`
}

type PageLink struct {
	ID          uint   `gorm:"primaryKey"`
	CrawlID     uint   `gorm:"not null;index:idx_crawl_source;index:idx_crawl_target"`
	SourceURL   string `gorm:"not null;index:idx_crawl_source"` // Page containing the link
	TargetURL   string `gorm:"not null;index:idx_crawl_target"` // Page being linked to
	LinkType    string `gorm:"not null"`                        // "anchor", "image", "script", etc.
	LinkText    string `gorm:"type:text"`                       // anchor text or alt text
	LinkContext string `gorm:"type:text"`                       // surrounding text context
	IsInternal  bool   `gorm:"not null"`                        // internal vs external
	Status      int    `gorm:"default:0"`                       // HTTP status of target (0 if not crawled)
	Title       string `gorm:"type:text"`                       // Title of target page (if crawled)
	ContentType string `gorm:"type:text"`                       // Content type of target (if crawled)
	Position    string `gorm:"type:text"`                       // Position: "content", "navigation", "header", "footer", "sidebar", "breadcrumbs", "pagination", "unknown"
	DOMPath     string `gorm:"type:text"`                       // Simplified DOM path showing link's location in HTML structure
	URLAction   string `gorm:"type:text;index"`                 // Action: "crawl" (normal), "record" (framework-specific), "skip" (ignored)
	CreatedAt   int64  `gorm:"autoCreateTime"`
}

type PageLinkData struct {
	URL         string
	Type        string
	Text        string
	Context     string
	IsInternal  bool
	Status      int
	Title       string
	ContentType string
	Position    string
	DOMPath     string
	URLAction   string
}

type Project struct {
	ID             uint    `gorm:"primaryKey"`
	URL            string  `gorm:"not null"`             // Normalized URL for the project
	Domain         string  `gorm:"uniqueIndex;not null"` // Domain identifier (includes subdomain)
	FaviconPath    string  `gorm:"type:text"`            // Path to cached favicon
	AICrawlerData  string  `gorm:"type:text"`            // JSON data for AI Crawler results
	SSRScreenshot  string  `gorm:"type:text"`            // Path to SSR screenshot
	JSScreenshot   string  `gorm:"type:text"`            // Path to JS-enabled screenshot
	NoJSScreenshot string  `gorm:"type:text"`            // Path to JS-disabled screenshot
	Crawls         []Crawl `gorm:"foreignKey:ProjectID;constraint:OnDelete:CASCADE"`
	CreatedAt      int64   `gorm:"autoCreateTime"`
	UpdatedAt      int64   `gorm:"autoUpdateTime"`
}

type QueueStats struct {
	Visited int64
	Pending int64
	Total   int64
}

type Store struct {
	// contains filtered or unexported fields
}