Documentation
¶
Overview ¶
Package crawlgraph implements a directed graph of URLs discovered during crawling, with support for SQLite persistence, multiple export formats, and replay strategies for re-crawling changed or failed pages.
Index ¶
- func ContentHash(content []byte) string
- func GetReplayURLs(graph *CrawlGraph, cfg ReplayConfig, logger *slog.Logger) ([]string, error)
- type CrawlGraph
- func (g *CrawlGraph) AddEdge(edge Edge) error
- func (g *CrawlGraph) AddNode(node Node) error
- func (g *CrawlGraph) Close() error
- func (g *CrawlGraph) ExportCSV() (string, error)
- func (g *CrawlGraph) ExportDOT() (string, error)
- func (g *CrawlGraph) ExportJSON() ([]byte, error)
- func (g *CrawlGraph) ExportMermaid() (string, error)
- func (g *CrawlGraph) GetEdges() ([]Edge, error)
- func (g *CrawlGraph) GetFailedNodes() ([]Node, error)
- func (g *CrawlGraph) GetNodes() ([]Node, error)
- func (g *CrawlGraph) GetNodesByDepth(depth int) ([]Node, error)
- func (g *CrawlGraph) SessionID() string
- func (g *CrawlGraph) StartSession(seedURL string) error
- func (g *CrawlGraph) Stats() (GraphStats, error)
- type Edge
- type GraphStats
- type Node
- type ReplayConfig
- type ReplayStrategy
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func ContentHash ¶
ContentHash computes a SHA256 hash of content for change detection.
func GetReplayURLs ¶
func GetReplayURLs(graph *CrawlGraph, cfg ReplayConfig, logger *slog.Logger) ([]string, error)
GetReplayURLs returns the list of URLs to re-crawl based on the strategy.
Types ¶
type CrawlGraph ¶
type CrawlGraph struct {
// contains filtered or unexported fields
}
CrawlGraph is a thread-safe directed graph with SQLite persistence.
func New ¶
func New(dbPath string, logger *slog.Logger) (*CrawlGraph, error)
New creates or opens a crawl graph database.
func (*CrawlGraph) AddEdge ¶
func (g *CrawlGraph) AddEdge(edge Edge) error
AddEdge adds a link between two URLs.
func (*CrawlGraph) AddNode ¶
func (g *CrawlGraph) AddNode(node Node) error
AddNode adds or updates a node in the graph.
func (*CrawlGraph) ExportCSV ¶
func (g *CrawlGraph) ExportCSV() (string, error)
ExportCSV exports nodes as CSV.
func (*CrawlGraph) ExportDOT ¶
func (g *CrawlGraph) ExportDOT() (string, error)
ExportDOT exports the graph in GraphViz DOT format.
func (*CrawlGraph) ExportJSON ¶
func (g *CrawlGraph) ExportJSON() ([]byte, error)
ExportJSON exports the graph as JSON.
func (*CrawlGraph) ExportMermaid ¶
func (g *CrawlGraph) ExportMermaid() (string, error)
ExportMermaid exports the graph in Mermaid format.
func (*CrawlGraph) GetEdges ¶
func (g *CrawlGraph) GetEdges() ([]Edge, error)
GetEdges retrieves all edges for the current session.
func (*CrawlGraph) GetFailedNodes ¶
func (g *CrawlGraph) GetFailedNodes() ([]Node, error)
GetFailedNodes retrieves nodes with errors or non-2xx status.
func (*CrawlGraph) GetNodes ¶
func (g *CrawlGraph) GetNodes() ([]Node, error)
GetNodes retrieves all nodes for the current session.
func (*CrawlGraph) GetNodesByDepth ¶
func (g *CrawlGraph) GetNodesByDepth(depth int) ([]Node, error)
GetNodesByDepth retrieves nodes at a specific depth.
func (*CrawlGraph) SessionID ¶
func (g *CrawlGraph) SessionID() string
SessionID returns the current session ID.
func (*CrawlGraph) StartSession ¶
func (g *CrawlGraph) StartSession(seedURL string) error
StartSession records a new crawl session.
func (*CrawlGraph) Stats ¶
func (g *CrawlGraph) Stats() (GraphStats, error)
Stats returns summary statistics for the current session.
type Edge ¶
type Edge struct {
FromURL string `json:"from_url"`
ToURL string `json:"to_url"`
LinkText string `json:"link_text"`
}
Edge represents a link between two nodes.
type GraphStats ¶
type GraphStats struct {
NodeCount int `json:"node_count"`
EdgeCount int `json:"edge_count"`
SuccessCount int `json:"success_count"`
ErrorCount int `json:"error_count"`
MaxDepth int `json:"max_depth"`
TotalTimeMs int64 `json:"total_time_ms"`
TotalItems int `json:"total_items"`
}
GraphStats holds summary statistics for a crawl graph.
type Node ¶
type Node struct {
ID string `json:"id"`
URL string `json:"url"`
StatusCode int `json:"status_code"`
ContentType string `json:"content_type"`
Depth int `json:"depth"`
CrawlTimeMs int64 `json:"crawl_time_ms"`
ContentHash string `json:"content_hash"`
ItemCount int `json:"item_count"`
Error string `json:"error,omitempty"`
ParentURL string `json:"parent_url,omitempty"`
DiscoveredAt time.Time `json:"discovered_at"`
}
Node represents a single URL in the crawl graph.
type ReplayConfig ¶
type ReplayConfig struct {
Strategy ReplayStrategy
ReferenceDB string // Path to reference crawl graph DB.
MaxDepth int // For depth strategy.
URLPattern string // For pattern strategy.
}
ReplayConfig configures a replay run.
type ReplayStrategy ¶
type ReplayStrategy string
ReplayStrategy determines which URLs to re-crawl.
const ( // ReplayErrors re-crawls nodes that had errors or non-2xx status codes. ReplayErrors ReplayStrategy = "errors" // ReplayChanged re-crawls nodes whose content hash differs from a reference session. ReplayChanged ReplayStrategy = "changed" // ReplayAll re-crawls all nodes from a previous session. ReplayAll ReplayStrategy = "all" // ReplayDepth re-crawls nodes at or below a specific depth. ReplayDepth ReplayStrategy = "depth" )
Source Files