Documentation
¶
Overview ¶
Package output provides output formatting for the crawler.
Index ¶
- type AuthCredentials
- type Config
- type CrawlError
- type CrawlMetadata
- type CrawlResult
- type CrawlStats
- type DetailedReport
- type Endpoint
- type EndpointSummary
- type ExposedAPI
- type Form
- type FormInput
- type FormSummary
- type JSONWriter
- func (j *JSONWriter) Close() error
- func (j *JSONWriter) Flush() error
- func (j *JSONWriter) WriteEndpoint(endpoint *Endpoint) error
- func (j *JSONWriter) WriteError(err *CrawlError) error
- func (j *JSONWriter) WriteForm(form *Form) error
- func (j *JSONWriter) WriteResult(result *CrawlResult) error
- func (j *JSONWriter) WriteWebSocket(ws *WebSocketEndpoint) error
- type Parameter
- type PathCount
- type ProgressStats
- type ProgressWriter
- type SecurityFindings
- type Statistics
- type StreamEvent
- type SummaryReport
- type TimingInfo
- type WebSocketEndpoint
- type WebSocketMsg
- type Writer
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type AuthCredentials ¶
type AuthCredentials struct {
Type string `json:"type"`
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Token string `json:"token,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
Cookies []*http.Cookie `json:"-"`
LoginURL string `json:"login_url,omitempty"`
FormFields map[string]string `json:"form_fields,omitempty"`
}
AuthCredentials holds authentication credentials (used for state).
type CrawlError ¶
type CrawlError struct {
URL string `json:"url"`
Error string `json:"error"`
Timestamp time.Time `json:"timestamp"`
}
CrawlError represents an error encountered during crawling.
type CrawlMetadata ¶
type CrawlMetadata struct {
CrawlerVersion string `json:"crawler_version"`
UserAgent string `json:"user_agent"`
Config map[string]interface{} `json:"config"`
Environment map[string]string `json:"environment,omitempty"`
}
CrawlMetadata contains metadata about the crawl.
type CrawlResult ¶
type CrawlResult struct {
Target string `json:"target"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at,omitempty"`
Stats CrawlStats `json:"stats"`
Endpoints []Endpoint `json:"endpoints"`
Forms []Form `json:"forms"`
WebSockets []WebSocketEndpoint `json:"websockets"`
Errors []CrawlError `json:"errors,omitempty"`
}
CrawlResult represents the complete result of a crawl session.
type CrawlStats ¶
type CrawlStats struct {
URLsDiscovered int `json:"urls_discovered"`
PagesCrawled int `json:"pages_crawled"`
FormsFound int `json:"forms_found"`
APIEndpoints int `json:"api_endpoints"`
WebSocketEndpoints int `json:"websocket_endpoints"`
ErrorCount int `json:"error_count"`
Duration time.Duration `json:"duration"`
BytesTransferred int64 `json:"bytes_transferred"`
}
CrawlStats contains statistics about the crawl.
type DetailedReport ¶
type DetailedReport struct {
Summary SummaryReport `json:"summary"`
Endpoints EndpointSummary `json:"endpoints"`
Forms FormSummary `json:"forms"`
Security SecurityFindings `json:"security"`
Metadata CrawlMetadata `json:"metadata"`
Timing TimingInfo `json:"timing"`
}
DetailedReport contains a detailed crawl report.
type Endpoint ¶
type Endpoint struct {
URL string `json:"url"`
Method string `json:"method"`
Source string `json:"source"`
Depth int `json:"depth"`
Parameters []Parameter `json:"parameters,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
DiscoveredFrom string `json:"discovered_from,omitempty"`
StatusCode int `json:"status_code,omitempty"`
ContentType string `json:"content_type,omitempty"`
ResponseSize int64 `json:"response_size,omitempty"`
Timestamp time.Time `json:"timestamp"`
}
Endpoint represents a discovered endpoint during crawling.
type EndpointSummary ¶
type EndpointSummary struct {
Total int `json:"total"`
ByMethod map[string]int `json:"by_method"`
BySource map[string]int `json:"by_source"`
ByStatus map[int]int `json:"by_status"`
TopPaths []PathCount `json:"top_paths"`
}
EndpointSummary contains a summary of discovered endpoints.
type ExposedAPI ¶
type ExposedAPI struct {
URL string `json:"url"`
Method string `json:"method"`
Reason string `json:"reason"`
}
ExposedAPI represents a potentially exposed API endpoint.
type Form ¶
type Form struct {
URL string `json:"url"`
Action string `json:"action"`
Method string `json:"method"`
Enctype string `json:"enctype"`
Inputs []FormInput `json:"inputs"`
HasCSRF bool `json:"has_csrf"`
Depth int `json:"depth"`
Timestamp time.Time `json:"timestamp"`
}
Form represents an HTML form discovered during crawling.
type FormInput ¶
type FormInput struct {
Name string `json:"name"`
Type string `json:"type"`
Value string `json:"value,omitempty"`
Required bool `json:"required"`
Placeholder string `json:"placeholder,omitempty"`
Pattern string `json:"pattern,omitempty"`
MaxLength int `json:"max_length,omitempty"`
MinLength int `json:"min_length,omitempty"`
}
FormInput represents an input field in a form.
type FormSummary ¶
type FormSummary struct {
Total int `json:"total"`
ByMethod map[string]int `json:"by_method"`
ByType map[string]int `json:"by_type"`
WithCSRF int `json:"with_csrf"`
WithCaptcha int `json:"with_captcha"`
FileUpload int `json:"file_upload"`
}
FormSummary contains a summary of discovered forms.
type JSONWriter ¶
type JSONWriter struct {
// contains filtered or unexported fields
}
JSONWriter writes output in JSON format.
func NewJSONWriter ¶
func NewJSONWriter(w io.Writer, pretty, stream bool) *JSONWriter
NewJSONWriter creates a new JSON writer.
func (*JSONWriter) WriteEndpoint ¶
func (j *JSONWriter) WriteEndpoint(endpoint *Endpoint) error
WriteEndpoint writes a single endpoint in streaming mode.
func (*JSONWriter) WriteError ¶
func (j *JSONWriter) WriteError(err *CrawlError) error
WriteError writes an error in streaming mode.
func (*JSONWriter) WriteForm ¶
func (j *JSONWriter) WriteForm(form *Form) error
WriteForm writes a single form in streaming mode.
func (*JSONWriter) WriteResult ¶
func (j *JSONWriter) WriteResult(result *CrawlResult) error
WriteResult writes the complete crawl result.
func (*JSONWriter) WriteWebSocket ¶
func (j *JSONWriter) WriteWebSocket(ws *WebSocketEndpoint) error
WriteWebSocket writes a single WebSocket endpoint in streaming mode.
type Parameter ¶
type Parameter struct {
Name string `json:"name"`
Type string `json:"type"`
Example string `json:"example,omitempty"`
Required bool `json:"required,omitempty"`
}
Parameter represents a request parameter.
type ProgressStats ¶
type ProgressStats struct {
URLsDiscovered int
PagesCrawled int
FormsFound int
APIEndpoints int
WebSockets int
Errors int
}
ProgressStats contains progress statistics.
type ProgressWriter ¶
type ProgressWriter struct {
Writer
// contains filtered or unexported fields
}
ProgressWriter wraps a writer and provides progress updates.
func NewProgressWriter ¶
func NewProgressWriter(w Writer, onProgress func(ProgressStats)) *ProgressWriter
NewProgressWriter creates a writer that reports progress.
func (*ProgressWriter) WriteEndpoint ¶
func (p *ProgressWriter) WriteEndpoint(endpoint *Endpoint) error
WriteEndpoint writes an endpoint and updates progress.
func (*ProgressWriter) WriteError ¶
func (p *ProgressWriter) WriteError(err *CrawlError) error
WriteError writes an error and updates progress.
func (*ProgressWriter) WriteForm ¶
func (p *ProgressWriter) WriteForm(form *Form) error
WriteForm writes a form and updates progress.
func (*ProgressWriter) WriteWebSocket ¶
func (p *ProgressWriter) WriteWebSocket(ws *WebSocketEndpoint) error
WriteWebSocket writes a WebSocket and updates progress.
type SecurityFindings ¶
type SecurityFindings struct {
ExposedAPIs []ExposedAPI `json:"exposed_apis,omitempty"`
MissingCSRF []string `json:"missing_csrf,omitempty"`
FileUploads []string `json:"file_uploads,omitempty"`
PotentialSecrets []string `json:"potential_secrets,omitempty"`
DebugEndpoints []string `json:"debug_endpoints,omitempty"`
}
SecurityFindings contains security-relevant findings.
type Statistics ¶
type Statistics struct {
TotalURLs int `json:"total_urls"`
CrawledPages int `json:"crawled_pages"`
DiscoveredForms int `json:"discovered_forms"`
DiscoveredAPIs int `json:"discovered_apis"`
DiscoveredWS int `json:"discovered_websockets"`
Errors int `json:"errors"`
SkippedOutOfScope int `json:"skipped_out_of_scope"`
SkippedDuplicate int `json:"skipped_duplicate"`
}
Statistics contains crawl statistics.
type StreamEvent ¶
type StreamEvent struct {
Type string `json:"type"`
Data interface{} `json:"data"`
}
StreamEvent represents a streaming output event.
type SummaryReport ¶
type SummaryReport struct {
Target string `json:"target"`
StartedAt time.Time `json:"started_at"`
CompletedAt time.Time `json:"completed_at"`
Duration time.Duration `json:"duration"`
Statistics Statistics `json:"statistics"`
}
SummaryReport contains a summary of the crawl.
type TimingInfo ¶
type TimingInfo struct {
TotalDuration time.Duration `json:"total_duration"`
AveragePageTime time.Duration `json:"average_page_time"`
FastestPage time.Duration `json:"fastest_page"`
SlowestPage time.Duration `json:"slowest_page"`
RequestsPerSec float64 `json:"requests_per_second"`
}
TimingInfo contains timing information.
type WebSocketEndpoint ¶
type WebSocketEndpoint struct {
URL string `json:"url"`
DiscoveredFrom string `json:"discovered_from"`
SampleMessages []WebSocketMsg `json:"sample_messages,omitempty"`
Protocols []string `json:"protocols,omitempty"`
Timestamp time.Time `json:"timestamp"`
}
WebSocketEndpoint represents a discovered WebSocket endpoint.
type WebSocketMsg ¶
type WebSocketMsg struct {
Direction string `json:"direction"`
Type string `json:"type"`
Data string `json:"data"`
Timestamp time.Time `json:"timestamp"`
}
WebSocketMsg represents a WebSocket message.
type Writer ¶
type Writer interface {
// WriteResult writes the complete crawl result
WriteResult(result *CrawlResult) error
// WriteEndpoint writes a single endpoint (for streaming)
WriteEndpoint(endpoint *Endpoint) error
// WriteForm writes a single form (for streaming)
WriteForm(form *Form) error
// WriteWebSocket writes a single WebSocket endpoint (for streaming)
WriteWebSocket(ws *WebSocketEndpoint) error
// WriteError writes an error (for streaming)
WriteError(err *CrawlError) error
// Flush flushes any buffered output
Flush() error
// Close closes the writer
Close() error
}
Writer defines the interface for output writers.