crawler

type AuthCredentials struct {
	Type        AuthType          `json:"type"`
	Username    string            `json:"username,omitempty"`
	Password    string            `json:"password,omitempty"`
	Token       string            `json:"token,omitempty"`
	Headers     map[string]string `json:"headers,omitempty"`
	Cookies     []*http.Cookie    `json:"-"`
	LoginURL    string            `json:"login_url,omitempty"`
	FormFields  map[string]string `json:"form_fields,omitempty"`
	OAuthConfig *OAuthConfig      `json:"oauth_config,omitempty"`
}

type AuthType string

type Config struct {
	// Target URL to crawl
	Target string `json:"target" yaml:"target"`

	// Number of concurrent workers
	Workers int `json:"workers" yaml:"workers"`

	// Maximum crawl depth
	MaxDepth int `json:"max_depth" yaml:"max_depth"`

	// Request timeout
	Timeout time.Duration `json:"timeout" yaml:"timeout"`

	// Scope rules
	Scope ScopeRules `json:"scope" yaml:"scope"`

	// Rate limiting
	RateLimit RateLimitConfig `json:"rate_limit" yaml:"rate_limit"`

	// Browser configuration
	Browser browser.Config `json:"browser" yaml:"browser"`

	// Authentication
	Auth AuthCredentials `json:"auth" yaml:"auth"`

	// Output configuration
	Output OutputConfig `json:"output" yaml:"output"`

	// State persistence
	State StateConfig `json:"state" yaml:"state"`

	// Enable passive API discovery
	PassiveAPIDiscovery bool `json:"passive_api_discovery" yaml:"passive_api_discovery"`

	// Enable active API probing
	ActiveAPIDiscovery bool `json:"active_api_discovery" yaml:"active_api_discovery"`

	// Enable WebSocket discovery
	WebSocketDiscovery bool `json:"websocket_discovery" yaml:"websocket_discovery"`

	// Enable form analysis
	FormAnalysis bool `json:"form_analysis" yaml:"form_analysis"`

	// Enable JavaScript analysis
	JSAnalysis bool `json:"js_analysis" yaml:"js_analysis"`

	// Enable AJAX discovery (intercept XHR/Fetch, trigger events, find AJAX forms)
	AJAXDiscovery bool `json:"ajax_discovery" yaml:"ajax_discovery"`

	// Fast mode skips heavy analysis (SPA framework detection, AJAX triggering) for speed
	FastMode bool `json:"fast_mode" yaml:"fast_mode"`

	// Custom headers to include in all requests
	CustomHeaders map[string]string `json:"custom_headers" yaml:"custom_headers"`

	// Cookies to include in all requests
	Cookies map[string]string `json:"cookies" yaml:"cookies"`

	// User agents to rotate (if empty, uses default)
	UserAgents []string `json:"user_agents" yaml:"user_agents"`

	// Proxy URL
	Proxy string `json:"proxy" yaml:"proxy"`

	// Verbose logging
	Verbose bool `json:"verbose" yaml:"verbose"`

	// Debug mode
	Debug bool `json:"debug" yaml:"debug"`

	// Enhanced discovery configuration
	EnhancedDiscovery EnhancedDiscoveryConfig `json:"enhanced_discovery" yaml:"enhanced_discovery"`
}

type CrawlError struct {
	URL       string    `json:"url"`
	Error     string    `json:"error"`
	Timestamp time.Time `json:"timestamp"`
}

type CrawlResult struct {
	Target       string              `json:"target"`
	StartedAt    time.Time           `json:"started_at"`
	CompletedAt  time.Time           `json:"completed_at,omitempty"`
	Stats        CrawlStats          `json:"stats"`
	Endpoints    []Endpoint          `json:"endpoints"`
	Forms        []Form              `json:"forms"`
	WebSockets   []WebSocketEndpoint `json:"websockets"`
	Technologies []Technology        `json:"technologies,omitempty"`
	Secrets      []SecretFinding     `json:"secrets,omitempty"`
	Errors       []CrawlError        `json:"errors,omitempty"`
}

type CrawlStats struct {
	URLsDiscovered     int           `json:"urls_discovered"`
	PagesCrawled       int           `json:"pages_crawled"`
	FormsFound         int           `json:"forms_found"`
	APIEndpoints       int           `json:"api_endpoints"`
	WebSocketEndpoints int           `json:"websocket_endpoints"`
	ErrorCount         int           `json:"error_count"`
	Duration           time.Duration `json:"duration"`
	BytesTransferred   int64         `json:"bytes_transferred"`
}

type Crawler struct {
	// contains filtered or unexported fields
}

type Endpoint struct {
	URL            string            `json:"url"`
	Method         string            `json:"method"`
	Source         string            `json:"source"` // passive, active, html, javascript
	Depth          int               `json:"depth"`
	Parameters     []Parameter       `json:"parameters,omitempty"`
	Headers        map[string]string `json:"headers,omitempty"`
	DiscoveredFrom string            `json:"discovered_from,omitempty"`
	StatusCode     int               `json:"status_code,omitempty"`
	ContentType    string            `json:"content_type,omitempty"`
	ResponseSize   int64             `json:"response_size,omitempty"`
	Timestamp      time.Time         `json:"timestamp"`
}

type EnhancedDiscoveryConfig struct {
	// Enable all enhanced discovery modules
	Enabled bool `json:"enabled" yaml:"enabled"`

	// Enable robots.txt parsing
	EnableRobots bool `json:"enable_robots" yaml:"enable_robots"`

	// Enable sitemap.xml discovery
	EnableSitemap bool `json:"enable_sitemap" yaml:"enable_sitemap"`

	// Enable JavaScript source map parsing
	EnableSourceMaps bool `json:"enable_source_maps" yaml:"enable_source_maps"`

	// Enable common path bruteforcing
	EnablePathBrute bool `json:"enable_path_brute" yaml:"enable_path_brute"`

	// Enable technology fingerprinting
	EnableFingerprint bool `json:"enable_fingerprint" yaml:"enable_fingerprint"`

	// Enable parameter discovery
	EnableParamDiscovery bool `json:"enable_param_discovery" yaml:"enable_param_discovery"`

	// Enable JavaScript extraction
	EnableJSExtract bool `json:"enable_js_extract" yaml:"enable_js_extract"`

	// Concurrency for enhanced discovery operations
	Concurrency int `json:"concurrency" yaml:"concurrency"`
}

type Form struct {
	URL       string      `json:"url"`
	Action    string      `json:"action"`
	Method    string      `json:"method"`
	Enctype   string      `json:"enctype"`
	Inputs    []FormInput `json:"inputs"`
	HasCSRF   bool        `json:"has_csrf"`
	Depth     int         `json:"depth"`
	Timestamp time.Time   `json:"timestamp"`
}

type FormAuth struct {
	LoginURL      string
	Username      string
	Password      string
	UsernameField string
	PasswordField string
	ExtraFields   map[string]string
}

type FormInput struct {
	Name        string `json:"name"`
	Type        string `json:"type"`
	Value       string `json:"value,omitempty"`
	Required    bool   `json:"required"`
	Placeholder string `json:"placeholder,omitempty"`
	Pattern     string `json:"pattern,omitempty"`
	MaxLength   int    `json:"max_length,omitempty"`
	MinLength   int    `json:"min_length,omitempty"`
}

type OAuthConfig struct {
	ClientID     string   `json:"client_id"`
	ClientSecret string   `json:"client_secret"`
	AuthURL      string   `json:"auth_url"`
	TokenURL     string   `json:"token_url"`
	RedirectURL  string   `json:"redirect_url"`
	Scopes       []string `json:"scopes"`
}

type Option func(*Crawler) error

type OutputConfig struct {
	Format     string `json:"format"` // json
	FilePath   string `json:"file_path"`
	Pretty     bool   `json:"pretty"`
	StreamMode bool   `json:"stream_mode"`
}

type Parameter struct {
	Name     string `json:"name"`
	Type     string `json:"type"` // query, body, header, path, cookie
	Example  string `json:"example,omitempty"`
	Required bool   `json:"required,omitempty"`
}

type QueueItem struct {
	URL       string            `json:"url"`
	Method    string            `json:"method"`
	Depth     int               `json:"depth"`
	ParentURL string            `json:"parent_url"`
	Headers   map[string]string `json:"headers,omitempty"`
	Body      []byte            `json:"body,omitempty"`
	Priority  int               `json:"priority"`
	Timestamp time.Time         `json:"timestamp"`
}

type RateLimitConfig struct {
	RequestsPerSecond float64       `json:"requests_per_second"`
	Burst             int           `json:"burst"`
	DelayBetween      time.Duration `json:"delay_between"`
	RespectRobotsTxt  bool          `json:"respect_robots_txt"`
}

type ScopeRules struct {
	IncludePatterns []string `json:"include_patterns"`
	ExcludePatterns []string `json:"exclude_patterns"`
	AllowedDomains  []string `json:"allowed_domains"`
	MaxDepth        int      `json:"max_depth"`
	FollowExternal  bool     `json:"follow_external"`
}

type SecretFinding struct {
	Type    string `json:"type"`
	Value   string `json:"value"`
	File    string `json:"file,omitempty"`
	Context string `json:"context,omitempty"`
}

type StateConfig struct {
	Enabled  bool   `json:"enabled"`
	FilePath string `json:"file_path"`
	AutoSave bool   `json:"auto_save"`
	Interval int    `json:"interval_seconds"`
}

type Technology struct {
	Name       string `json:"name"`
	Category   string `json:"category"`
	Version    string `json:"version,omitempty"`
	Confidence int    `json:"confidence"`
	Evidence   string `json:"evidence,omitempty"`
}

type WebSocketEndpoint struct {
	URL            string         `json:"url"`
	DiscoveredFrom string         `json:"discovered_from"`
	SampleMessages []WebSocketMsg `json:"sample_messages,omitempty"`
	Protocols      []string       `json:"protocols,omitempty"`
	Timestamp      time.Time      `json:"timestamp"`
}

type WebSocketMsg struct {
	Direction string    `json:"direction"` // sent, received
	Type      string    `json:"type"`      // text, binary
	Data      string    `json:"data"`
	Timestamp time.Time `json:"timestamp"`
}