source

type ConcurrentMultiSource struct {
	// contains filtered or unexported fields
}

type Countable interface {
	Count() int64
}

type DeparosDiscoveryConfig struct {
	Targets     []string      // Target URLs
	Concurrency int           // Worker threads (from -t flag), split across concurrent targets; default: 25
	MaxDuration time.Duration // default: 1h

	// RateLimit caps each discovery engine's outgoing requests per second.
	// 0 = unpaced (the historical behaviour; Concurrency is then the only
	// throttle). Set it only from an EXPLICIT operator setting — see
	// buildDeparosConfig in the runner — because imposing the global 100 rps
	// default on a crawl changes the speed of every existing scan.
	//
	// On its own the bucket is per engine, i.e. per target, so with several
	// targets in flight the phase as a whole can send up to that many times
	// RateLimit. The runner avoids that by passing Egress, which carries ONE
	// bucket at this rate for every target; RateLimit then only describes it.
	RateLimit int

	// Egress, when set, admits every physical attempt of every per-target engine
	// (deparos EngineConfig.Egress): the scan's host slots and quarantine, plus
	// the discovery token bucket. Because the same wrapper serves all targets,
	// its bucket is phase-wide. When set, the engines' own per-target bucket
	// (RateLimit) is not installed — the gate is expected to carry the rate.
	Egress func(nethttp.RoundTripper) nethttp.RoundTripper

	// TargetConcurrency is how many targets are discovered at once. 0 = default
	// (min(len(Targets), 3)); 1 = one at a time. Targets sharing a host:port are
	// never discovered concurrently, and Concurrency is split across the
	// targets in flight (max(5, Concurrency/active)) so the phase's total
	// request concurrency stays about where it was.
	TargetConcurrency int

	// FlushInterval is how often a running target's records (the referenced
	// class, plus the fuzz class unless FuzzAtEnd) are persisted and emitted.
	// 0 = default (15s); negative = only at the end of the target. A flush
	// also runs once defaultDiscoveryFlushNodes nodes landed since the last one.
	FlushInterval time.Duration

	// FuzzAtEnd keeps the fuzz class (wordlist, spider and observed records)
	// in memory until the target finishes, as before fuzz streaming: exact dedup
	// keeps the shorter of two identical paths and the cluster cap ranks
	// shortest-path-first (discovery.stream_fuzz: false). The default streams it
	// with the referenced class: exact dedup keeps the first record and the
	// cluster cap counts per cluster across flushes, first N wins.
	FuzzAtEnd bool

	EnableModules []string // Module selection for WorkItems

	// DrainTimeout bounds how long Close waits for the producer to finish
	// persisting what it already collected. <=0 uses defaultDiscoveryDrainTimeout.
	DrainTimeout time.Duration

	// Full deparos settings (from YAML config)
	Mode             string // "files_and_dirs" | "files_only" | "dirs_only"
	ScopeMode        string // "any" | "subdomain" | "exact"
	RecursionEnabled bool   // default: true
	RecursionDepth   int    // default: 5
	SaveResponseBody bool   // default: true

	// Wordlists
	ShortFilePath        string
	LongFilePath         string
	ShortDirPath         string
	LongDirPath          string
	FuzzWordlistPaths    []string // streamed in order, deduplicated across files
	UseObservedNames     bool
	UseObservedPaths     bool
	UseObservedFiles     bool
	EnableNumericFuzzing bool

	// Extensions
	TestCustom           bool
	CustomList           []string
	TestObserved         bool
	TestBackupExtensions bool
	BackupExtensions     []string
	TestNoExtension      bool

	// Confirmation-gated extension fuzzing. When ConfirmRequired is set, a
	// server-side extension (php/asp/aspx/jsp/action/cgi/…) is only wordlist-
	// fuzzed once the app is confirmed to serve it as a valid route via the
	// enabled sources below. Candidates/ProbeFilenames override the built-ins.
	ConfirmRequired       bool
	ConfirmViaObserved    bool
	ConfirmViaFingerprint bool
	ConfirmViaProbe       bool
	Candidates            []string
	ProbeFilenames        []string

	// JSBundleSweep enables the SPA-gated JS-bundle name sweep (main.js,
	// admin.js, config.js, …) on monolith apps; hits are fed to jstangle.
	// JSBundleNames overrides the curated name list (empty = built-in).
	JSBundleSweep bool
	JSBundleNames []string

	// Engine
	CaseSensitivity string // "auto_detect" | "sensitive" | "insensitive"
	EngineTimeout   time.Duration
	CustomHeaders   map[string]string
	// BrowserSessions carries WAF/bot-cleared sessions harvested by the spidering
	// browser, keyed by lowercased hostname. buildDeparosConfig injects the
	// session matching each target host (Cookie + optional pinned User-Agent) into
	// that target's request headers so content discovery crawls with the same
	// cleared session — set-if-absent, so a configured header always wins.
	BrowserSessions         map[string]httpmsg.CarriedSession
	EnableCookieJar         bool
	ProxyURL                string // HTTP proxy URL for discovery requests
	MaxConsecutiveErrors    int
	MaxConsecutiveWAFBlocks int
	ObservedMaxItems        int
	JSTangle                *deparosconfig.JSTangleConfig
	// JSTemplatePersist controls persistence of JS-derived request templates
	// that did not become discovery nodes (see deparos_discovery_jstangle.go).
	JSTemplatePersist JSTemplatePersistConfig
	// Sitemap overrides the deparos sitemap source settings; nil keeps the
	// deparos defaults (on, 2000 sampled URLs, 3 per shape, follow indexes).
	Sitemap *deparosconfig.SitemapConfig

	// Per-prefix circuit breaker (zero values = use deparos defaults).
	PrefixBreakerEnabled        *bool   // nil = default (true)
	PrefixBreakerMinSamples     int     // 0 = default
	PrefixBreakerTripRatio      float64 // 0 = default
	PrefixBreakerPrefixSegments int     // 0 = default
	PrefixBreakerLengthBucket   int64   // 0 = default

	// Malformed path probe
	EnableMalformedPathProbe bool

	// DedupClusterCap caps the number of near-identical responses kept per
	// cluster (same host/status/content-type, body size & word count within
	// 0.5%). 0 = use default (defaultDedupClusterCap); negative = disabled;
	// positive = that cap. Resolved via resolveClusterCap.
	DedupClusterCap int

	// RequestFilter is the egress policy for every request the discovery engine
	// makes: returning false refuses the request before it is sent, instead of
	// discarding its response afterwards. The engine also derives its spider-queue
	// exclusion from it, so a refused URL is never even queued.
	//
	// It carries the operator's EXPLICIT denials only (scope.host.exclude /
	// scope.path.exclude); how wide the crawl goes is still ScopeMode's job.
	// nil = no denials.
	RequestFilter func(*url.URL) bool

	// DB import: if set, results are saved to vigolium's http_records table
	Repository  RecordSaver
	ProjectUUID string
}

type DeparosDiscoverySource struct {
	// contains filtered or unexported fields
}

type Detection struct {
	// Format is the canonical registry name of the winning format, "urls" when
	// only the URL-list fallback matched, or "" when nothing did.
	Format string
	Score  int
	Reason string
	// Candidates are every format that scored above zero, best first, so an
	// error can name the closest alternative when the winner fails to parse.
	Candidates []string
}

type DiscoveryStats struct {
	TotalDiscovered    int
	HardDedupRemoved   int
	FuzzyCappedRemoved int // records dropped by the near-identical cluster cap
	ClusterCap         int // effective per-cluster cap used (0 = disabled)
	Imported           int
	AllCodes           [5]int // index 0=1xx, 1=2xx, 2=3xx, 3=4xx, 4=5xx
	DedupedCodes       [5]int // status codes of hard-dedup removed records
	CappedCodes        [5]int // status codes of cluster-capped removed records

	// TargetsFailed is the number of targets whose discovery returned an error.
	TargetsFailed int
	// TargetsTimedOut counts targets that exhausted their own MaxDuration. This
	// is designed behavior (per-target time-boxing), not a failure.
	TargetsTimedOut int
	// TargetsSkipped is the number of targets never attempted because the phase
	// was cancelled first.
	TargetsSkipped int
	// ImportFailed counts records that were collected but could not be persisted.
	ImportFailed int
	// TargetErrors holds up to maxReportedTargetErrors "<target>: <err>" strings.
	TargetErrors []string
	// Abandoned is set when Close gave up waiting for the producer to exit.
	Abandoned bool

	// JSFactsPersisted counts template stubs written (emitted + persist-only).
	JSFactsPersisted int
	// JSFactsNotFound counts persisted stubs whose replay was judged not found.
	JSFactsNotFound int
	// JSFactsWithheld counts templates never sent, by withheld_* outcome.
	JSFactsWithheld map[string]int
	// JSTemplatesCapped counts stubs dropped by max_persisted_js_templates; any
	// value above zero is the js_template_persist_cap limit.
	JSTemplatesCapped int
	// JSFactBundles counts jstangle-facts artifacts stored (one per JS asset).
	JSFactBundles int

	// SitemapCapped counts sitemap locs or documents dropped by a sitemap budget
	// (discovery.sitemap.max_urls and the fixed size/index caps); any value
	// above zero is the sitemap_cap limit.
	SitemapCapped int
}

type ExternalHarvesterInputSource struct {
	// contains filtered or unexported fields
}

type FileSource struct {
	// contains filtered or unexported fields
}

type FileSourceConfig struct {
	FilePath      string
	Format        string // canonical name or alias from formatRegistry ("urls", "nuclei", "openapi", …)
	BufferSize    int    // Channel buffer size (default: 100)
	EnableModules []string
	FormatOptions formats.InputFormatOptions

	// FormatExplicit marks Format as chosen by the user (-I given, a REST mode
	// other than "auto"). Only then does a URL-list Format skip content
	// detection; the "urls" default is otherwise indistinguishable from it.
	FormatExplicit bool
}

type FormatInfo struct {
	Name       string   // canonical -I value
	Aliases    []string // accepted alternatives, in registry order
	TargetList bool     // the file is a list of target URLs (see IsTargetListFormat)
}

type InputSource interface {
	// Next returns the next work item from the source.
	// It blocks until an item is available (for queue-based sources).
	//
	// Returns:
	//   - (*work.WorkItem, nil) - next item
	//   - (nil, io.EOF) - source exhausted, no more items
	//   - (nil, context.Canceled) - context was cancelled
	//   - (nil, error) - other error occurred
	Next(ctx context.Context) (*work.WorkItem, error)

	// Close releases any resources held by the source.
	// After Close, Next will return io.EOF.
	Close() error
}

type JSTemplatePersistConfig struct {
	// SkipUnreplayed stops persisting templates that were never sent
	// (withheld, deduped, unresolved). discovery.jstangle.persist_unreplayed:
	// false. Replayed-but-not-found templates are persisted regardless.
	SkipUnreplayed bool
	// EmitStatuses are the not-found replay statuses emitted to later phases.
	// nil means defaultJSNotFoundEmitStatuses; an empty non-nil slice emits none.
	EmitStatuses []int
	// MaxPerTarget caps the stubs written per target. 0 means
	// defaultMaxPersistedJSTemplates; negative means unlimited.
	MaxPerTarget int
}

type MultiSource struct {
	// contains filtered or unexported fields
}

type RecordSaver interface {
	SaveRecord(ctx context.Context, httpRR *httpmsg.HttpRequestResponse, source string, projectUUID string) (string, error)
	SaveRecordBatch(ctx context.Context, records []*httpmsg.HttpRequestResponse, source string, projectUUID string) ([]string, error)
}

type SingleSource struct {
	// contains filtered or unexported fields
}

type SliceSource struct {
	// contains filtered or unexported fields
}

type Sniff func(head []byte, name string) (score int, reason string)

type SourceConfig struct {
	// Direct targets (from -u flag)
	Targets []string

	// File-based input. FilePath is a single input file (e.g. -i/--input).
	// FilePaths carries additional files (e.g. repeated -T/--target-file); every
	// listed file is read and its items merged into the stream.
	FilePath  string
	FilePaths []string
	Format    string // "urls", "nuclei", "openapi", etc.
	// FormatExplicit: Format was chosen by the user, so a "urls" Format is not
	// content-detected (see FileSourceConfig.FormatExplicit).
	FormatExplicit bool

	// Stdin
	UseStdin bool
	Stdin    io.Reader // defaults to os.Stdin

	// Format options
	SkipFormatValidation  bool
	FormatUseRequiredOnly bool

	// Buffer size for channels
	BufferSize int

	// EnableModules for per-item module filtering (from -m flag)
	EnableModules []string
}

type SpecOptions struct {
	// BaseURL is the -t target: the OpenAPI base URL, the WSDL endpoint host
	// override and the Postman {{baseUrl}} / {{url}} value.
	BaseURL string
	// UseSpecServers makes OpenAPI use the spec's own servers (--spec-url).
	UseSpecServers bool
	// Headers are added to every generated OpenAPI / WSDL request.
	Headers map[string]string
	// Variables fill OpenAPI parameters, WSDL element values and Postman
	// {{name}} variables (--spec-var name=value).
	Variables map[string]string
	// DefaultFallbackValue fills an OpenAPI parameter nothing else does.
	DefaultFallbackValue string
	// FieldTypeDefaults are OpenAPI per-type defaults from the config.
	FieldTypeDefaults map[string][]string
}

type StdinSource struct {
	// contains filtered or unexported fields
}

type TargetSource struct {
	// contains filtered or unexported fields
}

type UnparsedInputError struct {
	Path      string
	Lines     int    // non-blank, non-comment lines examined
	Failed    int    // of those, lines the URL parser rejected
	Candidate string // closest other format, "" when none scored
	Explicit  bool   // the URL-list format was requested explicitly
}