runner

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 3, 2025 License: MIT Imports: 86 Imported by: 0

Documentation

Overview

TODO: move this to internal package

Package runner executes the enumeration process.

Index

Constants

View Source
const (
	DefaultResumeFile      = "resume.cfg"
	DefaultOutputDirectory = "output"
)
View Source
const Version = `v1.7.0`

Version is the current Version of httpx

Variables

View Source
var (
	PDCPApiKey = ""
	TeamIDEnv  = env.GetEnvOrDefault("PDCP_TEAM_ID", "")
)

Functions

func AuthWithPDCP

func AuthWithPDCP()

AuthWithPDCP is used to authenticate with PDCP

func DoHealthCheck

func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string

func GetUpdateCallback

func GetUpdateCallback() func()

GetUpdateCallback returns a callback function that updates httpx

func MustDisableSandbox

func MustDisableSandbox() bool

MustDisableSandbox determines if the current os and user needs sandbox mode disabled

Types

type AsnResponse

type AsnResponse struct {
	AsNumber  string   `json:"as_number" csv:"as_number"`
	AsName    string   `json:"as_name" csv:"as_name"`
	AsCountry string   `json:"as_country" csv:"as_country"`
	AsRange   []string `json:"as_range" csv:"as_range"`
}

func (AsnResponse) String

func (o AsnResponse) String() string

type Browser

type Browser struct {
	// contains filtered or unexported fields
}

func NewBrowser

func NewBrowser(proxy string, useLocal bool, optionalArgs map[string]string) (*Browser, error)

func (*Browser) Close

func (b *Browser) Close()

func (*Browser) ScreenshotWithBody

func (b *Browser) ScreenshotWithBody(url string, timeout time.Duration, idle time.Duration, headers []string, fullPage bool) ([]byte, string, error)

type Concurrency

type Concurrency struct {
	Threads int `json:"threads"`
}

type FilterOperator

type FilterOperator struct {
	// contains filtered or unexported fields
}

func (FilterOperator) Parse

func (f FilterOperator) Parse(flagValue string) (string, time.Duration, error)

Parse the given value into operator and value pair

type OnResultCallback

type OnResultCallback func(Result)

OnResultCallback (hostResult)

type Options

type Options struct {
	CustomHeaders customheader.CustomHeaders
	CustomPorts   customport.CustomPorts

	Output           string
	OutputAll        bool
	StoreResponseDir string
	OmitBody         bool
	// Deprecated: use Proxy
	HTTPProxy string
	// Deprecated: use Proxy
	SocksProxy      string
	Proxy           string
	InputFile       string
	InputTargetHost goflags.StringSlice
	Methods         string
	RequestURI      string
	RequestURIs     string

	OutputMatchStatusCode     string
	OutputMatchContentLength  string
	OutputFilterStatusCode    string
	OutputFilterErrorPage     bool
	FilterOutDuplicates       bool
	OutputFilterContentLength string
	InputRawRequest           string

	RequestBody        string
	OutputFilterString goflags.StringSlice
	OutputMatchString  goflags.StringSlice
	OutputFilterRegex  goflags.StringSlice
	OutputMatchRegex   goflags.StringSlice
	Retries            int
	Threads            int
	Timeout            int
	Delay              time.Duration

	VHost                   bool
	VHostInput              bool
	Smuggling               bool
	ExtractTitle            bool
	StatusCode              bool
	Location                bool
	ContentLength           bool
	FollowRedirects         bool
	RespectHSTS             bool
	StoreResponse           bool
	JSONOutput              bool
	CSVOutput               bool
	CSVOutputEncoding       string
	PdcpAuth                string
	PdcpAuthCredFile        string
	Silent                  bool
	Version                 bool
	Verbose                 bool
	NoColor                 bool
	OutputServerHeader      bool
	OutputWebSocket         bool
	ResponseHeadersInStdout bool
	ResponseInStdout        bool
	Base64ResponseInStdout  bool
	ChainInStdout           bool
	FollowHostRedirects     bool
	MaxRedirects            int
	OutputMethod            bool
	TLSProbe                bool
	CSPProbe                bool
	OutputContentType       bool
	OutputIP                bool
	OutputCName             bool
	ExtractFqdn             bool
	Unsafe                  bool
	Debug                   bool
	DebugRequests           bool
	DebugResponse           bool
	Pipeline                bool
	HTTP2Probe              bool
	OutputCDN               string
	OutputResponseTime      bool
	NoFallback              bool
	NoFallbackScheme        bool
	TechDetect              bool
	TLSGrab                 bool

	ShowStatistics            bool
	StatsInterval             int
	RandomAgent               bool
	StoreChain                bool
	StoreVisionReconClusters  bool
	Deny                      customlist.CustomList
	Allow                     customlist.CustomList
	MaxResponseBodySizeToSave int
	MaxResponseBodySizeToRead int
	ResponseBodyPreviewSize   int
	OutputExtractRegexs       goflags.StringSlice
	OutputExtractPresets      goflags.StringSlice
	RateLimit                 int
	RateLimitMinute           int
	Probe                     bool
	Resume                    bool

	Exclude               goflags.StringSlice
	HostMaxErrors         int
	Stream                bool
	SkipDedupe            bool
	ProbeAllIPS           bool
	Resolvers             goflags.StringSlice
	Favicon               bool
	OutputFilterFavicon   goflags.StringSlice
	OutputMatchFavicon    goflags.StringSlice
	LeaveDefaultPorts     bool
	ZTLS                  bool
	OutputLinesCount      bool
	OutputMatchLinesCount string

	OutputFilterLinesCount string
	Memprofile             string

	OutputWordsCount      bool
	OutputMatchWordsCount string

	OutputFilterWordsCount string

	Hashes                   string
	Jarm                     bool
	Asn                      bool
	OutputMatchCdn           goflags.StringSlice
	OutputFilterCdn          goflags.StringSlice
	SniName                  string
	OutputMatchResponseTime  string
	OutputFilterResponseTime string
	HealthCheck              bool
	ListDSLVariable          bool
	OutputFilterCondition    string
	OutputMatchCondition     string
	StripFilter              string
	//The OnResult callback function is invoked for each result. It is important to check for errors in the result before using Result.Err.
	OnResult             OnResultCallback
	DisableUpdateCheck   bool
	NoDecode             bool
	Screenshot           bool
	UseInstalledChrome   bool
	TlsImpersonate       bool
	DisableStdin         bool
	HttpApiEndpoint      string
	NoScreenshotBytes    bool
	NoHeadlessBody       bool
	NoScreenshotFullPage bool
	ScreenshotTimeout    time.Duration
	ScreenshotIdle       time.Duration
	// HeadlessOptionalArguments specifies optional arguments to pass to Chrome
	HeadlessOptionalArguments goflags.StringSlice
	Protocol                  string
	OutputFilterErrorPagePath string
	DisableStdout             bool
	// AssetUpload
	AssetUpload bool
	// AssetName
	AssetName string
	// AssetID
	AssetID string
	// AssetFileUpload
	AssetFileUpload string
	TeamID          string
	// OnClose adds a callback function that is invoked when httpx is closed
	// to be exact at end of existing closures
	OnClose func()

	Trace bool

	// Optional pre-created objects to reduce allocations
	Wappalyzer     *wappalyzer.Wappalyze
	Networkpolicy  *networkpolicy.NetworkPolicy
	CDNCheckClient *cdncheck.Client
	// contains filtered or unexported fields
}

Options contains configuration options for httpx.

func ParseOptions

func ParseOptions() *Options

ParseOptions parses the command line options for application

func (*Options) ParseHeadlessOptionalArguments

func (options *Options) ParseHeadlessOptionalArguments() map[string]string

redundant with katana

func (*Options) ShouldLoadResume

func (options *Options) ShouldLoadResume() bool

ShouldLoadResume resume file

func (*Options) ShouldSaveResume

func (options *Options) ShouldSaveResume() bool

ShouldSaveResume file

func (*Options) ValidateOptions

func (options *Options) ValidateOptions() error

type Result

type Result struct {
	Timestamp    time.Time              `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"`
	ASN          *AsnResponse           `json:"asn,omitempty" csv:"asn" mapstructure:"asn"`
	Err          error                  `json:"-" csv:"-" mapstructure:"-"`
	CSPData      *httpx.CSPData         `json:"csp,omitempty" csv:"csp" mapstructure:"csp"`
	TLSData      *clients.Response      `json:"tls,omitempty" csv:"tls" mapstructure:"tls"`
	Hashes       map[string]interface{} `json:"hash,omitempty" csv:"hash" mapstructure:"hash"`
	ExtractRegex []string               `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"`
	CDNName      string                 `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"`
	CDNType      string                 `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"`
	SNI          string                 `json:"sni,omitempty" csv:"sni" mapstructure:"sni"`
	Port         string                 `json:"port,omitempty" csv:"port" mapstructure:"port"`
	Raw          string                 `json:"-" csv:"-" mapstructure:"-"`
	URL          string                 `json:"url,omitempty" csv:"url" mapstructure:"url"`
	Input        string                 `json:"input,omitempty" csv:"input" mapstructure:"input"`
	Location     string                 `json:"location,omitempty" csv:"location" mapstructure:"location"`
	Title        string                 `json:"title,omitempty" csv:"title" mapstructure:"title"`

	Scheme             string                        `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"`
	Error              string                        `json:"error,omitempty" csv:"error" mapstructure:"error"`
	WebServer          string                        `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"`
	ResponseBody       string                        `json:"body,omitempty" csv:"-" mapstructure:"body"`
	BodyPreview        string                        `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"`
	ContentType        string                        `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"`
	Method             string                        `json:"method,omitempty" csv:"method" mapstructure:"method"`
	Host               string                        `json:"host,omitempty" csv:"host" mapstructure:"host"`
	Path               string                        `json:"path,omitempty" csv:"path" mapstructure:"path"`
	FavIconMMH3        string                        `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"`
	FavIconMD5         string                        `json:"favicon_md5,omitempty" csv:"favicon_md5" mapstructure:"favicon_md5"`
	FaviconPath        string                        `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"`
	FaviconURL         string                        `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"`
	FinalURL           string                        `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"`
	ResponseHeaders    map[string]interface{}        `json:"header,omitempty" csv:"-" mapstructure:"header"`
	RawHeaders         string                        `json:"raw_header,omitempty" csv:"-" mapstructure:"raw_header"`
	Request            string                        `json:"request,omitempty" csv:"-" mapstructure:"request"`
	ResponseTime       string                        `json:"time,omitempty" csv:"time" mapstructure:"time"`
	JarmHash           string                        `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"`
	ChainStatusCodes   []int                         `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"`
	A                  []string                      `json:"a,omitempty" csv:"a" mapstructure:"a"`
	AAAA               []string                      `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"`
	CNAMEs             []string                      `json:"cname,omitempty" csv:"cname" mapstructure:"cname"`
	Technologies       []string                      `json:"tech,omitempty" csv:"tech" mapstructure:"tech"`
	Extracts           map[string][]string           `json:"extracts,omitempty" csv:"extracts" mapstructure:"extracts"`
	Chain              []httpx.ChainItem             `json:"chain,omitempty" csv:"chain" mapstructure:"chain"`
	Words              int                           `json:"words" csv:"words" mapstructure:"words"`
	Lines              int                           `json:"lines" csv:"lines" mapstructure:"lines"`
	StatusCode         int                           `json:"status_code" csv:"status_code" mapstructure:"status_code"`
	ContentLength      int                           `json:"content_length" csv:"content_length" mapstructure:"content_length"`
	Failed             bool                          `json:"failed" csv:"failed" mapstructure:"failed"`
	VHost              bool                          `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"`
	WebSocket          bool                          `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"`
	CDN                bool                          `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"`
	HTTP2              bool                          `json:"http2,omitempty" csv:"http2" mapstructure:"http2"`
	Pipeline           bool                          `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"`
	HeadlessBody       string                        `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"`
	ScreenshotBytes    []byte                        `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"`
	StoredResponsePath string                        `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"`
	ScreenshotPath     string                        `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"`
	ScreenshotPathRel  string                        `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
	KnowledgeBase      map[string]interface{}        `json:"knowledgebase,omitempty" csv:"knowledgebase" mapstructure:"knowledgebase"`
	Resolvers          []string                      `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"`
	Fqdns              []string                      `json:"body_fqdn,omitempty" mapstructure:"body_fqdn"`
	Domains            []string                      `json:"body_domains,omitempty" mapstructure:"body_domains"`
	TechnologyDetails  map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"`
	RequestRaw         []byte                        `json:"-" csv:"-" mapstructure:"-"`
	Response           *httpx.Response               `json:"-" csv:"-" mapstructure:"-"`
	FaviconData        []byte                        `json:"-" csv:"-" mapstructure:"-"`
	Trace              *retryablehttp.TraceInfo      `json:"trace,omitempty" csv:"trace"  mapstructure:"trace"`
	// contains filtered or unexported fields
}

Result of a scan

func (Result) CSVHeader

func (r Result) CSVHeader() string

CSVHeader the CSV headers

func (Result) CSVRow

func (r Result) CSVRow(scanopts *ScanOptions) string

CSVRow the CSV Row

func (Result) JSON

func (r Result) JSON(scanopts *ScanOptions) string

JSON the result

type ResumeCfg

type ResumeCfg struct {
	ResumeFrom string
	Index      int
	// contains filtered or unexported fields
}

type Runner

type Runner struct {
	HostErrorsCache gcache.Cache[string, int]
	// contains filtered or unexported fields
}

Runner is a client for running the enumeration process.

func New

func New(options *Options) (*Runner, error)

New creates a new client for running enumeration process.

func (*Runner) Close

func (r *Runner) Close()

Close closes the httpx scan instance

func (*Runner) GetScanOpts

func (r *Runner) GetScanOpts() ScanOptions

func (*Runner) HTTPX

func (r *Runner) HTTPX() *httpx.HTTPX

func (*Runner) HandleFaviconHash

func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp []byte, defaultProbe bool) (string, string, string, []byte, string, error)

func (*Runner) Process

func (r *Runner) Process(t string, wg *syncutil.AdaptiveWaitGroup, protocol string, scanopts *ScanOptions, output chan Result)

func (*Runner) RunEnumeration

func (r *Runner) RunEnumeration()

RunEnumeration on targets for httpx client

func (*Runner) SaveResumeConfig

func (r *Runner) SaveResumeConfig() error

SaveResumeConfig to file

type ScanOptions

type ScanOptions struct {
	Methods                   []string
	StoreResponseDirectory    string
	RequestURI                string
	RequestBody               string
	VHost                     bool
	OutputTitle               bool
	OutputStatusCode          bool
	OutputLocation            bool
	OutputContentLength       bool
	StoreResponse             bool
	OmitBody                  bool
	OutputServerHeader        bool
	OutputWebSocket           bool
	OutputWithNoColor         bool
	OutputMethod              bool
	ResponseHeadersInStdout   bool
	ResponseInStdout          bool
	Base64ResponseInStdout    bool
	ChainInStdout             bool
	TLSProbe                  bool
	CSPProbe                  bool
	VHostInput                bool
	OutputContentType         bool
	Unsafe                    bool
	Pipeline                  bool
	HTTP2Probe                bool
	OutputIP                  bool
	OutputCName               bool
	OutputCDN                 string
	OutputResponseTime        bool
	PreferHTTPS               bool
	NoFallback                bool
	NoFallbackScheme          bool
	TechDetect                bool
	StoreChain                bool
	StoreVisionReconClusters  bool
	MaxResponseBodySizeToSave int
	MaxResponseBodySizeToRead int
	OutputExtractRegex        string

	ExcludeCDN           bool
	HostMaxErrors        int
	ProbeAllIPS          bool
	Favicon              bool
	LeaveDefaultPorts    bool
	OutputLinesCount     bool
	OutputWordsCount     bool
	Hashes               string
	Screenshot           bool
	UseInstalledChrome   bool
	DisableStdin         bool
	NoScreenshotBytes    bool
	NoHeadlessBody       bool
	NoScreenshotFullPage bool
	ScreenshotTimeout    time.Duration
	ScreenshotIdle       time.Duration
	// contains filtered or unexported fields
}

func (*ScanOptions) Clone

func (s *ScanOptions) Clone() *ScanOptions

func (*ScanOptions) IsScreenshotFullPage

func (s *ScanOptions) IsScreenshotFullPage() bool

type Server

type Server struct {
	// contains filtered or unexported fields
}

Server represents the HTTP server that handles the concurrency settings endpoints.

func NewServer

func NewServer(addr string, config *Options) *Server

New creates a new instance of Server.

func (*Server) Start

func (s *Server) Start() error

Start initializes the server and its routes, then starts listening on the specified address.

func (*Server) Stop

func (s *Server) Stop() error

Stop gracefully shuts down the server.

type Trace

type Trace struct {
	GetConn              time.Time `json:"get_conn,omitempty"`
	GotConn              time.Time `json:"got_conn,omitempty"`
	PutIdleConn          time.Time `json:"put_idle_conn,omitempty"`
	GotFirstResponseByte time.Time `json:"got_first_response_byte,omitempty"`
	Got100Continue       time.Time `json:"got_100_continue,omitempty"`
	DNSStart             time.Time `json:"dns_start,omitempty"`
	DNSDone              time.Time `json:"dns_done,omitempty"`
	ConnectStart         time.Time `json:"connect_start,omitempty"`
	ConnectDone          time.Time `json:"connect_done,omitempty"`
	TLSHandshakeStart    time.Time `json:"tls_handshake_start,omitempty"`
	TLSHandshakeDone     time.Time `json:"tls_handshake_done,omitempty"`
	WroteHeaderField     time.Time `json:"wrote_header_field,omitempty"`
	WroteHeaders         time.Time `json:"wrote_headers,omitempty"`
	Wait100Continue      time.Time `json:"wait_100_continue,omitempty"`
	WroteRequest         time.Time `json:"wrote_request,omitempty"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL