Documentation
¶
Index ¶
- Variables
- func BuildScanReport(result *url_scanner.ScanGetResponse, enrichment Enrichment, ...) map[string]any
- type CompanyProfile
- type ComplianceLink
- type CompliancePage
- type Config
- func (c *Config) GatherEnrichment(ctx context.Context, domain string, timeout time.Duration) (Enrichment, EnrichmentErrors)
- func (c *Config) GetCompanyData(ctx context.Context, url string) (*CompanyProfile, error)
- func (c *Config) GetComplianceData(ctx context.Context, domain string) (*CompliancePage, error)
- type DNSVendor
- type DNSVendorInfo
- type Enrichment
- type EnrichmentErrors
- type JSONSchema
- type JSONSchemaProperty
- type PromptType
- type ReportConfig
- type ResponseFormat
- type SocialLinks
- type SubdomainDNSInfo
- type System
- type TrustCenterPage
- type TrustDocument
Constants ¶
This section is empty.
Variables ¶
var ( // ErrInvalidDomain is returned when its not able to determine the apex domain ErrInvalidDomain = errors.New("could not determine domain") )
Functions ¶
func BuildScanReport ¶
func BuildScanReport(result *url_scanner.ScanGetResponse, enrichment Enrichment, nonVendorCategories, deniedVendorNames []string) map[string]any
BuildScanReport combines a Cloudflare URL Scanner result with the Enrichment gathered by GatherEnrichment into a single report unified vendors/technologies, assets, findings, meta, platform, systems, and compliance sections
Types ¶
type CompanyProfile ¶
type CompanyProfile struct {
Name string `json:"name"`
Description string `json:"description"`
Industry string `json:"industry"`
Systems []System `json:"systems"`
Location string `json:"location"`
EmployeeRange string `json:"employee_range"`
FoundedYear string `json:"founded_year"`
EstimatedRevenue string `json:"estimated_revenue"`
SocialLinks SocialLinks `json:"social_links"`
Customers []string `json:"customers"`
Technologies []string `json:"technologies"`
// StatusPageURL is the company's public status/uptime page, if found
StatusPageURL string `json:"status_page_url,omitempty"`
// SubdomainLinks are other subdomains of the same company's domain linked
// from the page (e.g. console.<domain>, app.<domain>, docs.<domain>)
SubdomainLinks []string `json:"subdomain_links,omitempty"`
// SSOSupported indicates the company's product advertises SSO support
SSOSupported bool `json:"sso_supported"`
// MFASupported indicates the company's product advertises MFA support
MFASupported bool `json:"mfa_supported"`
}
CompanyProfile is the company information extracted from a website by the browser rendering AI
type ComplianceLink ¶
type ComplianceLink struct {
// URL is the link target
URL string `json:"url"`
// Type categorizes the linked document (e.g., privacy_policy, terms_of_service, trust_center, dpa, soc2_report, security, subprocessors, gdpr, cookie_policy, or other)
Type string `json:"type"`
}
ComplianceLink is a single compliance-related link found on a page, categorized by type
type CompliancePage ¶
type CompliancePage struct {
// URL is the page URL that was analyzed
URL string `json:"url"`
// PageType categorizes the compliance document (e.g., privacy_policy, terms_of_service, trust_center, dpa, soc2_report, security, subprocessors, gdpr, cookie_policy)
PageType string `json:"page_type"`
// Title is the page title
Title string `json:"title"`
// Summary is a brief description of the page content
Summary string `json:"summary"`
// Frameworks lists compliance frameworks or certifications mentioned (e.g., SOC 2, ISO 27001, GDPR, HIPAA)
Frameworks []string `json:"frameworks"`
// SOC2Certified indicates whether the page claims SOC 2 (Type I or Type II) certification or compliance
SOC2Certified bool `json:"soc2_certified"`
// LastUpdated is the last updated or effective date mentioned on the page
LastUpdated string `json:"last_updated"`
// DownloadLinks contains URLs for downloadable reports or documents found on the page
DownloadLinks []string `json:"download_links"`
// Subprocessors lists third-party vendors or sub-processors mentioned on the page
Subprocessors []string `json:"subprocessors,omitempty"`
// Controls lists individual security or compliance controls/practices mentioned, such as
// those shown on a trust center page (e.g., MFA enforced, data encrypted at rest and in
// transit, background checks performed, vulnerability scanning, penetration testing, incident response plan)
Controls []string `json:"controls,omitempty"`
// TrustCenterHostedBy is the platform serving the company's trust center, if
// one was found (e.g., Vanta, Drata, SafeBase, Whistic, Conveyor, TrustArc,
// or "self-hosted") — itself a vendor/technology signal for the company
TrustCenterHostedBy string `json:"trust_center_hosted_by,omitempty"`
// Documents lists compliance documents found on a trust center, each
// flagged as public (directly accessible) or gated (requires a request/NDA)
Documents []TrustDocument `json:"documents,omitempty"`
// ComplianceLinks lists URLs to other compliance documents found on the page, each categorized by type
ComplianceLinks []ComplianceLink `json:"compliance_links,omitempty"`
}
CompliancePage holds structured compliance information extracted from a single page
type Config ¶
type Config struct {
// APIToken use to authenticate into with the cloudflare API
APIToken string
// AccountID the APIToken is associated with
AccountID string
// CacheTTL is the cache TTL, in seconds, Cloudflare applies to Browser Rendering requests.
// Zero disables caching for the request, forcing a fresh render instead of a cached one
CacheTTL int
}
Config holds the Cloudflare credentials used for browser rendering and browser-derived enrichment lookups
func (*Config) GatherEnrichment ¶
func (c *Config) GatherEnrichment(ctx context.Context, domain string, timeout time.Duration) (Enrichment, EnrichmentErrors)
GatherEnrichment runs the company profile, compliance, and DNS vendor lookups for domain concurrently, bounded by timeout. Each lookup is best-effort
func (*Config) GetCompanyData ¶
GetCompanyData builds a company profile from url's homepage, merged with whichever of companyProfileSubpaths resolve on the same host
func (*Config) GetComplianceData ¶
GetComplianceData fetches compliance information from the given domain and, if it can derive one, from a trust.<domain> subdomain as well
type DNSVendor ¶
DNSVendor is a single vendor identified from a DNS record. URL is only populated when the signal pointed at an actual hostname (MX, SPF include, or CNAME target); DKIM selector and TXT verification-tag signals carry a name only, since the selector/tag itself isn't a domain
type DNSVendorInfo ¶
type DNSVendorInfo struct {
// SPFRecord is the raw SPF TXT record found at the domain's apex, if any
SPFRecord string `json:"spf_record,omitempty"`
// DMARCRecord is the raw DMARC TXT record found at _dmarc.<domain>, if any
DMARCRecord string `json:"dmarc_record,omitempty"`
// DMARCPolicy is the enforcement policy from the DMARC record's p= tag (none, quarantine, or reject)
DMARCPolicy string `json:"dmarc_policy,omitempty"`
// MXHosts lists the mail exchanger hostnames for the domain's apex, in preference order
MXHosts []string `json:"mx_hosts,omitempty"`
// NSHosts lists the authoritative nameserver hostnames for the domain's apex, revealing
// the DNS hosting vendor (e.g. Cloudflare, Route 53, NS1)
NSHosts []string `json:"ns_hosts,omitempty"`
// TXTRecords lists every TXT record found at the domain's apex
TXTRecords []string `json:"txt_records,omitempty"`
// DKIMSelectors lists DKIM selector labels (see commonDKIMSelectors) found on the domain's apex
DKIMSelectors []string `json:"dkim_selectors,omitempty"`
// Subdomains holds MX, TXT, CNAME, and DKIM selector records found on conventional
// mail/vendor subdomains (see commonSubdomains), probed up to two levels deep
Subdomains []SubdomainDNSInfo `json:"subdomains,omitempty"`
// Vendors lists vendors identified from the MX, SPF, TXT, CNAME, DKIM selector, and
// NS records found at the apex and on Subdomains
Vendors []DNSVendor `json:"vendors,omitempty"`
}
DNSVendorInfo holds DNS-derived signals about a domain: mail routing/authentication at the apex, records on conventional subdomains (see commonSubdomains), and vendors those point to
func GetDNSVendorInfo ¶
func GetDNSVendorInfo(ctx context.Context, rawURL string) (*DNSVendorInfo, error)
GetDNSVendorInfo resolves MX/SPF/DMARC at the apex, probes commonSubdomains for MX/TXT/CNAME/DKIM, and derives vendor names from whatever's found rather than a maintained lookup table
type Enrichment ¶
type Enrichment struct {
// Company is the base company profile
Company *CompanyProfile
// Compliance is compliance specific data such as soc2 attestation, subprocessors, etc
Compliance *CompliancePage
// DNS includes DNS probed data
DNS *DNSVendorInfo
}
Enrichment holds the best-effort pkg/domainscan results for a single domain
type EnrichmentErrors ¶
EnrichmentErrors holds the per-lookup errors from GatherEnrichment, each nil on success
type JSONSchema ¶
type JSONSchema struct {
Type string `json:"type"`
Properties map[string]JSONSchemaProperty `json:"properties"`
}
JSONSchema is the JSON schema for structured extraction
type JSONSchemaProperty ¶
type JSONSchemaProperty struct {
Type string `json:"type"`
Description string `json:"description,omitempty"`
Items *JSONSchemaProperty `json:"items,omitempty"`
Properties map[string]JSONSchemaProperty `json:"properties,omitempty"`
}
JSONSchemaProperty describes a single property in a JSON schema
type PromptType ¶
type PromptType string
type ReportConfig ¶
type ReportConfig struct {
// NonVendorCategories lists wappalyzer categories treated as technologies
// instead of vendors when building an onboarding domain scan report
NonVendorCategories []string `` /* 145-byte string literal not displayed */
// DeniedVendorNames lists vendor names to always exclude from an onboarding domain scan report's vendor list
DeniedVendorNames []string `` /* 245-byte string literal not displayed */
// ScanTTL is the cache TTL, in seconds, for Browser Rendering requests issued during domain scan enrichment
ScanTTL int `json:"scanttl" koanf:"scanttl" default:"86400"`
}
ReportConfig configures how BuildScanReport classifies vendors versus technologies and which vendor names it always excludes
type ResponseFormat ¶
type ResponseFormat struct {
Type string `json:"type"`
Schema JSONSchema `json:"json_schema"`
}
ResponseFormat specifies JSON schema extraction
type SocialLinks ¶
type SocialLinks struct {
LinkedIn string `json:"linkedin"`
Twitter string `json:"twitter"`
GitHub string `json:"github"`
Discord string `json:"discord"`
Instagram string `json:"instagram"`
YouTube string `json:"youtube"`
Facebook string `json:"facebook"`
}
SocialLinks holds a company's social media and community profile URLs
type SubdomainDNSInfo ¶
type SubdomainDNSInfo struct {
// Host is the fully probed hostname, e.g. "mail.example.com"
Host string `json:"host"`
// MXHosts lists the mail exchanger hostnames found for this subdomain, if any
MXHosts []string `json:"mx_hosts,omitempty"`
// TXTRecords lists the TXT records found for this subdomain, if any
TXTRecords []string `json:"txt_records,omitempty"`
// CNAME is the canonical name this subdomain resolves to, if it's an
// alias to another domain (e.g. a hosted checkout, status page, or
// support portal pointed at a vendor's own domain)
CNAME string `json:"cname,omitempty"`
// DKIMSelectors lists DKIM selector labels (see commonDKIMSelectors)
// found configured on this subdomain, e.g. "resend" for
// "resend._domainkey.mail.example.com"
DKIMSelectors []string `json:"dkim_selectors,omitempty"`
}
SubdomainDNSInfo holds the DNS records found for a single conventional subdomain probed alongside the apex
type System ¶
type System struct {
Name string `json:"name"`
// Summary is a brief, 1-2 sentence description of the system
Summary string `json:"summary"`
// FullDescription is a more thorough description of the system, drawn
// from documentation, architecture pages, or other technical content when available
FullDescription string `json:"full_description"`
}
System describes a single technical system or component that makes up the company's offering (e.g. a web console, public API, mobile app, or storage backend)
type TrustCenterPage ¶
type TrustCenterPage struct {
// HostedBy is the platform or vendor hosting this trust center (e.g.,
// Vanta, Drata, SafeBase, Whistic, Conveyor, TrustArc, OneTrust), or
// "self-hosted" if it appears to be a custom-built page
HostedBy string `json:"hosted_by"`
// Frameworks lists compliance frameworks or certifications listed
Frameworks []string `json:"frameworks"`
// SOC2Certified indicates whether the trust center claims SOC 2 (Type I or Type II) certification or compliance
SOC2Certified bool `json:"soc2_certified"`
// Controls lists individual security or compliance controls or practices listed
Controls []string `json:"controls"`
// Documents lists compliance documents or reports listed, each flagged public or gated
Documents []TrustDocument `json:"documents"`
// Subprocessors lists third-party vendors or sub-processors mentioned
Subprocessors []string `json:"subprocessors,omitempty"`
}
TrustCenterPage holds structured information extracted from a trust center or trust portal page
type TrustDocument ¶
type TrustDocument struct {
// Name is the document title (e.g., "SOC 2 Type II Report")
Name string `json:"name"`
// URL is a direct link to the document, if one is available without requesting access
URL string `json:"url,omitempty"`
// Public indicates the document can be viewed or downloaded without requesting access or signing an NDA
Public bool `json:"public"`
}
TrustDocument is a single compliance document or report listed on a trust center