mailstrix

type Cache interface {
	// Get returns an immutable match slice owned by the cache. Callers must not
	// mutate the returned slice or its Match entries.
	Get(key string) ([]Match, bool)
	// Put stores matches by reference; callers must treat matches as immutable
	// after insertion.
	Put(key string, matches []Match)
	Flush()
	// Degraded returns a non-empty human-readable reason when the cache is
	// operating in a reduced capacity (e.g. the Redis circuit breaker is open).
	// An empty string means fully operational. Disabled caching (noopCache) is
	// not degraded — it is an intentional configuration.
	Degraded() string
}

type Config struct {
	Host           string        // MAILSTRIX_HOST            (default 0.0.0.0)
	Port           int           // MAILSTRIX_PORT            (default 8079)
	BackendTimeout time.Duration // MAILSTRIX_BACKEND_TIMEOUT (default 1s)
	MaxConcurrent  int           // MAILSTRIX_MAX_CONCURRENT  (default "auto" = CPU count)
	MaxInflight    int           // MAILSTRIX_MAX_INFLIGHT    (default 2×MaxConcurrent); admission gate
	MaxBody        int64         // MAILSTRIX_MAX_BODY bytes  (default 8 MiB)
	Token          string        // MAILSTRIX_TOKEN[_FILE]    (required for /scan; comma-separated for rotation)
	TokenNext      string        // MAILSTRIX_TOKEN_NEXT[_FILE] (incoming rotation token; empty = no rotation)

	// RulesDir is the directory of *.yar / *.yara source files compiled at boot
	// and on SIGHUP. RulesPath, if set, is a single precompiled (.yac) ruleset
	// loaded instead of compiling sources (faster startup, used when the image
	// bakes a compiled bundle). RulesPath wins when both are set.
	RulesDir  string // MAILSTRIX_RULES_DIR  (default /rules)
	RulesPath string // MAILSTRIX_RULES      (optional precompiled bundle)

	// CacheDir is the writable directory yarad keeps its live, updatable rule
	// bundle in (and later the abuse.ch feed snapshots). SeedRules is the baked,
	// read-only compiled .yac shipped in the image. On startup, when SeedRules is
	// set and CacheDir/compiled.yac is missing or unreadable, the seed is copied
	// into the cache and loaded from there — so a fresh deploy (or a wiped
	// bindmount) always self-heals to a known-good tested ruleset with no network.
	// `--fetch-rules` (a later step) refreshes the cache copy. Both empty keeps the
	// old behaviour (load RulesPath/RulesDir directly).
	CacheDir  string // MAILSTRIX_CACHE_DIR  (e.g. /var/cache/mailstrix; empty = disabled)
	SeedRules string // MAILSTRIX_SEED_RULES (baked read-only .yac to seed the cache from)

	// RulesMaxAge flags the loaded ruleset as STALE once its on-disk mtime is
	// older than this. The image bakes rules and a daily rebuild refreshes them;
	// if that rebuild silently breaks (fetch failed, image not redeployed) the
	// running container keeps serving old rules with no error. When set (>0) and
	// exceeded, /ready reports "stale" (503) so an orchestrator/alert notices —
	// but /health stays OK and scanning continues (fail-open: old rules still
	// catch most malware; a hard-down scanner is worse). 0 disables the check.
	RulesMaxAge time.Duration // MAILSTRIX_RULES_MAX_AGE (seconds; default 0 = off)

	// ScanTimeout bounds a single libyara scan so a pathological rule/input
	// cannot stall a worker (YARA's own internal timeout, seconds).
	ScanTimeout time.Duration // MAILSTRIX_SCAN_TIMEOUT (default 8s)

	// BigFileThreshold gates an oversized-buffer cost defence. A full-ruleset scan
	// of a multi-MB buffer is inherently unbounded (size × ~12k rules) and can
	// time out even at a large ScanTimeout, so the scanner fail-opens and a padded
	// dropper is MISSED. When a scanned buffer is larger than this threshold, the
	// scanner runs the small, high-signal "big-file" ruleset (BigFileRules — our
	// in-repo local rules only) INSTEAD of the full bundle, so the scan completes
	// fast and the local heuristics still fire. Below it, behaviour is unchanged
	// (full ruleset). Default 6 MiB (below the 8 MiB MaxBody default so a file
	// padded toward the body cap hits the gate). 0 disables the gate entirely.
	BigFileThreshold int64 // MAILSTRIX_BIGFILE_THRESHOLD bytes (default 6 MiB; 0 = off)

	// BigFileRules is the targeted ruleset used by the oversized-buffer gate. It is
	// a path to either a precompiled .yac bundle (loaded directly) or a directory
	// of *.yar/*.yara source files (compiled at boot, like RulesDir). It should
	// hold ONLY the in-repo high-signal local rules so the gated scan is cheap.
	// Empty disables the gate even if BigFileThreshold>0 — Scan then falls back to
	// the full ruleset for oversized buffers (logged once), never crashing.
	BigFileRules string // MAILSTRIX_BIGFILE_RULES (default = baked local.yac seed)

	// Verdict cache. At high volume mail is heavily duplicated (bulk campaigns,
	// one body to N recipients, MTA retries), so caching SHA256(body) -> matches
	// turns most scans into a microsecond lookup. The in-process LRU is always
	// on; RedisURL adds a shared layer across replicas (empty => LRU only).
	CacheTTL    time.Duration // MAILSTRIX_CACHE_TTL    (default 3600s; 0 disables caching)
	CacheSize   int           // MAILSTRIX_CACHE_SIZE   (default 65536 in-memory entries)
	RedisURL    string        // MAILSTRIX_REDIS_URL    (empty -> in-process LRU only)
	RedisPrefix string        // MAILSTRIX_REDIS_PREFIX (default yara:scan:)

	Verbose     bool // MAILSTRIX_VERBOSE
	LogStdout   bool // MAILSTRIX_LOG_STDOUT — info/access to stdout; errors stay stderr
	MetricsAuth bool // MAILSTRIX_METRICS_AUTH — require the token for /metrics and /version
	Pprof       bool // MAILSTRIX_PPROF — enable /debug/pprof (off by default, ops-only)
	Canary      bool // MAILSTRIX_CANARY — shadow/observe-only: tag ALL matches mailstrix_canary=1

	// Password-protected archive decryption (opt-in, default OFF). When OFF the
	// service behaves exactly as before — an encrypted zip/7z/rar member emits the
	// ARCHIVE-ENCRYPTED marker and is never decrypted. When ON, the extractor tries
	// a bounded candidate-password list against encrypted members and, on a hit,
	// feeds the plaintext through the normal child-scan path. The brute loop is
	// hard-capped (attempt budget + per-attempt deadline + candidate-list size cap);
	// see internal/extract for the bounds. Candidates come from the built-in list,
	// the optional wordlist file, the attachment filename, and (if the rspamd plugin
	// sends them) the mail subject/body.
	ArchivePW bool // MAILSTRIX_ARCHIVE_PW — enable encrypted-archive decrypt (default false)
	// ArchivePWords is the boot-loaded wordlist from MAILSTRIX_ARCHIVE_PW_FILE,
	// merged with the built-in defaults. Empty if the env is unset or the file is
	// unreadable (fail-open + a logged warning). Capped at load time
	// (archivePWFileMaxLines lines, archivePWWordMaxLen bytes/line, deduped).
	ArchivePWords []string // from MAILSTRIX_ARCHIVE_PW_FILE (optional)

	// DenylistFile is an optional path to a file of rule names (one per line,
	// # comments, case-insensitive) merged with the env-based RuleDenylist.
	// Re-read on every SIGHUP so rules can be suppressed without a restart.
	// If the file doesn't exist or is unreadable, a warning is logged and
	// scanning continues with the env-only denylist (fail-open).
	DenylistFile string // MAILSTRIX_DENYLIST_FILE (default empty = disabled)

	// URLhaus malware-URL lookup. Disabled unless an abuse.ch Auth-Key is set.
	URLhausKey     string        // MAILSTRIX_URLHAUS_KEY[_FILE] — abuse.ch Auth-Key
	URLhausRefresh time.Duration // MAILSTRIX_URLHAUS_REFRESH (default 360m, floor 5m)
	URLhausMaxURLs int           // MAILSTRIX_URLHAUS_MAX_URLS  (per message, default 64)

	// MalwareBazaar attachment-hash lookup (abuse.ch). The SHA256 of each scanned
	// buffer is matched against a cached set of known-malware sample hashes.
	// Disabled unless an Auth-Key is set (the SAME abuse.ch key as URLhaus).
	MBazaarKey     string        // MAILSTRIX_MBAZAAR_KEY[_FILE] — abuse.ch Auth-Key
	MBazaarRefresh time.Duration // MAILSTRIX_MBAZAAR_REFRESH (default 24h, floor 5m)
	MBazaarFeed    string        // MAILSTRIX_MBAZAAR_FEED (URL override; default full dump)

	// ThreatFox IOC lookup (abuse.ch). URL and domain IOCs from the ThreatFox
	// recent-IOC CSV are matched against URLs extracted from scanned buffers.
	// Complements URLhaus (delivery URLs) with botnet C&C indicators.
	// Disabled unless an Auth-Key is set (same abuse.ch key as URLhaus).
	ThreatFoxKey     string        // MAILSTRIX_THREATFOX_KEY[_FILE] — abuse.ch Auth-Key
	ThreatFoxRefresh time.Duration // MAILSTRIX_THREATFOX_REFRESH (default 360m, floor 5m)
	ThreatFoxMaxURLs int           // MAILSTRIX_THREATFOX_MAX_URLS (per message, default 64)

	ICAPAddr string // MAILSTRIX_ICAP_ADDR (empty = disabled; e.g. ":1344")

	// RuleDenylist suppresses matches for these rule names (case-insensitive).
	// Public rulesets ship demo/noise rules that are pure false positives for
	// mail — e.g. Didier Stevens' `http` rule (rtf.yara) is `$="http" nocase`,
	// so it fires on virtually every message. Defaults to "http"; override with a
	// comma-separated list, or set the var empty to disable filtering entirely.
	RuleDenylist map[string]struct{} // MAILSTRIX_RULE_DENYLIST (comma-sep, default "http")

	// RuleAllowlist names rules whose matches are KEPT but tagged log-only
	// (case-insensitive): yarad still reports them (so they show in the mail
	// history) but adds meta `mailstrix_allow=1`, and the rspamd plugin routes those
	// to a 0-weight symbol. This force-demotes a known-FP rule without dropping
	// its visibility (denylist) and without patching the upstream source. Empty
	// by default. A name in BOTH lists is denied (drop wins over demote).
	RuleAllowlist map[string]struct{} // MAILSTRIX_RULE_ALLOWLIST (comma-sep, default empty)

	// Effort tiers (EFFORT-1). A single 1..EffortMax dial scales every bounded
	// extraction/scan cap so one binary serves both a latency-tight front
	// (rspamd, pre-queue) and a deeper backend (LDA/sieve), and can shed work
	// under load. Level 1 = raw + shallowest extraction, EffortMax = everything.
	//
	// Resolution order per request (see ResolveEffort / scanMetaFromRequest):
	// X-MAILSTRIX-Effort header ?? Effort (env default), clamped to [1, EffortMax].
	// EffortMax is the DoS ceiling — an attacker-set header can never raise effort
	// above it. The resolved level folds into the verdict-cache key (the same
	// bytes scanned at effort 2 vs 9 can yield different verdicts).
	//
	// EFFORT-4 made the dial LIVE: a level now resolves to a real cap profile
	// (EffortProfileFor) that scales the MSD decode depth/iterations, the PDF
	// structural-indicator pass, and whether the URLhaus/MalwareBazaar reputation
	// feeds run. A low level is cheaper and shallower; EffortMax runs everything.
	Effort     int  // MAILSTRIX_EFFORT     (default = EffortMax; the env/default level)
	EffortMax  int  // MAILSTRIX_EFFORT_MAX (default 10; hard ceiling for header override)
	EffortAuto bool // MAILSTRIX_EFFORT=auto (EFFORT-2): derive the per-request level from

	Version string // build version string, set by main (not from env); for /version
	// contains filtered or unexported fields
}

type EffortProfile struct {
	// Level is the resolved effort (1..EffortMax) this profile was built for. It
	// is what folds into the verdict-cache key, so two scans of the same bytes at
	// different effort can hold distinct verdicts.
	Level int

	// DecodeDepth caps the MSD multi-layer static-decode recursion (decode.go
	// maxDecodeDepth). PDFDeepen enables the PDF action/JS indicator pass
	// (pdf.go fromPDFIndicators). ReputationFeeds enables the URLhaus/MalwareBazaar
	// lookups. These are the first caps EFFORT-4 wires; more (XLM formula/sheet
	// caps, fold/carve clamps, maxStreams) follow.
	DecodeDepth      int
	DecodeIterations int
	PDFDeepen        bool
	ReputationFeeds  bool

	// ScanTimeout is the per-request libyara wall-clock budget scaled by effort
	// level. At level 1 it is 50% of the base; at the ceiling it equals the base.
	// A zero base (no limit) keeps ScanTimeout at 0 (no limit).
	ScanTimeout time.Duration

	// XLMFoldSheets caps the number of macrosheets scanned per document by the
	// XLM constant-fold pass. Scaled linearly from 4 at level 1 to 64 at the
	// ceiling (EFFORT-4-XLMCAPS).
	XLMFoldSheets int
	// XLMFoldFormulas caps the number of formulas processed per macrosheet by the
	// XLM constant-fold pass. Scaled linearly from 256 at level 1 to 4096 at the
	// ceiling (EFFORT-4-XLMCAPS).
	XLMFoldFormulas int
}

type ExtractMetrics struct {
	Docs             uint64 // buffers recognised as an OLE2/OOXML container
	Streams          uint64 // decompressed macro blobs scanned (sum across docs)
	MacroDocs        uint64 // documents that yielded >=1 macro stream
	Failed           uint64 // container parse attempts that errored
	Panicked         uint64 // parser panics recovered (subset of Failed)
	Encrypted        uint64 // ECMA-376 encrypted OOXML (not decrypted)
	MSI              uint64 // OLE2 buffers recognised as MSI installers (streams dumped)
	MSG              uint64 // OLE2 buffers recognised as Outlook .msg (attachments extracted)
	OneNote          uint64 // buffers recognised as OneNote .one (embedded files carved)
	Archive          uint64 // buffers recognised as an archive (zip/gz/7z/rar/tar; members unpacked)
	ArchiveDecrypted uint64 // archives with >=1 password-protected member decrypted via a candidate
	OLEPackage       uint64 // OLE2 docs with an embedded OLE Package object (Ole10Native carved)
	LNK              uint64 // Windows shell links (.lnk) with StringData surfaced
	PDF              uint64 // PDFs with FlateDecode object streams inflated
	RTF              uint64 // RTF docs with \objdata embedded objects carved
	SLK              uint64 // SYLK (.slk) spreadsheets with XLM/DDE formulas scanned
	EncScript        uint64 // buffers with >=1 decoded MS-Script-Encoder (VBE/JSE) block
	Decoded          uint64 // buffers with >=1 base64/hex/reversed blob from the static decode pass
	// StreamMatches counts rule hits attributable ONLY to an extracted stream
	// (macro/MSI/VBE), i.e. rules that did NOT already fire on the raw bytes —
	// the direct measure of what pre-extraction adds over a raw-only scan.
	StreamMatches uint64
	// Deduped counts extracted streams skipped before YARA scanning because their
	// SHA256 matched a previously scanned stream (or the raw input buffer itself).
	Deduped     uint64
	DocProps    uint64 // documents with doc-property strings extracted
	XLMFold     uint64 // documents with XLM formula constant-folding applied
	ExtMismatch uint64 // attachments whose real container type contradicts a benign-looking extension (renamed dropper)
}

type FetchResult struct {
	Updated      bool // a new bundle was downloaded and swapped in
	LocalVersion int  // version before the run
	NewVersion   int  // version after (== LocalVersion when not updated)
	Reason       string
}

type Match struct {
	Rule string `json:"rule"`
	// Namespace is the libyara namespace the rule was compiled into. yarad
	// compiles each rule file into a namespace named after the file (see
	// compileDir / compile-rules.sh `ns:path`), so this is effectively the
	// source ruleset file (e.g. "sigbase-gen_url.yar") — surfaced so the rspamd
	// plugin can show WHICH ruleset fired, not just the (often generic) rule
	// name. Empty for synthetic matches (URLhaus) that aren't from a rule file.
	Namespace string            `json:"namespace,omitempty"`
	Tags      []string          `json:"tags,omitempty"`
	Meta      map[string]string `json:"meta,omitempty"`
}

type MatchCount struct {
	Rule  string `json:"rule"`
	Count uint64 `json:"count"`
}

type ReloadMetrics struct {
	Attempts        uint64 // Reload() calls (includes the initial boot load)
	Successes       uint64
	Failures        uint64
	LastUnix        int64  // unix seconds of the last successful reload
	LastMillis      int64  // wall-clock duration of the last reload attempt
	Rules           int64  // rule count after the last successful reload
	ModUnix         int64  // mtime (unix seconds) of the loaded ruleset on disk; 0 if unknown
	PrevFingerprint string // fingerprint before the last reload ("" on first load)
}

type RuleSource struct {
	Name    string `json:"name"`
	Repo    string `json:"repo"`
	License string `json:"license"`
	Ref     string `json:"ref"`
	Set     string `json:"set,omitempty"` // only yaraforge (core/extended/full)
}

type RulesManifest struct {
	Version   int          `json:"version"`           // monotonic; an update exists iff remote > local
	Generated string       `json:"generated"`         // RFC3339 UTC, display/audit
	Checksum  string       `json:"checksum"`          // "sha256:<hex>" of compiled.yac
	Libyara   string       `json:"libyara"`           // libyara version that compiled it (skew guard)
	Rules     int          `json:"rules"`             // rule count (display)
	Size      int64        `json:"size"`              // compiled.yac bytes (sanity)
	Sources   []RuleSource `json:"sources,omitempty"` // per-ruleset provenance
}

type ScanEngine interface {
	Scan(buf []byte, meta ScanMeta) ([]Match, error)
	RuleCount() int64
	// BigFileScans reports how many oversized buffers were scanned against the
	// targeted big-file ruleset (MAILSTRIX_BIGFILE_THRESHOLD gate), for /metrics.
	BigFileScans() uint64
	// BigFileStreamScans reports how many oversized extracted streams were scanned
	// against the targeted big-file ruleset instead of the full set, for /metrics.
	BigFileStreamScans() uint64
	// RawChannelScans / StreamChannelScans / MarkerChannelScans report per-channel
	// libyara scan counts (raw body / real-content stream / marker channel), for
	// /metrics (PERF-17).
	RawChannelScans() uint64
	StreamChannelScans() uint64
	MarkerChannelScans() uint64
	// RawScanErrs reports raw-scan failures that fell through to extraction
	// instead of aborting the request, for /metrics.
	RawScanErrs() uint64
	// Fingerprint identifies the active rule set; it is mixed into the cache key
	// so a reload that changes the rules invalidates old verdicts (L1 and Redis).
	Fingerprint() string
	// ExtractMetrics reports the OLE/OOXML pre-extraction counters for /metrics.
	ExtractMetrics() ExtractMetrics
	// ReloadMetrics reports rule-reload activity for /metrics.
	ReloadMetrics() ReloadMetrics
	// URLhausMetrics reports the URLhaus checker state for /metrics.
	URLhausMetrics() urlhaus.Metrics
	// MBazaarMetrics reports the MalwareBazaar checker state for /metrics.
	MBazaarMetrics() mbazaar.Metrics
	// ThreatFoxMetrics reports the ThreatFox checker state for /metrics.
	ThreatFoxMetrics() threatfox.Metrics
	// TopMatches returns the top n most-triggered rule names since last reload.
	TopMatches(n int) []MatchCount
}

type ScanMeta struct {
	Filename  string // sanitized basename, e.g. "invoice.exe" ("" if none)
	Extension string // lowercased extension WITH the leading dot, e.g. ".exe" (Loki/THOR convention); "" if none
	FileType  string // coarse optional type context for rules that use file_type, e.g. "outlook" for .msg/.oft
	// Effort is the resolved effort-tier level (1..EffortMax) for this scan
	// (EFFORT-1). It rides on ScanMeta so it is automatically part of the
	// verdict-cache key (see cacheKey) — the same bytes scanned at different
	// effort can yield different verdicts and must not share a cached one. Zero
	// means "unset" (legacy callers / internal scans); cacheKey treats 0 and the
	// resolved default identically only when both produce the same key string, so
	// resolution always fills a concrete level before Scan.
	Effort int
	// RawKey is the precomputed streamDedupKey([16]byte) of the raw request body,
	// set by the handler on the miss path so Scanner.Scan can seed the per-stream
	// dedup set without re-hashing the (potentially multi-MB) buffer (PERF-22).
	// Zero value signals "not precomputed"; Scan falls back to computing it inline.
	// The handler also uses string(RawKey[:]) as the body component of the
	// verdict-cache key (same domain as streamDedupKey, replaces bodyCacheHash).
	RawKey [16]byte
	// PWCandidates carries per-request candidate passwords (from the mail
	// subject/body via the rspamd plugin header) for the opt-in encrypted-archive
	// decrypt feature. Already capped+sanitized by the server header parse. These
	// change the verdict (a decrypted member yields different markers), so they
	// MUST fold into cacheKey — see cacheKey. Nil/empty when the feature is off or
	// no candidates were supplied; in that case cacheKey is byte-identical to the
	// pre-feature key (no cache split for the default-OFF path).
	PWCandidates []string
}

type Scanner struct {
	// contains filtered or unexported fields
}

type Server struct {
	// contains filtered or unexported fields
}