file

package
v0.9.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 9, 2026 License: BSD-3-Clause, GPL-3.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var Decoder = &decoder.AbstractDecoder{
	Type:        types.Type_NC_File,
	Name:        "File",
	Description: "A file that was transferred over the network",
}

Decoder for protocol analysis and writing audit records to disk.

Functions

func DetectContentType added in v0.9.0

func DetectContentType(data []byte) (string, bool)

DetectContentType performs enhanced MIME type detection using magic numbers Returns the MIME type and a boolean indicating if detection was accurate

func ExtensionForContentType

func ExtensionForContentType(typ string) string

ExtensionForContentType returns the file extension for the given MIME type.

func GetDedupCache added in v0.9.0

func GetDedupCache() *fileDeduplicationCache

GetDedupCache returns the global deduplication cache

func GetDedupStats added in v0.9.0

func GetDedupStats() dedupStats

GetDedupStats returns the current deduplication statistics

func GetMaxFileSize added in v0.9.0

func GetMaxFileSize() int64

GetMaxFileSize returns the configured maximum file size

func IsProtocolEnabled added in v0.9.0

func IsProtocolEnabled(protocol string) bool

IsProtocolEnabled checks if file extraction is enabled for a protocol

func IsReassemblyEnabled added in v0.9.0

func IsReassemblyEnabled() bool

IsReassemblyEnabled checks if file reassembly is enabled

func ListExtractors added in v0.9.0

func ListExtractors() []string

ListExtractors returns a list of all registered protocol extractors

func LogDedupStats added in v0.9.0

func LogDedupStats(logger *zap.Logger)

LogDedupStats logs the current deduplication statistics

func RegisterExtractor added in v0.9.0

func RegisterExtractor(extractor FileExtractor)

RegisterExtractor registers a file extractor for a specific protocol

func ResetDedupCache added in v0.9.0

func ResetDedupCache()

ResetDedupCache resets the deduplication cache (useful for testing or new capture sessions)

func ResetGlobalYaraScanner added in v0.9.0

func ResetGlobalYaraScanner()

ResetGlobalYaraScanner resets the singleton so it can be re-initialized (for testing or reconfiguration).

func SaveFileEnhanced added in v0.9.0

func SaveFileEnhanced(
	conv *core.ConversationInfo,
	source, name string,
	err error,
	body []byte,
	encoding []string,
	host string,
	contentType string,
	depth int,
	parentFileID string,
	flowDirection string,
	protocol string,
) error

SaveFileEnhanced is an enhanced version of SaveFile with support for: - Multiple hash algorithms (MD5, SHA1, SHA256) - Better content type detection - Depth tracking for nested files - Parent file tracking - Flow direction tracking - Protocol tracking (HTTP, FTP, SMB, SMTP, IRC, etc.) - Configuration-based filtering and settings

func SetGlobalConfig added in v0.9.0

func SetGlobalConfig(cfg *Config)

SetGlobalConfig sets the global configuration

func SetSaveFileLogger added in v0.9.0

func SetSaveFileLogger(logger *zap.Logger)

SetSaveFileLogger sets the logger for file saving operations

func ShouldComputeHash added in v0.9.0

func ShouldComputeHash(algorithm string) bool

ShouldComputeHash checks if a specific hash algorithm should be computed

func ShouldExtractMimeType added in v0.9.0

func ShouldExtractMimeType(mimeType string) bool

ShouldExtractMimeType checks if a MIME type should be extracted based on whitelist/blacklist

func ShouldUseMagicDetection added in v0.9.0

func ShouldUseMagicDetection() bool

ShouldUseMagicDetection checks if magic number detection should be used

func ValidateYaraSource added in v0.9.0

func ValidateYaraSource(source string) error

ValidateYaraSource compiles YARA source to check for errors. Returns nil if valid, or the compilation error.

func WriteFile

func WriteFile(f *types.File)

WriteFile writeDeviceProfile writes the profile.

func WriteFileEnhanced added in v0.9.0

func WriteFileEnhanced(f *types.File)

WriteFileEnhanced writes an enhanced file record with additional fields This is an alias to WriteFile for consistency

func YaraAvailable added in v0.9.0

func YaraAvailable() bool

YaraAvailable returns true when yara-x support is compiled in.

Types

type AdvancedConfig added in v0.9.0

type AdvancedConfig struct {
	UseMagicDetection bool `yaml:"use_magic_detection"`
	DecodeCompressed  bool `yaml:"decode_compressed"`
	DecodeBase64      bool `yaml:"decode_base64"`
	MaxFilenameLength int  `yaml:"max_filename_length"`
	DeduplicateFiles  bool `yaml:"deduplicate_files"` // Only save each unique file once based on content hash
	// Security analysis options
	ComputeEntropy        bool   `yaml:"compute_entropy"`         // Calculate Shannon entropy
	DetectExecutables     bool   `yaml:"detect_executables"`      // Detect PE/ELF/Mach-O executables
	DetectEmbeddedScripts bool   `yaml:"detect_embedded_scripts"` // Detect embedded VBA/JS/PowerShell
	EnableYaraScanning    bool   `yaml:"enable_yara_scanning"`    // Enable YARA rule matching (requires yara rules path)
	YaraRulesPath         string `yaml:"yara_rules_path"`         // Path to YARA rules directory
	EnableMagika          bool   `yaml:"enable_magika"`           // Enable AI-based file type classification (requires magika CLI in PATH)
	MagikaAssetsDir       string `yaml:"magika_assets_dir"`       // Unused (Rust CLI embeds model), kept for config compatibility
	MagikaModelName       string `yaml:"magika_model_name"`       // Unused (Rust CLI embeds model), kept for config compatibility
}

AdvancedConfig defines advanced file extraction options

type Config added in v0.9.0

type Config struct {
	FileExtraction FileExtractionConfig `yaml:"file_extraction"`
}

Config represents the file extraction configuration

func GetDefaultConfig added in v0.9.0

func GetDefaultConfig() *Config

GetDefaultConfig returns the default configuration

func GetGlobalConfig added in v0.9.0

func GetGlobalConfig() *Config

GetGlobalConfig returns the global configuration

func LoadConfig added in v0.9.0

func LoadConfig(path string) (*Config, error)

LoadConfig loads file extraction configuration from a YAML file

type ContentInfo added in v0.9.0

type ContentInfo struct {
	Hash            string // SHA256 hash of the decoded content
	DecodedContent  []byte // Decompressed content
	WasCompressed   bool   // Whether the content was compressed
	CompressionType string // Compression algorithm (gzip, deflate, or empty)
	CompressedSize  int64  // Original compressed size
}

ContentInfo contains information about decoded content

func ComputeContentHash added in v0.9.0

func ComputeContentHash(body []byte, encoding []string) (*ContentInfo, error)

ComputeContentHash computes SHA256 hash of the content, handling all encoding types. This function decodes gzip/deflate/base64 content before hashing to ensure we hash the actual content. It also returns compression information for audit records.

type FileAnalysis added in v0.9.0

type FileAnalysis struct {
	Entropy             float64
	MagicBytes          []byte
	TrueFileType        string
	TypeMismatch        bool
	IsPEExecutable      bool
	IsELFExecutable     bool
	IsMachO             bool
	HasEmbeddedScript   bool
	IsPasswordProtected bool
	YaraMatches         []string
	IsKnownMalware      bool
	ThreatName          string
}

FileAnalysis contains security analysis results for a file

func AnalyzeFile added in v0.9.0

func AnalyzeFile(content []byte, filename string) *FileAnalysis

AnalyzeFile performs security analysis on file content Respects configuration settings for which analyses to perform

type FileExtractionConfig added in v0.9.0

type FileExtractionConfig struct {
	Enabled         bool                  `yaml:"enabled"`
	Protocols       ProtocolsConfig       `yaml:"protocols"`
	SizeLimits      SizeLimitsConfig      `yaml:"size_limits"`
	HashAlgorithms  HashAlgorithmsConfig  `yaml:"hash_algorithms"`
	MimeTypes       MimeTypesConfig       `yaml:"mime_types"`
	Storage         StorageConfig         `yaml:"storage"`
	IncompleteFiles IncompleteFilesConfig `yaml:"incomplete_files"`
	Reassembly      ReassemblyConfig      `yaml:"reassembly"`
	Advanced        AdvancedConfig        `yaml:"advanced"`
}

FileExtractionConfig contains all settings for file extraction

type FileExtractor added in v0.9.0

type FileExtractor interface {
	// GetFileHandle generates a unique identifier for a file in the conversation
	GetFileHandle(conv *core.ConversationInfo, isOrigin bool, depth int) string

	// DescribeFile returns a human-readable description of the file
	DescribeFile(handle *FileHandle) string

	// ExtractFile performs the actual file extraction
	ExtractFile(conv *core.ConversationInfo, data []byte, metadata FileMetadata) error

	// ProtocolName returns the name of the protocol this extractor handles
	ProtocolName() string
}

FileExtractor is the interface that protocol-specific extractors must implement

func GetExtractor added in v0.9.0

func GetExtractor(protocol string) (FileExtractor, bool)

GetExtractor retrieves a file extractor for the given protocol

type FileHandle added in v0.9.0

type FileHandle struct {
	ID             string
	ConversationID string
	Protocol       string
	Timestamp      int64
	Depth          int
	IsComplete     bool
	TotalBytes     int64
	SeenBytes      int64
	MissingBytes   int64
	ParentFileID   string
}

FileHandle represents a unique file being extracted from network traffic

type FileHashes added in v0.9.0

type FileHashes struct {
	MD5    string
	SHA1   string
	SHA256 string
}

FileHashes contains multiple hash values for a file

func ComputeFileHashes added in v0.9.0

func ComputeFileHashes(filepath string) (FileHashes, error)

ComputeFileHashes computes hashes for an existing file on disk

func ComputeHashes added in v0.9.0

func ComputeHashes(data []byte) FileHashes

ComputeHashes calculates MD5, SHA1, and SHA256 hashes for the given data

type FileMetadata added in v0.9.0

type FileMetadata struct {
	// Network context
	ConnectionUID string
	FlowDirection string // "client_to_server" or "server_to_client"

	// Protocol-specific context
	HTTPMethod     string // For HTTP files
	HTTPStatusCode int
	HTTPURL        string
	FTPCommand     string // RETR/STOR
	SMBShare       string
	SMBPath        string

	// Content information
	Filename    string
	ContentType string
	Host        string
	Encoding    []string
}

FileMetadata contains contextual information about an extracted file

type FileReassembler added in v0.9.0

type FileReassembler struct {
	// contains filtered or unexported fields
}

FileReassembler handles reassembly of files that may arrive out-of-order or with missing chunks (e.g., due to packet loss)

func NewFileReassembler added in v0.9.0

func NewFileReassembler(totalSize int64) *FileReassembler

NewFileReassembler creates a new file reassembler

func (*FileReassembler) AddChunk added in v0.9.0

func (fr *FileReassembler) AddChunk(offset int64, data []byte)

AddChunk adds a chunk of data at the specified offset

func (*FileReassembler) GetMissingBytes added in v0.9.0

func (fr *FileReassembler) GetMissingBytes() int64

GetMissingBytes returns the number of missing bytes

func (*FileReassembler) GetSeenBytes added in v0.9.0

func (fr *FileReassembler) GetSeenBytes() int64

GetSeenBytes returns the number of bytes received

func (*FileReassembler) IsComplete added in v0.9.0

func (fr *FileReassembler) IsComplete() bool

IsComplete checks if all bytes have been received and are contiguous

func (*FileReassembler) Reassemble added in v0.9.0

func (fr *FileReassembler) Reassemble(includeMissing bool) ([]byte, error)

Reassemble reconstructs the file from chunks If includeMissing is false, the function returns an error if data is missing If includeMissing is true, gaps will be filled with zeros (sparse file)

type HashAlgorithmsConfig added in v0.9.0

type HashAlgorithmsConfig struct {
	MD5    bool `yaml:"md5"`
	SHA1   bool `yaml:"sha1"`
	SHA256 bool `yaml:"sha256"`
}

HashAlgorithmsConfig defines which hash algorithms to compute

type IncompleteFilesConfig added in v0.9.0

type IncompleteFilesConfig struct {
	WriteIncomplete  bool   `yaml:"write_incomplete"`
	IncompletePrefix string `yaml:"incomplete_prefix"`
}

IncompleteFilesConfig defines handling of incomplete files

type MIMESignature added in v0.9.0

type MIMESignature struct {
	Magic  []byte
	Offset int
	MIME   string
	Ext    string
}

MIMESignature represents a file signature for magic number detection

type MimeTypesConfig added in v0.9.0

type MimeTypesConfig struct {
	Whitelist []string `yaml:"whitelist"`
	Blacklist []string `yaml:"blacklist"`
}

MimeTypesConfig defines MIME type filtering

type ProtocolsConfig added in v0.9.0

type ProtocolsConfig struct {
	HTTP bool `yaml:"http"`
	FTP  bool `yaml:"ftp"`
	SMTP bool `yaml:"smtp"`
	POP3 bool `yaml:"pop3"`
	IMAP bool `yaml:"imap"`
	SMB  bool `yaml:"smb"`
	IRC  bool `yaml:"irc"`
}

ProtocolsConfig defines which protocols have file extraction enabled

type ReassemblyConfig added in v0.9.0

type ReassemblyConfig struct {
	Enabled          bool  `yaml:"enabled"`
	AllowSparseFiles bool  `yaml:"allow_sparse_files"`
	MaxBufferSize    int64 `yaml:"max_buffer_size"`
}

ReassemblyConfig defines file reassembly settings

type SizeLimitsConfig added in v0.9.0

type SizeLimitsConfig struct {
	MaxFileSize         int64 `yaml:"max_file_size"`
	IncludeMissingBytes bool  `yaml:"include_missing_bytes"`
	MaxFilesPerSession  int   `yaml:"max_files_per_session"`
}

SizeLimitsConfig defines size limits for file extraction

type StorageConfig added in v0.9.0

type StorageConfig struct {
	OrganizeByMime      bool `yaml:"organize_by_mime"`
	OrganizeByProtocol  bool `yaml:"organize_by_protocol"`
	OrganizeByDate      bool `yaml:"organize_by_date"`
	CompressStoredFiles bool `yaml:"compress_stored_files"`
	IncludeConnectionID bool `yaml:"include_connection_id"`
}

StorageConfig defines how extracted files are organized

type StreamingHashWriter added in v0.9.0

type StreamingHashWriter struct {
	// contains filtered or unexported fields
}

StreamingHashWriter computes multiple hashes while writing to a file This avoids having to re-read the file contents after writing

func NewStreamingHashWriter added in v0.9.0

func NewStreamingHashWriter(file *os.File) *StreamingHashWriter

NewStreamingHashWriter creates a new streaming hash writer

func (*StreamingHashWriter) Close added in v0.9.0

func (w *StreamingHashWriter) Close() error

Close closes the underlying file

func (*StreamingHashWriter) GetHashes added in v0.9.0

func (w *StreamingHashWriter) GetHashes() FileHashes

GetHashes returns the computed hashes

func (*StreamingHashWriter) Write added in v0.9.0

func (w *StreamingHashWriter) Write(p []byte) (n int, err error)

Write implements io.Writer and computes hashes while writing

type YaraScanner added in v0.9.0

type YaraScanner struct {
	// contains filtered or unexported fields
}

YaraScanner provides thread-safe YARA rule scanning using yara-x.

func GetGlobalYaraScanner added in v0.9.0

func GetGlobalYaraScanner() *YaraScanner

GetGlobalYaraScanner returns the global YARA scanner, or nil if not initialized.

func InitGlobalYaraScanner added in v0.9.0

func InitGlobalYaraScanner(rulesDir string) (*YaraScanner, error)

InitGlobalYaraScanner initializes the global YARA scanner with the given rules directory. Safe to call multiple times; only the first call takes effect.

func NewYaraScanner added in v0.9.0

func NewYaraScanner(rulesDir string) (*YaraScanner, error)

NewYaraScanner creates a scanner by compiling all .yar/.yara files from rulesDir.

func (*YaraScanner) LastLoad added in v0.9.0

func (ys *YaraScanner) LastLoad() time.Time

LastLoad returns the time the rules were last compiled.

func (*YaraScanner) Reload added in v0.9.0

func (ys *YaraScanner) Reload() error

Reload recompiles all enabled YARA rules from the rules directory.

func (*YaraScanner) RuleCount added in v0.9.0

func (ys *YaraScanner) RuleCount() int

RuleCount returns the number of compiled rules, or 0 if none loaded.

func (*YaraScanner) RulesDir added in v0.9.0

func (ys *YaraScanner) RulesDir() string

RulesDir returns the configured rules directory.

func (*YaraScanner) ScanBytes added in v0.9.0

func (ys *YaraScanner) ScanBytes(data []byte) ([]string, error)

ScanBytes scans byte content against loaded YARA rules and returns matched rule identifiers.

func (*YaraScanner) ScanFile added in v0.9.0

func (ys *YaraScanner) ScanFile(path string) ([]string, error)

ScanFile scans a file on disk against loaded YARA rules.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL