Documentation
¶
Index ¶
- Constants
- Variables
- func CheckCompressionBlock(block CarvedBlock) (bool, error)
- func CheckCompressionRaw(data []byte) bool
- func GenCarveName() string
- func GenCarveQuery(file string, glob bool) string
- func GenerateArchiveName(carve CarvedFile) string
- func GenerateS3Archive(bucket, env, uuid, sessionid, path string) string
- func GenerateS3Data(bucket, env, uuid, sessionid string, blockid int) string
- func GenerateS3File(env, uuid, sessionid, path string) string
- func GenerateS3Key(env, uuid, sessionid string, blockid int) string
- func S3URLtoKey(s3url, bucket string) string
- type CarveResult
- type CarveSample
- type CarveSampleCategory
- type CarveSamplePlatform
- type CarvedBlock
- type CarvedFile
- type CarverS3
- func (carveS3 *CarverS3) Archive(carve CarvedFile, blocks []CarvedBlock) (*CarveResult, error)
- func (carveS3 *CarverS3) Concatenate(key string, destKey string, part int, uploadid *string) (*string, error)
- func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error)
- func (carveS3 *CarverS3) GetDownloadLink(carve CarvedFile) (string, error)
- func (carveS3 *CarverS3) Settings(mgr *settings.Settings)
- func (carveS3 *CarverS3) Upload(block CarvedBlock, uuid, data string) error
- type CarverType
- type Carves
- func (c *Carves) Archive(sessionid, destPath string) (*CarveResult, error)
- func (c *Carves) ArchiveCarve(sessionid, archive string) error
- func (c *Carves) ArchiveLocal(destPath string, carve CarvedFile, blocks []CarvedBlock) (*CarveResult, error)
- func (c *Carves) ChangeStatus(status, sessionid string) error
- func (c *Carves) CheckCarve(sessionid, requestid string) bool
- func (c *Carves) CompleteBlock(sessionid string) error
- func (c *Carves) Completed(sessionid string) bool
- func (c *Carves) CreateBlock(block CarvedBlock, uuid, data string) error
- func (c *Carves) CreateCarve(carve CarvedFile) error
- func (c *Carves) Delete(carveid string) error
- func (c *Carves) DeleteBlocks(sessionid string) error
- func (c *Carves) GetBlocks(sessionid string) ([]CarvedBlock, error)
- func (c *Carves) GetByCarve(carveid string) (CarvedFile, error)
- func (c *Carves) GetByEnv(env uint) ([]CarvedFile, error)
- func (c *Carves) GetByQuery(name string, env uint) ([]CarvedFile, error)
- func (c *Carves) GetByRequest(requestid string) ([]CarvedFile, error)
- func (c *Carves) GetBySession(sessionid string) (CarvedFile, error)
- func (c *Carves) GetCheckCarve(sessionid, requestid string) (CarvedFile, error)
- func (c *Carves) GetNodeCarveBucketed(uuid string, since time.Time, bucketSeconds int) ([]dbutil.BucketedRow, error)
- func (c *Carves) GetNodeCarveTimestamps(uuid string, since time.Time) ([]time.Time, error)
- func (c *Carves) GetNodeCarves(uuid string) ([]CarvedFile, error)
- func (c *Carves) InitCarve(req types.CarveInitRequest, sessionid string) error
- func (c *Carves) InitateBlock(env, uuid, requestid, sessionid, data string, blockid int, envid uint) CarvedBlock
- type MappedCarves
- type QueriedCarve
Constants ¶
const ( // StatusQueried for queried carves that did not hit nodes yet StatusQueried string = "QUERIED" // StatusScheduled for initialized carves StatusScheduled string = "SCHEDULED" // StatusInProgress for carves that are on-going StatusInProgress string = "IN PROGRESS" // StatusCompleted for carves that finalized StatusCompleted string = "COMPLETED" // TarFileExtension to identify Tar files extension TarFileExtension string = ".tar" // ZstFileExtension to identify ZST compressed files ZstFileExtension string = ".zst" )
const ( // MaxUploadRetries to define how many times retry to upload MaxUploadRetries = 3 // MaxChunkSize to define max size for each part. AWS defines 5MB max per part MaxChunkSize = int64(5 * 1024 * 1024) // DownloadLinkExpiration in minutes to expire download links DownloadLinkExpiration = 5 )
const ( // S3proto to be used as s3 URL S3proto = "s3://" // S3URL to format the s3 URL S3URL = S3proto + "%s/%s" // S3Key to format the s3 key for a block S3Key = "%s:%s:%s:%d" // S3File to format the s3 key for a reconstructed file S3File = "%s:%s:%s:%s" + TarFileExtension // LocalFile to format the local file name LocalFile = "%s_%s_%s" + TarFileExtension )
Variables ¶
var CarveSamples = []CarveSample{ { Label: "/etc/passwd", Path: "/etc/passwd", Platform: CarvePlatformLinux, Category: CarveCategoryAuth, Notes: "Local user account database (read by every getpwnam call).", }, { Label: "/etc/shadow", Path: "/etc/shadow", Platform: CarvePlatformLinux, Category: CarveCategoryAuth, Notes: "Hashed password store — root-readable only; presence in carve output confirms agent ran as root.", }, { Label: "/etc/sudoers", Path: "/etc/sudoers", Platform: CarvePlatformLinux, Category: CarveCategoryAuth, Notes: "Sudo privilege configuration. Compare across hosts to spot drift.", }, { Label: "/var/log/auth.log", Path: "/var/log/auth.log", Platform: CarvePlatformLinux, Category: CarveCategoryLogs, Notes: "SSH / sudo / PAM authentication events (Debian / Ubuntu).", }, { Label: "/var/log/secure", Path: "/var/log/secure", Platform: CarvePlatformLinux, Category: CarveCategoryLogs, Notes: "SSH / sudo / PAM authentication events (RHEL / CentOS / Fedora).", }, { Label: "/var/log/syslog", Path: "/var/log/syslog", Platform: CarvePlatformLinux, Category: CarveCategoryLogs, Notes: "General system messages; correlate with auth.log for a fuller timeline.", }, { Label: "/root/.bash_history", Path: "/root/.bash_history", Platform: CarvePlatformLinux, Category: CarveCategoryHistory, Notes: "Root shell command history — first thing to grab on suspected compromise.", }, { Label: "/etc/crontab", Path: "/etc/crontab", Platform: CarvePlatformLinux, Category: CarveCategoryConfig, Notes: "System-wide cron schedule. Check for unfamiliar entries.", }, { Label: "/etc/hosts", Path: "/etc/hosts", Platform: CarvePlatformLinux, Category: CarveCategoryConfig, Notes: "Local hostname overrides. Tampered entries can redirect traffic.", }, { Label: "/etc/passwd", Path: "/etc/passwd", Platform: CarvePlatformDarwin, Category: CarveCategoryAuth, Notes: "Local user account database (legacy; macOS primarily uses OpenDirectory).", }, { Label: "/var/db/dslocal/nodes/Default/users", Path: "/var/db/dslocal/nodes/Default/users", Platform: CarvePlatformDarwin, Category: CarveCategoryAuth, Notes: "Local user records in OpenDirectory (plist files; carve the directory).", }, { Label: "~/Library/Keychains", Path: "/Users", Platform: CarvePlatformDarwin, Category: CarveCategoryKeychain, Notes: "User keychain directories. Carve a specific user's path: /Users/<user>/Library/Keychains.", }, { Label: "/var/log/system.log", Path: "/var/log/system.log", Platform: CarvePlatformDarwin, Category: CarveCategoryLogs, Notes: "Pre-unified-logging system messages.", }, { Label: "/var/log/install.log", Path: "/var/log/install.log", Platform: CarvePlatformDarwin, Category: CarveCategoryLogs, Notes: "Software install / update events — useful for spotting unexpected pkg installs.", }, { Label: "~/.zsh_history (root)", Path: "/var/root/.zsh_history", Platform: CarvePlatformDarwin, Category: CarveCategoryHistory, Notes: "Root zsh history. Adjust path for non-root users: /Users/<user>/.zsh_history.", }, { Label: "/etc/hosts", Path: "/etc/hosts", Platform: CarvePlatformDarwin, Category: CarveCategoryConfig, Notes: "Local hostname overrides.", }, { Label: `SAM hive`, Path: `C:\Windows\System32\config\SAM`, Platform: CarvePlatformWindows, Category: CarveCategoryRegistry, Notes: "Local account database hive. File is locked while Windows runs; carve from VSS shadow or live-running osquery as SYSTEM.", }, { Label: `SYSTEM hive`, Path: `C:\Windows\System32\config\SYSTEM`, Platform: CarvePlatformWindows, Category: CarveCategoryRegistry, Notes: "System configuration hive. Contains services, drivers, BootKey for SAM decryption.", }, { Label: `SECURITY hive`, Path: `C:\Windows\System32\config\SECURITY`, Platform: CarvePlatformWindows, Category: CarveCategoryRegistry, Notes: "Local security policy hive. Contains LSA secrets and cached domain credentials.", }, { Label: `Security event log`, Path: `C:\Windows\System32\winevt\Logs\Security.evtx`, Platform: CarvePlatformWindows, Category: CarveCategoryLogs, Notes: "Windows security audit log — logon events, privilege use, object access.", }, { Label: `System event log`, Path: `C:\Windows\System32\winevt\Logs\System.evtx`, Platform: CarvePlatformWindows, Category: CarveCategoryLogs, Notes: "System events — services, drivers, hardware. Pairs with Security.evtx for correlation.", }, { Label: `PowerShell op log`, Path: `C:\Windows\System32\winevt\Logs\Microsoft-Windows-PowerShell%4Operational.evtx`, Platform: CarvePlatformWindows, Category: CarveCategoryLogs, Notes: "PowerShell script-block and pipeline execution log. High-value for attacker activity.", }, { Label: `hosts file`, Path: `C:\Windows\System32\drivers\etc\hosts`, Platform: CarvePlatformWindows, Category: CarveCategoryConfig, Notes: "Local hostname overrides. Should rarely change in a managed fleet.", }, { Label: `NTUSER.DAT (per-user)`, Path: `C:\Users`, Platform: CarvePlatformWindows, Category: CarveCategoryConfig, Notes: "Per-user registry hive. Carve a specific user: C:\\Users\\<user>\\NTUSER.DAT (locked while user is logged in).", }, }
CarveSamples is the canonical starter library. ~24 entries across the three major platforms. Ordering is by platform then category so the SPA's template row reads in a predictable shape.
var ( // CompressionHeader to detect the usage of compressed carves (zstd header) // https://github.com/facebook/zstd CompressionHeader = []byte{0x28, 0xb5, 0x2f, 0xfd} )
Functions ¶
func CheckCompressionBlock ¶
func CheckCompressionBlock(block CarvedBlock) (bool, error)
Function to check if a block data is compressed using zstd https://github.com/facebook/zstd
func CheckCompressionRaw ¶
Function to check if data is compressed using zstd https://github.com/facebook/zstd
Returns false on inputs shorter than the 4-byte zstd magic — callers may feed arbitrary node-controlled bytes, so the length check is the only thing preventing a slice-bounds panic that would crash the service.
func GenCarveQuery ¶
GenCarveQuery builds the osquery SQL that selects matching `carves` rows on every targeted node. The carve `file` is treated as an untrusted SQL string literal: single quotes are doubled so a CarveLevel operator cannot break out of the literal to pivot from "carve this path" into arbitrary SELECTs (e.g. `'; SELECT 1; --`).
In glob mode `*` and `?` map to LIKE wildcards `%` and `_`, with any pre-existing `%`, `_`, or `\` in the path escaped via `ESCAPE '\'` so they are treated as literals.
Paths containing spaces (e.g. `C:\Program Files\...`, `/Library/Application Support/...`) and any UTF-8 characters are supported.
func GenerateArchiveName ¶
func GenerateArchiveName(carve CarvedFile) string
Function to generate a local file for carve archives
func GenerateS3Archive ¶
Function to generate a carve archived filename for s3
func GenerateS3Data ¶
Function to generate a carve block filename for s3
func GenerateS3File ¶
Function to generate the s3 file reconstructed from blocks
func GenerateS3Key ¶
Function to generate the s3 key for a carve block
func S3URLtoKey ¶
Function to translate from a s3:// URL to just the key
Types ¶
type CarveResult ¶
CarveResult holds metadata related to a carve
type CarveSample ¶ added in v0.5.2
type CarveSample struct {
Label string `json:"label"`
Path string `json:"path"`
Platform CarveSamplePlatform `json:"platform"`
Category CarveSampleCategory `json:"category"`
// Notes is a brief operator-facing description of why this file is
// worth grabbing during an investigation. Surfaced as a tooltip in
// the SPA template row.
Notes string `json:"notes"`
}
CarveSample is one starter target row.
type CarveSampleCategory ¶ added in v0.5.2
type CarveSampleCategory string
CarveSampleCategory groups paths so the SPA can label them for the operator (Auth / Logs / Registry / etc). Closed set; new categories require updating the SPA's label map too.
const ( CarveCategoryAuth CarveSampleCategory = "auth" CarveCategoryLogs CarveSampleCategory = "logs" CarveCategoryRegistry CarveSampleCategory = "registry" CarveCategoryKeychain CarveSampleCategory = "keychain" CarveCategoryHistory CarveSampleCategory = "history" CarveCategoryConfig CarveSampleCategory = "config" )
type CarveSamplePlatform ¶ added in v0.5.2
type CarveSamplePlatform string
CarveSamplePlatform — aligns with the platform buckets used elsewhere in osctrl. Each sample is single-platform because file paths are platform-specific by definition.
const ( CarvePlatformLinux CarveSamplePlatform = "linux" CarvePlatformDarwin CarveSamplePlatform = "darwin" CarvePlatformWindows CarveSamplePlatform = "windows" )
type CarvedBlock ¶
type CarvedBlock struct {
gorm.Model
RequestID string `gorm:"index"`
SessionID string `gorm:"index"`
Environment string
BlockID int
Data string
Size int
Carver string
EnvironmentID uint
}
CarvedBlock to store each block from a carve
type CarvedFile ¶
type CarvedFile struct {
gorm.Model
CarveID string `gorm:"unique;index"`
RequestID string
SessionID string
QueryName string
UUID string `gorm:"index"`
NodeID uint
Environment string
Path string
CarveSize int
BlockSize int
TotalBlocks int
CompletedBlocks int
Status string
CompletedAt time.Time
Carver string
Archived bool
ArchivePath string
EnvironmentID uint
}
CarvedFile to keep track of carved files from nodes
type CarverS3 ¶
type CarverS3 struct {
S3Config osctrl_config.S3Carver
AWSConfig aws.Config
Client *s3.Client
Enabled bool
Debug bool
}
CarverS3 will be used to carve files using S3 as destination
func CreateCarverS3 ¶
func CreateCarverS3(s3Config osctrl_config.S3Carver) (*CarverS3, error)
CreateCarverS3 to initialize the carver
func (*CarverS3) Archive ¶
func (carveS3 *CarverS3) Archive(carve CarvedFile, blocks []CarvedBlock) (*CarveResult, error)
Archive - Function to convert finalize a completed carve and create a file ready to download
func (*CarverS3) Concatenate ¶
func (carveS3 *CarverS3) Concatenate(key string, destKey string, part int, uploadid *string) (*string, error)
Concatenate - Function to concatenate a file that have been already uploaded in s3
func (*CarverS3) Download ¶
func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error)
Download - Function to download an archived carve from s3
func (*CarverS3) GetDownloadLink ¶
func (carveS3 *CarverS3) GetDownloadLink(carve CarvedFile) (string, error)
GetDownloadLink - Function to generate a pre-signed link to download directly from s3
type Carves ¶
Carves to handle file carves from nodes
func CreateFileCarves ¶
CreateFileCarves to initialize the carves struct and tables
func (*Carves) Archive ¶
func (c *Carves) Archive(sessionid, destPath string) (*CarveResult, error)
Archive to convert finalize a completed carve and create a file ready to download
func (*Carves) ArchiveCarve ¶
ArchiveCarve to mark one carve as archived and set the received file
func (*Carves) ArchiveLocal ¶
func (c *Carves) ArchiveLocal(destPath string, carve CarvedFile, blocks []CarvedBlock) (*CarveResult, error)
Archive to convert finalize a completed carve and create a file ready to download
func (*Carves) ChangeStatus ¶
ChangeStatus to change the status of a carve
func (*Carves) CheckCarve ¶
CheckCarve to verify a session belong to a carve
func (*Carves) CompleteBlock ¶
CompleteBlock to increase one block for a carve
func (*Carves) CreateBlock ¶
func (c *Carves) CreateBlock(block CarvedBlock, uuid, data string) error
CreateBlock to create a new block for a carve
func (*Carves) CreateCarve ¶
func (c *Carves) CreateCarve(carve CarvedFile) error
CreateCarve to create a new carved file for a node
func (*Carves) DeleteBlocks ¶
DeleteBlocks to delete all blocks by session id
func (*Carves) GetBlocks ¶
func (c *Carves) GetBlocks(sessionid string) ([]CarvedBlock, error)
GetBlocks to get a carve by session_id and ordered by block_id
func (*Carves) GetByCarve ¶
func (c *Carves) GetByCarve(carveid string) (CarvedFile, error)
GetByCarve to get a carve by carve id
func (*Carves) GetByEnv ¶
func (c *Carves) GetByEnv(env uint) ([]CarvedFile, error)
GetByEnv to get carves by environment
func (*Carves) GetByQuery ¶
func (c *Carves) GetByQuery(name string, env uint) ([]CarvedFile, error)
GetByQuery to get a carve by query name
func (*Carves) GetByRequest ¶
func (c *Carves) GetByRequest(requestid string) ([]CarvedFile, error)
GetByRequest to get a carve by request_id
func (*Carves) GetBySession ¶
func (c *Carves) GetBySession(sessionid string) (CarvedFile, error)
GetBySession to get a carve by session_id
func (*Carves) GetCheckCarve ¶
func (c *Carves) GetCheckCarve(sessionid, requestid string) (CarvedFile, error)
GetCarve to verify a session belong to a carve
func (*Carves) GetNodeCarveBucketed ¶ added in v0.5.2
func (c *Carves) GetNodeCarveBucketed(uuid string, since time.Time, bucketSeconds int) ([]dbutil.BucketedRow, error)
GetNodeCarveBucketed returns per-bucket row counts for carved_files rows produced by `uuid`. Same bucketing semantics as the logging-package variants — see pkg/dbutil.BucketExpr.
func (*Carves) GetNodeCarveTimestamps ¶ added in v0.5.2
GetNodeCarveTimestamps returns CreatedAt of every CarvedFile row from this node since the cutoff. Used by the per-node activity heatmap so it can bucket without dragging the full carve metadata.
func (*Carves) GetNodeCarves ¶
func (c *Carves) GetNodeCarves(uuid string) ([]CarvedFile, error)
GetNodeCarves to get all the carves for a given node
func (*Carves) InitCarve ¶
func (c *Carves) InitCarve(req types.CarveInitRequest, sessionid string) error
InitCarve to initialize an scheduled carve
func (*Carves) InitateBlock ¶
func (c *Carves) InitateBlock(env, uuid, requestid, sessionid, data string, blockid int, envid uint) CarvedBlock
InitateBlock to initiate a block based on the configured carver
type MappedCarves ¶
type MappedCarves map[string][]CarvedFile
MappedCarves to pass carves by query name / Request ID