importer

package
v5.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 22, 2025 License: GPL-3.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const ZeekDateTimeFmt = "2006-01-02-15-04-05"

ZeekDateTimeFmt is the common format for zeek header datetimes

Variables

View Source 
var ErrAllFilesPreviouslyImported = errors.New("all files were previously imported")

Functions

This section is empty.

Types

type ConnEntry 

type ConnEntry struct {
	ImportTime           time.Time        `ch:"import_time"`
	ZeekUID              util.FixedString `ch:"zeek_uid"`
	ImportID             util.FixedString `ch:"import_id"`
	Filtered             bool             `ch:"filtered"`
	Hash                 util.FixedString `ch:"hash"`
	Timestamp            time.Time        `ch:"ts"`
	Src                  net.IP           `ch:"src"`
	Dst                  net.IP           `ch:"dst"`
	SrcNUID              uuid.UUID        `ch:"src_nuid"`
	DstNUID              uuid.UUID        `ch:"dst_nuid"`
	SrcPort              uint32           `ch:"src_port"`
	DstPort              uint32           `ch:"dst_port"`
	MissingHostHeader    bool             `ch:"missing_host_header"`    // used to mark HTTP entries that have a missing host header
	MissingHostUseragent string           `ch:"missing_host_useragent"` // useragent for connections that have a missing host header
	Proto                string           `ch:"proto"`
	Service              string           `ch:"service"`
	Duration             float64          `ch:"duration"`
	SrcLocal             bool             `ch:"src_local"`
	DstLocal             bool             `ch:"dst_local"`
	ICMPType             int64            `ch:"icmp_type"`
	ICMPCode             int64            `ch:"icmp_code"`
	SrcBytes             uint64           `ch:"src_bytes"`
	DstBytes             uint64           `ch:"dst_bytes"`
	SrcIPBytes           uint64           `ch:"src_ip_bytes"`
	DstIPBytes           uint64           `ch:"dst_ip_bytes"`
	SrcPackets           uint64           `ch:"src_packets"`
	DstPackets           uint64           `ch:"dst_packets"`
	ConnState            string           `ch:"conn_state"`
	MissedBytes          uint64           `ch:"missed_bytes"`
	ZeekHistory          string           `ch:"zeek_history"`
}

type DNSEntry 

type DNSEntry struct {
	ImportTime          time.Time        `ch:"import_time"`
	ZeekUID             util.FixedString `ch:"zeek_uid"`
	Hash                util.FixedString `ch:"hash"`
	Timestamp           time.Time        `ch:"ts"`
	Src                 net.IP           `ch:"src"`
	Dst                 net.IP           `ch:"dst"`
	SrcNUID             uuid.UUID        `ch:"src_nuid"`
	DstNUID             uuid.UUID        `ch:"dst_nuid"`
	SrcPort             uint16           `ch:"src_port"`
	DstPort             uint16           `ch:"dst_port"`
	SrcLocal            bool             `ch:"src_local"`
	DstLocal            bool             `ch:"dst_local"`
	TransactionID       uint16           `ch:"transaction_id"`
	RoundTripTime       float64          `ch:"round_trip_time"`
	Query               string           `ch:"query"`
	QueryClassCode      uint16           `ch:"query_class_code"`
	QueryClassName      string           `ch:"query_class_name"`
	QueryTypeCode       uint16           `ch:"query_type_code"`
	QueryTypeName       string           `ch:"query_type_name"`
	ResponseCode        uint16           `ch:"response_code"`
	ResponseCodeName    string           `ch:"response_code_name"`
	AuthoritativeAnswer bool             `ch:"authoritative_answer"`
	RecursionDesired    bool             `ch:"recursion_desired"`
	RecursionAvailable  bool             `ch:"recursion_available"`
	Z                   uint16           `ch:"z"`
	Answers             []string         `ch:"answers"`
	TTLs                []float64        `ch:"ttls"`
	Rejected            bool             `ch:"rejected"`
	// PDNS field
	ResolvedIP net.IP `ch:"resolved_ip"`
}

type DoneChans 

type DoneChans struct {
	// contains filtered or unexported fields
}

type EntryChans 

type EntryChans struct {
	Conn     chan zeektypes.Conn
	OpenConn chan zeektypes.Conn
	DNS      chan zeektypes.DNS
	HTTP     chan zeektypes.HTTP
	OpenHTTP chan zeektypes.HTTP
	SSL      chan zeektypes.SSL
	OpenSSL  chan zeektypes.SSL
}

type HTTPEntry 

type HTTPEntry struct {
	ImportTime   time.Time        `ch:"import_time"`
	ZeekUID      util.FixedString `ch:"zeek_uid"`
	Hash         util.FixedString `ch:"hash"`
	Timestamp    time.Time        `ch:"ts"`
	Src          net.IP           `ch:"src"`
	Dst          net.IP           `ch:"dst"`
	SrcNUID      uuid.UUID        `ch:"src_nuid"`
	DstNUID      uuid.UUID        `ch:"dst_nuid"`
	MultiRequest bool             `ch:"multi_request"`
	SrcPort      uint32           `ch:"src_port"`
	DstPort      uint32           `ch:"dst_port"`
	Duration     float64          `ch:"duration"`
	SrcLocal     bool             `ch:"src_local"`
	DstLocal     bool             `ch:"dst_local"`
	SrcBytes     uint64           `ch:"src_bytes"`
	DstBytes     uint64           `ch:"dst_bytes"`
	SrcIPBytes   uint64           `ch:"src_ip_bytes"`
	DstIPBytes   uint64           `ch:"dst_ip_bytes"`
	SrcPackets   uint64           `ch:"src_packets"`
	DstPackets   uint64           `ch:"dst_packets"`
	Proto        string           `ch:"proto"`
	Service      string           `ch:"service"`
	ConnState    string           `ch:"conn_state"`
	TransDepth   uint64           `ch:"trans_depth"`
	Method       string           `ch:"method"`
	Host         string           `ch:"host"`
	URI          string           `ch:"uri"`
	Referrer     string           `ch:"referrer"`
	HTTPVersion  string           `ch:"http_version"`
	UserAgent    string           `ch:"useragent"`
	Origin       string           `ch:"origin"`
	StatusCode   uint64           `ch:"status_code"`
	StatusMsg    string           `ch:"status_msg"`
	InfoCode     uint64           `ch:"info_code"`
	InfoMsg      string           `ch:"info_msg"`
	Username     string           `ch:"username"`
	Password     string           `ch:"password"`
	SrcFUIDs     []string         `ch:"src_fuids"`
	SrcFileNames []string         `ch:"src_file_names"`
	SrcMIMETypes []string         `ch:"src_mime_types"`
	DstFUIDs     []string         `ch:"dst_fuids"`
	DstFileNames []string         `ch:"dst_file_names"`
	DstMIMETypes []string         `ch:"dst_mime_types"`
}

type Importer 

type Importer struct {
	Cfg               *config.Config
	Database          *database.DB
	ImportID          util.FixedString
	LogDirectory      string
	FileMap           map[string][]string
	EntryChannels     EntryChans
	MetaDBChannel     chan MetaDBFile
	Paths             chan string
	ErrChannel        chan error
	TotalFileCount    int
	DoneChannels      DoneChans
	Writers           writers
	WriteLimiter      *rate.Limiter
	ProgressBar       *mpb.Progress
	FileProgressBar   *mpb.Bar
	ProgressLogger    *log.Logger
	HTTPLinkMutex     sync.Mutex
	OpenHTTPLinkMutex sync.Mutex
	NumParsers        int
	NumDigesters      int
	NumWriters        int
	ResultCounts      ResultCounts
	// contains filtered or unexported fields
}

func NewImporter 

func NewImporter(db *database.DB, cfg *config.Config, importStartedAt time.Time, numDigesters int, numParsers int, numWriters int) (*Importer, error)

NewImporter creates and returns a new Importer object

func (*Importer) Import 

func (importer *Importer) Import(afs afero.Fs, files map[string][]string) error

type MetaDBFile 

type MetaDBFile struct {
	// contains filtered or unexported fields
}

type ResultCounts 

type ResultCounts struct {
	ZeekUIDs       uint64
	OpenZeekUIDs   uint64
	UnfilteredConn uint64
	Conn           uint64
	OpenConn       uint64
	HTTP           uint64
	OpenHTTP       uint64
	DNS            uint64
	UDNS           int64
	PDNSRaw        uint64
	SSL            uint64
	OpenSSL        uint64
}

type SSLEntry 

type SSLEntry struct {
	ImportTime       time.Time        `ch:"import_time"`
	ZeekUID          util.FixedString `ch:"zeek_uid"`
	Hash             util.FixedString `ch:"hash"`
	Timestamp        time.Time        `ch:"ts"`
	Src              net.IP           `ch:"src"`
	Dst              net.IP           `ch:"dst"`
	SrcNUID          uuid.UUID        `ch:"src_nuid"`
	DstNUID          uuid.UUID        `ch:"dst_nuid"`
	SrcPort          uint32           `ch:"src_port"`
	DstPort          uint32           `ch:"dst_port"`
	Duration         float64          `ch:"duration"`
	SrcLocal         bool             `ch:"src_local"`
	DstLocal         bool             `ch:"dst_local"`
	SrcBytes         uint64           `ch:"src_bytes"`
	DstBytes         uint64           `ch:"dst_bytes"`
	SrcIPBytes       uint64           `ch:"src_ip_bytes"`
	DstIPBytes       uint64           `ch:"dst_ip_bytes"`
	SrcPackets       uint64           `ch:"src_packets"`
	DstPackets       uint64           `ch:"dst_packets"`
	Proto            string           `ch:"proto"`
	Service          string           `ch:"service"`
	ConnState        string           `ch:"conn_state"`
	Version          string           `ch:"version"`
	Cipher           string           `ch:"cipher"`
	Curve            string           `ch:"curve"`
	ServerName       string           `ch:"server_name"`
	Resumed          bool             `ch:"resumed"`
	NextProtocol     string           `ch:"next_protocol"`
	Established      bool             `ch:"established"`
	ServerCertFUIDs  []string         `ch:"server_cert_fuids"`
	ClientCertFUIDs  []string         `ch:"client_cert_fuids"`
	ServerSubject    string           `ch:"server_subject"`
	ServerIssuer     string           `ch:"server_issuer"`
	ClientSubject    string           `ch:"client_subject"`
	ClientIssuer     string           `ch:"client_issuer"`
	ValidationStatus string           `ch:"validation_status"`
	JA3              string           `ch:"ja3"`
	JA3S             string           `ch:"ja3s"`
}

type UniqueConn 

type UniqueConn struct {
	Hash      util.FixedString `ch:"hash"`
	Src       net.IP           `ch:"src"`
	Dst       net.IP           `ch:"dst"`
	SrcNUID   uuid.UUID        `ch:"src_nuid"`
	DstNUID   uuid.UUID        `ch:"dst_nuid"`
	ConnCount uint64
	ConnType  string
}

type UniqueFQDN 

type UniqueFQDN struct {
	Hash    util.FixedString `ch:"hash"`
	Src     net.IP           `ch:"src"`
	Dst     net.IP           `ch:"dst"`
	FQDN    string           `ch:"fqdn"`
	SrcNUID uuid.UUID        `ch:"src_nuid"`
	DstNUID uuid.UUID        `ch:"dst_nuid"`
}

type WaitGroups 

type WaitGroups struct {
	Digester sync.WaitGroup
	MetaDB   sync.WaitGroup
	OpenConn sync.WaitGroup
	Conn     sync.WaitGroup
	DNS      sync.WaitGroup
	HTTP     sync.WaitGroup
	OpenHTTP sync.WaitGroup
	SSL      sync.WaitGroup
	OpenSSL  sync.WaitGroup
}

type ZeekHeader 

type ZeekHeader[Z zeekRecord] struct {
	// contains filtered or unexported fields
}

ZeekHeader stores vars in the header of the zeek log

type ZeekUIDRecord 

type ZeekUIDRecord struct {
	UID               util.FixedString
	Timestamp         time.Time
	UsedByFQDNBeacon  bool
	UsedByDNS         bool
	LinkedToHTTPEntry bool
	NumUsedByHTTP     int
	Duration          float64
	SrcBytes          uint64
	DstBytes          uint64
	SrcIPBytes        uint64
	DstIPBytes        uint64
	SrcPackets        uint64
	DstPackets        uint64
	ConnState         string
	Proto             string
	Service           string
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.