importer

package
v5.0.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 10, 2024 License: GPL-3.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source 
const ConnPrefix = "conn"
View Source 
const ConnSummaryPrefixHyphen = "conn-summary"
View Source 
const ConnSummaryPrefixUnderscore = "conn_summary"
View Source 
const DNSPrefix = "dns"
View Source 
const HTTPPrefix = "http"
View Source 
const OpenConnPrefix = "open_conn"
View Source 
const OpenHTTPPrefix = "open_http"
View Source 
const OpenSSLPrefix = "open_ssl"
View Source 
const SSLPrefix = "ssl"
View Source 
const ZeekDateTimeFmt = "2006-01-02-15-04-05"

ZeekDateTimeFmt is the common format for zeek header datetimes

Variables

View Source 
var ErrAllFilesPreviouslyImported = errors.New("all files were previously imported")

Functions

This section is empty.

Types

type ConnEntry 

type ConnEntry struct {
	ImportTime           time.Time        `ch:"import_time"`
	ZeekUID              util.FixedString `ch:"zeek_uid"`
	ImportID             util.FixedString `ch:"import_id"`
	Filtered             bool             `ch:"filtered"`
	Hash                 util.FixedString `ch:"hash"`
	Timestamp            time.Time        `ch:"ts"`
	Src                  net.IP           `ch:"src"`
	Dst                  net.IP           `ch:"dst"`
	SrcNUID              uuid.UUID        `ch:"src_nuid"`
	DstNUID              uuid.UUID        `ch:"dst_nuid"`
	SrcPort              uint16           `ch:"src_port"`
	DstPort              uint16           `ch:"dst_port"`
	MissingHostHeader    bool             `ch:"missing_host_header"`    // used to mark HTTP entries that have a missing host header
	MissingHostUseragent string           `ch:"missing_host_useragent"` // useragent for connections that have a missing host header
	Proto                string           `ch:"proto"`
	Service              string           `ch:"service"`
	Duration             float64          `ch:"duration"`
	SrcLocal             bool             `ch:"src_local"`
	DstLocal             bool             `ch:"dst_local"`
	ICMPType             int              `ch:"icmp_type"`
	ICMPCode             int              `ch:"icmp_code"`
	SrcBytes             int64            `ch:"src_bytes"`
	DstBytes             int64            `ch:"dst_bytes"`
	SrcIPBytes           int64            `ch:"src_ip_bytes"`
	DstIPBytes           int64            `ch:"dst_ip_bytes"`
	SrcPackets           int64            `ch:"src_packets"`
	DstPackets           int64            `ch:"dst_packets"`
	ConnState            string           `ch:"conn_state"`
	MissedBytes          int64            `ch:"missed_bytes"`
	ZeekHistory          string           `ch:"zeek_history"`
}

type DNSEntry 

type DNSEntry struct {
	ImportTime          time.Time        `ch:"import_time"`
	ZeekUID             util.FixedString `ch:"zeek_uid"`
	Hash                util.FixedString `ch:"hash"`
	Timestamp           time.Time        `ch:"ts"`
	Src                 net.IP           `ch:"src"`
	Dst                 net.IP           `ch:"dst"`
	SrcNUID             uuid.UUID        `ch:"src_nuid"`
	DstNUID             uuid.UUID        `ch:"dst_nuid"`
	SrcPort             uint16           `ch:"src_port"`
	DstPort             uint16           `ch:"dst_port"`
	SrcLocal            bool             `ch:"src_local"`
	DstLocal            bool             `ch:"dst_local"`
	TransactionID       uint16           `ch:"transaction_id"`
	RoundTripTime       float64          `ch:"round_trip_time"`
	Query               string           `ch:"query"`
	QueryClassCode      uint16           `ch:"query_class_code"`
	QueryClassName      string           `ch:"query_class_name"`
	QueryTypeCode       uint16           `ch:"query_type_code"`
	QueryTypeName       string           `ch:"query_type_name"`
	ResponseCode        uint16           `ch:"response_code"`
	ResponseCodeName    string           `ch:"response_code_name"`
	AuthoritativeAnswer bool             `ch:"authoritative_answer"`
	RecursionDesired    bool             `ch:"recursion_desired"`
	RecursionAvailable  bool             `ch:"recursion_available"`
	Z                   uint16           `ch:"z"`
	Answers             []string         `ch:"answers"`
	TTLs                []float64        `ch:"ttls"`
	Rejected            bool             `ch:"rejected"`
	// PDNS field
	ResolvedIP net.IP `ch:"resolved_ip"`
}

type DoneChans 

type DoneChans struct {
	// contains filtered or unexported fields
}

type EntryChans 

type EntryChans struct {
	Conn     chan zeektypes.Conn
	OpenConn chan zeektypes.Conn
	DNS      chan zeektypes.DNS
	HTTP     chan zeektypes.HTTP
	OpenHTTP chan zeektypes.HTTP
	SSL      chan zeektypes.SSL
	OpenSSL  chan zeektypes.SSL
}

type HTTPEntry 

type HTTPEntry struct {
	ImportTime   time.Time        `ch:"import_time"`
	ZeekUID      util.FixedString `ch:"zeek_uid"`
	Hash         util.FixedString `ch:"hash"`
	Timestamp    time.Time        `ch:"ts"`
	Src          net.IP           `ch:"src"`
	Dst          net.IP           `ch:"dst"`
	SrcNUID      uuid.UUID        `ch:"src_nuid"`
	DstNUID      uuid.UUID        `ch:"dst_nuid"`
	MultiRequest bool             `ch:"multi_request"`
	SrcPort      uint16           `ch:"src_port"`
	DstPort      uint16           `ch:"dst_port"`
	Duration     float64          `ch:"duration"`
	SrcLocal     bool             `ch:"src_local"`
	DstLocal     bool             `ch:"dst_local"`
	SrcBytes     int64            `ch:"src_bytes"`
	DstBytes     int64            `ch:"dst_bytes"`
	SrcIPBytes   int64            `ch:"src_ip_bytes"`
	DstIPBytes   int64            `ch:"dst_ip_bytes"`
	SrcPackets   int64            `ch:"src_packets"`
	DstPackets   int64            `ch:"dst_packets"`
	Proto        string           `ch:"proto"`
	Service      string           `ch:"service"`
	ConnState    string           `ch:"conn_state"`
	TransDepth   uint16           `ch:"trans_depth"`
	Method       string           `ch:"method"`
	Host         string           `ch:"host"`
	URI          string           `ch:"uri"`
	Referrer     string           `ch:"referrer"`
	HTTPVersion  string           `ch:"http_version"`
	UserAgent    string           `ch:"useragent"`
	Origin       string           `ch:"origin"`
	StatusCode   int64            `ch:"status_code"`
	StatusMsg    string           `ch:"status_msg"`
	InfoCode     int64            `ch:"info_code"`
	InfoMsg      string           `ch:"info_msg"`
	Username     string           `ch:"username"`
	Password     string           `ch:"password"`
	SrcFUIDs     []string         `ch:"src_fuids"`
	SrcFileNames []string         `ch:"src_file_names"`
	SrcMIMETypes []string         `ch:"src_mime_types"`
	DstFUIDs     []string         `ch:"dst_fuids"`
	DstFileNames []string         `ch:"dst_file_names"`
	DstMIMETypes []string         `ch:"dst_mime_types"`
}

type Importer 

type Importer struct {
	Database          *database.DB
	ImportID          util.FixedString
	LogDirectory      string
	FileMap           map[string][]string
	EntryChannels     EntryChans
	MetaDBChannel     chan MetaDBFile
	Paths             chan string
	ErrChannel        chan error
	TotalFileCount    int
	DoneChannels      DoneChans
	Writers           writers
	WriteLimiter      *rate.Limiter
	ProgressBar       *mpb.Progress
	FileProgressBar   *mpb.Bar
	ProgressLogger    *log.Logger
	HTTPLinkMutex     sync.Mutex
	OpenHTTPLinkMutex sync.Mutex
	NumParsers        int
	NumDigesters      int
	NumWriters        int
	ResultCounts      ResultCounts
	// contains filtered or unexported fields
}

func NewImporter 

func NewImporter(db *database.DB, cfg *config.Config, importStartedAt time.Time, numDigesters int, numParsers int, numWriters int) (*Importer, error)

NewImporter creates and returns a new Importer object

func (*Importer) Import 

func (importer *Importer) Import(afs afero.Fs, files map[string][]string) error

type MetaDBFile 

type MetaDBFile struct {
	// contains filtered or unexported fields
}

type ResultCounts 

type ResultCounts struct {
	ZeekUIDs       uint64
	OpenZeekUIDs   uint64
	UnfilteredConn uint64
	Conn           uint64
	OpenConn       uint64
	HTTP           uint64
	OpenHTTP       uint64
	DNS            uint64
	UDNS           int64
	PDNSRaw        uint64
	SSL            uint64
	OpenSSL        uint64
}

type SSLEntry 

type SSLEntry struct {
	ImportTime       time.Time        `ch:"import_time"`
	ZeekUID          util.FixedString `ch:"zeek_uid"`
	Hash             util.FixedString `ch:"hash"`
	Timestamp        time.Time        `ch:"ts"`
	Src              net.IP           `ch:"src"`
	Dst              net.IP           `ch:"dst"`
	SrcNUID          uuid.UUID        `ch:"src_nuid"`
	DstNUID          uuid.UUID        `ch:"dst_nuid"`
	SrcPort          uint16           `ch:"src_port"`
	DstPort          uint16           `ch:"dst_port"`
	Duration         float64          `ch:"duration"`
	SrcLocal         bool             `ch:"src_local"`
	DstLocal         bool             `ch:"dst_local"`
	SrcBytes         int64            `ch:"src_bytes"`
	DstBytes         int64            `ch:"dst_bytes"`
	SrcIPBytes       int64            `ch:"src_ip_bytes"`
	DstIPBytes       int64            `ch:"dst_ip_bytes"`
	SrcPackets       int64            `ch:"src_packets"`
	DstPackets       int64            `ch:"dst_packets"`
	Proto            string           `ch:"proto"`
	Service          string           `ch:"service"`
	ConnState        string           `ch:"conn_state"`
	Version          string           `ch:"version"`
	Cipher           string           `ch:"cipher"`
	Curve            string           `ch:"curve"`
	ServerName       string           `ch:"server_name"`
	Resumed          bool             `ch:"resumed"`
	NextProtocol     string           `ch:"next_protocol"`
	Established      bool             `ch:"established"`
	ServerCertFUIDs  []string         `ch:"server_cert_fuids"`
	ClientCertFUIDs  []string         `ch:"client_cert_fuids"`
	ServerSubject    string           `ch:"server_subject"`
	ServerIssuer     string           `ch:"server_issuer"`
	ClientSubject    string           `ch:"client_subject"`
	ClientIssuer     string           `ch:"client_issuer"`
	ValidationStatus string           `ch:"validation_status"`
	JA3              string           `ch:"ja3"`
	JA3S             string           `ch:"ja3s"`
}

type UniqueConn 

type UniqueConn struct {
	Hash      util.FixedString `ch:"hash"`
	Src       net.IP           `ch:"src"`
	Dst       net.IP           `ch:"dst"`
	SrcNUID   uuid.UUID        `ch:"src_nuid"`
	DstNUID   uuid.UUID        `ch:"dst_nuid"`
	ConnCount uint64
	ConnType  string
}

type UniqueFQDN 

type UniqueFQDN struct {
	Hash    util.FixedString `ch:"hash"`
	Src     net.IP           `ch:"src"`
	Dst     net.IP           `ch:"dst"`
	FQDN    string           `ch:"fqdn"`
	SrcNUID uuid.UUID        `ch:"src_nuid"`
	DstNUID uuid.UUID        `ch:"dst_nuid"`
}

type WaitGroups 

type WaitGroups struct {
	Digester sync.WaitGroup
	MetaDB   sync.WaitGroup
	OpenConn sync.WaitGroup
	Conn     sync.WaitGroup
	DNS      sync.WaitGroup
	HTTP     sync.WaitGroup
	OpenHTTP sync.WaitGroup
	SSL      sync.WaitGroup
	OpenSSL  sync.WaitGroup
}

type ZeekHeader 

type ZeekHeader[Z zeekRecord] struct {
	// contains filtered or unexported fields
}

ZeekHeader stores vars in the header of the zeek log

type ZeekUIDRecord 

type ZeekUIDRecord struct {
	UID               util.FixedString
	Timestamp         time.Time
	UsedByFQDNBeacon  bool
	UsedByDNS         bool
	LinkedToHTTPEntry bool
	NumUsedByHTTP     int
	Duration          float64
	SrcBytes          int64
	DstBytes          int64
	SrcIPBytes        int64
	DstIPBytes        int64
	SrcPackets        int64
	DstPackets        int64
	ConnState         string
	Proto             string
	Service           string
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.