Documentation
¶
Index ¶
Constants ¶
const SQLCheckForTrigramExtension = `SELECT COUNT(*) FROM pg_available_extensions WHERE name = 'pg_trgm';`
SQLCheckForTrigramExtension is an SQL query to check whether the trigram extension is available.
const SQLCopy = `COPY "%s" (ts, payload) FROM STDIN WITH CSV DELIMITER E'\t' QUOTE E'\b'`
SQLCopy is an SQL/DDL clause to bulk insert a chunk of JSON into the database
const SQLCreate = `` /* 150-byte string literal not displayed */
SQLCreate is an SQL/DDL clause to create a new event table
const SQLGenericQuery = `` /* 130-byte string literal not displayed */
SQLGenericQuery is the main kind of query used to pull out event metadata.
const SQLGetTableSizes = `SELECT relname as table,
pg_total_relation_size(relid) as size
FROM pg_catalog.pg_statio_user_tables
ORDER BY 1 DESC;`
SQLGetTableSizes is an SQL query to obtain the names of tables in the current schema and their size in bytes.
const SQLIndex = `` /* 315-byte string literal not displayed */
SQLIndex is an SQL/DDL clause to create indexes on event tables
const SQLQueryAllEvents = `` /* 2131-byte string literal not displayed */
SQLQueryAllEvents is a plpgsql function to enable queries over all hourly tables Example: SELECT COUNT(*) FROM all_events_query('WHERE trigram_string(payload) LIKE ”%%foo%%”');
const SQLTrigramFunction = `` /* 1241-byte string literal not displayed */
SQLTrigramFunction is a plpgsql function to pull out indexable content from event JSON
Variables ¶
var INDEXES = map[string]([]mgo.Index){ "dns": []mgo.Index{ mgo.Index{ Key: []string{"dns.rrname"}, Background: true, }, mgo.Index{ Key: []string{"timestamp"}, Background: true, }, }, "fileinfo": []mgo.Index{ mgo.Index{ Key: []string{"src_ip", "dest_ip"}, Background: true, }, mgo.Index{ Key: []string{"fileinfo.filename", "fileinfo.md5"}, Background: true, }, mgo.Index{ Key: []string{"timestamp"}, Background: true, }, }, "flow": []mgo.Index{ mgo.Index{ Key: []string{"src_ip", "dest_ip"}, Background: true, }, mgo.Index{ Key: []string{"timestamp"}, Background: true, }, }, "http": []mgo.Index{ mgo.Index{ Key: []string{"src_ip", "dest_ip"}, Background: true, }, mgo.Index{ Key: []string{"http.hostname", "http.http_user_agent"}, Background: true, }, mgo.Index{ Key: []string{"$text:http.url"}, Background: true, }, mgo.Index{ Key: []string{"timestamp"}, Background: true, }, }, "alert": []mgo.Index{ mgo.Index{ Key: []string{"src_ip", "dest_ip"}, Background: true, }, mgo.Index{ Key: []string{"$text:alert.payload_printable"}, Background: true, }, mgo.Index{ Key: []string{"timestamp"}, Background: true, }, }, "smtp": []mgo.Index{ mgo.Index{ Key: []string{"src_ip", "dest_ip"}, Background: true, }, mgo.Index{ Key: []string{"smtp.helo", "smtp.mail_from", "smtp.rcpt_to"}, Background: true, }, mgo.Index{ Key: []string{"email.attachment"}, Background: true, }, mgo.Index{ Key: []string{"timestamp"}, Background: true, }, }, "tls": []mgo.Index{ mgo.Index{ Key: []string{"src_ip", "dest_ip"}, Background: true, }, mgo.Index{ Key: []string{"tls.subject", "tls.issuerdn", "tls.fingerprint"}, Background: true, }, mgo.Index{ Key: []string{"timestamp"}, Background: true, }, }, "misc": []mgo.Index{ mgo.Index{ Key: []string{"src_ip", "dest_ip"}, Background: true, }, mgo.Index{ Key: []string{"timestamp"}, Background: true, }, }, }
INDEXES assigns index parameters to each collection, denoted by the corresponding event type
var MAXCOLLSIZEFRACTIONS = map[string]float64{
"dns": 0.25,
"http": 0.2,
"flow": 0.25,
"smtp": 0.05,
"ssh": 0.05,
"alert": 0.05,
"tls": 0.05,
"stats": 0.02,
"misc": 0.03,
"fileinfo": 0.05,
}
MAXCOLLSIZEFRACTIONS are the proportions of the general space cap to be assigned to the collections for each event type -- used to determine limits for capped collections
var TYPES = []string{
"alert", "dns", "fileinfo", "flow",
"http", "smtp", "ssh", "stats",
"tls", "misc",
}
TYPES are event types/collections supported by us
Functions ¶
This section is empty.
Types ¶
type DummySlurper ¶
type DummySlurper struct{}
DummySlurper is a slurper that just consumes entries with no action.
func (*DummySlurper) Finish ¶
func (s *DummySlurper) Finish()
Finish is a null operation in the DummySlurper implementation.
type MongoSlurper ¶
type MongoSlurper struct {
User string
Password string
Host string
Database string
TypeDispatch map[string](chan types.Entry)
ChunkSize int
MaxSize int64
Logger *log.Entry
}
MongoSlurper is a Slurper that stores events in an MongoDB database.
func MakeMongoSlurper ¶
func MakeMongoSlurper(host string, database string, user string, password string, chunkSize int, maxSize int64) *MongoSlurper
MakeMongoSlurper creates a new MongoSlurper instance.
func (*MongoSlurper) Finish ¶
func (s *MongoSlurper) Finish()
Finish is a null operation in the MongoSlurper implementation.
type PostgresSlurper ¶
type PostgresSlurper struct {
DB *pgxpool.Pool
DBUser string
// CopyFn allows injecting a custom COPY executor for testing.
// It should execute a COPY FROM STDIN using the provided SQL and reader and return rows copied.
CopyFn func(ctx context.Context, pool *pgxpool.Pool, sql string, r io.Reader) (int64, error)
// ExecFn allows injecting execution for DDL statements (e.g., CREATE TABLE ... GRANT ...)
ExecFn func(ctx context.Context, sql string) error
LastRotatedTime time.Time
IndexChan chan string
CurrentTableName string
RotationInterval time.Duration
MaxTableSize int64
ChunkSize int
Logger *log.Entry
}
PostgresSlurper is a Slurper that stores events in an PostgreSQL database.
func MakePostgresSlurper ¶
func MakePostgresSlurper(host string, database string, user string, password string, rotationInterval time.Duration, maxTableSize int64, chunkSize int) *PostgresSlurper
MakePostgresSlurper creates a new PostgresSlurper instance.
func (*PostgresSlurper) Finish ¶
func (s *PostgresSlurper) Finish()
Finish is a null operation in the PostgresSlurper implementation.
Source Files