streamer

package module
v0.8.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 10, 2025 License: BSD-3-Clause Imports: 23 Imported by: 0

README

Gitlab Log Streamer

License

Gitlab Log Streamer is a tool designed to overcome the limitations of Gitlab's audit_log.json and auth_log.json.

By default, Gitlab writes its audit events to the audit_log.json file, which limits their usefulness as they stay in your GItLab server filesystem.

This project parses the log files, stores the events in a SQLite database, and allows forwarding of new log entries using syslog format (RFC5424) or IBM QRadar's proprietary LEEF. It also supports defining an HTTP endpoint for POST requests with the event, enabling triggers and actions similar to Gitlab System hooks.

Table of Contents

Installation

Just head to https://github.com/juanfont/gitlab-log-streamer/releases and grab the latest version.

Then place it in your PATH (e.g., /usr/local/bin)

Usage

You need to create a file named config.yaml in the same directory as the binary or in /etc/gitlab-log-streamer:

---
db_path: "streamer.sqlite"
gitlab_hostname: "gitlab.font.eu"

sources:
  audit_log_path: "/var/log/gitlab/gitlab-rails/audit_json.log"
  auth_log_path: "/var/log/gitlab/gitlab-rails/auth_json.log"


destinations:
  http:
    url: "http://localhost:8080"
    headers:
      Authorization: "Bearer 1234567890"
      Content-Type: "application/json"
  syslog:
    server_addr: "localhost:1489"
    protocol: "udp"

    # Optional. If true, the syslog message will be in LEEF format. Otherwise, syslog in RFC5424 format.
    use_leef: false

And then just execute:

gitlab-log-streamer watch

Documentation

Index

Constants

View Source 
const (
	GitlabVersionManifestPath = "/opt/gitlab/version-manifest.txt"
	PreloadEventsPeriodDays   = 30

	CleanAuthLogEventsPeriod = 60 * time.Hour * 24 // time to keep the auth events in the DB
)
View Source 
const (
	AuditEventLoginWithTwoFactor = "two-factor"
	AuditEventLoginWithU2F       = "two-factor-via-u2f-device"
	AuditEventLoginWithWebAuthn  = "two-factor-via-webauthn-device"
	AuditEventLoginStandard      = "standard"
)

Variables

This section is empty.

Functions

This section is empty.

Types

type Add 

type Add string
const (
	CiGroupVariable Add = "ci_group_variable"
	Email           Add = "email"
	Group           Add = "group"
	Project         Add = "project"
	User            Add = "user"
	UserAccess      Add = "user_access"
)

type As 

type As string
const (
	Developer As = "Developer"
	Guest     As = "Guest"
	Owner     As = "Owner"
)

type AuditEvent 

type AuditEvent struct {
	ID            uint64 `gorm:"primary_key" json:"-"`
	CorrelationID string `gorm:"type:varchar(64)" json:"correlation_id"`

	Severity      string      `json:"severity"`
	Time          time.Time   `json:"time"`
	AuthorID      int64       `json:"author_id"`
	AuthorName    string      `json:"author_name"`
	EntityID      int64       `json:"entity_id"`
	EntityType    AuthorClass `json:"entity_type"`
	IPAddress     string      `json:"ip_address"`
	With          *With       `json:"with,omitempty"`
	TargetID      *int64      `json:"target_id"`
	TargetType    AuthorClass `json:"target_type"`
	TargetDetails *string     `json:"target_details"`
	EntityPath    string      `json:"entity_path"`
	Remove        *Add        `json:"remove,omitempty"`
	Add           *Add        `json:"add,omitempty"`
	// Details       *Details    `json:"details,omitempty"`
	// PushAccessLevels          []MergeAccessLevelElement `json:"push_access_levels,omitempty"`
	// MergeAccessLevels         []MergeAccessLevelElement `json:"merge_access_levels,omitempty"`
	AllowForcePush            *bool        `json:"allow_force_push,omitempty"`
	CodeOwnerApprovalRequired *bool        `json:"code_owner_approval_required,omitempty"`
	AuthorClass               *AuthorClass `json:"author_class,omitempty"`
	CustomMessage             *string      `json:"custom_message,omitempty"`
	As                        *As          `json:"as,omitempty"`
	MemberID                  *int64       `json:"member_id,omitempty"`
	Change                    *Change      `json:"change,omitempty"`
	From                      *string      `json:"from,omitempty"`
	To                        *string      `json:"to,omitempty"`
	Action                    *string      `json:"action,omitempty"`
	ExpiryFrom                *string      `json:"expiry_from"`
	ExpiryTo                  *string      `json:"expiry_to"`

	MetaCallerID        string `json:"meta.caller_id"`
	MetaRemoteIP        string `json:"meta.remote_ip"`
	MetaFeatureCategory string `json:"meta.feature_category"`
	MetaClientID        string `json:"meta.client_id"`
	MetaUser            string `json:"meta.user,omitempty"`
	MetaUserID          int    `json:"meta.user_id,omitempty"`

	OriginalData datatypes.JSON
}

type AuthEvent added in v0.5.0 

type AuthEvent struct {
	ID            uint64    `gorm:"primary_key" json:"-"`
	CorrelationID string    `gorm:"type:varchar(64)" json:"correlation_id"`
	Severity      string    `json:"severity"`
	Time          time.Time `json:"time"`

	Message              *string `json:"message,omitempty"`
	Env                  *string `json:"env,omitempty"`
	RemoteIP             *string `json:"remote_ip,omitempty"`
	RequestMethod        *string `json:"request_method,omitempty"`
	Path                 *string `json:"path,omitempty"`
	MetaCallerID         *string `json:"meta.caller_id,omitempty"`
	MetaRemoteIP         *string `json:"meta.remote_ip,omitempty"`
	MetaFeatureCategory  *string `json:"meta.feature_category,omitempty"`
	MetaClientID         *string `json:"meta.client_id,omitempty"`
	ScopeType            *string `json:"scope_type,omitempty"`
	RequestedProjectPath *string `json:"requested_project_path,omitempty"`
	// RequestedActions     []string `json:"requested_actions,omitempty"`
	// AuthorizedActions    []string `json:"authorized_actions,omitempty"`
	HTTPUser            *string `json:"http_user,omitempty"`
	AuthService         *string `json:"auth_service,omitempty"`
	AuthResultType      *string `json:"auth_result.type"`
	AuthResultActorType *string `json:"auth_result.actor_type"`
	PayloadType         *string `json:"payload_type,omitempty"`

	OriginalData datatypes.JSON
}

type AuthorClass 

type AuthorClass string
const (
	AuthorClassCiGroupVariable AuthorClass = "Ci::GroupVariable"
	AuthorClassEmail           AuthorClass = "Email"
	AuthorClassGroup           AuthorClass = "Group"
	AuthorClassProject         AuthorClass = "Project"
	AuthorClassUser            AuthorClass = "User"
	CiRunner                   AuthorClass = "Ci::Runner"
	PersonalAccessToken        AuthorClass = "PersonalAccessToken"
	ProtectedBranch            AuthorClass = "ProtectedBranch"
)

type Change 

type Change string
const (
	AccessLevel   Change = "access_level"
	AllowedToPush Change = "allowed to push"
	EmailAddress  Change = "email address"
	Name          Change = "name"
)

type Config 

type Config struct {
	ListenAddr                 string
	AuditLogForwardingEndpoint string
	AuthLogForwardingEndpoint  string
	GitlabHostname             string
	AuditLogPath               string
	AuthLogPath                string
	DBpath                     string

	SyslogServerAddr string
	SyslogProtocol   string
	UseLEEF          bool // Use QRadar propietary LEEF format
}

type GitLabLogStreamer added in v0.5.0 

type GitLabLogStreamer struct {
	// contains filtered or unexported fields
}

func NewGitLabLogStreamer added in v0.5.0 

func NewGitLabLogStreamer(config Config) (*GitLabLogStreamer, error)

func (*GitLabLogStreamer) Watch added in v0.5.0 

func (s *GitLabLogStreamer) Watch() error

type MergeAccessLevelElement 

type MergeAccessLevelElement string
const (
	Maintainers MergeAccessLevelElement = "Maintainers"
)

type Severity 

type Severity string

type With 

type With string
const (
	Saml                       With = "saml"
	TwoFactor                  With = "two-factor"
	TwoFactorViaWebauthnDevice With = "two-factor-via-webauthn-device"
)

Source Files

View all Source files

Directories

Path Synopsis
cmd
streamer command
streamer/cli
pkg
leef

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.