ingest

package
v1.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 25, 2020 License: Apache-2.0 Imports: 13 Imported by: 12

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type BatchEventsSender

type BatchEventsSender struct {
	PayLoadBytes int
	BatchSize    int
	EventsChan   chan Event
	EventsQueue  []Event
	QuitChan     chan struct{}
	EventService *Service
	IngestTicker *util.Ticker
	WaitGroup    *sync.WaitGroup
	ErrorChan    chan struct{}
	IsRunning    bool

	Errors []ingestError
	// contains filtered or unexported fields
}

BatchEventsSender sends events in batches or periodically if batch is not full to Splunk Cloud ingest service endpoints

func (*BatchEventsSender) AddEvent

func (b *BatchEventsSender) AddEvent(event Event) error

AddEvent pushes a single event into EventsChan

func (*BatchEventsSender) ResetQueue

func (b *BatchEventsSender) ResetQueue()

ResetQueue sets b.EventsQueue to empty, but keep memory allocated for underlying array

func (*BatchEventsSender) Restart

func (b *BatchEventsSender) Restart()

Restart will reset batch event sender, clear up queue, error msg, timer etc.

func (*BatchEventsSender) Run

func (b *BatchEventsSender) Run()

Run sets up ticker and starts a new goroutine

func (*BatchEventsSender) SetCallbackHandler

func (b *BatchEventsSender) SetCallbackHandler(callback UserErrHandler)

SetCallbackHandler allows users to pass their own callback function

func (*BatchEventsSender) Stop

func (b *BatchEventsSender) Stop()

Stop sends a signal to QuitChan, wait for all registered goroutines to finish, stop ticker and clear queue

type Error

type Error struct {
	Code    *string                `json:"code,omitempty"`
	Details map[string]interface{} `json:"details,omitempty"`
	Message *string                `json:"message,omitempty"`
}

type Event

type Event struct {
	// The raw event content. It can be a string, number, string array, number array, JSON object, map, list, a JSON array, or a byte array.
	Body interface{} `json:"body"`
	// Specifies a JSON object that contains explicit custom fields to be defined at index time.
	Attributes map[string]interface{} `json:"attributes,omitempty"`
	// The host value assigned to the event data. Typically, this is the hostname of the client from which you are sending data.
	Host *string `json:"host,omitempty"`
	// An optional ID that uniquely identifies the event data. It is used to deduplicate the data if same data is set multiple times. If ID is not specified, it will be assigned by the system.
	Id *string `json:"id,omitempty"`
	// Optional nanoseconds part of the timestamp.
	Nanos *int32 `json:"nanos,omitempty"`
	// The source value to assign to the event data. For example, if you are sending data from an app that you are developing, set this key to the name of the app.
	Source *string `json:"source,omitempty"`
	// The sourcetype value assigned to the event data.
	Sourcetype *string `json:"sourcetype,omitempty"`
	// Epoch time in milliseconds.
	Timestamp *int64 `json:"timestamp,omitempty"`
}

type HecResponse added in v1.5.0

type HecResponse struct {
	// code defines the status of the response. For a complete list of the possible error codes, see https://docs.splunk.com/Documentation/SplunkCloud/latest/Data/TroubleshootHTTPEventCollector#Possible_error_codes
	Code *int64 `json:"code,omitempty"`
	// invalid-event-number defines the index of the first event in error.
	InvalidEventNumber *int64 `json:"invalid-event-number,omitempty"`
	// text describes the response status
	Text *string `json:"text,omitempty"`
}

type HecTokenAccessResponse added in v1.5.0

type HecTokenAccessResponse struct {
	// created_at is a timestamp that captures when this token was created.  type: string format: date-time
	CreatedAt *string `json:"created_at,omitempty"`
	// created_by is the principal that created the token.  type: string
	CreatedBy *string `json:"created_by,omitempty"`
	// description is an optional description of the token.  type: string
	Description *string `json:"description,omitempty"`
	// index is the default value of the index field for records collected using this token.  type: string
	Index *string `json:"index,omitempty"`
	// last_modified_at is a timestamp that captures when this token was last modified.  type: string format: date-time
	LastModifiedAt *string `json:"last_modified_at,omitempty"`
	// last_modified_by is the principal that last modified the token.  type: string
	LastModifiedBy *string `json:"last_modified_by,omitempty"`
	// name is the name of the token (unique within the tenant that it belongs to).  type: string
	Name *string `json:"name,omitempty"`
	// source is the default value of the source field for records collected using this token.  type: string
	Source *string `json:"source,omitempty"`
	// sourcetype is the default value of the sourcetype field for records collected using this token.  type: string
	Sourcetype *string `json:"sourcetype,omitempty"`
	// tenant is the tenant that this token belongs to  type: string
	Tenant *string `json:"tenant,omitempty"`
}

GET /collector/tokens/{tokenName} PUT /collector/tokens/{tokenName}

type HecTokenCreateRequest added in v1.5.0

type HecTokenCreateRequest struct {
	// name is the name of the token (unique within the tenant that it belongs to).  type: string
	Name string `json:"name"`
	// description is an optional description of the token.  type: string
	Description *string `json:"description,omitempty"`
	// index is the default value of the index field for records collected using this token.  type: string
	Index *string `json:"index,omitempty"`
	// source is the default value of the source field for records collected using this token.   type: string
	Source *string `json:"source,omitempty"`
	// sourcetype is the default value of the sourcetype field for records collected using this token.  type: string
	Sourcetype *string `json:"sourcetype,omitempty"`
}

POST /collector/tokens

type HecTokenCreateResponse added in v1.5.0

type HecTokenCreateResponse struct {
	// created_at is a timestamp that captures when this token was created.  type: string format: date-time
	CreatedAt *string `json:"created_at,omitempty"`
	// created_by is the principal that created the token.  type: string
	CreatedBy *string `json:"created_by,omitempty"`
	// description is an optional description of the token.  type: string
	Description *string `json:"description,omitempty"`
	// index is the default value of the index field for records collected using this token.  type: string
	Index *string `json:"index,omitempty"`
	// last_modified_at is a timestamp that captures when this token was last modified.  type: string format: date-time
	LastModifiedAt *string `json:"last_modified_at,omitempty"`
	// last_modified_by is the principal that last modified the token.  type: string
	LastModifiedBy *string `json:"last_modified_by,omitempty"`
	// name is the name of the token (unique within the tenant that it belongs to).  type: string
	Name *string `json:"name,omitempty"`
	// source is the default value of the source field for records collected using this token.  type: string
	Source *string `json:"source,omitempty"`
	// sourcetype is the default value of the sourcetype field for records collected using this token.  type: string
	Sourcetype *string `json:"sourcetype,omitempty"`
	// tenant is the tenant that this token belongs to.  type: string
	Tenant *string `json:"tenant,omitempty"`
	// token is the token value.  type: string
	Token *string `json:"token,omitempty"`
}

POST /collector/tokens

type HecTokenDeleteResponse added in v1.5.0

type HecTokenDeleteResponse map[string]interface{}

DELETE /collector/tokens/{tokenName}

type HecTokenUpdateRequest added in v1.5.0

type HecTokenUpdateRequest struct {
	// description is an optional description of the token.  type: string
	Description *string `json:"description,omitempty"`
	// index is the default value of the index field for records collected using this token  type: string
	Index *string `json:"index,omitempty"`
	// source is the default value of the source field for records collected using this token  type: string
	Source *string `json:"source,omitempty"`
	// sourcetype is the default value of the sourcetype field for records collected using this token  type: string
	Sourcetype *string `json:"sourcetype,omitempty"`
}

PUT /collector/tokens/{tokenName}

type HttpResponse

type HttpResponse struct {
	Code    *string                `json:"code,omitempty"`
	Details map[string]interface{} `json:"details,omitempty"`
	Message *string                `json:"message,omitempty"`
}

type ListCollectorTokensQueryParams added in v1.5.0

type ListCollectorTokensQueryParams struct {
	Limit  *int64 `key:"limit"`
	Offset *int64 `key:"offset"`
}

ListCollectorTokensQueryParams represents valid query parameters for the ListCollectorTokens operation For convenience ListCollectorTokensQueryParams can be formed in a single statement, for example:

`v := ListCollectorTokensQueryParams{}.SetLimit(...).SetOffset(...)`

func (ListCollectorTokensQueryParams) SetLimit added in v1.5.0

func (ListCollectorTokensQueryParams) SetOffset added in v1.5.0

type Metric

type Metric struct {
	// Name of the metric e.g. CPU, Memory etc.
	Name string `json:"name"`
	// Dimensions allow metrics to be classified e.g. {\"Server\":\"nginx\", \"Region\":\"us-west-1\", ...}
	Dimensions map[string]string `json:"dimensions,omitempty"`
	// Type of metric. Default is g for gauge.
	Type *string `json:"type,omitempty"`
	// Unit of the metric e.g. percent, megabytes, seconds etc.
	Unit *string `json:"unit,omitempty"`
	// Value of the metric. If not specified, it will be defaulted to 0.
	Value *float64 `json:"value,omitempty"`
}

type MetricAttribute

type MetricAttribute struct {
	// Optional. If set, individual metrics inherit these dimensions and can override any and/or all of them.
	DefaultDimensions map[string]string `json:"defaultDimensions,omitempty"`
	// Optional. If set, individual metrics inherit this type and can optionally override.
	DefaultType *string `json:"defaultType,omitempty"`
	// Optional. If set, individual metrics inherit this unit and can optionally override.
	DefaultUnit *string `json:"defaultUnit,omitempty"`
}

type MetricEvent

type MetricEvent struct {
	// Specifies multiple related metrics e.g. Memory, CPU etc.
	Body       []Metric         `json:"body"`
	Attributes *MetricAttribute `json:"attributes,omitempty"`
	// The host value assigned to the event data. Typically, this is the hostname of the client from which you are sending data.
	Host *string `json:"host,omitempty"`
	// An optional ID that uniquely identifies the metric data. It is used to deduplicate the data if same data is set multiple times. If ID is not specified, it will be assigned by the system.
	Id *string `json:"id,omitempty"`
	// Optional nanoseconds part of the timestamp.
	Nanos *int32 `json:"nanos,omitempty"`
	// The source value to assign to the event data. For example, if you are sending data from an app that you are developing, set this key to the name of the app.
	Source *string `json:"source,omitempty"`
	// The sourcetype value assigned to the event data.
	Sourcetype *string `json:"sourcetype,omitempty"`
	// Epoch time in milliseconds.
	Timestamp *int64 `json:"timestamp,omitempty"`
}

type Service

type Service services.BaseService

func NewService

func NewService(config *services.Config) (*Service, error)

NewService creates a new ingest service client from the given Config

func (*Service) DeleteAllCollectorTokens added in v1.5.0

func (s *Service) DeleteAllCollectorTokens(resp ...*http.Response) (*map[string]interface{}, error)

DeleteAllCollectorTokens - Delete All dsphec tokens for a given tenant. Parameters:

resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) DeleteCollectorToken added in v1.5.0

func (s *Service) DeleteCollectorToken(tokenName string, resp ...*http.Response) (*map[string]interface{}, error)

DeleteCollectorToken - Delete dsphec token by name. Parameters:

tokenName
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) GetCollectorToken added in v1.5.0

func (s *Service) GetCollectorToken(tokenName string, resp ...*http.Response) (*HecTokenAccessResponse, error)

GetCollectorToken - Get the metadata of a dsphec token by name. Parameters:

tokenName
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) ListCollectorTokens added in v1.5.0

func (s *Service) ListCollectorTokens(query *ListCollectorTokensQueryParams, resp ...*http.Response) ([]HecTokenAccessResponse, error)

ListCollectorTokens - List dsphec tokens for a tenant. Parameters:

query: a struct pointer of valid query parameters for the endpoint, nil to send no query parameters
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) NewBatchEventsSender

func (s *Service) NewBatchEventsSender(batchSize int, interval int64, payLoadSize int) (*BatchEventsSender, error)
	NewBatchEventsSender initializes a BatchEventsSender to collect events and send them as a single batched
        request when a maximum event batch size, time interval, or maximum payload size is reached.
	Parameters:
		batchSize: maximum number of events to reach before sending the batch, default maximum is 500
		interval: milliseconds to wait before sending the batch if other conditions have not been met
		payLoadSize: bytes that the overall payload should not exceed before sending, default maximum is 1040000 ~1MiB

func (*Service) NewBatchEventsSenderWithMaxAllowedError

func (s *Service) NewBatchEventsSenderWithMaxAllowedError(batchSize int, interval int64, dataSize int, maxErrorsAllowed int) (*BatchEventsSender, error)
	NewBatchEventsSenderWithMaxAllowedError initializes a BatchEventsSender to collect events and send them as a single
        batched request when a maximum event batch size, time interval, or maximum payload size is reached. It also
        validates the user input for BatchEventSender.
	Parameters:
		batchSize: maximum number of events to reach before sending the batch, default maximum is 500
		interval: milliseconds to wait before sending the batch if other conditions have not been met
		dataSize: bytes that the overall payload should not exceed before sending, default maximum is 1040000 ~1MiB
		maxErrorsAllowed: number of errors after which the BatchEventsSender will stop

func (*Service) PostCollectorTokens added in v1.5.0

func (s *Service) PostCollectorTokens(hecTokenCreateRequest HecTokenCreateRequest, resp ...*http.Response) (*HecTokenCreateResponse, error)

PostCollectorTokens - Creates dsphec tokens. Parameters:

hecTokenCreateRequest: The API request schema for the token.
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) PostEvents

func (s *Service) PostEvents(event []Event, resp ...*http.Response) (*HttpResponse, error)

PostEvents - Sends events. Parameters:

event
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) PostMetrics

func (s *Service) PostMetrics(metricEvent []MetricEvent, resp ...*http.Response) (*HttpResponse, error)

PostMetrics - Sends metric events. Parameters:

metricEvent
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) PutCollectorToken added in v1.5.0

func (s *Service) PutCollectorToken(tokenName string, hecTokenUpdateRequest HecTokenUpdateRequest, resp ...*http.Response) (*HecTokenAccessResponse, error)

PutCollectorToken - Update the metadata of a dsphec token by name. Parameters:

tokenName
hecTokenUpdateRequest: The API request schema for the token.
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) UploadFiles

func (s *Service) UploadFiles(filename string, resp ...*http.Response) error

UploadFiles - Upload a CSV or text file that contains events. Parameters:

filename
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

func (*Service) UploadFilesStream

func (s *Service) UploadFilesStream(stream io.Reader, resp ...*http.Response) error

UploadFilesStream - Upload stream of io.Reader. Parameters:

stream
resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided

type Servicer

type Servicer interface {
	/*
	   	NewBatchEventsSenderWithMaxAllowedError initializes a BatchEventsSender to collect events and send them as a single
	           batched request when a maximum event batch size, time interval, or maximum payload size is reached. It also
	           validates the user input for BatchEventSender.
	   	Parameters:
	   		batchSize: maximum number of events to reach before sending the batch, default maximum is 500
	   		interval: milliseconds to wait before sending the batch if other conditions have not been met
	   		dataSize: bytes that the overall payload should not exceed before sending, default maximum is 1040000 ~1MiB
	   		maxErrorsAllowed: number of errors after which the BatchEventsSender will stop
	*/
	NewBatchEventsSenderWithMaxAllowedError(batchSize int, interval int64, dataSize int, maxErrorsAllowed int) (*BatchEventsSender, error)
	/*
	   	NewBatchEventsSender initializes a BatchEventsSender to collect events and send them as a single batched
	           request when a maximum event batch size, time interval, or maximum payload size is reached.
	   	Parameters:
	   		batchSize: maximum number of events to reach before sending the batch, default maximum is 500
	   		interval: milliseconds to wait before sending the batch if other conditions have not been met
	   		payLoadSize: bytes that the overall payload should not exceed before sending, default maximum is 1040000 ~1MiB
	*/
	NewBatchEventsSender(batchSize int, interval int64, payLoadSize int) (*BatchEventsSender, error)
	/*
		UploadFiles - Upload a CSV or text file that contains events.
		Parameters:
			filename
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	UploadFiles(filename string, resp ...*http.Response) error
	/*
		UploadFilesStream - Upload stream of io.Reader.
		Parameters:
			stream
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	UploadFilesStream(stream io.Reader, resp ...*http.Response) error
	/*
		DeleteAllCollectorTokens - Delete All dsphec tokens for a given tenant.
		Parameters:
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	DeleteAllCollectorTokens(resp ...*http.Response) (*map[string]interface{}, error)
	/*
		DeleteCollectorToken - Delete dsphec token by name.
		Parameters:
			tokenName
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	DeleteCollectorToken(tokenName string, resp ...*http.Response) (*map[string]interface{}, error)
	/*
		GetCollectorToken - Get the metadata of a dsphec token by name.
		Parameters:
			tokenName
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	GetCollectorToken(tokenName string, resp ...*http.Response) (*HecTokenAccessResponse, error)
	/*
		ListCollectorTokens - List dsphec tokens for a tenant.
		Parameters:
			query: a struct pointer of valid query parameters for the endpoint, nil to send no query parameters
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	ListCollectorTokens(query *ListCollectorTokensQueryParams, resp ...*http.Response) ([]HecTokenAccessResponse, error)
	/*
		PostCollectorTokens - Creates dsphec tokens.
		Parameters:
			hecTokenCreateRequest: The API request schema for the token.
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	PostCollectorTokens(hecTokenCreateRequest HecTokenCreateRequest, resp ...*http.Response) (*HecTokenCreateResponse, error)
	/*
		PostEvents - Sends events.
		Parameters:
			event
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	PostEvents(event []Event, resp ...*http.Response) (*HttpResponse, error)
	/*
		PostMetrics - Sends metric events.
		Parameters:
			metricEvent
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	PostMetrics(metricEvent []MetricEvent, resp ...*http.Response) (*HttpResponse, error)
	/*
		PutCollectorToken - Update the metadata of a dsphec token by name.
		Parameters:
			tokenName
			hecTokenUpdateRequest: The API request schema for the token.
			resp: an optional pointer to a http.Response to be populated by this method. NOTE: only the first resp pointer will be used if multiple are provided
	*/
	PutCollectorToken(tokenName string, hecTokenUpdateRequest HecTokenUpdateRequest, resp ...*http.Response) (*HecTokenAccessResponse, error)
}

Servicer represents the interface for implementing all endpoints for this service

type UserErrHandler

type UserErrHandler func(*BatchEventsSender)

UserErrHandler defines the type of user callback function for batchEventSender

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL