config

package

v0.1.3-rc1 Latest Latest Go to latest Published: Jul 24, 2022 License: Apache-2.0 Imports: 3 Imported by: 3

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/netobserv/flowlogs-pipeline

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func ParseConfig() error
type ConfigFileStruct
type Encode
type Extract
type File
type GenericMap
- func (m GenericMap) Copy() GenericMap
type Health
type Ingest
type Options
type PipelineBuilderStage
type Stage
type StageParam
type Transform
type Write

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	Opt        = Options{}
	PipeLine   []Stage
	Parameters []StageParam
)

Functions ¶

func ParseConfig ¶

func ParseConfig() error

ParseConfig creates the internal unmarshalled representation from the Pipeline and Parameters json

Types ¶

type ConfigFileStruct ¶ added in v0.1.3

type ConfigFileStruct struct {
	LogLevel   string       `yaml:"log-level,omitempty" json:"log-level,omitempty"`
	Pipeline   []Stage      `yaml:"pipeline,omitempty" json:"pipeline,omitempty"`
	Parameters []StageParam `yaml:"parameters,omitempty" json:"parameters,omitempty"`
}

type Encode ¶

type Encode struct {
	Type  string           `yaml:"type" json:"type"`
	Prom  *api.PromEncode  `yaml:"prom,omitempty" json:"prom,omitempty"`
	Kafka *api.EncodeKafka `yaml:"kafka,omitempty" json:"kafka,omitempty"`
}

type Extract ¶

type Extract struct {
	Type       string                    `yaml:"type" json:"type"`
	Aggregates []api.AggregateDefinition `yaml:"aggregates,omitempty" json:"aggregates,omitempty"`
}

type File ¶

type File struct {
	Filename string      `yaml:"filename" json:"filename"`
	Decoder  api.Decoder `yaml:"decoder" json:"decoder"`
	Loop     bool        `yaml:"loop" json:"loop"`
	Chunks   int         `yaml:"chunks" json:"chunks"`
}

type GenericMap ¶

type GenericMap map[string]interface{}

func (GenericMap) Copy ¶

func (m GenericMap) Copy() GenericMap

Copy will create a flat copy of GenericMap

type Health ¶

type Health struct {
	Port string
}

type Ingest ¶

type Ingest struct {
	Type      string               `yaml:"type" json:"type"`
	File      *File                `yaml:"file,omitempty" json:"file,omitempty"`
	Collector *api.IngestCollector `yaml:"collector,omitempty" json:"collector,omitempty"`
	Kafka     *api.IngestKafka     `yaml:"kafka,omitempty" json:"kafka,omitempty"`
	GRPC      *api.IngestGRPCProto `yaml:"grpc,omitempty" json:"grpc,omitempty"`
}

type Options ¶

type Options struct {
	PipeLine   string
	Parameters string
	Health     Health
}

type PipelineBuilderStage ¶

type PipelineBuilderStage struct {
	// contains filtered or unexported fields
}

PipelineBuilderStage holds information about a created pipeline stage. This stage can be used to chain a following stage, or several of them (resulting in a fork). Example:

firstStage := NewCollectorPipeline("first stage", ...)
secondStage := firstStage.DecodeJSON("second stage")
thirdStage := secondStage.WriteLoki("third stage", ...)
forkedStage := secondStage.WriteStdout("fork following second stage", ...)

All created stages hold a pointer to the whole pipeline, so that the resulting pipeline can be retrieve from any of the stages:

forkedStage.GetStages()
forkedStage.GetStageParams()
// is equivalent to:
firstStage.GetStages()
firstStage.GetStageParams()

func NewCollectorPipeline ¶

func NewCollectorPipeline(name string, ingest api.IngestCollector) PipelineBuilderStage

NewCollectorPipeline creates a new pipeline from an `IngestCollector` initial stage (listening for NetFlows / IPFIX)

func NewGRPCPipeline ¶

func NewGRPCPipeline(name string, ingest api.IngestGRPCProto) PipelineBuilderStage

NewGRPCPipeline creates a new pipeline from an `IngestGRPCProto` initial stage (listening for NetObserv's eBPF agent protobuf)

func NewKafkaPipeline ¶

func NewKafkaPipeline(name string, ingest api.IngestKafka) PipelineBuilderStage

NewKafkaPipeline creates a new pipeline from an `IngestKafka` initial stage (listening for flow events on Kafka)

func (*PipelineBuilderStage) Aggregate ¶

func (b *PipelineBuilderStage) Aggregate(name string, aggs []api.AggregateDefinition) PipelineBuilderStage

Aggregate chains the current stage with an aggregate stage and returns that new stage

func (*PipelineBuilderStage) EncodeKafka ¶

func (b *PipelineBuilderStage) EncodeKafka(name string, kafka api.EncodeKafka) PipelineBuilderStage

EncodeKafka chains the current stage with an EncodeKafka stage (writing to a Kafka topic) and returns that new stage

func (*PipelineBuilderStage) EncodePrometheus ¶

func (b *PipelineBuilderStage) EncodePrometheus(name string, prom api.PromEncode) PipelineBuilderStage

EncodePrometheus chains the current stage with a PromEncode stage (to expose metrics in Prometheus format) and returns that new stage

func (*PipelineBuilderStage) GetStageParams ¶

func (b *PipelineBuilderStage) GetStageParams() []StageParam

GetStageParams returns the current pipeline stage params. It can be called from any of the stages, they share the same pipeline reference.

func (*PipelineBuilderStage) GetStages ¶

func (b *PipelineBuilderStage) GetStages() []Stage

GetStages returns the current pipeline stages. It can be called from any of the stages, they share the same pipeline reference.

func (*PipelineBuilderStage) TransformFilter ¶

func (b *PipelineBuilderStage) TransformFilter(name string, filter api.TransformFilter) PipelineBuilderStage

TransformFilter chains the current stage with a TransformFilter stage and returns that new stage

func (*PipelineBuilderStage) TransformGeneric ¶

func (b *PipelineBuilderStage) TransformGeneric(name string, gen api.TransformGeneric) PipelineBuilderStage

TransformGeneric chains the current stage with a TransformGeneric stage and returns that new stage

func (*PipelineBuilderStage) TransformNetwork ¶

func (b *PipelineBuilderStage) TransformNetwork(name string, nw api.TransformNetwork) PipelineBuilderStage

TransformNetwork chains the current stage with a TransformNetwork stage and returns that new stage

func (*PipelineBuilderStage) WriteLoki ¶

func (b *PipelineBuilderStage) WriteLoki(name string, loki api.WriteLoki) PipelineBuilderStage

WriteLoki chains the current stage with a WriteLoki stage and returns that new stage

func (*PipelineBuilderStage) WriteStdout ¶

func (b *PipelineBuilderStage) WriteStdout(name string, stdout api.WriteStdout) PipelineBuilderStage

WriteStdout chains the current stage with a WriteStdout stage and returns that new stage

type Stage ¶

type Stage struct {
	Name    string `yaml:"name" json:"name"`
	Follows string `yaml:"follows,omitempty" json:"follows,omitempty"`
}

type StageParam ¶

type StageParam struct {
	Name      string     `yaml:"name" json:"name"`
	Ingest    *Ingest    `yaml:"ingest,omitempty" json:"ingest,omitempty"`
	Transform *Transform `yaml:"transform,omitempty" json:"transform,omitempty"`
	Extract   *Extract   `yaml:"extract,omitempty" json:"extract,omitempty"`
	Encode    *Encode    `yaml:"encode,omitempty" json:"encode,omitempty"`
	Write     *Write     `yaml:"write,omitempty" json:"write,omitempty"`
}

type Transform ¶

type Transform struct {
	Type    string                `yaml:"type" json:"type"`
	Generic *api.TransformGeneric `yaml:"generic,omitempty" json:"generic,omitempty"`
	Filter  *api.TransformFilter  `yaml:"filter,omitempty" json:"filter,omitempty"`
	Network *api.TransformNetwork `yaml:"network,omitempty" json:"network,omitempty"`
}

type Write ¶

type Write struct {
	Type   string           `yaml:"type" json:"type"`
	Loki   *api.WriteLoki   `yaml:"loki,omitempty" json:"loki,omitempty"`
	Stdout *api.WriteStdout `yaml:"stdout,omitempty" json:"stdout,omitempty"`
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL