model

package
v2.3.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 21, 2025 License: Apache-2.0 Imports: 9 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type DefaultSourceTuple

type DefaultSourceTuple struct {
	// contains filtered or unexported fields
}

func NewDefaultRawTuple

func NewDefaultRawTuple(raw []byte, meta map[string]any, ts time.Time) *DefaultSourceTuple

NewDefaultRawTuple creates a new DefaultSourceTuple with raw data. Use this when extend source connector

func NewDefaultRawTupleIgnoreTs

func NewDefaultRawTupleIgnoreTs(raw []byte, meta map[string]any) *DefaultSourceTuple

func NewDefaultSourceTuple

func NewDefaultSourceTuple(message map[string]any, meta map[string]any, timestamp time.Time) *DefaultSourceTuple

func (*DefaultSourceTuple) AllMeta

func (t *DefaultSourceTuple) AllMeta() map[string]any

func (*DefaultSourceTuple) Meta

func (t *DefaultSourceTuple) Meta(key, table string) (any, bool)

func (*DefaultSourceTuple) Range

func (t *DefaultSourceTuple) Range(f func(key string, value any) bool)

func (*DefaultSourceTuple) Raw

func (t *DefaultSourceTuple) Raw() []byte

func (*DefaultSourceTuple) Timestamp

func (t *DefaultSourceTuple) Timestamp() time.Time

func (*DefaultSourceTuple) ToMap

func (t *DefaultSourceTuple) ToMap() map[string]any

func (*DefaultSourceTuple) Value

func (t *DefaultSourceTuple) Value(key, table string) (any, bool)

type EncryptionConf added in v2.2.0

type EncryptionConf struct {
	Algorithm  string         `yaml:"algorithm,omitempty" json:"algorithm"`
	Properties map[string]any `yaml:"properties,omitempty" json:"properties"`
	Key        string         `yaml:"key,omitempty" json:"key"`
}

type IndexValuer added in v2.3.0

type IndexValuer interface {
	ValueByIndex(index int, sourceIndex int) (any, bool)
	SetByIndex(index int, value any)
	SetTempByIndex(index int, value any)
	TempByIndex(index int) any
}

type InfoNode

type InfoNode interface {
	Info() NodeInfo
	TransformType() api.Source
}

InfoNode explain the node itself. Mainly used for planner to decide the split of source/sink

type KuiperConf added in v2.2.0

type KuiperConf struct {
	Hack struct {
		Cold bool `yaml:"cold"`
	}
	Basic struct {
		LogLevel                string                `yaml:"logLevel"`
		Debug                   bool                  `yaml:"debug"`
		ConsoleLog              bool                  `yaml:"consoleLog"`
		FileLog                 bool                  `yaml:"fileLog"`
		LogDisableTimestamp     bool                  `yaml:"logDisableTimestamp"`
		Syslog                  *SyslogConf           `yaml:"syslog"`
		RotateTime              int                   `yaml:"rotateTime"`
		MaxAge                  int                   `yaml:"maxAge"`
		RotateSize              int64                 `yaml:"rotateSize"`
		RotateCount             int                   `yaml:"rotateCount"`
		TimeZone                string                `yaml:"timezone"`
		Ip                      string                `yaml:"ip"`
		Port                    int                   `yaml:"port"`
		RestIp                  string                `yaml:"restIp"`
		RestPort                int                   `yaml:"restPort"`
		RestTls                 *TlsConf              `yaml:"restTls"`
		Prometheus              bool                  `yaml:"prometheus"`
		PrometheusPort          int                   `yaml:"prometheusPort"`
		Pprof                   bool                  `yaml:"pprof"`
		PprofIp                 string                `yaml:"pprofIp"`
		PprofPort               int                   `yaml:"pprofPort"`
		PluginHosts             string                `yaml:"pluginHosts"`
		Authentication          bool                  `yaml:"authentication"`
		IgnoreCase              bool                  `yaml:"ignoreCase"`
		SQLConf                 *SQLConf              `yaml:"sql"`
		RulePatrolInterval      cast.DurationConf     `yaml:"rulePatrolInterval"`
		EnableOpenZiti          bool                  `yaml:"enableOpenZiti"`
		AesKey                  string                `yaml:"aesKey"`
		GracefulShutdownTimeout cast.DurationConf     `yaml:"gracefulShutdownTimeout"`
		ResourceProfileConfig   ResourceProfileConfig `yaml:"ResourceProfileConfig"`
		MetricsDumpConfig       MetricsDumpConfig     `yaml:"metricsDumpConfig"`
		EnableRestAuditLog      bool                  `yaml:"enableRestAuditLog"`
	}
	Rule   def.RuleOption
	Sink   *SinkConf
	Source *SourceConf
	Store  struct {
		Type         string `yaml:"type"`
		ExtStateType string `yaml:"extStateType"`
		Redis        struct {
			Host               string            `yaml:"host"`
			Port               int               `yaml:"port"`
			Password           string            `yaml:"password"`
			Timeout            cast.DurationConf `yaml:"timeout"`
			ConnectionSelector string            `yaml:"connectionSelector"`
		}
		Sqlite struct {
			Name string `yaml:"name"`
		}
		Fdb struct {
			Path string `yaml:"path"`
		}
	}
	Portable struct {
		PythonBin   string            `yaml:"pythonBin"`
		InitTimeout cast.DurationConf `yaml:"initTimeout"`
		SendTimeout time.Duration     `yaml:"sendTimeout"`
		RecvTimeout time.Duration     `yaml:"recvTimeout"`
	}
	Connection struct {
		BackoffMaxElapsedDuration cast.DurationConf `yaml:"backoffMaxElapsedDuration"`
	}
	OpenTelemetry OpenTelemetry `yaml:"openTelemetry"`
	AesKey        []byte
	Security      *SecurityConf
}

func (*KuiperConf) PprofSameAsRest added in v2.2.3

func (k *KuiperConf) PprofSameAsRest() bool

type MetricsDumpConfig added in v2.2.0

type MetricsDumpConfig struct {
	Enable           bool          `yaml:"enable"`
	RetainedDuration time.Duration `yaml:"retainedDuration"`
}

type NodeInfo

type NodeInfo struct {
	NeedDecode      bool
	NeedBatchDecode bool // like decrypt, decompress as a whole
	HasCompress     bool
	HasInterval     bool
}

type OpenTelemetry added in v2.2.0

type OpenTelemetry struct {
	ServiceName           string `yaml:"serviceName"`
	EnableRemoteCollector bool   `yaml:"enableRemoteCollector"`
	RemoteEndpoint        string `yaml:"remoteEndpoint"`
	LocalTraceCapacity    int    `yaml:"localTraceCapacity"`
	EnableLocalStorage    bool   `yaml:"enableLocalStorage"`
}

type PropsConsumer added in v2.1.1

type PropsConsumer interface {
	Consume(props map[string]any)
}

PropsConsumer Read in properties, swallow some and return new props

type ResourceProfileConfig added in v2.2.0

type ResourceProfileConfig struct {
	Enable   bool          `yaml:"enable"`
	Interval time.Duration `yaml:"interval"`
}

type SQLConf added in v2.2.0

type SQLConf struct {
	MaxConnections int `yaml:"maxConnections"`
}

type SecurityConf added in v2.2.0

type SecurityConf struct {
	Encryption *EncryptionConf          `yaml:"encryption,omitempty"`
	Tls        *TlsConfigurationOptions `yaml:"tls,omitempty"`
}

type SinkConf added in v2.2.0

type SinkConf struct {
	MemoryCacheThreshold int               `json:"memoryCacheThreshold" yaml:"memoryCacheThreshold"`
	MaxDiskCache         int               `json:"maxDiskCache" yaml:"maxDiskCache"`
	BufferPageSize       int               `json:"bufferPageSize" yaml:"bufferPageSize"`
	EnableCache          bool              `json:"enableCache" yaml:"enableCache"`
	ResendInterval       cast.DurationConf `json:"resendInterval" yaml:"resendInterval"`
	CleanCacheAtStop     bool              `json:"cleanCacheAtStop" yaml:"cleanCacheAtStop"`
	ResendAlterQueue     bool              `json:"resendAlterQueue" yaml:"resendAlterQueue"`
	ResendPriority       int               `json:"resendPriority" yaml:"resendPriority"`
	ResendIndicatorField string            `json:"resendIndicatorField" yaml:"resendIndicatorField"`
	ResendDestination    string            `json:"resendDestination" yaml:"resendDestination"`
}

func (*SinkConf) Validate added in v2.2.0

func (sc *SinkConf) Validate(logger api.Logger) error

Validate the configuration and reset to the default value for invalid values.

type SinkInfo added in v2.1.0

type SinkInfo struct {
	HasCompress bool
	HasBatch    bool
}

type SinkInfoNode added in v2.1.0

type SinkInfoNode interface {
	Info() SinkInfo
}

type SliceVal added in v2.3.0

type SliceVal []any

SliceVal this is the content type for slice tuple

func (SliceVal) IsEmpty added in v2.3.0

func (s SliceVal) IsEmpty() bool

func (SliceVal) SetByIndex added in v2.3.0

func (s SliceVal) SetByIndex(index int, val any) bool

type SourceConf added in v2.2.0

type SourceConf struct {
	HttpServerIp   string   `json:"httpServerIp" yaml:"httpServerIp"`
	HttpServerPort int      `json:"httpServerPort" yaml:"httpServerPort"`
	HttpServerTls  *TlsConf `json:"httpServerTls" yaml:"httpServerTls"`
}

func (*SourceConf) Validate added in v2.2.0

func (sc *SourceConf) Validate(logger api.Logger) error

type StreamReader

type StreamReader interface {
	CreateWriter(ctx api.StreamContext, currWriter io.Writer, compression string, encryption string) (io.Writer, error)
}

type StreamWriter

type StreamWriter interface {
	CreateWriter(ctx api.StreamContext, currWriter io.Writer, compression string, encryption string) (io.Writer, error)
}

type SyslogConf added in v2.2.0

type SyslogConf struct {
	Enable  bool   `yaml:"enable"`
	Network string `yaml:"network"`
	Address string `yaml:"address"`
	Tag     string `yaml:"tag"`
	Level   string `yaml:"level"`
}

func (*SyslogConf) Validate added in v2.2.0

func (s *SyslogConf) Validate() error

type TlsConf added in v2.2.0

type TlsConf struct {
	Certfile string `yaml:"certfile"`
	Keyfile  string `yaml:"keyfile"`
}

type TlsConfigurationOptions added in v2.2.0

type TlsConfigurationOptions struct {
	SkipCertVerify       bool            `json:"insecureSkipVerify" yaml:"insecureSkipVerify"`
	CertificationRaw     string          `json:"certificationRaw" yaml:"certificationRaw"`
	PrivateKeyRaw        string          `json:"privateKeyRaw" yaml:"privateKeyRaw"`
	RootCARaw            string          `json:"rootCARaw" yaml:"rootCARaw"`
	CertFile             string          `json:"certificationPath" yaml:"certificationPath"`
	KeyFile              string          `json:"privateKeyPath" yaml:"privateKeyPath"`
	CaFile               string          `json:"rootCaPath" yaml:"rootCaPath"`
	TLSMinVersion        string          `json:"tlsMinVersion" yaml:"tlsMinVersion"`
	RenegotiationSupport string          `json:"renegotiationSupport" yaml:"renegotiationSupport"`
	Decrypt              *EncryptionConf `json:"decrypt" yaml:"decrypt,omitempty"`
	// whether use default tls setting
	Tls string `json:"tls"`
}

func (*TlsConfigurationOptions) GenKeys added in v2.2.0

func (opts *TlsConfigurationOptions) GenKeys() (*TlsKeys, error)

type TlsKeys added in v2.2.0

type TlsKeys struct {
	RawCABytes   []byte
	RawCertBytes []byte
	RawKeyBytes  []byte
}

type UniqueConn

type UniqueConn interface {
	ConnId(props map[string]any) string
}

type UniqueSub

type UniqueSub interface {
	SubId(props map[string]any) string
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL