pathologicaleventlibrary

package
v1.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 29, 2026 License: Apache-2.0 Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ImagePullRedhatFlakeThreshold              = 5
	RequiredResourceMissingFlakeThreshold      = 10
	BackoffRestartingFlakeThreshold            = 10
	ErrorUpdatingEndpointSlicesFailedThreshold = -1 // flake only
	ErrorUpdatingEndpointSlicesFlakeThreshold  = 10

	DuplicateEventThreshold = 20
	PathologicalMark        = "pathological/true"
	InterestingMark         = "interesting/true"
)

Variables

View Source
var AllowBackOffRestartingFailedContainer = &SimplePathologicalEventMatcher{
	name:               "AllowBackOffRestartingFailedContainer",
	messageReasonRegex: regexp.MustCompile(`^BackOff$`),
	messageHumanRegex:  regexp.MustCompile(`Back-off restarting failed container`),
}

Separated out in testBackoffStartingFailedContainer

View Source
var AllowImagePullFromRedHatRegistry = &SimplePathologicalEventMatcher{
	name:              "AllowImagePullBackOffFromRedHatRegistry",
	messageHumanRegex: regexp.MustCompile(`Back-off pulling image .*registry.redhat.io`),
}

Separated out in testBackoffPullingRegistryRedhatImage

View Source
var AllowOVNReadiness = &SimplePathologicalEventMatcher{
	name: "OVNReadinessProbeFailed",
	locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{
		monitorapi.LocatorNamespaceKey: regexp.MustCompile(`openshift-ovn-kubernetes`),
		monitorapi.LocatorPodKey:       regexp.MustCompile(`ovnkube-node-`),
	},
	messageReasonRegex: regexp.MustCompile(`^Unhealthy$`),
	messageHumanRegex:  regexp.MustCompile(`Readiness probe failed:`),
}
View Source
var CertificateRotation = &SimplePathologicalEventMatcher{
	name:               "CertificateRotation",
	messageReasonRegex: regexp.MustCompile(`^(CABundleUpdateRequired|SignerUpdateRequired|TargetUpdateRequired|CertificateUpdated|CertificateRemoved|CertificateUpdateFailed|CSRCreated|CSRApproved|CertificateRotationStarted|ClientCertificateCreated|NoValidCertificateFound)$`),
}
View Source
var ErrorUpdatingEndpointSlices = &SimplePathologicalEventMatcher{
	name:               "ErrorUpdatingEndpointSlices",
	messageReasonRegex: regexp.MustCompile(`^FailedToUpdateEndpointSlices$`),
	messageHumanRegex:  regexp.MustCompile(`Error updating Endpoint Slices`),
}

Separated out in testErrorUpdatingEndpointSlices

View Source
var EtcdClusterOperatorStatusChanged = &SimplePathologicalEventMatcher{
	name: "EtcdClusterOperatorStatusChanged",
	locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{
		monitorapi.LocatorNamespaceKey: regexp.MustCompile(`openshift-etcd`),
		monitorapi.LocatorPodKey:       regexp.MustCompile(`^openshift-etcd`),
	},
	messageReasonRegex: regexp.MustCompile(`^OperatorStatusChanged$`),
	messageHumanRegex:  regexp.MustCompile(`Status for clusteroperator/etcd changed.*No unhealthy members found`),
}

reason/OperatorStatusChanged Status for clusteroperator/etcd changed: Degraded message changed from "NodeControllerDegraded: All master nodes are ready\nEtcdMembersDegraded: 2 of 3 members are available, ip-10-0-217-93.us-west-1.compute.internal is unhealthy" to "NodeControllerDegraded: All master nodes are ready\nEtcdMembersDegraded: No unhealthy members found"

View Source
var EtcdRequiredResourcesMissing = &SimplePathologicalEventMatcher{
	name:               "EtcdRequiredResourcesMissing",
	messageReasonRegex: regexp.MustCompile(`^RequiredInstallerResourcesMissing$`),
}

Separated out in testRequiredInstallerResourcesMissing

View Source
var FailedScheduling = &SimplePathologicalEventMatcher{
	name:               "FailedScheduling",
	messageReasonRegex: regexp.MustCompile(`^FailedScheduling$`),
	messageHumanRegex:  regexp.MustCompile(`nodes are available.*didn't match Pod's node affinity/selector`),
}

reason/FailedScheduling 0/6 nodes are available: 2 node(s) didn't match Pod's node affinity/selector, 2 node(s) didn't match pod anti-affinity rules, 2 node(s) were unschedulable. preemption: 0/6 nodes are available: 2 node(s) didn't match pod anti-affinity rules, 4 Preemption is not helpful for scheduling..

View Source
var KubeAPIServerAvoids500s = &SimplePathologicalEventMatcher{
	name:               "KubeAPIServerAvoids500s",
	messageReasonRegex: regexp.MustCompile(fmt.Sprintf("^%s$", monitorapi.ReasonKubeAPIServer500s)),
}

Separated out in auditloganalyzer

View Source
var MarketplaceStartupProbeFailure = &SimplePathologicalEventMatcher{
	name: "MarketplaceStartupProbeFailure",
	locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{
		monitorapi.LocatorNamespaceKey: regexp.MustCompile(`openshift-marketplace`),
		monitorapi.LocatorPodKey:       regexp.MustCompile(`(community-operators|redhat-operators)-[a-z0-9-]+`),
	},
	messageHumanRegex: regexp.MustCompile(`Startup probe failed`),
}

Separated out in testMarketplaceStartupProbeFailure

View Source
var NodeHasNoDiskPressure = &SimplePathologicalEventMatcher{
	name:               "NodeHasNoDiskPressure",
	messageReasonRegex: regexp.MustCompile(`^NodeHasNoDiskPressure$`),
	messageHumanRegex:  regexp.MustCompile(`status is now: NodeHasNoDiskPressure`),
}

Separated out in testNodeHasNoDiskPressure

View Source
var NodeHasSufficientMemory = &SimplePathologicalEventMatcher{
	name:               "NodeHasSufficientMemory",
	messageReasonRegex: regexp.MustCompile(`^NodeHasSufficientMemory$`),
	messageHumanRegex:  regexp.MustCompile(`status is now: NodeHasSufficientMemory`),
}

Separated out in testNodeHasSufficientMemory

View Source
var NodeHasSufficientPID = &SimplePathologicalEventMatcher{
	name:               "NodeHasSufficientPID",
	messageReasonRegex: regexp.MustCompile(`^NodeHasSufficientPID$`),
	messageHumanRegex:  regexp.MustCompile(`status is now: NodeHasSufficientPID`),
}

Separated out in testNodeHasSufficientPID

View Source
var ProbeErrorConnectionRefused = &SimplePathologicalEventMatcher{
	name: "ProbeErrorConnectionRefused",
	locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{
		monitorapi.LocatorNamespaceKey: regexp.MustCompile(`(openshift-config-operator|openshift-oauth-apiserver)`),
	},
	messageReasonRegex: regexp.MustCompile(`^ProbeError$`),
	messageHumanRegex:  regexp.MustCompile(`Readiness probe error.*connection refused`),
}

ProbeErrorConnectionRefused matches events in specific namespaces.

These namespaces have their own tests where you'll see this matcher re-used with additional checks on the namespace.

View Source
var ProbeErrorLiveness = &SimplePathologicalEventMatcher{
	name: "ProbeErrorLiveness",
	locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{
		monitorapi.LocatorNamespaceKey: regexp.MustCompile(`(openshift-config-operator|openshift-oauth-apiserver)`),
	},
	messageReasonRegex: regexp.MustCompile(`^(ProbeError|Unhealthy)$`),
	messageHumanRegex:  regexp.MustCompile(`Liveness probe error.*Client.Timeout exceeded while awaiting headers`),
}

ProbeErrorLiveness matches events in specific namespaces such as: Liveness probe error: Get "https://10.128.0.21:8443/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)

These namespaces have their own tests where you'll see this matcher re-used with additional checks on the namespace.

View Source
var ProbeErrorTimeoutAwaitingHeaders = &SimplePathologicalEventMatcher{
	name: "ProbeErrorTimeoutAwaitingHeaders",
	locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{
		monitorapi.LocatorNamespaceKey: regexp.MustCompile(`(openshift-config-operator|openshift-oauth-apiserver)`),
	},
	messageReasonRegex: regexp.MustCompile(`^ProbeError$`),
	messageHumanRegex:  regexp.MustCompile(`Readiness probe error.*Client.Timeout exceeded while awaiting headers`),
}

ProbeErrorTimeoutAwaitingHeaders matches events in specific namespaces such as: reason/ProbeError Readiness probe error: Get "https://10.130.0.15:8443/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)

These namespaces have their own tests where you'll see this matcher re-used with additional checks on the namespace.

View Source
var ReadinessFailed = &SimplePathologicalEventMatcher{
	name: "ReadinessFailed",
	locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{
		monitorapi.LocatorNamespaceKey: regexp.MustCompile(`(openshift-config-operator|openshift-oauth-apiserver)`),
	},
	messageReasonRegex: regexp.MustCompile(`^ReadinessFailed$`),
	messageHumanRegex:  regexp.MustCompile(`Get.*healthz.*net/http.*request canceled while waiting for connection.*Client.Timeout exceeded`),
}

ReadinessFailed matches events in specific namespaces such as: ...ReadinessFailed Get \"https://10.130.0.16:8443/healthz\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)

These namespaces have their own tests where you'll see this matcher re-used with additional checks on the namespace.

Functions

func BuildTestDupeKubeEvent

func BuildTestDupeKubeEvent(namespace, pod, reason, msg string, count int) monitorapi.Interval

BuildTestDupeKubeEvent is a test utility to make the process of creating these specific intervals a little more brief.

func EventExprMatchThresholdTest

func EventExprMatchThresholdTest(testName string, events monitorapi.Intervals, matcher *SimplePathologicalEventMatcher, eventFlakeThreshold int) []*junitapi.JUnitTestCase

func GetClusterInfraInfo

func GetClusterInfraInfo(c *rest.Config) (platform v1.PlatformType, topology v1.TopologyMode, err error)

func GetTimesAnEventHappened

func GetTimesAnEventHappened(msg monitorapi.Message) int

func IsDuringAPIServerProgressingOnSNO

func IsDuringAPIServerProgressingOnSNO(topology string, events monitorapi.Intervals) monitorapi.EventIntervalMatchesFunc

func IsEventAfterInstallation

func IsEventAfterInstallation(monitorEvent monitorapi.Interval, kubeClientConfig *rest.Config) (bool, error)

IsEventAfterInstallation returns true if the monitorEvent represents an event that happened after installation.

func MakeProbeTest

func MakeProbeTest(testName string, events monitorapi.Intervals, operatorName string,
	matcher *SimplePathologicalEventMatcher, eventFlakeThreshold int) []*junitapi.JUnitTestCase

func NewSingleEventThresholdCheck

func NewSingleEventThresholdCheck(testName string, matcher *SimplePathologicalEventMatcher, failThreshold, flakeThreshold int) *singleEventThresholdCheck

func TestDuplicatedEventForStableSystem

func TestDuplicatedEventForStableSystem(events monitorapi.Intervals, clientConfig *rest.Config) []*junitapi.JUnitTestCase

func TestDuplicatedEventForUpgrade

func TestDuplicatedEventForUpgrade(events monitorapi.Intervals, kubeClientConfig *rest.Config) []*junitapi.JUnitTestCase

Types

type AllowedPathologicalEventRegistry

type AllowedPathologicalEventRegistry struct {
	// contains filtered or unexported fields
}

func NewUniversalPathologicalEventMatchers

func NewUniversalPathologicalEventMatchers(kubeConfig *rest.Config, finalIntervals monitorapi.Intervals) *AllowedPathologicalEventRegistry

AllowedPathologicalEvents is the list of all allowed duplicate events on all jobs. Upgrade has an additional list which is combined with this one.

func NewUpgradePathologicalEventMatchers

func NewUpgradePathologicalEventMatchers(kubeConfig *rest.Config, finalIntervals monitorapi.Intervals) *AllowedPathologicalEventRegistry

NewUpgradePathologicalEventMatchers creates the registry for allowed events during upgrade. Contains everything in the universal set as well.

func (*AllowedPathologicalEventRegistry) AddPathologicalEventMatcher

func (r *AllowedPathologicalEventRegistry) AddPathologicalEventMatcher(eventMatcher EventMatcher) error

func (*AllowedPathologicalEventRegistry) AddPathologicalEventMatcherOrDie

func (r *AllowedPathologicalEventRegistry) AddPathologicalEventMatcherOrDie(eventMatcher EventMatcher)

func (*AllowedPathologicalEventRegistry) AllowedByAny

AllowedByAny checks if the given event locator and message match any registered matcher, and if the repeating event should be allowed. Returns true if so, the matcher name, and the matcher itself.

func (*AllowedPathologicalEventRegistry) GetMatcherByName

func (r *AllowedPathologicalEventRegistry) GetMatcherByName(name string) (EventMatcher, error)

func (*AllowedPathologicalEventRegistry) MatchesAny

MatchesAny checks if the given event locator and message match any registered matcher. Returns true if so, the matcher name, and the matcher itself. It does NOT check if the interval should be allowed.

type EventMatcher

type EventMatcher interface {
	// Name returns a unique name (enforced by registry) for this matcher.
	Name() string

	// Matches returns true if the given interval looks relevant to this matcher, but does not
	// indicate the interval should be allowed to repeat pathologically. For that use Allows.
	Matches(i monitorapi.Interval) bool

	// Allows returns true if the given interval should be allowed to repeat as many times
	// as it did. It performs the Matches, check, and layers in additional logic from runtime.
	Allows(i monitorapi.Interval, topology v1.TopologyMode) bool
}

type OverlapOtherIntervalsPathologicalEventMatcher

type OverlapOtherIntervalsPathologicalEventMatcher struct {
	// contains filtered or unexported fields
}

OverlapOtherIntervalsPathologicalEventMatcher is an implementation containing a regular matcher, plus additional logic that will allow the event only if it is contained within another set of intervals provided. (i.e. used to allow FailedScheduling pathological events if they are contained within NodeUpdate intervals)

func (*OverlapOtherIntervalsPathologicalEventMatcher) Allows

func (*OverlapOtherIntervalsPathologicalEventMatcher) Matches

func (*OverlapOtherIntervalsPathologicalEventMatcher) Name

type SimplePathologicalEventMatcher

type SimplePathologicalEventMatcher struct {
	// contains filtered or unexported fields
}

SimplePathologicalEventMatcher allows the definition of kube event intervals that can repeat more than the threshold we allow during a job run. All specified fields must match the interval for it to be allowed.

func (*SimplePathologicalEventMatcher) Allows

Allows checks if the given locator/messagebuilder matches this allowed dupe event, and if the interval should be allowed to repeat pathologically.

func (*SimplePathologicalEventMatcher) Matches

func (*SimplePathologicalEventMatcher) Name

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL