Documentation
¶
Index ¶
- Constants
- Variables
- func BuildTestDupeKubeEvent(namespace, pod, reason, msg string, count int) monitorapi.Interval
- func EventExprMatchThresholdTest(testName string, events monitorapi.Intervals, ...) []*junitapi.JUnitTestCase
- func GetClusterInfraInfo(c *rest.Config) (platform v1.PlatformType, topology v1.TopologyMode, err error)
- func GetTimesAnEventHappened(msg monitorapi.Message) int
- func IsDuringAPIServerProgressingOnSNO(topology string, events monitorapi.Intervals) monitorapi.EventIntervalMatchesFunc
- func IsEventAfterInstallation(monitorEvent monitorapi.Interval, kubeClientConfig *rest.Config) (bool, error)
- func MakeProbeTest(testName string, events monitorapi.Intervals, operatorName string, ...) []*junitapi.JUnitTestCase
- func NewSingleEventThresholdCheck(testName string, matcher *SimplePathologicalEventMatcher, ...) *singleEventThresholdCheck
- func TestDuplicatedEventForStableSystem(events monitorapi.Intervals, clientConfig *rest.Config) []*junitapi.JUnitTestCase
- func TestDuplicatedEventForUpgrade(events monitorapi.Intervals, kubeClientConfig *rest.Config) []*junitapi.JUnitTestCase
- type AllowedPathologicalEventRegistry
- func (r *AllowedPathologicalEventRegistry) AddPathologicalEventMatcher(eventMatcher EventMatcher) error
- func (r *AllowedPathologicalEventRegistry) AddPathologicalEventMatcherOrDie(eventMatcher EventMatcher)
- func (r *AllowedPathologicalEventRegistry) AllowedByAny(i monitorapi.Interval, topology v1.TopologyMode) (bool, EventMatcher)
- func (r *AllowedPathologicalEventRegistry) GetMatcherByName(name string) (EventMatcher, error)
- func (r *AllowedPathologicalEventRegistry) MatchesAny(i monitorapi.Interval) (bool, EventMatcher)
- type EventMatcher
- type OverlapOtherIntervalsPathologicalEventMatcher
- type SimplePathologicalEventMatcher
Constants ¶
const ( ImagePullRedhatFlakeThreshold = 5 RequiredResourceMissingFlakeThreshold = 10 BackoffRestartingFlakeThreshold = 10 ErrorUpdatingEndpointSlicesFailedThreshold = -1 // flake only ErrorUpdatingEndpointSlicesFlakeThreshold = 10 DuplicateEventThreshold = 20 PathologicalMark = "pathological/true" InterestingMark = "interesting/true" )
Variables ¶
var AllowBackOffRestartingFailedContainer = &SimplePathologicalEventMatcher{ name: "AllowBackOffRestartingFailedContainer", messageReasonRegex: regexp.MustCompile(`^BackOff$`), messageHumanRegex: regexp.MustCompile(`Back-off restarting failed container`), }
Separated out in testBackoffStartingFailedContainer
var AllowImagePullFromRedHatRegistry = &SimplePathologicalEventMatcher{ name: "AllowImagePullBackOffFromRedHatRegistry", messageHumanRegex: regexp.MustCompile(`Back-off pulling image .*registry.redhat.io`), }
Separated out in testBackoffPullingRegistryRedhatImage
var AllowOVNReadiness = &SimplePathologicalEventMatcher{ name: "OVNReadinessProbeFailed", locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ monitorapi.LocatorNamespaceKey: regexp.MustCompile(`openshift-ovn-kubernetes`), monitorapi.LocatorPodKey: regexp.MustCompile(`ovnkube-node-`), }, messageReasonRegex: regexp.MustCompile(`^Unhealthy$`), messageHumanRegex: regexp.MustCompile(`Readiness probe failed:`), }
var CertificateRotation = &SimplePathologicalEventMatcher{ name: "CertificateRotation", messageReasonRegex: regexp.MustCompile(`^(CABundleUpdateRequired|SignerUpdateRequired|TargetUpdateRequired|CertificateUpdated|CertificateRemoved|CertificateUpdateFailed|CSRCreated|CSRApproved|CertificateRotationStarted|ClientCertificateCreated|NoValidCertificateFound)$`), }
var ErrorUpdatingEndpointSlices = &SimplePathologicalEventMatcher{ name: "ErrorUpdatingEndpointSlices", messageReasonRegex: regexp.MustCompile(`^FailedToUpdateEndpointSlices$`), messageHumanRegex: regexp.MustCompile(`Error updating Endpoint Slices`), }
Separated out in testErrorUpdatingEndpointSlices
var EtcdClusterOperatorStatusChanged = &SimplePathologicalEventMatcher{ name: "EtcdClusterOperatorStatusChanged", locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ monitorapi.LocatorNamespaceKey: regexp.MustCompile(`openshift-etcd`), monitorapi.LocatorPodKey: regexp.MustCompile(`^openshift-etcd`), }, messageReasonRegex: regexp.MustCompile(`^OperatorStatusChanged$`), messageHumanRegex: regexp.MustCompile(`Status for clusteroperator/etcd changed.*No unhealthy members found`), }
reason/OperatorStatusChanged Status for clusteroperator/etcd changed: Degraded message changed from "NodeControllerDegraded: All master nodes are ready\nEtcdMembersDegraded: 2 of 3 members are available, ip-10-0-217-93.us-west-1.compute.internal is unhealthy" to "NodeControllerDegraded: All master nodes are ready\nEtcdMembersDegraded: No unhealthy members found"
var EtcdRequiredResourcesMissing = &SimplePathologicalEventMatcher{ name: "EtcdRequiredResourcesMissing", messageReasonRegex: regexp.MustCompile(`^RequiredInstallerResourcesMissing$`), }
Separated out in testRequiredInstallerResourcesMissing
var FailedScheduling = &SimplePathologicalEventMatcher{ name: "FailedScheduling", messageReasonRegex: regexp.MustCompile(`^FailedScheduling$`), messageHumanRegex: regexp.MustCompile(`nodes are available.*didn't match Pod's node affinity/selector`), }
reason/FailedScheduling 0/6 nodes are available: 2 node(s) didn't match Pod's node affinity/selector, 2 node(s) didn't match pod anti-affinity rules, 2 node(s) were unschedulable. preemption: 0/6 nodes are available: 2 node(s) didn't match pod anti-affinity rules, 4 Preemption is not helpful for scheduling..
var KubeAPIServerAvoids500s = &SimplePathologicalEventMatcher{ name: "KubeAPIServerAvoids500s", messageReasonRegex: regexp.MustCompile(fmt.Sprintf("^%s$", monitorapi.ReasonKubeAPIServer500s)), }
Separated out in auditloganalyzer
var MarketplaceStartupProbeFailure = &SimplePathologicalEventMatcher{ name: "MarketplaceStartupProbeFailure", locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ monitorapi.LocatorNamespaceKey: regexp.MustCompile(`openshift-marketplace`), monitorapi.LocatorPodKey: regexp.MustCompile(`(community-operators|redhat-operators)-[a-z0-9-]+`), }, messageHumanRegex: regexp.MustCompile(`Startup probe failed`), }
Separated out in testMarketplaceStartupProbeFailure
var NodeHasNoDiskPressure = &SimplePathologicalEventMatcher{ name: "NodeHasNoDiskPressure", messageReasonRegex: regexp.MustCompile(`^NodeHasNoDiskPressure$`), messageHumanRegex: regexp.MustCompile(`status is now: NodeHasNoDiskPressure`), }
Separated out in testNodeHasNoDiskPressure
var NodeHasSufficientMemory = &SimplePathologicalEventMatcher{ name: "NodeHasSufficientMemory", messageReasonRegex: regexp.MustCompile(`^NodeHasSufficientMemory$`), messageHumanRegex: regexp.MustCompile(`status is now: NodeHasSufficientMemory`), }
Separated out in testNodeHasSufficientMemory
var NodeHasSufficientPID = &SimplePathologicalEventMatcher{ name: "NodeHasSufficientPID", messageReasonRegex: regexp.MustCompile(`^NodeHasSufficientPID$`), messageHumanRegex: regexp.MustCompile(`status is now: NodeHasSufficientPID`), }
Separated out in testNodeHasSufficientPID
var ProbeErrorConnectionRefused = &SimplePathologicalEventMatcher{ name: "ProbeErrorConnectionRefused", locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ monitorapi.LocatorNamespaceKey: regexp.MustCompile(`(openshift-config-operator|openshift-oauth-apiserver)`), }, messageReasonRegex: regexp.MustCompile(`^ProbeError$`), messageHumanRegex: regexp.MustCompile(`Readiness probe error.*connection refused`), }
ProbeErrorConnectionRefused matches events in specific namespaces.
These namespaces have their own tests where you'll see this matcher re-used with additional checks on the namespace.
var ProbeErrorLiveness = &SimplePathologicalEventMatcher{ name: "ProbeErrorLiveness", locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ monitorapi.LocatorNamespaceKey: regexp.MustCompile(`(openshift-config-operator|openshift-oauth-apiserver)`), }, messageReasonRegex: regexp.MustCompile(`^(ProbeError|Unhealthy)$`), messageHumanRegex: regexp.MustCompile(`Liveness probe error.*Client.Timeout exceeded while awaiting headers`), }
ProbeErrorLiveness matches events in specific namespaces such as: Liveness probe error: Get "https://10.128.0.21:8443/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
These namespaces have their own tests where you'll see this matcher re-used with additional checks on the namespace.
var ProbeErrorTimeoutAwaitingHeaders = &SimplePathologicalEventMatcher{ name: "ProbeErrorTimeoutAwaitingHeaders", locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ monitorapi.LocatorNamespaceKey: regexp.MustCompile(`(openshift-config-operator|openshift-oauth-apiserver)`), }, messageReasonRegex: regexp.MustCompile(`^ProbeError$`), messageHumanRegex: regexp.MustCompile(`Readiness probe error.*Client.Timeout exceeded while awaiting headers`), }
ProbeErrorTimeoutAwaitingHeaders matches events in specific namespaces such as: reason/ProbeError Readiness probe error: Get "https://10.130.0.15:8443/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
These namespaces have their own tests where you'll see this matcher re-used with additional checks on the namespace.
var ReadinessFailed = &SimplePathologicalEventMatcher{ name: "ReadinessFailed", locatorKeyRegexes: map[monitorapi.LocatorKey]*regexp.Regexp{ monitorapi.LocatorNamespaceKey: regexp.MustCompile(`(openshift-config-operator|openshift-oauth-apiserver)`), }, messageReasonRegex: regexp.MustCompile(`^ReadinessFailed$`), messageHumanRegex: regexp.MustCompile(`Get.*healthz.*net/http.*request canceled while waiting for connection.*Client.Timeout exceeded`), }
ReadinessFailed matches events in specific namespaces such as: ...ReadinessFailed Get \"https://10.130.0.16:8443/healthz\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)
These namespaces have their own tests where you'll see this matcher re-used with additional checks on the namespace.
Functions ¶
func BuildTestDupeKubeEvent ¶
func BuildTestDupeKubeEvent(namespace, pod, reason, msg string, count int) monitorapi.Interval
BuildTestDupeKubeEvent is a test utility to make the process of creating these specific intervals a little more brief.
func EventExprMatchThresholdTest ¶
func EventExprMatchThresholdTest(testName string, events monitorapi.Intervals, matcher *SimplePathologicalEventMatcher, eventFlakeThreshold int) []*junitapi.JUnitTestCase
func GetClusterInfraInfo ¶
func GetClusterInfraInfo(c *rest.Config) (platform v1.PlatformType, topology v1.TopologyMode, err error)
func GetTimesAnEventHappened ¶
func GetTimesAnEventHappened(msg monitorapi.Message) int
func IsDuringAPIServerProgressingOnSNO ¶
func IsDuringAPIServerProgressingOnSNO(topology string, events monitorapi.Intervals) monitorapi.EventIntervalMatchesFunc
func IsEventAfterInstallation ¶
func IsEventAfterInstallation(monitorEvent monitorapi.Interval, kubeClientConfig *rest.Config) (bool, error)
IsEventAfterInstallation returns true if the monitorEvent represents an event that happened after installation.
func MakeProbeTest ¶
func MakeProbeTest(testName string, events monitorapi.Intervals, operatorName string, matcher *SimplePathologicalEventMatcher, eventFlakeThreshold int) []*junitapi.JUnitTestCase
func NewSingleEventThresholdCheck ¶
func NewSingleEventThresholdCheck(testName string, matcher *SimplePathologicalEventMatcher, failThreshold, flakeThreshold int) *singleEventThresholdCheck
func TestDuplicatedEventForStableSystem ¶
func TestDuplicatedEventForStableSystem(events monitorapi.Intervals, clientConfig *rest.Config) []*junitapi.JUnitTestCase
func TestDuplicatedEventForUpgrade ¶
func TestDuplicatedEventForUpgrade(events monitorapi.Intervals, kubeClientConfig *rest.Config) []*junitapi.JUnitTestCase
Types ¶
type AllowedPathologicalEventRegistry ¶
type AllowedPathologicalEventRegistry struct {
// contains filtered or unexported fields
}
func NewUniversalPathologicalEventMatchers ¶
func NewUniversalPathologicalEventMatchers(kubeConfig *rest.Config, finalIntervals monitorapi.Intervals) *AllowedPathologicalEventRegistry
AllowedPathologicalEvents is the list of all allowed duplicate events on all jobs. Upgrade has an additional list which is combined with this one.
func NewUpgradePathologicalEventMatchers ¶
func NewUpgradePathologicalEventMatchers(kubeConfig *rest.Config, finalIntervals monitorapi.Intervals) *AllowedPathologicalEventRegistry
NewUpgradePathologicalEventMatchers creates the registry for allowed events during upgrade. Contains everything in the universal set as well.
func (*AllowedPathologicalEventRegistry) AddPathologicalEventMatcher ¶
func (r *AllowedPathologicalEventRegistry) AddPathologicalEventMatcher(eventMatcher EventMatcher) error
func (*AllowedPathologicalEventRegistry) AddPathologicalEventMatcherOrDie ¶
func (r *AllowedPathologicalEventRegistry) AddPathologicalEventMatcherOrDie(eventMatcher EventMatcher)
func (*AllowedPathologicalEventRegistry) AllowedByAny ¶
func (r *AllowedPathologicalEventRegistry) AllowedByAny( i monitorapi.Interval, topology v1.TopologyMode) (bool, EventMatcher)
AllowedByAny checks if the given event locator and message match any registered matcher, and if the repeating event should be allowed. Returns true if so, the matcher name, and the matcher itself.
func (*AllowedPathologicalEventRegistry) GetMatcherByName ¶
func (r *AllowedPathologicalEventRegistry) GetMatcherByName(name string) (EventMatcher, error)
func (*AllowedPathologicalEventRegistry) MatchesAny ¶
func (r *AllowedPathologicalEventRegistry) MatchesAny(i monitorapi.Interval) (bool, EventMatcher)
MatchesAny checks if the given event locator and message match any registered matcher. Returns true if so, the matcher name, and the matcher itself. It does NOT check if the interval should be allowed.
type EventMatcher ¶
type EventMatcher interface {
// Name returns a unique name (enforced by registry) for this matcher.
Name() string
// Matches returns true if the given interval looks relevant to this matcher, but does not
// indicate the interval should be allowed to repeat pathologically. For that use Allows.
Matches(i monitorapi.Interval) bool
// Allows returns true if the given interval should be allowed to repeat as many times
// as it did. It performs the Matches, check, and layers in additional logic from runtime.
Allows(i monitorapi.Interval, topology v1.TopologyMode) bool
}
type OverlapOtherIntervalsPathologicalEventMatcher ¶
type OverlapOtherIntervalsPathologicalEventMatcher struct {
// contains filtered or unexported fields
}
OverlapOtherIntervalsPathologicalEventMatcher is an implementation containing a regular matcher, plus additional logic that will allow the event only if it is contained within another set of intervals provided. (i.e. used to allow FailedScheduling pathological events if they are contained within NodeUpdate intervals)
func (*OverlapOtherIntervalsPathologicalEventMatcher) Allows ¶
func (ade *OverlapOtherIntervalsPathologicalEventMatcher) Allows(i monitorapi.Interval, topology v1.TopologyMode) bool
func (*OverlapOtherIntervalsPathologicalEventMatcher) Matches ¶
func (ade *OverlapOtherIntervalsPathologicalEventMatcher) Matches(i monitorapi.Interval) bool
func (*OverlapOtherIntervalsPathologicalEventMatcher) Name ¶
func (ade *OverlapOtherIntervalsPathologicalEventMatcher) Name() string
type SimplePathologicalEventMatcher ¶
type SimplePathologicalEventMatcher struct {
// contains filtered or unexported fields
}
SimplePathologicalEventMatcher allows the definition of kube event intervals that can repeat more than the threshold we allow during a job run. All specified fields must match the interval for it to be allowed.
func (*SimplePathologicalEventMatcher) Allows ¶
func (ade *SimplePathologicalEventMatcher) Allows(i monitorapi.Interval, topology v1.TopologyMode) bool
Allows checks if the given locator/messagebuilder matches this allowed dupe event, and if the interval should be allowed to repeat pathologically.
func (*SimplePathologicalEventMatcher) Matches ¶
func (ade *SimplePathologicalEventMatcher) Matches(i monitorapi.Interval) bool
func (*SimplePathologicalEventMatcher) Name ¶
func (ade *SimplePathologicalEventMatcher) Name() string