services

package
v1.5.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 19, 2026 License: Apache-2.0 Imports: 11 Imported by: 0

Documentation

Overview

Package services provides etcd utilities: error classification, retry detection, and learner state handling.

Package services provides hypervisor utilities: configuration validation, SSH connectivity checks, and virsh availability verification.

Package services provides virsh/libvirt utilities: VM lifecycle, inspection, network config, and recreation via SSH.

Package services provides Pacemaker utilities: cluster status, etcd resource management, STONITH control, and job handling via SSH.

Index

Constants

This section is empty.

Variables

View Source 
var VMStateList = []VMState{
	VMStateUnknown,
	VMStateRunning,
	VMStateShutOff,
}

Functions

func CycleRemovedNode 

func CycleRemovedNode(failedNodeName, failedNodeIP, runningNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) error

CycleRemovedNode removes and re-adds a node in the pacemaker cluster configuration. 

err := CycleRemovedNode("master-0", "192.168.111.20", runningNodeIP, sshConfig, localKH, remoteKH)

func DeleteAfterSetupJob 

func DeleteAfterSetupJob(afterSetupJobName string, oc *exutil.CLI) error

DeleteAfterSetupJob deletes the TNF after-setup job for a node from openshift-etcd namespace. 

err := DeleteAfterSetupJob("tnf-after-setup-job-master-0", oc)

func DeleteAuthJob 

func DeleteAuthJob(authJobName string, oc *exutil.CLI) error

DeleteAuthJob deletes the TNF authentication job for a node from openshift-etcd namespace. 

err := DeleteAuthJob("tnf-auth-job-master-0", oc)

func DeleteJob 

func DeleteJob(jobName, namespace string, oc *exutil.CLI) error

DeleteJob deletes a Kubernetes job from a specified namespace. 

err := DeleteJob("tnf-auth-job-master-0", "openshift-etcd", oc)

func ExtractMACAddressFromXML 

func ExtractMACAddressFromXML(xmlContent string, networkBridge string) (string, error)

ExtractMACAddressFromXML extracts the MAC address for a network bridge from VM XML. 

mac, err := ExtractMACAddressFromXML(xmlContent, "ostestpr")

func FindVMByNodeName 

func FindVMByNodeName(nodeName string, sshConfig *core.SSHConfig, knownHostsPath string) (string, error)

FindVMByNodeName finds a VM that corresponds to an OpenShift node. Handles both simple names (master-0) and FQDNs (master-0.ostest.test.metalkube.org). Matches VM names like "ostest_master_0" by extracting the short name and converting dashes to underscores.

func GetVMNameByMACMatch 

func GetVMNameByMACMatch(nodeName, nodeMAC string, networkBridge string, sshConfig *core.SSHConfig, knownHostsPath string) (string, error)

GetVMNameByMACMatch finds the VM name with a specific MAC address by searching all VMs. 

vmName, err := GetVMNameByMACMatch("master-0", "52:54:00:12:34:56", "ostestpr", sshConfig, knownHostsPath)

func GetVMNetworkInfo 

func GetVMNetworkInfo(vmName string, networkBridge string, sshConfig *core.SSHConfig, knownHostsPath string) (string, string, error)

GetVMNetworkInfo retrieves the UUID and MAC address for a VM's network interface. 

uuid, mac, err := GetVMNetworkInfo("master-0", "ostestpr", sshConfig, knownHostsPath)

func IsEtcdLearnerError 

func IsEtcdLearnerError(err error) bool

IsEtcdLearnerError checks if an error is related to transient etcd learner state issues. 

err := RetryWithOptions(func() error { return createEtcdSecret(oc, secretFile) }, RetryOptions{ShouldRetry: IsEtcdLearnerError}, "create etcd secret")

func IsRetryableEtcdError 

func IsRetryableEtcdError(err error) bool

IsRetryableEtcdError checks if an etcd error should be retried (learner, network, timeout, rate limit). 

err := RetryWithOptions(func() error { return etcdOp() }, RetryOptions{ShouldRetry: IsRetryableEtcdError}, "etcd operation")

func PcsDebugRestart 

func PcsDebugRestart(remoteNodeIP string, fullOutput bool, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsDebugRestart restores etcd quorum by performing debug-stop then debug-start. 

stdout, stderr, err := PcsDebugRestart(nodeIP, false, sshConfig, localKH, remoteKH)

func PcsDebugStart 

func PcsDebugStart(remoteNodeIP string, fullOutput bool, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsDebugStart restores etcd quorum by performing debug-start (bypasses cluster checks for two-node scenarios). 

stdout, stderr, err := PcsDebugStart(nodeIP, false, sshConfig, localKH, remoteKH)

func PcsDebugStop 

func PcsDebugStop(remoteNodeIP string, fullOutput bool, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsDebugStop stops the etcd resource using debug-stop (controlled shutdown without triggering recovery). 

stdout, stderr, err := PcsDebugStop(nodeIP, false, sshConfig, localKH, remoteKH)

func PcsJournal 

func PcsJournal(pcsJournalTailLines int, remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsJournal retrieves the last N lines of pacemaker journal logs filtered for podman-etcd. 

stdout, stderr, err := PcsJournal(100, nodeIP, sshConfig, localKH, remoteKH)

func PcsProperty 

func PcsProperty(remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsProperty gets cluster properties from pacemaker. 

stdout, stderr, err := PcsProperty(nodeIP, sshConfig, localKH, remoteKH)

func PcsResourceCleanup 

func PcsResourceCleanup(remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsResourceCleanup cleans up resource failures in the pacemaker cluster. 

stdout, stderr, err := PcsResourceCleanup(nodeIP, sshConfig, localKH, remoteKH)

func PcsResourceCleanupViaDebug 

func PcsResourceCleanupViaDebug(ctx context.Context, oc *exutil.CLI, nodeName string) (string, error)

PcsResourceCleanupViaDebug clears any failed actions in the cluster's resource manager. This is the debug container equivalent of PcsResourceCleanup for use when SSH is unavailable. 

output, err := PcsResourceCleanupViaDebug(ctx, oc, "master-0")

func PcsResourceStatus 

func PcsResourceStatus(nodeName, remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsResourceStatus retrieves the status of a specific pacemaker resource (etcd) on a node. This is more targeted than PcsStatus and shows whether the etcd resource is started/stopped.

func PcsStatus 

func PcsStatus(remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsStatus retrieves the overall pacemaker cluster status. This shows the state of all cluster resources, nodes, and any failures.

func PcsStatusFull 

func PcsStatusFull(remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsStatusFull retrieves the full pacemaker cluster status with additional details. This includes all nodes, resources, fence status, and any pending operations.

func PcsStatusViaDebug 

func PcsStatusViaDebug(ctx context.Context, oc *exutil.CLI, nodeName string) (string, error)

PcsStatusViaDebug retrieves the overall pacemaker cluster status via debug container. This shows the state of all cluster resources, nodes, and any failures. Use instead of PcsStatus when SSH to the hypervisor is unavailable. 

output, err := PcsStatusViaDebug(ctx, oc, "master-0")

func PcsStonithCleanup 

func PcsStonithCleanup(remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsStonithCleanup cleans up STONITH failures in the pacemaker cluster. 

stdout, stderr, err := PcsStonithCleanup(nodeIP, sshConfig, localKH, remoteKH)

func PcsStonithCleanupViaDebug 

func PcsStonithCleanupViaDebug(ctx context.Context, oc *exutil.CLI, nodeName string) (string, error)

PcsStonithCleanupViaDebug clears any failed STONITH (fencing) actions in the cluster. This is the debug container equivalent of PcsStonithCleanup for use when SSH is unavailable. 

output, err := PcsStonithCleanupViaDebug(ctx, oc, "master-0")

func PcsStonithConfirm 

func PcsStonithConfirm(targetNodeName, remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsStonithConfirm manually confirms to the pacemaker cluster that a node has been fenced. This is used when the fencing device cannot fence the node (e.g., when the node has been destroyed and the BMC is unreachable). The --force flag bypasses interactive confirmation.

WARNING: Only use this when you have manually verified the node is powered off and has no access to shared resources. Using this incorrectly can cause data corruption. 

stdout, stderr, err := PcsStonithConfirm("master-1", nodeIP, sshConfig, localKH, remoteKH)

func PcsStonithDisable 

func PcsStonithDisable(remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsStonithDisable disables STONITH in the pacemaker cluster. 

stdout, stderr, err := PcsStonithDisable(nodeIP, sshConfig, localKH, remoteKH)

func PcsStonithEnable 

func PcsStonithEnable(remoteNodeIP string, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) (string, string, error)

PcsStonithEnable enables STONITH in the pacemaker cluster. 

stdout, stderr, err := PcsStonithEnable(nodeIP, sshConfig, localKH, remoteKH)

func PrintHypervisorConfigUsage 

func PrintHypervisorConfigUsage()

PrintHypervisorConfigUsage prints usage instructions for configuring hypervisor SSH access. Call this when HasHypervisorConfig() returns false to provide configuration guidance.

func VerifyHypervisorAvailability 

func VerifyHypervisorAvailability(sshConfig *core.SSHConfig, knownHostsPath string) error

VerifyHypervisorAvailability verifies SSH connectivity and checks virsh/libvirt availability. 

err := VerifyHypervisorAvailability(sshConfig, knownHostsPath)

func VerifyVirsh 

func VerifyVirsh(sshConfig *core.SSHConfig, knownHostsPath string) (string, error)

VerifyVirsh checks if virsh is available by executing 'virsh version'. 

output, err := VerifyVirsh(sshConfig, knownHostsPath)

func VirshAutostartVM 

func VirshAutostartVM(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) error

VirshAutostartVM enables autostart for a VM (starts on hypervisor boot). 

err := VirshAutostartVM("master-0", sshConfig, knownHostsPath)

func VirshCommand 

func VirshCommand(command string, sshConfig *core.SSHConfig, knownHostsPath string) (string, error)

VirshCommand executes a virsh command on the remote hypervisor via SSH. 

output, err := VirshCommand("list --all", sshConfig, knownHostsPath)

func VirshDefineVM 

func VirshDefineVM(xmlFilePath string, sshConfig *core.SSHConfig, knownHostsPath string) error

VirshDefineVM defines a new VM from an XML configuration file. 

err := VirshDefineVM("/tmp/master-0.xml", sshConfig, knownHostsPath)

func VirshDestroyVM 

func VirshDestroyVM(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) error

VirshDestroyVM forcefully stops a running VM (equivalent to power-off). 

err := VirshDestroyVM("master-0", sshConfig, knownHostsPath)

func VirshDumpXML 

func VirshDumpXML(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) (string, error)

VirshDumpXML retrieves the XML configuration of a VM. 

xml, err := VirshDumpXML("master-0", sshConfig, knownHostsPath)

func VirshGetVMUUID 

func VirshGetVMUUID(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) (string, error)

VirshGetVMUUID retrieves the UUID of a VM. 

uuid, err := VirshGetVMUUID("master-0", sshConfig, knownHostsPath)

func VirshList 

func VirshList(sshConfig *core.SSHConfig, knownHostsPath string, flags ...VirshListFlag) (string, error)

VirshList lists VMs on the hypervisor with configurable output format. Pass VirshListFlagAll to include inactive domains, VirshListFlagName to output only names.

Examples: 

vmList, err := VirshList(sshConfig, knownHostsPath, VirshListFlagAll, VirshListFlagName)  // names only
vmList, err := VirshList(sshConfig, knownHostsPath, VirshListFlagAll)                    // table with state

func VirshShutdownVM 

func VirshShutdownVM(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) error

VirshShutdownVM gracefully shuts down a running VM (allows guest OS to shutdown cleanly). 

err := VirshShutdownVM("master-0", sshConfig, knownHostsPath)

func VirshStartVM 

func VirshStartVM(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) error

VirshStartVM starts a defined VM. 

err := VirshStartVM("master-0", sshConfig, knownHostsPath)

func VirshUndefineVM 

func VirshUndefineVM(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) error

VirshUndefineVM undefines a VM (removes libvirt config, not disk images). 

err := VirshUndefineVM("master-0", sshConfig, knownHostsPath)

func VirshVMExists 

func VirshVMExists(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) (string, error)

VirshVMExists checks if a VM with the given name exists on the hypervisor. 

output, err := VirshVMExists("master-0", sshConfig, knownHostsPath)

func WaitForEtcdRevisionCreation 

func WaitForEtcdRevisionCreation(targetNodeIP string, timeout, pollInterval time.Duration, hypervisorConfig *core.SSHConfig, hypervisorKnownHostsPath, targetNodeKnownHostsPath string, oc *exutil.CLI) error

WaitForEtcdRevisionCreation polls until the /var/lib/etcd/revision.json file exists on the target node. This file is created by the Cluster Etcd Operator (CEO) after the node joins the cluster. 

err := WaitForEtcdRevisionCreation(targetNodeIP, tenMinuteTimeout, thirtySecondPollInterval, &hypervisorConfig, hypervisorKnownHosts, targetNodeKnownHosts, oc)

func WaitForJobCompletion 

func WaitForJobCompletion(jobName, namespace string, timeout, pollInterval time.Duration, oc *exutil.CLI) error

WaitForJobCompletion waits for a Kubernetes job to complete by polling status until Complete or Failed. 

err := WaitForJobCompletion("tnf-auth-job-master-0", "openshift-etcd", 5*time.Minute, 10*time.Second, oc)

func WaitForNodesOnline 

func WaitForNodesOnline(nodeNames []string, remoteNodeIP string, timeout, pollInterval time.Duration, sshConfig *core.SSHConfig, localKnownHostsPath, remoteKnownHostsPath string) error

WaitForNodesOnline waits for all specified nodes to be online in the pacemaker cluster by polling XML status. 

err := WaitForNodesOnline([]string{"master-0", "master-1"}, nodeIP, 5*time.Minute, 10*time.Second, sshConfig, localKH, remoteKH)

func WaitForVMState 

func WaitForVMState(vmName string, vmState VMState, timeout time.Duration, pollInterval time.Duration, sshConfig *core.SSHConfig, knownHostsPath string) error

WaitForVMState waits for a VM to reach a given state by polling domstate.

Types

type Devices 

type Devices struct {
	Interfaces []Interface `xml:"interface"`
}

Devices contains the hardware devices attached to a VM

type Domain 

type Domain struct {
	XMLName xml.Name `xml:"domain"`
	Name    string   `xml:"name"`
	UUID    string   `xml:"uuid"`
	Devices Devices  `xml:"devices"`
}

Domain represents a libvirt domain (virtual machine) configuration.

type EtcdErrorType 

type EtcdErrorType int

EtcdErrorType categorizes etcd errors for granular error handling. 

const (
	// EtcdErrorUnknown represents an unclassified error
	EtcdErrorUnknown EtcdErrorType = iota

	// EtcdErrorLearner represents errors related to etcd learner state transitions
	// These are typically transient and should be retried
	EtcdErrorLearner

	// EtcdErrorQuorum represents errors related to etcd cluster quorum issues
	// These may require operator intervention but can sometimes be retried
	EtcdErrorQuorum

	// EtcdErrorNetwork represents network connectivity errors
	// These are typically transient and should be retried
	EtcdErrorNetwork

	// EtcdErrorTimeout represents timeout errors
	// These are typically transient and should be retried
	EtcdErrorTimeout

	// EtcdErrorRateLimit represents rate limiting or overload errors
	// These should be retried with backoff
	EtcdErrorRateLimit
)

func ClassifyEtcdError 

func ClassifyEtcdError(err error) EtcdErrorType

ClassifyEtcdError categorizes an etcd error into a specific error type for intelligent retry strategies. 

errorType := ClassifyEtcdError(err)

func (EtcdErrorType) String 

func (e EtcdErrorType) String() string

String returns a human-readable name for the error type

type Interface 

type Interface struct {
	Type   string `xml:"type,attr"`
	MAC    MAC    `xml:"mac"`
	Source Source `xml:"source"`
}

Interface represents a network interface configuration in libvirt XML

type MAC 

type MAC struct {
	Address string `xml:"address,attr"`
}

MAC contains the MAC address of a network interface

type Source 

type Source struct {
	Bridge string `xml:"bridge,attr"`
}

Source specifies the network source (bridge, network, etc) for an interface

type VMState 

type VMState string
const (
	VMStateUnknown VMState = "unknown"
	VMStateRunning VMState = "running"
	VMStateShutOff VMState = "shut off"
)

func GetVMState 

func GetVMState(vmName string, sshConfig *core.SSHConfig, knownHostsPath string) (VMState, error)

GetVMState returns the current state of the VM.

type VirshListFlag 

type VirshListFlag string

VirshListFlag represents optional flags for the virsh list command. 

const (
	// VirshListFlagAll includes inactive domains in the output (--all)
	VirshListFlagAll VirshListFlag = "--all"
	// VirshListFlagName outputs only domain names, one per line (--name)
	VirshListFlagName VirshListFlag = "--name"
)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.