Documentation
¶
Overview ¶
Package cli sets up the CLI commands for the fleet apply binary.
Index ¶
- Constants
- func App() *cobra.Command
- func NewApply() *cobra.Command
- func NewCleanUp() *cobra.Command
- func NewClusterRegistration() *cobra.Command
- func NewDeploy() *cobra.Command
- func NewGitjob() *cobra.Command
- func NewMonitor() *cobra.Command
- func NewTarget() *cobra.Command
- func NewTest() *cobra.Command
- type APIConsistency
- type Apply
- type BundleDeploymentInfo
- type BundleInfo
- type BundleInputArgs
- type Cleanup
- type ClusterGroupInfo
- type ClusterInfo
- type ClusterRegistration
- type ContentInfo
- type ContentIssue
- type ControllerInfo
- type Deploy
- type Diagnostics
- type EventInfo
- type Fleet
- type FleetClient
- type GitRepoInfo
- type Gitjob
- type Monitor
- type OutputArgsNoDefault
- type ResourceWithFinalizers
- type Resources
- type SecretInfo
- type Target
- type Test
Constants ¶
const ( JSONOutputEnvVar = "FLEET_JSON_OUTPUT" JobNameEnvVar = "JOB_NAME" )
Variables ¶
This section is empty.
Functions ¶
func NewCleanUp ¶
NewCleanup returns a subcommand to `cleanup` cluster registrations
func NewClusterRegistration ¶ added in v0.10.3
func NewDeploy ¶ added in v0.10.0
NewDeploy returns a subcommand to deploy a bundledeployment/content resource to a cluster.
func NewMonitor ¶
Types ¶
type APIConsistency ¶
type Apply ¶
type Apply struct {
FleetClient
BundleInputArgs
OutputArgsNoDefault
Label map[string]string `usage:"Labels to apply to created bundles" short:"l"`
TargetsFile string `usage:"Addition source of targets and restrictions to be append"`
Compress bool `usage:"Force all resources to be compress" short:"c"`
ServiceAccount string `usage:"Service account to assign to bundle created" short:"a"`
SyncGeneration int `usage:"Generation number used to force sync the deployment"`
TargetNamespace string `usage:"Ensure this bundle goes to this target namespace"`
Paused bool `usage:"Create bundles in a paused state"`
Commit string `usage:"Commit to assign to the bundle" env:"COMMIT"`
Username string `usage:"Basic auth username for helm repo" env:"HELM_USERNAME"`
PasswordFile string `usage:"Path of file containing basic auth password for helm repo"`
CACertsFile string `usage:"Path of custom cacerts for helm repo" name:"cacerts-file"`
SSHPrivateKeyFile string `usage:"Path of ssh-private-key for helm repo" name:"ssh-privatekey-file"`
HelmRepoURLRegex string `` /* 166-byte string literal not displayed */
KeepResources bool `usage:"Keep resources created after the GitRepo or Bundle is deleted" name:"keep-resources"`
DeleteNamespace bool `usage:"Delete GitRepo target namespace after the GitRepo or Bundle is deleted" name:"delete-namespace"`
HelmCredentialsByPathFile string `usage:"Path of file containing helm credentials for paths" name:"helm-credentials-by-path-file"`
HelmBasicHTTP bool `usage:"Uses plain HTTP connections when downloading from helm repositories" name:"helm-basic-http"`
HelmInsecureSkipTLS bool `usage:"Skip TLS verification when downloading from helm repositories" name:"helm-insecure-skip-tls"`
CorrectDrift bool `usage:"Rollback any change made from outside of Fleet" name:"correct-drift"`
CorrectDriftForce bool `usage:"Use --force when correcting drift. Resources can be deleted and recreated" name:"correct-drift-force"`
CorrectDriftKeepFailHistory bool `usage:"Keep helm history for failed rollbacks" name:"correct-drift-keep-fail-history"`
OCIRegistrySecret string `usage:"OCI storage registry secret name" name:"oci-registry-secret"`
DrivenScan bool `usage:"Use driven scan. Bundles are defined by the user" name:"driven-scan"`
DrivenScanSeparator string `usage:"Separator to use for bundle folder and options file" name:"driven-scan-sep" default:":"`
}
func (*Apply) PersistentPre ¶ added in v0.10.0
type BundleDeploymentInfo ¶
type BundleDeploymentInfo struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
UID string `json:"uid"`
Generation int64 `json:"generation"`
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
Commit string `json:"commit,omitempty"`
ForceSyncGeneration int64 `json:"forceSyncGeneration,omitempty"`
SyncGeneration *int64 `json:"syncGeneration,omitempty"`
DeploymentID string `json:"deploymentID,omitempty"`
StagedDeploymentID string `json:"stagedDeploymentID,omitempty"`
AppliedDeploymentID string `json:"appliedDeploymentID,omitempty"`
DeletionTimestamp *string `json:"deletionTimestamp,omitempty"`
Finalizers []string `json:"finalizers,omitempty"`
Ready bool `json:"ready"`
ReadyMessage string `json:"readyMessage,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
}
type BundleInfo ¶
type BundleInfo struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
UID string `json:"uid"`
Generation int64 `json:"generation"`
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
Commit string `json:"commit,omitempty"`
ForceSyncGeneration int64 `json:"forceSyncGeneration,omitempty"`
ResourcesSHA256Sum string `json:"resourcesSHA256Sum,omitempty"`
DeletionTimestamp *string `json:"deletionTimestamp,omitempty"`
Finalizers []string `json:"finalizers,omitempty"`
Ready bool `json:"ready"`
ReadyMessage string `json:"readyMessage,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
}
type BundleInputArgs ¶
type ClusterGroupInfo ¶
type ClusterInfo ¶
type ClusterInfo struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
AgentNamespace string `json:"agentNamespace,omitempty"`
AgentLastSeen *string `json:"agentLastSeen,omitempty"`
AgentLastSeenAge string `json:"agentLastSeenAge,omitempty"`
Ready bool `json:"ready"`
ReadyMessage string `json:"readyMessage,omitempty"`
BundleDeployments int `json:"bundleDeployments,omitempty"`
ReadyBundleDeployments int `json:"readyBundleDeployments,omitempty"`
}
type ClusterRegistration ¶ added in v0.10.3
type ClusterRegistration struct {
FleetClient
Min string `usage:"Minimum delay between deletes (default: 10ms)" name:"min"`
Max string `usage:"Maximum delay between deletes (default: 5s)" name:"max"`
Factor string `usage:"Factor to increase delay between deletes (default: 1.1)" name:"factor"`
}
func (*ClusterRegistration) PersistentPre ¶ added in v0.10.3
func (r *ClusterRegistration) PersistentPre(_ *cobra.Command, _ []string) error
type ContentInfo ¶
type ContentIssue ¶
type ContentIssue struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
ContentName string `json:"contentName,omitempty"`
StagedContentName string `json:"stagedContentName,omitempty"`
AppliedContentName string `json:"appliedContentName,omitempty"`
ContentExists bool `json:"contentExists,omitempty"`
ContentDeletionTimestamp *string `json:"contentDeletionTimestamp,omitempty"`
StagedContentExists *bool `json:"stagedContentExists,omitempty"`
AppliedContentExists *bool `json:"appliedContentExists,omitempty"`
Issues []string `json:"issues"`
}
type ControllerInfo ¶
type Deploy ¶ added in v0.10.0
type Deploy struct {
InputFile string `usage:"Location of the YAML file containing the content and the bundledeployment resource" short:"i"`
DryRun bool `usage:"Print the resources that would be deployed, but do not actually deploy them" short:"d"`
Namespace string `usage:"Set the default namespace. Deploy helm chart into this namespace." short:"n"`
KubeVersion string `usage:"For dry runs, sets the Kubernetes version to assume when validating Chart Kubernetes version constraints."`
// AgentNamespace is set as an annotation on the chart.yaml in the helm release. Fleet-agent will manage charts with a matching label.
AgentNamespace string `` /* 182-byte string literal not displayed */
}
type Diagnostics ¶
type Diagnostics struct {
StuckBundles []BundleInfo `json:"stuckBundles,omitempty"`
StuckBundleDeployments []BundleDeploymentInfo `json:"stuckBundleDeployments,omitempty"`
GitRepoBundleInconsistencies []BundleInfo `json:"gitrepoBundleInconsistencies,omitempty"`
InvalidSecretOwners []SecretInfo `json:"invalidSecretOwners,omitempty"`
ResourcesWithMultipleFinalizers []ResourceWithFinalizers `json:"resourcesWithMultipleFinalizers,omitempty"`
LargeBundles []BundleInfo `json:"largeBundles,omitempty"`
BundlesWithMissingContent []BundleInfo `json:"bundlesWithMissingContent,omitempty"`
BundlesWithNoDeployments []BundleInfo `json:"bundlesWithNoDeployments,omitempty"`
GitReposWithNoBundles []GitRepoInfo `json:"gitreposWithNoBundles,omitempty"`
ClustersWithAgentIssues []ClusterInfo `json:"clustersWithAgentIssues,omitempty"`
ClusterGroupsWithNoClusters []ClusterGroupInfo `json:"clustergroupsWithNoClusters,omitempty"`
BundlesWithMissingGitRepo []BundleInfo `json:"bundlesWithMissingGitrepo,omitempty"`
BundleDeploymentsWithMissingBundle []BundleDeploymentInfo `json:"bundledeploymentsWithMissingBundle,omitempty"`
GitReposWithGenerationMismatch []GitRepoInfo `json:"gitreposWithGenerationMismatch,omitempty"`
BundlesWithGenerationMismatch []BundleInfo `json:"bundlesWithGenerationMismatch,omitempty"`
OrphanedSecretsCount int `json:"orphanedSecretsCount,omitempty"`
InvalidSecretOwnersCount int `json:"invalidSecretOwnersCount,omitempty"`
ContentIssuesCount int `json:"contentIssuesCount,omitempty"`
GitRepoBundleInconsistenciesCount int `json:"gitrepoBundleInconsistenciesCount,omitempty"`
ResourcesWithMultipleFinalizersCount int `json:"resourcesWithMultipleFinalizersCount,omitempty"`
BundlesWithDeletionTimestamp int `json:"bundlesWithDeletionTimestamp,omitempty"`
BundleDeploymentsWithDeletionTimestamp int `json:"bundleDeploymentsWithDeletionTimestamp,omitempty"`
ContentsWithDeletionTimestamp int `json:"contentsWithDeletionTimestamp,omitempty"`
}
type EventInfo ¶
type EventInfo struct {
Namespace string `json:"namespace"`
Type string `json:"type"`
Reason string `json:"reason"`
Message string `json:"message"`
InvolvedKind string `json:"involvedKind"`
InvolvedName string `json:"involvedName"`
Count int32 `json:"count"`
LastTimestamp string `json:"lastTimestamp,omitempty"`
}
type FleetClient ¶ added in v0.10.0
type FleetClient struct {
command.DebugConfig
Namespace string `usage:"namespace" env:"NAMESPACE" default:"fleet-local" short:"n"`
Kubeconfig string `usage:"kubeconfig for authentication" short:"k"`
Context string `usage:"kubeconfig context for authentication"`
}
type GitRepoInfo ¶
type GitRepoInfo struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
Generation int64 `json:"generation"`
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
Commit string `json:"commit,omitempty"`
ForceSyncGeneration int64 `json:"forceSyncGeneration,omitempty"`
Ready bool `json:"ready"`
ReadyMessage string `json:"readyMessage,omitempty"`
}
type Gitjob ¶ added in v0.10.3
type Gitjob struct {
FleetClient
BatchSize int `usage:"Number of git jobs to retrieve at once" name:"batch-size" default:"5000"`
}
func (*Gitjob) PersistentPre ¶ added in v0.10.3
type Monitor ¶
type Monitor struct {
FleetClient
Watch bool `usage:"Watch for changes and output continuously"`
Interval int `usage:"Interval in seconds between checks when watching (default: 5)" default:"5"`
System string `usage:"Fleet system namespace (default: cattle-fleet-system)" default:"cattle-fleet-system"`
}
Monitor provides diagnostic monitoring of Fleet resources to identify issues with bundle deployments.
Overview ¶
The monitor command collects and analyzes Fleet resources to diagnose why bundles get stuck during the targeting and deployment phases. It outputs compact JSON snapshots containing only the diagnostic-relevant fields from Fleet resources, making it easy to identify issues without the verbosity of full Kubernetes resource definitions. Use jq or similar tools to format and query the output.
What It Detects ¶
The monitor command checks for all critical issues that can cause bundles to get stuck:
- API Time Travel: Detects when the Kubernetes API server returns stale cached resource versions
- Old forceSyncGeneration: Identifies bundles that haven't updated to match their GitRepo's forceSyncGeneration
- UID Mismatches: Finds secrets with invalid owner references (owner was deleted and recreated)
- Deletion Timestamps: Tracks resources stuck with deletion timestamps due to blocking finalizers
- Stale Commit Hashes: Detects bundles that haven't updated to their GitRepo's latest commit
- DeploymentID Mismatches: Identifies BundleDeployments where spec.deploymentID != status.appliedDeploymentID
- Missing Content: Detects when referenced Content resources are missing or have deletion timestamps
- Controller Restarts: Tracks Fleet controller pod restarts which can indicate cache issues
- Multiple Finalizers: Identifies GitRepos, Bundles, and BundleDeployments with more than one finalizer (indicates a bug; only Contents use multiple finalizers for ref counting)
- Large Bundles: Bundles with content >1MB that may cause etcd performance issues
- Missing Content Resources: Bundles with resourcesSHA256Sum but no corresponding Content resource
- Target Matching Issues: Bundles with zero BundleDeployments (no clusters matched), GitRepos with zero Bundles, ClusterGroups with zero clusters
- Agent Connectivity: Clusters with non-ready agents, missing lastSeen timestamps, or stale lastSeen (>24h)
- Broken Ownership Chains: Bundles without GitRepos, BundleDeployments without Bundles
- Generation Mismatches: GitRepos and Bundles where generation != observedGeneration (reconciliation not progressing)
When to Use ¶
Use the monitor command when:
- Bundles are stuck and not deploying to target clusters
- GitRepo shows Ready but bundles aren't updating
- BundleDeployments are not applying new deploymentIDs
- Need to capture the state of Fleet resources for troubleshooting
- Debugging issues before/after making changes to GitRepos or bundles
- Investigating why resources have old commits or forceSyncGeneration values
- Checking if bundles are matching target clusters correctly
- Diagnosing agent connectivity issues
- Investigating etcd performance problems with large bundles
Output Format ¶
The command outputs compact JSON (use jq to format) with the following structure:
- timestamp: When the snapshot was taken (RFC3339)
- controller: Fleet controller pod status (name, restarts, uptime)
- gitrepos: Array of GitRepo info (generation, observedGeneration, commit, forceSyncGeneration, ready status)
- bundles: Array of Bundle info (UID, generation, observedGeneration, commit, resourcesSHA256Sum, finalizers, ready status)
- bundledeployments: Array of BundleDeployment info (UID, generation, forceSyncGeneration, syncGeneration, deploymentIDs, ready status)
- contents: Array of Content info (name, size, finalizers, deletion timestamps)
- clusters: Array of Cluster info (agent status, lastSeen, age, ready status, bundledeployment counts)
- clustergroups: Array of ClusterGroup info (cluster counts, selector)
- bundleSecrets: Array of bundle lifecycle secrets (commit, owners, finalizers)
- orphanedSecrets: Secrets with invalid owner references or deletion timestamps
- contentIssues: BundleDeployments with missing or problematic Content resources
- apiConsistency: Results of API server consistency check (detects time travel)
- recentEvents: Last 20 Fleet-related Kubernetes events
- diagnostics: Summary of all detected issues with arrays of affected resources
Usage Examples ¶
Basic usage - single snapshot:
fleetcli monitor | jq
Watch mode - continuous monitoring every 5 seconds:
fleetcli monitor --watch --interval 5
Monitor a specific namespace:
fleetcli monitor -n fleet-local | jq
Analyze the output with jq:
# Show summary with formatting
fleetcli monitor | jq '{timestamp, controller, diagnostics}'
# Check stuck bundles
fleetcli monitor | jq '.diagnostics.stuckBundles'
# Find bundles with old commits
fleetcli monitor | jq '.diagnostics.gitrepoBundleInconsistencies'
# Check for large bundles
fleetcli monitor | jq '.diagnostics.largeBundles'
# Check agent connectivity issues
fleetcli monitor | jq '.diagnostics.clustersWithAgentIssues'
# Check target matching problems
fleetcli monitor | jq '{
bundlesWithNoDeployments: (.diagnostics.bundlesWithNoDeployments | length),
gitreposWithNoBundles: (.diagnostics.gitreposWithNoBundles | length),
clustergroupsWithNoClusters: (.diagnostics.clustergroupsWithNoClusters | length)
}'
# Check generation mismatches
fleetcli monitor | jq '{
gitrepos: .diagnostics.gitreposWithGenerationMismatch,
bundles: .diagnostics.bundlesWithGenerationMismatch
}'
# Monitor agent health over time
while true; do
fleetcli monitor | jq '.clusters[] | {name, agentLastSeenAge, ready}'
sleep 30
done
# Check API consistency
fleetcli monitor | jq '.apiConsistency'
Compare before/after snapshots:
# Before change
fleetcli monitor > before.json
# Make your change (e.g., update GitRepo)
kubectl edit gitrepo/my-repo -n fleet-local
# After change
fleetcli monitor > after.json
# Compare commits
diff <(jq '.bundles[] | {name, commit}' before.json) \
<(jq '.bundles[] | {name, commit}' after.json)
Troubleshooting Workflow ¶
1. Capture initial state:
fleetcli monitor > initial.json
2. Check diagnostics for issues:
cat initial.json | jq '.diagnostics'
3. If bundles are stuck, check specific issues:
# Check for generation mismatches
cat initial.json | jq '.diagnostics.stuckBundles[] | {name, generation, observedGeneration}'
# Check for deployment ID mismatches
cat initial.json | jq '.diagnostics.stuckBundleDeployments[] | {name, deploymentID, appliedDeploymentID}'
# Check for orphaned secrets
cat initial.json | jq '.orphanedSecrets[] | {name, ownerUID}'
4. Fix the identified issues and capture new state:
fleetcli monitor > fixed.json
5. Verify the fix by comparing:
diff <(jq -S . initial.json) <(jq -S . fixed.json)
Output Field Reference ¶
GitRepoInfo fields:
- namespace, name: Resource identifiers
- generation, observedGeneration: Track reconciliation status
- commit: Current commit hash from the Git repository
- forceSyncGeneration: Used to force redeployment
- ready: Whether the GitRepo is in Ready state
- readyMessage: Error message if not ready
BundleInfo fields:
- namespace, name, uid: Resource identifiers
- generation, observedGeneration: Track reconciliation status
- commit: Commit hash from GitRepo (should match GitRepo.status.commit)
- forceSyncGeneration: Should match GitRepo.spec.forceSyncGeneration
- resourcesSHA256Sum: Hash of bundle resources
- deletionTimestamp: Set when bundle is being deleted
- finalizers: Finalizers blocking deletion
- ready: Whether bundle is in Ready state
- readyMessage, errorMessage: Error details if not ready
BundleDeploymentInfo fields:
- namespace, name, uid: Resource identifiers
- generation: Resource generation (incremented on spec changes)
- observedGeneration: Same as syncGeneration (kept for compatibility)
- commit: Commit hash from bundle
- forceSyncGeneration: Spec value for forcing redeployment (from spec.options.forceSyncGeneration)
- syncGeneration: Status value tracking if forceSyncGeneration has been applied (from status.syncGeneration)
- deploymentID: Target deployment (spec.deploymentID) - the content that should be deployed
- stagedDeploymentID: Staged deployment before applying
- appliedDeploymentID: Currently applied deployment (should eventually match deploymentID)
- deletionTimestamp: Set when being deleted
- finalizers: Finalizers blocking deletion
- ready: Whether deployment is in Ready state
- readyMessage, errorMessage: Error details if not ready
Note: syncGeneration tracks forceSyncGeneration application, NOT resource generation. A BundleDeployment is stuck if:
- forceSyncGeneration > 0 and syncGeneration != forceSyncGeneration (forced sync not applied)
- deploymentID != appliedDeploymentID (new content not applied)
- deletionTimestamp is set (being deleted but finalizers blocking)
Related Commands ¶
For live cluster monitoring, also consider:
- kubectl get bundles -A -w: Watch bundle resources
- kubectl get bundledeployments -A -w: Watch bundledeployment resources
- kubectl describe bundle <name>: Detailed bundle information
- kubectl logs -n cattle-fleet-system deploy/fleet-controller: Controller logs
type OutputArgsNoDefault ¶
type OutputArgsNoDefault struct {
Output string `usage:"Output contents to file or - for stdout" short:"o"`
}
type ResourceWithFinalizers ¶
type Resources ¶
type Resources struct {
Timestamp string `json:"timestamp"`
Controller *ControllerInfo `json:"controller,omitempty"`
GitRepos []GitRepoInfo `json:"gitrepos,omitempty"`
Bundles []BundleInfo `json:"bundles,omitempty"`
BundleDeployments []BundleDeploymentInfo `json:"bundledeployments,omitempty"`
Contents []ContentInfo `json:"contents,omitempty"`
Clusters []ClusterInfo `json:"clusters,omitempty"`
ClusterGroups []ClusterGroupInfo `json:"clustergroups,omitempty"`
BundleSecrets []SecretInfo `json:"bundleSecrets,omitempty"`
OrphanedSecrets []SecretInfo `json:"orphanedSecrets,omitempty"`
ContentIssues []ContentIssue `json:"contentIssues,omitempty"`
APIConsistency *APIConsistency `json:"apiConsistency,omitempty"`
RecentEvents []EventInfo `json:"recentEvents,omitempty"`
Diagnostics *Diagnostics `json:"diagnostics,omitempty"`
}
type SecretInfo ¶
type SecretInfo struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
Type string `json:"type"`
Commit string `json:"commit,omitempty"`
DeletionTimestamp *string `json:"deletionTimestamp,omitempty"`
Finalizers []string `json:"finalizers,omitempty"`
OwnerKind string `json:"ownerKind,omitempty"`
OwnerName string `json:"ownerName,omitempty"`
OwnerUID string `json:"ownerUID,omitempty"`
}
type Target ¶ added in v0.10.0
type Target struct {
BundleFile string `usage:"Location of the Bundle resource yaml" short:"b"`
DumpInputList bool `usage:"Dump the live resources, which impact targeting, like clusters, as YAML" short:"l"`
Namespace string `usage:"Override the namespace of the bundle. Targeting searches this namespace for clusters." short:"n"`
}
type Test ¶
type Test struct {
BundleInputArgs
Quiet bool `usage:"Just print the match and don't print the resources" short:"q"`
Group string `usage:"Cluster group to match against" short:"g"`
Name string `usage:"Cluster name to match against" short:"N"`
Label map[string]string `usage:"Cluster labels to match against" short:"l"`
GroupLabel map[string]string `usage:"Cluster group labels to match against" short:"L"`
Target string `usage:"Explicit target to match" short:"t"`
}
Source Files
¶
Directories
¶
| Path | Synopsis |
|---|---|
|
Package apply creates bundle resources from gitrepo resources.
|
Package apply creates bundle resources from gitrepo resources. |
|
Package match is used to test matching a bundles to a target on the command line.
|
Package match is used to test matching a bundles to a target on the command line. |
|
Package writer provides a writer that can be used to write to a file or stdout.
|
Package writer provides a writer that can be used to write to a file or stdout. |