mco: make MCN condition validation more robust against fast transitions

Author: isabella-janssen

test/extended/machine_config/helpers.go

@@ -762,7 +762,10 @@ func WaitForMCPConditionStatus(oc *exutil.CLI, mcpName string, conditionType mcf
 	return nil
 }
 
-// `WaitForMCNConditionStatus` waits up to a specified timeout for the desired MCN condition to match the desired status (ex. wait until "Updated" is "False")
+// `WaitForMCNConditionStatus` waits up to a specified timeout for the desired MCN condition to
+// match the desired status (ex. wait until "Updated" is "False"). If the desired condition is
+// "Unknown," the function will also return true if the condition is "True," which ensures that we
+// do not fail when an update progresses quickly through the intermediary "Unknown" phase.
 func WaitForMCNConditionStatus(clientSet *machineconfigclient.Clientset, mcnName string, conditionType mcfgv1.StateProgress, status metav1.ConditionStatus,
 	timeout time.Duration, interval time.Duration) (bool, error) {
 
@@ -781,6 +784,12 @@ func WaitForMCNConditionStatus(clientSet *machineconfigclient.Clientset, mcnName
 
 		// Check if the MCN status is as desired
 		conditionMet = CheckMCNConditionStatus(workerNodeMCN, conditionType, status)
+		// If the condition was not met and we are expecting it may have transitioned quickly
+		// trough the "Unknown" phase, check if the condition has flipped to `True`.
+		if !conditionMet && status == metav1.ConditionUnknown {
+			conditionMet = CheckMCNConditionStatus(workerNodeMCN, conditionType, metav1.ConditionTrue)
+			framework.Logf("MCN '%v' %v condition was %v, missed transition through %v.", mcnName, conditionType, metav1.ConditionTrue, status)
+		}
 		return conditionMet, nil
 	}); err != nil {
 		framework.Logf("The desired MCN condition was never met: %v", err)

test/extended/machine_config/machine_config_node.go

@@ -88,8 +88,10 @@ var _ = g.Describe("[sig-mco][OCPFeatureGate:MachineConfigNodes]", func() {
 
 		// When the cluster has machines in the "worker" MCP, use a custom MCP to test the update
 		if slices.Contains(poolNames, worker) {
+			framework.Logf("Validating MCN properties in custom MCP.")
 			ValidateMCNConditionTransitionsOnRebootlessUpdate(oc, clientSet, nodeDisruptionFixture, nodeDisruptionEmptyFixture, customMCFixture, infraMCPFixture)
 		} else { // When there are no machines in the "worker" MCP, test the update by applying a MC targeting the "master" MCP
+			framework.Logf("Validating MCN properties in master MCP.")
 			ValidateMCNConditionTransitionsOnRebootlessUpdateMaster(oc, clientSet, nodeDisruptionFixture, nodeDisruptionEmptyFixture, masterMCFixture)
 		}
 	})
@@ -303,9 +305,7 @@ func validateTransitionThroughConditions(clientSet *machineconfigclient.Clientse
 	framework.Logf("Waiting for UpdateExecuted=Unknown")
 	conditionMet, err = WaitForMCNConditionStatus(clientSet, updatingNodeName, mcfgv1.MachineConfigNodeUpdateExecuted, metav1.ConditionUnknown, 30*time.Second, 1*time.Second)
 	o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Error occured while waiting for UpdateExecuted=Unknown: %v", err))
-	if !conditionMet {
-		framework.Logf("Warning, could not detect UpdateExecuted=Unknown.")
-	}
+	o.Expect(conditionMet).To(o.BeTrue(), "Error, could not detect UpdateExecuted=Unknown.")
 
 	// On standard, non-rebootless, update, check that node transitions through "Cordoned" and "Drained" phases
 	if !isRebootless {
@@ -317,9 +317,7 @@ func validateTransitionThroughConditions(clientSet *machineconfigclient.Clientse
 		framework.Logf("Waiting for Drained=Unknown")
 		conditionMet, err = WaitForMCNConditionStatus(clientSet, updatingNodeName, mcfgv1.MachineConfigNodeUpdateDrained, metav1.ConditionUnknown, 15*time.Second, 1*time.Second)
 		o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Error occured while waiting for Drained=Unknown: %v", err))
-		if !conditionMet {
-			framework.Logf("Warning, could not detect Drained=Unknown.")
-		}
+		o.Expect(conditionMet).To(o.BeTrue(), "Error, could not detect Drained=Unknown.")
 
 		framework.Logf("Waiting for Drained=True")
 		conditionMet, err = WaitForMCNConditionStatus(clientSet, updatingNodeName, mcfgv1.MachineConfigNodeUpdateDrained, metav1.ConditionTrue, 4*time.Minute, 1*time.Second)
@@ -330,9 +328,7 @@ func validateTransitionThroughConditions(clientSet *machineconfigclient.Clientse
 	framework.Logf("Waiting for AppliedFilesAndOS=Unknown")
 	conditionMet, err = WaitForMCNConditionStatus(clientSet, updatingNodeName, mcfgv1.MachineConfigNodeUpdateFilesAndOS, metav1.ConditionUnknown, 30*time.Second, 1*time.Second)
 	o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Error occured while waiting for AppliedFilesAndOS=Unknown: %v", err))
-	if !conditionMet {
-		framework.Logf("Warning, could not detect AppliedFilesAndOS=Unknown.")
-	}
+	o.Expect(conditionMet).To(o.BeTrue(), "Error, could not detect AppliedFilesAndOS=Unknown.")
 
 	framework.Logf("Waiting for AppliedFilesAndOS=True")
 	conditionMet, err = WaitForMCNConditionStatus(clientSet, updatingNodeName, mcfgv1.MachineConfigNodeUpdateFilesAndOS, metav1.ConditionTrue, 3*time.Minute, 1*time.Second)
@@ -354,9 +350,7 @@ func validateTransitionThroughConditions(clientSet *machineconfigclient.Clientse
 		framework.Logf("Waiting for RebootedNode=Unknown")
 		conditionMet, err = WaitForMCNConditionStatus(clientSet, updatingNodeName, mcfgv1.MachineConfigNodeUpdateRebooted, metav1.ConditionUnknown, 15*time.Second, 1*time.Second)
 		o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Error occured while waiting for RebootedNode=Unknown: %v", err))
-		if !conditionMet {
-			framework.Logf("Warning, could not detect RebootedNode=Unknown.")
-		}
+		o.Expect(conditionMet).To(o.BeTrue(), "Error, could not detect RebootedNode=Unknown.")
 
 		framework.Logf("Waiting for RebootedNode=True")
 		conditionMet, err = WaitForMCNConditionStatus(clientSet, updatingNodeName, mcfgv1.MachineConfigNodeUpdateRebooted, metav1.ConditionTrue, 6*time.Minute, 1*time.Second)