Fix race condition in E2E test

Signed-off-by: Jonathan West <[email protected]>

Author: jgwest

tests/e2e/plugin_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"github.com/argoproj-labs/rollouts-plugin-trafficrouter-openshift/pkg/plugin"
 	"github.com/argoproj-labs/rollouts-plugin-trafficrouter-openshift/tests/fixtures"
@@ -248,12 +249,17 @@ var _ = Describe("OpenShift Route Traffic Plugin Tests", func() {
 			Eventually(routeA, "20s", "1s").Should(routeFixture.HaveWeights(50, 50))
 			Eventually(routeB, "20s", "1s").Should(routeFixture.HaveWeights(50, 50))
 
+			// There is a race condition in Rollouts where a call to PromoteRollout too quickly will cause 'Rollout phase mismatch, got Paused, want Healthy'
+			// I don't see a way to know when it's possible to promote the Rollout, so for now just wait X seconds
+			Eventually(rollout, "30s", "5s").Should(rolloutFixture.HaveExpectedObservedGeneration())
+			time.Sleep(10 * time.Second)
+
 			By("promote the Rollout for the final time and verify if the previous version is removed")
 			rollout, err = promote.PromoteRollout(rolloutClient.ArgoprojV1alpha1().Rollouts(namespace),
 				rolloutName, false, false, false)
 			Expect(err).ToNot(HaveOccurred())
 
-			Eventually(rollout, "30s", "1s").Should(rolloutFixture.HavePhase(rolloutsv1alpha1.RolloutPhaseHealthy))
+			Eventually(rollout, "2m", "5s").Should(rolloutFixture.HavePhase(rolloutsv1alpha1.RolloutPhaseHealthy))
 
 			By("verify if the traffic weight is updated in the Route")
 			Eventually(routeA, "20s", "1s").Should(routeFixture.HaveWeights(100, 0))

tests/fixtures/rollout/rollout.go

@@ -13,6 +13,38 @@ import (
 	"k8s.io/client-go/util/retry"
 )
 
+func HaveExpectedObservedGeneration() matcher.GomegaMatcher {
+
+	return WithTransform(func(rollout *rolloutsv1alpha1.Rollout) bool {
+		rolloutClient, err := fixtures.GetRolloutsClient()
+		if err != nil {
+			fmt.Println("failed to create the Rollout client", err)
+			return false
+		}
+
+		ctx := context.Background()
+		rollout, err = rolloutClient.ArgoprojV1alpha1().Rollouts(rollout.Namespace).Get(
+			ctx,
+			rollout.Name,
+			metav1.GetOptions{},
+		)
+		if err != nil {
+			fmt.Println("failed to get the Rollout", err)
+			return false
+		}
+
+		generation := fmt.Sprintf("%d", rollout.Generation)
+		observedGeneration := rollout.Status.ObservedGeneration
+
+		if generation != observedGeneration {
+			fmt.Println("HaveExpectedObservedGeneration - observedGeneration does not yet match generation", generation, observedGeneration)
+			return false
+		}
+
+		return true
+	}, BeTrue())
+}
+
 func HavePhase(status rolloutsv1alpha1.RolloutPhase) matcher.GomegaMatcher {
 	return WithTransform(func(rollout *rolloutsv1alpha1.Rollout) bool {
 		rolloutClient, err := fixtures.GetRolloutsClient()
@@ -121,7 +153,7 @@ func HasTransitionedToCanary(expectedReplicas int) matcher.GomegaMatcher {
 					return false
 				}
 
-				if *rs.Spec.Replicas != int32(expectedReplicas) {
+				if (int)(*rs.Spec.Replicas) != expectedReplicas {
 					fmt.Printf("expected the latest ReplicaSet %s to have %d replicas", rs.Name, expectedReplicas)
 					return false
 				}