Add NodePool STS permissions test

Author: Amarthya Valija

pkg/e2e/verify/nodepool.go

@@ -0,0 +1,324 @@
+package verify
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	viper "github.com/openshift/osde2e/pkg/common/concurrentviper"
+	"github.com/openshift/osde2e/pkg/common/config"
+	"github.com/openshift/osde2e/pkg/common/expect"
+	"github.com/openshift/osde2e/pkg/common/helper"
+	"github.com/openshift/osde2e/pkg/common/label"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+)
+
+const (
+	nodePoolLabel      = "hypershift.openshift.io/nodePool"
+	hostedClusterLabel = "hypershift.openshift.io/hosted-cluster"
+)
+
+var _ = ginkgo.Describe("[Suite: e2e] NodePool STS Permissions", ginkgo.Ordered, label.HyperShift, label.E2E, func() {
+	var h *helper.H
+	var client *resources.Resources
+	var clusterNamespace string
+	var testNodePoolName string
+
+	nodePoolGVR := schema.GroupVersionResource{
+		Group:    "hypershift.openshift.io",
+		Version:  "v1beta1",
+		Resource: "nodepools",
+	}
+
+	ginkgo.BeforeAll(func() {
+		if !viper.GetBool(config.Hypershift) {
+			ginkgo.Skip("NodePool tests are only supported on HyperShift clusters")
+		}
+
+		if viper.GetString(config.CloudProvider.CloudProviderID) != "aws" {
+			ginkgo.Skip("NodePool STS tests only supported on AWS HyperShift clusters")
+		}
+
+		h = helper.New()
+		client = h.AsUser("")
+
+		clusterNamespace = getClusterNamespace(h)
+		if clusterNamespace == "" {
+			ginkgo.Fail("Could not determine cluster namespace - cluster may not be properly configured")
+		}
+
+		testNodePoolName = fmt.Sprintf("test-%d", time.Now().Unix()%100000)
+	})
+
+	ginkgo.AfterAll(func() {
+		ctx := context.Background()
+		ginkgo.By("Cleaning up test NodePool")
+
+		err := h.Dynamic().Resource(nodePoolGVR).Namespace(clusterNamespace).
+			Delete(ctx, testNodePoolName, metav1.DeleteOptions{})
+		if err != nil && !apierrors.IsNotFound(err) {
+			ginkgo.GinkgoLogr.Error(err, "Failed to cleanup test NodePool", "name", testNodePoolName)
+		}
+		_ = wait.For(func(ctx context.Context) (bool, error) {
+			_, err := h.Dynamic().Resource(nodePoolGVR).Namespace(clusterNamespace).
+				Get(ctx, testNodePoolName, metav1.GetOptions{})
+			return apierrors.IsNotFound(err), nil
+		}, wait.WithTimeout(3*time.Minute), wait.WithInterval(5*time.Second))
+	})
+
+	ginkgo.It("should create NodePool with proper STS permissions", func(ctx context.Context) {
+		ginkgo.By("Creating test NodePool to validate STS permissions")
+
+		subnet := getExistingSubnet(ctx, h, clusterNamespace)
+		nodePoolSpec := buildNodePoolSpec(testNodePoolName, clusterNamespace, subnet)
+		nodePoolObj := &unstructured.Unstructured{Object: nodePoolSpec}
+
+		_, err := h.Dynamic().Resource(nodePoolGVR).Namespace(clusterNamespace).
+			Create(ctx, nodePoolObj, metav1.CreateOptions{})
+
+		if detectSTSPermissionError(err) {
+			expect.NoError(fmt.Errorf(
+				"STS_PERMISSION_ERROR: NodePool creation failed due to missing AWS permissions (ec2:RunInstances, ec2:CreateTags): %v. "+
+				"This blocks release until STS policies are updated", err))
+		}
+		expect.NoError(err, "NodePool creation failed")
+
+		ginkgo.GinkgoLogr.Info("NodePool created successfully - STS permissions validated", 
+			"nodepool", testNodePoolName, "permissions", []string{"ec2:RunInstances", "ec2:CreateTags"})
+	})
+
+	ginkgo.It("should provision nodes with AWS integration", func(ctx context.Context) {
+		ginkgo.By("Waiting for NodePool to become Ready")
+
+		err := wait.For(func(ctx context.Context) (bool, error) {
+			np, err := h.Dynamic().Resource(nodePoolGVR).Namespace(clusterNamespace).
+				Get(ctx, testNodePoolName, metav1.GetOptions{})
+			if err != nil { return false, err }
+			ready, _, _ := unstructured.NestedInt64(np.Object, "status", "readyReplicas")
+			return ready >= 1, nil
+		}, wait.WithTimeout(10*time.Minute), wait.WithInterval(15*time.Second))
+		expect.NoError(err, "NodePool did not report ready replicas")
+
+		ginkgo.By("Waiting for nodes to be provisioned")
+
+		var testNode *corev1.Node
+		err = wait.For(func(ctx context.Context) (bool, error) {
+			var nodeList corev1.NodeList
+			err := client.List(ctx, &nodeList)
+			if err != nil {
+				return false, err
+			}
+
+			for _, node := range nodeList.Items {
+				if labelValue, exists := node.Labels[nodePoolLabel]; exists {
+					if strings.Contains(labelValue, testNodePoolName) && isNodeReady(node) {
+						testNode = &node
+						return true, nil
+					}
+				}
+			}
+			return false, nil
+		}, wait.WithTimeout(15*time.Minute), wait.WithInterval(30*time.Second))
+
+		if detectSTSPermissionError(err) {
+			expect.NoError(fmt.Errorf(
+				"STS_PERMISSION_ERROR: Node provisioning failed due to missing AWS permissions (ec2:RunInstances): %v. "+
+				"This blocks release until STS policies are updated", err))
+		}
+		expect.NoError(err, "NodePool failed to provision nodes")
+
+		ginkgo.By("Validating AWS integration")
+		Expect(testNode).ToNot(BeNil(), "Test node should be available")
+
+		// Validate AWS provider ID
+		Expect(testNode.Spec.ProviderID).To(HavePrefix("aws://"),
+			"Node should have AWS provider ID - ec2:DescribeInstances permission may be missing")
+
+		hasInternalIP := false
+		for _, addr := range testNode.Status.Addresses {
+			if addr.Type == corev1.NodeInternalIP {
+				hasInternalIP = true
+				break
+			}
+		}
+		Expect(hasInternalIP).To(BeTrue(), "Node should have InternalIP")
+
+
+		ginkgo.GinkgoLogr.Info("Node provisioning validated - STS permissions working",
+			"node", testNode.Name, "permissions", []string{"ec2:RunInstances", "ec2:DescribeInstances"})
+	})
+
+	ginkgo.It("should schedule workloads on NodePool nodes", func(ctx context.Context) {
+		ginkgo.By("Creating test workload targeted at NodePool")
+
+		pod := &corev1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "nodepool-test-",
+				Namespace:    h.CurrentProject(),
+			},
+			Spec: corev1.PodSpec{
+				NodeSelector: map[string]string{
+					nodePoolLabel: fmt.Sprintf("%s-%s", clusterNamespace, testNodePoolName),
+				},
+				Containers: []corev1.Container{{
+					Name:    "test",
+					Image:   "registry.access.redhat.com/ubi8/ubi-minimal",
+					Command: []string{"/bin/sh", "-c", "echo 'NodePool test successful' && sleep 5"},
+					Resources: corev1.ResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceCPU:    resource.MustParse("100m"),
+							corev1.ResourceMemory: resource.MustParse("128Mi"),
+						},
+					},
+				}},
+				RestartPolicy: corev1.RestartPolicyNever,
+			},
+		}
+
+		err := client.Create(ctx, pod)
+		expect.NoError(err, "Failed to create test pod")
+		defer client.Delete(ctx, pod)
+
+		ginkgo.By("Waiting for workload to complete successfully")
+
+		err = wait.For(conditions.New(client).PodPhaseMatch(pod, corev1.PodSucceeded), wait.WithTimeout(5*time.Minute))
+		expect.NoError(err, "Workload scheduling failed on NodePool")
+
+		ginkgo.GinkgoLogr.Info("Workload scheduling validated - NodePool functional")
+	})
+
+	ginkgo.It("should reject duplicate NodePool names", func(ctx context.Context) {
+		ginkgo.By("Testing duplicate NodePool creation")
+
+		duplicateSpec := buildNodePoolSpec(testNodePoolName, clusterNamespace, "")
+		duplicateObj := &unstructured.Unstructured{Object: duplicateSpec}
+
+		_, err := h.Dynamic().Resource(nodePoolGVR).Namespace(clusterNamespace).
+			Create(ctx, duplicateObj, metav1.CreateOptions{})
+		Expect(err).To(HaveOccurred(), "Should fail when creating NodePool with duplicate name")
+		Expect(apierrors.IsAlreadyExists(err)).To(BeTrue(), "Expected AlreadyExists on duplicate NodePool")
+	})
+
+	ginkgo.It("should reject operations on non-existent NodePool", func(ctx context.Context) {
+		ginkgo.By("Testing access to non-existent NodePool")
+
+		_, err := h.Dynamic().Resource(nodePoolGVR).Namespace(clusterNamespace).
+			Get(ctx, "non-existent-nodepool", metav1.GetOptions{})
+
+		Expect(err).To(HaveOccurred(), "Getting non-existent NodePool should fail")
+		Expect(apierrors.IsNotFound(err)).To(BeTrue(), "Should return NotFound error")
+	})
+})
+
+func getClusterNamespace(h *helper.H) string {
+    gvr := schema.GroupVersionResource{
+        Group: "hypershift.openshift.io", Version: "v1beta1", Resource: "nodepools",
+    }
+    nps, err := h.Dynamic().Resource(gvr).List(context.Background(), metav1.ListOptions{})
+    if err == nil && len(nps.Items) > 0 {
+        return nps.Items[0].GetNamespace()
+    }
+    currentProject := h.CurrentProject()
+    if strings.HasPrefix(currentProject, "clusters-") {
+        return strings.TrimPrefix(currentProject, "clusters-")
+    }
+
+    return ""
+}
+
+func getExistingSubnet(ctx context.Context, h *helper.H, namespace string) string {
+	nodePoolGVR := schema.GroupVersionResource{
+		Group:    "hypershift.openshift.io",
+		Version:  "v1beta1",
+		Resource: "nodepools",
+	}
+
+	nodePoolList, err := h.Dynamic().Resource(nodePoolGVR).Namespace(namespace).
+		List(ctx, metav1.ListOptions{})
+	if err != nil || len(nodePoolList.Items) == 0 {
+		return ""
+	}
+
+	for _, nodePool := range nodePoolList.Items {
+		if spec, found, err := unstructured.NestedMap(nodePool.Object, "spec"); found && err == nil {
+			if platform, found, err := unstructured.NestedMap(spec, "platform"); found && err == nil {
+				if aws, found, err := unstructured.NestedMap(platform, "aws"); found && err == nil {
+					if subnet, found, err := unstructured.NestedString(aws, "subnet"); found && err == nil {
+						return subnet
+					}
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func buildNodePoolSpec(name, namespace, subnet string) map[string]interface{} {
+	awsConfig := map[string]interface{}{
+		"instanceType": "m5.large",
+	}
+
+	if subnet != "" {
+		awsConfig["subnet"] = subnet
+	}
+
+	return map[string]interface{}{
+		"apiVersion": "hypershift.openshift.io/v1beta1",
+		"kind":       "NodePool",
+		"metadata": map[string]interface{}{
+			"name":      name,
+			"namespace": namespace,
+		},
+		"spec": map[string]interface{}{
+			"clusterName": namespace,
+			"replicas":    1,
+			"management": map[string]interface{}{
+				"autoRepair":  true,
+				"upgradeType": "Replace",
+			},
+			"platform": map[string]interface{}{
+				"aws": awsConfig,
+			},
+		},
+	}
+}
+
+func detectSTSPermissionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errorMsg := strings.ToLower(err.Error())
+	stsPatterns := []string{
+		"accessdenied", "unauthorizedoperation", "forbidden",
+		"invalid iam role", "sts permissions", "assumerolewithwebidentity",
+	}
+
+	for _, pattern := range stsPatterns {
+		if strings.Contains(errorMsg, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+func isNodeReady(node corev1.Node) bool {
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}