Add NodePool STS permissions test

Author: Amarthya Valija

pkg/e2e/verify/nodepool.go

@@ -0,0 +1,232 @@
+package verify
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+
+	"github.com/openshift/osde2e/pkg/common/aws"
+	"github.com/openshift/osde2e/pkg/common/expect"
+	"github.com/openshift/osde2e/pkg/common/helper"
+	"github.com/openshift/osde2e/pkg/common/label"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+)
+
+// NodePool STS Permissions Test
+// This test validates day 2 operations for ROSA HCP clusters by creating NodePools.
+// It ensures required STS permissions (ec2:RunInstances, ec2:CreateTags, ec2:DescribeInstances) 
+
+var nodePoolTestName = "[Suite: e2e] NodePool STS Permissions"
+
+var _ = ginkgo.Describe(nodePoolTestName, ginkgo.Ordered, label.E2E, func() {
+	var h *helper.H
+	var client *resources.Resources
+
+	ginkgo.BeforeAll(func() {
+		h = helper.New()
+		client = h.AsUser("")
+	})
+
+	ginkgo.It("should create nodepool and provision EC2 instances with proper STS permissions", func(ctx context.Context) {
+		var nodes corev1.NodeList
+		expect.NoError(client.List(ctx, &nodes))
+		
+		if len(nodes.Items) == 0 {
+			ginkgo.Skip("No nodes found")
+		}
+
+		clusterName := extractClusterName(nodes.Items)
+		clusterID := getClusterID(nodes.Items)
+		
+		if clusterName == "" || clusterID == "" {
+			ginkgo.Skip("Could not determine cluster information")
+		}
+
+		nodePoolName := fmt.Sprintf("osde2e-sts-test-%d", time.Now().Unix())
+		
+		ginkgo.By("Creating NodePool to test STS permissions for day 2 operations")
+		err := createNodePool(clusterID, nodePoolName)
+		
+		if err != nil {
+			expect.NoError(fmt.Errorf("NodePool creation failed - STS permissions missing %v.", err))
+		}
+
+		// Cleanup
+		defer deleteNodePool(clusterID, nodePoolName)
+
+		ginkgo.By("Waiting for new nodes from NodePool (tests ec2:RunInstances, ec2:CreateTags)")
+		newNodes := waitForNewNodes(ctx, client, len(nodes.Items), nodePoolName)
+		
+		if len(newNodes) == 0 {
+			expect.NoError(fmt.Errorf("NodePool failed to provision nodes - STS permissions (ec2:RunInstances, ec2:CreateTags) missing."))
+		}
+		
+		ginkgo.By("Validating nodes exist in AWS (tests ec2:DescribeInstances)")
+		for _, node := range newNodes {
+			exists, err := aws.CcsAwsSession.CheckIfEC2ExistBasedOnNodeName(node.Name)
+			if err != nil {
+				expect.NoError(fmt.Errorf("AWS validation failed for node %s - STS permissions (ec2:DescribeInstances) missing: %v.", node.Name, err))
+			}
+			if !exists {
+				expect.NoError(fmt.Errorf("Node %s not found in AWS - STS permissions (ec2:DescribeInstances) missing.", node.Name))
+			}
+		}
+
+		ginkgo.By("Testing workload scheduling on new NodePool")
+		err = testWorkload(ctx, client, h, nodePoolName)
+		if err != nil {
+			expect.NoError(fmt.Errorf("Workload scheduling failed on NodePool: %v.", err))
+		}
+		
+		ginkgo.By("NodePool operations completed successfully - STS permissions working")
+		ginkgo.GinkgoLogr.Info("NodePool STS permissions test PASSED", 
+			"cluster", clusterName,
+			"nodepool", nodePoolName,
+			"nodes_created", len(newNodes))
+	})
+})
+
+func extractClusterName(nodes []corev1.Node) string {
+	for _, node := range nodes {
+		if label, exists := node.Labels["hypershift.openshift.io/nodePool"]; exists {
+			if strings.Contains(label, "-workers-") {
+				return strings.Split(label, "-workers-")[0]
+			}
+		}
+	}
+	return ""
+}
+
+func getClusterID(nodes []corev1.Node) string {
+	if clusterID := os.Getenv("CLUSTER_ID"); clusterID != "" {
+		return clusterID
+	}
+	
+	for _, node := range nodes {
+		if strings.Contains(node.Name, "-") {
+			parts := strings.Split(node.Name, "-")
+			if len(parts) > 0 && len(parts[0]) > 10 {
+				return parts[0]
+			}
+		}
+	}
+	return ""
+}
+
+func createNodePool(clusterID, nodePoolName string) error {
+	if _, err := exec.Command("rosa", "whoami").CombinedOutput(); err != nil {
+		if token := os.Getenv("OCM_TOKEN"); token != "" {
+			exec.Command("rosa", "login", "--env", "stage", "--token", token).Run()
+		}
+	}
+	
+	cmd := exec.Command("rosa", "create", "machinepool", 
+		"--cluster", clusterID,
+		"--name", nodePoolName,
+		"--replicas", "1",
+		"--instance-type", "m5.large",
+		"--yes")
+	
+	_, err := cmd.CombinedOutput()
+	return err
+}
+
+func deleteNodePool(clusterID, nodePoolName string) {
+	exec.Command("rosa", "delete", "machinepool", nodePoolName, "--cluster", clusterID, "--yes").Run()
+}
+
+func isWorkerNode(node corev1.Node) bool {
+	_, isMaster := node.Labels["node-role.kubernetes.io/master"]
+	_, isControlPlane := node.Labels["node-role.kubernetes.io/control-plane"]
+	return !isMaster && !isControlPlane
+}
+
+func waitForNewNodes(ctx context.Context, client *resources.Resources, initialCount int, nodePoolName string) []corev1.Node {
+	var newNodes []corev1.Node
+	
+	wait.For(func(ctx context.Context) (bool, error) {
+		var currentNodes corev1.NodeList
+		if err := client.List(ctx, &currentNodes); err != nil {
+			return false, err
+		}
+
+		newNodes = nil
+		if len(currentNodes.Items) > initialCount {
+			for _, node := range currentNodes.Items {
+				if label, exists := node.Labels["hypershift.openshift.io/nodePool"]; exists && label == nodePoolName {
+					if isNodeReady(node) {
+						newNodes = append(newNodes, node)
+					}
+				}
+			}
+		}
+		return len(newNodes) > 0, nil
+	}, wait.WithTimeout(20*time.Minute), wait.WithInterval(30*time.Second))
+	
+	return newNodes
+}
+
+func isNodeReady(node corev1.Node) bool {
+	for _, condition := range node.Status.Conditions {
+		if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}
+
+func testWorkload(ctx context.Context, client *resources.Resources, h *helper.H, nodePoolName string) error {
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "nodepool-test-",
+			Namespace:    h.CurrentProject(),
+		},
+		Spec: corev1.PodSpec{
+			NodeSelector: map[string]string{
+				"hypershift.openshift.io/nodePool": nodePoolName,
+			},
+			Containers: []corev1.Container{{
+				Name:    "test",
+				Image:   "registry.access.redhat.com/ubi8/ubi-minimal",
+				Command: []string{"/bin/sh", "-c", "echo 'Success' && sleep 10"},
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("100m"),
+						corev1.ResourceMemory: resource.MustParse("128Mi"),
+					},
+				},
+			}},
+			RestartPolicy: corev1.RestartPolicyNever,
+		},
+	}
+
+	if err := client.Create(ctx, pod); err != nil {
+		return fmt.Errorf("failed to create test pod: %v", err)
+	}
+	defer client.Delete(ctx, pod)
+
+	// Wait for pod success
+	err := wait.For(func(ctx context.Context) (bool, error) {
+		p := &corev1.Pod{}
+		if err := client.Get(ctx, pod.GetName(), pod.GetNamespace(), p); err != nil {
+			return false, err
+		}
+		return p.Status.Phase == corev1.PodSucceeded, nil
+	}, wait.WithTimeout(3*time.Minute))
+	
+	if err != nil {
+		return fmt.Errorf("test pod failed to complete successfully: %v", err)
+	}
+	
+	return nil
+}