trt-2246: lower parallelism based on worker nodes

Author: neisw

pkg/test/ginkgo/cmd_runsuite.go

@@ -352,19 +352,33 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 		}
 	}
 
-	parallelism := o.Parallelism
-	if parallelism == 0 {
-		parallelism = suite.Parallelism
+	// start with suite value
+	parallelism := suite.Parallelism
+	logrus.Infof("Suite defined parallelism %d", parallelism)
+
+	// adjust based on the number of workers
+	totalNodes, workerNodes, err := getClusterNodeCounts(ctx, restConfig)
+	if err != nil {
+		logrus.Errorf("Failed to get cluster node counts: %v", err)
+	} else {
+		// default to 10 concurrent tests per worker but use the min of that
+		// and the current parallelism value
+		if workerNodes > 0 {
+			workerParallelism := 10 * workerNodes
+			logrus.Infof("Parallelism based on worker node count: %d", workerParallelism)
+			parallelism = min(parallelism, workerParallelism)
+		}
 	}
 
+	// if 0 set our min value
 	if parallelism == 0 {
 		parallelism = 10
 	}
 
-	// adjust based on the number of workers
-	totalNodes, workerNodes, err := getClusterNodeCounts(ctx, restConfig)
-	if err != nil {
-		logrus.Errorf("Failed to get cluster node counts: %v", err)
+	// if explicitly set then use the specified value
+	if o.Parallelism > 0 {
+		parallelism = o.Parallelism
+		logrus.Infof("Using specified parallelism value: %d", parallelism)
 	}
 
 	logrus.Infof("Total nodes: %d, Worker nodes: %d, Parallelism: %d", totalNodes, workerNodes, parallelism)