WIP: Full in cluster registry test

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "cluster-deployment-automation"]
 	path = cluster-deployment-automation
-	url = https://github.com/bn222/cluster-deployment-automation
+	url = https://github.com/samd2021/cluster-deployment-automation
 [submodule "kubernetes-traffic-flow-tests"]
 	path = kubernetes-traffic-flow-tests
 	url = https://github.com/ovn-kubernetes/kubernetes-traffic-flow-tests

README.md

@@ -48,12 +48,34 @@ The build target will use previously built images if they are available to speed
 task clean-image-all
 ```
 
-6. **Alternative: Remote Registry**
+6. **Registry Configuration**
 
-Alternatively, if you set up a registry remotely, define the `REGISTERY` variable. Note that you need to do this for the build step and the push/run step:
+The DPU operator now uses in-cluster registries by default with predictable URLs based on your cluster configuration:
+
+- **Host Registry**: `default-route-openshift-image-registry.apps.ocpcluster.redhat.com`
+- **DPU Registry**: `default-route-openshift-image-registry.apps.172.16.3.16.nip.io`
+
+You can check the current configuration:
+```sh
+task show-registry-config
+```
+
+To verify the configuration matches your actual clusters (when they're running):
+```sh
+task verify-registry-routes
+```
+
+**Custom Registry Configuration**: If your cluster domains or IPs differ, you can override them:
 
 ```sh
-REGISTERY=... task ...
+# Override cluster domain (affects host registry)
+HOST_CLUSTER_DOMAIN=mycluster.example.com task build-image-all
+
+# Override DPU IP (affects DPU registry)  
+DPU_CLUSTER_IP=10.1.2.3 task build-image-all
+
+# Override registry URLs directly
+REGISTRY_HOST=my-host-registry.com REGISTRY_DPU=my-dpu-registry.com task build-image-all
 ```
 
 7. **Alternative: Deploy only one of the cluster**

hack/cluster-configs/config-dpu-host.yaml

@@ -6,6 +6,7 @@ clusters:
     version: "4.19.0-nightly"
     network_api_port: "{{api_network()}}"
     postconfig:
+    - name: "image_registry"
     - name: "dpu_operator_host"
       dpu_operator_path: "../../"
     - name: "monitoring_config"

hack/cluster-configs/config-dpu.yaml

@@ -5,6 +5,7 @@ clusters:
     network_api_port: "{{secondary_network_port()}}"
     kind: "iso"
     install_iso: "{{iso_server()}}/RHEL-9.6.0-20250416.8-aarch64-dvd1-w-kickstart.iso"
+    kubeconfig: "/root/kubeconfig.microshift"
     masters:
     - name: "{{worker_number(0)}}-acc"
       node: "localhost"
@@ -26,5 +27,7 @@ clusters:
       organization_id: "{{organization_id()}}"
       activation_key: "{{activation_key()}}"
     - name: "microshift"
+    - name: "image_registry"
+      registry_type: "microshift"
     - name: "dpu_operator_dpu"
       dpu_operator_path: "../../"

internal/testutils/testcluster.go

@@ -110,7 +110,7 @@ func GetDPUNodes(c client.Client) ([]corev1.Node, error) {
 func TrafficFlowTestsImage() string {
 	localContainer := ContainerImage{
 		Registry: os.Getenv("REGISTRY"),
-		Name:     "ovn-kubernetes/kubernetes-traffic-flow-tests",
+		Name:     "kubernetes-traffic-flow-tests",
 		Tag:      "latest",
 	}
 
@@ -125,6 +125,7 @@ func TrafficFlowTestsImage() string {
 		Expect(err).To(BeNil())
 		return localContainer.FullRef()
 	}
+
 	return remoteContainer.FullRef()
 }
 

taskfile.yaml

@@ -14,8 +14,11 @@ vars:
   BINDIR: bin
   BINDIR_ABS:
     sh: if [[ "{{.BINDIR}}" = /* ]]; then echo "{{.BINDIR}}"; else echo "$(pwd)/{{.BINDIR}}"; fi
-  REGISTRY:
-    sh: hostname | sed 's/$/:5000/'
+  HOST_CLUSTER_DOMAIN: '{{ default "ocpcluster.redhat.com" .HOST_CLUSTER_DOMAIN }}'
+  DPU_CLUSTER_IP: '{{ default "172.16.3.16" .DPU_CLUSTER_IP }}'
+  REGISTRY_HOST: '{{ default (printf "default-route-openshift-image-registry.apps.%s/in-cluster-registry" .HOST_CLUSTER_DOMAIN) .REGISTRY_HOST }}'
+  REGISTRY_DPU: '{{ default (printf "default-route-openshift-image-registry.apps.%s.nip.io/in-cluster-registry" .DPU_CLUSTER_IP) .REGISTRY_DPU }}'
+  REGISTRY: '{{.REGISTRY_HOST}}'
   ENVTEST_K8S_VERSION: 1.27.1
   KUSTOMIZE_VERSION: v5.6.0
   GINKGO_VERSION:
@@ -48,13 +51,52 @@ tasks:
     cmds:
       - echo "{{.BINDIR_ABS}}"
 
+  show-registry-config:
+    desc: "Show current registry configuration"
+    cmds:
+      - 'echo "Host Registry: {{.REGISTRY_HOST}}"'
+      - 'echo "DPU Registry:  {{.REGISTRY_DPU}}"'
+      - 'echo "Default Registry: {{.REGISTRY}}"'
+
+  verify-registry-routes:
+    desc: "Verify registry routes match actual cluster routes (requires running clusters)"
+    cmds:
+      - |
+        echo "Verifying registry routes..."
+        echo "Expected host registry: {{.REGISTRY_HOST}}"
+        if oc get route default-route -n openshift-image-registry --kubeconfig /root/kubeconfig.ocpcluster &>/dev/null; then
+          ACTUAL_HOST=$(oc get route default-route -n openshift-image-registry --template='{{`{{ .spec.host }}`}}' --kubeconfig /root/kubeconfig.ocpcluster)
+          echo "Actual host registry:   $ACTUAL_HOST"
+          if [ "{{.REGISTRY_HOST}}" = "$ACTUAL_HOST" ]; then
+            echo "Host registry matches"
+          else
+            echo "Host registry mismatch - consider setting HOST_CLUSTER_DOMAIN environment variable"
+          fi
+        else
+          echo "Host cluster not available or registry route not found"
+        fi
+
+        echo ""
+        echo "Expected DPU registry: {{.REGISTRY_DPU}}"
+        if oc get route default-route -n openshift-image-registry --kubeconfig /root/kubeconfig.microshift &>/dev/null; then
+          ACTUAL_DPU=$(oc get route default-route -n openshift-image-registry --template='{{`{{ .spec.host }}`}}' --kubeconfig /root/kubeconfig.microshift)
+          echo "Actual DPU registry:    $ACTUAL_DPU"
+          if [ "{{.REGISTRY_DPU}}" = "$ACTUAL_DPU" ]; then
+            echo "DPU registry matches"
+          else
+            echo "DPU registry mismatch - consider setting DPU_CLUSTER_IP environment variable"
+          fi
+        else
+          echo "DPU cluster not available or registry route not found"
+        fi
+
   push-image-helper:
     internal: true
     vars:
       SOURCE: '{{.SOURCE}}'
       IMAGE: '{{.IMAGE}}'
     cmds:
-      - buildah manifest push --all '{{.SOURCE}}-manifest' 'docker://{{.IMAGE}}'
+      - buildah manifest push --all --retry 3 --retry-delay 10s '{{.SOURCE}}-manifest' 'docker://{{.IMAGE}}'
 
   undeploy-helper:
     internal: true
@@ -95,12 +137,23 @@ tasks:
     deps:
       - task: kustomize
     cmds:
+      - mkdir -p bin/host bin/dpu
       - >
         go run ./tools/config/config.go
-        -registry-url {{.REGISTRY}}
+        -registry-url {{.REGISTRY_HOST}}
         -template-file config/dev/local-images-template.yaml
-        -output-file bin/local-images.yaml
-      - cp config/dev/kustomization.yaml bin
+        -output-file bin/host/local-images.yaml
+      - >
+        go run ./tools/config/config.go
+        -registry-url {{.REGISTRY_DPU}}
+        -template-file config/dev/local-images-template.yaml
+        -output-file bin/dpu/local-images.yaml
+      - cp config/dev/kustomization.yaml bin/host/
+      - cp config/dev/kustomization.yaml bin/dpu/
+      - sed -i 's/local-images-template.yaml/local-images.yaml/' bin/host/kustomization.yaml
+      - sed -i 's/local-images-template.yaml/local-images.yaml/' bin/dpu/kustomization.yaml
+      - sed -i 's|../config/default|../../config/default|' bin/host/kustomization.yaml
+      - sed -i 's|../config/default|../../config/default|' bin/dpu/kustomization.yaml
 
   ## Download envtest-setup locally if necessary
   envtest:
@@ -118,8 +171,8 @@ tasks:
         vars:
           KUBECONFIG_DPU: "/root/kubeconfig.microshift"
           KUBECONFIG_HOST: "/root/kubeconfig.ocpcluster"
-      - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.microshift" oc apply -f -
-      - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f -
+      - bin/kustomize build bin/dpu | KUBECONFIG="/root/kubeconfig.microshift" oc apply -f -
+      - bin/kustomize build bin/host | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f -
       - KUBECONFIG="/root/kubeconfig.microshift" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s
       - KUBECONFIG="/root/kubeconfig.ocpcluster" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s
 
@@ -130,16 +183,21 @@ tasks:
       - task: undeploy-1c
         vars:
           KUBECONFIG_HOST: "/root/kubeconfig.ocpcluster"
-      - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f -
+      - bin/kustomize build bin/host | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f -
       - KUBECONFIG="/root/kubeconfig.ocpcluster" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s
 
-  prepare-e2e-test:
+  ensure-test-images-in-both-registries:
+    desc: "Ensure test images are available in both registries"
     cmds:
-      - >
-        if [ "{{.SUBMODULES}}" = "true" ]; then
-            hack/prepare-submodules.sh
-        fi
-            hack/prepare-venv.sh
+      - |
+        echo "Ensuring test images are available in both registries..."
+        skopeo copy \
+          docker://ghcr.io/ovn-kubernetes/kubernetes-traffic-flow-tests:latest \
+          docker://{{.REGISTRY_HOST}}/kubernetes-traffic-flow-tests:latest
+        skopeo copy \
+          docker://ghcr.io/ovn-kubernetes/kubernetes-traffic-flow-tests:latest \
+          docker://{{.REGISTRY_DPU}}/kubernetes-traffic-flow-tests:latest
+        echo "Test images copied to both registries"
 
   e2e-test:
     deps:
@@ -159,6 +217,7 @@ tasks:
       - task: ginkgo
       - task: envtest
     cmds:
+      - task: ensure-test-images-in-both-registries
       - >
         FAST_TEST=true
         REGISTRY={{.REGISTRY}}

taskfiles/images.yaml

@@ -180,7 +180,7 @@ tasks:
   build-cpagent-service:
     internal: true
     vars:
-      IMAGE: '{{.REGISTRY}}/mrvl-cpagent:dev'
+      IMAGE: '{{.REGISTRY_HOST}}/mrvl-cpagent:dev'
       CPIMAGE: "Environment=IMAGE='{{.IMAGE}}'"
       CPFILE: 'internal/daemon/vendor-specific-plugins/marvell/cp-agent/cp-agent.service'
     cmds:
@@ -376,37 +376,71 @@ tasks:
       - task: push-image-all
 
   push-image-all:
-    deps:
+    cmds: # pushing images in parallel overwhelm the image registry and cause 503 errors
+      # Push to host registry
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/dpu-operator:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/dpu-operator:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/dpu-daemon:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/dpu-daemon:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/mrvl-vsp:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/mrvl-vsp:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/mrvl-cpagent:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/mrvl-cpagent:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/intel-vsp:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/intel-vsp:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/intel-vsp-p4:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/intel-vsp-p4:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/intel-netsec-vsp:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/intel-netsec-vsp:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/network-resources-injector:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/network-resources-injector:dev'
+      # Push to DPU registry
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/dpu-operator:dev'
-          IMAGE: '{{.REGISTRY}}/dpu-operator:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/dpu-operator:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/dpu-daemon:dev'
-          IMAGE: '{{.REGISTRY}}/dpu-daemon:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/dpu-daemon:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/mrvl-vsp:dev'
-          IMAGE: '{{.REGISTRY}}/mrvl-vsp:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/mrvl-vsp:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/mrvl-cpagent:dev'
-          IMAGE: '{{.REGISTRY}}/mrvl-cpagent:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/mrvl-cpagent:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/intel-vsp:dev'
-          IMAGE: '{{.REGISTRY}}/intel-vsp:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/intel-vsp:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/intel-vsp-p4:dev'
-          IMAGE: '{{.REGISTRY}}/intel-vsp-p4:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/intel-vsp-p4:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/intel-netsec-vsp:dev'
-          IMAGE: '{{.REGISTRY}}/intel-netsec-vsp:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/intel-netsec-vsp:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/network-resources-injector:dev'
-          IMAGE: '{{.REGISTRY}}/network-resources-injector:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/network-resources-injector:dev'