feat: Use in-cluster registries for CI deployment

SamD2021 · SamD2021 · commit b63eaa54847a · 2025-07-23T22:08:51.000-04:00
diff --git a/README.md b/README.md
@@ -48,12 +48,34 @@ The build target will use previously built images if they are available to speed
 task clean-image-all
 ```
 
-6. **Alternative: Remote Registry**
+6. **Registry Configuration**
 
-Alternatively, if you set up a registry remotely, define the `REGISTERY` variable. Note that you need to do this for the build step and the push/run step:
+The DPU operator now uses in-cluster registries by default with predictable URLs based on your cluster configuration:
+
+- **Host Registry**: `default-route-openshift-image-registry.apps.ocpcluster.redhat.com`
+- **DPU Registry**: `default-route-openshift-image-registry.apps.172.16.3.16.nip.io`
+
+You can check the current configuration:
+```sh
+task show-registry-config
+```
+
+To verify the configuration matches your actual clusters (when they're running):
+```sh
+task verify-registry-routes
+```
+
+**Custom Registry Configuration**: If your cluster domains or IPs differ, you can override them:
 
 ```sh
-REGISTERY=... task ...
+# Override cluster domain (affects host registry)
+HOST_CLUSTER_DOMAIN=mycluster.example.com task build-image-all
+
+# Override DPU IP (affects DPU registry)  
+DPU_CLUSTER_IP=10.1.2.3 task build-image-all
+
+# Override registry URLs directly
+REGISTRY_HOST=my-host-registry.com REGISTRY_DPU=my-dpu-registry.com task build-image-all
 ```
 
 7. **Alternative: Deploy only one of the cluster**
diff --git a/hack/cluster-configs/config-dpu.yaml b/hack/cluster-configs/config-dpu.yaml
@@ -5,6 +5,7 @@ clusters:
     network_api_port: "{{secondary_network_port()}}"
     kind: "iso"
     install_iso: "{{iso_server()}}/RHEL-9.6.0-20250416.8-aarch64-dvd1-w-kickstart.iso"
+    kubeconfig: "/root/kubeconfig.microshift"
     masters:
     - name: "{{worker_number(0)}}-acc"
       node: "localhost"
@@ -26,5 +27,7 @@ clusters:
       organization_id: "{{organization_id()}}"
       activation_key: "{{activation_key()}}"
     - name: "microshift"
+    - name: "image_registry"
+      registry_type: "microshift"
     - name: "dpu_operator_dpu"
       dpu_operator_path: "../../"
diff --git a/internal/testutils/testcluster.go b/internal/testutils/testcluster.go
@@ -110,7 +110,7 @@ func GetDPUNodes(c client.Client) ([]corev1.Node, error) {
 func TrafficFlowTestsImage() string {
 	localContainer := ContainerImage{
 		Registry: os.Getenv("REGISTRY"),
-		Name:     "ovn-kubernetes/kubernetes-traffic-flow-tests",
+		Name:     "kubernetes-traffic-flow-tests",
 		Tag:      "latest",
 	}
 
@@ -125,6 +125,7 @@ func TrafficFlowTestsImage() string {
 		Expect(err).To(BeNil())
 		return localContainer.FullRef()
 	}
+
 	return remoteContainer.FullRef()
 }
 
diff --git a/taskfile.yaml b/taskfile.yaml
@@ -14,8 +14,11 @@ vars:
   BINDIR: bin
   BINDIR_ABS:
     sh: if [[ "{{.BINDIR}}" = /* ]]; then echo "{{.BINDIR}}"; else echo "$(pwd)/{{.BINDIR}}"; fi
-  REGISTRY:
-    sh: hostname | sed 's/$/:5000/'
+  HOST_CLUSTER_DOMAIN: '{{ default "ocpcluster.redhat.com" .HOST_CLUSTER_DOMAIN }}'
+  DPU_CLUSTER_IP: '{{ default "172.16.3.16" .DPU_CLUSTER_IP }}'
+  REGISTRY_HOST: '{{ default (printf "default-route-openshift-image-registry.apps.%s/in-cluster-registry" .HOST_CLUSTER_DOMAIN) .REGISTRY_HOST }}'
+  REGISTRY_DPU: '{{ default (printf "default-route-openshift-image-registry.apps.%s.nip.io/in-cluster-registry" .DPU_CLUSTER_IP) .REGISTRY_DPU }}'
+  REGISTRY: '{{.REGISTRY_HOST}}'
   ENVTEST_K8S_VERSION: 1.27.1
   KUSTOMIZE_VERSION: v5.6.0
   GINKGO_VERSION:
@@ -48,13 +51,52 @@ tasks:
     cmds:
       - echo "{{.BINDIR_ABS}}"
 
+  show-registry-config:
+    desc: "Show current registry configuration"
+    cmds:
+      - 'echo "Host Registry: {{.REGISTRY_HOST}}"'
+      - 'echo "DPU Registry:  {{.REGISTRY_DPU}}"'
+      - 'echo "Default Registry: {{.REGISTRY}}"'
+
+  verify-registry-routes:
+    desc: "Verify registry routes match actual cluster routes (requires running clusters)"
+    cmds:
+      - |
+        echo "Verifying registry routes..."
+        echo "Expected host registry: {{.REGISTRY_HOST}}"
+        if oc get route default-route -n openshift-image-registry --kubeconfig /root/kubeconfig.ocpcluster &>/dev/null; then
+          ACTUAL_HOST=$(oc get route default-route -n openshift-image-registry --template='{{`{{ .spec.host }}`}}' --kubeconfig /root/kubeconfig.ocpcluster)
+          echo "Actual host registry:   $ACTUAL_HOST"
+          if [ "{{.REGISTRY_HOST}}" = "$ACTUAL_HOST" ]; then
+            echo "Host registry matches"
+          else
+            echo "Host registry mismatch - consider setting HOST_CLUSTER_DOMAIN environment variable"
+          fi
+        else
+          echo "Host cluster not available or registry route not found"
+        fi
+        
+        echo ""
+        echo "Expected DPU registry: {{.REGISTRY_DPU}}"
+        if oc get route default-route -n openshift-image-registry --kubeconfig /root/kubeconfig.microshift &>/dev/null; then
+          ACTUAL_DPU=$(oc get route default-route -n openshift-image-registry --template='{{`{{ .spec.host }}`}}' --kubeconfig /root/kubeconfig.microshift)
+          echo "Actual DPU registry:    $ACTUAL_DPU"
+          if [ "{{.REGISTRY_DPU}}" = "$ACTUAL_DPU" ]; then
+            echo "DPU registry matches"
+          else
+            echo "DPU registry mismatch - consider setting DPU_CLUSTER_IP environment variable"
+          fi
+        else
+          echo "DPU cluster not available or registry route not found"
+        fi
+
   push-image-helper:
     internal: true
     vars:
       SOURCE: '{{.SOURCE}}'
       IMAGE: '{{.IMAGE}}'
     cmds:
-      - buildah manifest push --all '{{.SOURCE}}-manifest' 'docker://{{.IMAGE}}'
+      - buildah manifest push --all --retry 3 --retry-delay 10s '{{.SOURCE}}-manifest' 'docker://{{.IMAGE}}'
 
   undeploy-helper:
     internal: true
@@ -95,12 +137,23 @@ tasks:
     deps:
       - task: kustomize
     cmds:
+      - mkdir -p bin/host bin/dpu
       - >
         go run ./tools/config/config.go
-        -registry-url {{.REGISTRY}}
+        -registry-url {{.REGISTRY_HOST}}
         -template-file config/dev/local-images-template.yaml
-        -output-file bin/local-images.yaml
-      - cp config/dev/kustomization.yaml bin
+        -output-file bin/host/local-images.yaml
+      - >
+        go run ./tools/config/config.go
+        -registry-url {{.REGISTRY_DPU}}
+        -template-file config/dev/local-images-template.yaml
+        -output-file bin/dpu/local-images.yaml
+      - cp config/dev/kustomization.yaml bin/host/
+      - cp config/dev/kustomization.yaml bin/dpu/
+      - sed -i 's/local-images-template.yaml/local-images.yaml/' bin/host/kustomization.yaml
+      - sed -i 's/local-images-template.yaml/local-images.yaml/' bin/dpu/kustomization.yaml
+      - sed -i 's|../config/default|../../config/default|' bin/host/kustomization.yaml
+      - sed -i 's|../config/default|../../config/default|' bin/dpu/kustomization.yaml
   
   ## Download envtest-setup locally if necessary
   envtest:
@@ -118,8 +171,8 @@ tasks:
         vars:
           KUBECONFIG_DPU: "/root/kubeconfig.microshift"
           KUBECONFIG_HOST: "/root/kubeconfig.ocpcluster"
-      - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.microshift" oc apply -f -
-      - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f -
+      - bin/kustomize build bin/dpu | KUBECONFIG="/root/kubeconfig.microshift" oc apply -f -
+      - bin/kustomize build bin/host | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f -
       - KUBECONFIG="/root/kubeconfig.microshift" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s
       - KUBECONFIG="/root/kubeconfig.ocpcluster" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s
 
@@ -130,16 +183,21 @@ tasks:
       - task: undeploy-1c
         vars:
           KUBECONFIG_HOST: "/root/kubeconfig.ocpcluster"
-      - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f -
+      - bin/kustomize build bin/host | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f -
       - KUBECONFIG="/root/kubeconfig.ocpcluster" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s
 
-  prepare-e2e-test:
+  ensure-test-images-in-both-registries:
+    desc: "Ensure test images are available in both registries"
     cmds:
-      - >
-        if [ "{{.SUBMODULES}}" = "true" ]; then
-            hack/prepare-submodules.sh
-        fi
-            hack/prepare-venv.sh
+      - |
+        echo "Ensuring test images are available in both registries..."
+        skopeo copy --dest-tls-verify=false \
+          docker://ghcr.io/ovn-kubernetes/kubernetes-traffic-flow-tests:latest \
+          docker://{{.REGISTRY_HOST}}/kubernetes-traffic-flow-tests:latest
+        skopeo copy --dest-tls-verify=false \
+          docker://ghcr.io/ovn-kubernetes/kubernetes-traffic-flow-tests:latest \
+          docker://{{.REGISTRY_DPU}}/kubernetes-traffic-flow-tests:latest
+        echo "Test images copied to both registries"
 
   e2e-test:
     deps:
@@ -159,6 +217,7 @@ tasks:
       - task: ginkgo
       - task: envtest
     cmds:
+      - task: ensure-test-images-in-both-registries
       - >
         FAST_TEST=true
         REGISTRY={{.REGISTRY}}
diff --git a/taskfiles/images.yaml b/taskfiles/images.yaml
@@ -180,7 +180,7 @@ tasks:
   build-cpagent-service:
     internal: true
     vars:
-      IMAGE: '{{.REGISTRY}}/mrvl-cpagent:dev'
+      IMAGE: '{{.REGISTRY_HOST}}/mrvl-cpagent:dev'
       CPIMAGE: "Environment=IMAGE='{{.IMAGE}}'"
       CPFILE: 'internal/daemon/vendor-specific-plugins/marvell/cp-agent/cp-agent.service'
     cmds:
@@ -377,36 +377,70 @@ tasks:
 
   push-image-all:
     deps:
+      # Push to host registry
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/dpu-operator:dev'
-          IMAGE: '{{.REGISTRY}}/dpu-operator:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/dpu-operator:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/dpu-daemon:dev'
-          IMAGE: '{{.REGISTRY}}/dpu-daemon:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/dpu-daemon:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/mrvl-vsp:dev'
-          IMAGE: '{{.REGISTRY}}/mrvl-vsp:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/mrvl-vsp:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/mrvl-cpagent:dev'
-          IMAGE: '{{.REGISTRY}}/mrvl-cpagent:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/mrvl-cpagent:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/intel-vsp:dev'
-          IMAGE: '{{.REGISTRY}}/intel-vsp:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/intel-vsp:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/intel-vsp-p4:dev'
-          IMAGE: '{{.REGISTRY}}/intel-vsp-p4:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/intel-vsp-p4:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/intel-netsec-vsp:dev'
-          IMAGE: '{{.REGISTRY}}/intel-netsec-vsp:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/intel-netsec-vsp:dev'
       - task: push-image-helper
         vars:
           SOURCE: 'localhost/network-resources-injector:dev'
-          IMAGE: '{{.REGISTRY}}/network-resources-injector:dev'
+          IMAGE: '{{.REGISTRY_HOST}}/network-resources-injector:dev'
+      # Push to DPU registry
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/dpu-operator:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/dpu-operator:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/dpu-daemon:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/dpu-daemon:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/mrvl-vsp:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/mrvl-vsp:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/mrvl-cpagent:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/mrvl-cpagent:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/intel-vsp:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/intel-vsp:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/intel-vsp-p4:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/intel-vsp-p4:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/intel-netsec-vsp:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/intel-netsec-vsp:dev'
+      - task: push-image-helper
+        vars:
+          SOURCE: 'localhost/network-resources-injector:dev'
+          IMAGE: '{{.REGISTRY_DPU}}/network-resources-injector:dev'
 

Original file line number	Diff line number	Diff line change
`@@ -110,7 +110,7 @@ func GetDPUNodes(c client.Client) ([]corev1.Node, error) {`
`110`	`110`	`func TrafficFlowTestsImage() string {`
`111`	`111`	`localContainer := ContainerImage{`
`112`	`112`	`Registry: os.Getenv("REGISTRY"),`
`113`		`- Name: "ovn-kubernetes/kubernetes-traffic-flow-tests",`
	`113`	`+ Name: "kubernetes-traffic-flow-tests",`
`114`	`114`	`Tag: "latest",`
`115`	`115`	`}`
`116`	`116`
`@@ -125,6 +125,7 @@ func TrafficFlowTestsImage() string {`
`125`	`125`	`Expect(err).To(BeNil())`
`126`	`126`	`return localContainer.FullRef()`
`127`	`127`	`}`
	`128`	`+`
`128`	`129`	`return remoteContainer.FullRef()`
`129`	`130`	`}`
`130`	`131`