use local proto go packages for ci

HumairAK · HumairAK · commit b8954374bbee · 2025-02-13T23:53:15.000-05:00
Signed-off-by: Humair Khan &lt;HumairAK@users.noreply.github.com&gt;
diff --git a/.github/resources/scripts/build-images.sh b/.github/resources/scripts/build-images.sh
@@ -22,38 +22,39 @@ REGISTRY="${REGISTRY:-kind-registry:5000}"
 echo "REGISTRY=$REGISTRY"
 TAG="${TAG:-latest}"
 EXIT_CODE=0
+BUILD_LOCAL_MODE=true
 
 docker system prune -a -f
 
-docker build --progress=plain -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain --build-arg LOCAL_MODE=${BUILD_LOCAL_MODE} -t "${REGISTRY}/apiserver:${TAG}" -f backend/Dockerfile . && docker push "${REGISTRY}/apiserver:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build apiserver image."
   exit $EXIT_CODE
 fi
 
-docker build --progress=plain -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain --build-arg LOCAL_MODE=${BUILD_LOCAL_MODE} -t "${REGISTRY}/persistenceagent:${TAG}" -f backend/Dockerfile.persistenceagent . && docker push "${REGISTRY}/persistenceagent:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build persistenceagent image."
   exit $EXIT_CODE
 fi
 
-docker build --progress=plain -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain --build-arg LOCAL_MODE=${BUILD_LOCAL_MODE} -t "${REGISTRY}/scheduledworkflow:${TAG}" -f backend/Dockerfile.scheduledworkflow . && docker push "${REGISTRY}/scheduledworkflow:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build scheduledworkflow image."
   exit $EXIT_CODE
 fi
 
-docker build --progress=plain -t "${REGISTRY}/driver:${TAG}" -f backend/Dockerfile.driver . && docker push "${REGISTRY}/driver:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain --build-arg LOCAL_MODE=${BUILD_LOCAL_MODE} -t "${REGISTRY}/driver:${TAG}" -f backend/Dockerfile.driver . && docker push "${REGISTRY}/driver:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build driver image."
   exit $EXIT_CODE
 fi
 
-docker build --progress=plain -t "${REGISTRY}/launcher:${TAG}" -f backend/Dockerfile.launcher . && docker push "${REGISTRY}/launcher:${TAG}" || EXIT_CODE=$?
+docker build --progress=plain --build-arg LOCAL_MODE=${BUILD_LOCAL_MODE} -t "${REGISTRY}/launcher:${TAG}" -f backend/Dockerfile.launcher . && docker push "${REGISTRY}/launcher:${TAG}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Failed to build launcher image."
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -14,6 +14,9 @@
 
 # 1. Build api server application
 FROM golang:1.22.10-bookworm as builder
+
+ARG LOCAL_MODE=false
+
 RUN apt-get update && apt-get install -y cmake clang musl-dev openssl
 WORKDIR /go/src/github.com/kubeflow/pipelines
 
@@ -23,6 +26,13 @@ COPY ./go.sum ./
 RUN GO111MODULE=on go mod download
 
 COPY . .
+
+RUN if [ "$LOCAL_MODE" = "true" ]; then \
+      echo "DEV MODE ENABVLED" && \
+      go mod edit -replace github.com/kubeflow/pipelines/kubernetes_platform=./kubernetes_platform &&  \
+      go mod edit -replace github.com/kubeflow/pipelines/api=./api && \
+      go mod tidy; fi
+
 RUN GO111MODULE=on go build -o /bin/apiserver backend/src/apiserver/*.go
 
 # 2. Compile preloaded pipeline samples
diff --git a/backend/Dockerfile.driver b/backend/Dockerfile.driver
@@ -15,6 +15,7 @@
 FROM golang:1.22.10-alpine3.21 as builder
 
 ARG GCFLAGS=""
+ARG LOCAL_MODE=false
 
 WORKDIR /go/src/github.com/kubeflow/pipelines
 
@@ -25,6 +26,12 @@ RUN GO111MODULE=on go mod download
 
 COPY . .
 
+RUN if [ "$LOCAL_MODE" = "true" ]; then \
+      echo "DEV MODE ENABVLED" && \
+      go mod edit -replace github.com/kubeflow/pipelines/kubernetes_platform=./kubernetes_platform &&  \
+      go mod edit -replace github.com/kubeflow/pipelines/api=./api && \
+      go mod tidy; fi
+
 RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -gcflags="${GCFLAGS}" -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go
 
 FROM alpine:3.19
diff --git a/backend/Dockerfile.launcher b/backend/Dockerfile.launcher
@@ -14,6 +14,8 @@
 
 FROM golang:1.22.10-alpine3.21 as builder
 
+ARG LOCAL_MODE=false
+
 WORKDIR /go/src/github.com/kubeflow/pipelines
 
 COPY ./go.mod ./
@@ -23,6 +25,12 @@ RUN GO111MODULE=on go mod download
 
 COPY . .
 
+RUN if [ "$LOCAL_MODE" = "true" ]; then \
+      echo "DEV MODE ENABVLED" && \
+      go mod edit -replace github.com/kubeflow/pipelines/kubernetes_platform=./kubernetes_platform &&  \
+      go mod edit -replace github.com/kubeflow/pipelines/api=./api && \
+      go mod tidy; fi
+
 RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go
 
 FROM alpine:3.19
diff --git a/backend/Dockerfile.persistenceagent b/backend/Dockerfile.persistenceagent
@@ -14,6 +14,8 @@
 
 FROM golang:1.22.10-alpine3.21 as builder
 
+ARG LOCAL_MODE=false
+
 WORKDIR /go/src/github.com/kubeflow/pipelines
 
 COPY ./go.mod ./
@@ -23,6 +25,12 @@ RUN GO111MODULE=on go mod download
 
 COPY . .
 
+RUN if [ "$LOCAL_MODE" = "true" ]; then \
+      echo "DEV MODE ENABVLED" && \
+      go mod edit -replace github.com/kubeflow/pipelines/kubernetes_platform=./kubernetes_platform &&  \
+      go mod edit -replace github.com/kubeflow/pipelines/api=./api && \
+      go mod tidy; fi
+
 # Needed musl-dev for github.com/mattn/go-sqlite3
 RUN apk update && apk upgrade && \
     apk add --no-cache bash git openssh gcc musl-dev
diff --git a/backend/Dockerfile.scheduledworkflow b/backend/Dockerfile.scheduledworkflow
@@ -14,6 +14,8 @@
 
 FROM golang:1.22.10-alpine3.21 as builder
 
+ARG LOCAL_MODE=false
+
 WORKDIR /go/src/github.com/kubeflow/pipelines
 
 COPY ./go.mod ./
@@ -23,6 +25,12 @@ RUN GO111MODULE=on go mod download
 
 COPY . .
 
+RUN if [ "$LOCAL_MODE" = "true" ]; then \
+      echo "DEV MODE ENABVLED" && \
+      go mod edit -replace github.com/kubeflow/pipelines/kubernetes_platform=./kubernetes_platform &&  \
+      go mod edit -replace github.com/kubeflow/pipelines/api=./api && \
+      go mod tidy; fi
+
 # Needed musl-dev for github.com/mattn/go-sqlite3
 RUN apk update && apk upgrade && \
     apk add --no-cache bash git openssh gcc musl-dev
diff --git a/test/presubmit-backend-test.sh b/test/presubmit-backend-test.sh
@@ -20,11 +20,14 @@ set -ex
 # Add installed go binaries to PATH.
 export PATH="${PATH}:$(go env GOPATH)/bin"
 
-# 1. Check go modules are tidy
+# 1. Reference local proto packages
+go mod edit -replace github.com/kubeflow/pipelines/kubernetes_platform=./kubernetes_platform
+go mod edit -replace github.com/kubeflow/pipelines/api=./api
+
+# 2. Check go modules are tidy
 # Reference: https://github.com/golang/go/issues/27005#issuecomment-564892876
 go mod download
 go mod tidy
-git diff --exit-code -- go.mod go.sum || (echo "go modules are not tidy, run 'go mod tidy'." && exit 1)
 
-# 2. Run tests in the backend directory
+# 1. Run tests in the backend directory
 go test -v -cover ./backend/...
