ART-10946 rhcos_sync nightlies to dev dir (#4295)

thegreyd · joepvd · web-flow · commit 9ccf2354db6a · 2024-11-20T15:31:56.000-05:00
Co-authored-by: Joep van Delft &lt;jpvndlft@gmail.com&gt;
diff --git a/jobs/build/rhcos_sync/Jenkinsfile b/jobs/build/rhcos_sync/Jenkinsfile
@@ -65,28 +65,46 @@ node {
     }
 
     tag = params.RELEASE_TAG
-    image = "quay.io/openshift-release-dev/ocp-release:${tag}"
-    if (tag.contains("/")) {
-        // assume instead of a tag it's a pullspec e.g. to registry.ci
-        if (!tag.contains(":")) error("RELEASE_TAG pullspec must include a :tag")
-        image = tag
-        tag = tag.split(":")[-1]
-    }
-
-    name = commonlib.shell(
-        returnStdout: true,
-        script: "oc adm release info -o template --template '{{ .metadata.version }}' ${image}"
-    )
 
     (major, minor) = commonlib.extractMajorMinorVersionNumbers(tag)
     ocpVersion = "$major.$minor"
 
     (arch, priv) = releaselib.getReleaseTagArchPriv(tag)
     suffix = releaselib.getArchPrivSuffix(arch, priv)
 
+    pullspec = ""
+    if (tag.contains("/")) {
+        // assume instead of a tag it's a pullspec e.g. to registry.ci
+        if (!tag.contains(":")) error("RELEASE_TAG pullspec must include a :tag")
+        pullspec = tag
+        tag = tag.split(":")[-1]
+    }
+    if (pullspec == "") {
+        if (tag.contains("nightly")) {
+            pullspec = "registry.ci.openshift.org/ocp${suffix}/release${suffix}:${tag}"
+        } else {
+            pullspec = "quay.io/openshift-release-dev/ocp-release:${tag}"
+        }
+    }
+
+    onlyIfDifferent = false
+    noLatest = params.NO_LATEST
+    // for nightlies, sync them to dev dir
+    // This is a special case. see: ART-10946
+    if (tag.contains("nightly")) {
+        name = "dev-${ocpVersion}"
+        noLatest = true
+        onlyIfDifferent = true
+    } else {
+        name = commonlib.shell(
+            returnStdout: true,
+            script: "oc adm release info -o template --template '{{ .metadata.version }}' ${pullspec}"
+        )
+    }
+
     cmd = """
         tmp=\$(mktemp -d /tmp/tmp.XXXXXX)
-        oc image extract --path /manifests/:\$tmp \$(oc adm release info --image-for installer ${image})
+        oc image extract --path /manifests/:\$tmp \$(oc adm release info --image-for installer ${pullspec})
         cat \$tmp/coreos-bootimages.yaml | yq -r .data.stream | jq -r .architectures.${arch}.artifacts.qemu.release
         rm -rf \$tmp
     """
@@ -101,23 +119,25 @@ node {
 
     print("RHCOS build: $rhcosBuild, arch: $arch, mirror prefix: $mirrorPrefix")
 
-    echo("Initializing RHCOS-${params.MIRROR_PREFIX} sync: #${currentBuild.number}")
+    echo("Initializing RHCOS-${mirrorPrefix} sync: #${currentBuild.number}")
     rhcoslib.initialize(ocpVersion, rhcosBuild, arch, name, mirrorPrefix)
 
     try {
         stage("Get/Generate sync list") {
             rhcoslib.rhcosSyncPrintArtifacts()
         }
         stage("Mirror artifacts") {
-            rhcoslib.rhcosSyncMirrorArtifacts(mirrorPrefix, arch, rhcosBuild, name)
+            rhcoslib.rhcosSyncMirrorArtifacts(mirrorPrefix, arch, rhcosBuild, name, noLatest, onlyIfDifferent)
         }
         stage("Slack notification to release channel") {
-            slacklib.to(ocpVersion).say("""
-            *:white_check_mark: rhcos_sync (${mirrorPrefix}) successful*
-            https://mirror.openshift.com/pub/openshift-v4/${arch}/dependencies/rhcos/${mirrorPrefix}/${name}/
+            if ( !params.DRY_RUN ) {
+                slacklib.to(ocpVersion).say("""
+                *:white_check_mark: rhcos_sync (${mirrorPrefix}) successful*
+                https://mirror.openshift.com/pub/openshift-v4/${arch}/dependencies/rhcos/${mirrorPrefix}/${name}/
 
-            buildvm job: ${commonlib.buildURL('console')}
-            """)
+                buildvm job: ${commonlib.buildURL('console')}
+                """)
+            }
         }
 
         // only run for x86_64 since no AMIs for other arches
@@ -136,9 +156,11 @@ node {
                 }
             }
             stage("Slack notification to release channel") {
-                slacklib.to(ocpVersion).say("""
-                *:white_check_mark: rosa_sync (${name}) successful*
-                """)
+                if ( !params.DRY_RUN ) {
+                    slacklib.to(ocpVersion).say("""
+                    *:white_check_mark: rosa_sync (${name}) successful*
+                    """)
+                }
             }
         }
 
@@ -158,10 +180,13 @@ node {
             }
         }
     } catch ( err ) {
-        slacklib.to(ocpVersion).say("""
-        *:heavy_exclamation_mark: rhcos_sync ${mirrorPrefix} failed*
-        buildvm job: ${commonlib.buildURL('console')}
-        """)
+        if ( !params.DRY_RUN ) {
+            slacklib.to(ocpVersion).say("""
+            *:heavy_exclamation_mark: rhcos_sync ${mirrorPrefix} failed*
+            buildvm job: ${commonlib.buildURL('console')}
+            """)
+        }
+        throw (err)
     } finally {
         commonlib.safeArchiveArtifacts(rhcoslib.artifacts)
         buildlib.cleanWorkspace()
diff --git a/jobs/build/rhcos_sync/S3-rhcossync.sh b/jobs/build/rhcos_sync/S3-rhcossync.sh
@@ -38,8 +38,8 @@ EOF
 }
 
 function downloadImages() {
-    for img in $(<${SYNCLIST}); do
-        curl -L --fail --retry 5 -O $img
+    for img in $(<"${SYNCLIST}"); do
+        curl -L --fail --silent --retry 5 -O "$img"
     done
     # rename files to indicate the release they match (including arch suffix by tradition).
     # also create an unversioned symlink to enable consistent incoming links.
@@ -49,15 +49,15 @@ function downloadImages() {
         # name like "rhcos-buildid-qemu..."
         file="${name/$BUILDID/$release}"  # rhcos-release-qemu...
         link="${name/-$BUILDID/}"         # rhcos-qemu...
-        [[ $name == $file ]] && continue  # skip files that aren't named that way
+        [[ $name == "$file" ]] && continue  # skip files that aren't named that way
         mv "$name" "$file"
         ln --symbolic "$file" "$link"
     done
     # Some customer portals point to the deprecated `rhcos-installer` names rather than `rhcos-live`.
     # Fix those links.
-    for f in $(find . -maxdepth 1 -type l -name 'rhcos-live-*'); do
-        ln -s "$(readlink $f)" "${f/rhcos-live-/rhcos-installer-}"
-    done
+    find . -maxdepth 1 -type l -name 'rhcos-live-*' -exec sh -c '
+        ln -s "$(readlink $1)" "${1/rhcos-live-/rhcos-installer-}"
+    ' sh {} \;
 }
 
 function genSha256() {
@@ -67,19 +67,19 @@ function genSha256() {
 }
 
 function genRhcosIdTxt() {
-    echo ${BUILDID} > rhcos-id.txt
+    echo "${BUILDID}" > rhcos-id.txt
 }
 
 function emulateSymlinks() {
     S3_SOURCE="$1"
-    MAJOR_MINOR=$(echo ${VERSION} |awk -F '[.-]' '{print $1 "." $2}')  # e.g. 4.3.0-0.nightly-2019-11-08-080321 -> 4.3
+    MAJOR_MINOR=$(echo "${VERSION}" |awk -F '[.-]' '{print $1 "." $2}')  # e.g. 4.3.0-0.nightly-2019-11-08-080321 -> 4.3
     # We also need to know what Y stream comes after this one.
-    MAJOR_NEXT_MINOR=$(echo ${MAJOR_MINOR} |awk -F '[.-]' '{print $1 "." $2+1}')  # e.g. 4.3 -> 4.4
+    MAJOR_NEXT_MINOR=$(echo "${MAJOR_MINOR}" |awk -F '[.-]' '{print $1 "." $2+1}')  # e.g. 4.3 -> 4.4
 
     if [[ "${RHCOS_MIRROR_PREFIX}" == "pre-release" ]]; then
         MAJOR_MINOR_LATEST="latest-${MAJOR_MINOR}"
-        aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${MAJOR_MINOR_LATEST}/
-        aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${MAJOR_MINOR_LATEST}/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+        "${SYNC_CMD[@]}" "${S3_SOURCE}" "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${MAJOR_MINOR_LATEST}/"
+        "${SYNC_CMD[@]}" "${S3_SOURCE}" "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${MAJOR_MINOR_LATEST}/" "${CLOUDFLARE_OPTS[@]}"
 
         # Is this major.minor the latest Y stream? If it is, we need to set
         # the overall 'latest'.
@@ -98,8 +98,8 @@ function emulateSymlinks() {
         # LATEST_LINK will end up being something like 4.9.0-fc.0 if the next major exists or "" if it does not.
 
         if [[ -z "${LATEST_LINK}" ]]; then
-            aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/
-            aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+            "${SYNC_CMD[@]}" "${S3_SOURCE}" "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/"
+            "${SYNC_CMD[@]}" "${S3_SOURCE}" "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/" "${CLOUDFLARE_OPTS[@]}"
         fi
 
     else
@@ -109,8 +109,8 @@ function emulateSymlinks() {
         LATEST_CONTENT=$(aws s3 ls "s3://art-srv-enterprise${BASEDIR}/${MAJOR_NEXT_MINOR}/" | grep PRE || true)
 
         if [[ -z "${LATEST_CONTENT}" ]]; then
-            aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/latest/
-            aws s3 sync --no-progress --delete --exact-timestamps ${S3_SOURCE} s3://art-srv-enterprise${BASEDIR}/latest/ --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+            "${SYNC_CMD[@]}" "${S3_SOURCE}" "s3://art-srv-enterprise${BASEDIR}/latest/"
+            "${SYNC_CMD[@]}" "${S3_SOURCE}" "s3://art-srv-enterprise${BASEDIR}/latest/" "${CLOUDFLARE_OPTS[@]}"
         fi
     fi
 
@@ -125,7 +125,7 @@ if [ "${#}" -lt "8" ]; then
     exit 1
 fi
 
-while [ $1 ]; do
+while [ "$1" ]; do
     case "$1" in
     "--prefix")
         shift
@@ -159,6 +159,14 @@ while [ $1 ]; do
     shift
 done
 
+DRY_RUN_FLAG=""
+if [ $TEST -eq 1 ]; then
+    DRY_RUN_FLAG="--dryrun"
+fi
+
+SYNC_CMD=(aws s3 sync --no-progress --exact-timestamps --delete ${DRY_RUN_FLAG})
+CLOUDFLARE_OPTS=(--profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT})
+
 DESTDIR="${PWD}/staging-${VERSION}"
 mkdir -p "${DESTDIR}"
 
@@ -170,28 +178,22 @@ Sync List: ${SYNCLIST}
 Basedir: ${BASEDIR}
 EOF
 
-pushd $DESTDIR
+pushd "${DESTDIR}"
 downloadImages
 genSha256
 genRhcosIdTxt
 
-if [ $TEST -eq 1 ]; then
-  echo Would have copied out to ${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/:
-  ls
-  exit 0
-fi
-
 # Copy the files out to their main location
-aws s3 sync --no-progress --delete --exact-timestamps ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/"
-aws s3 sync --no-progress --delete --exact-timestamps ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/" --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+"${SYNC_CMD[@]}" ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/"
+"${SYNC_CMD[@]}" ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/${VERSION}/" "${CLOUDFLARE_OPTS[@]}"
 if [ $NOLATEST -eq 0 ]; then
-    aws s3 sync --no-progress --delete --exact-timestamps ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/"
-    aws s3 sync --no-progress --delete --exact-timestamps ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/" --profile cloudflare --endpoint-url ${CLOUDFLARE_ENDPOINT}
+    "${SYNC_CMD[@]}" ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/"
+    "${SYNC_CMD[@]}" ./ "s3://art-srv-enterprise${BASEDIR}/${RHCOS_MIRROR_PREFIX}/latest/" "${CLOUDFLARE_OPTS[@]}"
     # CloudFlare does not support the full S3 API and we encountered unimplemented APIs using the AWS CLI to sync from one area of R2 to another area.
     # We therefore re-copy from local content to R2, which does not hit these limitations. This is technically slower, but works with R2.
     emulateSymlinks "./"
 else
     echo "INFO: Not updating 'latest' symlink because --nolatest was given"
 fi
 popd
-rm -rf $DESTDIR
+rm -rf "${DESTDIR}"
diff --git a/jobs/build/rhcos_sync/rhcoslib.groovy b/jobs/build/rhcos_sync/rhcoslib.groovy
@@ -3,7 +3,6 @@ buildlib = releaselib.buildlib
 commonlib = releaselib.commonlib
 rhcosWorking = "rhcos_working"
 logLevel = ""
-dryRun = ""
 artifacts = []
 baseUrl = ""
 metaUrl = ""
@@ -17,41 +16,15 @@ rhcos_allowlist = [ "gcp", "initramfs", "iso", "kernel", "metal", "openstack", "
 def initialize(ocpVersion, rhcosBuild, arch, name, mirrorPrefix) {
     buildlib.cleanWorkdir(rhcosWorking)
 
-    // Example URL paths (visit https://releases-rhcos-art.apps.ocp-virt.prod.psi.redhat.com/ to view yourself):
-    // 4.1 with x86 arch:       releases/rhcos-4.1/410.81.20200213.0/meta.json  (we do not plan to release a new 4.1 bootimage ever)
-    // 4.1 with non-x86 arch:   N/A
+    baseUrl = buildlib.doozer("--quiet --group=openshift-${ocpVersion} config:read-group urls.rhcos_release_base.multi --default ''", [capture: true]).trim()
+    baseUrl = "${baseUrl}/${rhcosBuild}/${arch}"
 
-    // 4.2 introduced an arch arch suffix
-    // 4.2 with x86 arch:       releases/rhcos-4.2/42.81.20200213.0/meta.json
-    // 4.2 with non-x86 arch:   releases/rhcos-4.2-s390x/42s390x.81.20200131.0/meta.json
-
-    // 4.3 introduced an arch subdirectory
-    // 4.3 with x86 arch:       releases/rhcos-4.3/43.81.202002130953.0/x86_64/meta.json
-    // 4.3 with non-x86 arch:   releases/rhcos-4.3-s390x/43.81.202001300441.0/s390x/meta.json
-
-    // Sub in some vars according to params
-    def archSuffix = commonlib.brewSuffixForArch(arch)
-    def archDir = ocpVersion == "4.2" ? "" : "/${arch}"
-    baseUrl = "https://art-rhcos-ci.s3.amazonaws.com/releases/rhcos-${ocpVersion}${archSuffix}/${rhcosBuild}${archDir}"
-    // Since 4.9+, rhcos meta is placed at a different location.
-    def (major, minor) = commonlib.extractMajorMinorVersionNumbers(ocpVersion)
-    if (major > 4 || (major == 4 && minor >=9)) {
-        baseUrl = "https://releases-rhcos-art.apps.ocp-virt.prod.psi.redhat.com/storage/prod/streams/$ocpVersion/builds/$rhcosBuild/$arch"
-        if (minor >= 16) {
-            baseUrl = "https://releases-rhcos-art.apps.ocp-virt.prod.psi.redhat.com/storage/prod/streams/$ocpVersion-9.4/builds/$rhcosBuild/$arch"
-        } else if (minor >= 13) {
-            baseUrl = "https://releases-rhcos-art.apps.ocp-virt.prod.psi.redhat.com/storage/prod/streams/$ocpVersion-9.2/builds/$rhcosBuild/$arch"
-        }
-    }
     s3MirrorBaseDir = "/pub/openshift-v4/${arch}/dependencies/rhcos"
-    // Actual meta.json
-    metaUrl = baseUrl + "/meta.json"
+    metaUrl = "${baseUrl}/meta.json"
 
     currentBuild.displayName = "${name} - ${rhcosBuild}:${arch} - ${mirrorPrefix}"
     currentBuild.description = "Meta JSON: ${metaUrl}"
-
     if ( params.DRY_RUN ) {
-        dryRun = "--dry-run=true"
         currentBuild.displayName += " [DRY_RUN]"
     }
 
@@ -77,7 +50,30 @@ def rhcosSyncPrintArtifacts() {
     writeFile file: syncList, text: "${imageUrls.join('\n')}"
 }
 
-def rhcosSyncMirrorArtifacts(rhcosMirrorPrefix, arch, rhcosBuild, name) {
+def getRhcosBuildFromMirror(rhcosMirrorPrefix, name) {
+    def rhcosIdUrl = "https://mirror.openshift.com${s3MirrorBaseDir}/${rhcosMirrorPrefix}/${name}/rhcos-id.txt"
+    def res = commonlib.shell(script: "curl --fail --silent --retry 3 --retry-delay 10 -L ${rhcosIdUrl}", returnAll: true)
+    def rhcosId = "[NOT FOUND]"
+    if (res.returnStatus == 0) {
+        rhcosId = res.stdout.trim()
+    }
+    return rhcosId
+}
+
+def rhcosSyncMirrorArtifacts(rhcosMirrorPrefix, arch, rhcosBuild, name, noLatest, onlyIfDifferent) {
+    // check if rhcos-id is already on the mirror
+    def rhcosBuildOnMirror = getRhcosBuildFromMirror(rhcosMirrorPrefix, name)
+    echo("RHCOS build requested to sync: ${rhcosBuild}")
+    echo("RHCOS build on mirror: ${rhcosBuildOnMirror}")
+    if (rhcosBuildOnMirror == rhcosBuild) {
+        if (!params.FORCE && onlyIfDifferent) {
+            echo("RHCOS build is already on mirror, skipping sync")
+            return
+        }
+    } else {
+        echo("RHCOS build ${rhcosBuild} not on mirror, syncing")
+    }
+
     def invokeOpts = " --prefix ${rhcosMirrorPrefix}" +
         " --arch ${arch}" +
         " --buildid ${rhcosBuild}" +
@@ -86,7 +82,7 @@ def rhcosSyncMirrorArtifacts(rhcosMirrorPrefix, arch, rhcosBuild, name) {
     if ( params.DRY_RUN ) {
             invokeOpts += " --test"
     }
-    if ( params.NO_LATEST ) {
+    if ( noLatest ) {
             invokeOpts += " --nolatest"
     }
 
diff --git a/scheduled-jobs/build/poll-payload/Jenkinsfile b/scheduled-jobs/build/poll-payload/Jenkinsfile
@@ -18,7 +18,9 @@ import groovy.json.JsonOutput
     "4-dev-preview-ppc64le": [this.&setDevPreviewLatest],
     "4-dev-preview-arm64": [this.&setDevPreviewLatest],
 
-    "4.18.0-0.nightly":  [this.&startBuildMicroshiftJob, this.&publishRPMsEl9, this.&publishRPMsEl8],
+    "4.19.0-0.nightly":  [this.&startRhcosSyncJob],
+
+    "4.18.0-0.nightly":  [this.&startBuildMicroshiftJob, this.&publishRPMsEl9, this.&publishRPMsEl8, this.&startRhcosSyncJob],
     "4.18.0-0.nightly-arm64": [this.&publishRPMsEl9, this.&publishRPMsEl8],
 
     "4.17.0-0.nightly":  [this.&startBuildMicroshiftJob, this.&publishRPMsEl9, this.&publishRPMsEl8],
@@ -34,6 +36,16 @@ import groovy.json.JsonOutput
     "4.14.0-0.nightly-arm64": [this.&publishRPMsEl9],
 ]
 
+def startRhcosSyncJob(String releaseStream, Map latestRelease, Map previousRelease) {
+    build(
+        job: '/aos-cd-builds/build%2Frhcos_sync',
+        propagate: false,
+        parameters: [
+            string(name: 'RELEASE_TAG', value: latestRelease.name),
+        ],
+    )
+}
+
 def setDevPreviewLatest(String releaseStream, Map latestRelease, Map previousRelease) {
     def buildVersion = commonlib.extractMajorMinorVersion(latestRelease.name)
     def arch = commonlib.extractArchFromReleaseName(releaseStream)
