From 7ed8bb046c27341fefb50f495aab77e417153397 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Tue, 1 Jul 2025 16:22:55 +0300 Subject: [PATCH 1/3] podvm: add fedora bootc based podvm image Containerfile with nvidia drivers installtion option. Signed-off-by: Snir Sheriber --- .../peerpods/podvm/bootc/Containerfile.fedora | 66 +++++++++++++++++++ .../podvm/bootc/nvidia/generate-nvidia-cdi.sh | 10 +++ .../podvm/bootc/nvidia/nvidia-cdi.service | 11 ++++ 3 files changed, 87 insertions(+) create mode 100644 config/peerpods/podvm/bootc/Containerfile.fedora create mode 100644 config/peerpods/podvm/bootc/nvidia/generate-nvidia-cdi.sh create mode 100755 config/peerpods/podvm/bootc/nvidia/nvidia-cdi.service diff --git a/config/peerpods/podvm/bootc/Containerfile.fedora b/config/peerpods/podvm/bootc/Containerfile.fedora new file mode 100644 index 000000000..faa26cf31 --- /dev/null +++ b/config/peerpods/podvm/bootc/Containerfile.fedora @@ -0,0 +1,66 @@ +# Get payload from upstream (including pause bundle) +FROM quay.io/confidential-containers/podvm-binaries-ubuntu-amd64:v0.13.0 AS payload + +# Build bootc rhel podvm +FROM quay.io/fedora/fedora-bootc:41 AS podvm-bootc + +ARG CLOUD_PROVIDER=azure + +COPY etc /etc +COPY usr /usr + +# afterburn is required for Azure +RUN if [[ "${CLOUD_PROVIDER}" == "azure" ]]; then \ + dnf install -y afterburn && dnf clean all && \ + ln -s ../afterburn-checkin.service /etc/systemd/system/multi-user.target.wants/afterburn-checkin.service; \ + fi + +# Cloud-init is required for Libvirt +RUN if [[ "${CLOUD_PROVIDER}" == "libvirt" ]]; then \ + dnf install -y cloud-init && dnf clean all; \ +fi + +# Copy pause bundle +COPY --from=payload /pause_bundle /pause_bundle + +# Extract podvm binaries +COPY --from=payload /podvm-binaries.tar.gz /podvm-binaries.tar.gz +RUN tar -xzvf podvm-binaries.tar.gz -C / +RUN sed -i 's#What=/kata-containers#What=/var/kata-containers#g' /etc/systemd/system/run-kata\\x2dcontainers.mount + +########## Nvidia podVM target ########## +FROM podvm-bootc AS nvidia-podvm-bootc + +# 570.172.08 or newer is required due to: https://github.com/NVIDIA/open-gpu-kernel-modules/issues/893 +ENV DRIVER_VERSION="570.172.08" + +RUN export KERNEL_VERSION=$(rpm -q --qf "%{VERSION}" kernel-core) && \ + export KERNEL_RELEASE=$(rpm -q --qf "%{RELEASE}" kernel-core | sed 's/\.el.\(_.\)*$//') && \ + export ARCH=$(uname -m) && \ + dnf install -y gcc kernel-devel-${KERNEL_VERSION}-${KERNEL_RELEASE} kernel-devel-matched-${KERNEL_VERSION}-${KERNEL_RELEASE} && \ + dnf install -y 'dnf5-command(config-manager)' && \ + dnf config-manager addrepo --from-repofile=https://developer.download.nvidia.com/compute/cuda/repos/fedora41/x86_64/cuda-fedora41.repo && \ + dnf config-manager --best --nodocs setopt install_weak_deps=False && \ + dnf install -y nvidia-driver-cuda-${DRIVER_VERSION} kmod-nvidia-open-dkms-${DRIVER_VERSION} --exclude=kernel\* && \ + export DRIVER_VERSION=${DRIVER_VERSION:-$(dkms status | grep -oP '\d+\.\d+\.\d+')} && \ + echo "DRIVER_VERSION: ${DRIVER_VERSION}, KERNEL_VERSION-KERNEL_RELEASE: ${KERNEL_VERSION}-${KERNEL_RELEASE}" && \ + sudo dkms build -m nvidia -v ${DRIVER_VERSION} -k ${KERNEL_VERSION}-${KERNEL_RELEASE}.${ARCH} && \ + sudo dkms install -m nvidia -v ${DRIVER_VERSION} -k ${KERNEL_VERSION}-${KERNEL_RELEASE}.${ARCH} && \ + dnf install -y nvidia-container-toolkit && \ + dnf clean all && rm /var/log/*.log* /var/lib/dnf -rf + +RUN echo -e "blacklist nouveau\nblacklist nova_core" > /etc/modprobe.d/blacklist_nv_alt.conf +RUN sed -i 's/^#no-cgroups = false/no-cgroups = true/' /etc/nvidia-container-runtime/config.toml + +ADD --chmod=644 nvidia/nvidia-cdi.service /etc/systemd/system/nvidia-cdi.service +ADD --chmod=755 nvidia/generate-nvidia-cdi.sh /usr/local/bin/generate-nvidia-cdi.sh +RUN ln -s /etc/systemd/system/nvidia-cdi.service /etc/systemd/system/multi-user.target.wants/nvidia-cdi.service + +# TODO: GPU attestation setup + +RUN bootc container lint +######################################### + +# a workaround to set podvm-bootc as default target +FROM podvm-bootc AS default-target +RUN bootc container lint diff --git a/config/peerpods/podvm/bootc/nvidia/generate-nvidia-cdi.sh b/config/peerpods/podvm/bootc/nvidia/generate-nvidia-cdi.sh new file mode 100644 index 000000000..e474b7e62 --- /dev/null +++ b/config/peerpods/podvm/bootc/nvidia/generate-nvidia-cdi.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +#load drivers +nvidia-ctk -d system create-device-nodes --control-devices --load-kernel-modules + +nvidia-persistenced +# set confidential compute to ready state +nvidia-smi conf-compute -srs 1 +# Generate NVIDIA CDI configuration +nvidia-ctk cdi generate --output=/var/run/cdi/nvidia.yaml > /var/log/nvidia-cdi-gen.log 2>&1 diff --git a/config/peerpods/podvm/bootc/nvidia/nvidia-cdi.service b/config/peerpods/podvm/bootc/nvidia/nvidia-cdi.service new file mode 100755 index 000000000..2f511fa78 --- /dev/null +++ b/config/peerpods/podvm/bootc/nvidia/nvidia-cdi.service @@ -0,0 +1,11 @@ +[Unit] +Description=Generate NVIDIA CDI Configuration +Before=kata-agent.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/generate-nvidia-cdi.sh +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target From 5e509438e054aa470ecdb33b2499be6f6a3ee1e7 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Mon, 14 Jul 2025 16:39:07 +0300 Subject: [PATCH 2/3] actions: add image building workflow --- .github/workflows/bootc-podvm-build.yaml | 172 +++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 .github/workflows/bootc-podvm-build.yaml diff --git a/.github/workflows/bootc-podvm-build.yaml b/.github/workflows/bootc-podvm-build.yaml new file mode 100644 index 000000000..b52d4f6ac --- /dev/null +++ b/.github/workflows/bootc-podvm-build.yaml @@ -0,0 +1,172 @@ +name: Build Bootc PodVM Image + +on: + push: + branches: [ devel ] + paths: + - 'config/peerpods/podvm/bootc/**' + workflow_dispatch: + inputs: + cloud_provider: + description: 'Cloud provider (azure, aws, gcp, libvirt)' + required: true + default: 'azure' + type: choice + options: + - azure + - aws + - gcp + - libvirt + build_target: + description: 'Container target to build' + required: false + default: 'podvm-bootc' + type: choice + options: + - podvm-bootc + - nvidia-podvm-bootc + password: + description: 'Password for the "peerpod" user (optional)' + required: false + type: string + ssh_key: + description: 'SSH key for the "peerpod" user (optional)' + required: false + type: string + push_to_quay: + description: 'Push oci image to quay.io' + required: false + default: true + type: boolean + +env: + CLOUD_PROVIDER: ${{ github.event.inputs.cloud_provider || 'azure' }} + +jobs: + build-container: + runs-on: ubuntu-latest + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Determine build target + id: target + run: | + # Default to nvidia, use standard only when explicitly requested + if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.build_target }}" ]]; then + echo "BUILD_TARGET=${{ github.event.inputs.build_target }}" >> $GITHUB_OUTPUT + echo "IMAGE_TAG=quay.io/openshift_sandboxed_containers/fedora-podvm-oci:custom-${{ github.sha }}" >> $GITHUB_OUTPUT + else # default to podvm-bootc + echo "BUILD_TARGET=podvm-bootc" >> $GITHUB_OUTPUT + echo "IMAGE_TAG=quay.io/openshift_sandboxed_containers/fedora-podvm-oci:${{ github.sha }}" >> $GITHUB_OUTPUT + fi + + - name: Build bootc container image + id: build + uses: docker/build-push-action@v6 + with: + context: config/peerpods/podvm/bootc + file: config/peerpods/podvm/bootc/Containerfile.fedora + target: ${{ steps.target.outputs.BUILD_TARGET }} + build-args: | + CLOUD_PROVIDER=${{ env.CLOUD_PROVIDER }} + tags: ${{ steps.target.outputs.IMAGE_TAG }} + # Use less aggressive caching for NVIDIA builds + cache-from: ${{ steps.target.outputs.BUILD_TARGET == 'nvidia-podvm-bootc' && 'type=gha,scope=nvidia' || 'type=gha' }} + cache-to: ${{ steps.target.outputs.BUILD_TARGET == 'nvidia-podvm-bootc' && 'type=gha,scope=nvidia,mode=min' || 'type=gha,mode=max' }} + platforms: linux/amd64 + load: true + + - name: Set up skopeo + uses: warjiang/setup-skopeo@main + with: + version: latest + + - name: Skopeo copy bootc container image to podman + run: | + sudo skopeo copy docker-daemon:${{ steps.target.outputs.IMAGE_TAG }} containers-storage:${{ steps.target.outputs.IMAGE_TAG }} + # Clean up docker image after copying to podman + docker rmi ${{ steps.target.outputs.IMAGE_TAG }} || true + + - name: Create output directory + working-directory: config/peerpods/podvm/bootc + run: | + mkdir -p output/qcow2 + + - name: Adapt config.toml file + working-directory: config/peerpods/podvm/bootc + run: | + [[ ! -f config.toml ]] && echo "default config.toml does not exist" && exit 1 + echo -e "\n[[customizations.user]]" >> config.toml + echo "name = \"peerpod\"" >> config.toml + echo "groups = [\"wheel\"]" >> config.toml + if [[ -n "${{ github.event.inputs.password }}" ]]; then + echo "Using custom password provided by user" + echo "password = \"${{ github.event.inputs.password }}\"" >> config.toml + fi + if [[ -n "${{ github.event.inputs.ssh_key }}" ]]; then + echo "Using custom SSH key provided by user" + echo "key = \"${{ github.event.inputs.ssh_key }}\"" >> config.toml + fi + + - name: Show config.toml file + working-directory: config/peerpods/podvm/bootc + run: | + cat config.toml + + - name: Build disk image + working-directory: config/peerpods/podvm/bootc + run: | + echo "Building disk image..." + sudo podman run \ + --rm \ + --privileged \ + --security-opt label=type:unconfined_t \ + -v $(pwd)/config.toml:/config.toml:ro \ + -v $(pwd)/output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ + quay.io/centos-bootc/bootc-image-builder:latest \ + --type qcow2 \ + --rootfs xfs \ + --local \ + ${{ steps.target.outputs.IMAGE_TAG }} + + - name: Verify disk image exists + working-directory: config/peerpods/podvm/bootc + run: ls -lh ${{ github.workspace }}/config/peerpods/podvm/bootc/output/qcow2/disk.qcow2 + + - name: Login to quay.io + if: ${{ github.event.inputs.push_to_quay == 'true' || github.event_name == 'push' }} + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_TOKEN }} + + - name: Wrap disk in oci image and push to quay.io + uses: docker/build-push-action@v6 + with: + context: config/peerpods/podvm + file: config/peerpods/podvm/Dockerfile.podvm-oci + tags: | + ${{ steps.target.outputs.IMAGE_TAG }} + ${{ github.event_name == 'push' && 'quay.io/openshift_sandboxed_containers/fedora-podvm-oci:latest' || '' }} + labels: | + org.opencontainers.image.created=${{ env.BUILD_DATE }} + org.opencontainers.image.authors=${{ github.actor }} + org.opencontainers.image.source=https://github.com/openshift-sandboxed-containers + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.build-target=${{ steps.target.outputs.BUILD_TARGET }} + org.opencontainers.image.cloud-provider=${{ env.CLOUD_PROVIDER }} + build-args: PODVM_IMAGE_SRC=bootc/output/qcow2/disk.qcow2 + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + push: ${{ github.event.inputs.push_to_quay == 'true' || github.event_name == 'push' }} + load: ${{ github.event.inputs.push_to_quay == 'false' && github.event_name == 'workflow_dispatch' }} From 88350e99cddb27ff942c00d12b70753f7d0fc370 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Mon, 14 Jul 2025 17:21:24 +0300 Subject: [PATCH 3/3] docs: update README Signed-off-by: Snir Sheriber --- config/peerpods/podvm/bootc/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/config/peerpods/podvm/bootc/README.md b/config/peerpods/podvm/bootc/README.md index 00908f145..100739ad0 100644 --- a/config/peerpods/podvm/bootc/README.md +++ b/config/peerpods/podvm/bootc/README.md @@ -1,7 +1,7 @@ # Image Mode (bootc) PodVM Builds Image Mode podVM builds enable OSC users to create PodVM images based -on a RHEL bootc container image. The resulting artifact can be either +on a RHEL/Fedora bootc container image. The resulting artifact can be either a podVM bootc container image or a podVM disk generated from that image. OSC includes updates to the Image Creation mechanism, supporting the conversion and upload of podVM images derived from pre-created bootc @@ -15,11 +15,13 @@ for image creation. Use Podman to build a podVM bootc image locally (push it to a container registry of your choice if needed). -**NOTE:** setting CLOUD_PROVIDER & RHEL Subscription credentials are required only for Azure +**NOTE:** setting CLOUD_PROVIDER & RHEL Subscription credentials are essential only for Azure on RHEL ``` IMG=quay.io/example/podvm-bootc AUTHFILE=/path/to/pull-secret -podman build --authfile ${AUTHFILE} --build-arg CLOUD_PROVIDER=azure --build-arg ORG_ID= --build-arg ACTIVATION_KEY= -f Containerfile.rhel -t ${IMG} +# OPTIONALS=" --build-arg CLOUD_PROVIDER=azure --build-arg ORG_ID= --build-arg ACTIVATION_KEY= " +OS_VARIANT= +podman build --authfile ${AUTHFILE} ${OPTIONALS} -f Containerfile.${OS_VARIANT} -t ${IMG} #podman push ${IMG} ``` @@ -43,7 +45,7 @@ Use [Bootc Image Builder](https://github.com/osbuild/bootc-image-builder) to convert the created podVM bootc container image to a podVM disk file **config.toml:** Use it to set custom bootc build configuration: https://osbuild.org/docs/bootc/#-build-config ``` -# podman pull ${IMG} # optional +# podman pull ${IMG} # if the image is not already available locally mkdir output sudo podman run \ -it --rm \ @@ -80,7 +82,6 @@ Once you have OSC operator installed and before applying KataConfig, ensure your `-podvm-image-cm` values are configured correctly: ``` -IMAGE_TYPE: pre-built PODVM_IMAGE_URI: ${IMG_URI} # Custom bootc build configuration: https://osbuild.org/docs/bootc/#-build-config # default is used if not set @@ -102,7 +103,7 @@ BOOTC_BUILD_CONFIG: | # Optional, custom bootc build configuration: https://osb #### AWS specifics -In order to convert image to AMI (Amazon Machine Image) in-cluster you'll need: +In order to convert image to AMI (Amazon Machine Image) in-cluster (**only needed if peer-pods-secret is manually set**) you'll need: * An existing s3 bucket in the region of your cluster * Your cluster's AWS credntials needs to have the following [permissions](https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html#iam-permissions-image) * [vmimport service role](https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html#vmimport-role) set