From c1bb932ba35a69e62fb3ee8f3989e4d853119d27 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Wed, 10 Sep 2025 10:30:45 +0200 Subject: [PATCH 1/5] host: honor timeout for ssh_connect() IPUBMC.is_ipu() calls self._version_via_ssh(), which tries to rh.ssh_connect(..., timeout="5s"). But ssh_connect() would first wait for one hour trying to ping the host. After timeout of one hour, wait_ping() will abort CDA. This ping check seems wrong altogether because SSH already retries with timeout. I think this ping check should be dropped. For now, just avoid the hanging, honor the timeout, and don't abort. --- host.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/host.py b/host.py index a8120565..cfca8109 100644 --- a/host.py +++ b/host.py @@ -160,7 +160,7 @@ def ssh_connect(self, username: str, password: Optional[str] = None, *, discover assert not self.is_localhost() if not self.ping(): logger.info(f"waiting for '{self._hostname}' to respond to ping") - self.wait_ping() + self.wait_ping(timeout=timeout) logger.info(f"{self._hostname} up, connecting with {username}") self._logins = [] @@ -371,11 +371,12 @@ def cold_boot(self) -> None: raise Exception(f"Can't cold boot host without bmc on {self.hostname()}") self._bmc.cold_boot() - def wait_ping(self) -> None: - t = timer.Timer("1h") + def wait_ping(self, *, timeout: str = "1h") -> None: + t = timer.Timer(timeout) while not self.ping(): if t.triggered(): - logger.error_and_exit(f"Waited for 1h for ping to {self.hostname()}") + logger.info(f"Timeout waiting for {t.elapsed()} for ping to {self.hostname()}") + return logger.info(f"Waited for {t.elapsed()} for {self.hostname()} to respond") def ping(self) -> bool: From 5fc672e7e8ed8221c32030a441b053c6f2e4ca83 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Fri, 5 Sep 2025 10:12:04 +0200 Subject: [PATCH 2/5] marvell: log a message before running pxeboot command Otherwise, there is nothing at info level telling that we are currently calling a 20 minute long command. When watching the log, you might think the program hangs. Log a message. --- marvell.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/marvell.py b/marvell.py index 9916818a..fe808d53 100644 --- a/marvell.py +++ b/marvell.py @@ -4,6 +4,7 @@ from bmc import BmcConfig import common import host +from logger import logger def marvell_bmc_rsh(bmc: BmcConfig) -> host.Host: @@ -44,6 +45,8 @@ def _pxeboot_marvell_dpu(name: str, bmc: BmcConfig, mac: str, ip: str, iso: str) image = os.environ.get("CDA_MARVELL_TOOLS_IMAGE", "quay.io/sdaniele/marvell-tools:latest") + logger.info(f"run pxeboot for {bmc.url} to install {image}") + r = rsh.run( "set -o pipefail ; " "sudo " From 8e021cff14b1da37dd8cc74dbfd41e478cd0b63b Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Thu, 4 Sep 2025 23:24:42 +0200 Subject: [PATCH 3/5] coreosBuilder: fix build() to initialize git submodule fedora-coreos-config has a git submodule "fedora-bootc". This needs to be initialized, otherwise the build is going to fail: # podman run --rm -ti --userns=host -u root --privileged -v /tmp/build/fcos:/srv/ --device /dev/kvm --device /dev/fuse --tmpfs /tmp -v /var/tmp:/var/tmp --name cosa -v /tmp/build/fedora-coreos-config:/git:ro quay.io/coreos-assembler/coreos-assembler:latest fetch Config commit: 9aebe7895a1f2055569ebc670a2212319c106d1a Using manifest: /srv/src/config/manifest.yaml error: Can't open file "/srv/src/config/manifests/../fedora-bootc/minimal-plus/manifest.yaml" for reading: No such file or directory (os error 2) failed to execute cmd-fetch: exit status 1 It is not clear to me how this ever worked. Fix this by initializing the submodules. --- coreosBuilder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/coreosBuilder.py b/coreosBuilder.py index 6f82dbee..2d2ef192 100644 --- a/coreosBuilder.py +++ b/coreosBuilder.py @@ -81,6 +81,8 @@ def build(self, dst: str) -> None: self._clone_if_not_exists("https://github.com/coreos/coreos-assembler.git") config_dir = self._clone_if_not_exists("https://github.com/coreos/fedora-coreos-config") + lh.run(f"git -C {config_dir} submodule update --init --recursive") + contents = "packages:\n - kernel-modules-extra\n" # contents = contents + " - python3\n - libvirt\n - qemu-img\n - qemu-kvm\n - virt-install\n - netcat\n - bridge-utils\n - tcpdump\n" From 6f1c9c1e41e0bb78acd58f06d33ca4a23022eed6 Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Wed, 3 Sep 2025 13:26:04 +0200 Subject: [PATCH 4/5] marvell: boot live ISO for accessing worker node We use pxeboot to install RHEL on the DPU. For that, we need to SSH into the host (that has the DPU inside) and run a podman command. That requires that the host is up and accessible via SSH. Also, CDA is not told by configuration which kind of DPU the host has inside. Instead, to detect that it should install a Marvell DPU, it needs to SSH into the host and check lspci. If the host is not accessible, then previously installation would fail. Well, the intended approach would have been to install the OCP cluster (including the worker) first. Then the worker node would be accessible via SSH, and all would be good. However, that is not done. Hence, if the machine is not accessible, fallback to boot a CoreOS Live iso. While at it, add MarvellBMC and MarvellClusterNode classes. https://issues.redhat.com/browse/MDC-108 https://issues.redhat.com/browse/IIC-806 --- dpuVendor.py | 11 +- extraConfigDpu.py | 2 +- extraConfigIsoBuilder.py | 2 +- isoDeployer.py | 13 ++- marvell.py | 240 ++++++++++++++++++++++++--------------- 5 files changed, 169 insertions(+), 99 deletions(-) diff --git a/dpuVendor.py b/dpuVendor.py index 8ec7f3e2..7db30595 100644 --- a/dpuVendor.py +++ b/dpuVendor.py @@ -1,5 +1,6 @@ import re import host +import typing from logger import logger from abc import ABC, abstractmethod import ipu @@ -7,7 +8,11 @@ import clustersConfig -def detect_dpu(node: clustersConfig.NodeConfig) -> str: +def detect_dpu( + node: clustersConfig.NodeConfig, + *, + get_external_port: typing.Callable[[], str], +) -> str: logger.info("Detecting DPU") assert node.kind == "dpu" assert node.bmc is not None @@ -17,7 +22,9 @@ def detect_dpu(node: clustersConfig.NodeConfig) -> str: ipu_bmc.ensure_started() if ipu_bmc.is_ipu(): return "ipu" - elif marvell.is_marvell(node.bmc): + + marvell_bmc = marvell.MarvellBMC(node.bmc, bmc_host=node.bmc_host, get_external_port=get_external_port) + if marvell_bmc.is_marvell(): return "marvell" else: logger.error_and_exit("Unknown DPU") diff --git a/extraConfigDpu.py b/extraConfigDpu.py index bcefe9e7..bf0b7b5f 100644 --- a/extraConfigDpu.py +++ b/extraConfigDpu.py @@ -141,7 +141,7 @@ def ExtraConfigDpu(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[str, acc.run("systemctl stop firewalld") acc.run("systemctl disable firewalld") - vendor_plugin = init_vendor_plugin(acc, detect_dpu(dpu_node)) + vendor_plugin = init_vendor_plugin(acc, detect_dpu(dpu_node, get_external_port=cc.get_external_port)) # TODO: For Intel, this configures hugepages. Figure out a better way vendor_plugin.setup(acc) diff --git a/extraConfigIsoBuilder.py b/extraConfigIsoBuilder.py index 0fd4e765..c8ec8631 100644 --- a/extraConfigIsoBuilder.py +++ b/extraConfigIsoBuilder.py @@ -48,7 +48,7 @@ def ExtraConfigIsoBuilder( if len(cc.masters) < 1: logger.error_and_exit("Error: At least one master is needed for the OS environment to match the DPU requirements") node = cc.masters[0] - dpu_flavor = detect_dpu(node) if node.kind == "dpu" else "agnostic" + dpu_flavor = detect_dpu(node, get_external_port=cc.get_external_port) if node.kind == "dpu" else "agnostic" if dpu_flavor == "ipu": extra_args = " ip=192.168.0.2:::255.255.255.0::enp0s1f0:off netroot=iscsi:192.168.0.1::::iqn.e2000:acc acpi=force" kernel_args = (kernel_args or "") + extra_args diff --git a/isoDeployer.py b/isoDeployer.py index ed8aa520..1556b917 100644 --- a/isoDeployer.py +++ b/isoDeployer.py @@ -6,6 +6,7 @@ import ipu from baseDeployer import BaseDeployer from clustersConfig import ClustersConfig +from clusterNode import ClusterNode from dpuVendor import detect_dpu from state_file import StateFile import sys @@ -70,17 +71,19 @@ def _deploy_master(self) -> None: self._setup_networking() assert self._master.kind == "dpu" assert self._master.bmc is not None - dpu_kind = detect_dpu(self._master) + cluster_node: ClusterNode + dpu_kind = detect_dpu(self._master, get_external_port=self._cc.get_external_port) if dpu_kind == "ipu": - node = ipu.IPUClusterNode(self._master, self._cc.get_external_port(), self._cc.network_api_port) - node.start(self._cc.install_iso) - node.post_boot() + cluster_node = ipu.IPUClusterNode(self._master, self._cc.get_external_port(), self._cc.network_api_port) elif dpu_kind == "marvell": - marvell.MarvellIsoBoot(self._master, self._cc.install_iso) + cluster_node = marvell.MarvellClusterNode(self._master) else: logger.error("Unknown DPU") sys.exit(-1) + cluster_node.start(self._cc.install_iso) + cluster_node.post_boot() + def _setup_networking(self) -> None: assert self._master.ip is not None gw = common.ip_to_gateway(self._master.ip, "255.255.255.0") diff --git a/marvell.py b/marvell.py index fe808d53..edc0e71c 100644 --- a/marvell.py +++ b/marvell.py @@ -1,96 +1,156 @@ import os import shlex -from clustersConfig import NodeConfig +import typing +from bmc import BMC from bmc import BmcConfig +from clustersConfig import NodeConfig +from clusterNode import ClusterNode import common import host from logger import logger - - -def marvell_bmc_rsh(bmc: BmcConfig) -> host.Host: - # For Marvell DPU, we require that our "BMC" is the host on has the DPU - # plugged in. - # - # We also assume, that the user name is "core" and that we can SSH into - # that host with public key authentication. We ignore the `bmc.user` - # setting. The reason for that is so that dpu-operator's - # "hack/cluster-config/config-dpu.yaml" (which should work with IPU and - # Marvell DPU) does not need to specify different BMC user name and - # passwords. If you solve how to express the BMC authentication in the - # cluster config in a way that is suitable for IPU and Marvell DPU at the - # same time (e.g. via Jinja2 templates), we can start honoring - # bmc.user/bmc.password. - rsh = host.RemoteHost(bmc.url) - rsh.ssh_connect("core") - return rsh - - -def is_marvell(bmc: BmcConfig) -> bool: - rsh = marvell_bmc_rsh(bmc) - return "177d:b900" in rsh.run("lspci -nn -d :b900").out - - -def _pxeboot_marvell_dpu(name: str, bmc: BmcConfig, mac: str, ip: str, iso: str) -> None: - rsh = marvell_bmc_rsh(bmc) - - ip_addr = f"{ip}/24" - ip_gateway = common.ip_to_gateway(ip, "255.255.255.0") - - # An empty entry means to use the host's "id_ed25519.pub". We want that. - ssh_keys = [""] - for _, pub_key_content, _ in common.iterate_ssh_keys(): - ssh_keys.append(pub_key_content) - - ssh_key_options = [f"--ssh-key={shlex.quote(s)}" for s in ssh_keys] - - image = os.environ.get("CDA_MARVELL_TOOLS_IMAGE", "quay.io/sdaniele/marvell-tools:latest") - - logger.info(f"run pxeboot for {bmc.url} to install {image}") - - r = rsh.run( - "set -o pipefail ; " - "sudo " - "podman " - "run " - "--pull always " - "--rm " - "--replace " - "--privileged " - "--pid host " - "--network host " - "--user 0 " - "--name marvell-tools " - "-i " - "-v /:/host " - "-v /dev:/dev " - f"{shlex.quote(image)} " - "./pxeboot.py " - f"--dpu-name={shlex.quote(name)} " - "--host-mode=coreos " - f"--nm-secondary-cloned-mac-address={shlex.quote(mac)} " - f"--nm-secondary-ip-address={shlex.quote(ip_addr)} " - f"--nm-secondary-ip-gateway={shlex.quote(ip_gateway)} " - "--yum-repos=rhel-nightly " - "--default-extra-packages " - "--octep-cp-agent-service-disable " - f"{' '.join(ssh_key_options)} " - f"{shlex.quote(iso)} " - "2>&1 " - "| tee \"/tmp/pxeboot-log-$(date '+%Y%m%d-%H%M%S')\"" - ) - if not r.success(): - raise RuntimeError(f"Failure to to pxeboot: {r}") - - -def MarvellIsoBoot(node: NodeConfig, iso: str) -> None: - assert node.ip is not None - assert node.bmc is not None - _pxeboot_marvell_dpu(node.name, node.bmc, node.mac, node.ip, iso) - - -def main() -> None: - pass - - -if __name__ == "__main__": - main() +import coreosBuilder +from nfs import NFS + + +class MarvellBMC: + def __init__( + self, + bmc: BmcConfig, + *, + bmc_host: typing.Optional[BmcConfig] = None, + get_external_port: typing.Optional[typing.Callable[[], str]] = None, + ) -> None: + assert (bmc_host is None) == (get_external_port is None) + self.bmc = bmc + self._bmc_host = bmc_host + self._get_external_port = get_external_port + + def _ssh_to_bmc(self, *, boot_coreos: bool = True) -> typing.Optional[host.Host]: + # For Marvell DPU, the "BMC" is the host where the DPU is plugged in. + # + # That host also has the serial console of the DPU connected to + # /dev/ttyUSB[01] and "eno4" is (by default) switched together with the + # primary interface enP2p3s0 on the DPU. This interface is also used + # for pxeboot installation. See + # https://github.com/wizhaoredhat/marvell-octeon-10-tools project. + # + # To access those interfaces, the host must be accessible via SSH. + # This function returns a Host instance with SSH connected (usually + # to the "core" user, use via sudo). + # + # If the host is not accessible, the function may first call _boot_coreos() + # method, to boot a CoreOS Live image. For that, the host needs a separate + # bmc_host (which is supposed to be a Redfish BMC of the host). + rsh = host.RemoteHost(self.bmc.url) + + try: + rsh.ssh_connect("core", timeout="2m") + except Exception as e: + logger.info(f"Cannot connect to core @ {self.bmc.url}: {e}") + else: + return rsh + + if self._bmc_host is None or not boot_coreos: + # There is no fallback to boot a CoreOS Live ISO. + return None + + self._boot_coreos() + + rsh = host.RemoteHost(self.bmc.url) + rsh.ssh_connect("core", timeout="15m") + return rsh + + def _boot_coreos(self) -> None: + assert self._bmc_host + assert self._get_external_port + + logger.info(f"For Marvell host {self.bmc.url} boot CoreOS Live via BMC {self._bmc_host.url}") + + coreosBuilder.ensure_fcos_exists() + lh = host.LocalHost() + nfs = NFS(lh, self._get_external_port()) + iso_url = nfs.host_file("/root/iso/fedora-coreos.iso") + + bmc2 = BMC.from_bmc_config(self._bmc_host) + bmc2.boot_iso_redfish(iso_url) + + def is_marvell(self) -> bool: + rsh = self._ssh_to_bmc() + if rsh is None: + return False + return "177d:b900" in rsh.run("lspci -nn -d :b900").out + + def pxeboot( + self, + name: str, + mac: str, + ip: str, + iso: str, + ) -> None: + rsh = self._ssh_to_bmc(boot_coreos=False) + + if rsh is None: + raise RuntimeError(f"Cannot connect to {self.bmc.url} for pxeboot of Marvell DPU") + + ip_addr = f"{ip}/24" + ip_gateway = common.ip_to_gateway(ip, "255.255.255.0") + + # An empty entry means to use the host's "id_ed25519.pub". We want that. + ssh_keys = [""] + for _, pub_key_content, _ in common.iterate_ssh_keys(): + ssh_keys.append(pub_key_content) + + ssh_key_options = [f"--ssh-key={shlex.quote(s)}" for s in ssh_keys] + + image = os.environ.get("CDA_MARVELL_TOOLS_IMAGE", "quay.io/sdaniele/marvell-tools:latest") + + logger.info(f"run pxeboot for {self.bmc.url} to install {image}") + + r = rsh.run( + "set -o pipefail ; " + "sudo " + "podman " + "run " + "--pull always " + "--rm " + "--replace " + "--privileged " + "--pid host " + "--network host " + "--user 0 " + "--name marvell-tools " + "-i " + "-v /:/host " + "-v /dev:/dev " + f"{shlex.quote(image)} " + "./pxeboot.py " + f"--dpu-name={shlex.quote(name)} " + "--host-mode=coreos " + f"--nm-secondary-cloned-mac-address={shlex.quote(mac)} " + f"--nm-secondary-ip-address={shlex.quote(ip_addr)} " + f"--nm-secondary-ip-gateway={shlex.quote(ip_gateway)} " + "--yum-repos=rhel-nightly " + "--default-extra-packages " + "--octep-cp-agent-service-disable " + f"{' '.join(ssh_key_options)} " + f"{shlex.quote(iso)} " + "2>&1 " + "| tee \"/tmp/pxeboot-log-$(date '+%Y%m%d-%H%M%S')\"" + ) + if not r.success(): + raise RuntimeError(f"Failure to to pxeboot: {r}") + + +class MarvellClusterNode(ClusterNode): + def __init__(self, node: NodeConfig) -> None: + assert node.ip is not None + assert node.bmc is not None + self._name = node.name + self._ip = node.ip + self._mac = node.mac + self._bmc = node.bmc + + def start(self, install_iso: str) -> bool: + bmc = MarvellBMC(self._bmc) + bmc.pxeboot(self._name, self._mac, self._ip, install_iso) + return True From 504f01da7766d6f221bb71157fad79f273e83c8e Mon Sep 17 00:00:00 2001 From: Thomas Haller Date: Tue, 9 Sep 2025 20:25:01 +0200 Subject: [PATCH 5/5] test/marvell: simular host unreachable for testing boot coreos --- marvell.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/marvell.py b/marvell.py index edc0e71c..fc8f2e13 100644 --- a/marvell.py +++ b/marvell.py @@ -44,6 +44,10 @@ def _ssh_to_bmc(self, *, boot_coreos: bool = True) -> typing.Optional[host.Host] rsh = host.RemoteHost(self.bmc.url) try: + if boot_coreos: + # FIXME: testing only. Drop this part. + raise RuntimeError("TEST: for testing simulate host is unrechable and boot coreos") + rsh.ssh_connect("core", timeout="2m") except Exception as e: logger.info(f"Cannot connect to core @ {self.bmc.url}: {e}")