[th/marvell-boot-live-iso] marvell: boot live ISO for accessing worker node

coreosBuilder.py

@@ -81,6 +81,8 @@ def build(self, dst: str) -> None:
         self._clone_if_not_exists("https://github.com/coreos/coreos-assembler.git")
         config_dir = self._clone_if_not_exists("https://github.com/coreos/fedora-coreos-config")
 
+        lh.run(f"git -C {config_dir} submodule update --init --recursive")
+
         contents = "packages:\n  - kernel-modules-extra\n"
         # contents = contents + "  - python3\n  - libvirt\n  - qemu-img\n  - qemu-kvm\n  - virt-install\n  - netcat\n  - bridge-utils\n  - tcpdump\n"
 

dpuVendor.py

@@ -1,13 +1,18 @@
 import re
 import host
+import typing
 from logger import logger
 from abc import ABC, abstractmethod
 import ipu
 import marvell
 import clustersConfig
 
 
-def detect_dpu(node: clustersConfig.NodeConfig) -> str:
+def detect_dpu(
+    node: clustersConfig.NodeConfig,
+    *,
+    get_external_port: typing.Callable[[], str],
+) -> str:
     logger.info("Detecting DPU")
     assert node.kind == "dpu"
     assert node.bmc is not None
@@ -17,7 +22,9 @@ def detect_dpu(node: clustersConfig.NodeConfig) -> str:
     ipu_bmc.ensure_started()
     if ipu_bmc.is_ipu():
         return "ipu"
-    elif marvell.is_marvell(node.bmc):
+
+    marvell_bmc = marvell.MarvellBMC(node.bmc, bmc_host=node.bmc_host, get_external_port=get_external_port)
+    if marvell_bmc.is_marvell():
         return "marvell"
     else:
         logger.error_and_exit("Unknown DPU")

extraConfigDpu.py

@@ -141,7 +141,7 @@ def ExtraConfigDpu(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[str,
     acc.run("systemctl stop firewalld")
     acc.run("systemctl disable firewalld")
 
-    vendor_plugin = init_vendor_plugin(acc, detect_dpu(dpu_node))
+    vendor_plugin = init_vendor_plugin(acc, detect_dpu(dpu_node, get_external_port=cc.get_external_port))
     # TODO: For Intel, this configures hugepages. Figure out a better way
     vendor_plugin.setup(acc)
 

extraConfigIsoBuilder.py

@@ -48,7 +48,7 @@ def ExtraConfigIsoBuilder(
     if len(cc.masters) < 1:
         logger.error_and_exit("Error: At least one master is needed for the OS environment to match the DPU requirements")
     node = cc.masters[0]
-    dpu_flavor = detect_dpu(node) if node.kind == "dpu" else "agnostic"
+    dpu_flavor = detect_dpu(node, get_external_port=cc.get_external_port) if node.kind == "dpu" else "agnostic"
     if dpu_flavor == "ipu":
         extra_args = " ip=192.168.0.2:::255.255.255.0::enp0s1f0:off netroot=iscsi:192.168.0.1::::iqn.e2000:acc acpi=force"
         kernel_args = (kernel_args or "") + extra_args

host.py

@@ -160,7 +160,7 @@ def ssh_connect(self, username: str, password: Optional[str] = None, *, discover
         assert not self.is_localhost()
         if not self.ping():
             logger.info(f"waiting for '{self._hostname}' to respond to ping")
-            self.wait_ping()
+            self.wait_ping(timeout=timeout)
         logger.info(f"{self._hostname} up, connecting with {username}")
 
         self._logins = []
@@ -371,11 +371,12 @@ def cold_boot(self) -> None:
             raise Exception(f"Can't cold boot host without bmc on {self.hostname()}")
         self._bmc.cold_boot()
 
-    def wait_ping(self) -> None:
-        t = timer.Timer("1h")
+    def wait_ping(self, *, timeout: str = "1h") -> None:
+        t = timer.Timer(timeout)
         while not self.ping():
             if t.triggered():
-                logger.error_and_exit(f"Waited for 1h for ping to {self.hostname()}")
+                logger.info(f"Timeout waiting for {t.elapsed()} for ping to {self.hostname()}")
+                return
         logger.info(f"Waited for {t.elapsed()} for {self.hostname()} to respond")
 
     def ping(self) -> bool:

isoDeployer.py

@@ -6,6 +6,7 @@
 import ipu
 from baseDeployer import BaseDeployer
 from clustersConfig import ClustersConfig
+from clusterNode import ClusterNode
 from dpuVendor import detect_dpu
 from state_file import StateFile
 import sys
@@ -70,17 +71,19 @@ def _deploy_master(self) -> None:
         self._setup_networking()
         assert self._master.kind == "dpu"
         assert self._master.bmc is not None
-        dpu_kind = detect_dpu(self._master)
+        cluster_node: ClusterNode
+        dpu_kind = detect_dpu(self._master, get_external_port=self._cc.get_external_port)
         if dpu_kind == "ipu":
-            node = ipu.IPUClusterNode(self._master, self._cc.get_external_port(), self._cc.network_api_port)
-            node.start(self._cc.install_iso)
-            node.post_boot()
+            cluster_node = ipu.IPUClusterNode(self._master, self._cc.get_external_port(), self._cc.network_api_port)
         elif dpu_kind == "marvell":
-            marvell.MarvellIsoBoot(self._master, self._cc.install_iso)
+            cluster_node = marvell.MarvellClusterNode(self._master)
         else:
             logger.error("Unknown DPU")
             sys.exit(-1)
 
+        cluster_node.start(self._cc.install_iso)
+        cluster_node.post_boot()
+
     def _setup_networking(self) -> None:
         assert self._master.ip is not None
         gw = common.ip_to_gateway(self._master.ip, "255.255.255.0")

marvell.py

@@ -1,93 +1,160 @@
 import os
 import shlex
-from clustersConfig import NodeConfig
+import typing
+from bmc import BMC
 from bmc import BmcConfig
+from clustersConfig import NodeConfig
+from clusterNode import ClusterNode
 import common
 import host
-
-
-def marvell_bmc_rsh(bmc: BmcConfig) -> host.Host:
-    # For Marvell DPU, we require that our "BMC" is the host on has the DPU
-    # plugged in.
-    #
-    # We also assume, that the user name is "core" and that we can SSH into
-    # that host with public key authentication. We ignore the `bmc.user`
-    # setting. The reason for that is so that dpu-operator's
-    # "hack/cluster-config/config-dpu.yaml" (which should work with IPU and
-    # Marvell DPU) does not need to specify different BMC user name and
-    # passwords. If you solve how to express the BMC authentication in the
-    # cluster config in a way that is suitable for IPU and Marvell DPU at the
-    # same time (e.g. via Jinja2 templates), we can start honoring
-    # bmc.user/bmc.password.
-    rsh = host.RemoteHost(bmc.url)
-    rsh.ssh_connect("core")
-    return rsh
-
-
-def is_marvell(bmc: BmcConfig) -> bool:
-    rsh = marvell_bmc_rsh(bmc)
-    return "177d:b900" in rsh.run("lspci -nn -d :b900").out
-
-
-def _pxeboot_marvell_dpu(name: str, bmc: BmcConfig, mac: str, ip: str, iso: str) -> None:
-    rsh = marvell_bmc_rsh(bmc)
-
-    ip_addr = f"{ip}/24"
-    ip_gateway = common.ip_to_gateway(ip, "255.255.255.0")
-
-    # An empty entry means to use the host's "id_ed25519.pub". We want that.
-    ssh_keys = [""]
-    for _, pub_key_content, _ in common.iterate_ssh_keys():
-        ssh_keys.append(pub_key_content)
-
-    ssh_key_options = [f"--ssh-key={shlex.quote(s)}" for s in ssh_keys]
-
-    image = os.environ.get("CDA_MARVELL_TOOLS_IMAGE", "quay.io/sdaniele/marvell-tools:latest")
-
-    r = rsh.run(
-        "set -o pipefail ; "
-        "sudo "
-        "podman "
-        "run "
-        "--pull always "
-        "--rm "
-        "--replace "
-        "--privileged "
-        "--pid host "
-        "--network host "
-        "--user 0 "
-        "--name marvell-tools "
-        "-i "
-        "-v /:/host "
-        "-v /dev:/dev "
-        f"{shlex.quote(image)} "
-        "./pxeboot.py "
-        f"--dpu-name={shlex.quote(name)} "
-        "--host-mode=coreos "
-        f"--nm-secondary-cloned-mac-address={shlex.quote(mac)} "
-        f"--nm-secondary-ip-address={shlex.quote(ip_addr)} "
-        f"--nm-secondary-ip-gateway={shlex.quote(ip_gateway)} "
-        "--yum-repos=rhel-nightly "
-        "--default-extra-packages "
-        "--octep-cp-agent-service-disable "
-        f"{' '.join(ssh_key_options)} "
-        f"{shlex.quote(iso)} "
-        "2>&1 "
-        "| tee \"/tmp/pxeboot-log-$(date '+%Y%m%d-%H%M%S')\""
-    )
-    if not r.success():
-        raise RuntimeError(f"Failure to to pxeboot: {r}")
-
-
-def MarvellIsoBoot(node: NodeConfig, iso: str) -> None:
-    assert node.ip is not None
-    assert node.bmc is not None
-    _pxeboot_marvell_dpu(node.name, node.bmc, node.mac, node.ip, iso)
-
-
-def main() -> None:
-    pass
-
-
-if __name__ == "__main__":
-    main()
+from logger import logger
+import coreosBuilder
+from nfs import NFS
+
+
+class MarvellBMC:
+    def __init__(
+        self,
+        bmc: BmcConfig,
+        *,
+        bmc_host: typing.Optional[BmcConfig] = None,
+        get_external_port: typing.Optional[typing.Callable[[], str]] = None,
+    ) -> None:
+        assert (bmc_host is None) == (get_external_port is None)
+        self.bmc = bmc
+        self._bmc_host = bmc_host
+        self._get_external_port = get_external_port
+
+    def _ssh_to_bmc(self, *, boot_coreos: bool = True) -> typing.Optional[host.Host]:
+        # For Marvell DPU, the "BMC" is the host where the DPU is plugged in.
+        #
+        # That host also has the serial console of the DPU connected to
+        # /dev/ttyUSB[01] and "eno4" is (by default) switched together with the
+        # primary interface enP2p3s0 on the DPU.  This interface is also used
+        # for pxeboot installation. See
+        # https://github.com/wizhaoredhat/marvell-octeon-10-tools project.
+        #
+        # To access those interfaces, the host must be accessible via SSH.
+        # This function returns a Host instance with SSH connected (usually
+        # to the "core" user, use via sudo).
+        #
+        # If the host is not accessible, the function may first call _boot_coreos()
+        # method, to boot a CoreOS Live image. For that, the host needs a separate
+        # bmc_host (which is supposed to be a Redfish BMC of the host).
+        rsh = host.RemoteHost(self.bmc.url)
+
+        try:
+            if boot_coreos:
+                # FIXME: testing only. Drop this part.
+                raise RuntimeError("TEST: for testing simulate host is unrechable and boot coreos")
+
+            rsh.ssh_connect("core", timeout="2m")
+        except Exception as e:
+            logger.info(f"Cannot connect to core @ {self.bmc.url}: {e}")
+        else:
+            return rsh
+
+        if self._bmc_host is None or not boot_coreos:
+            # There is no fallback to boot a CoreOS Live ISO.
+            return None
+
+        self._boot_coreos()
+
+        rsh = host.RemoteHost(self.bmc.url)
+        rsh.ssh_connect("core", timeout="15m")
+        return rsh
+
+    def _boot_coreos(self) -> None:
+        assert self._bmc_host
+        assert self._get_external_port
+
+        logger.info(f"For Marvell host {self.bmc.url} boot CoreOS Live via BMC {self._bmc_host.url}")
+
+        coreosBuilder.ensure_fcos_exists()
+        lh = host.LocalHost()
+        nfs = NFS(lh, self._get_external_port())
+        iso_url = nfs.host_file("/root/iso/fedora-coreos.iso")
+
+        bmc2 = BMC.from_bmc_config(self._bmc_host)
+        bmc2.boot_iso_redfish(iso_url)
+
+    def is_marvell(self) -> bool:
+        rsh = self._ssh_to_bmc()
+        if rsh is None:
+            return False
+        return "177d:b900" in rsh.run("lspci -nn -d :b900").out
+
+    def pxeboot(
+        self,
+        name: str,
+        mac: str,
+        ip: str,
+        iso: str,
+    ) -> None:
+        rsh = self._ssh_to_bmc(boot_coreos=False)
+
+        if rsh is None:
+            raise RuntimeError(f"Cannot connect to {self.bmc.url} for pxeboot of Marvell DPU")
+
+        ip_addr = f"{ip}/24"
+        ip_gateway = common.ip_to_gateway(ip, "255.255.255.0")
+
+        # An empty entry means to use the host's "id_ed25519.pub". We want that.
+        ssh_keys = [""]
+        for _, pub_key_content, _ in common.iterate_ssh_keys():
+            ssh_keys.append(pub_key_content)
+
+        ssh_key_options = [f"--ssh-key={shlex.quote(s)}" for s in ssh_keys]
+
+        image = os.environ.get("CDA_MARVELL_TOOLS_IMAGE", "quay.io/sdaniele/marvell-tools:latest")
+
+        logger.info(f"run pxeboot for {self.bmc.url} to install {image}")
+
+        r = rsh.run(
+            "set -o pipefail ; "
+            "sudo "
+            "podman "
+            "run "
+            "--pull always "
+            "--rm "
+            "--replace "
+            "--privileged "
+            "--pid host "
+            "--network host "
+            "--user 0 "
+            "--name marvell-tools "
+            "-i "
+            "-v /:/host "
+            "-v /dev:/dev "
+            f"{shlex.quote(image)} "
+            "./pxeboot.py "
+            f"--dpu-name={shlex.quote(name)} "
+            "--host-mode=coreos "
+            f"--nm-secondary-cloned-mac-address={shlex.quote(mac)} "
+            f"--nm-secondary-ip-address={shlex.quote(ip_addr)} "
+            f"--nm-secondary-ip-gateway={shlex.quote(ip_gateway)} "
+            "--yum-repos=rhel-nightly "
+            "--default-extra-packages "
+            "--octep-cp-agent-service-disable "
+            f"{' '.join(ssh_key_options)} "
+            f"{shlex.quote(iso)} "
+            "2>&1 "
+            "| tee \"/tmp/pxeboot-log-$(date '+%Y%m%d-%H%M%S')\""
+        )
+        if not r.success():
+            raise RuntimeError(f"Failure to to pxeboot: {r}")
+
+
+class MarvellClusterNode(ClusterNode):
+    def __init__(self, node: NodeConfig) -> None:
+        assert node.ip is not None
+        assert node.bmc is not None
+        self._name = node.name
+        self._ip = node.ip
+        self._mac = node.mac
+        self._bmc = node.bmc
+
+    def start(self, install_iso: str) -> bool:
+        bmc = MarvellBMC(self._bmc)
+        bmc.pxeboot(self._name, self._mac, self._ip, install_iso)
+        return True