diff --git a/coreosBuilder.py b/coreosBuilder.py index 6f82dbee..2d2ef192 100644 --- a/coreosBuilder.py +++ b/coreosBuilder.py @@ -81,6 +81,8 @@ def build(self, dst: str) -> None: self._clone_if_not_exists("https://github.com/coreos/coreos-assembler.git") config_dir = self._clone_if_not_exists("https://github.com/coreos/fedora-coreos-config") + lh.run(f"git -C {config_dir} submodule update --init --recursive") + contents = "packages:\n - kernel-modules-extra\n" # contents = contents + " - python3\n - libvirt\n - qemu-img\n - qemu-kvm\n - virt-install\n - netcat\n - bridge-utils\n - tcpdump\n" diff --git a/dpuVendor.py b/dpuVendor.py index 8ec7f3e2..7db30595 100644 --- a/dpuVendor.py +++ b/dpuVendor.py @@ -1,5 +1,6 @@ import re import host +import typing from logger import logger from abc import ABC, abstractmethod import ipu @@ -7,7 +8,11 @@ import clustersConfig -def detect_dpu(node: clustersConfig.NodeConfig) -> str: +def detect_dpu( + node: clustersConfig.NodeConfig, + *, + get_external_port: typing.Callable[[], str], +) -> str: logger.info("Detecting DPU") assert node.kind == "dpu" assert node.bmc is not None @@ -17,7 +22,9 @@ def detect_dpu(node: clustersConfig.NodeConfig) -> str: ipu_bmc.ensure_started() if ipu_bmc.is_ipu(): return "ipu" - elif marvell.is_marvell(node.bmc): + + marvell_bmc = marvell.MarvellBMC(node.bmc, bmc_host=node.bmc_host, get_external_port=get_external_port) + if marvell_bmc.is_marvell(): return "marvell" else: logger.error_and_exit("Unknown DPU") diff --git a/extraConfigDpu.py b/extraConfigDpu.py index bcefe9e7..bf0b7b5f 100644 --- a/extraConfigDpu.py +++ b/extraConfigDpu.py @@ -141,7 +141,7 @@ def ExtraConfigDpu(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[str, acc.run("systemctl stop firewalld") acc.run("systemctl disable firewalld") - vendor_plugin = init_vendor_plugin(acc, detect_dpu(dpu_node)) + vendor_plugin = init_vendor_plugin(acc, detect_dpu(dpu_node, get_external_port=cc.get_external_port)) # TODO: For Intel, this configures hugepages. Figure out a better way vendor_plugin.setup(acc) diff --git a/extraConfigIsoBuilder.py b/extraConfigIsoBuilder.py index 0fd4e765..c8ec8631 100644 --- a/extraConfigIsoBuilder.py +++ b/extraConfigIsoBuilder.py @@ -48,7 +48,7 @@ def ExtraConfigIsoBuilder( if len(cc.masters) < 1: logger.error_and_exit("Error: At least one master is needed for the OS environment to match the DPU requirements") node = cc.masters[0] - dpu_flavor = detect_dpu(node) if node.kind == "dpu" else "agnostic" + dpu_flavor = detect_dpu(node, get_external_port=cc.get_external_port) if node.kind == "dpu" else "agnostic" if dpu_flavor == "ipu": extra_args = " ip=192.168.0.2:::255.255.255.0::enp0s1f0:off netroot=iscsi:192.168.0.1::::iqn.e2000:acc acpi=force" kernel_args = (kernel_args or "") + extra_args diff --git a/host.py b/host.py index a8120565..cfca8109 100644 --- a/host.py +++ b/host.py @@ -160,7 +160,7 @@ def ssh_connect(self, username: str, password: Optional[str] = None, *, discover assert not self.is_localhost() if not self.ping(): logger.info(f"waiting for '{self._hostname}' to respond to ping") - self.wait_ping() + self.wait_ping(timeout=timeout) logger.info(f"{self._hostname} up, connecting with {username}") self._logins = [] @@ -371,11 +371,12 @@ def cold_boot(self) -> None: raise Exception(f"Can't cold boot host without bmc on {self.hostname()}") self._bmc.cold_boot() - def wait_ping(self) -> None: - t = timer.Timer("1h") + def wait_ping(self, *, timeout: str = "1h") -> None: + t = timer.Timer(timeout) while not self.ping(): if t.triggered(): - logger.error_and_exit(f"Waited for 1h for ping to {self.hostname()}") + logger.info(f"Timeout waiting for {t.elapsed()} for ping to {self.hostname()}") + return logger.info(f"Waited for {t.elapsed()} for {self.hostname()} to respond") def ping(self) -> bool: diff --git a/isoDeployer.py b/isoDeployer.py index ed8aa520..1556b917 100644 --- a/isoDeployer.py +++ b/isoDeployer.py @@ -6,6 +6,7 @@ import ipu from baseDeployer import BaseDeployer from clustersConfig import ClustersConfig +from clusterNode import ClusterNode from dpuVendor import detect_dpu from state_file import StateFile import sys @@ -70,17 +71,19 @@ def _deploy_master(self) -> None: self._setup_networking() assert self._master.kind == "dpu" assert self._master.bmc is not None - dpu_kind = detect_dpu(self._master) + cluster_node: ClusterNode + dpu_kind = detect_dpu(self._master, get_external_port=self._cc.get_external_port) if dpu_kind == "ipu": - node = ipu.IPUClusterNode(self._master, self._cc.get_external_port(), self._cc.network_api_port) - node.start(self._cc.install_iso) - node.post_boot() + cluster_node = ipu.IPUClusterNode(self._master, self._cc.get_external_port(), self._cc.network_api_port) elif dpu_kind == "marvell": - marvell.MarvellIsoBoot(self._master, self._cc.install_iso) + cluster_node = marvell.MarvellClusterNode(self._master) else: logger.error("Unknown DPU") sys.exit(-1) + cluster_node.start(self._cc.install_iso) + cluster_node.post_boot() + def _setup_networking(self) -> None: assert self._master.ip is not None gw = common.ip_to_gateway(self._master.ip, "255.255.255.0") diff --git a/marvell.py b/marvell.py index 9916818a..fc8f2e13 100644 --- a/marvell.py +++ b/marvell.py @@ -1,93 +1,160 @@ import os import shlex -from clustersConfig import NodeConfig +import typing +from bmc import BMC from bmc import BmcConfig +from clustersConfig import NodeConfig +from clusterNode import ClusterNode import common import host - - -def marvell_bmc_rsh(bmc: BmcConfig) -> host.Host: - # For Marvell DPU, we require that our "BMC" is the host on has the DPU - # plugged in. - # - # We also assume, that the user name is "core" and that we can SSH into - # that host with public key authentication. We ignore the `bmc.user` - # setting. The reason for that is so that dpu-operator's - # "hack/cluster-config/config-dpu.yaml" (which should work with IPU and - # Marvell DPU) does not need to specify different BMC user name and - # passwords. If you solve how to express the BMC authentication in the - # cluster config in a way that is suitable for IPU and Marvell DPU at the - # same time (e.g. via Jinja2 templates), we can start honoring - # bmc.user/bmc.password. - rsh = host.RemoteHost(bmc.url) - rsh.ssh_connect("core") - return rsh - - -def is_marvell(bmc: BmcConfig) -> bool: - rsh = marvell_bmc_rsh(bmc) - return "177d:b900" in rsh.run("lspci -nn -d :b900").out - - -def _pxeboot_marvell_dpu(name: str, bmc: BmcConfig, mac: str, ip: str, iso: str) -> None: - rsh = marvell_bmc_rsh(bmc) - - ip_addr = f"{ip}/24" - ip_gateway = common.ip_to_gateway(ip, "255.255.255.0") - - # An empty entry means to use the host's "id_ed25519.pub". We want that. - ssh_keys = [""] - for _, pub_key_content, _ in common.iterate_ssh_keys(): - ssh_keys.append(pub_key_content) - - ssh_key_options = [f"--ssh-key={shlex.quote(s)}" for s in ssh_keys] - - image = os.environ.get("CDA_MARVELL_TOOLS_IMAGE", "quay.io/sdaniele/marvell-tools:latest") - - r = rsh.run( - "set -o pipefail ; " - "sudo " - "podman " - "run " - "--pull always " - "--rm " - "--replace " - "--privileged " - "--pid host " - "--network host " - "--user 0 " - "--name marvell-tools " - "-i " - "-v /:/host " - "-v /dev:/dev " - f"{shlex.quote(image)} " - "./pxeboot.py " - f"--dpu-name={shlex.quote(name)} " - "--host-mode=coreos " - f"--nm-secondary-cloned-mac-address={shlex.quote(mac)} " - f"--nm-secondary-ip-address={shlex.quote(ip_addr)} " - f"--nm-secondary-ip-gateway={shlex.quote(ip_gateway)} " - "--yum-repos=rhel-nightly " - "--default-extra-packages " - "--octep-cp-agent-service-disable " - f"{' '.join(ssh_key_options)} " - f"{shlex.quote(iso)} " - "2>&1 " - "| tee \"/tmp/pxeboot-log-$(date '+%Y%m%d-%H%M%S')\"" - ) - if not r.success(): - raise RuntimeError(f"Failure to to pxeboot: {r}") - - -def MarvellIsoBoot(node: NodeConfig, iso: str) -> None: - assert node.ip is not None - assert node.bmc is not None - _pxeboot_marvell_dpu(node.name, node.bmc, node.mac, node.ip, iso) - - -def main() -> None: - pass - - -if __name__ == "__main__": - main() +from logger import logger +import coreosBuilder +from nfs import NFS + + +class MarvellBMC: + def __init__( + self, + bmc: BmcConfig, + *, + bmc_host: typing.Optional[BmcConfig] = None, + get_external_port: typing.Optional[typing.Callable[[], str]] = None, + ) -> None: + assert (bmc_host is None) == (get_external_port is None) + self.bmc = bmc + self._bmc_host = bmc_host + self._get_external_port = get_external_port + + def _ssh_to_bmc(self, *, boot_coreos: bool = True) -> typing.Optional[host.Host]: + # For Marvell DPU, the "BMC" is the host where the DPU is plugged in. + # + # That host also has the serial console of the DPU connected to + # /dev/ttyUSB[01] and "eno4" is (by default) switched together with the + # primary interface enP2p3s0 on the DPU. This interface is also used + # for pxeboot installation. See + # https://github.com/wizhaoredhat/marvell-octeon-10-tools project. + # + # To access those interfaces, the host must be accessible via SSH. + # This function returns a Host instance with SSH connected (usually + # to the "core" user, use via sudo). + # + # If the host is not accessible, the function may first call _boot_coreos() + # method, to boot a CoreOS Live image. For that, the host needs a separate + # bmc_host (which is supposed to be a Redfish BMC of the host). + rsh = host.RemoteHost(self.bmc.url) + + try: + if boot_coreos: + # FIXME: testing only. Drop this part. + raise RuntimeError("TEST: for testing simulate host is unrechable and boot coreos") + + rsh.ssh_connect("core", timeout="2m") + except Exception as e: + logger.info(f"Cannot connect to core @ {self.bmc.url}: {e}") + else: + return rsh + + if self._bmc_host is None or not boot_coreos: + # There is no fallback to boot a CoreOS Live ISO. + return None + + self._boot_coreos() + + rsh = host.RemoteHost(self.bmc.url) + rsh.ssh_connect("core", timeout="15m") + return rsh + + def _boot_coreos(self) -> None: + assert self._bmc_host + assert self._get_external_port + + logger.info(f"For Marvell host {self.bmc.url} boot CoreOS Live via BMC {self._bmc_host.url}") + + coreosBuilder.ensure_fcos_exists() + lh = host.LocalHost() + nfs = NFS(lh, self._get_external_port()) + iso_url = nfs.host_file("/root/iso/fedora-coreos.iso") + + bmc2 = BMC.from_bmc_config(self._bmc_host) + bmc2.boot_iso_redfish(iso_url) + + def is_marvell(self) -> bool: + rsh = self._ssh_to_bmc() + if rsh is None: + return False + return "177d:b900" in rsh.run("lspci -nn -d :b900").out + + def pxeboot( + self, + name: str, + mac: str, + ip: str, + iso: str, + ) -> None: + rsh = self._ssh_to_bmc(boot_coreos=False) + + if rsh is None: + raise RuntimeError(f"Cannot connect to {self.bmc.url} for pxeboot of Marvell DPU") + + ip_addr = f"{ip}/24" + ip_gateway = common.ip_to_gateway(ip, "255.255.255.0") + + # An empty entry means to use the host's "id_ed25519.pub". We want that. + ssh_keys = [""] + for _, pub_key_content, _ in common.iterate_ssh_keys(): + ssh_keys.append(pub_key_content) + + ssh_key_options = [f"--ssh-key={shlex.quote(s)}" for s in ssh_keys] + + image = os.environ.get("CDA_MARVELL_TOOLS_IMAGE", "quay.io/sdaniele/marvell-tools:latest") + + logger.info(f"run pxeboot for {self.bmc.url} to install {image}") + + r = rsh.run( + "set -o pipefail ; " + "sudo " + "podman " + "run " + "--pull always " + "--rm " + "--replace " + "--privileged " + "--pid host " + "--network host " + "--user 0 " + "--name marvell-tools " + "-i " + "-v /:/host " + "-v /dev:/dev " + f"{shlex.quote(image)} " + "./pxeboot.py " + f"--dpu-name={shlex.quote(name)} " + "--host-mode=coreos " + f"--nm-secondary-cloned-mac-address={shlex.quote(mac)} " + f"--nm-secondary-ip-address={shlex.quote(ip_addr)} " + f"--nm-secondary-ip-gateway={shlex.quote(ip_gateway)} " + "--yum-repos=rhel-nightly " + "--default-extra-packages " + "--octep-cp-agent-service-disable " + f"{' '.join(ssh_key_options)} " + f"{shlex.quote(iso)} " + "2>&1 " + "| tee \"/tmp/pxeboot-log-$(date '+%Y%m%d-%H%M%S')\"" + ) + if not r.success(): + raise RuntimeError(f"Failure to to pxeboot: {r}") + + +class MarvellClusterNode(ClusterNode): + def __init__(self, node: NodeConfig) -> None: + assert node.ip is not None + assert node.bmc is not None + self._name = node.name + self._ip = node.ip + self._mac = node.mac + self._bmc = node.bmc + + def start(self, install_iso: str) -> bool: + bmc = MarvellBMC(self._bmc) + bmc.pxeboot(self._name, self._mac, self._ip, install_iso) + return True