Merge branch 'master' into KVMAUTOMA-3534-cpuload

Author: 6-dehan

generic/tests/cfg/netperf.cfg

@@ -34,6 +34,7 @@
     # bridge_nic1 =
     #numa configration
     netperf_with_numa = yes
+    vdpa_add_flows = yes
     # configure netperf test parameters, some seconds will be took to
     # wait all the clients work, this wait time should be less than
     # 0.5 * l, the wait time will augments if you have move
@@ -88,7 +89,7 @@
     #    log_guestinfo_script = scripts/rh_perf_log_guestinfo_script.bat
     #    log_guestinfo_exec = cmd /c
     #    log_guestinfo_path = C:\log_guestinfo.bat
-        server_mtu_cmd = "netsh interface ipv4 set subinterface "%s" mtu=%s"
+        server_mtu_cmd = "netsh interface ipv4 set interface "%s" mtu=%s"
         i386, x86_64:
             cpu_model_flags = ",hv_time,hv_relaxed,hv_vapic,hv_spinlocks=0xfff"
         windows_disable_firewall = "netsh advfirewall set allprofiles state off"

generic/tests/netperf.py

@@ -15,7 +15,7 @@
     virt_vm,
 )
 
-from provider import netperf_base, win_driver_utils
+from provider import netperf_base, vdpa_utils, win_driver_utils
 
 LOG_JOB = logging.getLogger("avocado.test")
 
@@ -62,6 +62,7 @@ def mtu_set(mtu):
         server_mtu_cmd = params.get("server_mtu_cmd")
         client_mtu_cmd = params.get("client_mtu_cmd")
         host_mtu_cmd = params.get("host_mtu_cmd")
+        client_physical_nic = params.get("client_physical_nic")
         error_context.context("Changing the MTU of guest", test.log.info)
         if params.get("os_type") == "linux":
             ethname = utils_net.get_linux_ifname(server_ctl, mac)
@@ -73,9 +74,8 @@ def mtu_set(mtu):
             netperf_base.ssh_cmd(server_ctl, server_mtu_cmd % (connection_id, mtu))
 
         error_context.context("Changing the MTU of client", test.log.info)
-        netperf_base.ssh_cmd(
-            client, client_mtu_cmd % (params.get("client_physical_nic"), mtu)
-        )
+        if client_physical_nic:
+            netperf_base.ssh_cmd(client, client_mtu_cmd % (client_physical_nic, mtu))
 
         netdst = params.get("netdst", "switch")
         host_bridges = utils_net.Bridge()
@@ -84,52 +84,13 @@ def mtu_set(mtu):
         if netdst in br_in_use:
             ifaces_in_use = host_bridges.list_iface()
             target_ifaces = list(ifaces_in_use + br_in_use)
-        if (
-            process.system(
-                "which ovs-vsctl && systemctl status openvswitch.service",
-                ignore_status=True,
-                shell=True,
-            )
-            == 0
-        ):
-            ovs_br_all = netperf_base.ssh_cmd(host, "ovs-vsctl list-br")
-            ovs_br = []
-            if ovs_br_all:
-                for nic in vm.virtnet:
-                    if nic.netdst in ovs_br_all:
-                        ovs_br.append(nic.netdst)
-                    elif nic.nettype == "vdpa":
-                        vf_pci = netperf_base.ssh_cmd(
-                            host,
-                            "vdpa dev show |grep %s | grep -o 'pci/[^[:space:]]*' | "
-                            "awk -F/ '{print $2}'" % nic.netdst,
-                        )
-                        pf_pci = netperf_base.ssh_cmd(
-                            host,
-                            "grep PCI_SLOT_NAME /sys/bus/pci/devices/%s/physfn/uevent |"
-                            " cut -d'=' -f2" % vf_pci,
-                        )
-                        port = netperf_base.ssh_cmd(
-                            host, "ls /sys/bus/pci/devices/%s/net/ | head -n 1" % pf_pci
-                        )
-                        ovs_br_vdpa = netperf_base.ssh_cmd(
-                            host, "ovs-vsctl port-to-br %s" % port
-                        )
-                        cmd = (
-                            f"ovs-ofctl add-flow {ovs_br_vdpa} '"
-                            "in_port=1,idle_timeout=0 actions=output:2'"
-                        )
-                        cmd += (
-                            f"&&  ovs-ofctl add-flow {ovs_br_vdpa} '"
-                            "in_port=2,idle_timeout=0 actions=output:1'"
-                        )
-                        cmd += "&&  ovs-ofctl dump-flows {}".format(ovs_br_vdpa)
-                        netperf_base.ssh_cmd(host, cmd)
-                        ovs_br.append(ovs_br_vdpa)
-                for br in ovs_br:
-                    ovs_list = "ovs-vsctl list-ports %s" % br
-                    ovs_port = netperf_base.ssh_cmd(host, ovs_list)
-                    target_ifaces.extend(ovs_port.split() + [br])
+
+        add_flows = params.get("vdpa_ovs_add_flows", "yes") == "yes"
+        ovs_handler = vdpa_utils.OVSHandler(vm)
+        target_ifaces.extend(
+            ovs_handler.get_vdpa_ovs_info(add_flows=add_flows, return_ports=True)
+        )
+
         if vm.virtnet[0].nettype == "macvtap":
             target_ifaces.extend([vm.virtnet[0].netdst, vm.get_ifname(0)])
         error_context.context("Change all Bridge NICs MTU to %s" % mtu, test.log.info)
@@ -339,6 +300,7 @@ def mtu_set(mtu):
     client = params.get("client", "localhost")
     client_ip = client
     clients = []
+    client_pub_ip = None
     # client session 1 for control, session 2 for data communication
     for i in range(2):
         if client in params.get("vms"):

provider/vdpa_utils.py

@@ -0,0 +1,132 @@
+import logging
+
+from avocado.utils import process
+from virttest import openvswitch, utils_net
+
+LOG_JOB = logging.getLogger("avocado.test")
+
+
+def check_ovs_status():
+    """
+    Check if ovs-vsctl and openvswitch service are installed and running.
+    :return:  True if both are available and running, otherwise False
+    :rtype: bool
+    """
+    cmd = "which ovs-vsctl && systemctl status openvswitch.service"
+    return process.system(cmd, ignore_status=True, shell=True) == 0
+
+
+def get_vf_pci_address(nic_netdst):
+    """
+    Get vf pci address from a given network destination.
+
+    :param nic_netdst: Network destination address
+    :type nic_netdst: str
+
+    :return: VF pci address
+    :rtype: str
+    """
+    cmd = (
+        "vdpa dev show | grep {0} | grep -o 'pci/[^[:space:]]*' | "
+        "awk -F/ '{{print $2}}'"
+    ).format(nic_netdst)
+    return process.system_output(cmd, shell=True).decode().strip()
+
+
+def get_pf_pci_address(vf_pci):
+    """
+    Get pf pci address using vf pci address.
+
+    :param vf_pci: VF pci address
+    :type vf_pci: str
+
+    :return: VF pci address
+    :rtype: str
+    """
+    cmd = (
+        "grep PCI_SLOT_NAME /sys/bus/pci/devices/{0}/physfn/uevent | cut -d'=' -f2"
+    ).format(vf_pci)
+    return process.system_output(cmd, shell=True).decode().strip()
+
+
+def get_pf_port(pf_pci):
+    """
+    Get the port for the pf pci address.
+
+    :param pf_pci: PF pci address
+    :type pf_pci: str
+
+    :return: Port name
+    :rtype: str
+    """
+    cmd = "ls /sys/bus/pci/devices/{0}/net/ | head -n 1".format(pf_pci)
+    return process.system_output(cmd, shell=True).decode().strip()
+
+
+def add_flows_to_ovs_bridge(bridge, ovs):
+    """
+    Add flow rules to the given ovs bridge.
+
+    :parma bridge: OVS bridge name
+    :type bridge: str
+    :param ovs: OVS instance
+    :type ovs: OpenVSwitch
+    """
+    utils_net.openflow_manager(
+        bridge, "add-flow", flow_options="in_port=1,idle_timeout=0,actions=output:2"
+    )
+    utils_net.openflow_manager(
+        bridge, "add-flow", flow_options="in_port=2,idle_timeout=0,actions=output:1"
+    )
+    utils_net.openflow_manager(bridge, "dump-flows")
+
+
+class OVSHandler:
+    def __init__(self, vm):
+        self.vm = vm
+        if check_ovs_status():
+            self.ovs = openvswitch.OpenVSwitchControl()
+        else:
+            self.ovs = None
+
+    def get_vdpa_ovs_info(self, add_flows=True, return_ports=True):
+        """
+        Get OVS bridge and port information.
+
+        :param add_flows: Whether to add flows rules to the ovs bridge
+        :type add_flows: bool
+        :param return_ports: Whether to return port names
+        :type return_ports: bool
+
+        :return: list of target interfaces(bridges and ports) if return_port is Ture,
+        else empty list
+        :rtype: list
+        """
+        if not self.ovs:
+            LOG_JOB.error("Could not find existing Open vSwitch service")
+            return []
+
+        target_ifaces = []
+
+        for nic in self.vm.virtnet:
+            ovs_br = None
+            if nic.nettype == "vdpa":
+                vf_pci = get_vf_pci_address(nic.netdst)
+                pf_pci = get_pf_pci_address(vf_pci)
+                port = get_pf_port(pf_pci)
+                manager, ovs_br = utils_net.find_current_bridge(port)
+            else:
+                try:
+                    manager, ovs_br = utils_net.find_current_bridge(nic.netdst)
+                except NotImplementedError:
+                    ovs_br = None
+            if ovs_br:
+                if add_flows:
+                    add_flows_to_ovs_bridge(ovs_br, self.ovs)
+                if return_ports:
+                    if manager:
+                        ports = set(manager.list_ports(ovs_br))
+                        target_ifaces.extend(ports)
+                    target_ifaces.append(ovs_br)
+
+        return target_ifaces

qemu/tests/balloon_check.py

@@ -649,15 +649,25 @@ def run(test, params, env):
             session = balloon_test.vm.wait_for_login()
             session.cmd_output_safe(memhog_cmd)
             res2 = float(normalize_data_size(process.getoutput(get_res_cmd)))
-            time.sleep(30)
-            res3 = float(normalize_data_size(process.getoutput(get_res_cmd)))
+            release_timeout = int(params.get("balloon_release_time", 30))
+
+            def res_recovered():
+                res3 = float(normalize_data_size(process.getoutput(get_res_cmd)))
+                test.log.debug("Polling RES: %.2fM", res3)
+                return abs(res3 - res1) <= res1 * 0.1
+
+            if not utils_misc.wait_for(res_recovered, timeout=release_timeout, step=5):
+                test.fail("QEMU should consume same memory as before memhog")
+            res4 = float(normalize_data_size(process.getoutput(get_res_cmd)))
             test.log.info(
-                "The RES values are %sM, %sM, and %sM sequentially", res1, res2, res3
+                "The RES values are %sM, %sM, and %sM sequentially", res1, res2, res4
             )
             if res2 - res1 < consumed_mem * 0.5:
                 test.error("QEMU should consume more memory")
-            if res3 - res1 > res1 * 0.1:
-                test.fail("QEMU should consume same memory as before memhog ")
+            catch_call_trace_cmd = params.get("catch_call_trace")
+            call_trace = session.cmd_output_safe(catch_call_trace_cmd)
+            if call_trace:
+                test.fail("There's a Call trace:%s" % call_trace)
         # for windows guest, disable/uninstall driver to get memory leak based on
         # driver verifier is enabled
         if params.get("os_type") == "windows":

qemu/tests/cfg/balloon_check.cfg

@@ -30,6 +30,10 @@
             no RHEL.8.3 RHEL.8.2 RHEL.8.1 RHEL.8.0 RHEL.7
             balloon_opt_free_page_reporting = yes
             get_res_cmd = "top -n1 -b -p %s | grep qemu-kvm | awk -F ' ' '{print $6}'"
+            catch_call_trace = "dmesg -T | grep -i "Call Trace" -A 20"
+            s390x:
+                get_res_cmd = "ps -p %s -o rss= | awk '{printf "%%.2fM\n", $1/1024}'"
+                balloon_release_time = 300
             consumed_mem = 2G
             ppc64,ppc64le:
                 consumed_mem = 4G

qemu/tests/cfg/check_block_size.cfg

@@ -10,7 +10,7 @@
     kill_vm_on_error = yes
     index_enable = no
     image_name_stg = "images/check_block_size_image"
-    image_size_stg = 20G
+    image_size_stg = 30G
     image_verify_bootable = no
     force_create_image_stg = yes
     drive_serial_stg = "TARGET_DISK0"

qemu/tests/cfg/dump_guest_core.cfg

@@ -14,7 +14,7 @@
     gdb_command_file = "/home/gdb_command"
     crash_script = "/home/crash.cmd"
     vmcore_file = "/tmp/vmcore"
-    gdb_command = "gdb --core ${core_file}%s --command=${gdb_command_file}"
+    gdb_command = "gdb /usr/libexec/qemu-kvm --core ${core_file}%s --command=${gdb_command_file}"
     crash_cmd = "crash -i ${crash_script} /usr/lib/debug/lib/modules/%s/vmlinux ${vmcore_file}"
     dump_guest_memory_file = "/usr/share/qemu-kvm/dump-guest-memory.py"
     check_vmcore = 'yes'

qemu/tests/cfg/mlock_on_fault.cfg

@@ -0,0 +1,12 @@
+- mlock_on_fault:
+    virt_test_type = qemu
+    type = mlock_on_fault
+    required_qemu = [10.0,)
+    vms = ""
+    extra_qemu_options = ""
+    s390x:
+        extra_qemu_options = "--no-shutdown -nographic"
+    qemu_cmd_memlock = "%s -enable-kvm -overcommit mem-lock=%s -cpu host ${extra_qemu_options}"
+    memhog_extra_options = "-m %dG -object memory-backend-ram,id=mem0,size=%dG,prealloc=on"
+    qemu_cmd_memhog = "${qemu_cmd_memlock} ${memhog_extra_options} ${extra_qemu_options}"
+    memhog_cmd = "memhog %dG"

qemu/tests/cfg/netkvm_buffer_shortage.cfg

@@ -0,0 +1,31 @@
+- netkvm_buffer_shortage:
+    virt_test_type = qemu
+    type = netkvm_buffer_shortage
+    only Windows
+    only virtio_net
+    vhost = on
+    timeout = 360
+    cdroms += " virtio"
+    vms += " vm2"
+    image_snapshot = yes
+    start_vm = yes
+    start_vm_vm2 = no
+    smp = 2
+    queues = ${smp}
+    vectors = 1024
+    port_num = 12345
+    copy_dest = "C:\"
+    server_script = "server.py"
+    client_script = "client.py"
+    check_live_python = "tasklist | findstr /i python"
+    copy_all_cmd = 'xcopy "WIN_UTILS:\packet_loss_scripts\*" ${copy_dest}'
+    i386:
+        psutil_whl = "psutil-6.1.1-cp37-abi3-win32.whl"
+    x86_64:
+        psutil_whl = "psutil-6.1.1-cp37-abi3-win_amd64.whl"
+    pip_cmd = "py -m pip install ${psutil_whl}"
+    dest_location = "pushd ${copy_dest}"
+    server_cmd = "start cmd /c py ${server_script} ${port_num}"
+    client_cmd = "start cmd /c py ${client_script} 99999 %s ${port_num}"
+    param_name = "MinRxBufferPercent"
+    param_values = "0 25 50 75 100"