From 13ad716f300b2e78866978a1e4097e5ad016cbc1 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Thu, 12 May 2022 13:36:21 +1000 Subject: [PATCH] tests: Add pane_status command to check for success of issued commands When we use pane_wait to wait for a command issued to a tmux pane to finish we have no idea whether the command succeeded or not. This means that the test scripts can keep running long after the point something vital has failed, making it difficult to work out what went wrong. Add a new pane_status command that checks for success of the issued command and use it in most places instead of pane_wait. We still need explicit pane_wait where we're gathering explicit output with pane_parse, because the way we check the status with 'echo $?' means we lose track of that output. Signed-off-by: David Gibson [sbrivio: - instead of quitting the script, make a test fail if a command issued in a pane fails during a test, and loop until the status code is numeric in pane_status() as a hack to make it a bit more robust - retain usage of pane_wait() in iperf3 and teardown functions as we interrupt iperf3, passt, and pasta, so a non-zero exit code is expected - drop bogus ns_{1,2}_wait() calls in teardown_two_guests(), those functions were never implemented - use pane_status() for "guest" test directives too ] Signed-off-by: Stefano Brivio --- test/lib/setup | 57 +++++++++++++++++++++++--------------------------- test/lib/term | 14 +++++++++++++ test/lib/test | 38 ++++++++++++++++----------------- 3 files changed, 59 insertions(+), 50 deletions(-) diff --git a/test/lib/setup b/test/lib/setup index b954596..6139e8f 100755 --- a/test/lib/setup +++ b/test/lib/setup @@ -32,10 +32,10 @@ setup_passt() { __mbuto_dir="$(mktemp -d)" pane_run GUEST "git -C ${__mbuto_dir} clone git://mbuto.sh/mbuto" - pane_wait GUEST + pane_status GUEST pane_run GUEST "${__mbuto_dir}/mbuto/mbuto -p passt -c lz4 -f mbuto.img" - pane_wait GUEST + pane_status GUEST rm -rf "${__mbuto_dir}" @@ -52,9 +52,9 @@ setup_passt() { [ ${TRACE} -eq 1 ] && __opts="${__opts} --trace" pane_run PASST "make clean" - pane_wait PASST + pane_status PASST pane_run PASST "make valgrind" - pane_wait PASST + pane_status PASST pane_run PASST "valgrind --max-stackframe=$((4 * 1024 * 1024)) --trace-children=yes --vgdb=no --error-exitcode=1 --suppressions=test/valgrind.supp ./passt ${__opts} -f -t 10001 -u 10001 -P passt.pid" sleep 5 @@ -66,7 +66,7 @@ setup_passt() { 'virtio-net.napi_tx=1"' \ " -device virtio-net-pci,netdev=hostnet0,x-txburst=16384" \ " -netdev socket,fd=5,id=hostnet0" - pane_wait GUEST + pane_status GUEST } # setup_pasta() - Create a network and user namespace, connect pasta to it @@ -80,7 +80,7 @@ setup_pasta() { __tty_pid="$(pane_parse NS)" pane_run NS "unshare -rUnpf /bin/sh" - pane_wait NS + pane_status NS pane_run PASST "pstree -p ${__tty_pid} | sed -n 's/.*(\([0-9].*\))$/\1/p'" pane_wait PASST @@ -131,24 +131,24 @@ setup_passt_in_ns() { pane_run PASST "./pasta ${__opts} -t 10001,10002,10011,10012 -T 10003,10013 -u 10001,10002,10011,10012 -U 10003,10013 -P ${__pid_file}" sleep 1 pane_run PASST '' - pane_wait PASST + pane_status PASST __pasta_pid="$(cat "${__pid_file}")" __ns_pid="$(cat /proc/${__pasta_pid}/task/${__pasta_pid}/children | cut -f1 -d' ')" rm "${__pid_file}" pane_run GUEST "nsenter -t ${__ns_pid} -U -n --preserve-credentials" pane_run NS "nsenter -t ${__ns_pid} -U -n -p --preserve-credentials" - pane_wait GUEST - pane_wait NS + pane_status GUEST + pane_status NS pane_run NS "ip -j li sh | jq -rM '.[] | select(.link_type == \"ether\").ifname'" pane_wait NS __ifname="$(pane_parse NS)" pane_run NS "/sbin/udhcpc -i ${__ifname}" - pane_wait NS + pane_status NS sleep 2 pane_run NS "/sbin/dhclient -6 ${__ifname}" - pane_wait NS + pane_status NS __opts= [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/passt_in_pasta.pcap" @@ -157,15 +157,15 @@ setup_passt_in_ns() { if [ ${VALGRIND} -eq 1 ]; then pane_run PASST "make clean" - pane_wait PASST + pane_status PASST pane_run PASST "make valgrind" - pane_wait PASST + pane_status PASST pane_run PASST "valgrind --max-stackframe=$((4 * 1024 * 1024)) --trace-children=yes --vgdb=no --error-exitcode=1 --suppressions=test/valgrind.supp ./passt -f ${__opts} -t 10001,10011,10021,10031 -u 10001,10011,10021,10031 -P passt.pid" else pane_run PASST "make clean" - pane_wait PASST + pane_status PASST pane_run PASST "make" - pane_wait PASST + pane_status PASST pane_run PASST "./passt -f ${__opts} -t 10001,10011,10021,10031 -u 10001,10011,10021,10031 -P passt.pid" fi sleep 5 @@ -178,7 +178,7 @@ setup_passt_in_ns() { 'virtio-net.napi_tx=1"' \ " -device virtio-net-pci,netdev=hostnet0,x-txburst=524288" \ " -netdev socket,fd=5,id=hostnet0" - pane_wait GUEST + pane_status GUEST } # setup_two_guests() - Set up two namespace, run qemu and passt in both of them @@ -216,8 +216,8 @@ setup_two_guests() { pane_run PASST_1 '' pane_run PASST_2 '' - pane_wait PASST_1 - pane_wait PASST_2 + pane_status PASST_1 + pane_status PASST_2 __pasta1_pid="$(cat "${__pid1_file}")" __ns1_pid="$(cat /proc/${__pasta1_pid}/task/${__pasta1_pid}/children | cut -f1 -d' ')" rm "${__pid1_file}" @@ -234,13 +234,13 @@ setup_two_guests() { pane_run GUEST_1 "/sbin/udhcpc -i ${__ifname}" pane_run GUEST_2 "/sbin/udhcpc -i ${__ifname}" - pane_wait GUEST_1 - pane_wait GUEST_2 + pane_status GUEST_1 + pane_status GUEST_2 sleep 2 pane_run GUEST_1 "/sbin/dhclient -6 ${__ifname}" pane_run GUEST_2 "/sbin/dhclient -6 ${__ifname}" - pane_wait GUEST_1 - pane_wait GUEST_2 + pane_status GUEST_1 + pane_status GUEST_2 __opts= [ ${PCAP} -eq 1 ] && __opts="${__opts} -p /tmp/passt_1.pcap" @@ -257,7 +257,7 @@ setup_two_guests() { pane_run PASST_2 "./passt -f ${__opts} -t 10004 -u 10004" pane_run GUEST_2 'cp mbuto.img mbuto_2.img' - pane_wait GUEST_2 + pane_status GUEST_2 pane_run GUEST_1 './qrap 5 kvm -m '${VMEM}' -cpu host -smp '${VCPUS} \ ' -kernel ' "/boot/vmlinuz-$(uname -r)" \ @@ -275,8 +275,8 @@ setup_two_guests() { 'virtio-net.napi_tx=1"' \ " -device virtio-net-pci,netdev=hostnet0,x-txburst=16384" \ " -netdev socket,fd=5,id=hostnet0" - pane_wait GUEST_1 - pane_wait GUEST_2 + pane_status GUEST_1 + pane_status GUEST_2 } # teardown_passt() - Kill qemu, remove passt PID file @@ -301,7 +301,7 @@ teardown_passt_in_ns() { tmux send-keys -t ${PANE_GUEST} "C-d" [ ${VALGRIND} -eq 0 ] && tmux send-keys -t ${PANE_PASST} "C-c" - [ ${VALGRIND} -eq 0 ] && pane_wait GUEST + [ ${VALGRIND} -eq 0 ] && pane_status GUEST tmux send-keys -t ${PANE_PASST} "C-d" tmux send-keys -t ${PANE_NS} "C-d" @@ -331,13 +331,8 @@ teardown_two_guests() { pane_wait PASST_2 tmux send-keys -t ${PANE_PASST_2} "C-d" - tmux send-keys -t ${PANE_NS_1} "C-d" - tmux send-keys -t ${PANE_NS_2} "C-d" - pane_wait GUEST_1 pane_wait GUEST_2 - ns_1_wait - ns_2_wait pane_wait PASST_1 pane_wait PASST_2 } diff --git a/test/lib/term b/test/lib/term index 0093f90..004c2a2 100755 --- a/test/lib/term +++ b/test/lib/term @@ -228,6 +228,20 @@ pane_parse() { printf '%s' "${__buf}" || printf '@EMPTY@' } +# pane_status() - Wait for command to complete and return its exit status +# $1: Pane name +pane_status() { + pane_wait "${1}" + __status="$(pane_parse "${1}")" + while ! [ "${__status}" -eq "${__status}" ]; do + sleep 1 + pane_run "${1}" 'echo $?' + pane_wait "${1}" + __status="$(pane_parse "${1}")" + done + return ${__status} +} + # status_file_end() - Display and log messages when tests from one file are done status_file_end() { [ -z "${STATUS_FILE}" ] && return diff --git a/test/lib/test b/test/lib/test index 2cfec9a..4262343 100755 --- a/test/lib/test +++ b/test/lib/test @@ -52,7 +52,7 @@ test_iperf3() { pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do' \ ':> s${i}.bw; done' - pane_wait "${__pane}" + pane_status "${__pane}" if [ ${UDP_CLIENT} -eq 0 ]; then pane_run "${__pane}" 'for i in $(seq 0 '${__procs}');' \ @@ -70,7 +70,7 @@ test_iperf3() { 'done' fi - pane_wait "${__pane}" + pane_status "${__pane}" sleep 45 pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do' \ 'kill -INT $(cat s${i}.pid) 2>/dev/null; done' @@ -83,7 +83,7 @@ test_iperf3() { pane_parse "${__pane}" pane_run "${__pane}" 'for i in $(seq 0 '${__procs}'); do' \ 'rm -f s${i}.bw; done' - pane_wait "${__pane}" + pane_status "${__pane}" } test_one_line() { @@ -139,13 +139,13 @@ test_one_line() { ;; "host") pane_run HOST "${__arg}" - pane_wait HOST + pane_status HOST || TEST_ONE_nok=1 ;; "hostb") pane_run HOST "${__arg}" ;; "hostw") - pane_wait HOST + pane_status HOST || TEST_ONE_nok=1 ;; "hint") tmux send-keys -t ${PANE_HOST} "C-c" @@ -157,13 +157,13 @@ test_one_line() { ;; "passt") pane_run PASST "${__arg}" - pane_wait PASST + pane_status PASST || TEST_ONE_nok=1 ;; "passtb") pane_run PASST "${__arg}" ;; "passtw") - pane_wait PASST + pane_status PASST || TEST_ONE_nok=1 ;; "pout") __varname="${__arg%% *}" @@ -173,23 +173,23 @@ test_one_line() { ;; "guest") pane_run GUEST "${__arg}" - pane_wait GUEST + pane_status GUEST || TEST_ONE_nok=1 ;; "guestb") pane_run GUEST "${__arg}" ;; "guestw") - pane_wait GUEST + pane_status GUEST || TEST_ONE_nok=1 ;; "guest1") pane_run GUEST_1 "${__arg}" - pane_wait GUEST_1 + pane_status GUEST_1 || TEST_ONE_nok=1 ;; "guest1b") pane_run GUEST_1 "${__arg}" ;; "guest1w") - pane_wait GUEST_1 + pane_status GUEST_1 || TEST_ONE_nok=1 ;; "gtools") pane_run GUEST 'which '"${__arg}"' >/dev/null || echo skip' @@ -208,25 +208,25 @@ test_one_line() { ;; "guest2") pane_run GUEST_2 "${__arg}" - pane_wait GUEST_2 + pane_status GUEST_2 || TEST_ONE_nok=1 ;; "guest2b") pane_run GUEST_2 "${__arg}" ;; "guest2w") - pane_wait GUEST_2 + pane_status GUEST_2 || TEST_ONE_nok=1 ;; "ns") pane_run NS "${__arg}" - pane_wait NS + pane_status NS || TEST_ONE_nok=1 ;; "ns1") pane_run NS1 "${__arg}" - pane_wait NS1 + pane_status NS1 || TEST_ONE_nok=1 ;; "ns2") pane_run NS2 "${__arg}" - pane_wait NS2 + pane_status NS2 || TEST_ONE_nok=1 ;; "nsb") pane_run NS "${__arg}" @@ -238,13 +238,13 @@ test_one_line() { pane_run NS2 "${__arg}" ;; "nsw") - pane_wait NS + pane_status NS || TEST_ONE_nok=1 ;; "ns1w") - pane_wait NS1 + pane_status NS1 || TEST_ONE_nok=1 ;; "ns2w") - pane_wait NS2 + pane_status NS2 || TEST_ONE_nok=1 ;; "nstools") pane_run NS 'which '"${__arg}"' >/dev/null || echo skip'