[PATCH] RFC: Sandbox testsuite in a specified network environment

[PATCH] RFC: Sandbox testsuite in a specified network environment

[David Gibson]

Currently our testsuite runs passt and pasta directly on the host
machine.  This means that the "host" network environment that
passt/pasta uses is that of the host.  However passt and the tests in
particular have a number of assumptions and requirements about that
environment that may not always be the case.  In particular we require
both IPv4 and IPv6 default routes to be available, and on the same
external interface.

This makes the testsuite quite fragile.  Instead, alter our test
scripts so that we run all the tests inside a namespace where we
configure a specific environment that's suitable for them.  This does,
in a sense, reduce test coverage, because everyone is now testing the
same network environment rather than different onces.  This is
probably preferable to the current case, though, where people avoid
running the testsuite entirely because it doesn't work on their setup.

FIXME: currently disables podman tests, because they require some external
       access that doesn't work in the sandbox

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 test/lib/sandbox | 121 +++++++++++++++++++++++++++++++++++++++++++++++
 test/lib/term    |   6 +--
 test/run         |   7 +--
 3 files changed, 128 insertions(+), 6 deletions(-)
 create mode 100644 test/lib/sandbox

As noted, this isn't ready to merge, because it breaks the podman
tests.  However, Jon, Laurent and anyone else who's had trouble
getting the testsuite running, I'd appreciate if you can try this and
see how it works.

diff --git a/test/lib/sandbox b/test/lib/sandbox
new file mode 100644
index 00000000..d70c4ca5
--- /dev/null
+++ b/test/lib/sandbox
@@ -0,0 +1,121 @@
+#!/bin/sh
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# PASST - Plug A Simple Socket Transport
+#  for qemu/UNIX domain socket mode
+#
+# PASTA - Pack A Subtle Tap Abstraction
+#  for network namespace/tap device mode
+#
+# test/lib/sandbox - Run test programs in an isolated, controlled network environment
+#
+# Copyright Red Hat
+# Author: David Gibson <david@gibson.dropbear.id.au>
+
+# DOC: Theory of Operation
+#
+# We want to run our tests in an isolated network environment so that:
+#  * Bugs can't affect things beyond the test
+#  * We have a known environment, independent of the host's network
+#    configuration
+#
+# To do this, we set up 3 namespaces with nstool:
+#
+# SANDBOX - Parent namespace to isolate us from the host
+#    This has no network, it's just for isolation, and to create the
+#    other namespaces.
+#
+# SIMHOST - Namespace representing the host for passt/pasta tests
+#    This is a network namespace representing the host where we run
+#    passt or pasta.  It has a simple network configuration with both
+#    IPv4 and IPv6 over a default route to ROUTER.
+#
+# ROUTER - Namespace representing SIMHOST's default router
+#    This is a network namespace representing SIMHOST's router, and in
+#    a sense, all hosts external to SIMHOST.  It acts as default
+#    gateway for SIMHOST on both IPv4 and IPv6.
+
+SANDBOX_CTL=sandbox.nstool
+SANDBOX_CTL_HOST=sandbox-host.nstool
+SANDBOX_CTL_ROUTER=sandbox-router.nstool
+
+SANDBOX_IP4_BCAST=192.0.2.255
+SANDBOX_IP4_MASK=24
+SANDBOX_IP4_PREFIX=192.0.2.0/$IP4_MASK
+SANDBOX_IP4_HOST=192.0.2.1
+SANDBOX_IP4_ROUTER=192.0.2.2
+
+SANDBOX_IP6_MASK=64
+SANDBOX_IP6_PREFIX=2001:db8:9a55::/$IP6_MASK
+SANDBOX_IP6_HOST=2001:db8:9a55::1
+SANDBOX_IP6_ROUTER=2001:db8:9a55::2
+
+sandbox_cleanup() {
+        ${NSTOOL} stop "${SANDBOX_CTL_HOST}"
+        ${NSTOOL} stop "${SANDBOX_CTL_ROUTER}"
+        ${NSTOOL} stop "${SANDBOX_CTL}"
+        rm -f "${SANDBOX_CTL}" "${SANDBOX_CTL_HOST}" "${SANDBOX_CTL_ROUTER}"
+}
+
+# Run command in SANDBOX with privilege
+sb_priv() {
+        ${NSTOOL} exec --keep-caps "${SANDBOX_CTL}" -- "$@"
+}
+
+# Run command in SIMHOST ns with privilege
+sb_priv_host() {
+        ${NSTOOL} exec --keep-caps "${SANDBOX_CTL_HOST}" -- "$@"
+}
+
+# Run command in SIMHOST without privilege
+sb_uhost() {
+        ${NSTOOL} exec "${SANDBOX_CTL_HOST}" -- "$@"
+}
+
+# Run command in ROUTER with privilege
+sb_priv_router() {
+        ${NSTOOL} exec --keep-caps "${SANDBOX_CTL_ROUTER}" -- "$@"
+}
+
+sandbox() {
+        trap "sandbox_cleanup" EXIT
+
+        # Create SANDBOX
+        unshare -Ucnpfm --mount-proc $NSTOOL hold "${SANDBOX_CTL}" &
+        $NSTOOL info -w "${SANDBOX_CTL}" >/dev/null
+
+        # Create SIMHOST
+        sb_priv unshare -n ${NSTOOL} hold "${SANDBOX_CTL_HOST}" &
+        local host_rel_pid=$(${NSTOOL} exec "${SANDBOX_CTL}" -- ${NSTOOL} info -pw "${SANDBOX_CTL_HOST}")
+
+        # Create ROUTER
+        sb_priv unshare -n ${NSTOOL} hold "${SANDBOX_CTL_ROUTER}" &
+        local router_rel_pid=$(${NSTOOL} exec "${SANDBOX_CTL}" -- ${NSTOOL} info -pw "${SANDBOX_CTL_ROUTER}")
+
+        # Create veth between SIMHOST and ROUTER
+        sb_priv ip link add type veth
+        sb_priv ip link set veth0 netns $host_rel_pid
+        sb_priv ip link set veth1 netns $router_rel_pid
+
+        # Configure network in ROUTER
+        sb_priv_router ip link set lo up
+        sb_priv_router ip -4 addr add $SANDBOX_IP4_ROUTER/$SANDBOX_IP4_MASK dev veth1
+        sb_priv_router ip -6 addr add $SANDBOX_IP6_ROUTER/$SANDBOX_IP6_MASK dev veth1
+        sb_priv_router ip link set veth1 up
+
+        # Configure network in SIMHOST
+        sb_priv_host ip link set lo up
+        sb_priv_host ip -4 addr add $SANDBOX_IP4_HOST/$SANDBOX_IP4_MASK dev veth0
+        sb_priv_host ip -6 addr add $SANDBOX_IP6_HOST/$SANDBOX_IP6_MASK dev veth0
+        sb_priv_host ip link set veth0 up
+
+        # Configure SIMHOST's default gateway as ROUTER
+        sleep 2 # Wait for SLAAC
+        local ip6_ll_router=$(sb_priv_router ip -6 -j addr | jq -r '.[] | select(.ifname == "veth1") | .addr_info | .[] | select(.scope == "link") | .local')
+
+        sb_priv_host ip -4 route add default via $SANDBOX_IP4_ROUTER
+        sb_priv_host ip -6 route add default via $ip6_ll_router dev veth0
+
+        sb_uhost "$@"
+}
diff --git a/test/lib/term b/test/lib/term
index 262937eb..9e2fbbd6 100755
--- a/test/lib/term
+++ b/test/lib/term
@@ -669,14 +669,14 @@ run_term() {
 
 	if [ ${CI} -eq 1 ]; then
 		printf '\e[8;50;240t'
-		asciinema rec --overwrite "${STATEBASE}/ci.uncut" -c "$TMUX /bin/sh -c './ci from_term'"
+		sandbox asciinema rec --overwrite "${STATEBASE}/ci.uncut" -c "$TMUX /bin/sh -c './ci from_term'"
 		video_postprocess "${STATEBASE}/ci.uncut"
 	elif [ ${DEMO} -eq 1 ]; then
 		printf '\e[8;40;130t'
-		asciinema rec --overwrite "${STATEBASE}/demo.uncut" -c "$TMUX /bin/sh -c './run_demo from_term'"
+		sandbox asciinema rec --overwrite "${STATEBASE}/demo.uncut" -c "$TMUX /bin/sh -c './run_demo from_term'"
 		video_postprocess "${STATEBASE}/demo.uncut"
 	else
-		$TMUX /bin/sh -c "./run from_term ${*}"
+		sandbox $TMUX /bin/sh -c "./run from_term ${*}"
 	fi
 }
 
diff --git a/test/run b/test/run
index 3b376639..a278e23a 100755
--- a/test/run
+++ b/test/run
@@ -44,6 +44,7 @@ COMMIT="$(git log --oneline --no-decorate -1)"
 . lib/context
 . lib/setup
 . lib/setup_ugly
+. lib/sandbox
 . lib/term
 . lib/perf_report
 . lib/layout
@@ -82,9 +83,9 @@ run() {
 	test pasta_options/log_to_file
 	teardown pasta_options
 
-	setup build
-	test pasta_podman/bats
-	teardown build
+	#setup build
+	#test pasta_podman/bats
+	#teardown build
 
 	setup memory
 	test memory/passt
-- 
@@ -44,6 +44,7 @@ COMMIT="$(git log --oneline --no-decorate -1)"
 . lib/context
 . lib/setup
 . lib/setup_ugly
+. lib/sandbox
 . lib/term
 . lib/perf_report
 . lib/layout
@@ -82,9 +83,9 @@ run() {
 	test pasta_options/log_to_file
 	teardown pasta_options
 
-	setup build
-	test pasta_podman/bats
-	teardown build
+	#setup build
+	#test pasta_podman/bats
+	#teardown build
 
 	setup memory
 	test memory/passt
-- 
2.44.0