[PATCH 4/4] test: Add exeter+Avocado based build tests

[David Gibson <david@gibson.dropbear.id.au>]

Add a new test script to run the equivalent of the tests in build/all
using exeter and Avocado.  This new version of the tests is more robust
than the original, since it makes a temporary copy of the source tree so
will not be affected by concurrent manual builds.

Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
---
 test/Makefile              | 11 +++--
 test/build/.gitignore      |  1 +
 test/build/build.exeter.in | 89 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+), 3 deletions(-)
 create mode 100644 test/build/build.exeter.in

diff --git a/test/Makefile b/test/Makefile
index afc1334d..6c8a2820 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -63,13 +63,13 @@ LOCAL_ASSETS = mbuto.img mbuto.mem.img podman/bin/podman QEMU_EFI.fd \
 
 ASSETS = $(DOWNLOAD_ASSETS) $(LOCAL_ASSETS)
 
-EXETER_JOBS = build/static_checkers.json
+EXETER_JOBS = build/static_checkers.json build/build.json
 
 AVOCADO_JOBS = $(EXETER_JOBS) avocado/static_checkers.json
 
 SYSTEM_PYTHON = python3
 VENV = venv
-PYTHON = $(VENV)/bin/python3
+VENV_PYTHON = $(shell pwd)/$(VENV)/bin/python3
 PIP = $(VENV)/bin/pip3
 AVOCADO = $(VENV)/bin/avocado
 
@@ -130,13 +130,18 @@ big.bin:
 	dd if=/dev/urandom bs=1M count=10 of=$@
 
 .PHONY: venv
-venv:
+venv: pull-exeter
 	$(SYSTEM_PYTHON) -m venv $(VENV)
 	$(PIP) install avocado-framework
+	$(PIP) install exeter/
 
 %.json: %.exeter pull-exeter
 	cd ..; test/$< --avocado > test/$@
 
+%.exeter: %.exeter.in
+	sed 's!@PYTHON3@!$(VENV_PYTHON)!' < $< > $@
+	chmod +x $@
+
 .PHONY: avocado
 avocado: venv $(AVOCADO_JOBS)
 	cd .. && test/$(AVOCADO) run $(AVOCADO_JOBS:%=test/%)
diff --git a/test/build/.gitignore b/test/build/.gitignore
index a6c57f5f..4ef40dd0 100644
--- a/test/build/.gitignore
+++ b/test/build/.gitignore
@@ -1 +1,2 @@
 *.json
+build.exeter
diff --git a/test/build/build.exeter.in b/test/build/build.exeter.in
new file mode 100644
index 00000000..9b7da359
--- /dev/null
+++ b/test/build/build.exeter.in
@@ -0,0 +1,89 @@
+#! @PYTHON3@
+
+import contextlib
+import os.path
+import shutil
+import subprocess
+import tempfile
+
+import exeter
+
+def host_run(cmd, **kwargs):
+    return subprocess.run(cmd, shell=True, check=True, encoding='UTF-8', **kwargs)
+
+
+def host_out(cmd):
+    return host_run(cmd, capture_output=True).stdout
+
+
+@contextlib.contextmanager
+def clone_source_tree():
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=False) as tmpdir:
+        # Make a temporary copy of the sources
+        srcfiles = host_out('git ls-files').splitlines()
+        for src in srcfiles:
+            dst = os.path.join(tmpdir, src)
+            os.makedirs(os.path.dirname(dst), exist_ok=True)
+            shutil.copy(src, dst)
+        os.chdir(tmpdir)
+        yield tmpdir
+
+
+def build_target(target, outputs):
+    with clone_source_tree():
+        for o in outputs:
+            assert not os.path.exists(o)
+        host_run(f'make {target} CFLAGS="-Werror"')
+        for o in outputs:
+            assert os.path.exists(o)
+        host_run('make clean')
+        for o in outputs:
+            assert not os.path.exists(o)
+
+
+@exeter.test
+def test_make_passt():
+    build_target('passt', ['passt'])
+
+
+@exeter.test
+def test_make_pasta():
+    build_target('pasta', ['pasta'])
+
+
+@exeter.test
+def test_make_qrap():
+    build_target('qrap', ['qrap'])
+
+
+@exeter.test
+def test_make_all():
+    build_target('all', ['passt', 'pasta', 'qrap'])
+
+
+@exeter.test
+def test_make_install_uninstall():
+    with clone_source_tree():
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=False) as prefix:
+            bindir = os.path.join(prefix, 'bin')
+            mandir = os.path.join(prefix, 'share', 'man')
+            exes = ['passt', 'pasta', 'qrap']
+
+            # Install
+            host_run(f'make install CFLAGS="-Werror" prefix={prefix}')
+
+            for t in exes:
+                assert os.path.isfile(os.path.join(bindir, t))
+                host_run(f'man -M {mandir} -W passt')
+
+            # Uninstall
+            host_run(f'make uninstall prefix={prefix}')
+
+            for t in exes:
+                assert not os.path.exists(os.path.join(bindir, t))
+                exeter.assert_raises(subprocess.CalledProcessError,
+                                     host_run, f'man -M {mandir} -W passt')
+
+
+if __name__ == '__main__':
+    exeter.main()
-- 
@@ -0,0 +1,89 @@
+#! @PYTHON3@
+
+import contextlib
+import os.path
+import shutil
+import subprocess
+import tempfile
+
+import exeter
+
+def host_run(cmd, **kwargs):
+    return subprocess.run(cmd, shell=True, check=True, encoding='UTF-8', **kwargs)
+
+
+def host_out(cmd):
+    return host_run(cmd, capture_output=True).stdout
+
+
+@contextlib.contextmanager
+def clone_source_tree():
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=False) as tmpdir:
+        # Make a temporary copy of the sources
+        srcfiles = host_out('git ls-files').splitlines()
+        for src in srcfiles:
+            dst = os.path.join(tmpdir, src)
+            os.makedirs(os.path.dirname(dst), exist_ok=True)
+            shutil.copy(src, dst)
+        os.chdir(tmpdir)
+        yield tmpdir
+
+
+def build_target(target, outputs):
+    with clone_source_tree():
+        for o in outputs:
+            assert not os.path.exists(o)
+        host_run(f'make {target} CFLAGS="-Werror"')
+        for o in outputs:
+            assert os.path.exists(o)
+        host_run('make clean')
+        for o in outputs:
+            assert not os.path.exists(o)
+
+
+@exeter.test
+def test_make_passt():
+    build_target('passt', ['passt'])
+
+
+@exeter.test
+def test_make_pasta():
+    build_target('pasta', ['pasta'])
+
+
+@exeter.test
+def test_make_qrap():
+    build_target('qrap', ['qrap'])
+
+
+@exeter.test
+def test_make_all():
+    build_target('all', ['passt', 'pasta', 'qrap'])
+
+
+@exeter.test
+def test_make_install_uninstall():
+    with clone_source_tree():
+        with tempfile.TemporaryDirectory(ignore_cleanup_errors=False) as prefix:
+            bindir = os.path.join(prefix, 'bin')
+            mandir = os.path.join(prefix, 'share', 'man')
+            exes = ['passt', 'pasta', 'qrap']
+
+            # Install
+            host_run(f'make install CFLAGS="-Werror" prefix={prefix}')
+
+            for t in exes:
+                assert os.path.isfile(os.path.join(bindir, t))
+                host_run(f'man -M {mandir} -W passt')
+
+            # Uninstall
+            host_run(f'make uninstall prefix={prefix}')
+
+            for t in exes:
+                assert not os.path.exists(os.path.join(bindir, t))
+                exeter.assert_raises(subprocess.CalledProcessError,
+                                     host_run, f'man -M {mandir} -W passt')
+
+
+if __name__ == '__main__':
+    exeter.main()
-- 
2.45.2