[RFC PATCH 7/9] iotests/testenv: initialize an iotests venv

[John Snow]

Create a venv automatically when running iotests and install the QEMU
namespace package to it, which will also pull in qemu.qmp from PyPI.

The venv created will always use the same python binary as the one used
to launch the check process. Assuming that care is taken to run 'check'
with the python executable specified by the QEMU configure script, this
venv will use that same python binary.

When re-running the script with a different python binary, the virtual
environment will be re-created to match.

Packages that need to be installed for successful runs of iotests are
specified in requirements.txt. The "magic" here is that we can specify
the in-tree repository for the 'qemu' package, whose own requirements
(in python/setup.cfg) specify the now out-of-tree 'qemu.qmp' package.

As a bonus, iotest 297 is now easier to run than before.

Signed-off-by: John Snow <jsnow@redhat.com>
---
 tests/qemu-iotests/requirements.txt |  3 ++
 tests/qemu-iotests/testenv.py       | 74 ++++++++++++++++++++++++++---
 2 files changed, 70 insertions(+), 7 deletions(-)
 create mode 100644 tests/qemu-iotests/requirements.txt

diff --git a/tests/qemu-iotests/requirements.txt 
b/tests/qemu-iotests/requirements.txt
new file mode 100644
index 0000000000..923e6b26a3
--- /dev/null
+++ b/tests/qemu-iotests/requirements.txt
@@ -0,0 +1,3 @@
+../../python
+mypy >= 0.770
+pylint >= 2.8.0
diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
index c33454fa68..f258aceac0 100644
--- a/tests/qemu-iotests/testenv.py
+++ b/tests/qemu-iotests/testenv.py
@@ -26,6 +26,7 @@
 import subprocess
 import glob
 from typing import List, Dict, Any, Optional, ContextManager
+import venv
 
 DEF_GDB_OPTIONS = 'localhost:12345'
 
@@ -65,8 +66,9 @@ class TestEnv(ContextManager['TestEnv']):
     # lot of them. Silence pylint:
     # pylint: disable=too-many-instance-attributes
 
-    env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR',
-                     'OUTPUT_DIR', 'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG',
+    env_variables = ['PYTHONPATH', 'VIRTUAL_ENV', 'PYTHON',
+                     'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR',
+                     'OUTPUT_DIR', 'QEMU_PROG', 'QEMU_IMG_PROG',
                      'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG',
                      'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS',
                      'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT',
@@ -98,8 +100,68 @@ def get_env(self) -> Dict[str, str]:
             if val is not None:
                 env[v] = val
 
+        env['PATH'] = os.pathsep.join((
+            os.path.join(self.virtual_env, 'bin'),
+            os.environ['PATH']
+        ))
         return env
 
+    @staticmethod
+    def _make_venv(venv_dir: Path) -> None:
+        reqfile = Path(__file__, '../requirements.txt').resolve()
+
+        print("Making an iotest venv:")
+        print(f"  python: {sys.executable}")
+        print(f"  venv_dir: {venv_dir!s}")
+        print(f"  requirements: {reqfile!s}")
+        print("")
+
+        # same codepath as 'python3 -m venv iovenv'
+        venv.create(
+            venv_dir,
+            clear=True,
+            symlinks=True,
+            with_pip=True,
+        )
+
+        # enter the venv
+        env = os.environ.copy()
+        env['VIRTUAL_ENV'] = str(venv_dir)
+        env['PATH'] = os.pathsep.join((
+            str(venv_dir.joinpath('bin')),
+            os.environ['PATH']
+        ))
+
+        # install the qemu packages to the venv.
+        subprocess.run(
+            ['pip3', 'install', '-r', str(reqfile)],
+            env=env,
+            check=True,
+        )
+
+    def init_venv(self) -> None:
+        venv_dir = Path('iovenv').resolve()
+        py3_bin = venv_dir.joinpath('bin', 'python3')
+
+        self.virtual_env = str(venv_dir)
+
+        if py3_bin.exists():
+            # resolve symlinks on both python executables
+            target = py3_bin.resolve()
+            system = Path(sys.executable).resolve()
+            if target == system:
+                # venv executable matches ours, so presumably nothing to do.
+                return
+
+        try:
+            self._make_venv(venv_dir)
+        except:
+            try:
+                shutil.rmtree(venv_dir)
+            except FileNotFoundError:
+                pass
+            raise
+
     def init_directories(self) -> None:
         """Init directory variables:
              PYTHONPATH
@@ -109,12 +171,8 @@ def init_directories(self) -> None:
              OUTPUT_DIR
         """
 
-        # Path where qemu goodies live in this source tree.
-        qemu_srctree_path = Path(__file__, '../../../python').resolve()
-
         self.pythonpath = os.pathsep.join(filter(None, (
             self.source_iotests,
-            str(qemu_srctree_path),
             os.getenv('PYTHONPATH'),
         )))
 
@@ -141,7 +199,7 @@ def init_binaries(self) -> None:
              PYTHON (for bash tests)
              QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
         """
-        self.python = sys.executable
+        self.python = os.path.join(self.virtual_env, 'bin', 'python3')
 
         def root(*names: str) -> str:
             return os.path.join(self.build_root, *names)
@@ -225,6 +283,7 @@ def __init__(self, imgfmt: str, imgproto: str, aiomode: str,
 
         self.build_root = os.path.join(self.build_iotests, '..', '..')
 
+        self.init_venv()
         self.init_directories()
         self.init_binaries()
 
@@ -301,6 +360,7 @@ def print_env(self) -> None:
 GDB_OPTIONS   -- {GDB_OPTIONS}
 VALGRIND_QEMU -- {VALGRIND_QEMU}
 PRINT_QEMU_OUTPUT -- {PRINT_QEMU}
+VIRTUAL_ENV   -- {VIRTUAL_ENV}
 """
 
         args = collections.defaultdict(str, self.get_env())
-- 
2.31.1