testenv.py (11361B)
1 # TestEnv class to manage test environment variables. 2 # 3 # Copyright (c) 2020-2021 Virtuozzo International GmbH 4 # 5 # This program is free software; you can redistribute it and/or modify 6 # it under the terms of the GNU General Public License as published by 7 # the Free Software Foundation; either version 2 of the License, or 8 # (at your option) any later version. 9 # 10 # This program is distributed in the hope that it will be useful, 11 # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 # GNU General Public License for more details. 14 # 15 # You should have received a copy of the GNU General Public License 16 # along with this program. If not, see <http://www.gnu.org/licenses/>. 17 # 18 19 import os 20 import sys 21 import tempfile 22 from pathlib import Path 23 import shutil 24 import collections 25 import random 26 import subprocess 27 import glob 28 from typing import List, Dict, Any, Optional, ContextManager 29 30 DEF_GDB_OPTIONS = 'localhost:12345' 31 32 def isxfile(path: str) -> bool: 33 return os.path.isfile(path) and os.access(path, os.X_OK) 34 35 36 def get_default_machine(qemu_prog: str) -> str: 37 outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True, 38 universal_newlines=True, 39 stdout=subprocess.PIPE).stdout 40 41 machines = outp.split('\n') 42 try: 43 default_machine = next(m for m in machines if m.endswith(' (default)')) 44 except StopIteration: 45 return '' 46 default_machine = default_machine.split(' ', 1)[0] 47 48 alias_suf = ' (alias of {})'.format(default_machine) 49 alias = next((m for m in machines if m.endswith(alias_suf)), None) 50 if alias is not None: 51 default_machine = alias.split(' ', 1)[0] 52 53 return default_machine 54 55 56 class TestEnv(ContextManager['TestEnv']): 57 """ 58 Manage system environment for running tests 59 60 The following variables are supported/provided. They are represented by 61 lower-cased TestEnv attributes. 62 """ 63 64 # We store environment variables as instance attributes, and there are a 65 # lot of them. Silence pylint: 66 # pylint: disable=too-many-instance-attributes 67 68 env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR', 69 'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG', 70 'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG', 71 'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS', 72 'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT', 73 'QEMU_NBD_OPTIONS', 'IMGOPTS', 'IMGFMT', 'IMGPROTO', 74 'AIOMODE', 'CACHEMODE', 'VALGRIND_QEMU', 75 'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX', 76 'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_', 77 'GDB_OPTIONS', 'PRINT_QEMU'] 78 79 def prepare_subprocess(self, args: List[str]) -> Dict[str, str]: 80 if self.debug: 81 args.append('-d') 82 83 with open(args[0], encoding="utf-8") as f: 84 try: 85 if f.readline().rstrip() == '#!/usr/bin/env python3': 86 args.insert(0, self.python) 87 except UnicodeDecodeError: # binary test? for future. 88 pass 89 90 os_env = os.environ.copy() 91 os_env.update(self.get_env()) 92 return os_env 93 94 def get_env(self) -> Dict[str, str]: 95 env = {} 96 for v in self.env_variables: 97 val = getattr(self, v.lower(), None) 98 if val is not None: 99 env[v] = val 100 101 return env 102 103 def init_directories(self) -> None: 104 """Init directory variables: 105 PYTHONPATH 106 TEST_DIR 107 SOCK_DIR 108 SAMPLE_IMG_DIR 109 """ 110 111 # Path where qemu goodies live in this source tree. 112 qemu_srctree_path = Path(__file__, '../../../python').resolve() 113 114 self.pythonpath = os.pathsep.join(filter(None, ( 115 self.source_iotests, 116 str(qemu_srctree_path), 117 os.getenv('PYTHONPATH'), 118 ))) 119 120 self.test_dir = os.getenv('TEST_DIR', 121 os.path.join(os.getcwd(), 'scratch')) 122 Path(self.test_dir).mkdir(parents=True, exist_ok=True) 123 124 try: 125 self.sock_dir = os.environ['SOCK_DIR'] 126 self.tmp_sock_dir = False 127 Path(self.sock_dir).mkdir(parents=True, exist_ok=True) 128 except KeyError: 129 self.sock_dir = tempfile.mkdtemp() 130 self.tmp_sock_dir = True 131 132 self.sample_img_dir = os.getenv('SAMPLE_IMG_DIR', 133 os.path.join(self.source_iotests, 134 'sample_images')) 135 136 def init_binaries(self) -> None: 137 """Init binary path variables: 138 PYTHON (for bash tests) 139 QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG 140 """ 141 self.python = sys.executable 142 143 def root(*names: str) -> str: 144 return os.path.join(self.build_root, *names) 145 146 arch = os.uname().machine 147 if 'ppc64' in arch: 148 arch = 'ppc64' 149 150 self.qemu_prog = os.getenv('QEMU_PROG', root(f'qemu-system-{arch}')) 151 if not os.path.exists(self.qemu_prog): 152 pattern = root('qemu-system-*') 153 try: 154 progs = sorted(glob.iglob(pattern)) 155 self.qemu_prog = next(p for p in progs if isxfile(p)) 156 except StopIteration: 157 sys.exit("Not found any Qemu executable binary by pattern " 158 f"'{pattern}'") 159 160 self.qemu_img_prog = os.getenv('QEMU_IMG_PROG', root('qemu-img')) 161 self.qemu_io_prog = os.getenv('QEMU_IO_PROG', root('qemu-io')) 162 self.qemu_nbd_prog = os.getenv('QEMU_NBD_PROG', root('qemu-nbd')) 163 self.qsd_prog = os.getenv('QSD_PROG', root('storage-daemon', 164 'qemu-storage-daemon')) 165 166 for b in [self.qemu_img_prog, self.qemu_io_prog, self.qemu_nbd_prog, 167 self.qemu_prog, self.qsd_prog]: 168 if not os.path.exists(b): 169 sys.exit('No such file: ' + b) 170 if not isxfile(b): 171 sys.exit('Not executable: ' + b) 172 173 def __init__(self, imgfmt: str, imgproto: str, aiomode: str, 174 cachemode: Optional[str] = None, 175 imgopts: Optional[str] = None, 176 misalign: bool = False, 177 debug: bool = False, 178 valgrind: bool = False, 179 gdb: bool = False, 180 qprint: bool = False) -> None: 181 self.imgfmt = imgfmt 182 self.imgproto = imgproto 183 self.aiomode = aiomode 184 self.imgopts = imgopts 185 self.misalign = misalign 186 self.debug = debug 187 188 if qprint: 189 self.print_qemu = 'y' 190 191 if gdb: 192 self.gdb_options = os.getenv('GDB_OPTIONS', DEF_GDB_OPTIONS) 193 if not self.gdb_options: 194 # cover the case 'export GDB_OPTIONS=' 195 self.gdb_options = DEF_GDB_OPTIONS 196 elif 'GDB_OPTIONS' in os.environ: 197 # to not propagate it in prepare_subprocess() 198 del os.environ['GDB_OPTIONS'] 199 200 if valgrind: 201 self.valgrind_qemu = 'y' 202 203 if cachemode is None: 204 self.cachemode_is_default = 'true' 205 self.cachemode = 'writeback' 206 else: 207 self.cachemode_is_default = 'false' 208 self.cachemode = cachemode 209 210 # Initialize generic paths: build_root, build_iotests, source_iotests, 211 # which are needed to initialize some environment variables. They are 212 # used by init_*() functions as well. 213 214 if os.path.islink(sys.argv[0]): 215 # called from the build tree 216 self.source_iotests = os.path.dirname(os.readlink(sys.argv[0])) 217 self.build_iotests = os.path.dirname(os.path.abspath(sys.argv[0])) 218 else: 219 # called from the source tree 220 self.source_iotests = os.getcwd() 221 self.build_iotests = self.source_iotests 222 223 self.build_root = os.path.join(self.build_iotests, '..', '..') 224 225 self.init_directories() 226 self.init_binaries() 227 228 self.malloc_perturb_ = os.getenv('MALLOC_PERTURB_', 229 str(random.randrange(1, 255))) 230 231 # QEMU_OPTIONS 232 self.qemu_options = '-nodefaults -display none -accel qtest' 233 machine_map = ( 234 ('arm', 'virt'), 235 ('aarch64', 'virt'), 236 ('avr', 'mega2560'), 237 ('m68k', 'virt'), 238 ('riscv32', 'virt'), 239 ('riscv64', 'virt'), 240 ('rx', 'gdbsim-r5f562n8'), 241 ('tricore', 'tricore_testboard') 242 ) 243 for suffix, machine in machine_map: 244 if self.qemu_prog.endswith(f'qemu-system-{suffix}'): 245 self.qemu_options += f' -machine {machine}' 246 247 # QEMU_DEFAULT_MACHINE 248 self.qemu_default_machine = get_default_machine(self.qemu_prog) 249 250 self.qemu_img_options = os.getenv('QEMU_IMG_OPTIONS') 251 self.qemu_nbd_options = os.getenv('QEMU_NBD_OPTIONS') 252 253 is_generic = self.imgfmt not in ['bochs', 'cloop', 'dmg'] 254 self.imgfmt_generic = 'true' if is_generic else 'false' 255 256 self.qemu_io_options = f'--cache {self.cachemode} --aio {self.aiomode}' 257 if self.misalign: 258 self.qemu_io_options += ' --misalign' 259 260 self.qemu_io_options_no_fmt = self.qemu_io_options 261 262 if self.imgfmt == 'luks': 263 self.imgoptssyntax = 'true' 264 self.imgkeysecret = '123456' 265 if not self.imgopts: 266 self.imgopts = 'iter-time=10' 267 elif 'iter-time=' not in self.imgopts: 268 self.imgopts += ',iter-time=10' 269 else: 270 self.imgoptssyntax = 'false' 271 self.qemu_io_options += ' -f ' + self.imgfmt 272 273 if self.imgfmt == 'vmdk': 274 if not self.imgopts: 275 self.imgopts = 'zeroed_grain=on' 276 elif 'zeroed_grain=' not in self.imgopts: 277 self.imgopts += ',zeroed_grain=on' 278 279 def close(self) -> None: 280 if self.tmp_sock_dir: 281 shutil.rmtree(self.sock_dir) 282 283 def __enter__(self) -> 'TestEnv': 284 return self 285 286 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 287 self.close() 288 289 def print_env(self, prefix: str = '') -> None: 290 template = """\ 291 {prefix}QEMU -- "{QEMU_PROG}" {QEMU_OPTIONS} 292 {prefix}QEMU_IMG -- "{QEMU_IMG_PROG}" {QEMU_IMG_OPTIONS} 293 {prefix}QEMU_IO -- "{QEMU_IO_PROG}" {QEMU_IO_OPTIONS} 294 {prefix}QEMU_NBD -- "{QEMU_NBD_PROG}" {QEMU_NBD_OPTIONS} 295 {prefix}IMGFMT -- {IMGFMT}{imgopts} 296 {prefix}IMGPROTO -- {IMGPROTO} 297 {prefix}PLATFORM -- {platform} 298 {prefix}TEST_DIR -- {TEST_DIR} 299 {prefix}SOCK_DIR -- {SOCK_DIR} 300 {prefix}GDB_OPTIONS -- {GDB_OPTIONS} 301 {prefix}VALGRIND_QEMU -- {VALGRIND_QEMU} 302 {prefix}PRINT_QEMU_OUTPUT -- {PRINT_QEMU} 303 {prefix}""" 304 305 args = collections.defaultdict(str, self.get_env()) 306 307 if 'IMGOPTS' in args: 308 args['imgopts'] = f" ({args['IMGOPTS']})" 309 310 u = os.uname() 311 args['platform'] = f'{u.sysname}/{u.machine} {u.nodename} {u.release}' 312 args['prefix'] = prefix 313 print(template.format_map(args))