bench.py (9007B)
1 #!/usr/bin/python3 2 3 import os 4 import sys 5 if sys.version_info[0] < 3: raise Exception("Python 3 or a more recent version is required.") 6 import pprint 7 import argparse 8 import urllib.request 9 from datetime import datetime 10 import shutil 11 from time import sleep 12 13 # ============================================================================== 14 # == ARGUMENTS ================================================================= 15 # ============================================================================== 16 17 def addCommonFlags(parser): 18 parser.add_argument("compiler", choices=['msvc', 'gcc', 'clang'], default='msvc', help = "compiler to use") 19 parser.add_argument("--debug", action = "store_true", help = "build in debug") 20 parser.add_argument("--catch", action = "store_true", help = "use Catch instead of doctest") 21 parser.add_argument("--disabled", action = "store_true", help = "DOCTEST_CONFIG_DISABLE / CATCH_CONFIG_DISABLE") 22 parser.add_argument("--fast", action = "store_true", help = "define the doctest/Catch fast config identifier") 23 parser.add_argument("--files", type=int, default=1, help = "number of source files (besides the implementation)") 24 parser.add_argument("--tests", type=int, default=1, help = "number of test cases per source file") 25 parser.add_argument("--checks", type=int, default=1, help = "number of asserts per test case") 26 parser.add_argument("--asserts", choices=['normal', 'binary'], default="normal", 27 help = "<doctest> type of assert used - Catch: only normal") 28 29 parser = argparse.ArgumentParser() 30 subparsers = parser.add_subparsers() 31 parser_c = subparsers.add_parser('compile', help='benchmark compile times') 32 addCommonFlags(parser_c) 33 parser_c.add_argument("--implement", action = "store_true", help = "implement the framework test runner") 34 parser_c.add_argument("--header", action = "store_true", help = "include the framework header everywhere") 35 parser_r = subparsers.add_parser('runtime', help='benchmark runtime') 36 addCommonFlags(parser_r) 37 parser_r.add_argument("--loop-iters", type=int, default=1000, help = "loop N times all asserts in each test case") 38 parser_r.add_argument("--info", action = "store_true", help = "log the loop variable with INFO()") 39 40 def compile(args): args.compile = True; args.runtime = False 41 def runtime(args): args.compile = False; args.runtime = True 42 parser_c.set_defaults(func=compile) 43 parser_r.set_defaults(func=runtime) 44 args = parser.parse_args() 45 args.func(args) 46 47 print("== PASSED OPTIONS TO BENCHMARK SCRIPT:") 48 pprint.pprint(vars(args), width = 1) 49 50 # ============================================================================== 51 # == SETUP ENVIRONMENT ========================================================= 52 # ============================================================================== 53 54 # catch version 55 catch_ver = "2.3.0" 56 catch_header = "catch." + catch_ver + ".hpp" 57 58 # get the catch header 59 if not os.path.exists("catch." + catch_ver + ".hpp"): 60 urllib.request.urlretrieve("https://github.com/catchorg/Catch2/releases/download/v" + catch_ver + "/catch.hpp", catch_header) 61 62 # folder with generated code 63 the_folder = 'project' 64 65 # delete the folder 66 if os.path.exists(the_folder): 67 shutil.rmtree(the_folder) 68 69 # wait a bit or the script might fail... 70 sleep(2) 71 72 # create the folder 73 if not os.path.exists(the_folder): 74 os.makedirs(the_folder) 75 76 # enter folder 77 os.chdir(the_folder); 78 79 # ============================================================================== 80 # == DO STUFF ================================================================== 81 # ============================================================================== 82 83 # setup defines used 84 defines = "" 85 if args.catch and args.disabled: 86 defines += "#define CATCH_CONFIG_DISABLE\n" 87 if not args.catch and args.disabled: 88 defines += "#define DOCTEST_CONFIG_DISABLE\n" 89 if args.catch and args.fast: 90 defines += "#define CATCH_CONFIG_FAST_COMPILE\n" 91 if not args.catch and args.fast: 92 defines += "#define DOCTEST_CONFIG_SUPER_FAST_ASSERTS\n" 93 94 define_implement = "#define DOCTEST_CONFIG_IMPLEMENT\n" 95 if args.catch: 96 define_implement = "#define CATCH_CONFIG_RUNNER\n" 97 98 # setup the macros used 99 macro = " CHECK(a == b);\n" 100 if args.runtime: 101 macro = " CHECK(i == i);\n" 102 if not args.catch and args.asserts == "binary": 103 macro = " CHECK_EQ(a, b);\n" 104 105 # setup the header used 106 include = '#include "doctest.h"\n' 107 if args.catch: 108 include = '#include "' + catch_header + '"\n' 109 110 # ============================================================================== 111 # == GENERATE SOURCE CODE ====================================================== 112 # ============================================================================== 113 114 # make the source files 115 for i in range(0, args.files): 116 f = open(str(i) + '.cpp', 'w') 117 if args.runtime or args.header: 118 f.write(defines) 119 f.write(include) 120 for t in range(0, args.tests): 121 f.write('TEST_CASE("") {\n') 122 f.write(' int a = 5;\n') 123 f.write(' int b = 5;\n') 124 if args.runtime and args.loop_iters > 0: 125 f.write(' for(int i = 0; i < ' + str(args.loop_iters) + '; ++i) {\n') 126 if args.runtime and args.info: 127 f.write(' INFO(i);\n') 128 for a in range(0, args.checks): 129 if args.runtime and args.loop_iters > 0: 130 f.write(' ') 131 f.write(macro) 132 if args.runtime and args.loop_iters > 0: 133 f.write(' }\n') 134 f.write('}\n\n') 135 f.write('int f' + str(i) + '() { return ' + str(i) + '; }\n\n') 136 f.close() 137 138 # the main file 139 f = open('main.cpp', 'w') 140 if args.runtime or args.implement or args.header: 141 f.write(defines) 142 f.write(define_implement) 143 f.write(include) 144 f.write('int main(int argc, char** argv) {\n') 145 if args.runtime or args.implement or args.header: 146 if not args.catch: f.write(' int res = doctest::Context(argc, argv).run();\n') 147 else: f.write(' int res = Catch::Session().run(argc, argv);\n') 148 else: 149 f.write(' int res = 0;\n') 150 for i in range(0, args.files): 151 f.write(' int f' + str(i) + '(); res += f' + str(i) + '();\n') 152 f.write(' return res;\n}\n') 153 f.close() 154 155 # the cmake file 156 f = open('CMakeLists.txt', 'w') 157 f.write('cmake_minimum_required(VERSION 2.8)\n\n') 158 f.write('project(bench)\n\n') 159 f.write('if(NOT MSVC)\n') 160 f.write('set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")\n') 161 f.write('endif()\n\n') 162 if not args.catch: f.write('include_directories("../../../doctest/")\n\n') 163 else: f.write('include_directories("../")\n\n') 164 f.write('add_executable(bench main.cpp\n') 165 for i in range(0, args.files): 166 f.write(' ' + str(i) + '.cpp\n') 167 f.write(')\n') 168 f.close() 169 170 # ============================================================================== 171 # == INVOKE CMAKE ============================================================== 172 # ============================================================================== 173 174 compiler = "" 175 if args.compiler == 'clang': 176 compiler = " -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=-w" 177 if args.compiler == 'gcc': 178 compiler = " -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CXX_FLAGS=-w" 179 180 # setup cmake command 181 cmake_command = 'cmake . -G "Visual Studio 15 Win64"' # MSVC 2017 182 if args.compiler != 'msvc': 183 cmake_command = 'cmake . -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=' + ('Debug' if args.debug else 'Release') 184 if os.name != "nt": 185 cmake_command = 'cmake . -DCMAKE_BUILD_TYPE=' + ('Debug' if args.debug else 'Release') 186 187 os.system(cmake_command + compiler) 188 189 # ============================================================================== 190 # == BUILD PROJECT ============================================================= 191 # ============================================================================== 192 193 the_config = '' 194 if args.compiler == 'msvc': 195 if args.debug: the_config = ' --config Debug' 196 else: the_config = ' --config Release' 197 198 # build it 199 start = datetime.now() 200 os.system('cmake --build .' + the_config) 201 end = datetime.now() 202 203 if not args.runtime: 204 print("Time running compiler (+ linker) in seconds: " + str((end - start).total_seconds())) 205 206 # ============================================================================== 207 # == RUN PROJECT =============================================================== 208 # ============================================================================== 209 210 if args.runtime: 211 start = datetime.now() 212 if args.compiler == 'msvc': 213 os.system(('Debug' if args.debug else 'Release') + '\\bench.exe') 214 elif os.name == "nt": 215 os.system('bench.exe') 216 else: 217 os.system('./bench') 218 end = datetime.now() 219 220 print("Time running the tests in seconds: " + str((end - start).total_seconds())) 221 222 # leave folder 223 os.chdir("../"); 224 225 226 227 228 229 230 231 232 233 234 235 236 237