run_all.py (2138B)
1 #!/usr/bin/python3 2 3 import os 4 import sys 5 if sys.version_info[0] < 3: raise Exception("Python 3 or a more recent version is required.") 6 import json 7 import subprocess 8 9 average_num_times = 3 10 max_accum_time = 60 # don't take too long on a test - stop averaging if time exceeds some amount of seconds 11 12 with open('tests.json') as data_file: 13 data = json.load(data_file) 14 15 def runBench(prog): 16 result = subprocess.Popen(prog.split(), stdout = subprocess.PIPE).communicate()[0] 17 result = result.splitlines() 18 for line in result: 19 line = line.decode("utf-8") 20 if line.startswith("Time running "): 21 return str(line.rsplit(' ', 1)[-1]) 22 return "" 23 24 call = 'python ./bench.py' 25 the_os = 'linux' 26 if os.name == "nt": 27 call = 'python bench.py' 28 the_os = 'windows' 29 30 f = open('results.txt', 'w') 31 for test in ['header', 'asserts', 'runtime']: 32 print( '\n************** ' + test + '\n') 33 f.write('\n************** ' + test + '\n') 34 f.flush() 35 for framework in ['doctest', 'catch']: 36 print( '== ' + framework + '\n') 37 f.write('== ' + framework + '\n') 38 f.flush() 39 for config in data['compilers'][the_os]: 40 for curr in data[test][1]: 41 if curr[0] == framework or curr[0] == "any": 42 command = call + data[test][0] + config + curr[1] + (' --catch' if framework == 'catch' else '') 43 print(command) 44 45 accum = float(0) 46 num_times = 0 47 for i in range(0, average_num_times): 48 res = float(runBench(command)) 49 print(res) 50 accum += res 51 num_times += 1 52 53 if accum > max_accum_time: 54 break 55 56 average = "{:7.2f}".format(round(accum / num_times, 2)) 57 print("AVERAGE: " + average) 58 f.write(average + " | ") 59 f.flush() 60 f.write("\n") 61 f.flush() 62 63 f.close() 64 65