range_based_execution.py (1391B)
1 #!/usr/bin/python 2 3 import sys 4 import math 5 import multiprocessing 6 import subprocess 7 8 if len(sys.argv) < 2: 9 print("supply the path to the doctest executable as the first argument!") 10 sys.exit(1) 11 12 # get the number of tests in the doctest executable 13 num_tests = 0 14 15 program_with_args = [sys.argv[1], "--dt-count=1"] 16 for i in range(2, len(sys.argv)): 17 program_with_args.append(sys.argv[i]) 18 19 result = subprocess.Popen(program_with_args, stdout = subprocess.PIPE).communicate()[0] 20 result = result.splitlines(True) 21 for line in result: 22 if line.startswith("[doctest] unskipped test cases passing the current filters:"): 23 num_tests = int(line.rsplit(' ', 1)[-1]) 24 25 # calculate the ranges 26 cores = multiprocessing.cpu_count() 27 l = range(num_tests + 1) 28 n = int(math.ceil(float(len( l )) / cores)) 29 data = [l[i : i + n] for i in range(1, len( l ), n)] 30 data = tuple([[x[0], x[-1]] for x in data]) 31 32 # for 8 cores and 100 tests the ranges will look like this 33 # ([1, 13], [14, 26], [27, 39], [40, 52], [53, 65], [66, 78], [79, 91], [92, 100]) 34 35 # the worker callback that runs the executable for the given range of tests 36 def worker((first, last)): 37 program_with_args = [sys.argv[1], "--dt-first=" + str(first), "--dt-last=" + str(last)] 38 subprocess.Popen(program_with_args) 39 40 # run the tasks on a pool 41 if __name__ == '__main__': 42 p = multiprocessing.Pool(cores) 43 p.map(worker, data)