parallel_debug.py (12067B)
1 #! /usr/bin/env python 2 # encoding: utf-8 3 # Thomas Nagy, 2007-2010 (ita) 4 5 """ 6 Debugging helper for parallel compilation. 7 8 Copy it to your project and load it with:: 9 10 def options(opt): 11 opt.load('parallel_debug', tooldir='.') 12 def build(bld): 13 ... 14 15 The build will then output a file named pdebug.svg in the source directory. 16 """ 17 18 import re, sys, threading, time, traceback 19 try: 20 from Queue import Queue 21 except: 22 from queue import Queue 23 from waflib import Runner, Options, Task, Logs, Errors 24 25 SVG_TEMPLATE = """<?xml version="1.0" encoding="UTF-8" standalone="no"?> 26 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd"> 27 <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.0" 28 x="${project.x}" y="${project.y}" width="${project.width}" height="${project.height}" id="svg602" xml:space="preserve"> 29 30 <style type='text/css' media='screen'> 31 g.over rect { stroke:#FF0000; fill-opacity:0.4 } 32 </style> 33 34 <script type='text/javascript'><![CDATA[ 35 var svg = document.getElementsByTagName('svg')[0]; 36 37 svg.addEventListener('mouseover', function(e) { 38 var g = e.target.parentNode; 39 var x = document.getElementById('r_' + g.id); 40 if (x) { 41 g.setAttribute('class', g.getAttribute('class') + ' over'); 42 x.setAttribute('class', x.getAttribute('class') + ' over'); 43 showInfo(e, g.id, e.target.attributes.tooltip.value); 44 } 45 }, false); 46 47 svg.addEventListener('mouseout', function(e) { 48 var g = e.target.parentNode; 49 var x = document.getElementById('r_' + g.id); 50 if (x) { 51 g.setAttribute('class', g.getAttribute('class').replace(' over', '')); 52 x.setAttribute('class', x.getAttribute('class').replace(' over', '')); 53 hideInfo(e); 54 } 55 }, false); 56 57 function showInfo(evt, txt, details) { 58 ${if project.tooltip} 59 tooltip = document.getElementById('tooltip'); 60 61 var t = document.getElementById('tooltiptext'); 62 t.firstChild.data = txt + " " + details; 63 64 var x = evt.clientX + 9; 65 if (x > 250) { x -= t.getComputedTextLength() + 16; } 66 var y = evt.clientY + 20; 67 tooltip.setAttribute("transform", "translate(" + x + "," + y + ")"); 68 tooltip.setAttributeNS(null, "visibility", "visible"); 69 70 var r = document.getElementById('tooltiprect'); 71 r.setAttribute('width', t.getComputedTextLength() + 6); 72 ${endif} 73 } 74 75 function hideInfo(evt) { 76 var tooltip = document.getElementById('tooltip'); 77 tooltip.setAttributeNS(null,"visibility","hidden"); 78 } 79 ]]></script> 80 81 <!-- inkscape requires a big rectangle or it will not export the pictures properly --> 82 <rect 83 x='${project.x}' y='${project.y}' width='${project.width}' height='${project.height}' 84 style="font-size:10;fill:#ffffff;fill-opacity:0.01;fill-rule:evenodd;stroke:#ffffff;"></rect> 85 86 ${if project.title} 87 <text x="${project.title_x}" y="${project.title_y}" 88 style="font-size:15px; text-anchor:middle; font-style:normal;font-weight:normal;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Bitstream Vera Sans">${project.title}</text> 89 ${endif} 90 91 92 ${for cls in project.groups} 93 <g id='${cls.classname}'> 94 ${for rect in cls.rects} 95 <rect x='${rect.x}' y='${rect.y}' width='${rect.width}' height='${rect.height}' tooltip='${rect.name}' style="font-size:10;fill:${rect.color};fill-rule:evenodd;stroke:#000000;stroke-width:0.4;" /> 96 ${endfor} 97 </g> 98 ${endfor} 99 100 ${for info in project.infos} 101 <g id='r_${info.classname}'> 102 <rect x='${info.x}' y='${info.y}' width='${info.width}' height='${info.height}' style="font-size:10;fill:${info.color};fill-rule:evenodd;stroke:#000000;stroke-width:0.4;" /> 103 <text x="${info.text_x}" y="${info.text_y}" 104 style="font-size:12px;font-style:normal;font-weight:normal;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;font-family:Bitstream Vera Sans" 105 >${info.text}</text> 106 </g> 107 ${endfor} 108 109 ${if project.tooltip} 110 <g transform="translate(0,0)" visibility="hidden" id="tooltip"> 111 <rect id="tooltiprect" y="-15" x="-3" width="1" height="20" style="stroke:black;fill:#edefc2;stroke-width:1"/> 112 <text id="tooltiptext" style="font-family:Arial; font-size:12;fill:black;"> </text> 113 </g> 114 ${endif} 115 116 </svg> 117 """ 118 119 COMPILE_TEMPLATE = '''def f(project): 120 lst = [] 121 def xml_escape(value): 122 return value.replace("&", "&").replace('"', """).replace("'", "'").replace("<", "<").replace(">", ">") 123 124 %s 125 return ''.join(lst) 126 ''' 127 reg_act = re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<code>[^}]*?)\})", re.M) 128 def compile_template(line): 129 130 extr = [] 131 def repl(match): 132 g = match.group 133 if g('dollar'): 134 return "$" 135 elif g('backslash'): 136 return "\\" 137 elif g('subst'): 138 extr.append(g('code')) 139 return "<<|@|>>" 140 return None 141 142 line2 = reg_act.sub(repl, line) 143 params = line2.split('<<|@|>>') 144 assert(extr) 145 146 147 indent = 0 148 buf = [] 149 app = buf.append 150 151 def app(txt): 152 buf.append(indent * '\t' + txt) 153 154 for x in range(len(extr)): 155 if params[x]: 156 app("lst.append(%r)" % params[x]) 157 158 f = extr[x] 159 if f.startswith(('if', 'for')): 160 app(f + ':') 161 indent += 1 162 elif f.startswith('py:'): 163 app(f[3:]) 164 elif f.startswith(('endif', 'endfor')): 165 indent -= 1 166 elif f.startswith(('else', 'elif')): 167 indent -= 1 168 app(f + ':') 169 indent += 1 170 elif f.startswith('xml:'): 171 app('lst.append(xml_escape(%s))' % f[4:]) 172 else: 173 #app('lst.append((%s) or "cannot find %s")' % (f, f)) 174 app('lst.append(str(%s))' % f) 175 176 if extr: 177 if params[-1]: 178 app("lst.append(%r)" % params[-1]) 179 180 fun = COMPILE_TEMPLATE % "\n\t".join(buf) 181 # uncomment the following to debug the template 182 #for i, x in enumerate(fun.splitlines()): 183 # print i, x 184 return Task.funex(fun) 185 186 # red #ff4d4d 187 # green #4da74d 188 # lila #a751ff 189 190 color2code = { 191 'GREEN' : '#4da74d', 192 'YELLOW' : '#fefe44', 193 'PINK' : '#a751ff', 194 'RED' : '#cc1d1d', 195 'BLUE' : '#6687bb', 196 'CYAN' : '#34e2e2', 197 } 198 199 mp = {} 200 info = [] # list of (text,color) 201 202 def map_to_color(name): 203 if name in mp: 204 return mp[name] 205 try: 206 cls = Task.classes[name] 207 except KeyError: 208 return color2code['RED'] 209 if cls.color in mp: 210 return mp[cls.color] 211 if cls.color in color2code: 212 return color2code[cls.color] 213 return color2code['RED'] 214 215 def process(self): 216 m = self.generator.bld.producer 217 try: 218 # TODO another place for this? 219 del self.generator.bld.task_sigs[self.uid()] 220 except KeyError: 221 pass 222 223 self.generator.bld.producer.set_running(1, self) 224 225 try: 226 ret = self.run() 227 except Exception: 228 self.err_msg = traceback.format_exc() 229 self.hasrun = Task.EXCEPTION 230 231 # TODO cleanup 232 m.error_handler(self) 233 return 234 235 if ret: 236 self.err_code = ret 237 self.hasrun = Task.CRASHED 238 else: 239 try: 240 self.post_run() 241 except Errors.WafError: 242 pass 243 except Exception: 244 self.err_msg = traceback.format_exc() 245 self.hasrun = Task.EXCEPTION 246 else: 247 self.hasrun = Task.SUCCESS 248 if self.hasrun != Task.SUCCESS: 249 m.error_handler(self) 250 251 self.generator.bld.producer.set_running(-1, self) 252 253 Task.Task.process_back = Task.Task.process 254 Task.Task.process = process 255 256 old_start = Runner.Parallel.start 257 def do_start(self): 258 try: 259 Options.options.dband 260 except AttributeError: 261 self.bld.fatal('use def options(opt): opt.load("parallel_debug")!') 262 263 self.taskinfo = Queue() 264 old_start(self) 265 if self.dirty: 266 make_picture(self) 267 Runner.Parallel.start = do_start 268 269 lock_running = threading.Lock() 270 def set_running(self, by, tsk): 271 with lock_running: 272 try: 273 cache = self.lock_cache 274 except AttributeError: 275 cache = self.lock_cache = {} 276 277 i = 0 278 if by > 0: 279 vals = cache.values() 280 for i in range(self.numjobs): 281 if i not in vals: 282 cache[tsk] = i 283 break 284 else: 285 i = cache[tsk] 286 del cache[tsk] 287 288 self.taskinfo.put( (i, id(tsk), time.time(), tsk.__class__.__name__, self.processed, self.count, by, ",".join(map(str, tsk.outputs))) ) 289 Runner.Parallel.set_running = set_running 290 291 def name2class(name): 292 return name.replace(' ', '_').replace('.', '_') 293 294 def make_picture(producer): 295 # first, cast the parameters 296 if not hasattr(producer.bld, 'path'): 297 return 298 299 tmp = [] 300 try: 301 while True: 302 tup = producer.taskinfo.get(False) 303 tmp.append(list(tup)) 304 except: 305 pass 306 307 try: 308 ini = float(tmp[0][2]) 309 except: 310 return 311 312 if not info: 313 seen = [] 314 for x in tmp: 315 name = x[3] 316 if not name in seen: 317 seen.append(name) 318 else: 319 continue 320 321 info.append((name, map_to_color(name))) 322 info.sort(key=lambda x: x[0]) 323 324 thread_count = 0 325 acc = [] 326 for x in tmp: 327 thread_count += x[6] 328 acc.append("%d %d %f %r %d %d %d %s" % (x[0], x[1], x[2] - ini, x[3], x[4], x[5], thread_count, x[7])) 329 330 data_node = producer.bld.path.make_node('pdebug.dat') 331 data_node.write('\n'.join(acc)) 332 333 tmp = [lst[:2] + [float(lst[2]) - ini] + lst[3:] for lst in tmp] 334 335 st = {} 336 for l in tmp: 337 if not l[0] in st: 338 st[l[0]] = len(st.keys()) 339 tmp = [ [st[lst[0]]] + lst[1:] for lst in tmp ] 340 THREAD_AMOUNT = len(st.keys()) 341 342 st = {} 343 for l in tmp: 344 if not l[1] in st: 345 st[l[1]] = len(st.keys()) 346 tmp = [ [lst[0]] + [st[lst[1]]] + lst[2:] for lst in tmp ] 347 348 349 BAND = Options.options.dband 350 351 seen = {} 352 acc = [] 353 for x in range(len(tmp)): 354 line = tmp[x] 355 id = line[1] 356 357 if id in seen: 358 continue 359 seen[id] = True 360 361 begin = line[2] 362 thread_id = line[0] 363 for y in range(x + 1, len(tmp)): 364 line = tmp[y] 365 if line[1] == id: 366 end = line[2] 367 #print id, thread_id, begin, end 368 #acc.append( ( 10*thread_id, 10*(thread_id+1), 10*begin, 10*end ) ) 369 acc.append( (BAND * begin, BAND*thread_id, BAND*end - BAND*begin, BAND, line[3], line[7]) ) 370 break 371 372 if Options.options.dmaxtime < 0.1: 373 gwidth = 1 374 for x in tmp: 375 m = BAND * x[2] 376 if m > gwidth: 377 gwidth = m 378 else: 379 gwidth = BAND * Options.options.dmaxtime 380 381 ratio = float(Options.options.dwidth) / gwidth 382 gwidth = Options.options.dwidth 383 gheight = BAND * (THREAD_AMOUNT + len(info) + 1.5) 384 385 386 # simple data model for our template 387 class tobject(object): 388 pass 389 390 model = tobject() 391 model.x = 0 392 model.y = 0 393 model.width = gwidth + 4 394 model.height = gheight + 4 395 396 model.tooltip = not Options.options.dnotooltip 397 398 model.title = Options.options.dtitle 399 model.title_x = gwidth / 2 400 model.title_y = gheight + - 5 401 402 groups = {} 403 for (x, y, w, h, clsname, name) in acc: 404 try: 405 groups[clsname].append((x, y, w, h, name)) 406 except: 407 groups[clsname] = [(x, y, w, h, name)] 408 409 # groups of rectangles (else js highlighting is slow) 410 model.groups = [] 411 for cls in groups: 412 g = tobject() 413 model.groups.append(g) 414 g.classname = name2class(cls) 415 g.rects = [] 416 for (x, y, w, h, name) in groups[cls]: 417 r = tobject() 418 g.rects.append(r) 419 r.x = 2 + x * ratio 420 r.y = 2 + y 421 r.width = w * ratio 422 r.height = h 423 r.name = name 424 r.color = map_to_color(cls) 425 426 cnt = THREAD_AMOUNT 427 428 # caption 429 model.infos = [] 430 for (text, color) in info: 431 inf = tobject() 432 model.infos.append(inf) 433 inf.classname = name2class(text) 434 inf.x = 2 + BAND 435 inf.y = 5 + (cnt + 0.5) * BAND 436 inf.width = BAND/2 437 inf.height = BAND/2 438 inf.color = color 439 440 inf.text = text 441 inf.text_x = 2 + 2 * BAND 442 inf.text_y = 5 + (cnt + 0.5) * BAND + 10 443 444 cnt += 1 445 446 # write the file... 447 template1 = compile_template(SVG_TEMPLATE) 448 txt = template1(model) 449 450 node = producer.bld.path.make_node('pdebug.svg') 451 node.write(txt) 452 Logs.warn('Created the diagram %r', node) 453 454 def options(opt): 455 opt.add_option('--dtitle', action='store', default='Parallel build representation for %r' % ' '.join(sys.argv), 456 help='title for the svg diagram', dest='dtitle') 457 opt.add_option('--dwidth', action='store', type='int', help='diagram width', default=800, dest='dwidth') 458 opt.add_option('--dtime', action='store', type='float', help='recording interval in seconds', default=0.009, dest='dtime') 459 opt.add_option('--dband', action='store', type='int', help='band width', default=22, dest='dband') 460 opt.add_option('--dmaxtime', action='store', type='float', help='maximum time, for drawing fair comparisons', default=0, dest='dmaxtime') 461 opt.add_option('--dnotooltip', action='store_true', help='disable tooltips', default=False, dest='dnotooltip') 462