batched_cc.py (4694B)
1 #!/usr/bin/env python 2 # encoding: utf-8 3 # Thomas Nagy, 2006-2015 (ita) 4 5 """ 6 Instead of compiling object files one by one, c/c++ compilers are often able to compile at once: 7 cc -c ../file1.c ../file2.c ../file3.c 8 9 Files are output on the directory where the compiler is called, and dependencies are more difficult 10 to track (do not run the command on all source files if only one file changes) 11 As such, we do as if the files were compiled one by one, but no command is actually run: 12 replace each cc/cpp Task by a TaskSlave. A new task called TaskMaster collects the 13 signatures from each slave and finds out the command-line to run. 14 15 Just import this module to start using it: 16 def build(bld): 17 bld.load('batched_cc') 18 19 Note that this is provided as an example, unity builds are recommended 20 for best performance results (fewer tasks and fewer jobs to execute). 21 See waflib/extras/unity.py. 22 """ 23 24 from waflib import Task, Utils 25 from waflib.TaskGen import extension, feature, after_method 26 from waflib.Tools import c, cxx 27 28 MAX_BATCH = 50 29 30 c_str = '${CC} ${ARCH_ST:ARCH} ${CFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}' 31 c_fun, _ = Task.compile_fun_noshell(c_str) 32 33 cxx_str = '${CXX} ${ARCH_ST:ARCH} ${CXXFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}' 34 cxx_fun, _ = Task.compile_fun_noshell(cxx_str) 35 36 count = 70000 37 class batch(Task.Task): 38 color = 'PINK' 39 40 after = ['c', 'cxx'] 41 before = ['cprogram', 'cshlib', 'cstlib', 'cxxprogram', 'cxxshlib', 'cxxstlib'] 42 43 def uid(self): 44 return Utils.h_list([Task.Task.uid(self), self.generator.idx, self.generator.path.abspath(), self.generator.target]) 45 46 def __str__(self): 47 return 'Batch compilation for %d slaves' % len(self.slaves) 48 49 def __init__(self, *k, **kw): 50 Task.Task.__init__(self, *k, **kw) 51 self.slaves = [] 52 self.inputs = [] 53 self.hasrun = 0 54 55 global count 56 count += 1 57 self.idx = count 58 59 def add_slave(self, slave): 60 self.slaves.append(slave) 61 self.set_run_after(slave) 62 63 def runnable_status(self): 64 for t in self.run_after: 65 if not t.hasrun: 66 return Task.ASK_LATER 67 68 for t in self.slaves: 69 #if t.executed: 70 if t.hasrun != Task.SKIPPED: 71 return Task.RUN_ME 72 73 return Task.SKIP_ME 74 75 def get_cwd(self): 76 return self.slaves[0].outputs[0].parent 77 78 def batch_incpaths(self): 79 st = self.env.CPPPATH_ST 80 return [st % node.abspath() for node in self.generator.includes_nodes] 81 82 def run(self): 83 self.outputs = [] 84 85 srclst = [] 86 slaves = [] 87 for t in self.slaves: 88 if t.hasrun != Task.SKIPPED: 89 slaves.append(t) 90 srclst.append(t.inputs[0].abspath()) 91 92 self.env.SRCLST = srclst 93 94 if self.slaves[0].__class__.__name__ == 'c': 95 ret = c_fun(self) 96 else: 97 ret = cxx_fun(self) 98 99 if ret: 100 return ret 101 102 for t in slaves: 103 t.old_post_run() 104 105 def hook(cls_type): 106 def n_hook(self, node): 107 108 ext = '.obj' if self.env.CC_NAME == 'msvc' else '.o' 109 name = node.name 110 k = name.rfind('.') 111 if k >= 0: 112 basename = name[:k] + ext 113 else: 114 basename = name + ext 115 116 outdir = node.parent.get_bld().make_node('%d' % self.idx) 117 outdir.mkdir() 118 out = outdir.find_or_declare(basename) 119 120 task = self.create_task(cls_type, node, out) 121 122 try: 123 self.compiled_tasks.append(task) 124 except AttributeError: 125 self.compiled_tasks = [task] 126 127 if not getattr(self, 'masters', None): 128 self.masters = {} 129 self.allmasters = [] 130 131 def fix_path(tsk): 132 if self.env.CC_NAME == 'msvc': 133 tsk.env.append_unique('CXX_TGT_F_BATCHED', '/Fo%s\\' % outdir.abspath()) 134 135 if not node.parent in self.masters: 136 m = self.masters[node.parent] = self.master = self.create_task('batch') 137 fix_path(m) 138 self.allmasters.append(m) 139 else: 140 m = self.masters[node.parent] 141 if len(m.slaves) > MAX_BATCH: 142 m = self.masters[node.parent] = self.master = self.create_task('batch') 143 fix_path(m) 144 self.allmasters.append(m) 145 m.add_slave(task) 146 return task 147 return n_hook 148 149 extension('.c')(hook('c')) 150 extension('.cpp','.cc','.cxx','.C','.c++')(hook('cxx')) 151 152 @feature('cprogram', 'cshlib', 'cstaticlib', 'cxxprogram', 'cxxshlib', 'cxxstlib') 153 @after_method('apply_link') 154 def link_after_masters(self): 155 if getattr(self, 'allmasters', None): 156 for m in self.allmasters: 157 self.link_task.set_run_after(m) 158 159 # Modify the c and cxx task classes - in theory it would be best to 160 # create subclasses and to re-map the c/c++ extensions 161 for x in ('c', 'cxx'): 162 t = Task.classes[x] 163 def run(self): 164 pass 165 166 def post_run(self): 167 pass 168 169 setattr(t, 'oldrun', getattr(t, 'run', None)) 170 setattr(t, 'run', run) 171 setattr(t, 'old_post_run', t.post_run) 172 setattr(t, 'post_run', post_run) 173