msvcdeps.py (8835B)
1 #!/usr/bin/env python 2 # encoding: utf-8 3 # Copyright Garmin International or its subsidiaries, 2012-2013 4 5 ''' 6 Off-load dependency scanning from Python code to MSVC compiler 7 8 This tool is safe to load in any environment; it will only activate the 9 MSVC exploits when it finds that a particular taskgen uses MSVC to 10 compile. 11 12 Empirical testing shows about a 10% execution time savings from using 13 this tool as compared to c_preproc. 14 15 The technique of gutting scan() and pushing the dependency calculation 16 down to post_run() is cribbed from gccdeps.py. 17 18 This affects the cxx class, so make sure to load Qt5 after this tool. 19 20 Usage:: 21 22 def options(opt): 23 opt.load('compiler_cxx') 24 def configure(conf): 25 conf.load('compiler_cxx msvcdeps') 26 ''' 27 28 import os, sys, tempfile, threading 29 30 from waflib import Context, Errors, Logs, Task, Utils 31 from waflib.Tools import c_preproc, c, cxx, msvc 32 from waflib.TaskGen import feature, before_method 33 34 lock = threading.Lock() 35 36 PREPROCESSOR_FLAG = '/showIncludes' 37 INCLUDE_PATTERN = 'Note: including file:' 38 39 # Extensible by outside tools 40 supported_compilers = ['msvc'] 41 42 @feature('c', 'cxx') 43 @before_method('process_source') 44 def apply_msvcdeps_flags(taskgen): 45 if taskgen.env.CC_NAME not in supported_compilers: 46 return 47 48 for flag in ('CFLAGS', 'CXXFLAGS'): 49 if taskgen.env.get_flat(flag).find(PREPROCESSOR_FLAG) < 0: 50 taskgen.env.append_value(flag, PREPROCESSOR_FLAG) 51 52 53 def get_correct_path_case(base_path, path): 54 ''' 55 Return a case-corrected version of ``path`` by searching the filesystem for 56 ``path``, relative to ``base_path``, using the case returned by the filesystem. 57 ''' 58 components = Utils.split_path(path) 59 60 corrected_path = '' 61 if os.path.isabs(path): 62 corrected_path = components.pop(0).upper() + os.sep 63 64 for part in components: 65 part = part.lower() 66 search_path = os.path.join(base_path, corrected_path) 67 if part == '..': 68 corrected_path = os.path.join(corrected_path, part) 69 search_path = os.path.normpath(search_path) 70 continue 71 72 for item in sorted(os.listdir(search_path)): 73 if item.lower() == part: 74 corrected_path = os.path.join(corrected_path, item) 75 break 76 else: 77 raise ValueError("Can't find %r in %r" % (part, search_path)) 78 79 return corrected_path 80 81 82 def path_to_node(base_node, path, cached_nodes): 83 ''' 84 Take the base node and the path and return a node 85 Results are cached because searching the node tree is expensive 86 The following code is executed by threads, it is not safe, so a lock is needed... 87 ''' 88 # normalize the path to remove parent path components (..) 89 path = os.path.normpath(path) 90 91 # normalize the path case to increase likelihood of a cache hit 92 node_lookup_key = (base_node, os.path.normcase(path)) 93 94 try: 95 node = cached_nodes[node_lookup_key] 96 except KeyError: 97 # retry with lock on cache miss 98 with lock: 99 try: 100 node = cached_nodes[node_lookup_key] 101 except KeyError: 102 path = get_correct_path_case(base_node.abspath(), path) 103 node = cached_nodes[node_lookup_key] = base_node.find_node(path) 104 105 return node 106 107 def post_run(self): 108 if self.env.CC_NAME not in supported_compilers: 109 return super(self.derived_msvcdeps, self).post_run() 110 111 # TODO this is unlikely to work with netcache 112 if getattr(self, 'cached', None): 113 return Task.Task.post_run(self) 114 115 resolved_nodes = [] 116 unresolved_names = [] 117 bld = self.generator.bld 118 119 # Dynamically bind to the cache 120 try: 121 cached_nodes = bld.cached_nodes 122 except AttributeError: 123 cached_nodes = bld.cached_nodes = {} 124 125 for path in self.msvcdeps_paths: 126 node = None 127 if os.path.isabs(path): 128 node = path_to_node(bld.root, path, cached_nodes) 129 else: 130 # when calling find_resource, make sure the path does not begin with '..' 131 base_node = bld.bldnode 132 path = [k for k in Utils.split_path(path) if k and k != '.'] 133 while path[0] == '..': 134 path.pop(0) 135 base_node = base_node.parent 136 path = os.sep.join(path) 137 138 node = path_to_node(base_node, path, cached_nodes) 139 140 if not node: 141 raise ValueError('could not find %r for %r' % (path, self)) 142 else: 143 if not c_preproc.go_absolute: 144 if not (node.is_child_of(bld.srcnode) or node.is_child_of(bld.bldnode)): 145 # System library 146 Logs.debug('msvcdeps: Ignoring system include %r', node) 147 continue 148 149 if id(node) == id(self.inputs[0]): 150 # ignore the source file, it is already in the dependencies 151 # this way, successful config tests may be retrieved from the cache 152 continue 153 154 resolved_nodes.append(node) 155 156 Logs.debug('deps: msvcdeps for %s returned %s', self, resolved_nodes) 157 158 bld.node_deps[self.uid()] = resolved_nodes 159 bld.raw_deps[self.uid()] = unresolved_names 160 161 try: 162 del self.cache_sig 163 except AttributeError: 164 pass 165 166 Task.Task.post_run(self) 167 168 def scan(self): 169 if self.env.CC_NAME not in supported_compilers: 170 return super(self.derived_msvcdeps, self).scan() 171 172 resolved_nodes = self.generator.bld.node_deps.get(self.uid(), []) 173 unresolved_names = [] 174 return (resolved_nodes, unresolved_names) 175 176 def sig_implicit_deps(self): 177 if self.env.CC_NAME not in supported_compilers: 178 return super(self.derived_msvcdeps, self).sig_implicit_deps() 179 bld = self.generator.bld 180 181 try: 182 return self.compute_sig_implicit_deps() 183 except Errors.TaskNotReady: 184 raise ValueError("Please specify the build order precisely with msvcdeps (c/c++ tasks)") 185 except EnvironmentError: 186 # If a file is renamed, assume the dependencies are stale and must be recalculated 187 for x in bld.node_deps.get(self.uid(), []): 188 if not x.is_bld() and not x.exists(): 189 try: 190 del x.parent.children[x.name] 191 except KeyError: 192 pass 193 194 key = self.uid() 195 bld.node_deps[key] = [] 196 bld.raw_deps[key] = [] 197 return Utils.SIG_NIL 198 199 def exec_command(self, cmd, **kw): 200 if self.env.CC_NAME not in supported_compilers: 201 return super(self.derived_msvcdeps, self).exec_command(cmd, **kw) 202 203 if not 'cwd' in kw: 204 kw['cwd'] = self.get_cwd() 205 206 if self.env.PATH: 207 env = kw['env'] = dict(kw.get('env') or self.env.env or os.environ) 208 env['PATH'] = self.env.PATH if isinstance(self.env.PATH, str) else os.pathsep.join(self.env.PATH) 209 210 # The Visual Studio IDE adds an environment variable that causes 211 # the MS compiler to send its textual output directly to the 212 # debugging window rather than normal stdout/stderr. 213 # 214 # This is unrecoverably bad for this tool because it will cause 215 # all the dependency scanning to see an empty stdout stream and 216 # assume that the file being compiled uses no headers. 217 # 218 # See http://blogs.msdn.com/b/freik/archive/2006/04/05/569025.aspx 219 # 220 # Attempting to repair the situation by deleting the offending 221 # envvar at this point in tool execution will not be good enough-- 222 # its presence poisons the 'waf configure' step earlier. We just 223 # want to put a sanity check here in order to help developers 224 # quickly diagnose the issue if an otherwise-good Waf tree 225 # is then executed inside the MSVS IDE. 226 assert 'VS_UNICODE_OUTPUT' not in kw['env'] 227 228 cmd, args = self.split_argfile(cmd) 229 try: 230 (fd, tmp) = tempfile.mkstemp() 231 os.write(fd, '\r\n'.join(args).encode()) 232 os.close(fd) 233 234 self.msvcdeps_paths = [] 235 kw['env'] = kw.get('env', os.environ.copy()) 236 kw['cwd'] = kw.get('cwd', os.getcwd()) 237 kw['quiet'] = Context.STDOUT 238 kw['output'] = Context.STDOUT 239 240 out = [] 241 if Logs.verbose: 242 Logs.debug('argfile: @%r -> %r', tmp, args) 243 try: 244 raw_out = self.generator.bld.cmd_and_log(cmd + ['@' + tmp], **kw) 245 ret = 0 246 except Errors.WafError as e: 247 # Use e.msg if e.stdout is not set 248 raw_out = getattr(e, 'stdout', e.msg) 249 250 # Return non-zero error code even if we didn't 251 # get one from the exception object 252 ret = getattr(e, 'returncode', 1) 253 254 Logs.debug('msvcdeps: Running for: %s' % self.inputs[0]) 255 for line in raw_out.splitlines(): 256 if line.startswith(INCLUDE_PATTERN): 257 # Only strip whitespace after log to preserve 258 # dependency structure in debug output 259 inc_path = line[len(INCLUDE_PATTERN):] 260 Logs.debug('msvcdeps: Regex matched %s', inc_path) 261 self.msvcdeps_paths.append(inc_path.strip()) 262 else: 263 out.append(line) 264 265 # Pipe through the remaining stdout content (not related to /showIncludes) 266 if self.generator.bld.logger: 267 self.generator.bld.logger.debug('out: %s' % os.linesep.join(out)) 268 else: 269 sys.stdout.write(os.linesep.join(out) + os.linesep) 270 271 return ret 272 finally: 273 try: 274 os.remove(tmp) 275 except OSError: 276 # anti-virus and indexers can keep files open -_- 277 pass 278 279 280 def wrap_compiled_task(classname): 281 derived_class = type(classname, (Task.classes[classname],), {}) 282 derived_class.derived_msvcdeps = derived_class 283 derived_class.post_run = post_run 284 derived_class.scan = scan 285 derived_class.sig_implicit_deps = sig_implicit_deps 286 derived_class.exec_command = exec_command 287 288 for k in ('c', 'cxx'): 289 if k in Task.classes: 290 wrap_compiled_task(k) 291 292 def options(opt): 293 raise ValueError('Do not load msvcdeps options') 294