win32_opts.py (4708B)
1 #! /usr/bin/env python 2 # encoding: utf-8 3 4 """ 5 Windows-specific optimizations 6 7 This module can help reducing the overhead of listing files on windows 8 (more than 10000 files). Python 3.5 already provides the listdir 9 optimization though. 10 """ 11 12 import os 13 from waflib import Utils, Build, Node, Logs 14 15 try: 16 TP = '%s\\*'.decode('ascii') 17 except AttributeError: 18 TP = '%s\\*' 19 20 if Utils.is_win32: 21 from waflib.Tools import md5_tstamp 22 import ctypes, ctypes.wintypes 23 24 FindFirstFile = ctypes.windll.kernel32.FindFirstFileW 25 FindNextFile = ctypes.windll.kernel32.FindNextFileW 26 FindClose = ctypes.windll.kernel32.FindClose 27 FILE_ATTRIBUTE_DIRECTORY = 0x10 28 INVALID_HANDLE_VALUE = -1 29 UPPER_FOLDERS = ('.', '..') 30 try: 31 UPPER_FOLDERS = [unicode(x) for x in UPPER_FOLDERS] 32 except NameError: 33 pass 34 35 def cached_hash_file(self): 36 try: 37 cache = self.ctx.cache_listdir_cache_hash_file 38 except AttributeError: 39 cache = self.ctx.cache_listdir_cache_hash_file = {} 40 41 if id(self.parent) in cache: 42 try: 43 t = cache[id(self.parent)][self.name] 44 except KeyError: 45 raise IOError('Not a file') 46 else: 47 # an opportunity to list the files and the timestamps at once 48 findData = ctypes.wintypes.WIN32_FIND_DATAW() 49 find = FindFirstFile(TP % self.parent.abspath(), ctypes.byref(findData)) 50 51 if find == INVALID_HANDLE_VALUE: 52 cache[id(self.parent)] = {} 53 raise IOError('Not a file') 54 55 cache[id(self.parent)] = lst_files = {} 56 try: 57 while True: 58 if findData.cFileName not in UPPER_FOLDERS: 59 thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY 60 if not thatsadir: 61 ts = findData.ftLastWriteTime 62 d = (ts.dwLowDateTime << 32) | ts.dwHighDateTime 63 lst_files[str(findData.cFileName)] = d 64 if not FindNextFile(find, ctypes.byref(findData)): 65 break 66 except Exception: 67 cache[id(self.parent)] = {} 68 raise IOError('Not a file') 69 finally: 70 FindClose(find) 71 t = lst_files[self.name] 72 73 fname = self.abspath() 74 if fname in Build.hashes_md5_tstamp: 75 if Build.hashes_md5_tstamp[fname][0] == t: 76 return Build.hashes_md5_tstamp[fname][1] 77 78 try: 79 fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT) 80 except OSError: 81 raise IOError('Cannot read from %r' % fname) 82 f = os.fdopen(fd, 'rb') 83 m = Utils.md5() 84 rb = 1 85 try: 86 while rb: 87 rb = f.read(200000) 88 m.update(rb) 89 finally: 90 f.close() 91 92 # ensure that the cache is overwritten 93 Build.hashes_md5_tstamp[fname] = (t, m.digest()) 94 return m.digest() 95 Node.Node.cached_hash_file = cached_hash_file 96 97 def get_bld_sig_win32(self): 98 try: 99 return self.ctx.hash_cache[id(self)] 100 except KeyError: 101 pass 102 except AttributeError: 103 self.ctx.hash_cache = {} 104 self.ctx.hash_cache[id(self)] = ret = Utils.h_file(self.abspath()) 105 return ret 106 Node.Node.get_bld_sig = get_bld_sig_win32 107 108 def isfile_cached(self): 109 # optimize for nt.stat calls, assuming there are many files for few folders 110 try: 111 cache = self.__class__.cache_isfile_cache 112 except AttributeError: 113 cache = self.__class__.cache_isfile_cache = {} 114 115 try: 116 c1 = cache[id(self.parent)] 117 except KeyError: 118 c1 = cache[id(self.parent)] = [] 119 120 curpath = self.parent.abspath() 121 findData = ctypes.wintypes.WIN32_FIND_DATAW() 122 find = FindFirstFile(TP % curpath, ctypes.byref(findData)) 123 124 if find == INVALID_HANDLE_VALUE: 125 Logs.error("invalid win32 handle isfile_cached %r", self.abspath()) 126 return os.path.isfile(self.abspath()) 127 128 try: 129 while True: 130 if findData.cFileName not in UPPER_FOLDERS: 131 thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY 132 if not thatsadir: 133 c1.append(str(findData.cFileName)) 134 if not FindNextFile(find, ctypes.byref(findData)): 135 break 136 except Exception as e: 137 Logs.error('exception while listing a folder %r %r', self.abspath(), e) 138 return os.path.isfile(self.abspath()) 139 finally: 140 FindClose(find) 141 return self.name in c1 142 Node.Node.isfile_cached = isfile_cached 143 144 def find_or_declare_win32(self, lst): 145 # assuming that "find_or_declare" is called before the build starts, remove the calls to os.path.isfile 146 if isinstance(lst, str): 147 lst = [x for x in Utils.split_path(lst) if x and x != '.'] 148 149 node = self.get_bld().search_node(lst) 150 if node: 151 if not node.isfile_cached(): 152 try: 153 node.parent.mkdir() 154 except OSError: 155 pass 156 return node 157 self = self.get_src() 158 node = self.find_node(lst) 159 if node: 160 if not node.isfile_cached(): 161 try: 162 node.parent.mkdir() 163 except OSError: 164 pass 165 return node 166 node = self.get_bld().make_node(lst) 167 node.parent.mkdir() 168 return node 169 Node.Node.find_or_declare = find_or_declare_win32 170