wscript (13258B)
1 #! /usr/bin/env python 2 # encoding: utf-8 3 # Thomas Nagy, 2005-2018 4 5 """ 6 to make a custom waf file use the option --tools 7 8 To add a tool that does not exist in the folder compat15, pass an absolute path: 9 ./waf-light --tools=compat15,/comp/waf/aba.py --prelude=$'\tfrom waflib.extras import aba\n\taba.foo()' 10 """ 11 12 from __future__ import with_statement 13 14 VERSION="2.0.25" 15 APPNAME='waf' 16 REVISION='' 17 18 top = '.' 19 out = 'build' 20 21 zip_types = ['bz2', 'gz', 'xz'] 22 23 PRELUDE = '' 24 25 import os, sys, re, io, optparse, tokenize 26 from hashlib import md5 27 28 from waflib import Errors, Utils, Options, Logs, Scripting 29 from waflib import Configure 30 Configure.autoconfig = 1 31 32 def sub_file(fname, lst): 33 with open(fname, 'r') as f: 34 txt = f.read() 35 36 for (key, val) in lst: 37 re_pat = re.compile(key, re.M) 38 txt = re_pat.sub(val, txt) 39 40 with open(fname, 'w') as f: 41 f.write(txt) 42 43 def to_bytes(x): 44 if sys.hexversion>0x300000f: 45 return x.encode() 46 return x 47 48 Logs.warn('------> Executing code from the top-level wscript <-----') 49 def init(ctx): 50 if Options.options.setver: # maintainer only (ita) 51 ver = Options.options.setver 52 hexver = Utils.num2ver(ver) 53 hexver = '0x%x'%hexver 54 sub_file('wscript', (('^VERSION=(.*)', 'VERSION="%s"' % ver), )) 55 sub_file('waf-light', (('^VERSION=(.*)', 'VERSION="%s"' % ver), )) 56 57 pats = [] 58 pats.append(('^WAFVERSION=(.*)', 'WAFVERSION="%s"' % ver)) 59 pats.append(('^HEXVERSION(.*)', 'HEXVERSION=%s' % hexver)) 60 61 try: 62 rev = ctx.cmd_and_log("git rev-parse HEAD").strip() 63 except Errors.WafError: 64 rev = '' 65 else: 66 pats.append(('^WAFREVISION(.*)', 'WAFREVISION="%s"' % rev)) 67 68 sub_file('waflib/Context.py', pats) 69 70 sys.exit(0) 71 72 def check(ctx): 73 Logs.warn('Nothing to do') 74 75 # this function is called before any other for parsing the command-line 76 def options(opt): 77 78 # generate waf 79 opt.add_option('--make-waf', action='store_true', default=True, 80 help='creates the waf script', dest='waf') 81 82 opt.add_option('--interpreter', action='store', default=None, 83 help='specify the #! line on top of the waf file', dest='interpreter') 84 85 opt.add_option('--sign', action='store_true', default=False, help='make a signed file', dest='signed') 86 87 default_zip = 'bz2' 88 if os.name == 'java': 89 default_zip = 'gz' 90 opt.add_option('--zip-type', action='store', default=default_zip, 91 help='specify the zip type [Allowed values: %s]' % ' '.join(zip_types), dest='zip') 92 93 opt.add_option('--make-batch', action='store_true', default=False, 94 help='creates a convenience waf.bat file (done automatically on win32 systems)', 95 dest='make_batch') 96 97 opt.add_option('--yes', action='store_true', default=False, 98 help=optparse.SUPPRESS_HELP, 99 dest='yes') 100 101 # those ones are not too interesting 102 opt.add_option('--set-version', default='', 103 help='sets the version number for waf releases (for the maintainer)', dest='setver') 104 opt.add_option('--set-name', default='waf', help=optparse.SUPPRESS_HELP, dest='wafname') 105 106 opt.add_option('--strip', action='store_true', default=True, 107 help='shrinks waf (strip docstrings, saves 33kb)', 108 dest='strip_comments') 109 opt.add_option('--nostrip', action='store_false', help='no shrinking', 110 dest='strip_comments') 111 opt.add_option('--tools', action='store', help='Comma-separated 3rd party tools to add, eg: "compat,ocaml" [Default: "compat15"]', 112 dest='add3rdparty', default='compat15') 113 opt.add_option('--coretools', action='store', help='Comma-separated core tools to add, eg: "vala,tex" [Default: all of them]', 114 dest='coretools', default='default') 115 opt.add_option('--prelude', action='store', help='Code to execute before calling waf', dest='prelude', default=PRELUDE) 116 opt.add_option('--namesfrom', action='store', help='Obtain the file names from a model archive', dest='namesfrom', default=None) 117 opt.load('python') 118 119 def process_tokens(tokens): 120 accu = [] 121 prev = tokenize.NEWLINE 122 123 indent = 0 124 line_buf = [] 125 126 for (type, token, start, end, line) in tokens: 127 token = token.replace('\r\n', '\n') 128 if type == tokenize.NEWLINE: 129 if line_buf: 130 accu.append(indent * '\t') 131 ln = "".join(line_buf) 132 if ln == 'if __name__=="__main__":': break 133 #ln = ln.replace('\n', '') 134 accu.append(ln) 135 accu.append('\n') 136 line_buf = [] 137 prev = tokenize.NEWLINE 138 elif type == tokenize.INDENT: 139 indent += 1 140 elif type == tokenize.DEDENT: 141 indent -= 1 142 elif type == tokenize.NAME: 143 if prev == tokenize.NAME or prev == tokenize.NUMBER: line_buf.append(' ') 144 line_buf.append(token) 145 elif type == tokenize.NUMBER: 146 if prev == tokenize.NAME or prev == tokenize.NUMBER: line_buf.append(' ') 147 line_buf.append(token) 148 elif type == tokenize.STRING: 149 if not line_buf and token.startswith('"'): pass 150 else: line_buf.append(token) 151 elif type == tokenize.COMMENT: 152 pass 153 elif type == tokenize.OP: 154 line_buf.append(token) 155 else: 156 if token != "\n": line_buf.append(token) 157 158 if token != '\n': 159 prev = type 160 161 body = ''.join(accu) 162 return body 163 164 deco_re = re.compile('(def|class)\\s+(\\w+)\\(.*') 165 def process_decorators(body): 166 lst = body.splitlines() 167 accu = [] 168 all_deco = [] 169 buf = [] # put the decorator lines 170 for line in lst: 171 if line.startswith('@'): 172 buf.append(line[1:]) 173 elif buf: 174 name = deco_re.sub('\\2', line) 175 if not name: 176 raise IOError("decorator not followed by a function!" + line) 177 for x in buf: 178 all_deco.append('%s(%s)' % (x, name)) 179 accu.append(line) 180 buf = [] 181 else: 182 accu.append(line) 183 return '\n'.join(accu+all_deco) 184 185 def sfilter(path): 186 if path.endswith('.py') : 187 if Options.options.strip_comments: 188 if sys.version_info[0] >= 3: 189 with open(path, 'rb') as f: 190 tk = tokenize.tokenize(f.readline) 191 next(tk) # the first one is always tokenize.ENCODING for Python 3, ignore it 192 cnt = process_tokens(tk) 193 else: 194 with open(path, 'r') as f: 195 cnt = process_tokens(tokenize.generate_tokens(f.readline)) 196 else: 197 with open(path, 'r') as f: 198 cnt = f.read() 199 # WARNING: since python >= 2.5 is required, decorators are not processed anymore 200 # uncomment the following to enable decorator replacement: 201 #cnt = process_decorators(cnt) 202 #if cnt.find('set(') > -1: 203 # cnt = 'import sys\nif sys.hexversion < 0x020400f0: from sets import Set as set\n' + cnt 204 cnt = '#! /usr/bin/env python\n# encoding: utf-8\n# WARNING! Do not edit! https://waf.io/book/index.html#_obtaining_the_waf_file\n\n' + cnt 205 206 else: 207 with open(path, 'r') as f: 208 cnt = f.read() 209 210 if sys.hexversion > 0x030000f0: 211 return (io.BytesIO(cnt.encode('utf-8')), len(cnt.encode('utf-8')), cnt) 212 return (io.BytesIO(cnt), len(cnt), cnt) 213 214 def create_waf(self, *k, **kw): 215 mw = 'tmp-waf-'+VERSION 216 print('-> preparing %r' % mw) 217 218 import tarfile, zipfile 219 220 zipType = Options.options.zip.strip().lower() 221 if zipType not in zip_types: 222 zipType = zip_types[0] 223 224 directory_files = {} 225 files = [] 226 add3rdparty = [] 227 for x in Options.options.add3rdparty.split(','): 228 if os.path.isdir(x): 229 # Create mapping from files absolute path to path in module 230 # directory (for module mylib): 231 # 232 # {"/home/path/mylib/__init__.py": "mylib/__init__.py", 233 # "/home/path/mylib/lib.py": "mylib/lib.py", 234 # "/home/path/mylib/sub/sub.py": "mylib/sub/lib.py" 235 # } 236 # 237 x_dir = self.generator.bld.root.find_dir( 238 os.path.abspath(os.path.expanduser(x))) 239 240 file_list = x_dir.ant_glob('**/*.py') 241 242 for f in file_list: 243 244 file_from = f.abspath() 245 file_to = os.path.join(x_dir.name, f.path_from(x_dir)) 246 247 # If this is executed on Windows, then file_to will contain 248 # '\' path separators. These should be changed to '/', otherwise 249 # the added tools will not be accessible on Unix systems. 250 directory_files[file_from] = file_to.replace('\\', '/') 251 files.append(file_from) 252 253 elif os.path.isabs(x): 254 files.append(x) 255 else: 256 add3rdparty.append(x + '.py') 257 258 coretools = [] 259 for x in Options.options.coretools.split(','): 260 coretools.append(x + '.py') 261 262 up_node = self.generator.bld.path 263 for node in up_node.find_dir('waflib').ant_glob(incl=['*.py', 'Tools/*.py', 'extras/*.py']): 264 relpath = node.path_from(up_node) 265 if node.name == '__init__.py': 266 files.append(relpath) 267 continue 268 if node.parent.name == 'Tools' and Options.options.coretools != 'default': 269 if node.name not in coretools: 270 continue 271 if node.parent.name == 'extras': 272 if node.name not in add3rdparty: 273 continue 274 files.append(relpath) 275 276 if Options.options.namesfrom: 277 with tarfile.open(Options.options.namesfrom) as tar: 278 oldfiles = files 279 files = [x.name for x in tar.getmembers()] 280 if set(files) ^ set(oldfiles): 281 Logs.warn('The archive model has differences:') 282 Logs.warn('- Added %r', list(set(files) - set(oldfiles))) 283 Logs.warn('- Removed %r', list(set(oldfiles) - set(files))) 284 285 #open a file as tar.[extension] for writing 286 tar = tarfile.open('%s.tar.%s' % (mw, zipType), "w:%s" % zipType) 287 z = zipfile.ZipFile("zip/waflib.zip", "w", compression=zipfile.ZIP_DEFLATED) 288 for x in files: 289 try: 290 tarinfo = tar.gettarinfo(x, x) 291 except NotImplementedError: 292 # jython 2.7.0 workaround 293 tarinfo = tarfile.TarInfo(x) 294 tarinfo.uid = tarinfo.gid = 0 295 tarinfo.uname = tarinfo.gname = 'root' 296 (code, size, cnt) = sfilter(x) 297 tarinfo.size = size 298 299 if x in directory_files: 300 tarinfo.name = 'waflib/extras/' + directory_files[x] 301 elif os.path.isabs(x): 302 tarinfo.name = 'waflib/extras/' + os.path.split(x)[1] 303 304 print(' adding %s as %s' % (x, tarinfo.name)) 305 def dest(x): 306 if x in directory_files: 307 return os.path.join('waflib', 'extras', directory_files[x]) 308 elif os.path.isabs(x): 309 return os.path.join('waflib', 'extras', os.path.basename(x)) 310 else: 311 return os.path.normpath(os.path.relpath(x, ".")) 312 313 z.write(x, dest(x)) 314 tar.addfile(tarinfo, code) 315 tar.close() 316 z.close() 317 318 with open('waf-light', 'r') as f: 319 code1 = f.read() 320 321 # tune the application name if necessary 322 if Options.options.wafname != 'waf': 323 Options.options.prelude = '\tfrom waflib import Context\n\tContext.WAFNAME=%r\n' % Options.options.wafname + Options.options.prelude 324 325 # now store the revision unique number in waf 326 code1 = code1.replace("if sys.hexversion<0x206000f:\n\traise ImportError('Python >= 2.6 is required to create the waf file')\n", '') 327 code1 = code1.replace('\t#import waflib.extras.compat15#PRELUDE', Options.options.prelude) 328 329 # when possible, set the git revision in the waf file 330 bld = self.generator.bld 331 try: 332 rev = bld.cmd_and_log('git rev-parse HEAD', quiet=0).strip() 333 except Errors.WafError: 334 rev = '' 335 else: 336 reg = re.compile('^GIT(.*)', re.M) 337 code1 = reg.sub('GIT="%s"' % rev, code1) 338 339 # if the waf file is installed somewhere... but do not do that 340 prefix = '' 341 reg = re.compile('^INSTALL=(.*)', re.M) 342 code1 = reg.sub(r'INSTALL=%r' % prefix, code1) 343 #change the tarfile extension in the waf script 344 reg = re.compile('bz2', re.M) 345 code1 = reg.sub(zipType, code1) 346 if zipType == 'gz': 347 code1 = code1.replace('bunzip2', 'gzip -d') 348 elif zipType == 'xz': 349 code1 = code1.replace('bunzip2', 'xz -d') 350 351 with open('%s.tar.%s' % (mw, zipType), 'rb') as f: 352 cnt = f.read() 353 354 # the REVISION value is the md5 sum of the compressed data (facilitate audits) 355 m = md5() 356 m.update(cnt) 357 REVISION = m.hexdigest() 358 reg = re.compile('^REVISION=(.*)', re.M) 359 code1 = reg.sub(r'REVISION="%s"' % REVISION, code1) 360 361 def find_unused(kd, ch): 362 for i in range(35, 125): 363 for j in range(35, 125): 364 if i==j: continue 365 if i == 39 or j == 39: continue 366 if i == 92 or j == 92: continue 367 s = chr(i) + chr(j) 368 if -1 == kd.find(s.encode()): 369 return (kd.replace(ch.encode(), s.encode()), s) 370 raise ValueError('Could not find a proper encoding') 371 372 # The reverse order prevents collisions 373 (cnt, C3) = find_unused(cnt, '\x00') 374 (cnt, C2) = find_unused(cnt, '\r') 375 (cnt, C1) = find_unused(cnt, '\n') 376 ccc = code1.replace("C1='x'", "C1='%s'" % C1).replace("C2='x'", "C2='%s'" % C2).replace("C3='x'", "C3='%s'" % C3) 377 378 if getattr(Options.options, 'interpreter', None): 379 ccc = ccc.replace('#!/usr/bin/env python', Options.options.interpreter) 380 381 with open('waf', 'wb') as f: 382 f.write(ccc.encode()) 383 f.write(to_bytes('#==>\n#')) 384 f.write(cnt) 385 f.write(to_bytes('\n#<==\n')) 386 387 if Options.options.signed: 388 f.flush() 389 try: 390 os.remove('waf.asc') 391 except OSError: 392 pass 393 ret = Utils.subprocess.Popen('gpg -bass waf', shell=True).wait() 394 if ret: 395 raise ValueError('Could not sign the waf file!') 396 397 sig = Utils.readf('waf.asc') 398 sig = sig.replace('\r', '').replace('\n', '\\n') 399 f.write(to_bytes('#')) 400 f.write(to_bytes(sig)) 401 f.write(to_bytes('\n')) 402 os.remove('waf.asc') 403 404 405 if sys.platform == 'win32' or Options.options.make_batch: 406 with open('waf.bat', 'w') as f: 407 f.write('@setlocal\n@set PYEXE=python\n@where %PYEXE% 1>NUL 2>NUL\n@if %ERRORLEVEL% neq 0 set PYEXE=py\n@%PYEXE% -x "%~dp0waf" %*\n@exit /b %ERRORLEVEL%\n') 408 409 if sys.platform != 'win32': 410 os.chmod('waf', Utils.O755) 411 os.remove('%s.tar.%s' % (mw, zipType)) 412 413 def configure(conf): 414 conf.load('python') 415 416 def build(bld): 417 waf = bld.path.make_node('waf') # do not use a build directory for this file 418 bld(name='create_waf', rule=create_waf, target=waf, always=True, color='PINK') 419 420 class Dist(Scripting.Dist): 421 def get_excl(self): 422 return super(self.__class__, self).get_excl() + ' **/waflib.zip'