waf

FORK: waf with some random patches
git clone https://git.neptards.moe/neptards/waf.git
Log | Files | Refs | README

waf_xattr.py (4144B)

      1 #! /usr/bin/env python
      2 # encoding: utf-8
      3 
      4 """
      5 Use extended attributes instead of database files
      6 
      7 1. Input files will be made writable
      8 2. This is only for systems providing extended filesystem attributes
      9 3. By default, hashes are calculated only if timestamp/size change (HASH_CACHE below)
     10 4. The module enables "deep_inputs" on all tasks by propagating task signatures
     11 5. This module also skips task signature comparisons for task code changes due to point 4.
     12 6. This module is for Python3/Linux only, but it could be extended to Python2/other systems
     13    using the xattr library
     14 7. For projects in which tasks always declare output files, it should be possible to
     15    store the rest of build context attributes on output files (imp_sigs, raw_deps and node_deps)
     16    but this is not done here
     17 
     18 On a simple C++ project benchmark, the variations before and after adding waf_xattr.py were observed:
     19 total build time: 20s -> 22s
     20 no-op build time: 2.4s -> 1.8s
     21 pickle file size: 2.9MB -> 2.6MB
     22 """
     23 
     24 import os
     25 from waflib import Logs, Node, Task, Utils, Errors
     26 from waflib.Task import SKIP_ME, RUN_ME, CANCEL_ME, ASK_LATER, SKIPPED, MISSING
     27 
     28 HASH_CACHE = True
     29 SIG_VAR = 'user.waf.sig'
     30 SEP = ','.encode()
     31 TEMPLATE = '%b%d,%d'.encode()
     32 
     33 try:
     34 	PermissionError
     35 except NameError:
     36 	PermissionError = IOError
     37 
     38 def getxattr(self):
     39 	return os.getxattr(self.abspath(), SIG_VAR)
     40 
     41 def setxattr(self, val):
     42 	os.setxattr(self.abspath(), SIG_VAR, val)
     43 
     44 def h_file(self):
     45 	try:
     46 		ret = getxattr(self)
     47 	except OSError:
     48 		if HASH_CACHE:
     49 			st = os.stat(self.abspath())
     50 			mtime = st.st_mtime
     51 			size = st.st_size
     52 	else:
     53 		if len(ret) == 16:
     54 			# for build directory files
     55 			return ret
     56 
     57 		if HASH_CACHE:
     58 			# check if timestamp and mtime match to avoid re-hashing
     59 			st = os.stat(self.abspath())
     60 			mtime, size = ret[16:].split(SEP)
     61 			if int(1000 * st.st_mtime) == int(mtime) and st.st_size == int(size):
     62 				return ret[:16]
     63 
     64 	ret = Utils.h_file(self.abspath())
     65 	if HASH_CACHE:
     66 		val = TEMPLATE % (ret, int(1000 * st.st_mtime), int(st.st_size))
     67 		try:
     68 			setxattr(self, val)
     69 		except PermissionError:
     70 			os.chmod(self.abspath(), st.st_mode | 128)
     71 			setxattr(self, val)
     72 	return ret
     73 
     74 def runnable_status(self):
     75 	bld = self.generator.bld
     76 	if bld.is_install < 0:
     77 		return SKIP_ME
     78 
     79 	for t in self.run_after:
     80 		if not t.hasrun:
     81 			return ASK_LATER
     82 		elif t.hasrun < SKIPPED:
     83 			# a dependency has an error
     84 			return CANCEL_ME
     85 
     86 	# first compute the signature
     87 	try:
     88 		new_sig = self.signature()
     89 	except Errors.TaskNotReady:
     90 		return ASK_LATER
     91 
     92 	if not self.outputs:
     93 		# compare the signature to a signature computed previously
     94 		# this part is only for tasks with no output files
     95 		key = self.uid()
     96 		try:
     97 			prev_sig = bld.task_sigs[key]
     98 		except KeyError:
     99 			Logs.debug('task: task %r must run: it was never run before or the task code changed', self)
    100 			return RUN_ME
    101 		if new_sig != prev_sig:
    102 			Logs.debug('task: task %r must run: the task signature changed', self)
    103 			return RUN_ME
    104 
    105 	# compare the signatures of the outputs to make a decision
    106 	for node in self.outputs:
    107 		try:
    108 			sig = node.h_file()
    109 		except EnvironmentError:
    110 			Logs.debug('task: task %r must run: an output node does not exist', self)
    111 			return RUN_ME
    112 		if sig != new_sig:
    113 			Logs.debug('task: task %r must run: an output node is stale', self)
    114 			return RUN_ME
    115 
    116 	return (self.always_run and RUN_ME) or SKIP_ME
    117 
    118 def post_run(self):
    119 	bld = self.generator.bld
    120 	sig = self.signature()
    121 	for node in self.outputs:
    122 		if not node.exists():
    123 			self.hasrun = MISSING
    124 			self.err_msg = '-> missing file: %r' % node.abspath()
    125 			raise Errors.WafError(self.err_msg)
    126 		os.setxattr(node.abspath(), 'user.waf.sig', sig)
    127 	if not self.outputs:
    128 		# only for task with no outputs
    129 		bld.task_sigs[self.uid()] = sig
    130 	if not self.keep_last_cmd:
    131 		try:
    132 			del self.last_cmd
    133 		except AttributeError:
    134 			pass
    135 
    136 try:
    137 	os.getxattr
    138 except AttributeError:
    139 	pass
    140 else:
    141 	h_file.__doc__ = Node.Node.h_file.__doc__
    142 
    143 	# keep file hashes as file attributes
    144 	Node.Node.h_file = h_file
    145 
    146 	# enable "deep_inputs" on all tasks
    147 	Task.Task.runnable_status = runnable_status
    148 	Task.Task.post_run = post_run
    149 	Task.Task.sig_deep_inputs = Utils.nada
    150