build_cleaner.py (9561B)
1 #!/usr/bin/env python3 2 # Copyright (c) the JPEG XL Project Authors. All rights reserved. 3 # 4 # Use of this source code is governed by a BSD-style 5 # license that can be found in the LICENSE file. 6 7 8 """build_cleaner.py: Update build files. 9 10 This tool keeps certain parts of the build files up to date. 11 """ 12 13 import argparse 14 import locale 15 import os 16 import re 17 import subprocess 18 import sys 19 import tempfile 20 21 COPYRIGHT = [ 22 "Copyright (c) the JPEG XL Project Authors. All rights reserved.", 23 "", 24 "Use of this source code is governed by a BSD-style", 25 "license that can be found in the LICENSE file." 26 ] 27 28 DOC = [ 29 "This file is generated, do not modify by manually.", 30 "Run `tools/scripts/build_cleaner.py --update` to regenerate it." 31 ] 32 33 def RepoFiles(src_dir): 34 """Return the list of files from the source git repository""" 35 git_bin = os.environ.get('GIT_BIN', 'git') 36 files = subprocess.check_output([git_bin, '-C', src_dir, 'ls-files']) 37 ret = files.decode(locale.getpreferredencoding()).splitlines() 38 ret.sort() 39 return ret 40 41 42 def Check(condition, msg): 43 if not condition: 44 print(msg) 45 sys.exit(2) 46 47 48 def ContainsFn(*parts): 49 return lambda path: any(part in path for part in parts) 50 51 52 def HasPrefixFn(*prefixes): 53 return lambda path: any(path.startswith(prefix) for prefix in prefixes) 54 55 56 def HasSuffixFn(*suffixes): 57 return lambda path: any(path.endswith(suffix) for suffix in suffixes) 58 59 60 def Filter(src, fn): 61 yes_list = [] 62 no_list = [] 63 for item in src: 64 (yes_list if fn(item) else no_list).append(item) 65 return yes_list, no_list 66 67 68 def SplitLibFiles(repo_files): 69 """Splits the library files into the different groups.""" 70 71 srcs_base = 'lib/' 72 srcs, _ = Filter(repo_files, HasPrefixFn(srcs_base)) 73 srcs = [path[len(srcs_base):] for path in srcs] 74 srcs, _ = Filter(srcs, HasSuffixFn('.cc', '.h', '.ui')) 75 srcs.sort() 76 77 # Let's keep Jpegli sources a bit separate for a while. 78 jpegli_srcs, srcs = Filter(srcs, HasPrefixFn('jpegli')) 79 # TODO(eustas): move to tools? 80 _, srcs = Filter(srcs, HasSuffixFn('gbench_main.cc')) 81 # This stub compilation unit is manually referenced in CMake buildfile. 82 _, srcs = Filter(srcs, HasSuffixFn('nothing.cc')) 83 84 # First pick files scattered across directories. 85 tests, srcs = Filter(srcs, HasSuffixFn('_test.cc')) 86 jpegli_tests, jpegli_srcs = Filter(jpegli_srcs, HasSuffixFn('_test.cc')) 87 # TODO(eustas): move to separate list? 88 _, srcs = Filter(srcs, ContainsFn('testing.h')) 89 _, jpegli_srcs = Filter(jpegli_srcs, ContainsFn('testing.h')) 90 testlib_files, srcs = Filter(srcs, ContainsFn('test')) 91 jpegli_testlib_files, jpegli_srcs = Filter(jpegli_srcs, ContainsFn('test')) 92 jpegli_libjpeg_helper_files, jpegli_testlib_files = Filter( 93 jpegli_testlib_files, ContainsFn('libjpeg_test_util')) 94 gbench_sources, srcs = Filter(srcs, HasSuffixFn('_gbench.cc')) 95 96 extras_sources, srcs = Filter(srcs, HasPrefixFn('extras/')) 97 lib_srcs, srcs = Filter(srcs, HasPrefixFn('jxl/')) 98 public_headers, srcs = Filter(srcs, HasPrefixFn('include/jxl/')) 99 threads_sources, srcs = Filter(srcs, HasPrefixFn('threads/')) 100 101 Check(len(srcs) == 0, 'Orphan source files: ' + str(srcs)) 102 103 base_sources, lib_srcs = Filter(lib_srcs, HasPrefixFn('jxl/base/')) 104 105 jpegli_wrapper_sources, jpegli_srcs = Filter( 106 jpegli_srcs, HasSuffixFn('libjpeg_wrapper.cc')) 107 jpegli_sources = jpegli_srcs 108 109 threads_public_headers, public_headers = Filter( 110 public_headers, ContainsFn('_parallel_runner')) 111 112 codec_names = ['apng', 'exr', 'gif', 'jpegli', 'jpg', 'jxl', 'npy', 'pgx', 113 'pnm'] 114 codecs = {} 115 for codec in codec_names: 116 codec_sources, extras_sources = Filter(extras_sources, HasPrefixFn( 117 f'extras/dec/{codec}', f'extras/enc/{codec}')) 118 codecs[f'codec_{codec}_sources'] = codec_sources 119 120 # TODO(eustas): move to separate folder? 121 extras_for_tools_sources, extras_sources = Filter(extras_sources, ContainsFn( 122 '/codec', '/hlg', '/metrics', '/packed_image_convert', '/render_hdr', 123 '/tone_mapping')) 124 125 # Source files only needed by the encoder or by tools (including decoding 126 # tools), but not by the decoder library. 127 # TODO(eustas): investigate the status of codec_in_out.h 128 # TODO(eustas): rename butteraugli_wrapper.cc to butteraugli.cc? 129 # TODO(eustas): is it possible to make butteraugli more standalone? 130 enc_sources, lib_srcs = Filter(lib_srcs, ContainsFn('/enc_', '/butteraugli', 131 'jxl/encode.cc', 'jxl/encode_internal.h' 132 )) 133 134 # The remaining of the files are in the dec_library. 135 dec_jpeg_sources, dec_sources = Filter(lib_srcs, HasPrefixFn('jxl/jpeg/', 136 'jxl/decode_to_jpeg.cc', 'jxl/decode_to_jpeg.h')) 137 dec_box_sources, dec_sources = Filter(dec_sources, HasPrefixFn( 138 'jxl/box_content_decoder.cc', 'jxl/box_content_decoder.h')) 139 cms_sources, dec_sources = Filter(dec_sources, HasPrefixFn('jxl/cms/')) 140 141 # TODO(lode): further prune dec_srcs: only those files that the decoder 142 # absolutely needs, and or not only for encoding, should be listed here. 143 144 return codecs | {'base_sources': base_sources, 145 'cms_sources': cms_sources, 146 'dec_box_sources': dec_box_sources, 147 'dec_jpeg_sources': dec_jpeg_sources, 148 'dec_sources': dec_sources, 149 'enc_sources': enc_sources, 150 'extras_for_tools_sources': extras_for_tools_sources, 151 'extras_sources': extras_sources, 152 'gbench_sources': gbench_sources, 153 'jpegli_sources': jpegli_sources, 154 'jpegli_testlib_files': jpegli_testlib_files, 155 'jpegli_libjpeg_helper_files': jpegli_libjpeg_helper_files, 156 'jpegli_tests': jpegli_tests, 157 'jpegli_wrapper_sources' : jpegli_wrapper_sources, 158 'public_headers': public_headers, 159 'testlib_files': testlib_files, 160 'tests': tests, 161 'threads_public_headers': threads_public_headers, 162 'threads_sources': threads_sources, 163 } 164 165 166 def MaybeUpdateFile(args, filename, new_text): 167 """Optionally replace file with new contents. 168 169 If args.update is set, it will update the file with the new contents, 170 otherwise it will return True when no changes were needed. 171 """ 172 filepath = os.path.join(args.src_dir, filename) 173 with open(filepath, 'r') as f: 174 src_text = f.read() 175 176 if new_text == src_text: 177 return True 178 179 if args.update: 180 print('Updating %s' % filename) 181 with open(filepath, 'w') as f: 182 f.write(new_text) 183 return True 184 else: 185 prefix = os.path.basename(filename) 186 with tempfile.NamedTemporaryFile(mode='w', prefix=prefix) as new_file: 187 new_file.write(new_text) 188 new_file.flush() 189 subprocess.call(['diff', '-u', filepath, '--label', 'a/' + filename, 190 new_file.name, '--label', 'b/' + filename]) 191 return False 192 193 194 def FormatList(items, prefix, suffix): 195 return ''.join(f'{prefix}{item}{suffix}\n' for item in items) 196 197 198 def FormatGniVar(name, var): 199 if type(var) is list: 200 contents = FormatList(var, ' "', '",') 201 return f'{name} = [\n{contents}]\n' 202 else: # TODO(eustas): do we need scalar strings? 203 return f'{name} = {var}\n' 204 205 206 def FormatCMakeVar(name, var): 207 if type(var) is list: 208 contents = FormatList(var, ' ', '') 209 return f'set({name}\n{contents})\n' 210 else: # TODO(eustas): do we need scalar strings? 211 return f'set({name} {var})\n' 212 213 214 def GetJpegLibVersion(src_dir): 215 with open(os.path.join(src_dir, 'CMakeLists.txt'), 'r') as f: 216 cmake_text = f.read() 217 m = re.search(r'set\(JPEGLI_LIBJPEG_LIBRARY_SOVERSION "([0-9]+)"', 218 cmake_text) 219 version = m.group(1) 220 if len(version) == 1: 221 version += "0" 222 return version 223 224 def ToHashComment(lines): 225 return [("# " + line).rstrip() for line in lines] 226 def ToDocstringComment(lines): 227 return ["\"\"\""] + lines + ["\"\"\""] 228 229 def BuildCleaner(args): 230 repo_files = RepoFiles(args.src_dir) 231 232 with open(os.path.join(args.src_dir, 'lib/CMakeLists.txt'), 'r') as f: 233 cmake_text = f.read() 234 version = {'major_version': '', 'minor_version': '', 'patch_version': ''} 235 for var in version.keys(): 236 cmake_var = f'JPEGXL_{var.upper()}' 237 # TODO(eustas): use `cmake -L` 238 # Regexp: 239 # set(_varname_ _capture_decimal_) 240 match = re.search(r'set\(' + cmake_var + r' ([0-9]+)\)', cmake_text) 241 version[var] = match.group(1) 242 243 version['jpegli_lib_version'] = GetJpegLibVersion(args.src_dir) 244 245 lists = SplitLibFiles(repo_files) 246 247 cmake_chunks = ToHashComment(COPYRIGHT) + [""] + ToHashComment(DOC) 248 cmake_parts = lists 249 for var in sorted(cmake_parts): 250 cmake_chunks.append(FormatCMakeVar( 251 'JPEGXL_INTERNAL_' + var.upper(), cmake_parts[var])) 252 253 gni_bzl_parts = version | lists 254 gni_bzl_chunks = [] 255 for var in sorted(gni_bzl_parts): 256 gni_bzl_chunks.append(FormatGniVar('libjxl_' + var, gni_bzl_parts[var])) 257 258 bzl_chunks = ToHashComment(COPYRIGHT) + [""] + \ 259 ToDocstringComment(DOC) + [""] + gni_bzl_chunks 260 gni_chunks = ToHashComment(COPYRIGHT) + [""] + \ 261 ToHashComment(DOC) + [""] + gni_bzl_chunks 262 263 okay = [ 264 MaybeUpdateFile(args, 'lib/jxl_lists.bzl', '\n'.join(bzl_chunks)), 265 MaybeUpdateFile(args, 'lib/jxl_lists.cmake', '\n'.join(cmake_chunks)), 266 MaybeUpdateFile(args, 'lib/lib.gni', '\n'.join(gni_chunks)), 267 ] 268 return all(okay) 269 270 271 def main(): 272 parser = argparse.ArgumentParser(description=__doc__) 273 parser.add_argument('--src-dir', 274 default=os.path.realpath(os.path.join( os.path.dirname(__file__), '../..')), 275 help='path to the build directory') 276 parser.add_argument('--update', default=False, action='store_true', 277 help='update the build files instead of only checking') 278 args = parser.parse_args() 279 Check(BuildCleaner(args), 'Build files need update.') 280 281 282 if __name__ == '__main__': 283 main()