googletest-output-test.py (12617B)
1 #!/usr/bin/env python 2 # 3 # Copyright 2008, Google Inc. 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions are 8 # met: 9 # 10 # * Redistributions of source code must retain the above copyright 11 # notice, this list of conditions and the following disclaimer. 12 # * Redistributions in binary form must reproduce the above 13 # copyright notice, this list of conditions and the following disclaimer 14 # in the documentation and/or other materials provided with the 15 # distribution. 16 # * Neither the name of Google Inc. nor the names of its 17 # contributors may be used to endorse or promote products derived from 18 # this software without specific prior written permission. 19 # 20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 32 """Tests the text output of Google C++ Testing and Mocking Framework. 33 34 To update the golden file: 35 googletest_output_test.py --build_dir=BUILD/DIR --gengolden 36 where BUILD/DIR contains the built googletest-output-test_ file. 37 googletest_output_test.py --gengolden 38 googletest_output_test.py 39 """ 40 41 import difflib 42 import os 43 import re 44 import sys 45 import gtest_test_utils 46 47 48 # The flag for generating the golden file 49 GENGOLDEN_FLAG = '--gengolden' 50 CATCH_EXCEPTIONS_ENV_VAR_NAME = 'GTEST_CATCH_EXCEPTIONS' 51 52 # The flag indicating stacktraces are not supported 53 NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support' 54 55 IS_LINUX = os.name == 'posix' and os.uname()[0] == 'Linux' 56 IS_WINDOWS = os.name == 'nt' 57 58 GOLDEN_NAME = 'googletest-output-test-golden-lin.txt' 59 60 PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath('googletest-output-test_') 61 62 # At least one command we exercise must not have the 63 # 'internal_skip_environment_and_ad_hoc_tests' argument. 64 COMMAND_LIST_TESTS = ({}, [PROGRAM_PATH, '--gtest_list_tests']) 65 COMMAND_WITH_COLOR = ({}, [PROGRAM_PATH, '--gtest_color=yes']) 66 COMMAND_WITH_TIME = ({}, [PROGRAM_PATH, 67 '--gtest_print_time', 68 'internal_skip_environment_and_ad_hoc_tests', 69 '--gtest_filter=FatalFailureTest.*:LoggingTest.*']) 70 COMMAND_WITH_DISABLED = ( 71 {}, [PROGRAM_PATH, 72 '--gtest_also_run_disabled_tests', 73 'internal_skip_environment_and_ad_hoc_tests', 74 '--gtest_filter=*DISABLED_*']) 75 COMMAND_WITH_SHARDING = ( 76 {'GTEST_SHARD_INDEX': '1', 'GTEST_TOTAL_SHARDS': '2'}, 77 [PROGRAM_PATH, 78 'internal_skip_environment_and_ad_hoc_tests', 79 '--gtest_filter=PassingTest.*']) 80 81 GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(), GOLDEN_NAME) 82 83 84 def ToUnixLineEnding(s): 85 """Changes all Windows/Mac line endings in s to UNIX line endings.""" 86 87 return s.replace('\r\n', '\n').replace('\r', '\n') 88 89 90 def RemoveLocations(test_output): 91 """Removes all file location info from a Google Test program's output. 92 93 Args: 94 test_output: the output of a Google Test program. 95 96 Returns: 97 output with all file location info (in the form of 98 'DIRECTORY/FILE_NAME:LINE_NUMBER: 'or 99 'DIRECTORY\\FILE_NAME(LINE_NUMBER): ') replaced by 100 'FILE_NAME:#: '. 101 """ 102 103 return re.sub(r'.*[/\\]((googletest-output-test_|gtest).cc)(\:\d+|\(\d+\))\: ', 104 r'\1:#: ', test_output) 105 106 107 def RemoveStackTraceDetails(output): 108 """Removes all stack traces from a Google Test program's output.""" 109 110 # *? means "find the shortest string that matches". 111 return re.sub(r'Stack trace:(.|\n)*?\n\n', 112 'Stack trace: (omitted)\n\n', output) 113 114 115 def RemoveStackTraces(output): 116 """Removes all traces of stack traces from a Google Test program's output.""" 117 118 # *? means "find the shortest string that matches". 119 return re.sub(r'Stack trace:(.|\n)*?\n\n', '', output) 120 121 122 def RemoveTime(output): 123 """Removes all time information from a Google Test program's output.""" 124 125 return re.sub(r'\(\d+ ms', '(? ms', output) 126 127 128 def RemoveTypeInfoDetails(test_output): 129 """Removes compiler-specific type info from Google Test program's output. 130 131 Args: 132 test_output: the output of a Google Test program. 133 134 Returns: 135 output with type information normalized to canonical form. 136 """ 137 138 # some compilers output the name of type 'unsigned int' as 'unsigned' 139 return re.sub(r'unsigned int', 'unsigned', test_output) 140 141 142 def NormalizeToCurrentPlatform(test_output): 143 """Normalizes platform specific output details for easier comparison.""" 144 145 if IS_WINDOWS: 146 # Removes the color information that is not present on Windows. 147 test_output = re.sub('\x1b\\[(0;3\d)?m', '', test_output) 148 # Changes failure message headers into the Windows format. 149 test_output = re.sub(r': Failure\n', r': error: ', test_output) 150 # Changes file(line_number) to file:line_number. 151 test_output = re.sub(r'((\w|\.)+)\((\d+)\):', r'\1:\3:', test_output) 152 153 return test_output 154 155 156 def RemoveTestCounts(output): 157 """Removes test counts from a Google Test program's output.""" 158 159 output = re.sub(r'\d+ tests?, listed below', 160 '? tests, listed below', output) 161 output = re.sub(r'\d+ FAILED TESTS', 162 '? FAILED TESTS', output) 163 output = re.sub(r'\d+ tests? from \d+ test cases?', 164 '? tests from ? test cases', output) 165 output = re.sub(r'\d+ tests? from ([a-zA-Z_])', 166 r'? tests from \1', output) 167 return re.sub(r'\d+ tests?\.', '? tests.', output) 168 169 170 def RemoveMatchingTests(test_output, pattern): 171 """Removes output of specified tests from a Google Test program's output. 172 173 This function strips not only the beginning and the end of a test but also 174 all output in between. 175 176 Args: 177 test_output: A string containing the test output. 178 pattern: A regex string that matches names of test cases or 179 tests to remove. 180 181 Returns: 182 Contents of test_output with tests whose names match pattern removed. 183 """ 184 185 test_output = re.sub( 186 r'.*\[ RUN \] .*%s(.|\n)*?\[( FAILED | OK )\] .*%s.*\n' % ( 187 pattern, pattern), 188 '', 189 test_output) 190 return re.sub(r'.*%s.*\n' % pattern, '', test_output) 191 192 193 def NormalizeOutput(output): 194 """Normalizes output (the output of googletest-output-test_.exe).""" 195 196 output = ToUnixLineEnding(output) 197 output = RemoveLocations(output) 198 output = RemoveStackTraceDetails(output) 199 output = RemoveTime(output) 200 return output 201 202 203 def GetShellCommandOutput(env_cmd): 204 """Runs a command in a sub-process, and returns its output in a string. 205 206 Args: 207 env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra 208 environment variables to set, and element 1 is a string with 209 the command and any flags. 210 211 Returns: 212 A string with the command's combined standard and diagnostic output. 213 """ 214 215 # Spawns cmd in a sub-process, and gets its standard I/O file objects. 216 # Set and save the environment properly. 217 environ = os.environ.copy() 218 environ.update(env_cmd[0]) 219 p = gtest_test_utils.Subprocess(env_cmd[1], env=environ) 220 221 return p.output 222 223 224 def GetCommandOutput(env_cmd): 225 """Runs a command and returns its output with all file location 226 info stripped off. 227 228 Args: 229 env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra 230 environment variables to set, and element 1 is a string with 231 the command and any flags. 232 """ 233 234 # Disables exception pop-ups on Windows. 235 environ, cmdline = env_cmd 236 environ = dict(environ) # Ensures we are modifying a copy. 237 environ[CATCH_EXCEPTIONS_ENV_VAR_NAME] = '1' 238 return NormalizeOutput(GetShellCommandOutput((environ, cmdline))) 239 240 241 def GetOutputOfAllCommands(): 242 """Returns concatenated output from several representative commands.""" 243 244 return (GetCommandOutput(COMMAND_WITH_COLOR) + 245 GetCommandOutput(COMMAND_WITH_TIME) + 246 GetCommandOutput(COMMAND_WITH_DISABLED) + 247 GetCommandOutput(COMMAND_WITH_SHARDING)) 248 249 250 test_list = GetShellCommandOutput(COMMAND_LIST_TESTS) 251 SUPPORTS_DEATH_TESTS = 'DeathTest' in test_list 252 SUPPORTS_TYPED_TESTS = 'TypedTest' in test_list 253 SUPPORTS_THREADS = 'ExpectFailureWithThreadsTest' in test_list 254 SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv 255 256 CAN_GENERATE_GOLDEN_FILE = (SUPPORTS_DEATH_TESTS and 257 SUPPORTS_TYPED_TESTS and 258 SUPPORTS_THREADS and 259 SUPPORTS_STACK_TRACES) 260 261 class GTestOutputTest(gtest_test_utils.TestCase): 262 def RemoveUnsupportedTests(self, test_output): 263 if not SUPPORTS_DEATH_TESTS: 264 test_output = RemoveMatchingTests(test_output, 'DeathTest') 265 if not SUPPORTS_TYPED_TESTS: 266 test_output = RemoveMatchingTests(test_output, 'TypedTest') 267 test_output = RemoveMatchingTests(test_output, 'TypedDeathTest') 268 test_output = RemoveMatchingTests(test_output, 'TypeParamDeathTest') 269 if not SUPPORTS_THREADS: 270 test_output = RemoveMatchingTests(test_output, 271 'ExpectFailureWithThreadsTest') 272 test_output = RemoveMatchingTests(test_output, 273 'ScopedFakeTestPartResultReporterTest') 274 test_output = RemoveMatchingTests(test_output, 275 'WorksConcurrently') 276 if not SUPPORTS_STACK_TRACES: 277 test_output = RemoveStackTraces(test_output) 278 279 return test_output 280 281 def testOutput(self): 282 output = GetOutputOfAllCommands() 283 284 golden_file = open(GOLDEN_PATH, 'rb') 285 # A mis-configured source control system can cause \r appear in EOL 286 # sequences when we read the golden file irrespective of an operating 287 # system used. Therefore, we need to strip those \r's from newlines 288 # unconditionally. 289 golden = ToUnixLineEnding(golden_file.read().decode()) 290 golden_file.close() 291 292 # We want the test to pass regardless of certain features being 293 # supported or not. 294 295 # We still have to remove type name specifics in all cases. 296 normalized_actual = RemoveTypeInfoDetails(output) 297 normalized_golden = RemoveTypeInfoDetails(golden) 298 299 if CAN_GENERATE_GOLDEN_FILE: 300 self.assertEqual(normalized_golden, normalized_actual, 301 '\n'.join(difflib.unified_diff( 302 normalized_golden.split('\n'), 303 normalized_actual.split('\n'), 304 'golden', 'actual'))) 305 else: 306 normalized_actual = NormalizeToCurrentPlatform( 307 RemoveTestCounts(normalized_actual)) 308 normalized_golden = NormalizeToCurrentPlatform( 309 RemoveTestCounts(self.RemoveUnsupportedTests(normalized_golden))) 310 311 # This code is very handy when debugging golden file differences: 312 if os.getenv('DEBUG_GTEST_OUTPUT_TEST'): 313 open(os.path.join( 314 gtest_test_utils.GetSourceDir(), 315 '_googletest-output-test_normalized_actual.txt'), 'wb').write( 316 normalized_actual) 317 open(os.path.join( 318 gtest_test_utils.GetSourceDir(), 319 '_googletest-output-test_normalized_golden.txt'), 'wb').write( 320 normalized_golden) 321 322 self.assertEqual(normalized_golden, normalized_actual) 323 324 325 if __name__ == '__main__': 326 if NO_STACKTRACE_SUPPORT_FLAG in sys.argv: 327 # unittest.main() can't handle unknown flags 328 sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG) 329 330 if GENGOLDEN_FLAG in sys.argv: 331 if CAN_GENERATE_GOLDEN_FILE: 332 output = GetOutputOfAllCommands() 333 golden_file = open(GOLDEN_PATH, 'wb') 334 golden_file.write(output) 335 golden_file.close() 336 else: 337 message = ( 338 """Unable to write a golden file when compiled in an environment 339 that does not support all the required features (death tests, 340 typed tests, stack traces, and multiple threads). 341 Please build this test and generate the golden file using Blaze on Linux.""") 342 343 sys.stderr.write(message) 344 sys.exit(1) 345 else: 346 gtest_test_utils.Main()