You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
266 lines
12 KiB
Python
266 lines
12 KiB
Python
#===----------------------------------------------------------------------===##
|
|
#
|
|
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
# See https://llvm.org/LICENSE.txt for license information.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
#
|
|
#===----------------------------------------------------------------------===##
|
|
|
|
import copy
|
|
import errno
|
|
import os
|
|
import time
|
|
import random
|
|
|
|
import lit.Test # pylint: disable=import-error
|
|
import lit.TestRunner # pylint: disable=import-error
|
|
from lit.TestRunner import ParserKind, IntegratedTestKeywordParser \
|
|
# pylint: disable=import-error
|
|
|
|
from libcxx.test.executor import LocalExecutor as LocalExecutor
|
|
import libcxx.util
|
|
|
|
|
|
class LibcxxTestFormat(object):
|
|
"""
|
|
Custom test format handler for use with the test format use by libc++.
|
|
|
|
Tests fall into two categories:
|
|
FOO.pass.cpp - Executable test which should compile, run, and exit with
|
|
code 0.
|
|
FOO.fail.cpp - Negative test case which is expected to fail compilation.
|
|
FOO.sh.cpp - A test that uses LIT's ShTest format.
|
|
"""
|
|
|
|
def __init__(self, cxx, use_verify_for_fail, execute_external,
|
|
executor, exec_env):
|
|
self.cxx = copy.deepcopy(cxx)
|
|
self.use_verify_for_fail = use_verify_for_fail
|
|
self.execute_external = execute_external
|
|
self.executor = executor
|
|
self.exec_env = dict(exec_env)
|
|
|
|
@staticmethod
|
|
def _make_custom_parsers():
|
|
return [
|
|
IntegratedTestKeywordParser('FLAKY_TEST.', ParserKind.TAG,
|
|
initial_value=False),
|
|
IntegratedTestKeywordParser('MODULES_DEFINES:', ParserKind.LIST,
|
|
initial_value=[])
|
|
]
|
|
|
|
@staticmethod
|
|
def _get_parser(key, parsers):
|
|
for p in parsers:
|
|
if p.keyword == key:
|
|
return p
|
|
assert False and "parser not found"
|
|
|
|
# TODO: Move this into lit's FileBasedTest
|
|
def getTestsInDirectory(self, testSuite, path_in_suite,
|
|
litConfig, localConfig):
|
|
source_path = testSuite.getSourcePath(path_in_suite)
|
|
for filename in os.listdir(source_path):
|
|
# Ignore dot files and excluded tests.
|
|
if filename.startswith('.') or filename in localConfig.excludes:
|
|
continue
|
|
|
|
filepath = os.path.join(source_path, filename)
|
|
if not os.path.isdir(filepath):
|
|
if any([filename.endswith(ext)
|
|
for ext in localConfig.suffixes]):
|
|
yield lit.Test.Test(testSuite, path_in_suite + (filename,),
|
|
localConfig)
|
|
|
|
def execute(self, test, lit_config):
|
|
while True:
|
|
try:
|
|
return self._execute(test, lit_config)
|
|
except OSError as oe:
|
|
if oe.errno != errno.ETXTBSY:
|
|
raise
|
|
time.sleep(0.1)
|
|
|
|
def _execute(self, test, lit_config):
|
|
name = test.path_in_suite[-1]
|
|
name_root, name_ext = os.path.splitext(name)
|
|
is_libcxx_test = test.path_in_suite[0] == 'libcxx'
|
|
is_sh_test = name_root.endswith('.sh')
|
|
is_pass_test = name.endswith('.pass.cpp') or name.endswith('.pass.mm')
|
|
is_fail_test = name.endswith('.fail.cpp') or name.endswith('.fail.mm')
|
|
is_objcxx_test = name.endswith('.mm')
|
|
is_objcxx_arc_test = name.endswith('.arc.pass.mm') or \
|
|
name.endswith('.arc.fail.mm')
|
|
assert is_sh_test or name_ext == '.cpp' or name_ext == '.mm', \
|
|
'non-cpp file must be sh test'
|
|
|
|
if test.config.unsupported:
|
|
return (lit.Test.UNSUPPORTED,
|
|
"A lit.local.cfg marked this unsupported")
|
|
|
|
if is_objcxx_test and not \
|
|
'objective-c++' in test.config.available_features:
|
|
return (lit.Test.UNSUPPORTED, "Objective-C++ is not supported")
|
|
|
|
parsers = self._make_custom_parsers()
|
|
script = lit.TestRunner.parseIntegratedTestScript(
|
|
test, additional_parsers=parsers, require_script=is_sh_test)
|
|
# Check if a result for the test was returned. If so return that
|
|
# result.
|
|
if isinstance(script, lit.Test.Result):
|
|
return script
|
|
if lit_config.noExecute:
|
|
return lit.Test.Result(lit.Test.PASS)
|
|
|
|
# Check that we don't have run lines on tests that don't support them.
|
|
if not is_sh_test and len(script) != 0:
|
|
lit_config.fatal('Unsupported RUN line found in test %s' % name)
|
|
|
|
tmpDir, tmpBase = lit.TestRunner.getTempPaths(test)
|
|
substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir,
|
|
tmpBase)
|
|
script = lit.TestRunner.applySubstitutions(script, substitutions)
|
|
|
|
test_cxx = copy.deepcopy(self.cxx)
|
|
if is_fail_test:
|
|
test_cxx.useCCache(False)
|
|
test_cxx.useWarnings(False)
|
|
extra_modules_defines = self._get_parser('MODULES_DEFINES:',
|
|
parsers).getValue()
|
|
if '-fmodules' in test.config.available_features:
|
|
test_cxx.compile_flags += [('-D%s' % mdef.strip()) for
|
|
mdef in extra_modules_defines]
|
|
test_cxx.addWarningFlagIfSupported('-Wno-macro-redefined')
|
|
# FIXME: libc++ debug tests #define _LIBCPP_ASSERT to override it
|
|
# If we see this we need to build the test against uniquely built
|
|
# modules.
|
|
if is_libcxx_test:
|
|
with open(test.getSourcePath(), 'rb') as f:
|
|
contents = f.read()
|
|
if b'#define _LIBCPP_ASSERT' in contents:
|
|
test_cxx.useModules(False)
|
|
|
|
if is_objcxx_test:
|
|
test_cxx.source_lang = 'objective-c++'
|
|
if is_objcxx_arc_test:
|
|
test_cxx.compile_flags += ['-fobjc-arc']
|
|
else:
|
|
test_cxx.compile_flags += ['-fno-objc-arc']
|
|
test_cxx.link_flags += ['-framework', 'Foundation']
|
|
|
|
# Dispatch the test based on its suffix.
|
|
if is_sh_test:
|
|
if not isinstance(self.executor, LocalExecutor):
|
|
# We can't run ShTest tests with a executor yet.
|
|
# For now, bail on trying to run them
|
|
return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
|
|
test.config.environment = dict(self.exec_env)
|
|
return lit.TestRunner._runShTest(test, lit_config,
|
|
self.execute_external, script,
|
|
tmpBase)
|
|
elif is_fail_test:
|
|
return self._evaluate_fail_test(test, test_cxx, parsers)
|
|
elif is_pass_test:
|
|
return self._evaluate_pass_test(test, tmpBase, lit_config,
|
|
test_cxx, parsers)
|
|
else:
|
|
# No other test type is supported
|
|
assert False
|
|
|
|
def _clean(self, exec_path): # pylint: disable=no-self-use
|
|
libcxx.util.cleanFile(exec_path)
|
|
|
|
def _evaluate_pass_test(self, test, tmpBase, lit_config,
|
|
test_cxx, parsers):
|
|
execDir = os.path.dirname(test.getExecPath())
|
|
source_path = test.getSourcePath()
|
|
exec_path = tmpBase + '.exe'
|
|
object_path = tmpBase + '.o'
|
|
# Create the output directory if it does not already exist.
|
|
libcxx.util.mkdir_p(os.path.dirname(tmpBase))
|
|
try:
|
|
# Compile the test
|
|
cmd, out, err, rc = test_cxx.compileLinkTwoSteps(
|
|
source_path, out=exec_path, object_file=object_path,
|
|
cwd=execDir)
|
|
compile_cmd = cmd
|
|
if rc != 0:
|
|
report = libcxx.util.makeReport(cmd, out, err, rc)
|
|
report += "Compilation failed unexpectedly!"
|
|
return lit.Test.Result(lit.Test.FAIL, report)
|
|
# Run the test
|
|
local_cwd = os.path.dirname(source_path)
|
|
env = None
|
|
if self.exec_env:
|
|
env = self.exec_env
|
|
# TODO: Only list actually needed files in file_deps.
|
|
# Right now we just mark all of the .dat files in the same
|
|
# directory as dependencies, but it's likely less than that. We
|
|
# should add a `// FILE-DEP: foo.dat` to each test to track this.
|
|
data_files = [os.path.join(local_cwd, f)
|
|
for f in os.listdir(local_cwd) if f.endswith('.dat')]
|
|
is_flaky = self._get_parser('FLAKY_TEST.', parsers).getValue()
|
|
max_retry = 3 if is_flaky else 1
|
|
for retry_count in range(max_retry):
|
|
cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
|
|
local_cwd, data_files,
|
|
env)
|
|
report = "Compiled With: '%s'\n" % ' '.join(compile_cmd)
|
|
report += libcxx.util.makeReport(cmd, out, err, rc)
|
|
if rc == 0:
|
|
res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
|
|
return lit.Test.Result(res, report)
|
|
elif rc != 0 and retry_count + 1 == max_retry:
|
|
report += "Compiled test failed unexpectedly!"
|
|
return lit.Test.Result(lit.Test.FAIL, report)
|
|
|
|
assert False # Unreachable
|
|
finally:
|
|
# Note that cleanup of exec_file happens in `_clean()`. If you
|
|
# override this, cleanup is your reponsibility.
|
|
libcxx.util.cleanFile(object_path)
|
|
self._clean(exec_path)
|
|
|
|
def _evaluate_fail_test(self, test, test_cxx, parsers):
|
|
source_path = test.getSourcePath()
|
|
# FIXME: lift this detection into LLVM/LIT.
|
|
with open(source_path, 'rb') as f:
|
|
contents = f.read()
|
|
verify_tags = [b'expected-note', b'expected-remark',
|
|
b'expected-warning', b'expected-error',
|
|
b'expected-no-diagnostics']
|
|
use_verify = self.use_verify_for_fail and \
|
|
any([tag in contents for tag in verify_tags])
|
|
# FIXME(EricWF): GCC 5 does not evaluate static assertions that
|
|
# are dependant on a template parameter when '-fsyntax-only' is passed.
|
|
# This is fixed in GCC 6. However for now we only pass "-fsyntax-only"
|
|
# when using Clang.
|
|
if test_cxx.type != 'gcc':
|
|
test_cxx.flags += ['-fsyntax-only']
|
|
if use_verify:
|
|
test_cxx.useVerify()
|
|
test_cxx.useWarnings()
|
|
if '-Wuser-defined-warnings' in test_cxx.warning_flags:
|
|
test_cxx.warning_flags += ['-Wno-error=user-defined-warnings']
|
|
else:
|
|
# We still need to enable certain warnings on .fail.cpp test when
|
|
# -verify isn't enabled. Such as -Werror=unused-result. However,
|
|
# we don't want it enabled too liberally, which might incorrectly
|
|
# allow unrelated failure tests to 'pass'.
|
|
#
|
|
# Therefore, we check if the test was expected to fail because of
|
|
# nodiscard before enabling it
|
|
test_str_list = [b'ignoring return value', b'nodiscard',
|
|
b'NODISCARD']
|
|
if any(test_str in contents for test_str in test_str_list):
|
|
test_cxx.flags += ['-Werror=unused-result']
|
|
cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull)
|
|
expected_rc = 0 if use_verify else 1
|
|
report = libcxx.util.makeReport(cmd, out, err, rc)
|
|
if rc == expected_rc:
|
|
return lit.Test.Result(lit.Test.PASS, report)
|
|
else:
|
|
report += ('Expected compilation to fail!\n' if not use_verify else
|
|
'Expected compilation using verify to pass!\n')
|
|
return lit.Test.Result(lit.Test.FAIL, report)
|