mirror of
https://github.com/crystalidea/qt6windows7.git
synced 2024-11-23 20:10:48 +08:00
304 lines
13 KiB
Python
304 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
# Copyright (C) 2021 The Qt Company Ltd.
|
|
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
|
|
|
|
|
|
import sys
|
|
import os
|
|
import re
|
|
import subprocess
|
|
|
|
from subprocess import STDOUT, PIPE
|
|
from tempfile import TemporaryDirectory, mkstemp
|
|
|
|
MY_NAME = os.path.basename(__file__)
|
|
my_dir = os.path.dirname(__file__)
|
|
testrunner = os.path.join(my_dir, "..", "qt-testrunner.py")
|
|
mock_test = os.path.join(my_dir, "qt_mock_test.py")
|
|
xml_log_template = os.path.join(my_dir, "qt_mock_test-log.xml")
|
|
|
|
with open(xml_log_template) as f:
|
|
XML_TEMPLATE = f.read()
|
|
|
|
|
|
import unittest
|
|
|
|
def setUpModule():
|
|
global TEMPDIR
|
|
TEMPDIR = TemporaryDirectory(prefix="tst_testrunner-")
|
|
|
|
filename = os.path.join(TEMPDIR.name, "file_1")
|
|
print("setUpModule(): setting up temporary directory and env var"
|
|
" QT_MOCK_TEST_STATE_FILE=" + filename + " and"
|
|
" QT_MOCK_TEST_XML_TEMPLATE_FILE=" + xml_log_template)
|
|
|
|
os.environ["QT_MOCK_TEST_STATE_FILE"] = filename
|
|
os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = xml_log_template
|
|
|
|
def tearDownModule():
|
|
print("\ntearDownModule(): Cleaning up temporary directory:",
|
|
TEMPDIR.name)
|
|
del os.environ["QT_MOCK_TEST_STATE_FILE"]
|
|
TEMPDIR.cleanup()
|
|
|
|
|
|
# Helper to run a command and always capture output
|
|
def run(*args, **kwargs):
|
|
if DEBUG:
|
|
print("Running: ", args, flush=True)
|
|
proc = subprocess.run(*args, stdout=PIPE, stderr=STDOUT, **kwargs)
|
|
if DEBUG and proc.stdout:
|
|
print(proc.stdout.decode(), flush=True)
|
|
return proc
|
|
|
|
# Helper to run qt-testrunner.py with proper testing arguments.
|
|
def run_testrunner(xml_filename=None, extra_args=None, env=None):
|
|
|
|
args = [ testrunner, mock_test ]
|
|
if xml_filename:
|
|
args += [ "--parse-xml-testlog", xml_filename ]
|
|
if extra_args:
|
|
args += extra_args
|
|
|
|
return run(args, env=env)
|
|
|
|
# Write the XML_TEMPLATE to filename, replacing the templated results.
|
|
def write_xml_log(filename, failure=None):
|
|
data = XML_TEMPLATE
|
|
# Replace what was asked to fail with "fail"
|
|
if type(failure) in (list, tuple):
|
|
for template in failure:
|
|
data = data.replace("{{"+template+"_result}}", "fail")
|
|
elif type(failure) is str:
|
|
data = data.replace("{{"+failure+"_result}}", "fail")
|
|
# Replace the rest with "pass"
|
|
data = re.sub(r"{{[^}]+}}", "pass", data)
|
|
with open(filename, "w") as f:
|
|
f.write(data)
|
|
|
|
|
|
# Test that qt_mock_test.py behaves well. This is necessary to properly
|
|
# test qt-testrunner.
|
|
class Test_qt_mock_test(unittest.TestCase):
|
|
def setUp(self):
|
|
state_file = os.environ["QT_MOCK_TEST_STATE_FILE"]
|
|
if os.path.exists(state_file):
|
|
os.remove(state_file)
|
|
def test_always_pass(self):
|
|
proc = run([mock_test, "always_pass"])
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_always_fail(self):
|
|
proc = run([mock_test, "always_fail"])
|
|
self.assertEqual(proc.returncode, 1)
|
|
def test_fail_then_pass_2(self):
|
|
proc = run([mock_test, "fail_then_pass:2"])
|
|
self.assertEqual(proc.returncode, 1)
|
|
proc = run([mock_test, "fail_then_pass:2"])
|
|
self.assertEqual(proc.returncode, 1)
|
|
proc = run([mock_test, "fail_then_pass:2"])
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_fail_then_pass_1(self):
|
|
proc = run([mock_test, "fail_then_pass:1"])
|
|
self.assertEqual(proc.returncode, 1)
|
|
proc = run([mock_test, "fail_then_pass:1"])
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_fail_then_pass_many_tests(self):
|
|
proc = run([mock_test, "fail_then_pass:1"])
|
|
self.assertEqual(proc.returncode, 1)
|
|
proc = run([mock_test, "fail_then_pass:2"])
|
|
self.assertEqual(proc.returncode, 1)
|
|
proc = run([mock_test, "fail_then_pass:1"])
|
|
self.assertEqual(proc.returncode, 0)
|
|
proc = run([mock_test, "fail_then_pass:2"])
|
|
self.assertEqual(proc.returncode, 1)
|
|
proc = run([mock_test, "fail_then_pass:2"])
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_xml_file_is_written(self):
|
|
filename = os.path.join(TEMPDIR.name, "testlog.xml")
|
|
proc = run([mock_test, "-o", filename+",xml"])
|
|
self.assertEqual(proc.returncode, 0)
|
|
self.assertTrue(os.path.exists(filename))
|
|
self.assertGreater(os.path.getsize(filename), 0)
|
|
os.remove(filename)
|
|
|
|
# Test regular invocations of qt-testrunner.
|
|
class Test_testrunner(unittest.TestCase):
|
|
def setUp(self):
|
|
state_file = os.environ["QT_MOCK_TEST_STATE_FILE"]
|
|
if os.path.exists(state_file):
|
|
os.remove(state_file)
|
|
old_logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml")
|
|
if os.path.exists(old_logfile):
|
|
os.remove(old_logfile)
|
|
self.env = dict()
|
|
self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"] = os.environ["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
|
|
self.env["QT_MOCK_TEST_STATE_FILE"] = state_file
|
|
self.extra_args = [ "--log-dir", TEMPDIR.name ]
|
|
def prepare_env(self, run_list=None):
|
|
if run_list is not None:
|
|
self.env['QT_MOCK_TEST_RUN_LIST'] = ",".join(run_list)
|
|
def run2(self):
|
|
return run_testrunner(extra_args=self.extra_args, env=self.env)
|
|
def test_simple_invocation(self):
|
|
# All tests pass.
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_always_pass(self):
|
|
self.prepare_env(run_list=["always_pass"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_always_fail(self):
|
|
self.prepare_env(run_list=["always_fail"])
|
|
proc = self.run2()
|
|
# TODO verify that re-runs==max_repeats
|
|
self.assertEqual(proc.returncode, 2)
|
|
def test_flaky_pass_1(self):
|
|
self.prepare_env(run_list=["always_pass,fail_then_pass:1"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_flaky_pass_5(self):
|
|
self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:5"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_flaky_fail(self):
|
|
self.prepare_env(run_list=["always_pass,fail_then_pass:6"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 2)
|
|
def test_flaky_pass_fail(self):
|
|
self.prepare_env(run_list=["always_pass,fail_then_pass:1,fail_then_pass:6"])
|
|
proc = self.run2()
|
|
# TODO verify that one func was re-run and passed but the other failed.
|
|
self.assertEqual(proc.returncode, 2)
|
|
def test_initTestCase_fail_crash(self):
|
|
self.prepare_env(run_list=["initTestCase,always_pass"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 3)
|
|
|
|
# If no XML file is found by qt-testrunner, it is usually considered a
|
|
# CRASH and the whole test is re-run. Even when the return code is zero.
|
|
# It is a PASS only if the test is not capable of XML output (see no_extra_args, TODO test it).
|
|
def test_no_xml_log_written_pass_crash(self):
|
|
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
|
|
self.prepare_env(run_list=["always_pass"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 3)
|
|
# On the 2nd iteration of the full test, both of the tests pass.
|
|
# Still it's a CRASH because no XML file was found.
|
|
def test_no_xml_log_written_fail_then_pass_crash(self):
|
|
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
|
|
self.prepare_env(run_list=["always_pass,fail_then_pass:1"])
|
|
proc = self.run2()
|
|
# TODO verify that the whole test has run twice.
|
|
self.assertEqual(proc.returncode, 3)
|
|
# Even after 2 iterations of the full test we still get failures but no XML file,
|
|
# and this is considered a CRASH.
|
|
def test_no_xml_log_written_crash(self):
|
|
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
|
|
self.prepare_env(run_list=["fail_then_pass:2"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 3)
|
|
|
|
# If a test returns success but XML contains failures, it's a CRASH.
|
|
def test_wrong_xml_log_written_1_crash(self):
|
|
logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml")
|
|
write_xml_log(logfile, failure="always_fail")
|
|
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
|
|
self.prepare_env(run_list=["always_pass"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 3)
|
|
# If a test returns failure but XML contains only pass, it's a CRASH.
|
|
def test_wrong_xml_log_written_2_crash(self):
|
|
logfile = os.path.join(TEMPDIR.name, os.path.basename(mock_test) + ".xml")
|
|
write_xml_log(logfile)
|
|
del self.env["QT_MOCK_TEST_XML_TEMPLATE_FILE"]
|
|
self.prepare_env(run_list=["always_fail"])
|
|
proc = self.run2()
|
|
self.assertEqual(proc.returncode, 3)
|
|
|
|
|
|
# Test qt-testrunner script with an existing XML log file:
|
|
# qt-testrunner.py qt_mock_test.py --parse-xml-testlog file.xml
|
|
# qt-testrunner should repeat the testcases that are logged as
|
|
# failures and fail or pass depending on how the testcases behave.
|
|
# Different XML files are generated for the following test cases.
|
|
# + No failure logged. qt-testrunner should exit(0)
|
|
# + The "always_pass" test has failed. qt-testrunner should exit(0).
|
|
# + The "always_fail" test has failed. qt-testrunner should exit(2).
|
|
# + The "always_crash" test has failed. qt-testrunner should exit(2).
|
|
# + The "fail_then_pass:2" test failed. qt-testrunner should exit(0).
|
|
# + The "fail_then_pass:5" test failed. qt-testrunner should exit(2).
|
|
# + The "initTestCase" failed which is listed as NO_RERUN thus
|
|
# qt-testrunner should exit(3).
|
|
class Test_testrunner_with_xml_logfile(unittest.TestCase):
|
|
# Runs before every single test function, creating a unique temp file.
|
|
def setUp(self):
|
|
(_handle, self.xml_file) = mkstemp(
|
|
suffix=".xml", prefix="qt_mock_test-log-",
|
|
dir=TEMPDIR.name)
|
|
if os.path.exists(os.environ["QT_MOCK_TEST_STATE_FILE"]):
|
|
os.remove(os.environ["QT_MOCK_TEST_STATE_FILE"])
|
|
def tearDown(self):
|
|
os.remove(self.xml_file)
|
|
del self.xml_file
|
|
|
|
def test_no_failure(self):
|
|
write_xml_log(self.xml_file, failure=None)
|
|
proc = run_testrunner(self.xml_file)
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_always_pass_failed(self):
|
|
write_xml_log(self.xml_file, failure="always_pass")
|
|
proc = run_testrunner(self.xml_file)
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_always_pass_failed_max_repeats_0(self):
|
|
write_xml_log(self.xml_file, failure="always_pass")
|
|
proc = run_testrunner(self.xml_file,
|
|
extra_args=["--max-repeats", "0"])
|
|
self.assertEqual(proc.returncode, 2)
|
|
def test_always_fail_failed(self):
|
|
write_xml_log(self.xml_file, failure="always_fail")
|
|
proc = run_testrunner(self.xml_file)
|
|
self.assertEqual(proc.returncode, 2)
|
|
# Assert that one of the re-runs was in verbose mode
|
|
matches = re.findall("VERBOSE RUN",
|
|
proc.stdout.decode())
|
|
self.assertEqual(len(matches), 1)
|
|
# Assert that the environment was altered too
|
|
self.assertIn("QT_LOGGING_RULES", proc.stdout.decode())
|
|
def test_always_crash_failed(self):
|
|
write_xml_log(self.xml_file, failure="always_crash")
|
|
proc = run_testrunner(self.xml_file)
|
|
self.assertEqual(proc.returncode, 2)
|
|
def test_fail_then_pass_2_failed(self):
|
|
write_xml_log(self.xml_file, failure="fail_then_pass:2")
|
|
proc = run_testrunner(self.xml_file)
|
|
self.assertEqual(proc.returncode, 0)
|
|
def test_fail_then_pass_5_failed(self):
|
|
write_xml_log(self.xml_file, failure="fail_then_pass:5")
|
|
proc = run_testrunner(self.xml_file)
|
|
self.assertEqual(proc.returncode, 2)
|
|
def test_with_two_failures(self):
|
|
write_xml_log(self.xml_file,
|
|
failure=["always_pass", "fail_then_pass:2"])
|
|
proc = run_testrunner(self.xml_file)
|
|
self.assertEqual(proc.returncode, 0)
|
|
# Check that test output is properly interleaved with qt-testrunner's logging.
|
|
matches = re.findall(r"(PASS|FAIL!).*\n.*Test process exited with code",
|
|
proc.stdout.decode())
|
|
self.assertEqual(len(matches), 4)
|
|
def test_initTestCase_fail_crash(self):
|
|
write_xml_log(self.xml_file, failure="initTestCase")
|
|
proc = run_testrunner(self.xml_file)
|
|
self.assertEqual(proc.returncode, 3)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
DEBUG = False
|
|
if "--debug" in sys.argv:
|
|
sys.argv.remove("--debug")
|
|
DEBUG = True
|
|
|
|
# We set failfast=True as we do not want the test suite to continue if the
|
|
# tests of qt_mock_test failed. The next ones depend on it.
|
|
unittest.main(failfast=True)
|