diff --git a/python_files/tests/pytestadapter/.data/subtests_execution.py b/python_files/tests/pytestadapter/.data/subtests_execution.py new file mode 100644 index 000000000000..11dae3d78631 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/subtests_execution.py @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import unittest + +import pytest + + +# Each subtest reports with its parent's node id, so the parent must fail if any subtest fails, +# no matter which subtest reports first. +def test_pass_then_fail(subtests): + with subtests.test("passes"): + assert True + with subtests.test("fails"): + assert 1 == 2 + + +def test_fail_then_pass(subtests): + with subtests.test("fails"): + assert 1 == 2 + with subtests.test("passes"): + assert True + + +def test_all_pass(subtests): + with subtests.test("first"): + assert True + with subtests.test("second"): + assert True + + +def test_pass_then_body_fails(subtests): + with subtests.test("passes"): + assert True + pytest.fail("the test body fails after its subtests") + + +class TestUnittestSubTest(unittest.TestCase): + def test_pass_then_fail(self): + with self.subTest(i=0): + self.assertEqual(1, 1) + with self.subTest(i=1): + self.assertEqual(1, 2) diff --git a/python_files/tests/pytestadapter/expected_execution_test_output.py b/python_files/tests/pytestadapter/expected_execution_test_output.py index fa6743d0e112..9eda23c18d66 100644 --- a/python_files/tests/pytestadapter/expected_execution_test_output.py +++ b/python_files/tests/pytestadapter/expected_execution_test_output.py @@ -747,3 +747,69 @@ "subtest": None, } } + +# This is the expected output for the subtests_execution.py tests. +# A failed subtest fails its parent test, whichever subtest reports first. +# └── subtests_execution.py +# ├── test_pass_then_fail: failure +# ├── test_fail_then_pass: failure +# ├── test_all_pass: success +# ├── test_pass_then_body_fails: failure +# └── TestUnittestSubTest +# └── test_pass_then_fail: failure +subtests_execution_path = TEST_DATA_PATH / "subtests_execution.py" +subtests_execution_expected_output = { + get_absolute_test_id("subtests_execution.py::test_pass_then_fail", subtests_execution_path): { + "test": get_absolute_test_id( + "subtests_execution.py::test_pass_then_fail", subtests_execution_path + ), + "outcome": FAILURE, + "message": "ERROR MESSAGE", + "traceback": None, + "subtest": None, + }, + get_absolute_test_id("subtests_execution.py::test_fail_then_pass", subtests_execution_path): { + "test": get_absolute_test_id( + "subtests_execution.py::test_fail_then_pass", subtests_execution_path + ), + "outcome": FAILURE, + "message": "ERROR MESSAGE", + "traceback": None, + "subtest": None, + }, + get_absolute_test_id("subtests_execution.py::test_all_pass", subtests_execution_path): { + "test": get_absolute_test_id( + "subtests_execution.py::test_all_pass", subtests_execution_path + ), + "outcome": SUCCESS, + "message": None, + "traceback": None, + "subtest": None, + }, + get_absolute_test_id( + "subtests_execution.py::test_pass_then_body_fails", subtests_execution_path + ): { + "test": get_absolute_test_id( + "subtests_execution.py::test_pass_then_body_fails", subtests_execution_path + ), + "outcome": FAILURE, + "message": "ERROR MESSAGE", + "traceback": None, + "subtest": None, + }, +} + +unittest_subtest_execution_expected_output = { + get_absolute_test_id( + "subtests_execution.py::TestUnittestSubTest::test_pass_then_fail", subtests_execution_path + ): { + "test": get_absolute_test_id( + "subtests_execution.py::TestUnittestSubTest::test_pass_then_fail", + subtests_execution_path, + ), + "outcome": FAILURE, + "message": "ERROR MESSAGE", + "traceback": None, + "subtest": None, + } +} diff --git a/python_files/tests/pytestadapter/test_execution.py b/python_files/tests/pytestadapter/test_execution.py index 95a66e0e7b87..a5acc4f38737 100644 --- a/python_files/tests/pytestadapter/test_execution.py +++ b/python_files/tests/pytestadapter/test_execution.py @@ -1,5 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import importlib.util import json import os import pathlib @@ -199,6 +200,26 @@ def test_rootdir_specified(): expected_execution_test_output.skip_test_fixture_execution_expected_output, id="skip_test_fixture", ), + pytest.param( + [ + "subtests_execution.py::test_pass_then_fail", + "subtests_execution.py::test_fail_then_pass", + "subtests_execution.py::test_all_pass", + "subtests_execution.py::test_pass_then_body_fails", + ], + expected_execution_test_output.subtests_execution_expected_output, + id="pytest_subtests", + marks=pytest.mark.skipif( + pytest.version_tuple < (9, 0) + and importlib.util.find_spec("pytest_subtests") is None, + reason="needs pytest 9 or the pytest-subtests plugin", + ), + ), + pytest.param( + ["subtests_execution.py::TestUnittestSubTest::test_pass_then_fail"], + expected_execution_test_output.unittest_subtest_execution_expected_output, + id="unittest_subtest", + ), ], ) def test_pytest_execution(test_ids, expected_const): diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 88f6327f6e8b..7e4a8bff6d62 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -90,6 +90,10 @@ def __init__(self, message): IS_DISCOVERY = False map_id_to_path = {} collected_tests_so_far = set() +# Failure messages of subtests, keyed by the absolute id of their parent test. Subtest reports +# share the parent's node id, so they are collected here and folded into the parent's single +# outcome when its own call report arrives. +subtest_failures: dict[str, list[str]] = {} TEST_RUN_PIPE = os.getenv("TEST_RUN_PIPE") PROJECT_ROOT_PATH = os.getenv( "PROJECT_ROOT_PATH" @@ -202,6 +206,9 @@ def pytest_exception_interact(node, call, report): else: ERRORS.append(report.longreprtext + "\n Check Python Logs for more details.") else: + if is_subtest_report(report): + # Folded into the parent's outcome by pytest_report_teststatus. + return # If during execution, send this data that the given node failed. report_value = "error" if call.excinfo.typename == "AssertionError": @@ -303,6 +310,18 @@ class TestRunResultDict(Dict[str, Dict[str, TestOutcome]]): tests: dict[str, TestOutcome] +def is_subtest_report(report) -> bool: + """Return True for a report produced by a subtest rather than by the test itself. + + pytest 9 reports `subtests.test()` blocks and unittest `subTest()` blocks as + `SubtestReport`; the pytest-subtests plugin used on older pytest versions reports + them as `SubTestReport`. Both are call phase reports that reuse the parent's node id. + """ + return type(report).__name__ in ("SubtestReport", "SubTestReport") and hasattr( + report, "context" + ) + + @pytest.hookimpl(hookwrapper=True, trylast=True) def pytest_report_teststatus(report, config): # noqa: ARG001 """A pytest hook that is called when a test is called. @@ -332,8 +351,25 @@ def pytest_report_teststatus(report, config): # noqa: ARG001 node_path = cwd # Calculate the absolute test id and use this as the ID moving forward. absolute_node_id = get_absolute_test_id(report.nodeid, node_path) - if absolute_node_id not in collected_tests_so_far: + if is_subtest_report(report): + # A subtest report carries its parent's node id, so it must not publish or claim that + # id; its failure is kept for the parent's own call report. The terminal summary can + # call this hook again after the parent is published, hence the id check. + if report.failed and absolute_node_id not in collected_tests_so_far: + subtest_failures.setdefault(absolute_node_id, []).append( + f"{report.head_line}\n{report.longreprtext}" + ) + elif absolute_node_id not in collected_tests_so_far: collected_tests_so_far.add(absolute_node_id) + failed_subtests = subtest_failures.pop(absolute_node_id, None) + if failed_subtests: + # The parent's report can still say passed here: pytest 9 marks it failed only + # in its own hook, which runs after this code, and never does for unittest + # subTest() or the pytest-subtests plugin. Any failed subtest fails the parent. + if message: + failed_subtests.append(message) + report_value = "failure" + message = "\n\n".join(failed_subtests) item_result = create_test_outcome( absolute_node_id, report_value,