testcheck: sort inline assertions (#15290)

The order in which inline error assertions are collected into the
`TestCase.output` (the "expected") can differ from the actual errors'
order. Specifically, it's the order of the files themselves (as they're
traversed during build), rather than individual lines.

To allow inline assertions across multiple modules, we can sort the
collected errors to match the actual errors (only when it concerns
module order; within the modules the order remains stable).

Author: ikonst

mypy/test/data.py

@@ -208,6 +208,7 @@ def parse_test_case(case: DataDrivenTestCase) -> None:
                 ).format(passnum, case.file, first_item.line)
             )
 
+    output_inline_start = len(output)
     input = first_item.data
     expand_errors(input, output, "main")
     for file_path, contents in files:
@@ -225,6 +226,7 @@ def parse_test_case(case: DataDrivenTestCase) -> None:
 
     case.input = input
     case.output = output
+    case.output_inline_start = output_inline_start
     case.output2 = output2
     case.last_line = case.line + item.line + len(item.data) - 2
     case.files = files
@@ -246,6 +248,7 @@ class DataDrivenTestCase(pytest.Item):
 
     input: list[str]
     output: list[str]  # Output for the first pass
+    output_inline_start: int
     output2: dict[int, list[str]]  # Output for runs 2+, indexed by run number
 
     # full path of test suite

mypy/test/testcheck.py

@@ -84,6 +84,19 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         else:
             self.run_case_once(testcase)
 
+    def _sort_output_if_needed(self, testcase: DataDrivenTestCase, a: list[str]) -> None:
+        idx = testcase.output_inline_start
+        if not testcase.files or idx == len(testcase.output):
+            return
+
+        def _filename(_msg: str) -> str:
+            return _msg.partition(":")[0]
+
+        file_weights = {file: idx for idx, file in enumerate(_filename(msg) for msg in a)}
+        testcase.output[idx:] = sorted(
+            testcase.output[idx:], key=lambda msg: file_weights.get(_filename(msg), -1)
+        )
+
     def run_case_once(
         self,
         testcase: DataDrivenTestCase,
@@ -163,21 +176,20 @@ def run_case_once(
             a = normalize_error_messages(a)
 
         # Make sure error messages match
-        if incremental_step == 0:
-            # Not incremental
-            msg = "Unexpected type checker output ({}, line {})"
+        if incremental_step < 2:
+            if incremental_step == 1:
+                msg = "Unexpected type checker output in incremental, run 1 ({}, line {})"
+            else:
+                assert incremental_step == 0
+                msg = "Unexpected type checker output ({}, line {})"
+            self._sort_output_if_needed(testcase, a)
             output = testcase.output
-        elif incremental_step == 1:
-            msg = "Unexpected type checker output in incremental, run 1 ({}, line {})"
-            output = testcase.output
-        elif incremental_step > 1:
+        else:
             msg = (
                 f"Unexpected type checker output in incremental, run {incremental_step}"
                 + " ({}, line {})"
             )
             output = testcase.output2.get(incremental_step, [])
-        else:
-            raise AssertionError()
 
         if output != a and testcase.config.getoption("--update-data", False):
             update_testcase_output(testcase, a)

test-data/unit/check-basic.test

@@ -493,3 +493,13 @@ class A:
 
 [file test.py]
 def foo(s: str) -> None: ...
+
+[case testInlineAssertions]
+import a, b
+s1: str = 42  # E: Incompatible types in assignment (expression has type "int", variable has type "str")
+[file a.py]
+s2: str = 42  # E: Incompatible types in assignment (expression has type "int", variable has type "str")
+[file b.py]
+s3: str = 42  # E: Incompatible types in assignment (expression has type "int", variable has type "str")
+[file c.py]
+s3: str = 'foo'