Add docs and make junit xml filename more human-readable

sklam · Mar 4, 2024 · Feb 12, 2024 · Feb 13, 2024 · Feb 13, 2024 · Feb 13, 2024
commit 08132eeb9aa925e9b95cd2ee24dc7f891a3e2234
diff --git a/buildscripts/azure/azure-linux-macos.yml b/buildscripts/azure/azure-linux-macos.yml
@@ -54,5 +54,5 @@ jobs:
       condition: succeededOrFailed()
       displayName: Publish Test Results
       inputs:
-        testResultsFiles: 'junit_testsuites_*.xml'
+        testResultsFiles: 'junit_numba_*.xml'
         mergeTestResults: true
diff --git a/buildscripts/azure/azure-windows.yml b/buildscripts/azure/azure-windows.yml
@@ -69,5 +69,5 @@ jobs:
       condition: succeededOrFailed()
       displayName: Publish Test Results
       inputs:
-        testResultsFiles: 'junit_testsuites_*.xml'
+        testResultsFiles: 'junit_numba_*.xml'
         mergeTestResults: true
diff --git a/docs/source/developer/contributing.rst b/docs/source/developer/contributing.rst
@@ -214,6 +214,9 @@ instead. For example::
     $ NUMBA_USE_TYPEGUARD=1 python runtests.py
 
 
+See also: :ref:`Debugging the Test Suite <developer-testsuite>`
+
+
 Running coverage
 ''''''''''''''''
 

diff --git a/docs/source/developer/index.rst b/docs/source/developer/index.rst
@@ -27,6 +27,7 @@ Developer Manual
    literal.rst
    llvm_timings.rst
    debugging.rst
+   testsuite.rst
    event_api.rst
    target_extension.rst
    bytecode.rst

diff --git a/docs/source/developer/testsuite.rst b/docs/source/developer/testsuite.rst
@@ -0,0 +1,77 @@
+.. _developer-testsuite:
+
+==============================
+Debugging the Numba Test Suite
+==============================
+
+The Numba test suite provides some useful features for debugging test failures 
+or crashes.
+
+Generating JUnit XML Output
+---------------------------
+
+You can generate a JUnit-compatible XML report by running the tests with the 
+``--junit`` flag:
+
+.. code-block:: bash
+
+    $ python -m numba.runtests <args> --junit
+
+This will produce a ``junit_numba_<timestamp>.xml`` file containing the test results.
+
+The XML contains some extra metadata as ``testcase`` properties:
+
+- ``pid``: The ID of the worker process running each test case
+- ``start_time``: The start time for each test case relative to when the worker 
+  process started
+
+
+Example of the XML file:
+
+.. code-block:: xml
+
+    <?xml version='1.0' encoding='utf-8'?>
+    <testsuites time="20.745398832">
+        <testsuite name="numba_testsuite">
+            <testcase name="TestUsecases.test_andor" classname="numba.tests.test_usecases"
+                time="1.7033392500000002">
+                <properties>
+                    <property name="pid" value="73274" />
+                    <property name="start_time" value="1.337771667" />
+                </properties>
+            </testcase>
+            <testcase name="TestUsecases.test_blackscholes_cnd" classname="numba.tests.test_usecases"
+                time="1.723800292">
+                <properties>
+                    <property name="pid" value="73275" />
+                    <property name="start_time" value="1.3350285" />
+                </properties>
+            </testcase>
+            ...
+        </testsuite>
+    </testsuites>
+
+
+Debugging Timeouts caused by Crashes
+------------------------------------
+
+If the test suite times out, it may indicate a crashed worker process. 
+In this case, the timeout error will show:
+
+- The active tests that didn't finish before the timeout
+- For each affected process ID, the list of tests that process was running
+
+For example:
+
+.. code-block:: none
+
+  TimeoutError: Active tests didn't finish before timeout:
+  - test_foo [PID 1234]
+
+  Tests ran by each affected process: 
+  - [PID 1234]: test_bar test_baz test_foo
+
+This information can help reproduce crashes that depend on the test execution 
+order or state within a process. You can retry just the tests that were running 
+in the crashed process.
+
diff --git a/numba/testing/main.py b/numba/testing/main.py
@@ -16,9 +16,9 @@
 from functools import lru_cache
 from io import StringIO
 from unittest import result, runner, signals, suite, loader, case
+from datetime import datetime
 
 from .loader import TestLoader
-from numba.core import config
 
 try:
     from multiprocessing import TimeoutError
@@ -979,7 +979,8 @@ def run(self, test):
             self.stream.write(f"Total test runtime (seconds): {result.test_runtime}\n")
         # Write junit xml
         if self.write_junit:
-            filename = f"junit_testsuites_{time.time()}.xml"
+            tstamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+            filename = f"junit_numba_{tstamp}.xml"
             print("Writing junit xml file:", filename)
             result.write_junit_xml(filename)
         return result