[WIP] Write a JUnit XML report from databricks pipelines test

.nextchanges/cli/pipelines-test-junit.md

@@ -0,0 +1 @@
+* Add `--junit-xml` to `databricks pipelines test`, which writes the test results as a JUnit XML report. ([#7299](https://github.com/databricks/cli/pull/7299))

acceptance/pipelines/test/arguments/out.flag-error.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="unsupported pytest argument &#34;-x&#34;: only -k and -m are supported">unsupported pytest argument &#34;-x&#34;: only -k and -m are supported</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/out.no-wait.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="1234" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="--junit-xml cannot be used with --no-wait because no results are available when the command returns">--junit-xml cannot be used with --no-wait because no results are available when the command returns</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/arguments/output.txt

@@ -28,10 +28,16 @@ collected and none failed or errored. --no-wait returns after the update starts.
 --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume
 to read the results later. To stop the update, run 'pipelines stop'.
 
+--junit-xml writes one entry per test with its outcome and duration. An
+incomplete run, a run without tests, or a timeout adds a failing entry that
+names the cause. With workspace.sync_mode immutable, test names keep the path
+of the deployed snapshot, which changes with each deployment.
+
 Usage:
   databricks pipelines test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...]
 
 Pipeline Test Flags:
+      --junit-xml string         Write the results as a JUnit XML report to this file, replacing it.
       --no-wait                  Return after the test update starts, without waiting for results.
       --pipeline-id string       ID of the pipeline to test, instead of a PIPELINE_KEY.
       --pytest-arg stringArray   Argument to pass to pytest (repeatable). Only -k and -m are supported.
@@ -82,6 +88,13 @@ Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that p
 >>> musterr [CLI] pipelines test --pipeline-id 1234 --resume 
 Error: --resume "" is not an update ID; pass the update ID that pipelines test printed
 
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml
+Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns
+
+=== An invalid flag replaces an older report
+>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml
+Error: unsupported pytest argument "-x": only -k and -m are supported
+
 === JSON output is not supported
 >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json
 Error: --output json is not supported by pipelines test

acceptance/pipelines/test/arguments/script

@@ -17,6 +17,11 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait
 trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m
 trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1"
 trace musterr $CLI pipelines test --pipeline-id 1234 --resume ""
+trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml
+
+title "An invalid flag replaces an older report"
+echo '<testsuites><testsuite name="1234" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.flag-error.xml
+trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml
 
 title "JSON output is not supported"
 trace musterr $CLI pipelines test --pipeline-id 1234 -o json

acceptance/pipelines/test/results/out.failures.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="4" failures="1" errors="1" skipped="1">
+    <testcase classname="tests.test_orders" name="test_valid_order" time="1.000"></testcase>
+    <testcase classname="tests.test_orders" name="test_backfill_fails" time="1.000">
+      <failure message="AssertionError: assert 100 == 150">AssertionError: assert 100 == 150</failure>
+    </testcase>
+    <testcase classname="tests.test_orders" name="test_legacy_format_skipped" time="1.000">
+      <skipped></skipped>
+    </testcase>
+    <testcase classname="tests.test_orders" name="test_fixture_error" time="1.000">
+      <error message="RuntimeError: fixture &#39;orders&#39; setup failed">RuntimeError: fixture &#39;orders&#39; setup failed</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/out.incomplete.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="4" failures="0" errors="3" skipped="0">
+    <testcase classname="tests.test_compute" name="test_small_order" time="1.000"></testcase>
+    <testcase classname="tests.test_compute" name="test_crash_on_large_order">
+      <error message="no result received"></error>
+    </testcase>
+    <testcase classname="tests.test_compute" name="test_last_order">
+      <error message="no result received"></error>
+    </testcase>
+    <testcase name="update [UUID]">
+      <error message="RUN INCOMPLETE: the update failed before all test results were received.">RUN INCOMPLETE: the update failed before all test results were received.&#xA;update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly.</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/out.late-failure.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="3" failures="1" errors="1" skipped="0">
+    <testcase classname="tests.test_late_failure" name="test_backfill_fails" time="1.000">
+      <failure message="AssertionError: assert 100 == 150">AssertionError: assert 100 == 150</failure>
+    </testcase>
+    <testcase classname="tests.test_late_failure" name="test_breaks_update_after_summary" time="1.000"></testcase>
+    <testcase name="update [UUID]">
+      <error message="update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified.">update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified.</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/out.no-tests.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="update [UUID]">
+      <error message="No tests were collected.">No tests were collected.</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/out.passed.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="1" failures="0" errors="0" skipped="0">
+    <testcase classname="tests.test_orders" name="test_valid_order" time="1.000"></testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/out.update-failed.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="2" failures="0" errors="1" skipped="0">
+    <testcase classname="tests.test_delivery" name="test_breaks_update_after_summary" time="1.000"></testcase>
+    <testcase name="update [UUID]">
+      <error message="update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified.">update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified.</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/results/output.txt

@@ -7,7 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID]
 
 === All selected tests pass
->>> [CLI] pipelines test my_pipeline tests/test_orders.py::test_valid_order
+>>> [CLI] pipelines test my_pipeline tests/test_orders.py::test_valid_order --junit-xml out.passed.xml
 Recommendation: This command runs the last deployed version of the code
 
 If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
@@ -19,6 +19,7 @@ Running tests...
 tests/test_orders.py::test_valid_order  PASSED (1.00s)
 
 1 test: 1 passed, 0 failed, 0 skipped, 0 errors
+JUnit report: out.passed.xml
 
 >>> print_requests.py --get //events
 {
@@ -59,7 +60,7 @@ tests/test_orders.py::test_valid_order  PASSED (1.00s)
 }
 
 === Failed, errored, and skipped tests
->>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py
+>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py --junit-xml out.failures.xml
 Recommendation: This command runs the last deployed version of the code
 
 If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
@@ -80,9 +81,10 @@ tests/test_orders.py::test_fixture_error
   RuntimeError: fixture 'orders' setup failed
 
 4 tests: 1 passed, 1 failed, 1 skipped, 1 error
+JUnit report: out.failures.xml
 
 === No tests collected
->>> musterr [CLI] pipelines test my_pipeline tests/helpers.py
+>>> musterr [CLI] pipelines test my_pipeline tests/helpers.py --junit-xml out.no-tests.xml
 Recommendation: This command runs the last deployed version of the code
 
 If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
@@ -93,9 +95,10 @@ Waiting for compute...
 Running tests...
 
 No tests were collected.
+JUnit report: out.no-tests.xml
 
 === The update fails after all tests passed
->>> musterr [CLI] pipelines test my_pipeline tests/test_delivery.py
+>>> musterr [CLI] pipelines test my_pipeline tests/test_delivery.py --junit-xml out.update-failed.xml
 Recommendation: This command runs the last deployed version of the code
 
 If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
@@ -107,10 +110,11 @@ Running tests...
 tests/test_delivery.py::test_breaks_update_after_summary  PASSED (1.00s)
 
 1 test: 1 passed, 0 failed, 0 skipped, 0 errors
+JUnit report: out.update-failed.xml
 Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified.
 
 === The update fails with an error of its own after a test failed
->>> musterr [CLI] pipelines test my_pipeline tests/test_late_failure.py
+>>> musterr [CLI] pipelines test my_pipeline tests/test_late_failure.py --junit-xml out.late-failure.xml
 Recommendation: This command runs the last deployed version of the code
 
 If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
@@ -127,10 +131,11 @@ tests/test_late_failure.py::test_backfill_fails
   AssertionError: assert 100 == 150
 
 2 tests: 1 passed, 1 failed, 0 skipped, 0 errors
+JUnit report: out.late-failure.xml
 Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified.
 
 === Compute terminates before all results are received
->>> musterr [CLI] pipelines test my_pipeline tests/test_compute.py
+>>> musterr [CLI] pipelines test my_pipeline tests/test_compute.py --junit-xml out.incomplete.xml
 Recommendation: This command runs the last deployed version of the code
 
 If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.
@@ -143,4 +148,5 @@ tests/test_compute.py::test_small_order  PASSED (1.00s)
 
 RUN INCOMPLETE: the update failed before all test results were received.
 Results received: 1 passed.
+JUnit report: out.incomplete.xml
 Error: update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly.

acceptance/pipelines/test/results/script

@@ -2,21 +2,21 @@ trace $CLI pipelines deploy
 rm "$OUT_REQUESTS"
 
 title "All selected tests pass"
-trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order
+trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order --junit-xml out.passed.xml
 trace print_requests.py --get //events
 
 title "Failed, errored, and skipped tests"
-trace musterr $CLI pipelines test my_pipeline tests/test_orders.py
+trace musterr $CLI pipelines test my_pipeline tests/test_orders.py --junit-xml out.failures.xml
 
 title "No tests collected"
-trace musterr $CLI pipelines test my_pipeline tests/helpers.py
+trace musterr $CLI pipelines test my_pipeline tests/helpers.py --junit-xml out.no-tests.xml
 
 title "The update fails after all tests passed"
-trace musterr $CLI pipelines test my_pipeline tests/test_delivery.py
+trace musterr $CLI pipelines test my_pipeline tests/test_delivery.py --junit-xml out.update-failed.xml
 
 title "The update fails with an error of its own after a test failed"
-trace musterr $CLI pipelines test my_pipeline tests/test_late_failure.py
+trace musterr $CLI pipelines test my_pipeline tests/test_late_failure.py --junit-xml out.late-failure.xml
 
 title "Compute terminates before all results are received"
-trace musterr $CLI pipelines test my_pipeline tests/test_compute.py
+trace musterr $CLI pipelines test my_pipeline tests/test_compute.py --junit-xml out.incomplete.xml
 rm "$OUT_REQUESTS"

acceptance/pipelines/test/run-control/out.invalid.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="reference does not exist: ${var.missing}">reference does not exist: ${var.missing}</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/run-control/out.not-test-update.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="update [UUID] is not a test update; pass the update ID that pipelines test printed">update [UUID] is not a test update; pass the update ID that pipelines test printed</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/run-control/out.outside.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="" tests="1" failures="0" errors="1" skipped="0">
+    <testcase name="pipelines test">
+      <error message="outside a project, identify the pipeline with --pipeline-id">outside a project, identify the pipeline with --pipeline-id</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/run-control/out.resumed.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="2" failures="0" errors="2" skipped="0">
+    <testcase classname="tests.test_slow" name="test_hang_on_lock">
+      <error message="no result received"></error>
+    </testcase>
+    <testcase name="update [UUID]">
+      <error message="RUN INCOMPLETE: the update was canceled before all test results were received.">RUN INCOMPLETE: the update was canceled before all test results were received.&#xA;update [UUID] is CANCELED</error>
+    </testcase>
+  </testsuite>
+</testsuites>

acceptance/pipelines/test/run-control/output.txt

@@ -33,11 +33,25 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure
 Pipeline: my_pipeline ([UUID])
 Update: [UUID]
 
->>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s
+>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s --junit-xml out.resumed.xml
 Pipeline: my_pipeline ([UUID])
 Update: [UUID]
 Running tests...
-Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --resume [UUID]
+JUnit report: out.resumed.xml
+Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID]
+
+>>> cat out.resumed.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="my_pipeline" tests="2" failures="0" errors="2" skipped="0">
+    <testcase classname="tests.test_slow" name="test_hang_on_lock">
+      <error message="no result received"></error>
+    </testcase>
+    <testcase name="update [UUID]">
+      <error message="stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID]">stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID]</error>
+    </testcase>
+  </testsuite>
+</testsuites>
 
 === The command to read the results keeps the profile
 >>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s -p run-control
@@ -51,14 +65,29 @@ Error: stopped waiting for update [UUID] after 3s; the update keeps running, to
 Stopping my_pipeline...
 my_pipeline has been stopped.
 
->>> musterr [CLI] pipelines test my_pipeline --resume [UUID]
+>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --junit-xml out.resumed.xml
 Pipeline: my_pipeline ([UUID])
 Update: [UUID]
 
 RUN INCOMPLETE: the update was canceled before all test results were received.
 Results received: none.
+JUnit report: out.resumed.xml
 Error: update [UUID] is CANCELED
 
-=== --resume rejects an update that is not a test update
->>> musterr [CLI] pipelines test my_pipeline --resume [UUID]
+=== --resume rejects an update that is not a test update, and an older report is replaced
+>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --junit-xml out.not-test-update.xml
 Error: update [UUID] is not a test update; pass the update ID that pipelines test printed
+
+=== Outside a project, an older passing report is replaced
+Error: outside a project, identify the pipeline with --pipeline-id
+
+=== A project with errors names the first error in the report
+>>> musterr [CLI] pipelines test my_pipeline --junit-xml ../out.invalid.xml
+Error: reference does not exist: ${var.missing}
+
+
+=== When the report cannot be written either, the write error is shown
+>>> musterr [CLI] pipelines test my_pipeline --junit-xml missing/report.xml
+Error: reference does not exist: ${var.missing}
+
+Error: failed to write JUnit report: open missing/report.xml: no such file or directory

acceptance/pipelines/test/run-control/script

@@ -11,7 +11,8 @@ title "--timeout stops waiting but not the update"
 trace $CLI pipelines test my_pipeline tests/test_slow.py --no-wait | tee submit.txt
 hang_update_id=$(sed -n 's/^Update: //p' submit.txt)
 rm submit.txt
-trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout 3s
+trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout 3s --junit-xml out.resumed.xml
+trace cat out.resumed.xml
 
 title "The command to read the results keeps the profile"
 cat > .databrickscfg <<EOF
@@ -26,9 +27,30 @@ rm .databrickscfg
 
 title "Stopping the pipeline cancels the test update"
 trace $CLI pipelines stop my_pipeline
-trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id
+trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --junit-xml out.resumed.xml
 
-title "--resume rejects an update that is not a test update"
+title "--resume rejects an update that is not a test update, and an older report is replaced"
 pipeline_id=$($CLI bundle summary -o json | jq -r '.resources.pipelines.my_pipeline.id')
 regular_update_id=$($CLI pipelines start-update $pipeline_id | jq -r .update_id)
-trace musterr $CLI pipelines test my_pipeline --resume $regular_update_id
+echo '<testsuites><testsuite name="my_pipeline" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.not-test-update.xml
+trace musterr $CLI pipelines test my_pipeline --resume $regular_update_id --junit-xml out.not-test-update.xml
+
+title "Outside a project, an older passing report is replaced\n"
+echo '<testsuites><testsuite name="my_pipeline" tests="1"><testcase name="test_old"></testcase></testsuite></testsuites>' > out.outside.xml
+here=$PWD
+outside=$(mktemp -d)
+cd $outside
+musterr $CLI pipelines test --junit-xml "$here/out.outside.xml"
+cd $here
+rm -r $outside
+
+title "A project with errors names the first error in the report"
+mkdir invalid
+printf 'bundle:\n  name: invalid\nresources:\n  pipelines:\n    my_pipeline:\n      name: ${var.missing}\n' > invalid/databricks.yml
+cd invalid
+trace musterr $CLI pipelines test my_pipeline --junit-xml ../out.invalid.xml
+
+title "When the report cannot be written either, the write error is shown"
+trace musterr $CLI pipelines test my_pipeline --junit-xml missing/report.xml
+cd ..
+rm -r invalid