From 34b6330cf699e6041889cd76b2c4ac72705a46c1 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:44:37 +0000 Subject: [PATCH 1/9] Write a JUnit XML report from databricks pipelines test --- .nextchanges/cli/pipelines-test-junit.md | 1 + .../pipelines/test/arguments/output.txt | 8 + acceptance/pipelines/test/arguments/script | 1 + .../pipelines/test/results/out.failures.xml | 15 ++ .../pipelines/test/results/out.incomplete.xml | 15 ++ .../test/results/out.late-failure.xml | 12 ++ .../pipelines/test/results/out.no-tests.xml | 8 + .../test/results/out.update-failed.xml | 9 + acceptance/pipelines/test/results/output.txt | 15 +- acceptance/pipelines/test/results/script | 10 +- .../test/run-control/out.invalid.xml | 8 + .../test/run-control/out.outside.xml | 8 + .../test/run-control/out.resumed.xml | 11 ++ .../pipelines/test/run-control/output.txt | 35 +++- acceptance/pipelines/test/run-control/script | 25 ++- .../test/submit-error/out.start-failed.xml | 8 + .../pipelines/test/submit-error/output.txt | 4 + acceptance/pipelines/test/submit-error/script | 4 + cmd/pipelines/test.go | 59 +++++- cmd/pipelines/test_junit.go | 187 ++++++++++++++++++ cmd/pipelines/test_junit_test.go | 137 +++++++++++++ cmd/pipelines/test_test.go | 7 + 22 files changed, 563 insertions(+), 24 deletions(-) create mode 100644 .nextchanges/cli/pipelines-test-junit.md create mode 100644 acceptance/pipelines/test/results/out.failures.xml create mode 100644 acceptance/pipelines/test/results/out.incomplete.xml create mode 100644 acceptance/pipelines/test/results/out.late-failure.xml create mode 100644 acceptance/pipelines/test/results/out.no-tests.xml create mode 100644 acceptance/pipelines/test/results/out.update-failed.xml create mode 100644 acceptance/pipelines/test/run-control/out.invalid.xml create mode 100644 acceptance/pipelines/test/run-control/out.outside.xml create mode 100644 acceptance/pipelines/test/run-control/out.resumed.xml create mode 100644 acceptance/pipelines/test/submit-error/out.start-failed.xml create mode 100644 cmd/pipelines/test_junit.go create mode 100644 cmd/pipelines/test_junit_test.go diff --git a/.nextchanges/cli/pipelines-test-junit.md b/.nextchanges/cli/pipelines-test-junit.md new file mode 100644 index 00000000000..fff9ab2c679 --- /dev/null +++ b/.nextchanges/cli/pipelines-test-junit.md @@ -0,0 +1 @@ +* Add `--junit-xml` to `databricks pipelines test`, which writes the test results as a JUnit XML report. ([#7273](https://github.com/databricks/cli/pull/7273)) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index c2d17764f80..780ee4e6cb3 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -28,10 +28,15 @@ collected and none failed or errored. --no-wait returns after the update starts. --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume to read the results later. To stop the update, run 'pipelines stop'. +--junit-xml writes one entry per test with its outcome and duration. An +incomplete run, a run without tests, or a timeout adds a failing entry that +names the cause. + Usage: databricks pipelines test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...] Pipeline Test Flags: + --junit-xml string Write the results as a JUnit XML report to this file, replacing it. --no-wait Return after the test update starts, without waiting for results. --pipeline-id string ID of the pipeline to test, instead of a PIPELINE_KEY. --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. @@ -82,6 +87,9 @@ Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that p >>> musterr [CLI] pipelines test --pipeline-id 1234 --resume Error: --resume "" is not an update ID; pass the update ID that pipelines test printed +>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml report.xml +Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns + === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index ae91c307706..25ad2cce729 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -17,6 +17,7 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1" trace musterr $CLI pipelines test --pipeline-id 1234 --resume "" +trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml report.xml title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/results/out.failures.xml b/acceptance/pipelines/test/results/out.failures.xml new file mode 100644 index 00000000000..64da8c3634d --- /dev/null +++ b/acceptance/pipelines/test/results/out.failures.xml @@ -0,0 +1,15 @@ + + + + + + AssertionError: assert 100 == 150 + + + + + + RuntimeError: fixture 'orders' setup failed + + + diff --git a/acceptance/pipelines/test/results/out.incomplete.xml b/acceptance/pipelines/test/results/out.incomplete.xml new file mode 100644 index 00000000000..096c3289d8a --- /dev/null +++ b/acceptance/pipelines/test/results/out.incomplete.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + RUN INCOMPLETE: the update failed before all test results were received. update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. + + + diff --git a/acceptance/pipelines/test/results/out.late-failure.xml b/acceptance/pipelines/test/results/out.late-failure.xml new file mode 100644 index 00000000000..110219d1016 --- /dev/null +++ b/acceptance/pipelines/test/results/out.late-failure.xml @@ -0,0 +1,12 @@ + + + + + AssertionError: assert 100 == 150 + + + + update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + + + diff --git a/acceptance/pipelines/test/results/out.no-tests.xml b/acceptance/pipelines/test/results/out.no-tests.xml new file mode 100644 index 00000000000..68f35e0ab2c --- /dev/null +++ b/acceptance/pipelines/test/results/out.no-tests.xml @@ -0,0 +1,8 @@ + + + + + No tests were collected. + + + diff --git a/acceptance/pipelines/test/results/out.update-failed.xml b/acceptance/pipelines/test/results/out.update-failed.xml new file mode 100644 index 00000000000..87562fd44a5 --- /dev/null +++ b/acceptance/pipelines/test/results/out.update-failed.xml @@ -0,0 +1,9 @@ + + + + + + update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + + + diff --git a/acceptance/pipelines/test/results/output.txt b/acceptance/pipelines/test/results/output.txt index 6f3ccfb2842..9a27e25b31f 100644 --- a/acceptance/pipelines/test/results/output.txt +++ b/acceptance/pipelines/test/results/output.txt @@ -59,7 +59,7 @@ tests/test_orders.py::test_valid_order PASSED (1.00s) } === Failed, errored, and skipped tests ->>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py --junit-xml out.failures.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -80,9 +80,10 @@ tests/test_orders.py::test_fixture_error RuntimeError: fixture 'orders' setup failed 4 tests: 1 passed, 1 failed, 1 skipped, 1 error +JUnit report: out.failures.xml === No tests collected ->>> musterr [CLI] pipelines test my_pipeline tests/helpers.py +>>> musterr [CLI] pipelines test my_pipeline tests/helpers.py --junit-xml out.no-tests.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -93,9 +94,10 @@ Waiting for compute... Running tests... No tests were collected. +JUnit report: out.no-tests.xml === The update fails after all tests passed ->>> musterr [CLI] pipelines test my_pipeline tests/test_delivery.py +>>> musterr [CLI] pipelines test my_pipeline tests/test_delivery.py --junit-xml out.update-failed.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -107,10 +109,11 @@ Running tests... tests/test_delivery.py::test_breaks_update_after_summary PASSED (1.00s) 1 test: 1 passed, 0 failed, 0 skipped, 0 errors +JUnit report: out.update-failed.xml Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. === The update fails with an error of its own after a test failed ->>> musterr [CLI] pipelines test my_pipeline tests/test_late_failure.py +>>> musterr [CLI] pipelines test my_pipeline tests/test_late_failure.py --junit-xml out.late-failure.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -127,10 +130,11 @@ tests/test_late_failure.py::test_backfill_fails AssertionError: assert 100 == 150 2 tests: 1 passed, 1 failed, 0 skipped, 0 errors +JUnit report: out.late-failure.xml Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. === Compute terminates before all results are received ->>> musterr [CLI] pipelines test my_pipeline tests/test_compute.py +>>> musterr [CLI] pipelines test my_pipeline tests/test_compute.py --junit-xml out.incomplete.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -143,4 +147,5 @@ tests/test_compute.py::test_small_order PASSED (1.00s) RUN INCOMPLETE: the update failed before all test results were received. Results received: 1 passed. +JUnit report: out.incomplete.xml Error: update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. diff --git a/acceptance/pipelines/test/results/script b/acceptance/pipelines/test/results/script index e19daac850c..a1f18dcd9b0 100644 --- a/acceptance/pipelines/test/results/script +++ b/acceptance/pipelines/test/results/script @@ -6,17 +6,17 @@ trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order trace print_requests.py --get //events title "Failed, errored, and skipped tests" -trace musterr $CLI pipelines test my_pipeline tests/test_orders.py +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py --junit-xml out.failures.xml title "No tests collected" -trace musterr $CLI pipelines test my_pipeline tests/helpers.py +trace musterr $CLI pipelines test my_pipeline tests/helpers.py --junit-xml out.no-tests.xml title "The update fails after all tests passed" -trace musterr $CLI pipelines test my_pipeline tests/test_delivery.py +trace musterr $CLI pipelines test my_pipeline tests/test_delivery.py --junit-xml out.update-failed.xml title "The update fails with an error of its own after a test failed" -trace musterr $CLI pipelines test my_pipeline tests/test_late_failure.py +trace musterr $CLI pipelines test my_pipeline tests/test_late_failure.py --junit-xml out.late-failure.xml title "Compute terminates before all results are received" -trace musterr $CLI pipelines test my_pipeline tests/test_compute.py +trace musterr $CLI pipelines test my_pipeline tests/test_compute.py --junit-xml out.incomplete.xml rm "$OUT_REQUESTS" diff --git a/acceptance/pipelines/test/run-control/out.invalid.xml b/acceptance/pipelines/test/run-control/out.invalid.xml new file mode 100644 index 00000000000..ff48bd9dac9 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.invalid.xml @@ -0,0 +1,8 @@ + + + + + reference does not exist: ${var.missing} + + + diff --git a/acceptance/pipelines/test/run-control/out.outside.xml b/acceptance/pipelines/test/run-control/out.outside.xml new file mode 100644 index 00000000000..1e1f250b228 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.outside.xml @@ -0,0 +1,8 @@ + + + + + outside a project, identify the pipeline with --pipeline-id + + + diff --git a/acceptance/pipelines/test/run-control/out.resumed.xml b/acceptance/pipelines/test/run-control/out.resumed.xml new file mode 100644 index 00000000000..ff8b40372a2 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.resumed.xml @@ -0,0 +1,11 @@ + + + + + + + + RUN INCOMPLETE: the update was canceled before all test results were received. update [UUID] is CANCELED + + + diff --git a/acceptance/pipelines/test/run-control/output.txt b/acceptance/pipelines/test/run-control/output.txt index e1a1895d5f3..3ca8abd6ef2 100644 --- a/acceptance/pipelines/test/run-control/output.txt +++ b/acceptance/pipelines/test/run-control/output.txt @@ -33,11 +33,25 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Pipeline: my_pipeline ([UUID]) Update: [UUID] ->>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s --junit-xml out.resumed.xml Pipeline: my_pipeline ([UUID]) Update: [UUID] Running tests... -Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --resume [UUID] +JUnit report: out.resumed.xml +Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID] + +>>> cat out.resumed.xml + + + + + + + + stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID] + + + === The command to read the results keeps the profile >>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s -p run-control @@ -51,14 +65,29 @@ Error: stopped waiting for update [UUID] after 3s; the update keeps running, to Stopping my_pipeline... my_pipeline has been stopped. ->>> musterr [CLI] pipelines test my_pipeline --resume [UUID] +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --junit-xml out.resumed.xml Pipeline: my_pipeline ([UUID]) Update: [UUID] RUN INCOMPLETE: the update was canceled before all test results were received. Results received: none. +JUnit report: out.resumed.xml Error: update [UUID] is CANCELED === --resume rejects an update that is not a test update >>> musterr [CLI] pipelines test my_pipeline --resume [UUID] Error: update [UUID] is not a test update; pass the update ID that pipelines test printed + +=== Outside a project, an older passing report is replaced +Error: outside a project, identify the pipeline with --pipeline-id + +=== A project with errors names the first error in the report +>>> musterr [CLI] pipelines test my_pipeline --junit-xml ../out.invalid.xml +Error: reference does not exist: ${var.missing} + + +=== When the report cannot be written either, the write error is shown +>>> musterr [CLI] pipelines test my_pipeline --junit-xml missing/report.xml +Error: reference does not exist: ${var.missing} + +Error: failed to write JUnit report: open missing/report.xml: no such file or directory diff --git a/acceptance/pipelines/test/run-control/script b/acceptance/pipelines/test/run-control/script index c640f6174a7..9dbebd00397 100644 --- a/acceptance/pipelines/test/run-control/script +++ b/acceptance/pipelines/test/run-control/script @@ -11,7 +11,8 @@ title "--timeout stops waiting but not the update" trace $CLI pipelines test my_pipeline tests/test_slow.py --no-wait | tee submit.txt hang_update_id=$(sed -n 's/^Update: //p' submit.txt) rm submit.txt -trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout 3s +trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout 3s --junit-xml out.resumed.xml +trace cat out.resumed.xml title "The command to read the results keeps the profile" cat > .databrickscfg <' > out.outside.xml +here=$PWD +outside=$(mktemp -d) +cd $outside +musterr $CLI pipelines test --junit-xml "$here/out.outside.xml" +cd $here +rm -r $outside + +title "A project with errors names the first error in the report" +mkdir invalid +printf 'bundle:\n name: invalid\nresources:\n pipelines:\n my_pipeline:\n name: ${var.missing}\n' > invalid/databricks.yml +cd invalid +trace musterr $CLI pipelines test my_pipeline --junit-xml ../out.invalid.xml + +title "When the report cannot be written either, the write error is shown" +trace musterr $CLI pipelines test my_pipeline --junit-xml missing/report.xml +cd .. +rm -r invalid diff --git a/acceptance/pipelines/test/submit-error/out.start-failed.xml b/acceptance/pipelines/test/submit-error/out.start-failed.xml new file mode 100644 index 00000000000..93125b3d911 --- /dev/null +++ b/acceptance/pipelines/test/submit-error/out.start-failed.xml @@ -0,0 +1,8 @@ + + + + + An active update '[UUID]' already exists for pipeline '1234'. + + + diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index 7f0fcedf532..7d3e524451f 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -11,3 +11,7 @@ Error: An active update '[UUID]' already exists for pipeline '1234'. "test_only": true } } + +=== A submit error replaces an older report +>>> musterr [CLI] pipelines test --pipeline-id [UUID] --junit-xml out.start-failed.xml +Error: An active update '[UUID]' already exists for pipeline '1234'. diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script index 29d720cc633..4586be78c10 100644 --- a/acceptance/pipelines/test/submit-error/script +++ b/acceptance/pipelines/test/submit-error/script @@ -5,3 +5,7 @@ rm "$OUT_REQUESTS" title "A submit error from the service is shown as is, and no update ID is printed" trace musterr $CLI pipelines test --pipeline-id $pipeline_id trace print_requests.py //updates + +title "A submit error replaces an older report" +echo '' > out.start-failed.xml +trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.start-failed.xml diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 1f1c4cc1b14..4b2a2804f08 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -412,6 +412,10 @@ func (t testTarget) resumeCommand(cmd *cobra.Command, updateID string) string { if profile := cmd.Flag("profile").Value.String(); profile != "" { parts = append(parts, "--profile", shellquote.BashArg(profile)) } + // Reading the results again replaces the partial report. + if flag := cmd.Flag("junit-xml"); flag != nil && flag.Value.String() != "" { + parts = append(parts, "--junit-xml", shellquote.BashArg(flag.Value.String())) + } return strings.Join(append(parts, "--resume", updateID), " ") } @@ -507,6 +511,22 @@ func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun return fmt.Errorf("update %s is %s", updateID, state) } +// testRunProblem describes why a run that ended did not pass for a reason other than +// failed tests, or returns "" otherwise. +func testRunProblem(run *testRun, incompleteReason string, runErr error) string { + var lines []string + switch { + case incompleteReason != "": + lines = append(lines, fmt.Sprintf("RUN INCOMPLETE: %s.", incompleteReason)) + case run.summaries[0].Total == 0: + lines = append(lines, "No tests were collected.") + } + if runErr != nil && !errors.Is(runErr, root.ErrAlreadyPrinted) { + lines = append(lines, runErr.Error()) + } + return strings.Join(lines, "\n") +} + var errTestWaitTimeout = errors.New("test wait timed out") func testCommand() *cobra.Command { @@ -539,7 +559,11 @@ example: The command waits for the results and succeeds only if at least one test was collected and none failed or errored. --no-wait returns after the update starts. --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume -to read the results later. To stop the update, run 'pipelines stop'.`, +to read the results later. To stop the update, run 'pipelines stop'. + +--junit-xml writes one entry per test with its outcome and duration. An +incomplete run, a run without tests, or a timeout adds a failing entry that +names the cause.`, } var pipelineID string @@ -547,6 +571,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, var noWait bool var timeout time.Duration var resumeUpdateID string + var junitPath string testGroup := cmdgroup.NewFlagGroup("Pipeline Test") testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a PIPELINE_KEY.") @@ -554,6 +579,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, testGroup.FlagSet().BoolVar(&noWait, "no-wait", false, "Return after the test update starts, without waiting for results.") testGroup.FlagSet().DurationVar(&timeout, "timeout", 0, "Stop waiting for results after this duration, for example 30m. The update keeps running.") testGroup.FlagSet().StringVar(&resumeUpdateID, "resume", "", "Read the results of an existing test update instead of starting one.") + testGroup.FlagSet().StringVar(&junitPath, "junit-xml", "", "Write the results as a JUnit XML report to this file, replacing it.") wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(testGroup) @@ -579,6 +605,9 @@ to read the results later. To stop the update, run 'pipelines stop'.`, if noWait && timeout > 0 { return errors.New("--timeout cannot be used with --no-wait") } + if noWait && junitPath != "" { + return errors.New("--junit-xml cannot be used with --no-wait because no results are available when the command returns") + } // An empty --resume, such as from an unset shell variable, must not start a new run. resume := cmd.Flags().Changed("resume") if resume { @@ -599,6 +628,12 @@ to read the results later. To stop the update, run 'pipelines stop'.`, } } + // From here on, every exit writes the report, so that an older report never survives. + report := &testReport{path: junitPath, out: cmd.OutOrStdout(), suiteName: pipelineID} + if report.suiteName == "" && len(args) > 0 { + report.suiteName = args[0] + } + var target testTarget var err error if pipelineID != "" { @@ -607,12 +642,16 @@ to read the results later. To stop the update, run 'pipelines stop'.`, ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) if !bundle.HasRoot(ctx) { - return errors.New("outside a project, identify the pipeline with --pipeline-id") + return report.failed(ctx, errors.New("outside a project, identify the pipeline with --pipeline-id")) } target, err = resolveTestTargetInBundle(cmd, args, resume) } if err != nil { - return err + return report.failed(cmd.Context(), err) + } + report.suiteName = target.key + if report.suiteName == "" { + report.suiteName = target.pipelineID } // The CLI root doesn't cancel the context on signals, so stop waiting here. ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) @@ -624,7 +663,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, req := newStartTestUpdateRequest(target.selections, pytestArgs) updateID, err = startTestUpdate(ctx, target.w, target.pipelineID, req) if err != nil { - return err + return report.failed(ctx, err) } } else if err := checkTestUpdate(ctx, target.w, target.pipelineID, updateID); err != nil { return err @@ -636,7 +675,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, events, err := newTestEventReader(target.w, target.pipelineID, updateID) if err != nil { - return err + return report.failed(ctx, err) } watcher := &testWatcher{ w: target.w, @@ -656,18 +695,20 @@ to read the results later. To stop the update, run 'pipelines stop'.`, state, err := watcher.watch(waitCtx) if err != nil { if waitCtx.Err() == nil { - return err + return report.finish(updateID, watcher.run, err.Error(), err) } resume := target.resumeCommand(cmd, updateID) + stopErr := fmt.Errorf("stopped waiting for update %s; the update keeps running, to get its results run: %s", updateID, resume) if errors.Is(context.Cause(waitCtx), errTestWaitTimeout) { - return fmt.Errorf("stopped waiting for update %s after %s; the update keeps running, to get its results run: %s", updateID, timeout, resume) + stopErr = fmt.Errorf("stopped waiting for update %s after %s; the update keeps running, to get its results run: %s", updateID, timeout, resume) } - return fmt.Errorf("stopped waiting for update %s; the update keeps running, to get its results run: %s", updateID, resume) + return report.finish(updateID, watcher.run, stopErr.Error(), stopErr) } incompleteReason := watcher.run.incompleteReason(state) renderTestResult(out, watcher.run, incompleteReason, target.shortID) - return testRunError(updateID, state, watcher.run, incompleteReason) + runErr := testRunError(updateID, state, watcher.run, incompleteReason) + return report.finish(updateID, watcher.run, testRunProblem(watcher.run, incompleteReason, runErr), runErr) } return cmd diff --git a/cmd/pipelines/test_junit.go b/cmd/pipelines/test_junit.go new file mode 100644 index 00000000000..c354849b1eb --- /dev/null +++ b/cmd/pipelines/test_junit.go @@ -0,0 +1,187 @@ +package pipelines + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/logdiag" +) + +type junitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []junitTestSuite `xml:"testsuite"` +} + +type junitTestSuite struct { + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr"` + Cases []junitTestCase `xml:"testcase"` +} + +type junitTestCase struct { + ClassName string `xml:"classname,attr,omitempty"` + Name string `xml:"name,attr"` + Time string `xml:"time,attr,omitempty"` + Failure *junitResult `xml:"failure"` + Error *junitResult `xml:"error"` + Skipped *junitResult `xml:"skipped"` +} + +type junitResult struct { + Message string `xml:"message,attr,omitempty"` + Text string `xml:",chardata"` +} + +// junitCaseName splits a pytest test ID the way pytest's JUnit report does: the classname +// is the module path plus any class names, joined by dots. See +// https://docs.pytest.org/en/stable/how-to/output.html#creating-junitxml-format-files. +// A parameter ID in brackets, such as test_cast[a::int], stays with the test name. +func junitCaseName(testID string) (string, string) { + params := "" + if i := strings.Index(testID, "["); i >= 0 { + testID, params = testID[:i], testID[i:] + } + parts := strings.Split(testID, "::") + parts[len(parts)-1] += params + module := strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(parts[0], "/"), ".py"), "/", ".") + if len(parts) == 1 { + return "", module + } + classes := append([]string{module}, parts[1:len(parts)-1]...) + return strings.Join(classes, "."), parts[len(parts)-1] +} + +func newJUnitResult(text string) *junitResult { + message, _, _ := strings.Cut(text, "\n") + return &junitResult{Message: message, Text: text} +} + +// junitNoResult is the error message of a collected case that has no final outcome. +const junitNoResult = "no result received" + +// newJUnitReport has one entry per collected case; a case without a final outcome is an error, +// because CI tools treat skipped entries as passing. +// A non-empty problem, such as an incomplete run, adds a failing entry for the update so the +// report never passes by omission. +func newJUnitReport(suiteName, updateID string, run *testRun, problem string) junitTestSuites { + suite := junitTestSuite{Name: suiteName} + for _, c := range run.cases { + entry := junitTestCase{} + entry.ClassName, entry.Name = junitCaseName(c.id) + if d, ok := c.duration(); ok { + entry.Time = fmt.Sprintf("%.3f", d.Seconds()) + } + + details := "" + if c.errorDetail != nil { + details = errorDetailText(c.errorDetail) + } + if c.detailsTruncated { + details = strings.TrimPrefix(details+"\n(details truncated)", "\n") + } + switch c.status { + case testCasePassed: + case testCaseFailed: + entry.Failure = newJUnitResult(details) + suite.Failures++ + case testCaseError: + entry.Error = newJUnitResult(details) + suite.Errors++ + case testCaseSkipped: + entry.Skipped = &junitResult{} + suite.Skipped++ + default: + entry.Error = &junitResult{Message: junitNoResult} + suite.Errors++ + } + suite.Cases = append(suite.Cases, entry) + } + + if problem != "" { + suite.Cases = append(suite.Cases, junitTestCase{ + Name: "update " + updateID, + Error: newJUnitResult(problem), + }) + suite.Errors++ + } + suite.Tests = len(suite.Cases) + return junitTestSuites{Suites: []junitTestSuite{suite}} +} + +// testReport writes the JUnit report of a test run to path, if --junit-xml is set. +type testReport struct { + path string + out io.Writer + suiteName string +} + +// finish writes the report of a run that ended with runErr. A write error is joined with runErr, +// so that neither is lost. +func (r *testReport) finish(updateID string, run *testRun, problem string, runErr error) error { + if r.path == "" { + return runErr + } + if err := writeJUnitReport(r.path, newJUnitReport(r.suiteName, updateID, run, problem)); err != nil { + if runErr == nil || errors.Is(runErr, root.ErrAlreadyPrinted) { + return err + } + return errors.Join(runErr, err) + } + fmt.Fprintf(r.out, "JUnit report: %s\n", filepath.ToSlash(r.path)) + return runErr +} + +// failed writes a report with one failing entry for an error that ended the command before the +// update could be read, so that an older report does not survive. A write error is joined with +// err, as in finish. +func (r *testReport) failed(ctx context.Context, err error) error { + if r.path == "" { + return err + } + message := err.Error() + printed := errors.Is(err, root.ErrAlreadyPrinted) + if printed { + // The error was printed as a diagnostic. Name it instead of the sentinel's text. + message = "the project has errors, see the command output" + if logdiag.IsSetup(ctx) && logdiag.GetFirstErrorSummary(ctx) != "" { + message = logdiag.GetFirstErrorSummary(ctx) + } + } + report := junitTestSuites{Suites: []junitTestSuite{{ + Name: r.suiteName, + Tests: 1, + Errors: 1, + Cases: []junitTestCase{{Name: "pipelines test", Error: newJUnitResult(message)}}, + }}} + if writeErr := writeJUnitReport(r.path, report); writeErr != nil { + if printed { + return writeErr + } + return errors.Join(err, writeErr) + } + return err +} + +// writeJUnitReport replaces the file at path with the report. +func writeJUnitReport(path string, report junitTestSuites) error { + data, err := xml.MarshalIndent(report, "", " ") + if err != nil { + return err + } + data = append([]byte(xml.Header), data...) + data = append(data, '\n') + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("failed to write JUnit report: %w", err) + } + return nil +} diff --git a/cmd/pipelines/test_junit_test.go b/cmd/pipelines/test_junit_test.go new file mode 100644 index 00000000000..ef9caf8f205 --- /dev/null +++ b/cmd/pipelines/test_junit_test.go @@ -0,0 +1,137 @@ +package pipelines + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJUnitCaseName(t *testing.T) { + tests := []struct { + testID, className, name string + }{ + {"tests/test_orders.py::test_backfill", "tests.test_orders", "test_backfill"}, + {"tests/test_orders.py::TestOrders::test_tax[a-b]", "tests.test_orders.TestOrders", "test_tax[a-b]"}, + // A parameter ID may contain "::". + {"tests/test_a.py::test_cast[a::int]", "tests.test_a", "test_cast[a::int]"}, + {"tests/test_a.py::TestCast::test_cast[a::int]", "tests.test_a.TestCast", "test_cast[a::int]"}, + {"tests/test_orders.py", "", "tests.test_orders"}, + {"/Workspace/Shared/tests/test_x.py::test_x", "Workspace.Shared.tests.test_x", "test_x"}, + } + for _, tc := range tests { + className, name := junitCaseName(tc.testID) + assert.Equal(t, tc.className, className, tc.testID) + assert.Equal(t, tc.name, name, tc.testID) + } +} + +func TestWriteJUnitReport(t *testing.T) { + run := newTestRun() + for _, e := range []testPipelineEvent{ + caseEvent(t, "tests/test_a.py::test_ok", "RUNNING", "2026-01-01T00:00:00.000Z"), + caseEvent(t, "tests/test_a.py::test_ok", "PASSED", "2026-01-01T00:00:01.840Z"), + caseEvent(t, "tests/test_a.py::test_bad", "FAILED", "", + `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`, + `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), + caseEvent(t, "tests/test_a.py::test_setup", "ERROR", "", + `"error": {"exceptions": [{"class_name": "RuntimeError", "message": "fixture failed"}]}`), + caseEvent(t, "tests/test_a.py::test_old", "SKIPPED", ""), + caseEvent(t, "tests/test_a.py::test_unfinished", "RUNNING", ""), + } { + run.add(e) + } + + path := filepath.Join(t.TempDir(), "report.xml") + require.NoError(t, os.WriteFile(path, []byte("previous report"), 0o644)) + require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", run, "RUN INCOMPLETE: received 4 of 5 test results."))) + + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, ` + + + + + AssertionError: assert 1 == 2 where 1 = f() (details truncated) + + + RuntimeError: fixture <db> failed + + + + + + + + + RUN INCOMPLETE: received 4 of 5 test results. + + + +`, string(data)) +} + +func TestTestRunProblem(t *testing.T) { + empty := newTestRun() + empty.add(summaryEvent(t, 0, 0, 0, 0, 0)) + assert.Equal(t, "No tests were collected.", testRunProblem(empty, "", root.ErrAlreadyPrinted)) + + failed := newTestRun() + failed.add(caseEvent(t, "t::a", "FAILED", "")) + failed.add(summaryEvent(t, 1, 0, 1, 0, 0)) + assert.Empty(t, testRunProblem(failed, "", root.ErrAlreadyPrinted)) + + passed := newTestRun() + passed.add(caseEvent(t, "t::a", "PASSED", "")) + passed.add(summaryEvent(t, 1, 1, 0, 0, 0)) + assert.Equal(t, "update u1 is FAILED", testRunProblem(passed, "", errors.New("update u1 is FAILED"))) + + assert.Equal(t, "RUN INCOMPLETE: the update was canceled before all test results were received.\nupdate u1 is CANCELED", + testRunProblem(newTestRun(), "the update was canceled before all test results were received", errors.New("update u1 is CANCELED"))) +} + +func TestTestReportJoinsWriteErrors(t *testing.T) { + report := &testReport{path: filepath.Join(t.TempDir(), "missing", "report.xml"), out: io.Discard, suiteName: "orders"} + runErr := errors.New("update u1 is FAILED") + err := report.finish("u1", newTestRun(), runErr.Error(), runErr) + require.ErrorIs(t, err, runErr) + assert.ErrorContains(t, err, "failed to write JUnit report") +} + +func TestTestReportReplacesAStaleReportOnError(t *testing.T) { + path := filepath.Join(t.TempDir(), "report.xml") + require.NoError(t, os.WriteFile(path, []byte(""), 0o644)) + report := &testReport{path: path, out: io.Discard, suiteName: "orders"} + startErr := errors.New("An active update already exists.") + assert.Equal(t, startErr, report.failed(t.Context(), startErr)) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(data), ``) + assert.Contains(t, string(data), ``) +} + +func TestTestReportNamesAPrintedError(t *testing.T) { + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + logdiag.LogDiag(ctx, diag.Diagnostic{Severity: diag.Error, Summary: "reference does not exist: ${var.missing}"}) + path := filepath.Join(t.TempDir(), "report.xml") + report := &testReport{path: path, out: io.Discard, suiteName: "orders"} + assert.Equal(t, root.ErrAlreadyPrinted, report.failed(ctx, root.ErrAlreadyPrinted)) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(data), ``) + + // The sentinel would hide a joined write error, so the write error is returned alone. + report.path = filepath.Join(t.TempDir(), "missing", "report.xml") + err = report.failed(ctx, root.ErrAlreadyPrinted) + assert.NotErrorIs(t, err, root.ErrAlreadyPrinted) + assert.ErrorContains(t, err, "failed to write JUnit report") +} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index ad4844213ad..30aea1be34d 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -150,6 +150,12 @@ func TestTestTargetResumeCommand(t *testing.T) { flags: map[string]string{"target": "dev west", "profile": "CI West"}, want: "databricks pipelines test my_pipeline --target 'dev west' --profile 'CI West' --resume u1", }, + { + name: "JUnit report", + target: testTarget{key: "my_pipeline", pipelineID: "p1"}, + flags: map[string]string{"junit-xml": "reports/test results.xml"}, + want: "databricks pipelines test my_pipeline --junit-xml 'reports/test results.xml' --resume u1", + }, { name: "pipeline ID with profile", target: testTarget{pipelineID: "p1"}, @@ -164,6 +170,7 @@ func TestTestTargetResumeCommand(t *testing.T) { rootCmd.PersistentFlags().StringP("target", "t", "", "") pipelinesCmd := &cobra.Command{Use: "pipelines"} cmd := &cobra.Command{Use: "test"} + cmd.Flags().String("junit-xml", "", "") rootCmd.AddCommand(pipelinesCmd) pipelinesCmd.AddCommand(cmd) for name, value := range tc.flags { From 5c6e0a10eaf189f3b4c55afbdab32ad720586e65 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:46:12 +0000 Subject: [PATCH 2/9] Name pipelines test JUnit cases relative to the project folder, as pytest does --- .../pipelines/test/results/out.failures.xml | 8 ++++---- .../pipelines/test/results/out.incomplete.xml | 6 +++--- .../test/results/out.late-failure.xml | 4 ++-- .../test/results/out.update-failed.xml | 2 +- .../pipelines/test/run-control/out.resumed.xml | 2 +- .../pipelines/test/run-control/output.txt | 2 +- cmd/pipelines/test.go | 18 +++++++++++------- cmd/pipelines/test_junit.go | 15 +++++++++------ cmd/pipelines/test_junit_test.go | 16 +++++++++------- cmd/pipelines/test_test.go | 12 ++++++------ 10 files changed, 47 insertions(+), 38 deletions(-) diff --git a/acceptance/pipelines/test/results/out.failures.xml b/acceptance/pipelines/test/results/out.failures.xml index 64da8c3634d..5bf009266fd 100644 --- a/acceptance/pipelines/test/results/out.failures.xml +++ b/acceptance/pipelines/test/results/out.failures.xml @@ -1,14 +1,14 @@ - - + + AssertionError: assert 100 == 150 - + - + RuntimeError: fixture 'orders' setup failed diff --git a/acceptance/pipelines/test/results/out.incomplete.xml b/acceptance/pipelines/test/results/out.incomplete.xml index 096c3289d8a..c6e324f0f6b 100644 --- a/acceptance/pipelines/test/results/out.incomplete.xml +++ b/acceptance/pipelines/test/results/out.incomplete.xml @@ -1,11 +1,11 @@ - - + + - + diff --git a/acceptance/pipelines/test/results/out.late-failure.xml b/acceptance/pipelines/test/results/out.late-failure.xml index 110219d1016..5e9a004942b 100644 --- a/acceptance/pipelines/test/results/out.late-failure.xml +++ b/acceptance/pipelines/test/results/out.late-failure.xml @@ -1,10 +1,10 @@ - + AssertionError: assert 100 == 150 - + update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. diff --git a/acceptance/pipelines/test/results/out.update-failed.xml b/acceptance/pipelines/test/results/out.update-failed.xml index 87562fd44a5..a115d599dc0 100644 --- a/acceptance/pipelines/test/results/out.update-failed.xml +++ b/acceptance/pipelines/test/results/out.update-failed.xml @@ -1,7 +1,7 @@ - + update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. diff --git a/acceptance/pipelines/test/run-control/out.resumed.xml b/acceptance/pipelines/test/run-control/out.resumed.xml index ff8b40372a2..c2fb2be3d65 100644 --- a/acceptance/pipelines/test/run-control/out.resumed.xml +++ b/acceptance/pipelines/test/run-control/out.resumed.xml @@ -1,7 +1,7 @@ - + diff --git a/acceptance/pipelines/test/run-control/output.txt b/acceptance/pipelines/test/run-control/output.txt index 3ca8abd6ef2..9d1c19b4cd9 100644 --- a/acceptance/pipelines/test/run-control/output.txt +++ b/acceptance/pipelines/test/run-control/output.txt @@ -44,7 +44,7 @@ Error: stopped waiting for update [UUID] after 3s; the update keeps running, to - + diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 4b2a2804f08..e080de62675 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -188,10 +188,10 @@ func bundleShortTestID(cwd, syncRoot, filesRoot, id string) string { return filepath.ToSlash(local) + testID } -// rootPathShortTestID shows a test ID whose path is under rootPath relative to it, which --pipeline-id -// resolves against root_path to select the test again. Other test IDs are returned unchanged. -func rootPathShortTestID(rootPath, id string) string { - rel, testID, ok := cutTestIDFolder(rootPath, id) +// relativeTestID shows a test ID whose path is under folder relative to it. Other test IDs are returned +// unchanged. Relative to root_path, it is what --pipeline-id resolves to select the test again. +func relativeTestID(folder, id string) string { + rel, testID, ok := cutTestIDFolder(folder, id) if !ok { return id } @@ -309,6 +309,9 @@ type testTarget struct { w *databricks.WorkspaceClient // shortID shows a test ID in the shortest form that selects the test again. shortID func(id string) string + // projectFolder is the deployed folder that names test cases independently of the working + // directory: the bundle's synchronized files, or root_path. It is "" when unknown. + projectFolder string } func (t testTarget) printHeader(out io.Writer, updateID string) { @@ -346,8 +349,8 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string, if err != nil { return testTarget{}, err } - shortID := func(id string) string { return rootPathShortTestID(rootPath, id) } - return testTarget{pipelineID: pipelineID, selections: selections, w: w, shortID: shortID}, nil + shortID := func(id string) string { return relativeTestID(rootPath, id) } + return testTarget{pipelineID: pipelineID, selections: selections, w: w, shortID: shortID, projectFolder: rootPath}, nil } // resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. @@ -394,7 +397,7 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string, resume bool) ( return testTarget{}, err } shortID := func(id string) string { return bundleShortTestID(cwd, b.SyncRootPath, filesRoot, id) } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), shortID: shortID}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), shortID: shortID, projectFolder: filesRoot}, nil } // resumeCommand returns the command that reads the result of the update later. @@ -653,6 +656,7 @@ names the cause.`, if report.suiteName == "" { report.suiteName = target.pipelineID } + report.projectFolder = target.projectFolder // The CLI root doesn't cancel the context on signals, so stop waiting here. ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/cmd/pipelines/test_junit.go b/cmd/pipelines/test_junit.go index c354849b1eb..75939821082 100644 --- a/cmd/pipelines/test_junit.go +++ b/cmd/pipelines/test_junit.go @@ -73,11 +73,13 @@ const junitNoResult = "no result received" // because CI tools treat skipped entries as passing. // A non-empty problem, such as an incomplete run, adds a failing entry for the update so the // report never passes by omission. -func newJUnitReport(suiteName, updateID string, run *testRun, problem string) junitTestSuites { +// Cases are named relative to projectFolder, as pytest names them, so that the names do not depend on +// the working directory. +func newJUnitReport(suiteName, updateID, projectFolder string, run *testRun, problem string) junitTestSuites { suite := junitTestSuite{Name: suiteName} for _, c := range run.cases { entry := junitTestCase{} - entry.ClassName, entry.Name = junitCaseName(c.id) + entry.ClassName, entry.Name = junitCaseName(relativeTestID(projectFolder, c.id)) if d, ok := c.duration(); ok { entry.Time = fmt.Sprintf("%.3f", d.Seconds()) } @@ -120,9 +122,10 @@ func newJUnitReport(suiteName, updateID string, run *testRun, problem string) ju // testReport writes the JUnit report of a test run to path, if --junit-xml is set. type testReport struct { - path string - out io.Writer - suiteName string + path string + out io.Writer + suiteName string + projectFolder string } // finish writes the report of a run that ended with runErr. A write error is joined with runErr, @@ -131,7 +134,7 @@ func (r *testReport) finish(updateID string, run *testRun, problem string, runEr if r.path == "" { return runErr } - if err := writeJUnitReport(r.path, newJUnitReport(r.suiteName, updateID, run, problem)); err != nil { + if err := writeJUnitReport(r.path, newJUnitReport(r.suiteName, updateID, r.projectFolder, run, problem)); err != nil { if runErr == nil || errors.Is(runErr, root.ErrAlreadyPrinted) { return err } diff --git a/cmd/pipelines/test_junit_test.go b/cmd/pipelines/test_junit_test.go index ef9caf8f205..ae60f20a582 100644 --- a/cmd/pipelines/test_junit_test.go +++ b/cmd/pipelines/test_junit_test.go @@ -34,24 +34,26 @@ func TestJUnitCaseName(t *testing.T) { } func TestWriteJUnitReport(t *testing.T) { + // Cases are named relative to the project folder, as pytest names them, wherever the command runs. + const project = "/Workspace/Users/someone/.bundle/orders/dev/files" run := newTestRun() for _, e := range []testPipelineEvent{ - caseEvent(t, "tests/test_a.py::test_ok", "RUNNING", "2026-01-01T00:00:00.000Z"), - caseEvent(t, "tests/test_a.py::test_ok", "PASSED", "2026-01-01T00:00:01.840Z"), - caseEvent(t, "tests/test_a.py::test_bad", "FAILED", "", + caseEvent(t, project+"/tests/test_a.py::test_ok", "RUNNING", "2026-01-01T00:00:00.000Z"), + caseEvent(t, project+"/tests/test_a.py::test_ok", "PASSED", "2026-01-01T00:00:01.840Z"), + caseEvent(t, project+"/tests/test_a.py::test_bad", "FAILED", "", `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`, `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), - caseEvent(t, "tests/test_a.py::test_setup", "ERROR", "", + caseEvent(t, project+"/tests/test_a.py::test_setup", "ERROR", "", `"error": {"exceptions": [{"class_name": "RuntimeError", "message": "fixture failed"}]}`), - caseEvent(t, "tests/test_a.py::test_old", "SKIPPED", ""), - caseEvent(t, "tests/test_a.py::test_unfinished", "RUNNING", ""), + caseEvent(t, project+"/tests/test_a.py::test_old", "SKIPPED", ""), + caseEvent(t, project+"/tests/test_a.py::test_unfinished", "RUNNING", ""), } { run.add(e) } path := filepath.Join(t.TempDir(), "report.xml") require.NoError(t, os.WriteFile(path, []byte("previous report"), 0o644)) - require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", run, "RUN INCOMPLETE: received 4 of 5 test results."))) + require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", project, run, "RUN INCOMPLETE: received 4 of 5 test results."))) data, err := os.ReadFile(path) require.NoError(t, err) diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 30aea1be34d..b21a0ce94ea 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -93,20 +93,20 @@ func TestBundleShortTestID(t *testing.T) { assert.Equal(t, id, bundleShortTestID(syncRoot, syncRoot, "", id)) } -func TestRootPathShortTestID(t *testing.T) { +func TestRelativeTestID(t *testing.T) { rootPath := "/Workspace/Users/someone/orders" - assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID(rootPath, rootPath+"/tests/test_a.py::test_x")) - assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID(rootPath+"/", rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID(rootPath, rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID(rootPath+"/", rootPath+"/tests/test_a.py::test_x")) // The service accepts workspace paths with or without the /Workspace prefix. - assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID("/Users/someone/orders", rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Users/someone/orders", rootPath+"/tests/test_a.py::test_x")) for _, other := range []string{ "/Workspace/Users/someone/shared/test_s.py::test_s", rootPath + "2/test_a.py::test_x", "tests/test_a.py::test_x", } { - assert.Equal(t, other, rootPathShortTestID(rootPath, other)) + assert.Equal(t, other, relativeTestID(rootPath, other)) } - assert.Equal(t, rootPath+"/tests/test_a.py::test_x", rootPathShortTestID("", rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, rootPath+"/tests/test_a.py::test_x", relativeTestID("", rootPath+"/tests/test_a.py::test_x")) } // Source-linked deployments run the source files, as TranslatePaths maps them. From 9c1ba5d42cc9e4af498d87be22bd9de699b2c7a7 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:48:47 +0000 Subject: [PATCH 3/9] Replace an older pipelines test JUnit report when a flag is invalid --- .../test/arguments/out.flag-error.xml | 8 +++ .../pipelines/test/arguments/out.no-wait.xml | 8 +++ .../pipelines/test/arguments/output.txt | 6 +- acceptance/pipelines/test/arguments/script | 6 +- cmd/pipelines/test.go | 70 ++++++++++--------- 5 files changed, 64 insertions(+), 34 deletions(-) create mode 100644 acceptance/pipelines/test/arguments/out.flag-error.xml create mode 100644 acceptance/pipelines/test/arguments/out.no-wait.xml diff --git a/acceptance/pipelines/test/arguments/out.flag-error.xml b/acceptance/pipelines/test/arguments/out.flag-error.xml new file mode 100644 index 00000000000..475a397f5fe --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.flag-error.xml @@ -0,0 +1,8 @@ + + + + + unsupported pytest argument "-x": only -k and -m are supported + + + diff --git a/acceptance/pipelines/test/arguments/out.no-wait.xml b/acceptance/pipelines/test/arguments/out.no-wait.xml new file mode 100644 index 00000000000..5f025801a52 --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.no-wait.xml @@ -0,0 +1,8 @@ + + + + + --junit-xml cannot be used with --no-wait because no results are available when the command returns + + + diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 780ee4e6cb3..883b2ecc162 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -87,9 +87,13 @@ Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that p >>> musterr [CLI] pipelines test --pipeline-id 1234 --resume Error: --resume "" is not an update ID; pass the update ID that pipelines test printed ->>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml report.xml +>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns +=== An invalid flag replaces an older report +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml +Error: unsupported pytest argument "-x": only -k and -m are supported + === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index 25ad2cce729..5e5db690f86 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -17,7 +17,11 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1" trace musterr $CLI pipelines test --pipeline-id 1234 --resume "" -trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml report.xml +trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml + +title "An invalid flag replaces an older report" +echo '' > out.flag-error.xml +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index e080de62675..39cc886a46b 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -593,49 +593,55 @@ names the cause.`, if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled { return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar) } - if cmd.ArgsLenAtDash() >= 0 { - return errors.New("the -- separator is not supported: " + pytestArgHint) - } - if root.OutputType(cmd) != flags.OutputText { - return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) - } - if err := validatePytestArgs(pytestArgs); err != nil { - return err - } - if timeout < 0 { - return errors.New("--timeout must not be negative") - } - if noWait && timeout > 0 { - return errors.New("--timeout cannot be used with --no-wait") - } - if noWait && junitPath != "" { - return errors.New("--junit-xml cannot be used with --no-wait because no results are available when the command returns") - } // An empty --resume, such as from an unset shell variable, must not start a new run. resume := cmd.Flags().Changed("resume") - if resume { - selectionCount := len(args) - if pipelineID == "" { - selectionCount = max(len(args)-1, 0) + checkFlags := func() error { + if cmd.ArgsLenAtDash() >= 0 { + return errors.New("the -- separator is not supported: " + pytestArgHint) } - switch { - case noWait: - return errors.New("--resume cannot be used with --no-wait") - case len(pytestArgs) > 0: - return errors.New("--resume cannot be used with --pytest-arg") - case selectionCount > 0: - return errors.New("--resume cannot be used with test selections") + if root.OutputType(cmd) != flags.OutputText { + return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) } - if _, err := uuid.Parse(resumeUpdateID); err != nil { - return fmt.Errorf("--resume %q is not an update ID; pass the update ID that pipelines test printed", resumeUpdateID) + if err := validatePytestArgs(pytestArgs); err != nil { + return err } + if timeout < 0 { + return errors.New("--timeout must not be negative") + } + if noWait && timeout > 0 { + return errors.New("--timeout cannot be used with --no-wait") + } + if noWait && junitPath != "" { + return errors.New("--junit-xml cannot be used with --no-wait because no results are available when the command returns") + } + if resume { + selectionCount := len(args) + if pipelineID == "" { + selectionCount = max(len(args)-1, 0) + } + switch { + case noWait: + return errors.New("--resume cannot be used with --no-wait") + case len(pytestArgs) > 0: + return errors.New("--resume cannot be used with --pytest-arg") + case selectionCount > 0: + return errors.New("--resume cannot be used with test selections") + } + if _, err := uuid.Parse(resumeUpdateID); err != nil { + return fmt.Errorf("--resume %q is not an update ID; pass the update ID that pipelines test printed", resumeUpdateID) + } + } + return nil } - // From here on, every exit writes the report, so that an older report never survives. + // Every exit writes the report, so that an older report never survives. report := &testReport{path: junitPath, out: cmd.OutOrStdout(), suiteName: pipelineID} if report.suiteName == "" && len(args) > 0 { report.suiteName = args[0] } + if err := checkFlags(); err != nil { + return report.failed(cmd.Context(), err) + } var target testTarget var err error From 88bd03ff6bacee8558d808dae3a5974998bef664 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:57:20 +0000 Subject: [PATCH 4/9] Replace an older pipelines test JUnit report when --resume names an update that is not a test update --- .../pipelines/test/run-control/out.not-test-update.xml | 8 ++++++++ acceptance/pipelines/test/run-control/output.txt | 4 ++-- acceptance/pipelines/test/run-control/script | 5 +++-- cmd/pipelines/test.go | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 acceptance/pipelines/test/run-control/out.not-test-update.xml diff --git a/acceptance/pipelines/test/run-control/out.not-test-update.xml b/acceptance/pipelines/test/run-control/out.not-test-update.xml new file mode 100644 index 00000000000..89adb9dfed7 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.not-test-update.xml @@ -0,0 +1,8 @@ + + + + + update [UUID] is not a test update; pass the update ID that pipelines test printed + + + diff --git a/acceptance/pipelines/test/run-control/output.txt b/acceptance/pipelines/test/run-control/output.txt index 9d1c19b4cd9..1c1f5806d17 100644 --- a/acceptance/pipelines/test/run-control/output.txt +++ b/acceptance/pipelines/test/run-control/output.txt @@ -74,8 +74,8 @@ Results received: none. JUnit report: out.resumed.xml Error: update [UUID] is CANCELED -=== --resume rejects an update that is not a test update ->>> musterr [CLI] pipelines test my_pipeline --resume [UUID] +=== --resume rejects an update that is not a test update, and an older report is replaced +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --junit-xml out.not-test-update.xml Error: update [UUID] is not a test update; pass the update ID that pipelines test printed === Outside a project, an older passing report is replaced diff --git a/acceptance/pipelines/test/run-control/script b/acceptance/pipelines/test/run-control/script index 9dbebd00397..29d026169f6 100644 --- a/acceptance/pipelines/test/run-control/script +++ b/acceptance/pipelines/test/run-control/script @@ -29,10 +29,11 @@ title "Stopping the pipeline cancels the test update" trace $CLI pipelines stop my_pipeline trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --junit-xml out.resumed.xml -title "--resume rejects an update that is not a test update" +title "--resume rejects an update that is not a test update, and an older report is replaced" pipeline_id=$($CLI bundle summary -o json | jq -r '.resources.pipelines.my_pipeline.id') regular_update_id=$($CLI pipelines start-update $pipeline_id | jq -r .update_id) -trace musterr $CLI pipelines test my_pipeline --resume $regular_update_id +echo '' > out.not-test-update.xml +trace musterr $CLI pipelines test my_pipeline --resume $regular_update_id --junit-xml out.not-test-update.xml title "Outside a project, an older passing report is replaced\n" echo '' > out.outside.xml diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 39cc886a46b..b44e243cd73 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -676,7 +676,7 @@ names the cause.`, return report.failed(ctx, err) } } else if err := checkTestUpdate(ctx, target.w, target.pipelineID, updateID); err != nil { - return err + return report.failed(ctx, err) } target.printHeader(out, updateID) if noWait { From f360375f6b53c5793fd49f6dbee6fa129e06282e Mon Sep 17 00:00:00 2001 From: Bhukya Date: Sat, 31 Oct 2026 18:57:20 +0000 Subject: [PATCH 5/9] Cover the JUnit report of a passing pipelines test run --- acceptance/pipelines/test/results/out.passed.xml | 6 ++++++ acceptance/pipelines/test/results/output.txt | 3 ++- acceptance/pipelines/test/results/script | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 acceptance/pipelines/test/results/out.passed.xml diff --git a/acceptance/pipelines/test/results/out.passed.xml b/acceptance/pipelines/test/results/out.passed.xml new file mode 100644 index 00000000000..95d1e6f3d0a --- /dev/null +++ b/acceptance/pipelines/test/results/out.passed.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/acceptance/pipelines/test/results/output.txt b/acceptance/pipelines/test/results/output.txt index 9a27e25b31f..d0b3b88be8a 100644 --- a/acceptance/pipelines/test/results/output.txt +++ b/acceptance/pipelines/test/results/output.txt @@ -7,7 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === All selected tests pass ->>> [CLI] pipelines test my_pipeline tests/test_orders.py::test_valid_order +>>> [CLI] pipelines test my_pipeline tests/test_orders.py::test_valid_order --junit-xml out.passed.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -19,6 +19,7 @@ Running tests... tests/test_orders.py::test_valid_order PASSED (1.00s) 1 test: 1 passed, 0 failed, 0 skipped, 0 errors +JUnit report: out.passed.xml >>> print_requests.py --get //events { diff --git a/acceptance/pipelines/test/results/script b/acceptance/pipelines/test/results/script index a1f18dcd9b0..56779522991 100644 --- a/acceptance/pipelines/test/results/script +++ b/acceptance/pipelines/test/results/script @@ -2,7 +2,7 @@ trace $CLI pipelines deploy rm "$OUT_REQUESTS" title "All selected tests pass" -trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order +trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order --junit-xml out.passed.xml trace print_requests.py --get //events title "Failed, errored, and skipped tests" From 5f18da5924613f062605256ad438f2422c9b545a Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 2 Nov 2026 19:46:41 +0000 Subject: [PATCH 6/9] Link the pipelines test changelog entries to #7299 --- .nextchanges/cli/pipelines-test-junit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/pipelines-test-junit.md b/.nextchanges/cli/pipelines-test-junit.md index fff9ab2c679..4d44bd2935e 100644 --- a/.nextchanges/cli/pipelines-test-junit.md +++ b/.nextchanges/cli/pipelines-test-junit.md @@ -1 +1 @@ -* Add `--junit-xml` to `databricks pipelines test`, which writes the test results as a JUnit XML report. ([#7273](https://github.com/databricks/cli/pull/7273)) +* Add `--junit-xml` to `databricks pipelines test`, which writes the test results as a JUnit XML report. ([#7299](https://github.com/databricks/cli/pull/7299)) From 9245151533809970355ca95294877b1983963730 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 3 Nov 2026 00:34:26 +0000 Subject: [PATCH 7/9] Test each pipelines test JUnit behavior once and use /Workspace/Users paths --- cmd/pipelines/test_junit_test.go | 78 +++----------------------------- cmd/pipelines/test_test.go | 6 --- 2 files changed, 6 insertions(+), 78 deletions(-) diff --git a/cmd/pipelines/test_junit_test.go b/cmd/pipelines/test_junit_test.go index ae60f20a582..c2e973db909 100644 --- a/cmd/pipelines/test_junit_test.go +++ b/cmd/pipelines/test_junit_test.go @@ -7,9 +7,6 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/logdiag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,7 +21,7 @@ func TestJUnitCaseName(t *testing.T) { {"tests/test_a.py::test_cast[a::int]", "tests.test_a", "test_cast[a::int]"}, {"tests/test_a.py::TestCast::test_cast[a::int]", "tests.test_a.TestCast", "test_cast[a::int]"}, {"tests/test_orders.py", "", "tests.test_orders"}, - {"/Workspace/Shared/tests/test_x.py::test_x", "Workspace.Shared.tests.test_x", "test_x"}, + {"/Workspace/Users/someone@example.com/tests/test_x.py::test_x", "Workspace.Users.someone@example.com.tests.test_x", "test_x"}, } for _, tc := range tests { className, name := junitCaseName(tc.testID) @@ -34,72 +31,39 @@ func TestJUnitCaseName(t *testing.T) { } func TestWriteJUnitReport(t *testing.T) { - // Cases are named relative to the project folder, as pytest names them, wherever the command runs. - const project = "/Workspace/Users/someone/.bundle/orders/dev/files" + // Acceptance tests cover each outcome. This covers what the test server does not send: + // multi-line and truncated details, and text that XML must escape. + const project = "/Workspace/Users/someone@example.com/.bundle/orders/dev/files" run := newTestRun() for _, e := range []testPipelineEvent{ - caseEvent(t, project+"/tests/test_a.py::test_ok", "RUNNING", "2026-01-01T00:00:00.000Z"), - caseEvent(t, project+"/tests/test_a.py::test_ok", "PASSED", "2026-01-01T00:00:01.840Z"), caseEvent(t, project+"/tests/test_a.py::test_bad", "FAILED", "", `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`, `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), caseEvent(t, project+"/tests/test_a.py::test_setup", "ERROR", "", `"error": {"exceptions": [{"class_name": "RuntimeError", "message": "fixture failed"}]}`), - caseEvent(t, project+"/tests/test_a.py::test_old", "SKIPPED", ""), - caseEvent(t, project+"/tests/test_a.py::test_unfinished", "RUNNING", ""), } { run.add(e) } path := filepath.Join(t.TempDir(), "report.xml") - require.NoError(t, os.WriteFile(path, []byte("previous report"), 0o644)) - require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", project, run, "RUN INCOMPLETE: received 4 of 5 test results."))) + require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", project, run, ""))) data, err := os.ReadFile(path) require.NoError(t, err) assert.Equal(t, ` - - + AssertionError: assert 1 == 2 where 1 = f() (details truncated) RuntimeError: fixture <db> failed - - - - - - - - RUN INCOMPLETE: received 4 of 5 test results. - `, string(data)) } -func TestTestRunProblem(t *testing.T) { - empty := newTestRun() - empty.add(summaryEvent(t, 0, 0, 0, 0, 0)) - assert.Equal(t, "No tests were collected.", testRunProblem(empty, "", root.ErrAlreadyPrinted)) - - failed := newTestRun() - failed.add(caseEvent(t, "t::a", "FAILED", "")) - failed.add(summaryEvent(t, 1, 0, 1, 0, 0)) - assert.Empty(t, testRunProblem(failed, "", root.ErrAlreadyPrinted)) - - passed := newTestRun() - passed.add(caseEvent(t, "t::a", "PASSED", "")) - passed.add(summaryEvent(t, 1, 1, 0, 0, 0)) - assert.Equal(t, "update u1 is FAILED", testRunProblem(passed, "", errors.New("update u1 is FAILED"))) - - assert.Equal(t, "RUN INCOMPLETE: the update was canceled before all test results were received.\nupdate u1 is CANCELED", - testRunProblem(newTestRun(), "the update was canceled before all test results were received", errors.New("update u1 is CANCELED"))) -} - func TestTestReportJoinsWriteErrors(t *testing.T) { report := &testReport{path: filepath.Join(t.TempDir(), "missing", "report.xml"), out: io.Discard, suiteName: "orders"} runErr := errors.New("update u1 is FAILED") @@ -107,33 +71,3 @@ func TestTestReportJoinsWriteErrors(t *testing.T) { require.ErrorIs(t, err, runErr) assert.ErrorContains(t, err, "failed to write JUnit report") } - -func TestTestReportReplacesAStaleReportOnError(t *testing.T) { - path := filepath.Join(t.TempDir(), "report.xml") - require.NoError(t, os.WriteFile(path, []byte(""), 0o644)) - report := &testReport{path: path, out: io.Discard, suiteName: "orders"} - startErr := errors.New("An active update already exists.") - assert.Equal(t, startErr, report.failed(t.Context(), startErr)) - data, err := os.ReadFile(path) - require.NoError(t, err) - assert.Contains(t, string(data), ``) - assert.Contains(t, string(data), ``) -} - -func TestTestReportNamesAPrintedError(t *testing.T) { - ctx := logdiag.InitContext(t.Context()) - logdiag.SetCollect(ctx, true) - logdiag.LogDiag(ctx, diag.Diagnostic{Severity: diag.Error, Summary: "reference does not exist: ${var.missing}"}) - path := filepath.Join(t.TempDir(), "report.xml") - report := &testReport{path: path, out: io.Discard, suiteName: "orders"} - assert.Equal(t, root.ErrAlreadyPrinted, report.failed(ctx, root.ErrAlreadyPrinted)) - data, err := os.ReadFile(path) - require.NoError(t, err) - assert.Contains(t, string(data), ``) - - // The sentinel would hide a joined write error, so the write error is returned alone. - report.path = filepath.Join(t.TempDir(), "missing", "report.xml") - err = report.failed(ctx, root.ErrAlreadyPrinted) - assert.NotErrorIs(t, err, root.ErrAlreadyPrinted) - assert.ErrorContains(t, err, "failed to write JUnit report") -} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index b21a0ce94ea..d94781ae606 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -150,12 +150,6 @@ func TestTestTargetResumeCommand(t *testing.T) { flags: map[string]string{"target": "dev west", "profile": "CI West"}, want: "databricks pipelines test my_pipeline --target 'dev west' --profile 'CI West' --resume u1", }, - { - name: "JUnit report", - target: testTarget{key: "my_pipeline", pipelineID: "p1"}, - flags: map[string]string{"junit-xml": "reports/test results.xml"}, - want: "databricks pipelines test my_pipeline --junit-xml 'reports/test results.xml' --resume u1", - }, { name: "pipeline ID with profile", target: testTarget{pipelineID: "p1"}, From f61716e31fed97ebec4248ee7b040704e4b56676 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 3 Nov 2026 00:43:56 +0000 Subject: [PATCH 8/9] Say that pipelines test JUnit names keep the snapshot path with workspace.sync_mode immutable --- acceptance/pipelines/test/arguments/output.txt | 3 ++- cmd/pipelines/test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index 883b2ecc162..71c5f18c7a4 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -30,7 +30,8 @@ to read the results later. To stop the update, run 'pipelines stop'. --junit-xml writes one entry per test with its outcome and duration. An incomplete run, a run without tests, or a timeout adds a failing entry that -names the cause. +names the cause. With workspace.sync_mode immutable, test names keep the path +of the deployed snapshot, which changes with each deployment. Usage: databricks pipelines test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...] diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index b44e243cd73..822bd9afe25 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -566,7 +566,8 @@ to read the results later. To stop the update, run 'pipelines stop'. --junit-xml writes one entry per test with its outcome and duration. An incomplete run, a run without tests, or a timeout adds a failing entry that -names the cause.`, +names the cause. With workspace.sync_mode immutable, test names keep the path +of the deployed snapshot, which changes with each deployment.`, } var pipelineID string From 7c61f68c687a1ce5032317926ccd4e97889b3752 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 3 Nov 2026 17:31:04 +0000 Subject: [PATCH 9/9] Accept the Windows error text in the pipelines test JUnit report case Co-authored-by: Isaac --- acceptance/pipelines/test/run-control/test.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/acceptance/pipelines/test/run-control/test.toml b/acceptance/pipelines/test/run-control/test.toml index a030353d571..40e8a1be234 100644 --- a/acceptance/pipelines/test/run-control/test.toml +++ b/acceptance/pipelines/test/run-control/test.toml @@ -1 +1,6 @@ RecordRequests = false + +# Replace error message from windows +[[Repls]] +Old = 'open missing/report.xml: The system cannot find the path specified\.' +New = 'open missing/report.xml: no such file or directory'