diff --git a/.nextchanges/cli/pipelines-test-junit.md b/.nextchanges/cli/pipelines-test-junit.md new file mode 100644 index 00000000000..4d44bd2935e --- /dev/null +++ b/.nextchanges/cli/pipelines-test-junit.md @@ -0,0 +1 @@ +* Add `--junit-xml` to `databricks pipelines test`, which writes the test results as a JUnit XML report. ([#7299](https://github.com/databricks/cli/pull/7299)) diff --git a/acceptance/pipelines/test/arguments/out.flag-error.xml b/acceptance/pipelines/test/arguments/out.flag-error.xml new file mode 100644 index 00000000000..475a397f5fe --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.flag-error.xml @@ -0,0 +1,8 @@ + + + + + unsupported pytest argument "-x": only -k and -m are supported + + + diff --git a/acceptance/pipelines/test/arguments/out.no-wait.xml b/acceptance/pipelines/test/arguments/out.no-wait.xml new file mode 100644 index 00000000000..5f025801a52 --- /dev/null +++ b/acceptance/pipelines/test/arguments/out.no-wait.xml @@ -0,0 +1,8 @@ + + + + + --junit-xml cannot be used with --no-wait because no results are available when the command returns + + + diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index c2d17764f80..71c5f18c7a4 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -28,10 +28,16 @@ collected and none failed or errored. --no-wait returns after the update starts. --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume to read the results later. To stop the update, run 'pipelines stop'. +--junit-xml writes one entry per test with its outcome and duration. An +incomplete run, a run without tests, or a timeout adds a failing entry that +names the cause. With workspace.sync_mode immutable, test names keep the path +of the deployed snapshot, which changes with each deployment. + Usage: databricks pipelines test [PIPELINE_KEY] [TEST_SELECTION...] [flags] [--pytest-arg=ARG ...] Pipeline Test Flags: + --junit-xml string Write the results as a JUnit XML report to this file, replacing it. --no-wait Return after the test update starts, without waiting for results. --pipeline-id string ID of the pipeline to test, instead of a PIPELINE_KEY. --pytest-arg stringArray Argument to pass to pytest (repeatable). Only -k and -m are supported. @@ -82,6 +88,13 @@ Error: --resume "5678' OR '1'='1" is not an update ID; pass the update ID that p >>> musterr [CLI] pipelines test --pipeline-id 1234 --resume Error: --resume "" is not an update ID; pass the update ID that pipelines test printed +>>> musterr [CLI] pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml +Error: --junit-xml cannot be used with --no-wait because no results are available when the command returns + +=== An invalid flag replaces an older report +>>> musterr [CLI] pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml +Error: unsupported pytest argument "-x": only -k and -m are supported + === JSON output is not supported >>> musterr [CLI] pipelines test --pipeline-id 1234 -o json Error: --output json is not supported by pipelines test diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index ae91c307706..5e5db690f86 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -17,6 +17,11 @@ trace musterr $CLI pipelines test --pipeline-id 1234 --resume 5678 --no-wait trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --timeout 10m trace musterr $CLI pipelines test --pipeline-id 1234 --resume "5678' OR '1'='1" trace musterr $CLI pipelines test --pipeline-id 1234 --resume "" +trace musterr $CLI pipelines test --pipeline-id 1234 --no-wait --junit-xml out.no-wait.xml + +title "An invalid flag replaces an older report" +echo '' > out.flag-error.xml +trace musterr $CLI pipelines test --pipeline-id 1234 --pytest-arg=-x --junit-xml out.flag-error.xml title "JSON output is not supported" trace musterr $CLI pipelines test --pipeline-id 1234 -o json diff --git a/acceptance/pipelines/test/results/out.failures.xml b/acceptance/pipelines/test/results/out.failures.xml new file mode 100644 index 00000000000..5bf009266fd --- /dev/null +++ b/acceptance/pipelines/test/results/out.failures.xml @@ -0,0 +1,15 @@ + + + + + + AssertionError: assert 100 == 150 + + + + + + RuntimeError: fixture 'orders' setup failed + + + diff --git a/acceptance/pipelines/test/results/out.incomplete.xml b/acceptance/pipelines/test/results/out.incomplete.xml new file mode 100644 index 00000000000..c6e324f0f6b --- /dev/null +++ b/acceptance/pipelines/test/results/out.incomplete.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + RUN INCOMPLETE: the update failed before all test results were received. update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. + + + diff --git a/acceptance/pipelines/test/results/out.late-failure.xml b/acceptance/pipelines/test/results/out.late-failure.xml new file mode 100644 index 00000000000..5e9a004942b --- /dev/null +++ b/acceptance/pipelines/test/results/out.late-failure.xml @@ -0,0 +1,12 @@ + + + + + AssertionError: assert 100 == 150 + + + + update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + + + diff --git a/acceptance/pipelines/test/results/out.no-tests.xml b/acceptance/pipelines/test/results/out.no-tests.xml new file mode 100644 index 00000000000..68f35e0ab2c --- /dev/null +++ b/acceptance/pipelines/test/results/out.no-tests.xml @@ -0,0 +1,8 @@ + + + + + No tests were collected. + + + diff --git a/acceptance/pipelines/test/results/out.passed.xml b/acceptance/pipelines/test/results/out.passed.xml new file mode 100644 index 00000000000..95d1e6f3d0a --- /dev/null +++ b/acceptance/pipelines/test/results/out.passed.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/acceptance/pipelines/test/results/out.update-failed.xml b/acceptance/pipelines/test/results/out.update-failed.xml new file mode 100644 index 00000000000..a115d599dc0 --- /dev/null +++ b/acceptance/pipelines/test/results/out.update-failed.xml @@ -0,0 +1,9 @@ + + + + + + update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. + + + diff --git a/acceptance/pipelines/test/results/output.txt b/acceptance/pipelines/test/results/output.txt index 6f3ccfb2842..d0b3b88be8a 100644 --- a/acceptance/pipelines/test/results/output.txt +++ b/acceptance/pipelines/test/results/output.txt @@ -7,7 +7,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[WORKSPACE_ID] === All selected tests pass ->>> [CLI] pipelines test my_pipeline tests/test_orders.py::test_valid_order +>>> [CLI] pipelines test my_pipeline tests/test_orders.py::test_valid_order --junit-xml out.passed.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -19,6 +19,7 @@ Running tests... tests/test_orders.py::test_valid_order PASSED (1.00s) 1 test: 1 passed, 0 failed, 0 skipped, 0 errors +JUnit report: out.passed.xml >>> print_requests.py --get //events { @@ -59,7 +60,7 @@ tests/test_orders.py::test_valid_order PASSED (1.00s) } === Failed, errored, and skipped tests ->>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py +>>> musterr [CLI] pipelines test my_pipeline tests/test_orders.py --junit-xml out.failures.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -80,9 +81,10 @@ tests/test_orders.py::test_fixture_error RuntimeError: fixture 'orders' setup failed 4 tests: 1 passed, 1 failed, 1 skipped, 1 error +JUnit report: out.failures.xml === No tests collected ->>> musterr [CLI] pipelines test my_pipeline tests/helpers.py +>>> musterr [CLI] pipelines test my_pipeline tests/helpers.py --junit-xml out.no-tests.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -93,9 +95,10 @@ Waiting for compute... Running tests... No tests were collected. +JUnit report: out.no-tests.xml === The update fails after all tests passed ->>> musterr [CLI] pipelines test my_pipeline tests/test_delivery.py +>>> musterr [CLI] pipelines test my_pipeline tests/test_delivery.py --junit-xml out.update-failed.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -107,10 +110,11 @@ Running tests... tests/test_delivery.py::test_breaks_update_after_summary PASSED (1.00s) 1 test: 1 passed, 0 failed, 0 skipped, 0 errors +JUnit report: out.update-failed.xml Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. === The update fails with an error of its own after a test failed ->>> musterr [CLI] pipelines test my_pipeline tests/test_late_failure.py +>>> musterr [CLI] pipelines test my_pipeline tests/test_late_failure.py --junit-xml out.late-failure.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -127,10 +131,11 @@ tests/test_late_failure.py::test_backfill_fails AssertionError: assert 100 == 150 2 tests: 1 passed, 1 failed, 0 skipped, 0 errors +JUnit report: out.late-failure.xml Error: update [UUID] is FAILED: ResultDeliveryException: Test results could not be verified. === Compute terminates before all results are received ->>> musterr [CLI] pipelines test my_pipeline tests/test_compute.py +>>> musterr [CLI] pipelines test my_pipeline tests/test_compute.py --junit-xml out.incomplete.xml Recommendation: This command runs the last deployed version of the code If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. @@ -143,4 +148,5 @@ tests/test_compute.py::test_small_order PASSED (1.00s) RUN INCOMPLETE: the update failed before all test results were received. Results received: 1 passed. +JUnit report: out.incomplete.xml Error: update [UUID] is FAILED: ComputeTerminatedException: Compute terminated unexpectedly. diff --git a/acceptance/pipelines/test/results/script b/acceptance/pipelines/test/results/script index e19daac850c..56779522991 100644 --- a/acceptance/pipelines/test/results/script +++ b/acceptance/pipelines/test/results/script @@ -2,21 +2,21 @@ trace $CLI pipelines deploy rm "$OUT_REQUESTS" title "All selected tests pass" -trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order +trace $CLI pipelines test my_pipeline tests/test_orders.py::test_valid_order --junit-xml out.passed.xml trace print_requests.py --get //events title "Failed, errored, and skipped tests" -trace musterr $CLI pipelines test my_pipeline tests/test_orders.py +trace musterr $CLI pipelines test my_pipeline tests/test_orders.py --junit-xml out.failures.xml title "No tests collected" -trace musterr $CLI pipelines test my_pipeline tests/helpers.py +trace musterr $CLI pipelines test my_pipeline tests/helpers.py --junit-xml out.no-tests.xml title "The update fails after all tests passed" -trace musterr $CLI pipelines test my_pipeline tests/test_delivery.py +trace musterr $CLI pipelines test my_pipeline tests/test_delivery.py --junit-xml out.update-failed.xml title "The update fails with an error of its own after a test failed" -trace musterr $CLI pipelines test my_pipeline tests/test_late_failure.py +trace musterr $CLI pipelines test my_pipeline tests/test_late_failure.py --junit-xml out.late-failure.xml title "Compute terminates before all results are received" -trace musterr $CLI pipelines test my_pipeline tests/test_compute.py +trace musterr $CLI pipelines test my_pipeline tests/test_compute.py --junit-xml out.incomplete.xml rm "$OUT_REQUESTS" diff --git a/acceptance/pipelines/test/run-control/out.invalid.xml b/acceptance/pipelines/test/run-control/out.invalid.xml new file mode 100644 index 00000000000..ff48bd9dac9 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.invalid.xml @@ -0,0 +1,8 @@ + + + + + reference does not exist: ${var.missing} + + + diff --git a/acceptance/pipelines/test/run-control/out.not-test-update.xml b/acceptance/pipelines/test/run-control/out.not-test-update.xml new file mode 100644 index 00000000000..89adb9dfed7 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.not-test-update.xml @@ -0,0 +1,8 @@ + + + + + update [UUID] is not a test update; pass the update ID that pipelines test printed + + + diff --git a/acceptance/pipelines/test/run-control/out.outside.xml b/acceptance/pipelines/test/run-control/out.outside.xml new file mode 100644 index 00000000000..1e1f250b228 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.outside.xml @@ -0,0 +1,8 @@ + + + + + outside a project, identify the pipeline with --pipeline-id + + + diff --git a/acceptance/pipelines/test/run-control/out.resumed.xml b/acceptance/pipelines/test/run-control/out.resumed.xml new file mode 100644 index 00000000000..c2fb2be3d65 --- /dev/null +++ b/acceptance/pipelines/test/run-control/out.resumed.xml @@ -0,0 +1,11 @@ + + + + + + + + RUN INCOMPLETE: the update was canceled before all test results were received. update [UUID] is CANCELED + + + diff --git a/acceptance/pipelines/test/run-control/output.txt b/acceptance/pipelines/test/run-control/output.txt index e1a1895d5f3..1c1f5806d17 100644 --- a/acceptance/pipelines/test/run-control/output.txt +++ b/acceptance/pipelines/test/run-control/output.txt @@ -33,11 +33,25 @@ If you've made local changes, run 'databricks pipelines deploy' first to ensure Pipeline: my_pipeline ([UUID]) Update: [UUID] ->>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s --junit-xml out.resumed.xml Pipeline: my_pipeline ([UUID]) Update: [UUID] Running tests... -Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --resume [UUID] +JUnit report: out.resumed.xml +Error: stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID] + +>>> cat out.resumed.xml + + + + + + + + stopped waiting for update [UUID] after 3s; the update keeps running, to get its results run: databricks pipelines test my_pipeline --junit-xml out.resumed.xml --resume [UUID] + + + === The command to read the results keeps the profile >>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --timeout 3s -p run-control @@ -51,14 +65,29 @@ Error: stopped waiting for update [UUID] after 3s; the update keeps running, to Stopping my_pipeline... my_pipeline has been stopped. ->>> musterr [CLI] pipelines test my_pipeline --resume [UUID] +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --junit-xml out.resumed.xml Pipeline: my_pipeline ([UUID]) Update: [UUID] RUN INCOMPLETE: the update was canceled before all test results were received. Results received: none. +JUnit report: out.resumed.xml Error: update [UUID] is CANCELED -=== --resume rejects an update that is not a test update ->>> musterr [CLI] pipelines test my_pipeline --resume [UUID] +=== --resume rejects an update that is not a test update, and an older report is replaced +>>> musterr [CLI] pipelines test my_pipeline --resume [UUID] --junit-xml out.not-test-update.xml Error: update [UUID] is not a test update; pass the update ID that pipelines test printed + +=== Outside a project, an older passing report is replaced +Error: outside a project, identify the pipeline with --pipeline-id + +=== A project with errors names the first error in the report +>>> musterr [CLI] pipelines test my_pipeline --junit-xml ../out.invalid.xml +Error: reference does not exist: ${var.missing} + + +=== When the report cannot be written either, the write error is shown +>>> musterr [CLI] pipelines test my_pipeline --junit-xml missing/report.xml +Error: reference does not exist: ${var.missing} + +Error: failed to write JUnit report: open missing/report.xml: no such file or directory diff --git a/acceptance/pipelines/test/run-control/script b/acceptance/pipelines/test/run-control/script index c640f6174a7..29d026169f6 100644 --- a/acceptance/pipelines/test/run-control/script +++ b/acceptance/pipelines/test/run-control/script @@ -11,7 +11,8 @@ title "--timeout stops waiting but not the update" trace $CLI pipelines test my_pipeline tests/test_slow.py --no-wait | tee submit.txt hang_update_id=$(sed -n 's/^Update: //p' submit.txt) rm submit.txt -trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout 3s +trace musterr $CLI pipelines test my_pipeline --resume $hang_update_id --timeout 3s --junit-xml out.resumed.xml +trace cat out.resumed.xml title "The command to read the results keeps the profile" cat > .databrickscfg <' > out.not-test-update.xml +trace musterr $CLI pipelines test my_pipeline --resume $regular_update_id --junit-xml out.not-test-update.xml + +title "Outside a project, an older passing report is replaced\n" +echo '' > out.outside.xml +here=$PWD +outside=$(mktemp -d) +cd $outside +musterr $CLI pipelines test --junit-xml "$here/out.outside.xml" +cd $here +rm -r $outside + +title "A project with errors names the first error in the report" +mkdir invalid +printf 'bundle:\n name: invalid\nresources:\n pipelines:\n my_pipeline:\n name: ${var.missing}\n' > invalid/databricks.yml +cd invalid +trace musterr $CLI pipelines test my_pipeline --junit-xml ../out.invalid.xml + +title "When the report cannot be written either, the write error is shown" +trace musterr $CLI pipelines test my_pipeline --junit-xml missing/report.xml +cd .. +rm -r invalid diff --git a/acceptance/pipelines/test/run-control/test.toml b/acceptance/pipelines/test/run-control/test.toml index a030353d571..40e8a1be234 100644 --- a/acceptance/pipelines/test/run-control/test.toml +++ b/acceptance/pipelines/test/run-control/test.toml @@ -1 +1,6 @@ RecordRequests = false + +# Replace error message from windows +[[Repls]] +Old = 'open missing/report.xml: The system cannot find the path specified\.' +New = 'open missing/report.xml: no such file or directory' diff --git a/acceptance/pipelines/test/submit-error/out.start-failed.xml b/acceptance/pipelines/test/submit-error/out.start-failed.xml new file mode 100644 index 00000000000..93125b3d911 --- /dev/null +++ b/acceptance/pipelines/test/submit-error/out.start-failed.xml @@ -0,0 +1,8 @@ + + + + + An active update '[UUID]' already exists for pipeline '1234'. + + + diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index 7f0fcedf532..7d3e524451f 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -11,3 +11,7 @@ Error: An active update '[UUID]' already exists for pipeline '1234'. "test_only": true } } + +=== A submit error replaces an older report +>>> musterr [CLI] pipelines test --pipeline-id [UUID] --junit-xml out.start-failed.xml +Error: An active update '[UUID]' already exists for pipeline '1234'. diff --git a/acceptance/pipelines/test/submit-error/script b/acceptance/pipelines/test/submit-error/script index 29d720cc633..4586be78c10 100644 --- a/acceptance/pipelines/test/submit-error/script +++ b/acceptance/pipelines/test/submit-error/script @@ -5,3 +5,7 @@ rm "$OUT_REQUESTS" title "A submit error from the service is shown as is, and no update ID is printed" trace musterr $CLI pipelines test --pipeline-id $pipeline_id trace print_requests.py //updates + +title "A submit error replaces an older report" +echo '' > out.start-failed.xml +trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.start-failed.xml diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 1f1c4cc1b14..822bd9afe25 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -188,10 +188,10 @@ func bundleShortTestID(cwd, syncRoot, filesRoot, id string) string { return filepath.ToSlash(local) + testID } -// rootPathShortTestID shows a test ID whose path is under rootPath relative to it, which --pipeline-id -// resolves against root_path to select the test again. Other test IDs are returned unchanged. -func rootPathShortTestID(rootPath, id string) string { - rel, testID, ok := cutTestIDFolder(rootPath, id) +// relativeTestID shows a test ID whose path is under folder relative to it. Other test IDs are returned +// unchanged. Relative to root_path, it is what --pipeline-id resolves to select the test again. +func relativeTestID(folder, id string) string { + rel, testID, ok := cutTestIDFolder(folder, id) if !ok { return id } @@ -309,6 +309,9 @@ type testTarget struct { w *databricks.WorkspaceClient // shortID shows a test ID in the shortest form that selects the test again. shortID func(id string) string + // projectFolder is the deployed folder that names test cases independently of the working + // directory: the bundle's synchronized files, or root_path. It is "" when unknown. + projectFolder string } func (t testTarget) printHeader(out io.Writer, updateID string) { @@ -346,8 +349,8 @@ func resolveTestTargetByID(cmd *cobra.Command, args []string, pipelineID string, if err != nil { return testTarget{}, err } - shortID := func(id string) string { return rootPathShortTestID(rootPath, id) } - return testTarget{pipelineID: pipelineID, selections: selections, w: w, shortID: shortID}, nil + shortID := func(id string) string { return relativeTestID(rootPath, id) } + return testTarget{pipelineID: pipelineID, selections: selections, w: w, shortID: shortID, projectFolder: rootPath}, nil } // resolveTestTargetInBundle identifies the pipeline by its bundle KEY, which must precede any selections. @@ -394,7 +397,7 @@ func resolveTestTargetInBundle(cmd *cobra.Command, args []string, resume bool) ( return testTarget{}, err } shortID := func(id string) string { return bundleShortTestID(cwd, b.SyncRootPath, filesRoot, id) } - return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), shortID: shortID}, nil + return testTarget{key: key, pipelineID: pipelineID, selections: selections, w: b.WorkspaceClient(ctx), shortID: shortID, projectFolder: filesRoot}, nil } // resumeCommand returns the command that reads the result of the update later. @@ -412,6 +415,10 @@ func (t testTarget) resumeCommand(cmd *cobra.Command, updateID string) string { if profile := cmd.Flag("profile").Value.String(); profile != "" { parts = append(parts, "--profile", shellquote.BashArg(profile)) } + // Reading the results again replaces the partial report. + if flag := cmd.Flag("junit-xml"); flag != nil && flag.Value.String() != "" { + parts = append(parts, "--junit-xml", shellquote.BashArg(flag.Value.String())) + } return strings.Join(append(parts, "--resume", updateID), " ") } @@ -507,6 +514,22 @@ func testRunError(updateID string, state pipelines.UpdateInfoState, run *testRun return fmt.Errorf("update %s is %s", updateID, state) } +// testRunProblem describes why a run that ended did not pass for a reason other than +// failed tests, or returns "" otherwise. +func testRunProblem(run *testRun, incompleteReason string, runErr error) string { + var lines []string + switch { + case incompleteReason != "": + lines = append(lines, fmt.Sprintf("RUN INCOMPLETE: %s.", incompleteReason)) + case run.summaries[0].Total == 0: + lines = append(lines, "No tests were collected.") + } + if runErr != nil && !errors.Is(runErr, root.ErrAlreadyPrinted) { + lines = append(lines, runErr.Error()) + } + return strings.Join(lines, "\n") +} + var errTestWaitTimeout = errors.New("test wait timed out") func testCommand() *cobra.Command { @@ -539,7 +562,12 @@ example: The command waits for the results and succeeds only if at least one test was collected and none failed or errored. --no-wait returns after the update starts. --timeout and Ctrl+C stop waiting but not the update; pass its ID to --resume -to read the results later. To stop the update, run 'pipelines stop'.`, +to read the results later. To stop the update, run 'pipelines stop'. + +--junit-xml writes one entry per test with its outcome and duration. An +incomplete run, a run without tests, or a timeout adds a failing entry that +names the cause. With workspace.sync_mode immutable, test names keep the path +of the deployed snapshot, which changes with each deployment.`, } var pipelineID string @@ -547,6 +575,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, var noWait bool var timeout time.Duration var resumeUpdateID string + var junitPath string testGroup := cmdgroup.NewFlagGroup("Pipeline Test") testGroup.FlagSet().StringVar(&pipelineID, "pipeline-id", "", "ID of the pipeline to test, instead of a PIPELINE_KEY.") @@ -554,6 +583,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, testGroup.FlagSet().BoolVar(&noWait, "no-wait", false, "Return after the test update starts, without waiting for results.") testGroup.FlagSet().DurationVar(&timeout, "timeout", 0, "Stop waiting for results after this duration, for example 30m. The update keeps running.") testGroup.FlagSet().StringVar(&resumeUpdateID, "resume", "", "Read the results of an existing test update instead of starting one.") + testGroup.FlagSet().StringVar(&junitPath, "junit-xml", "", "Write the results as a JUnit XML report to this file, replacing it.") wrappedCmd := cmdgroup.NewCommandWithGroupFlag(cmd) wrappedCmd.AddFlagGroup(testGroup) @@ -564,39 +594,54 @@ to read the results later. To stop the update, run 'pipelines stop'.`, if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled { return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar) } - if cmd.ArgsLenAtDash() >= 0 { - return errors.New("the -- separator is not supported: " + pytestArgHint) - } - if root.OutputType(cmd) != flags.OutputText { - return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) - } - if err := validatePytestArgs(pytestArgs); err != nil { - return err - } - if timeout < 0 { - return errors.New("--timeout must not be negative") - } - if noWait && timeout > 0 { - return errors.New("--timeout cannot be used with --no-wait") - } // An empty --resume, such as from an unset shell variable, must not start a new run. resume := cmd.Flags().Changed("resume") - if resume { - selectionCount := len(args) - if pipelineID == "" { - selectionCount = max(len(args)-1, 0) + checkFlags := func() error { + if cmd.ArgsLenAtDash() >= 0 { + return errors.New("the -- separator is not supported: " + pytestArgHint) + } + if root.OutputType(cmd) != flags.OutputText { + return fmt.Errorf("--output %s is not supported by pipelines test", root.OutputType(cmd)) } - switch { - case noWait: - return errors.New("--resume cannot be used with --no-wait") - case len(pytestArgs) > 0: - return errors.New("--resume cannot be used with --pytest-arg") - case selectionCount > 0: - return errors.New("--resume cannot be used with test selections") + if err := validatePytestArgs(pytestArgs); err != nil { + return err + } + if timeout < 0 { + return errors.New("--timeout must not be negative") + } + if noWait && timeout > 0 { + return errors.New("--timeout cannot be used with --no-wait") + } + if noWait && junitPath != "" { + return errors.New("--junit-xml cannot be used with --no-wait because no results are available when the command returns") } - if _, err := uuid.Parse(resumeUpdateID); err != nil { - return fmt.Errorf("--resume %q is not an update ID; pass the update ID that pipelines test printed", resumeUpdateID) + if resume { + selectionCount := len(args) + if pipelineID == "" { + selectionCount = max(len(args)-1, 0) + } + switch { + case noWait: + return errors.New("--resume cannot be used with --no-wait") + case len(pytestArgs) > 0: + return errors.New("--resume cannot be used with --pytest-arg") + case selectionCount > 0: + return errors.New("--resume cannot be used with test selections") + } + if _, err := uuid.Parse(resumeUpdateID); err != nil { + return fmt.Errorf("--resume %q is not an update ID; pass the update ID that pipelines test printed", resumeUpdateID) + } } + return nil + } + + // Every exit writes the report, so that an older report never survives. + report := &testReport{path: junitPath, out: cmd.OutOrStdout(), suiteName: pipelineID} + if report.suiteName == "" && len(args) > 0 { + report.suiteName = args[0] + } + if err := checkFlags(); err != nil { + return report.failed(cmd.Context(), err) } var target testTarget @@ -607,13 +652,18 @@ to read the results later. To stop the update, run 'pipelines stop'.`, ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) if !bundle.HasRoot(ctx) { - return errors.New("outside a project, identify the pipeline with --pipeline-id") + return report.failed(ctx, errors.New("outside a project, identify the pipeline with --pipeline-id")) } target, err = resolveTestTargetInBundle(cmd, args, resume) } if err != nil { - return err + return report.failed(cmd.Context(), err) } + report.suiteName = target.key + if report.suiteName == "" { + report.suiteName = target.pipelineID + } + report.projectFolder = target.projectFolder // The CLI root doesn't cancel the context on signals, so stop waiting here. ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -624,10 +674,10 @@ to read the results later. To stop the update, run 'pipelines stop'.`, req := newStartTestUpdateRequest(target.selections, pytestArgs) updateID, err = startTestUpdate(ctx, target.w, target.pipelineID, req) if err != nil { - return err + return report.failed(ctx, err) } } else if err := checkTestUpdate(ctx, target.w, target.pipelineID, updateID); err != nil { - return err + return report.failed(ctx, err) } target.printHeader(out, updateID) if noWait { @@ -636,7 +686,7 @@ to read the results later. To stop the update, run 'pipelines stop'.`, events, err := newTestEventReader(target.w, target.pipelineID, updateID) if err != nil { - return err + return report.failed(ctx, err) } watcher := &testWatcher{ w: target.w, @@ -656,18 +706,20 @@ to read the results later. To stop the update, run 'pipelines stop'.`, state, err := watcher.watch(waitCtx) if err != nil { if waitCtx.Err() == nil { - return err + return report.finish(updateID, watcher.run, err.Error(), err) } resume := target.resumeCommand(cmd, updateID) + stopErr := fmt.Errorf("stopped waiting for update %s; the update keeps running, to get its results run: %s", updateID, resume) if errors.Is(context.Cause(waitCtx), errTestWaitTimeout) { - return fmt.Errorf("stopped waiting for update %s after %s; the update keeps running, to get its results run: %s", updateID, timeout, resume) + stopErr = fmt.Errorf("stopped waiting for update %s after %s; the update keeps running, to get its results run: %s", updateID, timeout, resume) } - return fmt.Errorf("stopped waiting for update %s; the update keeps running, to get its results run: %s", updateID, resume) + return report.finish(updateID, watcher.run, stopErr.Error(), stopErr) } incompleteReason := watcher.run.incompleteReason(state) renderTestResult(out, watcher.run, incompleteReason, target.shortID) - return testRunError(updateID, state, watcher.run, incompleteReason) + runErr := testRunError(updateID, state, watcher.run, incompleteReason) + return report.finish(updateID, watcher.run, testRunProblem(watcher.run, incompleteReason, runErr), runErr) } return cmd diff --git a/cmd/pipelines/test_junit.go b/cmd/pipelines/test_junit.go new file mode 100644 index 00000000000..75939821082 --- /dev/null +++ b/cmd/pipelines/test_junit.go @@ -0,0 +1,190 @@ +package pipelines + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/logdiag" +) + +type junitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []junitTestSuite `xml:"testsuite"` +} + +type junitTestSuite struct { + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr"` + Cases []junitTestCase `xml:"testcase"` +} + +type junitTestCase struct { + ClassName string `xml:"classname,attr,omitempty"` + Name string `xml:"name,attr"` + Time string `xml:"time,attr,omitempty"` + Failure *junitResult `xml:"failure"` + Error *junitResult `xml:"error"` + Skipped *junitResult `xml:"skipped"` +} + +type junitResult struct { + Message string `xml:"message,attr,omitempty"` + Text string `xml:",chardata"` +} + +// junitCaseName splits a pytest test ID the way pytest's JUnit report does: the classname +// is the module path plus any class names, joined by dots. See +// https://docs.pytest.org/en/stable/how-to/output.html#creating-junitxml-format-files. +// A parameter ID in brackets, such as test_cast[a::int], stays with the test name. +func junitCaseName(testID string) (string, string) { + params := "" + if i := strings.Index(testID, "["); i >= 0 { + testID, params = testID[:i], testID[i:] + } + parts := strings.Split(testID, "::") + parts[len(parts)-1] += params + module := strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(parts[0], "/"), ".py"), "/", ".") + if len(parts) == 1 { + return "", module + } + classes := append([]string{module}, parts[1:len(parts)-1]...) + return strings.Join(classes, "."), parts[len(parts)-1] +} + +func newJUnitResult(text string) *junitResult { + message, _, _ := strings.Cut(text, "\n") + return &junitResult{Message: message, Text: text} +} + +// junitNoResult is the error message of a collected case that has no final outcome. +const junitNoResult = "no result received" + +// newJUnitReport has one entry per collected case; a case without a final outcome is an error, +// because CI tools treat skipped entries as passing. +// A non-empty problem, such as an incomplete run, adds a failing entry for the update so the +// report never passes by omission. +// Cases are named relative to projectFolder, as pytest names them, so that the names do not depend on +// the working directory. +func newJUnitReport(suiteName, updateID, projectFolder string, run *testRun, problem string) junitTestSuites { + suite := junitTestSuite{Name: suiteName} + for _, c := range run.cases { + entry := junitTestCase{} + entry.ClassName, entry.Name = junitCaseName(relativeTestID(projectFolder, c.id)) + if d, ok := c.duration(); ok { + entry.Time = fmt.Sprintf("%.3f", d.Seconds()) + } + + details := "" + if c.errorDetail != nil { + details = errorDetailText(c.errorDetail) + } + if c.detailsTruncated { + details = strings.TrimPrefix(details+"\n(details truncated)", "\n") + } + switch c.status { + case testCasePassed: + case testCaseFailed: + entry.Failure = newJUnitResult(details) + suite.Failures++ + case testCaseError: + entry.Error = newJUnitResult(details) + suite.Errors++ + case testCaseSkipped: + entry.Skipped = &junitResult{} + suite.Skipped++ + default: + entry.Error = &junitResult{Message: junitNoResult} + suite.Errors++ + } + suite.Cases = append(suite.Cases, entry) + } + + if problem != "" { + suite.Cases = append(suite.Cases, junitTestCase{ + Name: "update " + updateID, + Error: newJUnitResult(problem), + }) + suite.Errors++ + } + suite.Tests = len(suite.Cases) + return junitTestSuites{Suites: []junitTestSuite{suite}} +} + +// testReport writes the JUnit report of a test run to path, if --junit-xml is set. +type testReport struct { + path string + out io.Writer + suiteName string + projectFolder string +} + +// finish writes the report of a run that ended with runErr. A write error is joined with runErr, +// so that neither is lost. +func (r *testReport) finish(updateID string, run *testRun, problem string, runErr error) error { + if r.path == "" { + return runErr + } + if err := writeJUnitReport(r.path, newJUnitReport(r.suiteName, updateID, r.projectFolder, run, problem)); err != nil { + if runErr == nil || errors.Is(runErr, root.ErrAlreadyPrinted) { + return err + } + return errors.Join(runErr, err) + } + fmt.Fprintf(r.out, "JUnit report: %s\n", filepath.ToSlash(r.path)) + return runErr +} + +// failed writes a report with one failing entry for an error that ended the command before the +// update could be read, so that an older report does not survive. A write error is joined with +// err, as in finish. +func (r *testReport) failed(ctx context.Context, err error) error { + if r.path == "" { + return err + } + message := err.Error() + printed := errors.Is(err, root.ErrAlreadyPrinted) + if printed { + // The error was printed as a diagnostic. Name it instead of the sentinel's text. + message = "the project has errors, see the command output" + if logdiag.IsSetup(ctx) && logdiag.GetFirstErrorSummary(ctx) != "" { + message = logdiag.GetFirstErrorSummary(ctx) + } + } + report := junitTestSuites{Suites: []junitTestSuite{{ + Name: r.suiteName, + Tests: 1, + Errors: 1, + Cases: []junitTestCase{{Name: "pipelines test", Error: newJUnitResult(message)}}, + }}} + if writeErr := writeJUnitReport(r.path, report); writeErr != nil { + if printed { + return writeErr + } + return errors.Join(err, writeErr) + } + return err +} + +// writeJUnitReport replaces the file at path with the report. +func writeJUnitReport(path string, report junitTestSuites) error { + data, err := xml.MarshalIndent(report, "", " ") + if err != nil { + return err + } + data = append([]byte(xml.Header), data...) + data = append(data, '\n') + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("failed to write JUnit report: %w", err) + } + return nil +} diff --git a/cmd/pipelines/test_junit_test.go b/cmd/pipelines/test_junit_test.go new file mode 100644 index 00000000000..c2e973db909 --- /dev/null +++ b/cmd/pipelines/test_junit_test.go @@ -0,0 +1,73 @@ +package pipelines + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJUnitCaseName(t *testing.T) { + tests := []struct { + testID, className, name string + }{ + {"tests/test_orders.py::test_backfill", "tests.test_orders", "test_backfill"}, + {"tests/test_orders.py::TestOrders::test_tax[a-b]", "tests.test_orders.TestOrders", "test_tax[a-b]"}, + // A parameter ID may contain "::". + {"tests/test_a.py::test_cast[a::int]", "tests.test_a", "test_cast[a::int]"}, + {"tests/test_a.py::TestCast::test_cast[a::int]", "tests.test_a.TestCast", "test_cast[a::int]"}, + {"tests/test_orders.py", "", "tests.test_orders"}, + {"/Workspace/Users/someone@example.com/tests/test_x.py::test_x", "Workspace.Users.someone@example.com.tests.test_x", "test_x"}, + } + for _, tc := range tests { + className, name := junitCaseName(tc.testID) + assert.Equal(t, tc.className, className, tc.testID) + assert.Equal(t, tc.name, name, tc.testID) + } +} + +func TestWriteJUnitReport(t *testing.T) { + // Acceptance tests cover each outcome. This covers what the test server does not send: + // multi-line and truncated details, and text that XML must escape. + const project = "/Workspace/Users/someone@example.com/.bundle/orders/dev/files" + run := newTestRun() + for _, e := range []testPipelineEvent{ + caseEvent(t, project+"/tests/test_a.py::test_bad", "FAILED", "", + `"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`, + `"truncation": {"truncated_fields": [{"field_name": "error"}]}`), + caseEvent(t, project+"/tests/test_a.py::test_setup", "ERROR", "", + `"error": {"exceptions": [{"class_name": "RuntimeError", "message": "fixture failed"}]}`), + } { + run.add(e) + } + + path := filepath.Join(t.TempDir(), "report.xml") + require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", project, run, ""))) + + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, ` + + + + AssertionError: assert 1 == 2 where 1 = f() (details truncated) + + + RuntimeError: fixture <db> failed + + + +`, string(data)) +} + +func TestTestReportJoinsWriteErrors(t *testing.T) { + report := &testReport{path: filepath.Join(t.TempDir(), "missing", "report.xml"), out: io.Discard, suiteName: "orders"} + runErr := errors.New("update u1 is FAILED") + err := report.finish("u1", newTestRun(), runErr.Error(), runErr) + require.ErrorIs(t, err, runErr) + assert.ErrorContains(t, err, "failed to write JUnit report") +} diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index ad4844213ad..d94781ae606 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -93,20 +93,20 @@ func TestBundleShortTestID(t *testing.T) { assert.Equal(t, id, bundleShortTestID(syncRoot, syncRoot, "", id)) } -func TestRootPathShortTestID(t *testing.T) { +func TestRelativeTestID(t *testing.T) { rootPath := "/Workspace/Users/someone/orders" - assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID(rootPath, rootPath+"/tests/test_a.py::test_x")) - assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID(rootPath+"/", rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID(rootPath, rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID(rootPath+"/", rootPath+"/tests/test_a.py::test_x")) // The service accepts workspace paths with or without the /Workspace prefix. - assert.Equal(t, "tests/test_a.py::test_x", rootPathShortTestID("/Users/someone/orders", rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, "tests/test_a.py::test_x", relativeTestID("/Users/someone/orders", rootPath+"/tests/test_a.py::test_x")) for _, other := range []string{ "/Workspace/Users/someone/shared/test_s.py::test_s", rootPath + "2/test_a.py::test_x", "tests/test_a.py::test_x", } { - assert.Equal(t, other, rootPathShortTestID(rootPath, other)) + assert.Equal(t, other, relativeTestID(rootPath, other)) } - assert.Equal(t, rootPath+"/tests/test_a.py::test_x", rootPathShortTestID("", rootPath+"/tests/test_a.py::test_x")) + assert.Equal(t, rootPath+"/tests/test_a.py::test_x", relativeTestID("", rootPath+"/tests/test_a.py::test_x")) } // Source-linked deployments run the source files, as TranslatePaths maps them. @@ -164,6 +164,7 @@ func TestTestTargetResumeCommand(t *testing.T) { rootCmd.PersistentFlags().StringP("target", "t", "", "") pipelinesCmd := &cobra.Command{Use: "pipelines"} cmd := &cobra.Command{Use: "test"} + cmd.Flags().String("junit-xml", "", "") rootCmd.AddCommand(pipelinesCmd) pipelinesCmd.AddCommand(cmd) for name, value := range tc.flags {