do-not-merge/testing-nuts-failure-logs

.github/workflows/test.yml

@@ -21,6 +21,18 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
+        # Split the NUTs across jobs so each group runs on its own runner (its own
+        # egress IP). The `login rate exceeded` OAuth cap is per-IP, and running the
+        # whole suite serially on one IP exhausts the budget before run-eval. run-eval
+        # is isolated on its own job for maximum headroom; the z0-z5 pipeline stays in
+        # one job because its tests share one org in create->...->trace order.
+        command:
+          - 'yarn test:nuts:run-eval'
+          - 'yarn test:nuts:test'
+          - 'yarn test:nuts:generate'
+          - 'yarn test:nuts:z'
+          - 'yarn test:nuts:adl-mcp'
       fail-fast: false
     with:
       os: ${{ matrix.os }}
+      command: ${{ matrix.command }}

README.md

@@ -2498,3 +2498,5 @@ EXAMPLES
 _See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/2.2.3/src/commands/agent/validate/authoring-bundle.ts)_
 
 <!-- commandsstop -->
+
+<!-- CI trigger: verifying NUT failure-log output ([NUT-CONTEXT] / [NUT-FAILURE]). Do not merge. -->

package.json

@@ -139,6 +139,11 @@
     "prepare": "sf-install",
     "test": "wireit",
     "test:nuts": "nyc mocha \"test/nuts/**/*.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit",
+    "test:nuts:run-eval": "nyc mocha \"test/nuts/agent.test.run-eval.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit",
+    "test:nuts:test": "nyc mocha \"test/nuts/agent.test.nut.ts\" \"test/nuts/agent.test.create.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit",
+    "test:nuts:generate": "nyc mocha \"test/nuts/agent.activate.nut.ts\" \"test/nuts/agent.generate.*.nut.ts\" \"test/nuts/template.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit",
+    "test:nuts:z": "nyc mocha \"test/nuts/z*.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit",
+    "test:nuts:adl-mcp": "nyc mocha \"test/nuts/agent.adl.nut.ts\" \"test/nuts/agent.mcp.nut.ts\" --slow 4500 --timeout 600000 --reporter-options maxDiffSize=15000 --exit",
     "test:only": "wireit",
     "version": "oclif readme"
   },

test/nuts/agent.test.run-eval.nut.ts

@@ -29,55 +29,65 @@ describe('agent test run-eval', function () {
   const yamlSpecPath = join(mockProjectDir, 'eval-test-spec.yaml');
   const yamlWithContextPath = join(mockProjectDir, 'eval-with-context.yaml');
 
+  // The JSON-payload run-eval is expensive (~18 min) and hammers the shared admin user's
+  // OAuth refresh rate limit when invoked repeatedly. Run it exactly once here and share the
+  // parsed result across every test that asserts against it, so a rate-limit failure surfaces
+  // once rather than per-test.
+  let jsonResult: RunEvalResult;
+
   before(async function () {
     this.timeout(30 * 60 * 1000); // 30 minutes for setup
     await getTestSession();
+
+    const jsonCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
+    const jsonExec = execCmd<RunEvalResult>(jsonCommand, { ensureExitCode: 0 });
+    expect(jsonExec.jsonOutput?.result).to.be.ok;
+    jsonResult = jsonExec.jsonOutput!.result;
   });
 
   describe('run-eval with JSON file', () => {
     it('should run evaluation with JSON payload file', async () => {
-      const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
-      const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });
-
-      expect(result.jsonOutput?.result).to.be.ok;
-      expect(result.jsonOutput?.result.tests).to.be.an('array');
-      expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0);
-      expect(result.jsonOutput?.result.summary).to.be.ok;
-      expect(result.jsonOutput?.result.summary.passed).to.be.a('number');
-      expect(result.jsonOutput?.result.summary.failed).to.be.a('number');
-      expect(result.jsonOutput?.result.summary.scored).to.be.a('number');
-      expect(result.jsonOutput?.result.summary.errors).to.be.a('number');
+      expect(jsonResult).to.be.ok;
+      expect(jsonResult.tests).to.be.an('array');
+      expect(jsonResult.tests.length).to.be.greaterThan(0);
+      expect(jsonResult.summary).to.be.ok;
+      expect(jsonResult.summary.passed).to.be.a('number');
+      expect(jsonResult.summary.failed).to.be.a('number');
+      expect(jsonResult.summary.scored).to.be.a('number');
+      expect(jsonResult.summary.errors).to.be.a('number');
     });
 
     it('should run evaluation with normalized payload', async () => {
-      const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
-      const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });
-
-      expect(result.jsonOutput?.result.tests[0]).to.be.ok;
-      expect(result.jsonOutput?.result.tests[0].id).to.equal('test-topic-routing');
-      expect(result.jsonOutput?.result.tests[0].status).to.be.oneOf(['passed', 'failed']);
-      expect(result.jsonOutput?.result.tests[0].evaluations).to.be.an('array');
+      expect(jsonResult.tests[0]).to.be.ok;
+      expect(jsonResult.tests[0].id).to.equal('test-topic-routing');
+      expect(jsonResult.tests[0].status).to.be.oneOf(['passed', 'failed']);
+      expect(jsonResult.tests[0].evaluations).to.be.an('array');
     });
   });
 
   describe('run-eval with YAML file', () => {
-    it('should run evaluation with YAML test spec file', async () => {
+    // The YAML-spec run-eval is run once and shared across the two tests that assert against it,
+    // for the same OAuth-rate-limit reason as the JSON-payload run above.
+    let yamlResult: RunEvalResult;
+
+    before(() => {
       const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`;
-      const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });
+      const output = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });
+      expect(output.jsonOutput?.result).to.be.ok;
+      yamlResult = output.jsonOutput!.result;
+    });
 
-      expect(result.jsonOutput?.result).to.be.ok;
-      expect(result.jsonOutput?.result.tests).to.be.an('array');
-      expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0);
-      expect(result.jsonOutput?.result.summary).to.be.ok;
+    it('should run evaluation with YAML test spec file', async () => {
+      expect(yamlResult).to.be.ok;
+      expect(yamlResult.tests).to.be.an('array');
+      expect(yamlResult.tests.length).to.be.greaterThan(0);
+      expect(yamlResult.summary).to.be.ok;
     });
 
     it('should auto-infer agent name from YAML subjectName', async () => {
-      const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`;
-      const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });
-
       // Should succeed without explicit --api-name flag
-      expect(result.jsonOutput?.result).to.be.ok;
-      expect(result.jsonOutput?.result.tests).to.be.an('array');
+      expect(yamlResult).to.be.ok;
+      expect(yamlResult.tests).to.be.an('array');
     });
 
     it('should handle YAML spec with contextVariables', async () => {
@@ -173,11 +183,8 @@ describe('agent test run-eval', function () {
 
   describe('run-eval output structure', () => {
     it('should include test summaries with correct structure', async () => {
-      const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
-      const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });
-
-      expect(result.jsonOutput?.result.tests).to.be.an('array');
-      const firstTest = result.jsonOutput?.result.tests[0];
+      expect(jsonResult.tests).to.be.an('array');
+      const firstTest = jsonResult.tests[0];
       expect(firstTest).to.have.property('id');
       expect(firstTest).to.have.property('status');
       expect(firstTest).to.have.property('evaluations');
@@ -187,10 +194,7 @@ describe('agent test run-eval', function () {
     });
 
     it('should include summary with all metrics', async () => {
-      const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
-      const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });
-
-      const summary = result.jsonOutput?.result.summary;
+      const summary = jsonResult.summary;
       expect(summary).to.have.property('passed');
       expect(summary).to.have.property('failed');
       expect(summary).to.have.property('scored');