Show every Ruby's speed side by side on the results site

.github/workflows/benchmarks.yml

@@ -1,5 +1,7 @@
 name: Benchmarks
 
+# Checks that every benchmark runs on every Ruby (PRs: only the files they change).
+# It publishes nothing: the results site comes from results-site.yml.
 # Always runs, so the benchmarks-ok check at the end always reports and can be
 # required. The changes job decides whether there is anything to benchmark.
 on:
@@ -67,14 +69,12 @@ jobs:
           - { ruby: ruby_4.0, variant: zjit, flags: --zjit }
           - { ruby: ruby_head, variant: zjit, flags: --zjit }
 
-    # Read by docker/run-benchmarks.sh and docker/collect_results.rb.
     env:
+      # Read by docker/run-benchmarks.sh.
       RUBY_VARIANT: ${{ matrix.variant }}
       RUBY_VARIANT_FLAGS: ${{ matrix.flags }}
-      RESULTS_DIR: results
-      RESULTS_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
-      RESULTS_PR: ${{ github.event.pull_request.number }}
-      RESULTS_LABEL: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }}
+      # The build's name in the step names and the failure report, for example ruby_3.4+yjit.
+      BUILD: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }}
 
     steps:
       # Plain Rubies only: shared reports do not record the Ruby or its flags,
@@ -84,113 +84,13 @@ jobs:
         run: |
           echo "SHARE=1" >> "$GITHUB_ENV"
       - uses: actions/checkout@v4
-      - name: Run benchmarks on ${{ env.RESULTS_LABEL }}
+      - name: Run benchmarks on ${{ env.BUILD }}
         env:
           FILES: ${{ needs.changes.outputs.files }}
         run: .github/scripts/run-benchmarks.sh ${{ matrix.ruby }}
       - name: Report failed benchmarks
         if: failure()
-        run: .github/scripts/report-failures.sh benchmarks.log ${{ env.RESULTS_LABEL }}
-      - name: Upload results
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: results-${{ env.RESULTS_LABEL }}
-          path: results/
-          if-no-files-found: warn
-
-  # Builds the results site from every job's results, also when some jobs
-  # failed, so a PR gets a preview (the site-preview artifact, named outside
-  # the results-* pattern so a re-run never downloads it). Skipped when the
-  # benchmark jobs did not run (a lint failure). Only a push to
-  # main where every job passed publishes it, so a partial run never replaces
-  # the live site. Not required: a Pages problem never blocks a merge.
-  site:
-    needs: [changes, rake]
-    if: ${{ !cancelled() && needs.changes.outputs.run == 'true' && needs.rake.result != 'skipped' }}
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: results-*
-          path: results/
-          merge-multiple: true
-      - name: Build the results site
-        # A failed benchmark job leaves no result file, like a benchmark that needs a newer Ruby, so the page says missing results may be crashes.
-        env:
-          RESULTS_INCOMPLETE: ${{ needs.rake.result != 'success' && '1' || '' }}
-        run: docker compose run --rm -T -e RESULTS_INCOMPLETE --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site
-      - name: Upload the site preview
-        id: preview
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-preview
-          path: _site/
-          # A re-run replaces the earlier attempt's preview.
-          overwrite: true
-      # A one-click link on the run's Summary page; it needs no extra permissions, so it works for PRs from forks too.
-      - name: Link the site preview in the run summary
-        env:
-          PREVIEW_URL: ${{ steps.preview.outputs.artifact-url }}
-        run: |
-          {
-            echo "### Results site preview"
-            echo ""
-            echo "[Download the site preview]($PREVIEW_URL) (a zip, needs a GitHub login), unzip it and open \`index.html\` in a browser."
-          } >> "$GITHUB_STEP_SUMMARY"
-      - name: Upload the site for GitHub Pages
-        if: github.event_name == 'push' && github.ref_name == 'main' && needs.rake.result == 'success'
-        uses: actions/upload-pages-artifact@v5
-        with:
-          path: _site/
-
-  deploy:
-    needs: [rake, site]
-    if: github.event_name == 'push' && github.ref_name == 'main' && needs.rake.result == 'success' && needs.site.result == 'success'
-    runs-on: ubuntu-latest
-    # Only this job can publish; the rest of the workflow keeps the default
-    # token permissions.
-    permissions:
-      pages: write
-      id-token: write
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    # One deploy at a time; a newer merge waits instead of cancelling one that
-    # is halfway through.
-    concurrency:
-      group: pages
-      cancel-in-progress: false
-
-    steps:
-      # Runs can finish out of order, so an older run must not put its results back over a newer one.
-      # When main moved on, ask pick-benchmarks.sh (the same rule CI uses) whether the newer commits run benchmarks.
-      # If they do, their own run deploys newer results; if not (a README-only merge), this run's results are still the newest.
-      - uses: actions/checkout@v4
-        with:
-          ref: main
-          fetch-depth: 0
-      - name: Check no newer run on main will deploy
-        id: newest
-        run: |
-          if [ "$(git rev-parse HEAD)" = "$GITHUB_SHA" ]; then
-            echo "deploy=true" >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          GITHUB_EVENT_NAME=push GITHUB_OUTPUT=newer.txt .github/scripts/pick-benchmarks.sh "$GITHUB_SHA"
-          if grep -q '^run=true' newer.txt; then
-            echo "main moved on and its newer commits run benchmarks, so their run deploys; skipping."
-            echo "deploy=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "main moved on, but nothing since this run affects benchmarks; deploying."
-            echo "deploy=true" >> "$GITHUB_OUTPUT"
-          fi
-      - name: Deploy to GitHub Pages
-        id: deployment
-        if: steps.newest.outputs.deploy == 'true'
-        uses: actions/deploy-pages@v5
+        run: .github/scripts/report-failures.sh benchmarks.log ${{ env.BUILD }}
 
   # The check to require on main. Passes when every benchmark job passed, or
   # when there was nothing to benchmark.

.github/workflows/results-site.yml

@@ -0,0 +1,96 @@
+name: Results site
+
+# Builds and publishes the results site (https://fastruby.github.io/fast-ruby/).
+# Every Ruby runs each benchmark on the same machine, so the site can compare i/s across Rubies; benchmarks.yml runs each Ruby on its own machine and only checks that the benchmarks run.
+# Weekly, because a full run takes about 3 hours per shard; run it by hand to publish sooner.
+on:
+  schedule:
+    # Sunday 03:00 UTC, away from the working day's PR runs.
+    - cron: '0 3 * * 0'
+  workflow_dispatch:
+
+jobs:
+  # 6 machines, each running every 6th benchmark file on all 26 builds, with the newest released Ruby run again between them (script/run_cross_ruby.rb).
+  shard:
+    name: shard ${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    # 35 passes of about 13 files each; the job limit is 6 hours.
+    timeout-minutes: 330
+
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2, 3, 4, 5]
+
+    # Read by docker/collect_results.rb.
+    env:
+      RESULTS_COMMIT: ${{ github.sha }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run every build on this machine
+        run: ruby script/run_cross_ruby.rb --shard ${{ matrix.shard }} --shards 6 --fresh-images --out results
+      # Also when a pass failed: the shard's other results are kept, to see what went wrong (the site is not built from an incomplete run).
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: results-shard-${{ matrix.shard }}
+          path: results/
+          retention-days: 90
+          # A re-run of a failed shard replaces its earlier attempt.
+          overwrite: true
+
+  # Builds both views from all six shards and publishes them, only for a complete run on main.
+  site:
+    needs: shard
+    if: github.ref_name == 'main'
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+      # Reads this workflow's earlier runs (the newest-results check below).
+      actions: read
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    # One deploy at a time; a newer run waits instead of cancelling one that is halfway through.
+    concurrency:
+      group: pages
+      cancel-in-progress: false
+
+    steps:
+      - uses: actions/checkout@v4
+      # Never put older results back: a re-run of an old run, or two runs finishing out of order, must not publish over a newer run's results.
+      # Run ids only grow and a re-run keeps its id, so a newer run has a higher id (and, since main cannot be force-pushed, a commit at least as new).
+      - name: Check no newer results are live
+        id: newest
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # The highest id among recent successful runs, so the order the list comes back in does not matter.
+          live=$(gh run list --workflow results-site.yml --branch main --status success --limit 20 --json databaseId --jq 'map(.databaseId) | max // empty')
+          if [ -n "$live" ] && [ "$live" -gt "$GITHUB_RUN_ID" ]; then
+            echo "The live site has results from run $live, newer than this run ($GITHUB_RUN_ID); not publishing."
+            echo "publish=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "publish=true" >> "$GITHUB_OUTPUT"
+          fi
+      - uses: actions/download-artifact@v4
+        if: steps.newest.outputs.publish == 'true'
+        with:
+          pattern: results-shard-*
+          path: results/
+          merge-multiple: true
+      - name: Build the results site
+        if: steps.newest.outputs.publish == 'true'
+        run: docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site
+      - name: Upload the site for GitHub Pages
+        if: steps.newest.outputs.publish == 'true'
+        uses: actions/upload-pages-artifact@v5
+        with:
+          path: _site/
+      - name: Deploy to GitHub Pages
+        id: deployment
+        if: steps.newest.outputs.publish == 'true'
+        uses: actions/deploy-pages@v5

.gitignore

@@ -3,3 +3,4 @@
 /Gemfile.lock
 /results/
 /_site/
+/cross-ruby/

CONTRIBUTING.md

@@ -12,6 +12,7 @@ These idioms list here are trying to satisfy following goals:
 
 - [Note on entry](#note-on-entry)
 - [Running it on other Rubies](#running-it-on-other-rubies)
+- [The results site](#the-results-site)
 - [Benchmarks that need a newer Ruby](#benchmarks-that-need-a-newer-ruby)
 - [License](#license)
 
@@ -104,23 +105,25 @@ RUBY_VARIANT=yjit RUBY_VARIANT_FLAGS=--yjit docker compose run --rm ruby_3.4 cod
 RUBY_VARIANT=zjit RUBY_VARIANT_FLAGS=--zjit docker compose run --rm ruby_4.0 code/your-new/entry.rb
 ```
 
-To keep the results, set `RESULTS_DIR`. Each benchmark then also writes its
-report as JSON to `results/<label>/`, with the Ruby, its flags and the machine
-it ran on:
+## The results site
+
+The results site (https://fastruby.github.io/fast-ruby/) compares Rubies, so every build of a benchmark has to run on the same machine.
+`script/run_cross_ruby.rb` (Ruby 3 on your machine, it calls Docker) does that, and runs the newest released MRI again after every 3 builds, so the site can show how steady the machine was.
+To see a few benchmarks the way the site shows them, run them on a few builds, build the site into `_site/`, then open `_site/index.html` in a browser:
 
 ```
-RESULTS_DIR=results RESULTS_LABEL=ruby_3.4 docker compose run --rm ruby_3.4 code/your-new/entry.rb
+ruby script/run_cross_ruby.rb --files code/date/iso8601-vs-parse.rb,code/string/gsub-vs-tr.rb --builds ruby_3.4,ruby_3.4+yjit,ruby_4.0 --out cross-ruby
+docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb cross-ruby _site
 ```
 
-To see those results the way the results site shows them, build the site
-into `_site/`, then open `_site/index.html` in a browser:
+`ruby script/run_cross_ruby.rb --help` lists the options.
+The builds are the ones in the CI matrix (`.github/workflows/benchmarks.yml`), so a new Ruby added there and in `compose.yaml` is measured too, and once released it becomes both the reference and the Ruby the site opens on.
+On an Apple silicon Mac, leave out `ruby_2.1` and `jruby_9.1` (they run under emulation, so their numbers are not comparable) and `ruby_3.1+yjit` (Ruby 3.1's YJIT only exists on x86-64).
 
-```
-docker compose run --rm -T --entrypoint ruby ruby_4.0 script/build_results_site.rb results _site
-```
+CI has two workflows:
 
-CI does the same after every run: the site is attached to the run as the
-`site-preview` artifact, and published to GitHub Pages from `main`.
+- `.github/workflows/benchmarks.yml` checks that every benchmark runs on every Ruby: on a PR, the benchmark files it changes; on `main`, all of them when a benchmark or a shared file changed. It publishes nothing.
+- `.github/workflows/results-site.yml` runs every build on every file once a week, split over 6 machines, and publishes the site. Run it by hand from the Actions tab to publish sooner.
 
 ## Benchmarks that need a newer Ruby
 

compose.yaml

@@ -20,7 +20,11 @@ x-benchmark: &benchmark
     - RESULTS_DIR
     - RESULTS_LABEL
     - RESULTS_COMMIT
-    - RESULTS_PR
+    # Cross-Ruby runs only (script/run_cross_ruby.rb).
+    - RESULTS_SHARD
+    - RESULTS_SHARDS
+    - RESULTS_PASS
+    - RESULTS_REFERENCE
     - GITHUB_RUN_ID
     - GITHUB_RUN_ATTEMPT
     - RUNNER_NAME

docker/collect_results.rb

@@ -48,10 +48,16 @@ def write(report)
         "flags" => env("RUBY_VARIANT_FLAGS"),
         "jit" => jit,
         "commit" => env("RESULTS_COMMIT"),
-        "pr" => env("RESULTS_PR") && env("RESULTS_PR").to_i,
         "run_id" => env("GITHUB_RUN_ID"),
         "run_attempt" => env("GITHUB_RUN_ATTEMPT"),
         "runner" => env("RUNNER_NAME"),
+        # Set by script/run_cross_ruby.rb, which runs every build on one machine and the reference build between them; nil otherwise.
+        "shard" => env("RESULTS_SHARD") && env("RESULTS_SHARD").to_i,
+        "shards" => env("RESULTS_SHARDS") && env("RESULTS_SHARDS").to_i,
+        "pass" => env("RESULTS_PASS") && env("RESULTS_PASS").to_i,
+        "reference" => env("RESULTS_REFERENCE") == "1",
+        # When this report finished, to place it between the reference passes.
+        "measured_at" => Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
         "environment" => environment,
         "entries" => report.data
       }))