From e4d0dbc19d07f42d2b01e3cfb010131ab9bdb252 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 2 Nov 2026 19:38:29 +0900 Subject: [PATCH 1/2] ci: run examples/ against the installed native wheel (#192) New job "Examples on the installed wheel" in test-native.yml: the four platforms, Python 3.12 and 3.14 (a pull request: linux-x86_64, 3.12), the wheel the build job already uploads, installed with uv. The per-example arguments, timeouts, dataset handling and the answer check of example 10 move out of test-python-examples.yml into scripts/run_examples.sh, which both workflows call, so the JVM job behaves as before. scripts/native_examples_skip.txt lists, with a reason, the examples that need the JVM package (empty to start); the runner prints it. examples-summary is the one check for all legs. The native workflow also runs when examples/ or the runner change. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-native.yml | 229 ++++++++++- .github/workflows/test-python-examples.yml | 354 +---------------- scripts/native_examples_skip.txt | 5 + scripts/run_examples.sh | 434 +++++++++++++++++++++ scripts/select_native_matrix.py | 14 +- tests/test_select_native_matrix.py | 48 ++- 6 files changed, 732 insertions(+), 352 deletions(-) create mode 100644 scripts/native_examples_skip.txt create mode 100755 scripts/run_examples.sh diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index a52fa06..2dc5266 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -16,6 +16,10 @@ # needs jpype1 and ladybug, which have no cp315 wheel yet, so 3.15 runs without the test files that need jpype1) and into the free-threaded 3.14t, and runs a small subset of the tests there (about 20 s of tests per Python). It is informational: release-native.yml does not need it. # scripts/select_native_matrix.py decides its legs. # +# The `examples` job runs every script of examples/ against the installed wheel of each platform (Python 3.12 and 3.14; a pull request: linux-x86_64, 3.12), through +# scripts/run_examples.sh, the runner test-python-examples.yml uses too. scripts/native_examples_skip.txt lists, with a reason, the examples that need the JVM package. +# `examples-summary` is the one check for all its legs. They do not decide what a release publishes (release-native.yml reads the markers and the wheels only). +# # One wheel per platform, py3-none-: # linux-x86_64 manylinux_2_28 container manylinux_2_28_x86_64 libarcadedb.so + libjvector-x86_64.so # linux-aarch64 manylinux_2_28 aarch64 container manylinux_2_28_aarch64 libarcadedb.so + libjvector-aarch64.so @@ -41,6 +45,10 @@ on: - "tests/**" - "scripts/select_native_matrix.py" - "scripts/smoke_wheel.py" + - "scripts/run_examples.sh" + - "scripts/native_examples_skip.txt" + - "examples/**" + - "!examples/README.md" - "pyproject.toml" - "setup.py" - ".github/workflows/test-native.yml" @@ -114,6 +122,7 @@ jobs: build: ${{ steps.plan.outputs.build }} test: ${{ steps.plan.outputs.test }} pyver: ${{ steps.plan.outputs.pyver }} + examples: ${{ steps.plan.outputs.examples }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -154,7 +163,7 @@ jobs: event="$EVENT" if [ "$RELEASE" = true ]; then event=release; fi if [ "$EVENT" = "schedule" ]; then - changed="$(git log --since='25 hours ago' --format=%h -- src native tests scripts pyproject.toml .github/workflows/test-native.yml | wc -l)" + changed="$(git log --since='25 hours ago' --format=%h -- src native tests scripts examples pyproject.toml .github/workflows/test-native.yml | wc -l)" monday="$(date -u +%u)" if [ "$changed" = 0 ] && [ "$monday" != 1 ]; then run=false @@ -610,6 +619,224 @@ jobs: if-no-files-found: warn retention-days: 30 + # The examples of examples/ against the INSTALLED native wheel, so the scripts a new user copies are run on what `uv add arcadedb` installs. + # Nothing is built here: the wheel is the one the build job uploaded (native-wheel-), the file the parity job installs. The platforms are + # those of the parity job and the Pythons are 3.12 and 3.14 (scripts/select_native_matrix.py, `examples`). The runner is scripts/run_examples.sh, the + # same script test-python-examples.yml calls for the JVM package, so the arguments, timeouts, datasets and answer checks are the same. + # An example that needs the JVM package is listed with its reason in scripts/native_examples_skip.txt; the step prints that list. + # A failing example turns this job, and so the workflow, red. release-native.yml reads only the native-passed-* markers and the native-wheel-* + # wheels, so these legs do not decide what a release publishes. + examples: + name: Examples on the installed wheel (${{ matrix.key }}, Python ${{ matrix.python }}) + needs: [plan, build] + # A platform whose build failed has no wheel to download and shows red here; the other platforms still run. + if: ${{ !cancelled() && needs.plan.outputs.run == 'true' }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 240 + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.examples) }} + env: + KEY: ${{ matrix.key }} + PY: ${{ matrix.python }} + EXAMPLES: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" + EXAMPLES_SKIP_FILE: scripts/native_examples_skip.txt + # The Hugging Face cache inside the workspace, as in test-python-examples.yml: one path on every platform, cacheable. + HF_HOME: ${{ github.workspace }}/.hf-cache + PYTHONIOENCODING: utf-8 + steps: + - name: Keep LF line endings + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Paths + run: | + if [ "$RUNNER_OS" = Windows ]; then T="$(cygpath -m "$RUNNER_TEMP")"; else T="$RUNNER_TEMP"; fi + echo "T=$T" >> "$GITHUB_ENV" + + - name: Install uv + if: ${{ runner.os != 'Windows' }} + run: | + installer="$(mktemp)" + curl -LsSf --retry 5 --retry-all-errors --retry-delay 5 -o "$installer" "https://astral.sh/uv/${UV_VERSION}/install.sh" + sh "$installer" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Install uv (Windows) + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: | + $env:UV_NO_MODIFY_PATH = "1" + Invoke-RestMethod "https://astral.sh/uv/$env:UV_VERSION/install.ps1" | Invoke-Expression + Add-Content -Path $env:GITHUB_PATH -Value "$env:USERPROFILE\.local\bin" + + # The wheel of the build job of this platform, not a rebuild. + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: native-wheel-${{ matrix.key }} + path: ${{ runner.temp }}/wheel + + # The wheel and the example dependencies live in a uv project of their own (examples-env/, as in test-python-examples.yml), synced with + # `uv sync`. The wheel has no dependencies (jpype1 is not installed: a user of `uv add arcadedb` does not have it). torch comes from the + # CPU-only PyTorch index (explicit: only torch is looked up there). numpy is not capped (see test-python-examples.yml). + - name: Install the wheel and the example dependencies + run: | + set -euo pipefail + wheels=("$T"/wheel/arcadedb-*.whl) + [ "${#wheels[@]}" -eq 1 ] || { echo "expected one wheel in $T/wheel, found ${#wheels[@]}" >&2; exit 1; } + mkdir -p examples-env + cat > examples-env/pyproject.toml <> "$GITHUB_ENV" + "$ex_py" -c "import arcadedb; print('arcadedb', arcadedb.__version__, arcadedb.__file__)" + + - name: Cache Hugging Face model + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.hf-cache + key: hf-all-MiniLM-L6-v2-${{ runner.os }}-${{ runner.arch }} + restore-keys: | + hf-all-MiniLM-L6-v2-${{ runner.os }}- + + # Not fail-fast, as in test-python-examples.yml: a hub outage here must not hide behind a retry, the dataset step fails with its own message. + - name: Pre-warm embedding model + continue-on-error: true + run: | + for attempt in 1 2 3; do + if "$EX_PY" -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2'); print('model ready')"; then + echo "model cached on attempt $attempt" + exit 0 + fi + echo "::warning::model fetch failed (attempt $attempt/3)" + [ "$attempt" -lt 3 ] && sleep $((attempt * 20)) + done + echo "::warning::could not pre-warm the embedding model after 3 attempts; the dataset step will retry" + + - name: Download datasets + run: bash scripts/run_examples.sh datasets + + - name: Install timeout command (macOS only) + if: ${{ runner.os == 'macOS' }} + run: brew install coreutils + + - name: Run all examples + id: run_examples + run: bash scripts/run_examples.sh run + + - name: Generate test summary + if: ${{ always() }} + run: | + cd examples + { + echo "## Examples on the installed wheel ($KEY, Python $PY)" + echo + if [ "${{ steps.run_examples.outcome }}" = "success" ]; then + echo "**Status**: all examples passed (${{ steps.run_examples.outputs.passed || '0' }}/${{ steps.run_examples.outputs.total || '0' }})" + else + echo "**Status**: some examples failed (${{ steps.run_examples.outputs.passed || '0' }}/${{ steps.run_examples.outputs.total || '0' }} passed)" + fi + echo + echo "| Metric | Count |" + echo "|--------|------:|" + echo "| Total | ${{ steps.run_examples.outputs.total || '0' }} |" + echo "| Passed | ${{ steps.run_examples.outputs.passed || '0' }} |" + echo "| Failed | ${{ steps.run_examples.outputs.failed || '0' }} |" + echo "| Skipped | ${{ steps.run_examples.outputs.skipped || '0' }} |" + echo + echo "### Skip list (scripts/native_examples_skip.txt)" + echo + echo '```' + grep -v -e '^[[:space:]]*#' -e '^[[:space:]]*$' ../scripts/native_examples_skip.txt || echo "(empty: no example is skipped)" + echo '```' + if [ -f example-results.txt ]; then + echo + echo "### Detailed results" + echo + echo '```' + cat example-results.txt + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: native-examples-${{ matrix.key }}-py${{ matrix.python }} + path: | + examples/*.log + examples/example-results.txt + if-no-files-found: warn + retention-days: 7 + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ failure() }} + with: + name: native-examples-databases-${{ matrix.key }}-py${{ matrix.python }} + path: examples/my_test_databases/ + if-no-files-found: ignore + retention-days: 3 + + # One check for the examples of every platform and Python: red when any leg of the `examples` job failed or the job did not run to the end. + examples-summary: + name: Examples summary + needs: [plan, examples] + if: ${{ always() && needs.plan.outputs.run == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check the examples legs + env: + RESULT: ${{ needs.examples.result }} + run: | + { + echo "## Examples on the installed wheel" + echo + if [ "$RESULT" = success ]; then + echo "All example legs passed." + else + echo "The examples job ended with: $RESULT. See the legs of the job \"Examples on the installed wheel\"." + fi + } >> "$GITHUB_STEP_SUMMARY" + [ "$RESULT" = success ] + # The cheap Python-version coverage of the wheel of linux-x86_64 (py3-none: the same file for every Python, nothing is rebuilt): a subset of # the test files, not the whole suite. The legs are the supported Pythons the full parity runs of this event leave out, plus the # free-threaded 3.14t (scripts/select_native_matrix.py). diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml index b0df568..eb08699 100644 --- a/.github/workflows/test-python-examples.yml +++ b/.github/workflows/test-python-examples.yml @@ -22,6 +22,7 @@ on: - "native/**" - ".github/workflows/test-native.yml" - ".github/workflows/release-native.yml" + - "scripts/native_examples_skip.txt" pull_request: # ready_for_review: draft PRs skip the platform matrices (see the job conditions), # so marking one ready is what starts them. @@ -37,6 +38,7 @@ on: - "native/**" - ".github/workflows/test-native.yml" - ".github/workflows/release-native.yml" + - "scripts/native_examples_skip.txt" # A new upstream snapshot is published as arcadedata/arcadedb:latest without # any change here, so build and test against it every night as well. The tag @@ -102,7 +104,9 @@ env: EXAMPLES: ${{ inputs.examples || '0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py' }} # Note: Example 21 runs in CI at reduced scale; its arguments and 15-minute -# limit are set in the per-example case in the "Run all examples" step below. +# limit are set in the per-example case in scripts/run_examples.sh, which the +# "Download datasets" and "Run all examples" steps call. test-native.yml calls +# the same script on the native wheel. permissions: contents: read @@ -427,75 +431,7 @@ jobs: - name: Download datasets shell: bash run: | - cd examples - - # Determine which examples are selected - examples=$(ls $EXAMPLES 2>/dev/null || true) - - # EXIT 75 MEANS "THE DATASET HOST IS DOWN", NOT "THE BINDINGS BROKE". - # - # On 2026-08-29 the TLS certificate for files.grouplens.org expired and - # every job here went red on BOTH this fork and upstream's main -- 11 - # and 12 jobs, every OS, Python 3.10 through 3.14 -- at this step, - # before a single example ran. Re-running does not help: a re-run - # happens now, and the certificate is still expired. - # - # The cost is not the lost coverage, it is that upstream's main showed - # a failing run titled with a Python bindings PR. A third party letting - # a certificate lapse must not be indistinguishable from a regression - # we introduced. - # - # So: skip the examples that need the dataset, keep running the rest, - # and say plainly why. Any other non-zero exit is still a hard failure. - SKIPPED_DATASETS="" - fetch() { # fetch