From 87e548c4cc97e4c2aca9ce433ea672873054364f Mon Sep 17 00:00:00 2001 From: AlexanderMitrofanov Date: Thu, 22 Oct 2026 15:30:13 +0200 Subject: [PATCH 1/2] fix(wasm): build against the latest stable ViennaRNA 2.x --- .github/workflows/build-wasm.yml | 11 ++- ChangeLog | 23 +++++ scripts/wasm/build.sh | 53 +++++++----- scripts/wasm/patch-vienna.py | 25 ------ scripts/wasm/smoke-protocol.mjs | 4 +- scripts/wasm/stage.sh | 28 +++++-- scripts/wasm/test-browser.py | 11 ++- scripts/wasm/test-vienna-release.py | 90 ++++++++++++++++++++ scripts/wasm/vienna-release.json | 5 ++ scripts/wasm/vienna-release.py | 93 +++++++++++++++++++++ scripts/wasm/write-build-info.mjs | 6 +- webinterface/docs/step1-wasm-compilation.md | 53 ++++++++---- 12 files changed, 327 insertions(+), 75 deletions(-) delete mode 100644 scripts/wasm/patch-vienna.py create mode 100644 scripts/wasm/test-vienna-release.py create mode 100644 scripts/wasm/vienna-release.json create mode 100644 scripts/wasm/vienna-release.py diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index c6baf0d0..aadec7f3 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -31,9 +31,18 @@ jobs: with: python-version: '3.12' - name: Test dependency download integrity and cache recovery - run: python scripts/wasm/test-downloads.py + run: | + python scripts/wasm/test-downloads.py + python scripts/wasm/test-vienna-release.py - name: Test input, worker lifecycle and CSV handling run: npm test --prefix webinterface + - name: Resolve newest stable ViennaRNA 2.x release and checksum + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python scripts/wasm/vienna-release.py --latest > "$RUNNER_TEMP/vienna-release.json" + cat "$RUNNER_TEMP/vienna-release.json" + echo "WASM_VIENNA_RELEASE=$RUNNER_TEMP/vienna-release.json" >> "$GITHUB_ENV" - name: Cross-compile dependencies and IntaRNA run: bash scripts/wasm/build.sh - name: Check real WASM predictions against the native reference diff --git a/ChangeLog b/ChangeLog index 01272691..cafc21d7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -84,6 +84,10 @@ ## Technical changes and Optimizations +- build WASM with ViennaRNA 2.7.2 locally and the newest stable 2.x in CI; + verify published archive checksums, isolate versioned caches and retain + selected dependency provenance and notices (PR #292 review) + - BUGFIX : restore WASM dependency downloads using the pinned official ViennaRNA release asset; verify downloads before caching and recover invalid caches (PR #291) @@ -249,6 +253,25 @@ energies, restricted partition sums, and trackers. ################################################################################ ################################################################################ +261022 Alexander Mitrofanov + * scripts/wasm/{vienna-release.py,vienna-release.json}, build.sh + + Resolve the newest stable ViennaRNA >=2.7.2 and <3 from all official GitHub + release pages in CI; require its release tarball and published SHA-256. + * Pin 2.7.2 for reproducible local builds and accept saved release manifests. + Isolate RNAlib sources/installations and IntaRNA objects by archive identity + to prevent reuse of libraries or headers from another release (PR #292). + * scripts/wasm/patch-vienna.py + - Remove the dlib 19.24 compatibility patch; ViennaRNA 2.7.2 bundles dlib 20.0. + * scripts/wasm/{write-build-info.mjs,stage.sh,smoke-protocol.mjs,test-browser.py} + * Record the selected archive URL/checksum, check the compiled ViennaRNA + version, stage matching dependency notices and test actual version exports. + * scripts/wasm/test-vienna-release.py, .github/workflows/build-wasm.yml + + Cover numeric version selection, pagination, major-3/prerelease exclusion, + missing/invalid asset metadata, API failures and manifest validation. + * webinterface/docs/step1-wasm-compilation.md + * Document latest-2.x CI selection, pinned local reproduction, versioned + caches, current bundled dlib and archive provenance. + 261022 Alexander Mitrofanov * scripts/wasm/{build,download}.sh * Use the official GitHub ViennaRNA 2.7.0 release asset and pin its SHA-256; diff --git a/scripts/wasm/build.sh b/scripts/wasm/build.sh index 60140119..1ca91ac0 100644 --- a/scripts/wasm/build.sh +++ b/scripts/wasm/build.sh @@ -7,7 +7,7 @@ mkdir -p "$build" build=$(cd "$build" && pwd) prefix="$build/deps" jobs=${JOBS:-2} -for tool in emcc em++ emar emranlib emconfigure emmake curl pkg-config node; do +for tool in emcc em++ emar emranlib emconfigure emmake curl pkg-config node python3; do command -v "$tool" >/dev/null || { echo "Missing tool: $tool" >&2; exit 1; } done mkdir -p "$build/downloads" "$build/sources" "$prefix" "$build/site/wasm" @@ -18,16 +18,27 @@ fetch https://zlib.net/fossils/zlib-1.3.1.tar.gz zlib-1.3.1.tar.gz \ 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 fetch https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.bz2 boost_1_87_0.tar.bz2 \ af57be25cb4c4f4b413ed692fe378affb4352ea50fbe294a11ef548f4d527d89 -fetch https://github.com/ViennaRNA/ViennaRNA/releases/download/v2.7.0/ViennaRNA-2.7.0.tar.gz ViennaRNA-2.7.0.tar.gz \ - e2addf33e7c0dd4fff31d8e9a46fda8e9efcde0d199f789f20eb0a13b7f7135d +# CI supplies the newest stable 2.x manifest; local builds use the checked-in pin. +read -r vienna_version vienna_url vienna_sha256 < <( + python3 "$repo/scripts/wasm/vienna-release.py" --fields \ + --manifest "${WASM_VIENNA_RELEASE:-$repo/scripts/wasm/vienna-release.json}" +) +echo "Building ViennaRNA $vienna_version (SHA-256 $vienna_sha256)" +fetch "$vienna_url" "ViennaRNA-$vienna_version.tar.gz" "$vienna_sha256" +# Separate source, installed RNAlib and IntaRNA objects by archive identity so +# changing the release cannot reuse headers, libraries or objects from an old one. +vienna_id="vienna-$vienna_version-$vienna_sha256" +vienna_prefix="$prefix/$vienna_id" +vienna_sources="$build/sources/$vienna_id" +intarna_build="$build/intarna-$vienna_id" # Keep pkg-config searches inside the WASM prefix, including configure probes. -export PKG_CONFIG_LIBDIR="$prefix/lib/pkgconfig" +export PKG_CONFIG_LIBDIR="$vienna_prefix/lib/pkgconfig:$prefix/lib/pkgconfig" export PKG_CONFIG_PATH="$PKG_CONFIG_LIBDIR" export CFLAGS='-O3 -fexceptions' export CXXFLAGS='-O3 -fexceptions' -export CPPFLAGS="-I$prefix/include" -export LDFLAGS="-L$prefix/lib -fexceptions" +export CPPFLAGS="-I$vienna_prefix/include -I$prefix/include" +export LDFLAGS="-L$vienna_prefix/lib -L$prefix/lib -fexceptions" if [[ ! -f "$prefix/lib/libz.a" ]]; then tar -xzf "$build/downloads/zlib-1.3.1.tar.gz" -C "$build/sources" (cd "$build/sources/zlib-1.3.1" @@ -47,27 +58,26 @@ if [[ ! -f "$prefix/.boost-1.87.0-js-exceptions-complete" ]]; then -sZLIB_LIBPATH="$prefix/lib" --prefix="$prefix" install) touch "$prefix/.boost-1.87.0-js-exceptions-complete" fi -if [[ ! -f "$prefix/lib/libRNA.a" ]]; then - tar -xzf "$build/downloads/ViennaRNA-2.7.0.tar.gz" -C "$build/sources" - (cd "$build/sources/ViennaRNA-2.7.0" - CPPFLAGS="$CPPFLAGS -I$PWD/src" emconfigure ./configure --host=wasm32-unknown-emscripten --prefix="$prefix" \ +if [[ ! -f "$vienna_prefix/.complete" ]]; then + mkdir -p "$vienna_sources" + tar -xzf "$build/downloads/ViennaRNA-$vienna_version.tar.gz" -C "$vienna_sources" + (cd "$vienna_sources/ViennaRNA-$vienna_version" + CPPFLAGS="$CPPFLAGS -I$PWD/src" emconfigure ./configure --host=wasm32-unknown-emscripten --prefix="$vienna_prefix" \ --disable-shared --enable-static --disable-openmp --disable-pthreads \ --disable-lto --disable-simd --disable-vectorize --disable-mpfr \ --without-svm --without-gsl --without-swig --without-perl --without-python \ --without-doc --without-cla --without-check --without-kinfold \ --without-forester --without-rnalocmin --without-rnaxplorer - # Backport dlib's upstream char32_t typedef for modern libc++ char_traits. - # Only the downloaded dependency is patched; RNAlib algorithms are unchanged. - python3 "$repo/scripts/wasm/patch-vienna.py" "$PWD" emmake make -C src/ViennaRNA -j "$jobs" emmake make -C src/ViennaRNA install emmake make install-pkgconfigDATA) + touch "$vienna_prefix/.complete" fi # Legacy Autoconf probes declare exit()/inflateEnd() with the wrong signatures, # which wasm-ld rejects. Verify real dependency APIs before caching those probes. -em++ -std=c++23 -O2 -fexceptions -I"$prefix/include" \ - "$repo/scripts/wasm/dependency-probe.cpp" -L"$prefix/lib" \ +em++ -std=c++23 -O2 -fexceptions -I"$vienna_prefix/include" -I"$prefix/include" \ + "$repo/scripts/wasm/dependency-probe.cpp" -L"$vienna_prefix/lib" -L"$prefix/lib" \ -lboost_filesystem -lboost_system -lboost_program_options -lboost_regex \ -lboost_iostreams -lboost_serialization -lRNA -lz -o "$build/dependency-probe.js" node "$build/dependency-probe.js" @@ -78,15 +88,15 @@ export ac_cv_lib_boost_regex_exit=yes export ac_cv_lib_boost_iostreams_exit=yes export ac_cv_lib_z_inflateEnd=yes # RNAlib's .pc embeds a native-only -Wl,-fno-lto; this is a static WASM build. -export VRNA_LIBS="-L$prefix/lib -lRNA -lm" +export VRNA_LIBS="-L$vienna_prefix/lib -lRNA -lm" # Use the native Autotools source lists and CLI entry point, with a JS output. (cd "$repo" && bash autotools-init.sh) -mkdir -p "$build/intarna" -(cd "$build/intarna" - CXXFLAGS="$CXXFLAGS -Werror=inconsistent-missing-override" CPPFLAGS="-I$build/intarna/src -I$repo/src $CPPFLAGS" emconfigure "$repo/configure" --host=wasm32-unknown-emscripten \ +mkdir -p "$intarna_build" +(cd "$intarna_build" + CXXFLAGS="$CXXFLAGS -Werror=inconsistent-missing-override" CPPFLAGS="-I$intarna_build/src -I$repo/src $CPPFLAGS" emconfigure "$repo/configure" --host=wasm32-unknown-emscripten \ --disable-multithreading --disable-log-coloring --with-mdspan=kokkos \ - --with-vrna="$prefix" --with-boost="$prefix" \ + --with-vrna="$vienna_prefix" --with-boost="$prefix" \ --with-boost-libdir="$prefix/lib" --with-zlib="$prefix" emmake make -C src -j "$jobs" EXEEXT=.js \ LDFLAGS="$LDFLAGS -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=createIntaRNA -sENVIRONMENT=web,worker,node -sINVOKE_RUN=0 -sEXIT_RUNTIME=1 -sEXPORTED_RUNTIME_METHODS=FS,callMain -sALLOW_MEMORY_GROWTH=1 -sMAXIMUM_MEMORY=1073741824 -sSTACK_SIZE=8388608" @@ -96,5 +106,6 @@ mkdir -p "$build/intarna" printf '{"type":"module"}\n' > "$build/site/wasm/package.json" node "$repo/scripts/wasm/write-build-info.mjs" "$repo" "$build/site" \ "$(emcc -dumpversion)" "$(git -C "$repo" rev-parse HEAD)" \ - "$(git -C "$repo" status --porcelain --untracked-files=no)" + "$(git -C "$repo" status --porcelain --untracked-files=no)" \ + "$vienna_version" "$vienna_url" "$vienna_sha256" echo "WASM assets: $build/site/wasm" diff --git a/scripts/wasm/patch-vienna.py b/scripts/wasm/patch-vienna.py deleted file mode 100644 index 4b636aad..00000000 --- a/scripts/wasm/patch-vienna.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Apply narrow dlib 19.24 compatibility fixes in the downloaded ViennaRNA tree. - -Modern libc++ only defines char_traits for standard character types. dlib now -uses char32_t upstream. Its old serialization overloads and dependent template -call also need the matching C++ fixes. RNA models and recurrences are untouched. -""" -import sys -from pathlib import Path - -root = Path(sys.argv[1]) / "src/dlib-19.24/dlib" -patches = [ - ("unicode/unicode.h", "typedef uint32 unichar;", "typedef char32_t unichar;"), - ("serialize.h", " USE_DEFAULT_INT_SERIALIZATION_FOR(wchar_t)", - " USE_DEFAULT_INT_SERIALIZATION_FOR(wchar_t)\n USE_DEFAULT_INT_SERIALIZATION_FOR(char32_t)"), - ("global_optimization/find_max_global.h", "::template go(std::forward(f),a)", - "::template go(std::forward(f),a)"), -] -for filename, before, after in patches: - path = root / filename - source = path.read_text() - if after in source: - continue - if source.count(before) != 1: - raise SystemExit(f"Unexpected dlib source: {path}; review compatibility patch") - path.write_text(source.replace(before, after)) diff --git a/scripts/wasm/smoke-protocol.mjs b/scripts/wasm/smoke-protocol.mjs index f168a7b1..ee27ad8b 100644 --- a/scripts/wasm/smoke-protocol.mjs +++ b/scripts/wasm/smoke-protocol.mjs @@ -1,5 +1,6 @@ // Integration coverage of browser argument translation against the real CLI. import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { DEFAULTS, EVALUATION_EXAMPLE, prepareRequest, makeArguments } from '../../webinterface/protocol.js'; @@ -8,6 +9,7 @@ import { parseResults, projectCsv } from '../../webinterface/results.js'; import { archiveFiles } from '../../webinterface/archive.js'; const asset = resolve(process.argv[2] || 'build-wasm/site/wasm/intarna.js'); const { default: createIntaRNA } = await import(pathToFileURL(asset)); +const compiledVersion = readFileSync(new URL('../build-info.txt', pathToFileURL(asset)), 'utf8'); async function run(input, format = 'C') { const request = prepareRequest({ ...DEFAULTS, ...input }); const diagnostics = []; @@ -54,7 +56,7 @@ async function replayArchive(input) { const outputs = fs.readdir('/output').filter((name) => name !== '.' && name !== '..').map((name) => ({ name, text: fs.readFile(`/output/${name}`, { encoding: 'utf8' }) })); const reports = [['normal', 'N'], ['detailed', 'D'], ['ensemble', 'E']].filter(([key]) => request.outputs.includes(key)); for (const [key, format] of reports) outputs.push({ name: `intarna-${key}.txt`, text: (await run(input, format)).text }); - const files = archiveFiles(request, { csv: original.text, files: outputs }, 'IntaRNA 3.4.1\nViennaRNA 2.7.0\n'); + const files = archiveFiles(request, { csv: original.text, files: outputs }, compiledVersion); for (const config of ['parameters', ...reports.map(([key]) => key)]) { const diagnostics = []; const module = await createIntaRNA({ noInitialRun: true, locateFile: () => new URL('./intarna.wasm', pathToFileURL(asset)).href, print: (line) => diagnostics.push(line), printErr: (line) => diagnostics.push(line) }); diff --git a/scripts/wasm/stage.sh b/scripts/wasm/stage.sh index d123f2b1..20367c2e 100644 --- a/scripts/wasm/stage.sh +++ b/scripts/wasm/stage.sh @@ -6,6 +6,24 @@ build=${WASM_BUILD_DIR:-"$repo/build-wasm"} site="$build/site" test -s "$site/wasm/intarna.js" test -s "$site/wasm/intarna.wasm" +# Select notices from the archive used by these assets, including dlib upgrades. +read -r vienna_version vienna_sha256 dlib_directory < <(python3 - "$site" "$build" <<'PYSOURCE' +import json, re, sys +from pathlib import Path +site, build = map(Path, sys.argv[1:]) +metadata = json.loads((site / "build-info.json").read_text()) +archive = metadata["viennarnaArchive"] +version, digest = archive["version"], archive["sha256"] +if version != metadata["viennarna"] or not re.fullmatch(r"2\.\d+\.\d+", version) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise SystemExit("Invalid compiled ViennaRNA archive identity") +source = build / "sources" / f"vienna-{version}-{digest}" / f"ViennaRNA-{version}" +dlib = list((source / "src").glob("dlib-*/dlib/LICENSE.txt")) +if len(dlib) != 1 or not re.fullmatch(r"dlib-[0-9.]+", dlib[0].parent.parent.name): + raise SystemExit("Expected one bundled dlib license in the compiled ViennaRNA source") +print(version, digest, dlib[0].parent.parent.name) +PYSOURCE +) +vienna_source="$build/sources/vienna-$vienna_version-$vienna_sha256/ViennaRNA-$vienna_version" mkdir -p "$site/licenses" cp -R "$repo/webinterface/assets" "$site/" for file in index.html styles.css app.js mark.svg protocol.js runner.js results.js options.js columns.js archive.js visualization.js; do @@ -18,20 +36,20 @@ cp "$repo/src/easylogging++.LICENCE.txt" "$site/licenses/easylogging.txt" cp "$repo/src/mdspan/LICENSE" "$site/licenses/mdspan.txt" cp "$build/sources/boost_1_87_0/LICENSE_1_0.txt" "$site/licenses/Boost.txt" cp "$build/sources/zlib-1.3.1/zlib.h" "$site/licenses/zlib.h" -cp "$build/sources/ViennaRNA-2.7.0/COPYING" "$site/licenses/ViennaRNA.txt" -cp "$build/sources/ViennaRNA-2.7.0/src/dlib-19.24/dlib/LICENSE.txt" "$site/licenses/dlib.txt" -cp "$build/sources/ViennaRNA-2.7.0/src/ViennaRNA/plotting/naview/naview.c" "$site/licenses/naview.c" +cp "$vienna_source/COPYING" "$site/licenses/ViennaRNA.txt" +cp "$vienna_source/src/$dlib_directory/dlib/LICENSE.txt" "$site/licenses/dlib.txt" +cp "$vienna_source/src/ViennaRNA/plotting/naview/naview.c" "$site/licenses/naview.c" emscripten_dir=$(dirname "$(command -v emcc)") cp "$emscripten_dir/LICENSE" "$site/licenses/Emscripten.txt" cp "$emscripten_dir/system/lib/libc/musl/COPYRIGHT" "$site/licenses/musl.txt" for library in libcxx libcxxabi libunwind compiler-rt; do cp "$emscripten_dir/system/lib/$library/LICENSE.TXT" "$site/licenses/$library.txt" done -cat > "$site/licenses/index.html" <<'HTML' +cat > "$site/licenses/index.html" <IntaRNA licenses

IntaRNA and third-party notices

IntaRNA: Bioinformatics Group of Prof. Backofen, University of Freiburg.
ViennaRNA: authors and Institute for Theoretical Chemistry, University of Vienna.

-

This distribution uses ViennaRNA 2.7.0, Boost 1.87.0, zlib 1.3.1, bundled dlib 19.24 (with Unicode/template compatibility fixes), Kokkos mdspan, Easylogging++ and the Emscripten 4.0.23 runtime.

+

This distribution uses ViennaRNA $vienna_version, Boost 1.87.0, zlib 1.3.1, bundled dlib ${dlib_directory#dlib-}, Kokkos mdspan, Easylogging++ and the Emscripten 4.0.23 runtime.

  • IntaRNA
  • ViennaRNA
  • Boost
  • zlib
    • diff --git a/scripts/wasm/test-browser.py b/scripts/wasm/test-browser.py index 3bcdadbc..079fbdf5 100644 --- a/scripts/wasm/test-browser.py +++ b/scripts/wasm/test-browser.py @@ -2,7 +2,8 @@ import argparse from pathlib import Path from io import BytesIO -from urllib.parse import parse_qs, urlparse +import re +from urllib.parse import parse_qs, urljoin, urlparse from zipfile import ZipFile from playwright.sync_api import sync_playwright, expect @@ -53,7 +54,11 @@ def predict(): expect(status).to_contain_text('complete:', timeout=120000) expect(page.locator('#input-error')).not_to_be_visible() - expect(page.locator('#version-info')).to_contain_text('ViennaRNA 2.7.0') + metadata = page.request.get(urljoin(args.url, 'build-info.json')).json() + vienna_version = metadata['viennarna'] + assert re.fullmatch(r'2\.\d+\.\d+', vienna_version) + assert metadata['viennarnaArchive']['version'] == vienna_version + expect(page.locator('#version-info')).to_contain_text(f'ViennaRNA {vienna_version}') for logo in page.locator('.site-header img').all(): assert logo.evaluate('node => node.complete && node.naturalWidth > 0') for key in ['query-options', 'target-options', 'seed-options', 'helix-options', 'shape-options', 'interaction-options', 'output-options', 'general-options', 'sequence-input']: @@ -122,7 +127,7 @@ def predict(): assert archive['intarna-interactions.csv'] == csv assert archive['query.fa'] == '>query\nGGGAGGAGGG\n' assert '# IntaRNA 3.4.1' in archive['parameters.cfg'] - assert 'Vienna RNA package 2.7.0' in archive['parameters.cfg'] + assert f'Vienna RNA package {vienna_version}' in archive['parameters.cfg'] assert 'outNumber = 10' in archive['parameters.cfg'] assert 'IntaRNA --parameterFile=parameters.cfg' in archive['parameters.cfg'] assert 'temperature =' not in archive['parameters.cfg'] diff --git a/scripts/wasm/test-vienna-release.py b/scripts/wasm/test-vienna-release.py new file mode 100644 index 00000000..7357a88e --- /dev/null +++ b/scripts/wasm/test-vienna-release.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Check stable-major selection, pagination and archive identity without GitHub.""" +from copy import deepcopy +import importlib.util +from io import BytesIO +import json +from pathlib import Path +import subprocess +from tempfile import TemporaryDirectory +import unittest +from unittest.mock import patch +from urllib.error import HTTPError + + +SCRIPT = Path(__file__).with_name("vienna-release.py") +SPEC = importlib.util.spec_from_file_location("vienna_release", SCRIPT) +resolver = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(resolver) +DIGEST = "a" * 64 + + +def release(version, **fields): + return { + "tag_name": f"v{version}", "draft": False, "prerelease": False, + "assets": [{ + "name": f"ViennaRNA-{version}.tar.gz", "digest": f"sha256:{DIGEST}", + "browser_download_url": f"{resolver.REPOSITORY}/releases/download/v{version}/ViennaRNA-{version}.tar.gz", + }], **fields, + } + + +class ReleaseTests(unittest.TestCase): + def resolve(self, pages): + with patch.object(resolver, "urlopen", side_effect=[BytesIO(json.dumps(page).encode()) for page in pages]) as fetch: + result = resolver.latest_release() + return result, fetch + + def test_numeric_selection_excludes_major3_drafts_and_prereleases(self): + result, _ = self.resolve([[ + release("3.0.0"), release("2.9.0"), release("2.10.0"), + release("2.11.0", prerelease=True), release("2.12.0", draft=True), + release("2.13.0-rc.1"), release("2.7.2"), release("2.7.1"), + ]]) + self.assertEqual(result["version"], "2.10.0") + self.assertEqual(result["sha256"], DIGEST) + + def test_pagination_finds_version2_after_one_hundred_newer_releases(self): + result, fetch = self.resolve([[release(f"3.0.{n}") for n in range(100)], [release("2.7.2")]]) + self.assertEqual(result["version"], "2.7.2") + self.assertEqual(fetch.call_count, 2) + self.assertTrue(fetch.call_args.args[0].full_url.endswith("per_page=100&page=2")) + + def test_newest_release_must_have_exact_asset_url_and_digest(self): + latest = release("2.8.0") + for field, value in [("name", "source.tar.gz"), ("digest", None), ("digest", "sha256:bad"), ("digest", "md5:" + DIGEST), ("browser_download_url", "https://example.com/archive.tar.gz")]: + with self.subTest(field=field, value=value): + invalid = deepcopy(latest) + invalid["assets"][0][field] = value + with self.assertRaises(ValueError): + self.resolve([[release("2.7.2"), invalid]]) + with self.assertRaises(ValueError): + self.resolve([[release("2.8.0", assets=latest["assets"] * 2)]]) + + def test_no_supported_release_or_api_failure_is_an_error(self): + with self.assertRaises(ValueError): + self.resolve([[release("3.0.0"), release("2.7.1"), release("2.8.0", prerelease=True)]]) + with patch.object(resolver, "urlopen", side_effect=HTTPError(resolver.API, 403, "rate limited", {}, None)): + with self.assertRaises(HTTPError): + resolver.latest_release() + + def test_checked_in_pin_and_shell_fields(self): + manifest = json.loads(subprocess.check_output(["python3", str(SCRIPT)], text=True)) + self.assertEqual(manifest["version"], "2.7.2") + fields = subprocess.check_output(["python3", str(SCRIPT), "--fields"], text=True).split() + self.assertEqual(fields, list(manifest.values())) + + def test_invalid_pin_fails_without_emitting_shell_fields(self): + manifest = json.loads(SCRIPT.with_suffix(".json").read_text()) + with TemporaryDirectory(prefix="vienna manifest ") as directory: + path = Path(directory) / "release.json" + for field, value in [("version", "3.0.0"), ("version", "2.7.0"), ("url", "https://example.com"), ("sha256", "bad")]: + with self.subTest(field=field): + path.write_text(json.dumps({**manifest, field: value})) + result = subprocess.run(["python3", str(SCRIPT), "--manifest", str(path), "--fields"], capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/wasm/vienna-release.json b/scripts/wasm/vienna-release.json new file mode 100644 index 00000000..d54b5517 --- /dev/null +++ b/scripts/wasm/vienna-release.json @@ -0,0 +1,5 @@ +{ + "version": "2.7.2", + "url": "https://github.com/ViennaRNA/ViennaRNA/releases/download/v2.7.2/ViennaRNA-2.7.2.tar.gz", + "sha256": "4cb7aaca00dfac92cb36b030c9111487c170b90568759bd4ffedd8318cc9384f" +} diff --git a/scripts/wasm/vienna-release.py b/scripts/wasm/vienna-release.py new file mode 100644 index 00000000..f869e786 --- /dev/null +++ b/scripts/wasm/vienna-release.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Resolve a stable ViennaRNA 2.x release asset, or validate a pinned manifest.""" +import argparse +import json +import os +from pathlib import Path +import re +from urllib.request import Request, urlopen + + +REPOSITORY = "https://github.com/ViennaRNA/ViennaRNA" +API = "https://api.github.com/repos/ViennaRNA/ViennaRNA/releases" +MINIMUM = (2, 7, 2) + + +def version_tuple(version): + if not isinstance(version, str) or not re.fullmatch(r"2\.\d+\.\d+", version): + raise ValueError("Expected a stable ViennaRNA 2.x version") + result = tuple(map(int, version.split("."))) + if result < MINIMUM: + raise ValueError("The WASM build requires ViennaRNA >= 2.7.2") + return result + + +def validate(manifest): + """Accept only official release tarballs with an explicit SHA-256.""" + version = manifest["version"] + version_tuple(version) + expected = f"{REPOSITORY}/releases/download/v{version}/ViennaRNA-{version}.tar.gz" + if manifest["url"] != expected: + raise ValueError("Expected the official ViennaRNA release asset URL") + if not isinstance(manifest["sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}", manifest["sha256"]): + raise ValueError("Missing or invalid ViennaRNA archive SHA-256") + return {key: manifest[key] for key in ("version", "url", "sha256")} + + +def latest_release(): + # Enumerate all pages: /latest may select major 3, and release dates need not + # follow version order (for example, a maintenance release of an older minor). + headers = {"Accept": "application/vnd.github+json", "User-Agent": "IntaRNA-WASM"} + if os.environ.get("GITHUB_TOKEN"): + headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}" + candidates = [] + page = 1 + while True: + request = Request(f"{API}?per_page=100&page={page}", headers=headers) + with urlopen(request, timeout=30) as response: + releases = json.load(response) + if not isinstance(releases, list): + raise ValueError("Expected a GitHub release list") + for release in releases: + tag = release.get("tag_name", "") + if release.get("draft") or release.get("prerelease") or not re.fullmatch(r"v2\.\d+\.\d+", tag): + continue + version = tag[1:] + if tuple(map(int, version.split("."))) >= MINIMUM: + candidates.append(release) + if len(releases) < 100: + break + page += 1 + if not candidates: + raise ValueError("No stable ViennaRNA 2.x release >= 2.7.2 was found") + release = max(candidates, key=lambda item: version_tuple(item["tag_name"][1:])) + version = release["tag_name"][1:] + # Fail if the newest release lacks the asset/digest; do not silently select + # an older release or GitHub's differently packaged autogenerated source. + name = f"ViennaRNA-{version}.tar.gz" + assets = [asset for asset in release.get("assets", []) if asset["name"] == name] + if len(assets) != 1: + raise ValueError(f"Expected exactly one {name} release asset") + asset = assets[0] + digest = asset.get("digest") or "" + if not digest.startswith("sha256:"): + raise ValueError(f"Missing published SHA-256 for {name}") + return validate({"version": version, "url": asset["browser_download_url"], "sha256": digest[7:]}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group() + source.add_argument("--latest", action="store_true", help="query the official GitHub releases API") + source.add_argument("--manifest", type=Path, default=Path(__file__).with_suffix(".json")) + parser.add_argument("--fields", action="store_true", help="emit version, URL and SHA-256 for build.sh") + args = parser.parse_args() + try: + manifest = latest_release() if args.latest else validate(json.loads(args.manifest.read_text())) + except (OSError, ValueError, KeyError, TypeError) as error: + parser.exit(1, f"Cannot resolve ViennaRNA release: {error}\n") + print(" ".join(manifest.values()) if args.fields else json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/wasm/write-build-info.mjs b/scripts/wasm/write-build-info.mjs index 959aacc4..4bf9fd47 100644 --- a/scripts/wasm/write-build-info.mjs +++ b/scripts/wasm/write-build-info.mjs @@ -3,7 +3,7 @@ import { createHash } from 'node:crypto'; import { readFileSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -const [repo, site, emscripten, commit, changes] = process.argv.slice(2); +const [repo, site, emscripten, commit, changes, viennaVersion, viennaUrl, viennaSha256] = process.argv.slice(2); const { default: createIntaRNA } = await import(pathToFileURL(resolve(site, 'wasm/intarna.js'))); const lines = []; const module = await createIntaRNA({ noInitialRun: true, locateFile: () => pathToFileURL(resolve(site, 'wasm/intarna.wasm')).href, print: (line) => lines.push(line), printErr: (line) => lines.push(line) }); @@ -11,13 +11,15 @@ if (module.callMain(['--version']) !== 0) throw new Error('Could not read compil const version = lines.join('\n'); const match = /IntaRNA\s+(\S+)[\s\S]*Vienna RNA package\s+(\S+)\s+and boost\s+(\S+)/i.exec(version); if (!match) throw new Error(`Unrecognized engine version output: ${version}`); +if (match[2] !== viennaVersion) throw new Error(`Compiled ViennaRNA ${match[2]} differs from selected ${viennaVersion}`); const metadata = { source: 'https://github.com/BackofenLab/IntaRNA', commit, dirty: !!changes, intarna: match[1], viennarna: match[2], boost: match[3], emscripten, zlib: '1.3.1', + viennarnaArchive: { version: viennaVersion, url: viennaUrl, sha256: viennaSha256 }, threading: 'single worker per task', sha256: Object.fromEntries(['intarna.js', 'intarna.wasm'].map((name) => [name, createHash('sha256').update(readFileSync(resolve(site, 'wasm', name))).digest('hex')])), }; writeFileSync(resolve(site, 'build-info.json'), JSON.stringify(metadata, null, 2) + '\n'); -writeFileSync(resolve(site, 'build-info.txt'), `${version}\nIntaRNA repository: ${metadata.source}\nIntaRNA revision: ${metadata.commit}${metadata.dirty ? ' (modified sources)' : ''}\nEmscripten: ${emscripten}\nzlib: ${metadata.zlib}\n${Object.entries(metadata.sha256).map(([name, hash]) => `SHA-256 ${name}: ${hash}`).join('\n')}\n`); +writeFileSync(resolve(site, 'build-info.txt'), `${version}\nIntaRNA repository: ${metadata.source}\nIntaRNA revision: ${metadata.commit}${metadata.dirty ? ' (modified sources)' : ''}\nEmscripten: ${emscripten}\nzlib: ${metadata.zlib}\nViennaRNA archive: ${viennaUrl}\nViennaRNA archive SHA-256: ${viennaSha256}\n${Object.entries(metadata.sha256).map(([name, hash]) => `SHA-256 ${name}: ${hash}`).join('\n')}\n`); console.log(`Recorded compiled versions: IntaRNA ${match[1]}, ViennaRNA ${match[2]}, Boost ${match[3]}`); diff --git a/webinterface/docs/step1-wasm-compilation.md b/webinterface/docs/step1-wasm-compilation.md index 06aba756..8135c2f5 100644 --- a/webinterface/docs/step1-wasm-compilation.md +++ b/webinterface/docs/step1-wasm-compilation.md @@ -17,31 +17,50 @@ node scripts/wasm/smoke.mjs node scripts/wasm/smoke-protocol.mjs ``` -`WASM_BUILD_DIR` changes the build directory; `JOBS` defaults to 2. Source -archives are downloaded into `build-wasm/downloads` and verified by pinned -SHA-256 hashes on every invocation. ViennaRNA 2.7.0 uses the official GitHub -release asset, with its own pinned checksum; the university download endpoint +`WASM_BUILD_DIR` changes the build directory; `JOBS` defaults to 2. Local builds +pin ViennaRNA 2.7.2 and its official GitHub release asset SHA-256 in +`scripts/wasm/vienna-release.json`. On every run, the GitHub workflow resolves +the newest stable **2.x** release (at least 2.7.2), excludes drafts/prereleases +and major 3, and uses the asset's published SHA-256. Selection compares numeric +versions across all API pages, rather than using GitHub's `/latest` endpoint. +Missing release assets, digests or failed API requests stop the build. + +To use that same selection locally (optionally set `GITHUB_TOKEN` for API limits): + +```sh +python3 scripts/wasm/vienna-release.py --latest > /tmp/vienna-release.json +WASM_VIENNA_RELEASE=/tmp/vienna-release.json bash scripts/wasm/build.sh +``` + +Keep the resolved manifest to reproduce that exact release later with +`WASM_VIENNA_RELEASE`. `build-info.json` and `build-info.txt` record the selected +archive URL and checksum as well as the actual compiled library version. + +Source archives are downloaded into `build-wasm/downloads` and verified by the +selected SHA-256 hashes on every invocation. Use the official GitHub release +asset, rather than an autogenerated source archive; the university endpoint returned an HTTP 200 text response instead of the archive in PR #291's build. `scripts/wasm/download.sh` removes invalid cached archives and checks new downloads before moving them from `.part` into the cache. Failed transfers and checksum mismatches leave no cached archive or partial file, so a subsequent build can retry. Run `python3 scripts/wasm/test-downloads.py` to check download, -cache recovery and failure handling against a local HTTP server, without an SDK -or external network access. The build directory is ignored. Delete -`build-wasm/deps` and `build-wasm/sources` for a fresh dependency build after -changing dependency versions or compiler flags. Do not reuse this prefix with a -different SDK. Native libraries are never accepted from system pkg-config paths. +cache recovery and failure handling against a local HTTP server, and +`python3 scripts/wasm/test-vienna-release.py` for release selection and manifest +validation, without an SDK or external network access. The build directory is +ignored. ViennaRNA sources, its installation prefix and IntaRNA objects are +separated by the selected version and checksum, so upgrades cannot reuse an +older RNAlib or IntaRNA object cache (including the old 2.7.0 prefix). zlib and +Boost caches remain reusable. Delete the build directory for a fresh build +after changing compiler flags or SDKs. Native libraries are never accepted +from system pkg-config paths. The script supplies absolute source and generated-header include paths for the out-of-tree IntaRNA build, and uses generated files from the build directory. ViennaRNA also needs an explicit source include path when its no-SVM build -omits the target's normal include flags. ViennaRNA 2.7.0 bundles dlib 19.24, whose Unicode -`uint32` string requires a generic `std::char_traits` removed in modern libc++. -The script backports [dlib's upstream `char32_t` typedef](https://github.com/davisking/dlib/blob/master/dlib/unicode/unicode.h) -in the downloaded source only, adds its matching serialization overload and -supplies a missing dependent template argument rejected by current Clang. -`scripts/wasm/patch-vienna.py` checks the expected source before each replacement. -It does not change the RNA algorithms. +omits the target's normal include flags. ViennaRNA 2.7.2 bundles dlib 20.0, +which already supports modern libc++; no downloaded sources are patched. +Staging takes the ViennaRNA, dlib and Naview notices from the source archive +identified by the compiled provenance, and displays their selected versions. The legacy Boost and zlib configure macros declare dummy test functions with incorrect signatures (`exit()` and `inflateEnd()`), which wasm-ld correctly @@ -57,7 +76,7 @@ RNAlib link flags because the upstream `.pc` includes native-only linker flags. - Boost 1.87.0: headers plus static filesystem, program_options, regex, iostreams, serialization and system libraries, built with Boost.Build's Emscripten toolset. Optional bzip2, lzma and zstd backends are disabled. -- ViennaRNA 2.7.0: static RNAlib, without OpenMP, pthreads, SIMD, external +- ViennaRNA 2.7.2 locally, newest stable 2.x in CI: static RNAlib, without OpenMP, pthreads, SIMD, external solvers, bindings, documentation or companion applications. Compiling only RNAlib avoids executing cross-compiled helper applications on the host. - IntaRNA: C++23, bundled Kokkos mdspan, standard CLI, default energy models From d44d7a6ca00a519563aebb688f98194d66960ff3 Mon Sep 17 00:00:00 2001 From: AlexanderMitrofanov Date: Thu, 22 Oct 2026 15:35:38 +0200 Subject: [PATCH 2/2] fix(wasm): isolate legacy ViennaRNA headers from Boost includes --- ChangeLog | 6 +++-- scripts/wasm/build.sh | 6 +++-- scripts/wasm/test-vienna-release.py | 30 +++++++++++++++++++++ webinterface/docs/step1-wasm-compilation.md | 4 ++- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index cafc21d7..02b0b5f7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -259,7 +259,8 @@ energies, restricted partition sums, and trackers. release pages in CI; require its release tarball and published SHA-256. * Pin 2.7.2 for reproducible local builds and accept saved release manifests. Isolate RNAlib sources/installations and IntaRNA objects by archive identity - to prevent reuse of libraries or headers from another release (PR #292). + and keep Boost/zlib in a separate prefix to prevent reuse or include-path + shadowing of libraries/headers from another release (PR #292). * scripts/wasm/patch-vienna.py - Remove the dlib 19.24 compatibility patch; ViennaRNA 2.7.2 bundles dlib 20.0. * scripts/wasm/{write-build-info.mjs,stage.sh,smoke-protocol.mjs,test-browser.py} @@ -267,7 +268,8 @@ energies, restricted partition sums, and trackers. version, stage matching dependency notices and test actual version exports. * scripts/wasm/test-vienna-release.py, .github/workflows/build-wasm.yml + Cover numeric version selection, pagination, major-3/prerelease exclusion, - missing/invalid asset metadata, API failures and manifest validation. + missing/invalid asset metadata, API failures, manifest validation and + rejection of compiled engines with a stale ViennaRNA version. * webinterface/docs/step1-wasm-compilation.md * Document latest-2.x CI selection, pinned local reproduction, versioned caches, current bundled dlib and archive provenance. diff --git a/scripts/wasm/build.sh b/scripts/wasm/build.sh index 1ca91ac0..10499d68 100644 --- a/scripts/wasm/build.sh +++ b/scripts/wasm/build.sh @@ -5,7 +5,9 @@ repo=$(cd "$(dirname "$0")/../.." && pwd) build=${WASM_BUILD_DIR:-"$repo/build-wasm"} mkdir -p "$build" build=$(cd "$build" && pwd) -prefix="$build/deps" +# Keep legacy RNAlib headers out of Boost's -I path: ordinary -I searches take +# precedence over the -isystem ViennaRNA paths produced by IntaRNA's configure. +prefix="$build/deps/common" jobs=${JOBS:-2} for tool in emcc em++ emar emranlib emconfigure emmake curl pkg-config node python3; do command -v "$tool" >/dev/null || { echo "Missing tool: $tool" >&2; exit 1; } @@ -28,7 +30,7 @@ fetch "$vienna_url" "ViennaRNA-$vienna_version.tar.gz" "$vienna_sha256" # Separate source, installed RNAlib and IntaRNA objects by archive identity so # changing the release cannot reuse headers, libraries or objects from an old one. vienna_id="vienna-$vienna_version-$vienna_sha256" -vienna_prefix="$prefix/$vienna_id" +vienna_prefix="$build/deps/$vienna_id" vienna_sources="$build/sources/$vienna_id" intarna_build="$build/intarna-$vienna_id" diff --git a/scripts/wasm/test-vienna-release.py b/scripts/wasm/test-vienna-release.py index 7357a88e..2bf50995 100644 --- a/scripts/wasm/test-vienna-release.py +++ b/scripts/wasm/test-vienna-release.py @@ -85,6 +85,36 @@ def test_invalid_pin_fails_without_emitting_shell_fields(self): self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, "") + def test_provenance_rejects_stale_engine_and_records_selected_archive(self): + manifest = json.loads(SCRIPT.with_suffix(".json").read_text()) + with TemporaryDirectory(prefix="vienna engine ") as directory: + site = Path(directory) + assets = site / "wasm" + assets.mkdir() + (assets / "package.json").write_text('{"type":"module"}') + (assets / "intarna.wasm").write_bytes(b"test engine") + for version in ["2.7.0", manifest["version"]]: + (assets / "intarna.js").write_text( + "export default async ({print}) => ({callMain() {" + f"print('IntaRNA 3.4.1 using Vienna RNA package {version} and boost 1.87.0');" + "return 0;}});" + ) + result = subprocess.run([ + "node", str(SCRIPT.with_name("write-build-info.mjs")), + str(SCRIPT.parents[2]), str(site), "4.0.23", "test-commit", "", + manifest["version"], manifest["url"], manifest["sha256"], + ], capture_output=True, text=True) + if version == "2.7.0": + self.assertNotEqual(result.returncode, 0) + self.assertIn("differs from selected", result.stderr) + self.assertFalse((site / "build-info.json").exists()) + else: + self.assertEqual(result.returncode, 0, result.stderr) + metadata = json.loads((site / "build-info.json").read_text()) + self.assertEqual(metadata["viennarna"], version) + self.assertEqual(metadata["viennarnaArchive"], manifest) + self.assertIn(manifest["sha256"], (site / "build-info.txt").read_text()) + if __name__ == "__main__": unittest.main() diff --git a/webinterface/docs/step1-wasm-compilation.md b/webinterface/docs/step1-wasm-compilation.md index 8135c2f5..dc09f76c 100644 --- a/webinterface/docs/step1-wasm-compilation.md +++ b/webinterface/docs/step1-wasm-compilation.md @@ -50,7 +50,9 @@ validation, without an SDK or external network access. The build directory is ignored. ViennaRNA sources, its installation prefix and IntaRNA objects are separated by the selected version and checksum, so upgrades cannot reuse an older RNAlib or IntaRNA object cache (including the old 2.7.0 prefix). zlib and -Boost caches remain reusable. Delete the build directory for a fresh build +Boost use a separate `build-wasm/deps/common` prefix, so legacy ViennaRNA headers +cannot override the selected release through Boost's include path. Their caches +remain reusable across ViennaRNA upgrades. Delete the build directory for a fresh build after changing compiler flags or SDKs. Native libraries are never accepted from system pkg-config paths.