diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index c6baf0d0..aadec7f3 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -31,9 +31,18 @@ jobs: with: python-version: '3.12' - name: Test dependency download integrity and cache recovery - run: python scripts/wasm/test-downloads.py + run: | + python scripts/wasm/test-downloads.py + python scripts/wasm/test-vienna-release.py - name: Test input, worker lifecycle and CSV handling run: npm test --prefix webinterface + - name: Resolve newest stable ViennaRNA 2.x release and checksum + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python scripts/wasm/vienna-release.py --latest > "$RUNNER_TEMP/vienna-release.json" + cat "$RUNNER_TEMP/vienna-release.json" + echo "WASM_VIENNA_RELEASE=$RUNNER_TEMP/vienna-release.json" >> "$GITHUB_ENV" - name: Cross-compile dependencies and IntaRNA run: bash scripts/wasm/build.sh - name: Check real WASM predictions against the native reference diff --git a/ChangeLog b/ChangeLog index 01272691..02b0b5f7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -84,6 +84,10 @@ ## Technical changes and Optimizations +- build WASM with ViennaRNA 2.7.2 locally and the newest stable 2.x in CI; + verify published archive checksums, isolate versioned caches and retain + selected dependency provenance and notices (PR #292 review) + - BUGFIX : restore WASM dependency downloads using the pinned official ViennaRNA release asset; verify downloads before caching and recover invalid caches (PR #291) @@ -249,6 +253,27 @@ energies, restricted partition sums, and trackers. ################################################################################ ################################################################################ +261022 Alexander Mitrofanov + * scripts/wasm/{vienna-release.py,vienna-release.json}, build.sh + + Resolve the newest stable ViennaRNA >=2.7.2 and <3 from all official GitHub + release pages in CI; require its release tarball and published SHA-256. + * Pin 2.7.2 for reproducible local builds and accept saved release manifests. + Isolate RNAlib sources/installations and IntaRNA objects by archive identity + and keep Boost/zlib in a separate prefix to prevent reuse or include-path + shadowing of libraries/headers from another release (PR #292). + * scripts/wasm/patch-vienna.py + - Remove the dlib 19.24 compatibility patch; ViennaRNA 2.7.2 bundles dlib 20.0. + * scripts/wasm/{write-build-info.mjs,stage.sh,smoke-protocol.mjs,test-browser.py} + * Record the selected archive URL/checksum, check the compiled ViennaRNA + version, stage matching dependency notices and test actual version exports. + * scripts/wasm/test-vienna-release.py, .github/workflows/build-wasm.yml + + Cover numeric version selection, pagination, major-3/prerelease exclusion, + missing/invalid asset metadata, API failures, manifest validation and + rejection of compiled engines with a stale ViennaRNA version. + * webinterface/docs/step1-wasm-compilation.md + * Document latest-2.x CI selection, pinned local reproduction, versioned + caches, current bundled dlib and archive provenance. + 261022 Alexander Mitrofanov * scripts/wasm/{build,download}.sh * Use the official GitHub ViennaRNA 2.7.0 release asset and pin its SHA-256; diff --git a/scripts/wasm/build.sh b/scripts/wasm/build.sh index 60140119..10499d68 100644 --- a/scripts/wasm/build.sh +++ b/scripts/wasm/build.sh @@ -5,9 +5,11 @@ repo=$(cd "$(dirname "$0")/../.." && pwd) build=${WASM_BUILD_DIR:-"$repo/build-wasm"} mkdir -p "$build" build=$(cd "$build" && pwd) -prefix="$build/deps" +# Keep legacy RNAlib headers out of Boost's -I path: ordinary -I searches take +# precedence over the -isystem ViennaRNA paths produced by IntaRNA's configure. +prefix="$build/deps/common" jobs=${JOBS:-2} -for tool in emcc em++ emar emranlib emconfigure emmake curl pkg-config node; do +for tool in emcc em++ emar emranlib emconfigure emmake curl pkg-config node python3; do command -v "$tool" >/dev/null || { echo "Missing tool: $tool" >&2; exit 1; } done mkdir -p "$build/downloads" "$build/sources" "$prefix" "$build/site/wasm" @@ -18,16 +20,27 @@ fetch https://zlib.net/fossils/zlib-1.3.1.tar.gz zlib-1.3.1.tar.gz \ 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 fetch https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.bz2 boost_1_87_0.tar.bz2 \ af57be25cb4c4f4b413ed692fe378affb4352ea50fbe294a11ef548f4d527d89 -fetch https://github.com/ViennaRNA/ViennaRNA/releases/download/v2.7.0/ViennaRNA-2.7.0.tar.gz ViennaRNA-2.7.0.tar.gz \ - e2addf33e7c0dd4fff31d8e9a46fda8e9efcde0d199f789f20eb0a13b7f7135d +# CI supplies the newest stable 2.x manifest; local builds use the checked-in pin. +read -r vienna_version vienna_url vienna_sha256 < <( + python3 "$repo/scripts/wasm/vienna-release.py" --fields \ + --manifest "${WASM_VIENNA_RELEASE:-$repo/scripts/wasm/vienna-release.json}" +) +echo "Building ViennaRNA $vienna_version (SHA-256 $vienna_sha256)" +fetch "$vienna_url" "ViennaRNA-$vienna_version.tar.gz" "$vienna_sha256" +# Separate source, installed RNAlib and IntaRNA objects by archive identity so +# changing the release cannot reuse headers, libraries or objects from an old one. +vienna_id="vienna-$vienna_version-$vienna_sha256" +vienna_prefix="$build/deps/$vienna_id" +vienna_sources="$build/sources/$vienna_id" +intarna_build="$build/intarna-$vienna_id" # Keep pkg-config searches inside the WASM prefix, including configure probes. -export PKG_CONFIG_LIBDIR="$prefix/lib/pkgconfig" +export PKG_CONFIG_LIBDIR="$vienna_prefix/lib/pkgconfig:$prefix/lib/pkgconfig" export PKG_CONFIG_PATH="$PKG_CONFIG_LIBDIR" export CFLAGS='-O3 -fexceptions' export CXXFLAGS='-O3 -fexceptions' -export CPPFLAGS="-I$prefix/include" -export LDFLAGS="-L$prefix/lib -fexceptions" +export CPPFLAGS="-I$vienna_prefix/include -I$prefix/include" +export LDFLAGS="-L$vienna_prefix/lib -L$prefix/lib -fexceptions" if [[ ! -f "$prefix/lib/libz.a" ]]; then tar -xzf "$build/downloads/zlib-1.3.1.tar.gz" -C "$build/sources" (cd "$build/sources/zlib-1.3.1" @@ -47,27 +60,26 @@ if [[ ! -f "$prefix/.boost-1.87.0-js-exceptions-complete" ]]; then -sZLIB_LIBPATH="$prefix/lib" --prefix="$prefix" install) touch "$prefix/.boost-1.87.0-js-exceptions-complete" fi -if [[ ! -f "$prefix/lib/libRNA.a" ]]; then - tar -xzf "$build/downloads/ViennaRNA-2.7.0.tar.gz" -C "$build/sources" - (cd "$build/sources/ViennaRNA-2.7.0" - CPPFLAGS="$CPPFLAGS -I$PWD/src" emconfigure ./configure --host=wasm32-unknown-emscripten --prefix="$prefix" \ +if [[ ! -f "$vienna_prefix/.complete" ]]; then + mkdir -p "$vienna_sources" + tar -xzf "$build/downloads/ViennaRNA-$vienna_version.tar.gz" -C "$vienna_sources" + (cd "$vienna_sources/ViennaRNA-$vienna_version" + CPPFLAGS="$CPPFLAGS -I$PWD/src" emconfigure ./configure --host=wasm32-unknown-emscripten --prefix="$vienna_prefix" \ --disable-shared --enable-static --disable-openmp --disable-pthreads \ --disable-lto --disable-simd --disable-vectorize --disable-mpfr \ --without-svm --without-gsl --without-swig --without-perl --without-python \ --without-doc --without-cla --without-check --without-kinfold \ --without-forester --without-rnalocmin --without-rnaxplorer - # Backport dlib's upstream char32_t typedef for modern libc++ char_traits. - # Only the downloaded dependency is patched; RNAlib algorithms are unchanged. - python3 "$repo/scripts/wasm/patch-vienna.py" "$PWD" emmake make -C src/ViennaRNA -j "$jobs" emmake make -C src/ViennaRNA install emmake make install-pkgconfigDATA) + touch "$vienna_prefix/.complete" fi # Legacy Autoconf probes declare exit()/inflateEnd() with the wrong signatures, # which wasm-ld rejects. Verify real dependency APIs before caching those probes. -em++ -std=c++23 -O2 -fexceptions -I"$prefix/include" \ - "$repo/scripts/wasm/dependency-probe.cpp" -L"$prefix/lib" \ +em++ -std=c++23 -O2 -fexceptions -I"$vienna_prefix/include" -I"$prefix/include" \ + "$repo/scripts/wasm/dependency-probe.cpp" -L"$vienna_prefix/lib" -L"$prefix/lib" \ -lboost_filesystem -lboost_system -lboost_program_options -lboost_regex \ -lboost_iostreams -lboost_serialization -lRNA -lz -o "$build/dependency-probe.js" node "$build/dependency-probe.js" @@ -78,15 +90,15 @@ export ac_cv_lib_boost_regex_exit=yes export ac_cv_lib_boost_iostreams_exit=yes export ac_cv_lib_z_inflateEnd=yes # RNAlib's .pc embeds a native-only -Wl,-fno-lto; this is a static WASM build. -export VRNA_LIBS="-L$prefix/lib -lRNA -lm" +export VRNA_LIBS="-L$vienna_prefix/lib -lRNA -lm" # Use the native Autotools source lists and CLI entry point, with a JS output. (cd "$repo" && bash autotools-init.sh) -mkdir -p "$build/intarna" -(cd "$build/intarna" - CXXFLAGS="$CXXFLAGS -Werror=inconsistent-missing-override" CPPFLAGS="-I$build/intarna/src -I$repo/src $CPPFLAGS" emconfigure "$repo/configure" --host=wasm32-unknown-emscripten \ +mkdir -p "$intarna_build" +(cd "$intarna_build" + CXXFLAGS="$CXXFLAGS -Werror=inconsistent-missing-override" CPPFLAGS="-I$intarna_build/src -I$repo/src $CPPFLAGS" emconfigure "$repo/configure" --host=wasm32-unknown-emscripten \ --disable-multithreading --disable-log-coloring --with-mdspan=kokkos \ - --with-vrna="$prefix" --with-boost="$prefix" \ + --with-vrna="$vienna_prefix" --with-boost="$prefix" \ --with-boost-libdir="$prefix/lib" --with-zlib="$prefix" emmake make -C src -j "$jobs" EXEEXT=.js \ LDFLAGS="$LDFLAGS -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=createIntaRNA -sENVIRONMENT=web,worker,node -sINVOKE_RUN=0 -sEXIT_RUNTIME=1 -sEXPORTED_RUNTIME_METHODS=FS,callMain -sALLOW_MEMORY_GROWTH=1 -sMAXIMUM_MEMORY=1073741824 -sSTACK_SIZE=8388608" @@ -96,5 +108,6 @@ mkdir -p "$build/intarna" printf '{"type":"module"}\n' > "$build/site/wasm/package.json" node "$repo/scripts/wasm/write-build-info.mjs" "$repo" "$build/site" \ "$(emcc -dumpversion)" "$(git -C "$repo" rev-parse HEAD)" \ - "$(git -C "$repo" status --porcelain --untracked-files=no)" + "$(git -C "$repo" status --porcelain --untracked-files=no)" \ + "$vienna_version" "$vienna_url" "$vienna_sha256" echo "WASM assets: $build/site/wasm" diff --git a/scripts/wasm/patch-vienna.py b/scripts/wasm/patch-vienna.py deleted file mode 100644 index 4b636aad..00000000 --- a/scripts/wasm/patch-vienna.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Apply narrow dlib 19.24 compatibility fixes in the downloaded ViennaRNA tree. - -Modern libc++ only defines char_traits for standard character types. dlib now -uses char32_t upstream. Its old serialization overloads and dependent template -call also need the matching C++ fixes. RNA models and recurrences are untouched. -""" -import sys -from pathlib import Path - -root = Path(sys.argv[1]) / "src/dlib-19.24/dlib" -patches = [ - ("unicode/unicode.h", "typedef uint32 unichar;", "typedef char32_t unichar;"), - ("serialize.h", " USE_DEFAULT_INT_SERIALIZATION_FOR(wchar_t)", - " USE_DEFAULT_INT_SERIALIZATION_FOR(wchar_t)\n USE_DEFAULT_INT_SERIALIZATION_FOR(char32_t)"), - ("global_optimization/find_max_global.h", "::template go(std::forward(f),a)", - "::template go(std::forward(f),a)"), -] -for filename, before, after in patches: - path = root / filename - source = path.read_text() - if after in source: - continue - if source.count(before) != 1: - raise SystemExit(f"Unexpected dlib source: {path}; review compatibility patch") - path.write_text(source.replace(before, after)) diff --git a/scripts/wasm/smoke-protocol.mjs b/scripts/wasm/smoke-protocol.mjs index f168a7b1..ee27ad8b 100644 --- a/scripts/wasm/smoke-protocol.mjs +++ b/scripts/wasm/smoke-protocol.mjs @@ -1,5 +1,6 @@ // Integration coverage of browser argument translation against the real CLI. import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { DEFAULTS, EVALUATION_EXAMPLE, prepareRequest, makeArguments } from '../../webinterface/protocol.js'; @@ -8,6 +9,7 @@ import { parseResults, projectCsv } from '../../webinterface/results.js'; import { archiveFiles } from '../../webinterface/archive.js'; const asset = resolve(process.argv[2] || 'build-wasm/site/wasm/intarna.js'); const { default: createIntaRNA } = await import(pathToFileURL(asset)); +const compiledVersion = readFileSync(new URL('../build-info.txt', pathToFileURL(asset)), 'utf8'); async function run(input, format = 'C') { const request = prepareRequest({ ...DEFAULTS, ...input }); const diagnostics = []; @@ -54,7 +56,7 @@ async function replayArchive(input) { const outputs = fs.readdir('/output').filter((name) => name !== '.' && name !== '..').map((name) => ({ name, text: fs.readFile(`/output/${name}`, { encoding: 'utf8' }) })); const reports = [['normal', 'N'], ['detailed', 'D'], ['ensemble', 'E']].filter(([key]) => request.outputs.includes(key)); for (const [key, format] of reports) outputs.push({ name: `intarna-${key}.txt`, text: (await run(input, format)).text }); - const files = archiveFiles(request, { csv: original.text, files: outputs }, 'IntaRNA 3.4.1\nViennaRNA 2.7.0\n'); + const files = archiveFiles(request, { csv: original.text, files: outputs }, compiledVersion); for (const config of ['parameters', ...reports.map(([key]) => key)]) { const diagnostics = []; const module = await createIntaRNA({ noInitialRun: true, locateFile: () => new URL('./intarna.wasm', pathToFileURL(asset)).href, print: (line) => diagnostics.push(line), printErr: (line) => diagnostics.push(line) }); diff --git a/scripts/wasm/stage.sh b/scripts/wasm/stage.sh index d123f2b1..20367c2e 100644 --- a/scripts/wasm/stage.sh +++ b/scripts/wasm/stage.sh @@ -6,6 +6,24 @@ build=${WASM_BUILD_DIR:-"$repo/build-wasm"} site="$build/site" test -s "$site/wasm/intarna.js" test -s "$site/wasm/intarna.wasm" +# Select notices from the archive used by these assets, including dlib upgrades. +read -r vienna_version vienna_sha256 dlib_directory < <(python3 - "$site" "$build" <<'PYSOURCE' +import json, re, sys +from pathlib import Path +site, build = map(Path, sys.argv[1:]) +metadata = json.loads((site / "build-info.json").read_text()) +archive = metadata["viennarnaArchive"] +version, digest = archive["version"], archive["sha256"] +if version != metadata["viennarna"] or not re.fullmatch(r"2\.\d+\.\d+", version) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise SystemExit("Invalid compiled ViennaRNA archive identity") +source = build / "sources" / f"vienna-{version}-{digest}" / f"ViennaRNA-{version}" +dlib = list((source / "src").glob("dlib-*/dlib/LICENSE.txt")) +if len(dlib) != 1 or not re.fullmatch(r"dlib-[0-9.]+", dlib[0].parent.parent.name): + raise SystemExit("Expected one bundled dlib license in the compiled ViennaRNA source") +print(version, digest, dlib[0].parent.parent.name) +PYSOURCE +) +vienna_source="$build/sources/vienna-$vienna_version-$vienna_sha256/ViennaRNA-$vienna_version" mkdir -p "$site/licenses" cp -R "$repo/webinterface/assets" "$site/" for file in index.html styles.css app.js mark.svg protocol.js runner.js results.js options.js columns.js archive.js visualization.js; do @@ -18,20 +36,20 @@ cp "$repo/src/easylogging++.LICENCE.txt" "$site/licenses/easylogging.txt" cp "$repo/src/mdspan/LICENSE" "$site/licenses/mdspan.txt" cp "$build/sources/boost_1_87_0/LICENSE_1_0.txt" "$site/licenses/Boost.txt" cp "$build/sources/zlib-1.3.1/zlib.h" "$site/licenses/zlib.h" -cp "$build/sources/ViennaRNA-2.7.0/COPYING" "$site/licenses/ViennaRNA.txt" -cp "$build/sources/ViennaRNA-2.7.0/src/dlib-19.24/dlib/LICENSE.txt" "$site/licenses/dlib.txt" -cp "$build/sources/ViennaRNA-2.7.0/src/ViennaRNA/plotting/naview/naview.c" "$site/licenses/naview.c" +cp "$vienna_source/COPYING" "$site/licenses/ViennaRNA.txt" +cp "$vienna_source/src/$dlib_directory/dlib/LICENSE.txt" "$site/licenses/dlib.txt" +cp "$vienna_source/src/ViennaRNA/plotting/naview/naview.c" "$site/licenses/naview.c" emscripten_dir=$(dirname "$(command -v emcc)") cp "$emscripten_dir/LICENSE" "$site/licenses/Emscripten.txt" cp "$emscripten_dir/system/lib/libc/musl/COPYRIGHT" "$site/licenses/musl.txt" for library in libcxx libcxxabi libunwind compiler-rt; do cp "$emscripten_dir/system/lib/$library/LICENSE.TXT" "$site/licenses/$library.txt" done -cat > "$site/licenses/index.html" <<'HTML' +cat > "$site/licenses/index.html" <IntaRNA licenses

IntaRNA and third-party notices

IntaRNA: Bioinformatics Group of Prof. Backofen, University of Freiburg.
ViennaRNA: authors and Institute for Theoretical Chemistry, University of Vienna.

-

This distribution uses ViennaRNA 2.7.0, Boost 1.87.0, zlib 1.3.1, bundled dlib 19.24 (with Unicode/template compatibility fixes), Kokkos mdspan, Easylogging++ and the Emscripten 4.0.23 runtime.

+

This distribution uses ViennaRNA $vienna_version, Boost 1.87.0, zlib 1.3.1, bundled dlib ${dlib_directory#dlib-}, Kokkos mdspan, Easylogging++ and the Emscripten 4.0.23 runtime.

  • IntaRNA
  • ViennaRNA
  • Boost
  • zlib
  • diff --git a/scripts/wasm/test-browser.py b/scripts/wasm/test-browser.py index 3bcdadbc..079fbdf5 100644 --- a/scripts/wasm/test-browser.py +++ b/scripts/wasm/test-browser.py @@ -2,7 +2,8 @@ import argparse from pathlib import Path from io import BytesIO -from urllib.parse import parse_qs, urlparse +import re +from urllib.parse import parse_qs, urljoin, urlparse from zipfile import ZipFile from playwright.sync_api import sync_playwright, expect @@ -53,7 +54,11 @@ def predict(): expect(status).to_contain_text('complete:', timeout=120000) expect(page.locator('#input-error')).not_to_be_visible() - expect(page.locator('#version-info')).to_contain_text('ViennaRNA 2.7.0') + metadata = page.request.get(urljoin(args.url, 'build-info.json')).json() + vienna_version = metadata['viennarna'] + assert re.fullmatch(r'2\.\d+\.\d+', vienna_version) + assert metadata['viennarnaArchive']['version'] == vienna_version + expect(page.locator('#version-info')).to_contain_text(f'ViennaRNA {vienna_version}') for logo in page.locator('.site-header img').all(): assert logo.evaluate('node => node.complete && node.naturalWidth > 0') for key in ['query-options', 'target-options', 'seed-options', 'helix-options', 'shape-options', 'interaction-options', 'output-options', 'general-options', 'sequence-input']: @@ -122,7 +127,7 @@ def predict(): assert archive['intarna-interactions.csv'] == csv assert archive['query.fa'] == '>query\nGGGAGGAGGG\n' assert '# IntaRNA 3.4.1' in archive['parameters.cfg'] - assert 'Vienna RNA package 2.7.0' in archive['parameters.cfg'] + assert f'Vienna RNA package {vienna_version}' in archive['parameters.cfg'] assert 'outNumber = 10' in archive['parameters.cfg'] assert 'IntaRNA --parameterFile=parameters.cfg' in archive['parameters.cfg'] assert 'temperature =' not in archive['parameters.cfg'] diff --git a/scripts/wasm/test-vienna-release.py b/scripts/wasm/test-vienna-release.py new file mode 100644 index 00000000..2bf50995 --- /dev/null +++ b/scripts/wasm/test-vienna-release.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check stable-major selection, pagination and archive identity without GitHub.""" +from copy import deepcopy +import importlib.util +from io import BytesIO +import json +from pathlib import Path +import subprocess +from tempfile import TemporaryDirectory +import unittest +from unittest.mock import patch +from urllib.error import HTTPError + + +SCRIPT = Path(__file__).with_name("vienna-release.py") +SPEC = importlib.util.spec_from_file_location("vienna_release", SCRIPT) +resolver = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(resolver) +DIGEST = "a" * 64 + + +def release(version, **fields): + return { + "tag_name": f"v{version}", "draft": False, "prerelease": False, + "assets": [{ + "name": f"ViennaRNA-{version}.tar.gz", "digest": f"sha256:{DIGEST}", + "browser_download_url": f"{resolver.REPOSITORY}/releases/download/v{version}/ViennaRNA-{version}.tar.gz", + }], **fields, + } + + +class ReleaseTests(unittest.TestCase): + def resolve(self, pages): + with patch.object(resolver, "urlopen", side_effect=[BytesIO(json.dumps(page).encode()) for page in pages]) as fetch: + result = resolver.latest_release() + return result, fetch + + def test_numeric_selection_excludes_major3_drafts_and_prereleases(self): + result, _ = self.resolve([[ + release("3.0.0"), release("2.9.0"), release("2.10.0"), + release("2.11.0", prerelease=True), release("2.12.0", draft=True), + release("2.13.0-rc.1"), release("2.7.2"), release("2.7.1"), + ]]) + self.assertEqual(result["version"], "2.10.0") + self.assertEqual(result["sha256"], DIGEST) + + def test_pagination_finds_version2_after_one_hundred_newer_releases(self): + result, fetch = self.resolve([[release(f"3.0.{n}") for n in range(100)], [release("2.7.2")]]) + self.assertEqual(result["version"], "2.7.2") + self.assertEqual(fetch.call_count, 2) + self.assertTrue(fetch.call_args.args[0].full_url.endswith("per_page=100&page=2")) + + def test_newest_release_must_have_exact_asset_url_and_digest(self): + latest = release("2.8.0") + for field, value in [("name", "source.tar.gz"), ("digest", None), ("digest", "sha256:bad"), ("digest", "md5:" + DIGEST), ("browser_download_url", "https://example.com/archive.tar.gz")]: + with self.subTest(field=field, value=value): + invalid = deepcopy(latest) + invalid["assets"][0][field] = value + with self.assertRaises(ValueError): + self.resolve([[release("2.7.2"), invalid]]) + with self.assertRaises(ValueError): + self.resolve([[release("2.8.0", assets=latest["assets"] * 2)]]) + + def test_no_supported_release_or_api_failure_is_an_error(self): + with self.assertRaises(ValueError): + self.resolve([[release("3.0.0"), release("2.7.1"), release("2.8.0", prerelease=True)]]) + with patch.object(resolver, "urlopen", side_effect=HTTPError(resolver.API, 403, "rate limited", {}, None)): + with self.assertRaises(HTTPError): + resolver.latest_release() + + def test_checked_in_pin_and_shell_fields(self): + manifest = json.loads(subprocess.check_output(["python3", str(SCRIPT)], text=True)) + self.assertEqual(manifest["version"], "2.7.2") + fields = subprocess.check_output(["python3", str(SCRIPT), "--fields"], text=True).split() + self.assertEqual(fields, list(manifest.values())) + + def test_invalid_pin_fails_without_emitting_shell_fields(self): + manifest = json.loads(SCRIPT.with_suffix(".json").read_text()) + with TemporaryDirectory(prefix="vienna manifest ") as directory: + path = Path(directory) / "release.json" + for field, value in [("version", "3.0.0"), ("version", "2.7.0"), ("url", "https://example.com"), ("sha256", "bad")]: + with self.subTest(field=field): + path.write_text(json.dumps({**manifest, field: value})) + result = subprocess.run(["python3", str(SCRIPT), "--manifest", str(path), "--fields"], capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + def test_provenance_rejects_stale_engine_and_records_selected_archive(self): + manifest = json.loads(SCRIPT.with_suffix(".json").read_text()) + with TemporaryDirectory(prefix="vienna engine ") as directory: + site = Path(directory) + assets = site / "wasm" + assets.mkdir() + (assets / "package.json").write_text('{"type":"module"}') + (assets / "intarna.wasm").write_bytes(b"test engine") + for version in ["2.7.0", manifest["version"]]: + (assets / "intarna.js").write_text( + "export default async ({print}) => ({callMain() {" + f"print('IntaRNA 3.4.1 using Vienna RNA package {version} and boost 1.87.0');" + "return 0;}});" + ) + result = subprocess.run([ + "node", str(SCRIPT.with_name("write-build-info.mjs")), + str(SCRIPT.parents[2]), str(site), "4.0.23", "test-commit", "", + manifest["version"], manifest["url"], manifest["sha256"], + ], capture_output=True, text=True) + if version == "2.7.0": + self.assertNotEqual(result.returncode, 0) + self.assertIn("differs from selected", result.stderr) + self.assertFalse((site / "build-info.json").exists()) + else: + self.assertEqual(result.returncode, 0, result.stderr) + metadata = json.loads((site / "build-info.json").read_text()) + self.assertEqual(metadata["viennarna"], version) + self.assertEqual(metadata["viennarnaArchive"], manifest) + self.assertIn(manifest["sha256"], (site / "build-info.txt").read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/wasm/vienna-release.json b/scripts/wasm/vienna-release.json new file mode 100644 index 00000000..d54b5517 --- /dev/null +++ b/scripts/wasm/vienna-release.json @@ -0,0 +1,5 @@ +{ + "version": "2.7.2", + "url": "https://github.com/ViennaRNA/ViennaRNA/releases/download/v2.7.2/ViennaRNA-2.7.2.tar.gz", + "sha256": "4cb7aaca00dfac92cb36b030c9111487c170b90568759bd4ffedd8318cc9384f" +} diff --git a/scripts/wasm/vienna-release.py b/scripts/wasm/vienna-release.py new file mode 100644 index 00000000..f869e786 --- /dev/null +++ b/scripts/wasm/vienna-release.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Resolve a stable ViennaRNA 2.x release asset, or validate a pinned manifest.""" +import argparse +import json +import os +from pathlib import Path +import re +from urllib.request import Request, urlopen + + +REPOSITORY = "https://github.com/ViennaRNA/ViennaRNA" +API = "https://api.github.com/repos/ViennaRNA/ViennaRNA/releases" +MINIMUM = (2, 7, 2) + + +def version_tuple(version): + if not isinstance(version, str) or not re.fullmatch(r"2\.\d+\.\d+", version): + raise ValueError("Expected a stable ViennaRNA 2.x version") + result = tuple(map(int, version.split("."))) + if result < MINIMUM: + raise ValueError("The WASM build requires ViennaRNA >= 2.7.2") + return result + + +def validate(manifest): + """Accept only official release tarballs with an explicit SHA-256.""" + version = manifest["version"] + version_tuple(version) + expected = f"{REPOSITORY}/releases/download/v{version}/ViennaRNA-{version}.tar.gz" + if manifest["url"] != expected: + raise ValueError("Expected the official ViennaRNA release asset URL") + if not isinstance(manifest["sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}", manifest["sha256"]): + raise ValueError("Missing or invalid ViennaRNA archive SHA-256") + return {key: manifest[key] for key in ("version", "url", "sha256")} + + +def latest_release(): + # Enumerate all pages: /latest may select major 3, and release dates need not + # follow version order (for example, a maintenance release of an older minor). + headers = {"Accept": "application/vnd.github+json", "User-Agent": "IntaRNA-WASM"} + if os.environ.get("GITHUB_TOKEN"): + headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}" + candidates = [] + page = 1 + while True: + request = Request(f"{API}?per_page=100&page={page}", headers=headers) + with urlopen(request, timeout=30) as response: + releases = json.load(response) + if not isinstance(releases, list): + raise ValueError("Expected a GitHub release list") + for release in releases: + tag = release.get("tag_name", "") + if release.get("draft") or release.get("prerelease") or not re.fullmatch(r"v2\.\d+\.\d+", tag): + continue + version = tag[1:] + if tuple(map(int, version.split("."))) >= MINIMUM: + candidates.append(release) + if len(releases) < 100: + break + page += 1 + if not candidates: + raise ValueError("No stable ViennaRNA 2.x release >= 2.7.2 was found") + release = max(candidates, key=lambda item: version_tuple(item["tag_name"][1:])) + version = release["tag_name"][1:] + # Fail if the newest release lacks the asset/digest; do not silently select + # an older release or GitHub's differently packaged autogenerated source. + name = f"ViennaRNA-{version}.tar.gz" + assets = [asset for asset in release.get("assets", []) if asset["name"] == name] + if len(assets) != 1: + raise ValueError(f"Expected exactly one {name} release asset") + asset = assets[0] + digest = asset.get("digest") or "" + if not digest.startswith("sha256:"): + raise ValueError(f"Missing published SHA-256 for {name}") + return validate({"version": version, "url": asset["browser_download_url"], "sha256": digest[7:]}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group() + source.add_argument("--latest", action="store_true", help="query the official GitHub releases API") + source.add_argument("--manifest", type=Path, default=Path(__file__).with_suffix(".json")) + parser.add_argument("--fields", action="store_true", help="emit version, URL and SHA-256 for build.sh") + args = parser.parse_args() + try: + manifest = latest_release() if args.latest else validate(json.loads(args.manifest.read_text())) + except (OSError, ValueError, KeyError, TypeError) as error: + parser.exit(1, f"Cannot resolve ViennaRNA release: {error}\n") + print(" ".join(manifest.values()) if args.fields else json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/wasm/write-build-info.mjs b/scripts/wasm/write-build-info.mjs index 959aacc4..4bf9fd47 100644 --- a/scripts/wasm/write-build-info.mjs +++ b/scripts/wasm/write-build-info.mjs @@ -3,7 +3,7 @@ import { createHash } from 'node:crypto'; import { readFileSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -const [repo, site, emscripten, commit, changes] = process.argv.slice(2); +const [repo, site, emscripten, commit, changes, viennaVersion, viennaUrl, viennaSha256] = process.argv.slice(2); const { default: createIntaRNA } = await import(pathToFileURL(resolve(site, 'wasm/intarna.js'))); const lines = []; const module = await createIntaRNA({ noInitialRun: true, locateFile: () => pathToFileURL(resolve(site, 'wasm/intarna.wasm')).href, print: (line) => lines.push(line), printErr: (line) => lines.push(line) }); @@ -11,13 +11,15 @@ if (module.callMain(['--version']) !== 0) throw new Error('Could not read compil const version = lines.join('\n'); const match = /IntaRNA\s+(\S+)[\s\S]*Vienna RNA package\s+(\S+)\s+and boost\s+(\S+)/i.exec(version); if (!match) throw new Error(`Unrecognized engine version output: ${version}`); +if (match[2] !== viennaVersion) throw new Error(`Compiled ViennaRNA ${match[2]} differs from selected ${viennaVersion}`); const metadata = { source: 'https://github.com/BackofenLab/IntaRNA', commit, dirty: !!changes, intarna: match[1], viennarna: match[2], boost: match[3], emscripten, zlib: '1.3.1', + viennarnaArchive: { version: viennaVersion, url: viennaUrl, sha256: viennaSha256 }, threading: 'single worker per task', sha256: Object.fromEntries(['intarna.js', 'intarna.wasm'].map((name) => [name, createHash('sha256').update(readFileSync(resolve(site, 'wasm', name))).digest('hex')])), }; writeFileSync(resolve(site, 'build-info.json'), JSON.stringify(metadata, null, 2) + '\n'); -writeFileSync(resolve(site, 'build-info.txt'), `${version}\nIntaRNA repository: ${metadata.source}\nIntaRNA revision: ${metadata.commit}${metadata.dirty ? ' (modified sources)' : ''}\nEmscripten: ${emscripten}\nzlib: ${metadata.zlib}\n${Object.entries(metadata.sha256).map(([name, hash]) => `SHA-256 ${name}: ${hash}`).join('\n')}\n`); +writeFileSync(resolve(site, 'build-info.txt'), `${version}\nIntaRNA repository: ${metadata.source}\nIntaRNA revision: ${metadata.commit}${metadata.dirty ? ' (modified sources)' : ''}\nEmscripten: ${emscripten}\nzlib: ${metadata.zlib}\nViennaRNA archive: ${viennaUrl}\nViennaRNA archive SHA-256: ${viennaSha256}\n${Object.entries(metadata.sha256).map(([name, hash]) => `SHA-256 ${name}: ${hash}`).join('\n')}\n`); console.log(`Recorded compiled versions: IntaRNA ${match[1]}, ViennaRNA ${match[2]}, Boost ${match[3]}`); diff --git a/webinterface/docs/step1-wasm-compilation.md b/webinterface/docs/step1-wasm-compilation.md index 06aba756..dc09f76c 100644 --- a/webinterface/docs/step1-wasm-compilation.md +++ b/webinterface/docs/step1-wasm-compilation.md @@ -17,31 +17,52 @@ node scripts/wasm/smoke.mjs node scripts/wasm/smoke-protocol.mjs ``` -`WASM_BUILD_DIR` changes the build directory; `JOBS` defaults to 2. Source -archives are downloaded into `build-wasm/downloads` and verified by pinned -SHA-256 hashes on every invocation. ViennaRNA 2.7.0 uses the official GitHub -release asset, with its own pinned checksum; the university download endpoint +`WASM_BUILD_DIR` changes the build directory; `JOBS` defaults to 2. Local builds +pin ViennaRNA 2.7.2 and its official GitHub release asset SHA-256 in +`scripts/wasm/vienna-release.json`. On every run, the GitHub workflow resolves +the newest stable **2.x** release (at least 2.7.2), excludes drafts/prereleases +and major 3, and uses the asset's published SHA-256. Selection compares numeric +versions across all API pages, rather than using GitHub's `/latest` endpoint. +Missing release assets, digests or failed API requests stop the build. + +To use that same selection locally (optionally set `GITHUB_TOKEN` for API limits): + +```sh +python3 scripts/wasm/vienna-release.py --latest > /tmp/vienna-release.json +WASM_VIENNA_RELEASE=/tmp/vienna-release.json bash scripts/wasm/build.sh +``` + +Keep the resolved manifest to reproduce that exact release later with +`WASM_VIENNA_RELEASE`. `build-info.json` and `build-info.txt` record the selected +archive URL and checksum as well as the actual compiled library version. + +Source archives are downloaded into `build-wasm/downloads` and verified by the +selected SHA-256 hashes on every invocation. Use the official GitHub release +asset, rather than an autogenerated source archive; the university endpoint returned an HTTP 200 text response instead of the archive in PR #291's build. `scripts/wasm/download.sh` removes invalid cached archives and checks new downloads before moving them from `.part` into the cache. Failed transfers and checksum mismatches leave no cached archive or partial file, so a subsequent build can retry. Run `python3 scripts/wasm/test-downloads.py` to check download, -cache recovery and failure handling against a local HTTP server, without an SDK -or external network access. The build directory is ignored. Delete -`build-wasm/deps` and `build-wasm/sources` for a fresh dependency build after -changing dependency versions or compiler flags. Do not reuse this prefix with a -different SDK. Native libraries are never accepted from system pkg-config paths. +cache recovery and failure handling against a local HTTP server, and +`python3 scripts/wasm/test-vienna-release.py` for release selection and manifest +validation, without an SDK or external network access. The build directory is +ignored. ViennaRNA sources, its installation prefix and IntaRNA objects are +separated by the selected version and checksum, so upgrades cannot reuse an +older RNAlib or IntaRNA object cache (including the old 2.7.0 prefix). zlib and +Boost use a separate `build-wasm/deps/common` prefix, so legacy ViennaRNA headers +cannot override the selected release through Boost's include path. Their caches +remain reusable across ViennaRNA upgrades. Delete the build directory for a fresh build +after changing compiler flags or SDKs. Native libraries are never accepted +from system pkg-config paths. The script supplies absolute source and generated-header include paths for the out-of-tree IntaRNA build, and uses generated files from the build directory. ViennaRNA also needs an explicit source include path when its no-SVM build -omits the target's normal include flags. ViennaRNA 2.7.0 bundles dlib 19.24, whose Unicode -`uint32` string requires a generic `std::char_traits` removed in modern libc++. -The script backports [dlib's upstream `char32_t` typedef](https://github.com/davisking/dlib/blob/master/dlib/unicode/unicode.h) -in the downloaded source only, adds its matching serialization overload and -supplies a missing dependent template argument rejected by current Clang. -`scripts/wasm/patch-vienna.py` checks the expected source before each replacement. -It does not change the RNA algorithms. +omits the target's normal include flags. ViennaRNA 2.7.2 bundles dlib 20.0, +which already supports modern libc++; no downloaded sources are patched. +Staging takes the ViennaRNA, dlib and Naview notices from the source archive +identified by the compiled provenance, and displays their selected versions. The legacy Boost and zlib configure macros declare dummy test functions with incorrect signatures (`exit()` and `inflateEnd()`), which wasm-ld correctly @@ -57,7 +78,7 @@ RNAlib link flags because the upstream `.pc` includes native-only linker flags. - Boost 1.87.0: headers plus static filesystem, program_options, regex, iostreams, serialization and system libraries, built with Boost.Build's Emscripten toolset. Optional bzip2, lzma and zstd backends are disabled. -- ViennaRNA 2.7.0: static RNAlib, without OpenMP, pthreads, SIMD, external +- ViennaRNA 2.7.2 locally, newest stable 2.x in CI: static RNAlib, without OpenMP, pthreads, SIMD, external solvers, bindings, documentation or companion applications. Compiling only RNAlib avoids executing cross-compiled helper applications on the host. - IntaRNA: C++23, bundled Kokkos mdspan, standard CLI, default energy models