diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..24f5e42 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# The build recipes run in bash on Windows too, so they keep LF line ends there. +# The patches go to git apply byte for byte: some of the files they change use CRLF. +recipes/**/*.sh text eol=lf +recipes/**/*.py text eol=lf +recipes/versions.env text eol=lf +recipes/**/*.patch -text diff --git a/.github/workflows/rebuild-engines.yml b/.github/workflows/rebuild-engines.yml new file mode 100644 index 0000000..7650094 --- /dev/null +++ b/.github/workflows/rebuild-engines.yml @@ -0,0 +1,303 @@ +name: Rebuild engines + +# Rebuilds the engines whose upstream binaries bundle an outdated zlib or expat +# from the sources pinned in recipes/versions.env (see recipes/README.md), with +# the build procedure of their upstream release builds, checks the zlib and expat +# versions they carry, and compares their results with those of the binaries +# they replace. The binaries are uploaded in this repository's folder layout. + +on: + push: + branches: [master] # other branches build in their pull requests + paths: + - 'recipes/**' + - '.github/workflows/rebuild-engines.yml' + pull_request: + paths: + - 'recipes/**' + - '.github/workflows/rebuild-engines.yml' + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + comet: + name: Comet ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + # the runners and compilers of Comet's own release builds + - platform: Linux/x86_64 + runner: ubuntu-24.04 + container: ubuntu:22.04 + - platform: Linux/aarch64 + runner: ubuntu-24.04-arm + container: ubuntu:22.04 + - platform: MacOS/arm64 + runner: macos-14 + macos_min: '14.0' + - platform: MacOS/x86_64 + runner: macos-15-intel + macos_min: '13.0' + - platform: Windows/x86_64 + runner: windows-2022 + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container }} + timeout-minutes: 60 + steps: + - name: Install the build tools + if: runner.os == 'Linux' + run: | + apt-get update + apt-get install -y --no-install-recommends ca-certificates curl git g++ make xz-utils python3 binutils + + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up the work folder + run: | + { + if [ "$RUNNER_OS" = Windows ]; then + echo "WORK=D:/w" + echo "PYTHON=python" + else + echo "WORK=$RUNNER_TEMP/w" + fi + grep -E '^[A-Z0-9_]+=' recipes/versions.env + } >> "$GITHUB_ENV" + # for the git commands of the later steps (in a container, the checkout has another owner) + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch the sources + run: recipes/fetch-sources.sh "$WORK" zlib expat comet openms-testdata + + - name: Swap MSToolkit's zlib and expat + run: recipes/comet/prepare.sh "$WORK/comet" "$WORK" + + - name: Build (make) + if: runner.os != 'Windows' + working-directory: ${{ env.WORK }}/comet + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_min }} + # GITHUB_SHA: Comet shows the first seven characters of the commit it was built from + run: | + make GITHUB_SHA="$COMET_COMMIT" + echo "BINARY=$PWD/comet.exe" >> "$GITHUB_ENV" + echo "LIBRARIES=$PWD/MSToolkit/libmstoolkitlite.a" >> "$GITHUB_ENV" + + # MSToolkit's RAW reader #imports the type library of Thermo's MSFileReader, + # whose DLLs need the Visual C++ 2010 runtime: the windows-2019 image of + # Comet's release builds had it, windows-2022 has not. + - name: Install the Visual C++ 2010 runtime (Windows) + if: runner.os == 'Windows' + run: choco install vcredist2010 --no-progress -y + + - name: Unpack Thermo's MSFileReader (Windows) + if: runner.os == 'Windows' + working-directory: ${{ env.WORK }}/comet + run: unzip -q -P iacceptthelicense extern/msfreader.zip -d dlls + + - name: Register Thermo's MSFileReader (Windows) + if: runner.os == 'Windows' + working-directory: ${{ env.WORK }}/comet + shell: cmd + run: regsvr32 /S dlls\XRawfile2_x64.dll + + - uses: microsoft/setup-msbuild@v2 + if: runner.os == 'Windows' + + - name: Build (MSBuild) + if: runner.os == 'Windows' + working-directory: ${{ env.WORK }}/comet + shell: cmd # Git Bash does not find msbuild + run: msbuild Comet.sln -t:Comet -p:Configuration=Release -p:Platform=x64 -p:PlatformToolset=v142 -tv:Current + + - name: Locate the binary (Windows) + if: runner.os == 'Windows' + working-directory: ${{ env.WORK }}/comet + run: | + echo "BINARY=$PWD/x64/Release/comet.exe" >> "$GITHUB_ENV" + echo "LIBRARIES=$(find "$PWD" -name MSToolkitLite.lib | tr '\n' ' ')" >> "$GITHUB_ENV" + + - name: Check the zlib and expat versions + run: | + # shellcheck disable=SC2086 # LIBRARIES lists several files + recipes/check-bundled-libs.sh "$BINARY" $LIBRARIES + + - name: Compare with the binary it replaces + run: | + "$BINARY" -p > /dev/null && rm comet.params.new + git fetch -q --depth 1 origin "$BASELINE_COMMIT" + mkdir -p "$WORK/baseline" + git cat-file blob "FETCH_HEAD:${{ matrix.platform }}/Comet/comet.exe" > "$WORK/baseline/comet.exe" + chmod +x "$WORK/baseline/comet.exe" + recipes/compare/comet.sh "$WORK/baseline/comet.exe" "$BINARY" "$WORK/compare" "$WORK/openms" + + - name: Stage the binary + run: | + mkdir -p "$WORK/out/${{ matrix.platform }}/Comet" + cp "$BINARY" "$WORK/out/${{ matrix.platform }}/Comet/comet.exe" + + - uses: actions/upload-artifact@v7 + with: + name: comet-${{ runner.os }}-${{ runner.arch }} + path: ${{ env.WORK }}/out/ + if-no-files-found: error + + maracluster: + name: MaRaCluster ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + # the runners of MaRaCluster's rel-1-04-1 release build + - platform: Linux/x86_64 + runner: ubuntu-24.04 + container: ubuntu:22.04 + - platform: MacOS/arm64 + runner: macos-14 + - platform: Windows/x86_64 + runner: windows-2022 + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container }} + timeout-minutes: 300 + steps: + - name: Install the build tools + if: runner.os == 'Linux' + run: | + apt-get update + apt-get install -y --no-install-recommends ca-certificates curl git python3 binutils sudo wget rsync g++ make cmake lbzip2 + + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set up the work folder + run: | + { + if [ "$RUNNER_OS" = Windows ]; then + echo "WORK=D:/w" # short paths for ProteoWizard's build + echo 'WORK_WIN=D:\w' + echo "PYTHON=python" + echo "PWIZ_SOURCE=pwiz-with-vendor-apis" + else + echo "WORK=$RUNNER_TEMP/w" + echo "PWIZ_SOURCE=pwiz" + fi + grep -E '^[A-Z0-9_]+=' recipes/versions.env + } >> "$GITHUB_ENV" + # for the git commands of the later steps (in a container, the checkout has another owner) + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch the sources + run: recipes/fetch-sources.sh "$WORK" zlib maracluster "$PWIZ_SOURCE" openms-testdata + + - name: Put the pinned ProteoWizard and zlib in place + run: recipes/maracluster/prepare.sh "$WORK/maracluster" "$WORK/pwiz" "$WORK" "$WORK/build" + + # MaRaCluster's own builders, as its release build ran them (without the GUI). + # JAVA_HOME has to be empty, or ProteoWizard adds its Java bindings, which + # break its "libraries" target (MaRaCluster's CI clears it on macOS too). + - name: Build (admin/builders/tarball64_build.sh) + if: runner.os == 'Linux' + env: + JAVA_HOME: '' + run: | + "$WORK/maracluster/admin/builders/tarball64_build.sh" -s "$WORK" -r "$WORK/release" -b "$WORK/build" -g + echo "BINARY=$WORK/build/maracluster/src/maracluster" >> "$GITHUB_ENV" + echo "LIBRARIES=$WORK/build/tools/lib/libzlib.a" >> "$GITHUB_ENV" + + - name: Build (admin/builders/osx64_build.sh) + if: runner.os == 'macOS' + env: + JAVA_HOME: '' + run: | + "$WORK/maracluster/admin/builders/osx64_build.sh" -s "$WORK" -r "$WORK/release" -b "$WORK/build" -g + echo "BINARY=$WORK/build/maracluster/src/maracluster" >> "$GITHUB_ENV" + echo "LIBRARIES=$WORK/build/tools/lib/libzlib.a" >> "$GITHUB_ENV" + + - name: Build (admin/builders/nativew64_build.bat) + if: runner.os == 'Windows' + shell: cmd + run: | + CALL %WORK_WIN%\maracluster\admin\builders\nativew64_build.bat -s %WORK_WIN% -r %WORK_WIN%\release -b %WORK_WIN%\build -g + IF NOT EXIST %WORK_WIN%\build\maracluster\src\Release\maracluster.exe EXIT /B 1 + ECHO BINARY=%WORK%/build/maracluster/src/Release/maracluster.exe>> %GITHUB_ENV% + ECHO LIBRARIES=%WORK%/build/tools/proteowizard/lib/zlib.lib>> %GITHUB_ENV% + + - name: Show ProteoWizard's build log + if: failure() + run: | + for log in "$WORK"/build/tools/pwiz_installation.log "$WORK"/build/tools/proteowizard/pwiz_installation.log; do + if [ -f "$log" ]; then echo "== $log"; tail -n 200 "$log"; fi + done + + - name: Check the zlib and expat versions + run: | + # shellcheck disable=SC2086 # LIBRARIES lists several files + recipes/check-bundled-libs.sh "$BINARY" $LIBRARIES + + - name: Compare with the binary it replaces + env: + REPEAT: 25 + run: | + name=maracluster + [ "$RUNNER_OS" = Windows ] && name=maracluster.exe + git fetch -q --depth 1 origin "$BASELINE_COMMIT" + mkdir -p "$WORK/baseline" + git cat-file blob "FETCH_HEAD:${{ matrix.platform }}/MaRaCluster/$name" > "$WORK/baseline/$name" + chmod +x "$WORK/baseline/$name" + recipes/compare/maracluster.sh "$WORK/baseline/$name" "$BINARY" "$WORK/compare" "$WORK/openms" + + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: maracluster-diagnostics-${{ runner.os }}-${{ runner.arch }} + path: | + ${{ env.WORK }}/compare/out/**/*.log + ${{ env.BINARY }} + if-no-files-found: ignore + + - name: Stage the binary + run: | + mkdir -p "$WORK/out/${{ matrix.platform }}/MaRaCluster" + cp "$BINARY" "$WORK/out/${{ matrix.platform }}/MaRaCluster/" + + - uses: actions/upload-artifact@v7 + with: + name: maracluster-${{ runner.os }}-${{ runner.arch }} + path: ${{ env.WORK }}/out/ + if-no-files-found: error + + engines: + name: All engines + needs: [comet, maracluster] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: actions/download-artifact@v8 + with: + path: engines + merge-multiple: true + + - name: Check the zlib and expat versions + run: | + find engines -type f | sort + find engines -type f -print0 | sort -z | xargs -0 recipes/check-bundled-libs.sh + (cd engines && find . -type f -print0 | sort -z | xargs -0 sha256sum) + + - uses: actions/upload-artifact@v7 + with: + name: engines + path: engines/ + if-no-files-found: error diff --git a/Licenses.txt b/Licenses.txt index 5d221ba..c912eb2 100644 --- a/Licenses.txt +++ b/Licenses.txt @@ -66,12 +66,6 @@ SIRIUS: - License: https://www.gnu.org/licenses/agpl-3.0 -X!Tandem: - - URL and source code: - http://www.thegpm.org/tandem/ - - License: - Artistic License (Perl), https://spdx.org/licenses/Artistic-1.0-Perl.html - Sage: - URL and source code: https://github.com/lazear/sage diff --git a/Linux/aarch64/Comet/LICENSE-expat.txt b/Linux/aarch64/Comet/LICENSE-expat.txt new file mode 100644 index 0000000..c6d184a --- /dev/null +++ b/Linux/aarch64/Comet/LICENSE-expat.txt @@ -0,0 +1,21 @@ +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2025 Expat maintainers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Linux/aarch64/Comet/README.md b/Linux/aarch64/Comet/README.md index 41be329..a71ec4b 100644 --- a/Linux/aarch64/Comet/README.md +++ b/Linux/aarch64/Comet/README.md @@ -1,23 +1,11 @@ - - -# Comet MS/MS - -Comet is an open source tandem mass spectrometry (MS/MS) sequence database search tool written primarily in C/C++. The original Comet repository lived on [SourceForge](https://sourceforge.net/projects/comet-ms/) since 2012. It was migrated to GitHub on September 2021. - -The project website [can be found here](https://uwpr.github.io/Comet/). This includes release notes and search parameters documentation. - -To compile on linux and macOS: - -- Type 'make'. This will generate a binary "comet.exe". - -To compile with Microsoft Visual Studio: - -- Use build tools v142 with Microsoft Visual Studio 2019. - -- First install [MSFileReader from Thermo Fischer Scientific](https://uwpr.github.io/Comet/notes/20220228_rawfile.html). - -- Load "Comet.sln" in Visual Studio. - -- Set the build to "Release" and "x64". - -- Right-mouse-click on the "Comet" project and choose "Build". This should generate a binary "Comet.exe" in x64/Release. +# Comet 2025.01 rev. 1 + +`comet.exe` is Comet 2025.01 rev. 1 (UWPR/Comet v2025.01.1, commit 4181df6), +built by OpenMS from Comet's source with zlib 1.3.2 and expat 2.8.5 in place of +the zlib 1.2.11 and expat 2.2.9 that Comet's MSToolkit bundles, which have known +vulnerabilities. Otherwise it is built as Comet's own release binaries are; the +recipe is in https://github.com/OpenMS/THIRDPARTY/tree/master/recipes. + +- Comet: https://github.com/UWPR/Comet, Apache License 2.0 (LICENSE) +- expat: https://libexpat.github.io, MIT License (LICENSE-expat.txt) +- zlib: https://zlib.net, zlib License diff --git a/Linux/aarch64/Comet/comet.exe b/Linux/aarch64/Comet/comet.exe index 76be6ce..ffeb4e3 100755 Binary files a/Linux/aarch64/Comet/comet.exe and b/Linux/aarch64/Comet/comet.exe differ diff --git a/Linux/x86_64/Comet/LICENSE-expat.txt b/Linux/x86_64/Comet/LICENSE-expat.txt new file mode 100644 index 0000000..c6d184a --- /dev/null +++ b/Linux/x86_64/Comet/LICENSE-expat.txt @@ -0,0 +1,21 @@ +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2025 Expat maintainers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Linux/x86_64/Comet/README.md b/Linux/x86_64/Comet/README.md index 400ed06..a71ec4b 100644 --- a/Linux/x86_64/Comet/README.md +++ b/Linux/x86_64/Comet/README.md @@ -1,9 +1,11 @@ -Recompiled from source on Linux 5.4.0-65-x86_64 with: +# Comet 2025.01 rev. 1 -- ldd (Ubuntu GLIBC 2.31-0ubuntu9.7) 2.31 +`comet.exe` is Comet 2025.01 rev. 1 (UWPR/Comet v2025.01.1, commit 4181df6), +built by OpenMS from Comet's source with zlib 1.3.2 and expat 2.8.5 in place of +the zlib 1.2.11 and expat 2.2.9 that Comet's MSToolkit bundles, which have known +vulnerabilities. Otherwise it is built as Comet's own release binaries are; the +recipe is in https://github.com/OpenMS/THIRDPARTY/tree/master/recipes. -- g++ (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0 - -statically. - -Tested on Ubuntu 20.04.2 LTS +- Comet: https://github.com/UWPR/Comet, Apache License 2.0 (LICENSE) +- expat: https://libexpat.github.io, MIT License (LICENSE-expat.txt) +- zlib: https://zlib.net, zlib License diff --git a/Linux/x86_64/Comet/comet.exe b/Linux/x86_64/Comet/comet.exe index 9f86dc1..e331b13 100755 Binary files a/Linux/x86_64/Comet/comet.exe and b/Linux/x86_64/Comet/comet.exe differ diff --git a/Linux/x86_64/MaRaCluster/README.md b/Linux/x86_64/MaRaCluster/README.md new file mode 100644 index 0000000..2b4f7c9 --- /dev/null +++ b/Linux/x86_64/MaRaCluster/README.md @@ -0,0 +1,13 @@ +# MaRaCluster 1.04.1 + +`maracluster` is MaRaCluster 1.04.1 (statisticalbiotechnology/maracluster +rel-1-04-1, commit c122929), built by OpenMS from MaRaCluster's source with zlib +1.3.2 in place of the zlib 1.2.3 that ProteoWizard bundles, which has known +vulnerabilities. Otherwise it is built as MaRaCluster's own release binaries +are, with its builder scripts and the ProteoWizard revision its release build +used (ProteoWizard/pwiz 9ba7317); the recipe is in +https://github.com/OpenMS/THIRDPARTY/tree/master/recipes. + +- MaRaCluster: https://github.com/statisticalbiotechnology/maracluster, Apache License 2.0 (LICENSE, NOTICE) +- ProteoWizard: https://github.com/ProteoWizard/pwiz, Apache License 2.0 +- zlib: https://zlib.net, zlib License diff --git a/Linux/x86_64/MaRaCluster/maracluster b/Linux/x86_64/MaRaCluster/maracluster index bb84ba1..c6574f6 100755 Binary files a/Linux/x86_64/MaRaCluster/maracluster and b/Linux/x86_64/MaRaCluster/maracluster differ diff --git a/Linux/x86_64/XTandem/LICENSE b/Linux/x86_64/XTandem/LICENSE deleted file mode 100644 index 33a9314..0000000 --- a/Linux/x86_64/XTandem/LICENSE +++ /dev/null @@ -1,121 +0,0 @@ -The Artistic License for all X! software, binaries and documentation - -Preamble -The intent of this document is to state the conditions under which a -Package may be copied, such that the Copyright Holder maintains some -semblance of artistic control over the development of the package, -while giving the users of the package the right to use and distribute -the Package in a more-or-less customary fashion, plus the right to -make reasonable modifications. - -Definitions -"Package" refers to the collection of files distributed by the Copyright - Holder, and derivatives of that collection of files created through - textual modification. - -"Standard Version" refers to such a Package if it has not been modified, - or has been modified in accordance with the wishes of the Copyright - Holder as specified below. - -"Copyright Holder" is whoever is named in the copyright or copyrights - for the package. - -"You" is you, if you're thinking about copying or distributing this Package. - -"Reasonable copying fee" is whatever you can justify on the basis of - media cost, duplication charges, time of people involved, and so on. - (You will not be required to justify it to the Copyright Holder, but - only to the computing community at large as a market that must bear - the fee.) - -"Freely Available" means that no fee is charged for the item itself, - though there may be fees involved in handling the item. It also means - that recipients of the item may redistribute it under the same - conditions they received it. - -1. You may make and give away verbatim copies of the source form of the -Standard Version of this Package without restriction, provided that -you duplicate all of the original copyright notices and associated -disclaimers. - -2. You may apply bug fixes, portability fixes and other modifications -derived from the Public Domain or from the Copyright Holder. A -Package modified in such a way shall still be considered the Standard -Version. - -3. You may otherwise modify your copy of this Package in any way, provided -that you insert a prominent notice in each changed file stating how and -when you changed that file, and provided that you do at least ONE of the -following: - - a. place your modifications in the Public Domain or otherwise make them - Freely Available, such as by posting said modifications to Usenet - or an equivalent medium, or placing the modifications on a major - archive site such as uunet.uu.net, or by allowing the Copyright Holder - to include your modifications in the Standard Version of the Package. - b. use the modified Package only within your corporation or organization. - c. rename any non-standard executables so the names do not conflict - with standard executables, which must also be provided, and provide - a separate manual page for each non-standard executable that clearly - documents how it differs from the Standard Version. - d. make other distribution arrangements with the Copyright Holder. - -4. You may distribute the programs of this Package in object code or -executable form, provided that you do at least ONE of the following: - - a. distribute a Standard Version of the executables and library files, - together with instructions (in the manual page or equivalent) on - where to get the Standard Version. - b. accompany the distribution with the machine-readable source of the - Package with your modifications. - c. give non-standard executables non-standard names, and clearly - document the differences in manual pages (or equivalent), together - with instructions on where to get the Standard Version. - d. make other distribution arrangements with the Copyright Holder. - -5. You may charge a reasonable copying fee for any distribution of -this Package. You may charge any fee you choose for support of -this Package. You may not charge a fee for this Package itself. -However, you may distribute this Package in aggregate with other -(possibly commercial) programs as part of a larger (possibly -commercial) software distribution provided that you do not a -dvertise this Package as a product of your own. You may embed this -Package's interpreter within an executable of yours (by linking); -this shall be construed as a mere form of aggregation, provided that -the complete Standard Version of the interpreter is so embedded. - -6. The scripts and library files supplied as input to or produced as -output from the programs of this Package do not automatically fall -under the copyright of this Package, but belong to whomever generated -them, and may be sold commercially, and may be aggregated with this -Package. If such scripts or library files are aggregated with this -Package via the so-called "undump" or "unexec" methods of producing -a binary executable image, then distribution of such an image shall -neither be construed as a distribution of this Package nor shall it -fall under the restrictions of Paragraphs 3 and 4, provided that you -do not represent such an executable image as a Standard Version of -this Package. - -7. C subroutines (or comparably compiled subroutines in other languages) -supplied by you and linked into this Package in order to emulate -subroutines and variables of the language defined by this Package -shall not be considered part of this Package, but are the equivalent -of input as in Paragraph 6, provided these subroutines do not change -the language in any way that would cause it to fail the regression -tests for the language. - -8. Aggregation of this Package with a commercial distribution is always -permitted provided that the use of this Package is embedded; that is, -when no overt attempt is made to make this Package's interfaces visible -to the end user of the commercial distribution. Such use shall not be -construed as a distribution of this Package. - -9. The name of the Copyright Holder may not be used to endorse or promote -products derived from this software without specific prior written permission. - -10. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. - -The End - diff --git a/Linux/x86_64/XTandem/tandem.exe b/Linux/x86_64/XTandem/tandem.exe deleted file mode 100755 index 8a2e4bf..0000000 Binary files a/Linux/x86_64/XTandem/tandem.exe and /dev/null differ diff --git a/MacOS/arm64/Comet/LICENSE-expat.txt b/MacOS/arm64/Comet/LICENSE-expat.txt new file mode 100644 index 0000000..c6d184a --- /dev/null +++ b/MacOS/arm64/Comet/LICENSE-expat.txt @@ -0,0 +1,21 @@ +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2025 Expat maintainers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/MacOS/arm64/Comet/README.md b/MacOS/arm64/Comet/README.md index 41be329..a71ec4b 100644 --- a/MacOS/arm64/Comet/README.md +++ b/MacOS/arm64/Comet/README.md @@ -1,23 +1,11 @@ - - -# Comet MS/MS - -Comet is an open source tandem mass spectrometry (MS/MS) sequence database search tool written primarily in C/C++. The original Comet repository lived on [SourceForge](https://sourceforge.net/projects/comet-ms/) since 2012. It was migrated to GitHub on September 2021. - -The project website [can be found here](https://uwpr.github.io/Comet/). This includes release notes and search parameters documentation. - -To compile on linux and macOS: - -- Type 'make'. This will generate a binary "comet.exe". - -To compile with Microsoft Visual Studio: - -- Use build tools v142 with Microsoft Visual Studio 2019. - -- First install [MSFileReader from Thermo Fischer Scientific](https://uwpr.github.io/Comet/notes/20220228_rawfile.html). - -- Load "Comet.sln" in Visual Studio. - -- Set the build to "Release" and "x64". - -- Right-mouse-click on the "Comet" project and choose "Build". This should generate a binary "Comet.exe" in x64/Release. +# Comet 2025.01 rev. 1 + +`comet.exe` is Comet 2025.01 rev. 1 (UWPR/Comet v2025.01.1, commit 4181df6), +built by OpenMS from Comet's source with zlib 1.3.2 and expat 2.8.5 in place of +the zlib 1.2.11 and expat 2.2.9 that Comet's MSToolkit bundles, which have known +vulnerabilities. Otherwise it is built as Comet's own release binaries are; the +recipe is in https://github.com/OpenMS/THIRDPARTY/tree/master/recipes. + +- Comet: https://github.com/UWPR/Comet, Apache License 2.0 (LICENSE) +- expat: https://libexpat.github.io, MIT License (LICENSE-expat.txt) +- zlib: https://zlib.net, zlib License diff --git a/MacOS/arm64/Comet/comet.exe b/MacOS/arm64/Comet/comet.exe index 3ed29f0..632083c 100755 Binary files a/MacOS/arm64/Comet/comet.exe and b/MacOS/arm64/Comet/comet.exe differ diff --git a/MacOS/arm64/MaRaCluster/README.md b/MacOS/arm64/MaRaCluster/README.md new file mode 100644 index 0000000..2b4f7c9 --- /dev/null +++ b/MacOS/arm64/MaRaCluster/README.md @@ -0,0 +1,13 @@ +# MaRaCluster 1.04.1 + +`maracluster` is MaRaCluster 1.04.1 (statisticalbiotechnology/maracluster +rel-1-04-1, commit c122929), built by OpenMS from MaRaCluster's source with zlib +1.3.2 in place of the zlib 1.2.3 that ProteoWizard bundles, which has known +vulnerabilities. Otherwise it is built as MaRaCluster's own release binaries +are, with its builder scripts and the ProteoWizard revision its release build +used (ProteoWizard/pwiz 9ba7317); the recipe is in +https://github.com/OpenMS/THIRDPARTY/tree/master/recipes. + +- MaRaCluster: https://github.com/statisticalbiotechnology/maracluster, Apache License 2.0 (LICENSE, NOTICE) +- ProteoWizard: https://github.com/ProteoWizard/pwiz, Apache License 2.0 +- zlib: https://zlib.net, zlib License diff --git a/MacOS/arm64/MaRaCluster/maracluster b/MacOS/arm64/MaRaCluster/maracluster index 056bfcb..ea7b714 100755 Binary files a/MacOS/arm64/MaRaCluster/maracluster and b/MacOS/arm64/MaRaCluster/maracluster differ diff --git a/MacOS/x86_64/Comet/LICENSE-expat.txt b/MacOS/x86_64/Comet/LICENSE-expat.txt new file mode 100644 index 0000000..c6d184a --- /dev/null +++ b/MacOS/x86_64/Comet/LICENSE-expat.txt @@ -0,0 +1,21 @@ +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2025 Expat maintainers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/MacOS/x86_64/Comet/README.md b/MacOS/x86_64/Comet/README.md index f9232e0..a71ec4b 100644 --- a/MacOS/x86_64/Comet/README.md +++ b/MacOS/x86_64/Comet/README.md @@ -1,22 +1,11 @@ -2023/03/31 +# Comet 2025.01 rev. 1 -Comet version "2023.01 rev. 2". -https://uwpr.github.io/Comet/releases/release_202301.html +`comet.exe` is Comet 2025.01 rev. 1 (UWPR/Comet v2025.01.1, commit 4181df6), +built by OpenMS from Comet's source with zlib 1.3.2 and expat 2.8.5 in place of +the zlib 1.2.11 and expat 2.2.9 that Comet's MSToolkit bundles, which have known +vulnerabilities. Otherwise it is built as Comet's own release binaries are; the +recipe is in https://github.com/OpenMS/THIRDPARTY/tree/master/recipes. -release 2023.01 rev. 2 (2023.01.2), release date 2023/03/30 - -mzML/mzXML files without the optional scan index would not be searched because their spectra could not be read. Support for this functionality was implemented in v2019.01.4 thru v2022.01.1 but was lost as of v2022.01.2 when MSToolkit code was updated from that library’s repository. This functionality is re-implemented in this release. Thanks to J. Wang for reporting the issue. -A parameter “export_additional_pepxml_scores” has been implemented. This is an optional/hidden parameter in that it is not written by default in the comet.params files; it needs to be added manually. When this parameter is present and it’s value set to “1”, additional search scores (lnrSp, deltLCn, lnExpect, and IonFrac) are reported in the pep.xml output. Feature request by J. Scheid, OpenMS group. -release 2023.01 rev. 1 (2023.01.1), release date 2023/03/15 - -For pep.xml output when “num_output_lines = 1”, the deltaCn scores will always be reported as “1.0”. This occurs when only the top hit is reported and has been this behavior since the original release of Comet. This update will correctly report the deltaCn value for this case. Thanks to J. Scheid for reporting this issue. -Comet and MSToolkit had input file name/path limits of 512 and 256 characters, respectively. Any input file strings longer than 256 would cause an error in reading the input spectra data. The file name buffer has been expanded to 4096 to mitigate this issue. Thanks to B. Connolly for reporting this issue. -The Linux/Ubuntu and Mac runners for compiling the release binaries via GitHub Actions have been changed from “ubuntu-latest” and “macos-latest” to “ubuntu-20.04” and “macos-11”. This is to give these binaries wider, backwards compatibility with older OS’s. Thanks to T. Sachsenberg for reporting this issue. -release 2023.01 rev. 0 (2023.01.0), release date 2023/01/31 - -Address issue where a peptide with a low xcorr identified in a very poor/sparse spectrum would be assigned a good E-value. This is due to the majority of matched xcorr scores being “0” so any poor scoring match looks like a good outlier. Thanks to D. Shteynberg for reporting the issue. -Add “scale_fragmentNL” parameter entry which scales (multiplies) the neutral loss mass value by the number of modified residues in the fragment. Feature requested by A. Keller. -Add contributions of fragment neutral loss peaks in preliminary (Sp) score; previously they only applied to the cross-correlation score. -Fix bug where the fragment neutral loss peak was not analyzed if the primary fragment peak was not matched. -Fix bug where Comet files to analyze a variable modification if more than 19 residues are specified for that mod. Thanks to D. Tabb for reporting the issue. -Fix minor typo in command line help. Thanks to M. Riffle for the pull request. +- Comet: https://github.com/UWPR/Comet, Apache License 2.0 (LICENSE) +- expat: https://libexpat.github.io, MIT License (LICENSE-expat.txt) +- zlib: https://zlib.net, zlib License diff --git a/MacOS/x86_64/Comet/comet.exe b/MacOS/x86_64/Comet/comet.exe index 441336d..2be90e0 100755 Binary files a/MacOS/x86_64/Comet/comet.exe and b/MacOS/x86_64/Comet/comet.exe differ diff --git a/MacOS/x86_64/XTandem/LICENSE b/MacOS/x86_64/XTandem/LICENSE deleted file mode 100644 index 33a9314..0000000 --- a/MacOS/x86_64/XTandem/LICENSE +++ /dev/null @@ -1,121 +0,0 @@ -The Artistic License for all X! software, binaries and documentation - -Preamble -The intent of this document is to state the conditions under which a -Package may be copied, such that the Copyright Holder maintains some -semblance of artistic control over the development of the package, -while giving the users of the package the right to use and distribute -the Package in a more-or-less customary fashion, plus the right to -make reasonable modifications. - -Definitions -"Package" refers to the collection of files distributed by the Copyright - Holder, and derivatives of that collection of files created through - textual modification. - -"Standard Version" refers to such a Package if it has not been modified, - or has been modified in accordance with the wishes of the Copyright - Holder as specified below. - -"Copyright Holder" is whoever is named in the copyright or copyrights - for the package. - -"You" is you, if you're thinking about copying or distributing this Package. - -"Reasonable copying fee" is whatever you can justify on the basis of - media cost, duplication charges, time of people involved, and so on. - (You will not be required to justify it to the Copyright Holder, but - only to the computing community at large as a market that must bear - the fee.) - -"Freely Available" means that no fee is charged for the item itself, - though there may be fees involved in handling the item. It also means - that recipients of the item may redistribute it under the same - conditions they received it. - -1. You may make and give away verbatim copies of the source form of the -Standard Version of this Package without restriction, provided that -you duplicate all of the original copyright notices and associated -disclaimers. - -2. You may apply bug fixes, portability fixes and other modifications -derived from the Public Domain or from the Copyright Holder. A -Package modified in such a way shall still be considered the Standard -Version. - -3. You may otherwise modify your copy of this Package in any way, provided -that you insert a prominent notice in each changed file stating how and -when you changed that file, and provided that you do at least ONE of the -following: - - a. place your modifications in the Public Domain or otherwise make them - Freely Available, such as by posting said modifications to Usenet - or an equivalent medium, or placing the modifications on a major - archive site such as uunet.uu.net, or by allowing the Copyright Holder - to include your modifications in the Standard Version of the Package. - b. use the modified Package only within your corporation or organization. - c. rename any non-standard executables so the names do not conflict - with standard executables, which must also be provided, and provide - a separate manual page for each non-standard executable that clearly - documents how it differs from the Standard Version. - d. make other distribution arrangements with the Copyright Holder. - -4. You may distribute the programs of this Package in object code or -executable form, provided that you do at least ONE of the following: - - a. distribute a Standard Version of the executables and library files, - together with instructions (in the manual page or equivalent) on - where to get the Standard Version. - b. accompany the distribution with the machine-readable source of the - Package with your modifications. - c. give non-standard executables non-standard names, and clearly - document the differences in manual pages (or equivalent), together - with instructions on where to get the Standard Version. - d. make other distribution arrangements with the Copyright Holder. - -5. You may charge a reasonable copying fee for any distribution of -this Package. You may charge any fee you choose for support of -this Package. You may not charge a fee for this Package itself. -However, you may distribute this Package in aggregate with other -(possibly commercial) programs as part of a larger (possibly -commercial) software distribution provided that you do not a -dvertise this Package as a product of your own. You may embed this -Package's interpreter within an executable of yours (by linking); -this shall be construed as a mere form of aggregation, provided that -the complete Standard Version of the interpreter is so embedded. - -6. The scripts and library files supplied as input to or produced as -output from the programs of this Package do not automatically fall -under the copyright of this Package, but belong to whomever generated -them, and may be sold commercially, and may be aggregated with this -Package. If such scripts or library files are aggregated with this -Package via the so-called "undump" or "unexec" methods of producing -a binary executable image, then distribution of such an image shall -neither be construed as a distribution of this Package nor shall it -fall under the restrictions of Paragraphs 3 and 4, provided that you -do not represent such an executable image as a Standard Version of -this Package. - -7. C subroutines (or comparably compiled subroutines in other languages) -supplied by you and linked into this Package in order to emulate -subroutines and variables of the language defined by this Package -shall not be considered part of this Package, but are the equivalent -of input as in Paragraph 6, provided these subroutines do not change -the language in any way that would cause it to fail the regression -tests for the language. - -8. Aggregation of this Package with a commercial distribution is always -permitted provided that the use of this Package is embedded; that is, -when no overt attempt is made to make this Package's interfaces visible -to the end user of the commercial distribution. Such use shall not be -construed as a distribution of this Package. - -9. The name of the Copyright Holder may not be used to endorse or promote -products derived from this software without specific prior written permission. - -10. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. - -The End - diff --git a/MacOS/x86_64/XTandem/tandem b/MacOS/x86_64/XTandem/tandem deleted file mode 100755 index 0b1887c..0000000 Binary files a/MacOS/x86_64/XTandem/tandem and /dev/null differ diff --git a/VERSION b/VERSION index 386109e..09db63e 100644 --- a/VERSION +++ b/VERSION @@ -1,10 +1,10 @@ -Comet 2025.01 rev. 1 (Linux x86_64 and aarch64, MacOS arm64 and x86_64, Windows x86_64) +Comet 2025.01 rev. 1 (Linux x86_64 and aarch64, MacOS arm64 and x86_64, Windows x86_64; built from source with zlib 1.3.2 and expat 2.8.5, see recipes/README.md) https://github.com/UWPR/Comet/releases/tag/v2025.01.1 LuciPHOr2 2.1-SNAPSHOT (all platforms, Java) http://luciphor2.sourceforge.net/ -MaRaCluster 1.04.1 (Linux x86_64, MacOS arm64, Windows x86_64) +MaRaCluster 1.04.1 (Linux x86_64, MacOS arm64, Windows x86_64; built from source with zlib 1.3.2, see recipes/README.md) https://github.com/statisticalbiotechnology/maracluster/releases/tag/rel-1-04-1 MS-GF+ v2024.03.26 (all platforms, Java) @@ -21,6 +21,3 @@ https://github.com/lazear/sage/releases/tag/v0.14.6 ThermoRawFileParser 1.4.4 (all platforms, .NET) https://github.com/compomics/ThermoRawFileParser/releases/tag/v1.4.4 - -X! Tandem Vengeance 2015.12.15.2 (Linux x86_64, MacOS x86_64, Windows x86_64) -http://www.thegpm.org/tandem/ diff --git a/Windows/x86_64/Comet/LICENSE-expat.txt b/Windows/x86_64/Comet/LICENSE-expat.txt new file mode 100644 index 0000000..c6d184a --- /dev/null +++ b/Windows/x86_64/Comet/LICENSE-expat.txt @@ -0,0 +1,21 @@ +Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper +Copyright (c) 2001-2025 Expat maintainers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Windows/x86_64/Comet/README.md b/Windows/x86_64/Comet/README.md new file mode 100644 index 0000000..a71ec4b --- /dev/null +++ b/Windows/x86_64/Comet/README.md @@ -0,0 +1,11 @@ +# Comet 2025.01 rev. 1 + +`comet.exe` is Comet 2025.01 rev. 1 (UWPR/Comet v2025.01.1, commit 4181df6), +built by OpenMS from Comet's source with zlib 1.3.2 and expat 2.8.5 in place of +the zlib 1.2.11 and expat 2.2.9 that Comet's MSToolkit bundles, which have known +vulnerabilities. Otherwise it is built as Comet's own release binaries are; the +recipe is in https://github.com/OpenMS/THIRDPARTY/tree/master/recipes. + +- Comet: https://github.com/UWPR/Comet, Apache License 2.0 (LICENSE) +- expat: https://libexpat.github.io, MIT License (LICENSE-expat.txt) +- zlib: https://zlib.net, zlib License diff --git a/Windows/x86_64/Comet/comet.exe b/Windows/x86_64/Comet/comet.exe index 82c892d..64d8106 100755 Binary files a/Windows/x86_64/Comet/comet.exe and b/Windows/x86_64/Comet/comet.exe differ diff --git a/Windows/x86_64/MaRaCluster/README.md b/Windows/x86_64/MaRaCluster/README.md new file mode 100644 index 0000000..18e2533 --- /dev/null +++ b/Windows/x86_64/MaRaCluster/README.md @@ -0,0 +1,13 @@ +# MaRaCluster 1.04.1 + +`maracluster.exe` is MaRaCluster 1.04.1 (statisticalbiotechnology/maracluster +rel-1-04-1, commit c122929), built by OpenMS from MaRaCluster's source with zlib +1.3.2 in place of the zlib 1.2.3 that ProteoWizard bundles, which has known +vulnerabilities. Otherwise it is built as MaRaCluster's own release binaries +are, with its builder scripts and the ProteoWizard revision its release build +used (ProteoWizard/pwiz 9ba7317); the recipe is in +https://github.com/OpenMS/THIRDPARTY/tree/master/recipes. + +- MaRaCluster: https://github.com/statisticalbiotechnology/maracluster, Apache License 2.0 (LICENSE, NOTICE) +- ProteoWizard: https://github.com/ProteoWizard/pwiz, Apache License 2.0 +- zlib: https://zlib.net, zlib License diff --git a/Windows/x86_64/MaRaCluster/maracluster.exe b/Windows/x86_64/MaRaCluster/maracluster.exe index e3afc92..c961ab7 100644 Binary files a/Windows/x86_64/MaRaCluster/maracluster.exe and b/Windows/x86_64/MaRaCluster/maracluster.exe differ diff --git a/Windows/x86_64/XTandem/LICENSE b/Windows/x86_64/XTandem/LICENSE deleted file mode 100644 index 33a9314..0000000 --- a/Windows/x86_64/XTandem/LICENSE +++ /dev/null @@ -1,121 +0,0 @@ -The Artistic License for all X! software, binaries and documentation - -Preamble -The intent of this document is to state the conditions under which a -Package may be copied, such that the Copyright Holder maintains some -semblance of artistic control over the development of the package, -while giving the users of the package the right to use and distribute -the Package in a more-or-less customary fashion, plus the right to -make reasonable modifications. - -Definitions -"Package" refers to the collection of files distributed by the Copyright - Holder, and derivatives of that collection of files created through - textual modification. - -"Standard Version" refers to such a Package if it has not been modified, - or has been modified in accordance with the wishes of the Copyright - Holder as specified below. - -"Copyright Holder" is whoever is named in the copyright or copyrights - for the package. - -"You" is you, if you're thinking about copying or distributing this Package. - -"Reasonable copying fee" is whatever you can justify on the basis of - media cost, duplication charges, time of people involved, and so on. - (You will not be required to justify it to the Copyright Holder, but - only to the computing community at large as a market that must bear - the fee.) - -"Freely Available" means that no fee is charged for the item itself, - though there may be fees involved in handling the item. It also means - that recipients of the item may redistribute it under the same - conditions they received it. - -1. You may make and give away verbatim copies of the source form of the -Standard Version of this Package without restriction, provided that -you duplicate all of the original copyright notices and associated -disclaimers. - -2. You may apply bug fixes, portability fixes and other modifications -derived from the Public Domain or from the Copyright Holder. A -Package modified in such a way shall still be considered the Standard -Version. - -3. You may otherwise modify your copy of this Package in any way, provided -that you insert a prominent notice in each changed file stating how and -when you changed that file, and provided that you do at least ONE of the -following: - - a. place your modifications in the Public Domain or otherwise make them - Freely Available, such as by posting said modifications to Usenet - or an equivalent medium, or placing the modifications on a major - archive site such as uunet.uu.net, or by allowing the Copyright Holder - to include your modifications in the Standard Version of the Package. - b. use the modified Package only within your corporation or organization. - c. rename any non-standard executables so the names do not conflict - with standard executables, which must also be provided, and provide - a separate manual page for each non-standard executable that clearly - documents how it differs from the Standard Version. - d. make other distribution arrangements with the Copyright Holder. - -4. You may distribute the programs of this Package in object code or -executable form, provided that you do at least ONE of the following: - - a. distribute a Standard Version of the executables and library files, - together with instructions (in the manual page or equivalent) on - where to get the Standard Version. - b. accompany the distribution with the machine-readable source of the - Package with your modifications. - c. give non-standard executables non-standard names, and clearly - document the differences in manual pages (or equivalent), together - with instructions on where to get the Standard Version. - d. make other distribution arrangements with the Copyright Holder. - -5. You may charge a reasonable copying fee for any distribution of -this Package. You may charge any fee you choose for support of -this Package. You may not charge a fee for this Package itself. -However, you may distribute this Package in aggregate with other -(possibly commercial) programs as part of a larger (possibly -commercial) software distribution provided that you do not a -dvertise this Package as a product of your own. You may embed this -Package's interpreter within an executable of yours (by linking); -this shall be construed as a mere form of aggregation, provided that -the complete Standard Version of the interpreter is so embedded. - -6. The scripts and library files supplied as input to or produced as -output from the programs of this Package do not automatically fall -under the copyright of this Package, but belong to whomever generated -them, and may be sold commercially, and may be aggregated with this -Package. If such scripts or library files are aggregated with this -Package via the so-called "undump" or "unexec" methods of producing -a binary executable image, then distribution of such an image shall -neither be construed as a distribution of this Package nor shall it -fall under the restrictions of Paragraphs 3 and 4, provided that you -do not represent such an executable image as a Standard Version of -this Package. - -7. C subroutines (or comparably compiled subroutines in other languages) -supplied by you and linked into this Package in order to emulate -subroutines and variables of the language defined by this Package -shall not be considered part of this Package, but are the equivalent -of input as in Paragraph 6, provided these subroutines do not change -the language in any way that would cause it to fail the regression -tests for the language. - -8. Aggregation of this Package with a commercial distribution is always -permitted provided that the use of this Package is embedded; that is, -when no overt attempt is made to make this Package's interfaces visible -to the end user of the commercial distribution. Such use shall not be -construed as a distribution of this Package. - -9. The name of the Copyright Holder may not be used to endorse or promote -products derived from this software without specific prior written permission. - -10. THIS PACKAGE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED -WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF -MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. - -The End - diff --git a/Windows/x86_64/XTandem/tandem.exe b/Windows/x86_64/XTandem/tandem.exe deleted file mode 100644 index 7db9112..0000000 Binary files a/Windows/x86_64/XTandem/tandem.exe and /dev/null differ diff --git a/recipes/README.md b/recipes/README.md new file mode 100644 index 0000000..270cef9 --- /dev/null +++ b/recipes/README.md @@ -0,0 +1,80 @@ +# Rebuilt engines + +The upstream binaries of two engines carry copies of zlib and expat with known +vulnerabilities, so this repository holds binaries built from the same engine +versions against current releases of those libraries: + +| Engine | Folders | Upstream binaries carry | Rebuilt with | +| --- | --- | --- | --- | +| Comet 2025.01 rev. 1 | `Linux/x86_64`, `Linux/aarch64`, `MacOS/arm64`, `MacOS/x86_64`, `Windows/x86_64` | zlib 1.2.11 and expat 2.2.9, bundled by its MSToolkit | zlib 1.3.2, expat 2.8.5 | +| MaRaCluster 1.04.1 | `Linux/x86_64`, `MacOS/arm64`, `Windows/x86_64` | zlib 1.2.3, bundled by ProteoWizard | zlib 1.3.2 | + +## How they are built + +`versions.env` pins every source: the zlib and expat release archives by their +SHA-256, and the engines and ProteoWizard by commit (which `fetch-sources.sh` +checks against the release tags). The builds are those of the upstream release +binaries, with the same runners, compilers and build scripts; what differs is +in the patches here. + +- **Comet** (`comet/`): `prepare.sh` replaces the zlib and expat archives and + source trees in MSToolkit's `src` folder with the pinned releases, and the + patch points Comet's Makefiles and Visual Studio projects at them. For the + Visual Studio build it also adds expat's `random_rand_s.c`, and `prepare.sh` + writes the `expat_config.h` that expat's CMake build generates for MSVC: + unlike expat 2.2.9's, the sources of expat 2.8.5 include it on Windows too. The binaries + are built with `make` on Linux and macOS and with MSBuild (v142 toolset) on + Windows, as Comet's release workflows do. +- **MaRaCluster** (`maracluster/`): MaRaCluster links ProteoWizard, which builds + zlib, Boost and itself from the zlib in its `libraries` folder; ProteoWizard and + MaRaCluster have to use the same zlib. The patch makes MaRaCluster's builder + scripts (`admin/builders`) build the ProteoWizard tree that `prepare.sh` puts + in place, with ProteoWizard's `--zlib-src` option pointing to zlib 1.3.2; + `prepare.sh` removes ProteoWizard's zlib 1.2.3, and on Linux and macOS runs + zlib's `configure`, which ProteoWizard's build of zlib does not (without it, + `zconf.h` leaves out ``, which clang does not accept for zlib 1.3.2's + gz* functions). MaRaCluster's release build + took the newest ProteoWizard at build time; `versions.env` pins the commit + that was (26 June 2025). The ProteoWizard tree is a sparse checkout with the + content of ProteoWizard's source tarballs (`pwiz-src-without-tv`, and on + Windows `pwiz-src-without-t`, which has the vendor APIs). + +`.github/workflows/rebuild-engines.yml` runs the builds on pull requests and +pushes to master that change this folder, and by hand (workflow_dispatch). + +## How they are checked + +- `check-bundled-libs.sh` lists the zlib and expat version strings in a binary + (as the OpenMS release check finds them) and fails on the copyright strings of + any other zlib, the version strings of any other expat, and the ZLIB_VERSION + strings of the zlib releases the upstream binaries bundled. MSVC drops most of + these strings from the executables, so on Windows it also checks the static + libraries that are linked. +- `compare/comet.sh` and `compare/maracluster.sh` run the rebuilt binary and the + upstream binary it replaces (`BASELINE_COMMIT`) on OpenMS's test data, with the + spectra uncompressed, zlib-compressed and gzipped, and require the same + results: Comet's txt, pepXML, mzIdentML, pin and SQT output, and MaRaCluster's + clusters and consensus spectra (the latter name ProteoWizard's version). +- They run the engines as OpenMS's adapters do. MaRaClusterAdapter runs + MaRaCluster's `index` step in one thread, then `batch` and `consensus` + (OpenMS/OpenMS#10259): `batch` alone converts the input files in one OpenMP + thread per file, and opening several files at once crashes MaRaCluster's Windows + build with an access violation. On a windows-2022 runner, `batch` alone on the + two test files crashed in 42 of 100 runs of the 1.04.1 release binary, in 35 of + 100 runs of the rebuilt one, and in none of 100 runs of the rebuilt one in one + thread; on Linux, neither binary crashed in 500 runs. `compare/maracluster.sh` + runs `index` and `batch` of both binaries 25 more times (`REPEAT`), and each run + has to succeed. + +## Updating the binaries + +1. Run the workflow, or push a change to this folder. +2. Download the `engines` artifact of the run. It holds the binaries in this + repository's folder layout (without their file modes: the Linux and macOS + binaries need `chmod +x`). +3. Copy them over the old ones and commit them, with the run's URL. + +## Not covered + +MaRaCluster also links ProteoWizard's HDF5 1.8.7 (for mzMLb and mz5 files), and +ProteoWizard's master still bundles that version. Only zlib is replaced here. diff --git a/recipes/check-bundled-libs.sh b/recipes/check-bundled-libs.sh new file mode 100755 index 0000000..c0f15e9 --- /dev/null +++ b/recipes/check-bundled-libs.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Lists the zlib and expat versions that binaries carry, found the way the OpenMS +# release check finds them: zlib's "deflate/inflate x.y.z Copyright" strings and +# its ZLIB_VERSION string, and expat's "expat_x.y.z" string. Fails if one of them +# names another version than versions.env. +# +# An MSVC build keeps only the code it uses, which may leave none of these strings +# in the executable; pass the static libraries it was linked from as well. +# +# usage: check-bundled-libs.sh ... +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR source=versions.env +. "$here/versions.env" + +# printable : its printable strings, like "strings -a", which Git Bash on Windows does not have +printable() { + if command -v strings > /dev/null; then + strings -a "$1" + else + "${PYTHON:-python3}" -c 'import re, sys +for s in re.findall(rb"[\t\x20-\x7e]{4,}", open(sys.argv[1], "rb").read()): + print(s.decode())' "$1" + fi +} + +status=0 +for file in "$@"; do + echo "== $file" + # zlib's ZLIB_VERSION is a bare "1.x.y" string, which other version numbers can + # be as well; such a string fails the check only if it is the version of a zlib + # that the upstream binaries bundled. Only full versions count: the objects of + # MSVC's link-time code generation hold fragments such as "expat_2". + markers=$(printable "$file" | grep -oE '(de|in)flate [0-9]+(\.[0-9]+)+ Copyright|expat_[0-9]+\.[0-9]+\.[0-9]+|^1\.[0-9]\.[0-9]+(\.[0-9]+)?$' | sort | uniq -c || true) + if [ -z "$markers" ]; then + echo " no zlib or expat version string" + continue + fi + while read -r count marker; do + case $marker in + "deflate $ZLIB_VERSION Copyright" | "inflate $ZLIB_VERSION Copyright" | "$ZLIB_VERSION") verdict="zlib $ZLIB_VERSION" ;; + "expat_$EXPAT_VERSION") verdict="expat $EXPAT_VERSION" ;; + *flate* | 1.2.3 | 1.2.11) verdict="NOT zlib $ZLIB_VERSION"; status=1 ;; + expat_*) verdict="NOT expat $EXPAT_VERSION"; status=1 ;; + *) verdict="(another version string)" ;; + esac + printf ' %3s x %-28s %s\n' "$count" "$marker" "$verdict" + done <<< "$markers" +done +exit $status diff --git a/recipes/comet/prepare.sh b/recipes/comet/prepare.sh new file mode 100755 index 0000000..030d712 --- /dev/null +++ b/recipes/comet/prepare.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Replaces the zlib 1.2.11 and expat 2.2.9 that Comet's MSToolkit bundles with +# the releases in versions.env. The patch points Comet's Makefiles and Visual +# Studio projects to them, and their release archives take the place of the old +# ones in MSToolkit/src, extracted next to them as the old ones were. +# +# usage: prepare.sh +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR source=../versions.env +. "$here/../versions.env" + +comet=$1 +downloads=$(cd "$2" && pwd) +patch=$here/zlib-$ZLIB_VERSION-expat-$EXPAT_VERSION.patch # written for these versions + +cd "$comet" +test "$(git rev-parse HEAD)" = "$COMET_COMMIT" +git apply "$patch" + +rm -r MSToolkit/src/expat-2.2.9 MSToolkit/src/zlib-1.2.11 +rm MSToolkit/src/expat-2.2.9.tar.gz MSToolkit/src/zlib1211.zip +cp "$downloads/zlib-$ZLIB_VERSION.tar.gz" "$downloads/expat-$EXPAT_VERSION.tar.xz" MSToolkit/src/ +tar -xzf "MSToolkit/src/zlib-$ZLIB_VERSION.tar.gz" -C MSToolkit/src +tar -xJf "MSToolkit/src/expat-$EXPAT_VERSION.tar.xz" -C MSToolkit/src + +if [ "${OS:-}" = Windows_NT ]; then + # The Visual Studio projects compile expat's sources into MSToolkitLite. Unlike + # those of expat 2.2.9, they include expat_config.h also on Windows; this is the + # one that expat's CMake build writes for MSVC. + cmake -S "MSToolkit/src/expat-$EXPAT_VERSION" -B expat-config -A x64 -DEXPAT_SHARED_LIBS=OFF \ + -DEXPAT_BUILD_TOOLS=OFF -DEXPAT_BUILD_EXAMPLES=OFF -DEXPAT_BUILD_TESTS=OFF -DEXPAT_BUILD_DOCS=OFF + cp expat-config/expat_config.h "MSToolkit/src/expat-$EXPAT_VERSION/lib/" +fi + +build_files="Makefile CometSearch/Makefile MSToolkit/Makefile MSToolkit/MAKEFILE.nmake MSToolkit/mstoolkitlite.mri + Comet.vcxproj CometSearch/CometSearch.vcxproj CometWrapper/CometWrapper.vcxproj + MSToolkit/MSToolkitLite.vcxproj MSToolkit/MSToolkitLite.vcxproj.filters" +# shellcheck disable=SC2086 +if grep -n -e 'expat-2\.2\.9' -e 'zlib-1\.2\.11' -e 'zlib1211' $build_files; then + echo "Comet's build files still refer to the bundled zlib 1.2.11 or expat 2.2.9" >&2 + exit 1 +fi +ls -d MSToolkit/src/zlib-* MSToolkit/src/expat-* diff --git a/recipes/comet/zlib-1.3.2-expat-2.8.5.patch b/recipes/comet/zlib-1.3.2-expat-2.8.5.patch new file mode 100644 index 0000000..9eb63d5 --- /dev/null +++ b/recipes/comet/zlib-1.3.2-expat-2.8.5.patch @@ -0,0 +1,716 @@ +diff --git a/Comet.vcxproj b/Comet.vcxproj +index aba3de6..db1fc44 100644 +--- a/Comet.vcxproj ++++ b/Comet.vcxproj +@@ -113,7 +113,7 @@ + + Level3 + Disabled +- MSToolkit\include;MSToolkit\src\expat-2.2.9\lib;MSToolkit\src\zlib-1.2.11;CometSearch;%(AdditionalIncludeDirectories) ++ MSToolkit\include;MSToolkit\src\expat-2.8.5\lib;MSToolkit\src\zlib-1.3.2;CometSearch;%(AdditionalIncludeDirectories) + Default + false + Neither +@@ -143,7 +143,7 @@ + MaxSpeed + false + true +- pthreads-w32\include;MSToolkit\include;MSToolkit\src\expat-2.2.9\lib;MSToolkit\src\zlib-1.2.11;CometSearch;%(AdditionalIncludeDirectories) ++ pthreads-w32\include;MSToolkit\include;MSToolkit\src\expat-2.8.5\lib;MSToolkit\src\zlib-1.3.2;CometSearch;%(AdditionalIncludeDirectories) + MultiThreadedDLL + AnySuitable + Speed +@@ -172,7 +172,7 @@ + MaxSpeed + false + true +- MSToolkit\include;MSToolkit\src\expat-2.2.9\lib;MSToolkit\src\zlib-1.2.11;CometSearch;%(AdditionalIncludeDirectories) ++ MSToolkit\include;MSToolkit\src\expat-2.8.5\lib;MSToolkit\src\zlib-1.3.2;CometSearch;%(AdditionalIncludeDirectories) + WIN32;WIN64;_WIN64;_CRT_SECURE_NO_DEPRECATE;_NOSQLITE + Fast + false +diff --git a/CometSearch/CometSearch.vcxproj b/CometSearch/CometSearch.vcxproj +index e6c8c3f..c5c59fb 100644 +--- a/CometSearch/CometSearch.vcxproj ++++ b/CometSearch/CometSearch.vcxproj +@@ -109,7 +109,7 @@ + + Level3 + Disabled +- ..\MSToolkit\include;..\MSToolkit\src\expat-2.2.9\lib;..\MSToolkit\src\zlib-1.2.11 ++ ..\MSToolkit\include;..\MSToolkit\src\expat-2.8.5\lib;..\MSToolkit\src\zlib-1.3.2 + Fast + WIN32;WIN64;_WIN64;_MBCS;_CRT_SECURE_NO_DEPRECATE;_NOSQLITE;%(PreprocessorDefinitions) + ProgramDatabase +@@ -125,7 +125,7 @@ + MaxSpeed + false + true +- ..\pthreads-w32\include;..\MSToolkit\include;..\MSToolkit\src\expat-2.2.9\lib;..\MSToolkit\src\zlib-1.2.11;..\MSToolkit\src\sqlite-3.32.1 ++ ..\pthreads-w32\include;..\MSToolkit\include;..\MSToolkit\src\expat-2.8.5\lib;..\MSToolkit\src\zlib-1.3.2;..\MSToolkit\src\sqlite-3.32.1 + AnySuitable + Speed + true +@@ -148,7 +148,7 @@ + MaxSpeed + false + true +- ..\MSToolkit\include;..\MSToolkit\src\expat-2.2.9\lib;..\MSToolkit\src\zlib-1.2.11; ++ ..\MSToolkit\include;..\MSToolkit\src\expat-2.8.5\lib;..\MSToolkit\src\zlib-1.3.2; + AnySuitable + Speed + true +diff --git a/CometSearch/Makefile b/CometSearch/Makefile +index 4af7032..537c71e 100644 +--- a/CometSearch/Makefile ++++ b/CometSearch/Makefile +@@ -14,9 +14,9 @@ endif + + UNAME_S := $(shell uname -s) + ifeq ($(UNAME_S),Darwin) +- override CXXFLAGS += -O3 -static -std=c++14 -fpermissive -Wall -Wextra -Wno-write-strings -DGITHUBSHA='"$(GITHUB_SHA)"' -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DGCC -D_NOSQLITE -D__int64=off64_t -I. -I$(MSTPATH)/include -I$(MSTPATH)/src/expat-2.2.9/lib -I$(MSTPATH)/src/zlib-1.2.11 ++ override CXXFLAGS += -O3 -static -std=c++14 -fpermissive -Wall -Wextra -Wno-write-strings -DGITHUBSHA='"$(GITHUB_SHA)"' -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DGCC -D_NOSQLITE -D__int64=off64_t -I. -I$(MSTPATH)/include -I$(MSTPATH)/src/expat-2.8.5/lib -I$(MSTPATH)/src/zlib-1.3.2 + else +- override CXXFLAGS += -O3 -static -std=c++14 -fpermissive -Wall -Wextra -Wno-write-strings -DGITHUBSHA='"$(GITHUB_SHA)"' -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DGCC -D_NOSQLITE -D__int64=off64_t -I. -I$(MSTPATH)/include -I$(MSTPATH)/src/expat-2.2.9/lib -I$(MSTPATH)/src/zlib-1.2.11 ++ override CXXFLAGS += -O3 -static -std=c++14 -fpermissive -Wall -Wextra -Wno-write-strings -DGITHUBSHA='"$(GITHUB_SHA)"' -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -DGCC -D_NOSQLITE -D__int64=off64_t -I. -I$(MSTPATH)/include -I$(MSTPATH)/src/expat-2.8.5/lib -I$(MSTPATH)/src/zlib-1.3.2 + + endif + +diff --git a/CometWrapper/CometWrapper.vcxproj b/CometWrapper/CometWrapper.vcxproj +index e080cd1..e528095 100644 +--- a/CometWrapper/CometWrapper.vcxproj ++++ b/CometWrapper/CometWrapper.vcxproj +@@ -110,7 +110,7 @@ + Disabled + WIN32;WIN64;_WIN64;_DEBUG;_NOSQLITE;_CRT_SECURE_NO_WARNINGS%(PreprocessorDefinitions) + Use +- ..\MSToolkit\include;..\MSToolkit\src\expat-2.2.9\lib;..\MSToolkit\src\zlib-1.2.11;..\CometSearch ++ ..\MSToolkit\include;..\MSToolkit\src\expat-2.8.5\lib;..\MSToolkit\src\zlib-1.3.2;..\CometSearch + ProgramDatabase + + +@@ -124,7 +124,7 @@ + Level3 + WIN32;NDEBUG;_NOSQLITE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + Use +- ..\MSToolkit\include;..\MSToolkit\src\expat-2.2.9\lib;..\MSToolkit\src\zlib-1.2.11;..\MSToolkit\src\sqlite-3.32.1;..\CometSearch ++ ..\MSToolkit\include;..\MSToolkit\src\expat-2.8.5\lib;..\MSToolkit\src\zlib-1.3.2;..\MSToolkit\src\sqlite-3.32.1;..\CometSearch + + + true +@@ -137,7 +137,7 @@ + Level3 + WIN32;WIN64;_WIN64;NDEBUG;_NOSQLITE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + Use +- ..\MSToolkit\include;..\MSToolkit\src\expat-2.2.9\lib;..\MSToolkit\src\zlib-1.2.11;..\CometSearch ++ ..\MSToolkit\include;..\MSToolkit\src\expat-2.8.5\lib;..\MSToolkit\src\zlib-1.3.2;..\CometSearch + AnySuitable + true + Speed +diff --git a/MSToolkit/MAKEFILE.nmake b/MSToolkit/MAKEFILE.nmake +index 51fc79f..3e725ca 100644 +--- a/MSToolkit/MAKEFILE.nmake ++++ b/MSToolkit/MAKEFILE.nmake +@@ -39,7 +39,7 @@ clean: + # -- Expat -------------------------------------------- + # XML support + #------------------------------------------------------ +-EXPAT_SRCDIR=$(BUILD_SRC)\expat-2.2.9\lib ++EXPAT_SRCDIR=$(BUILD_SRC)\expat-2.8.5\lib + + expat : $(EXPAT_SRCDIR)\\*.c + cl $(CFLAGS) $(CDEFS) /I"$(BUILD_INC)" /c /Fo$(BUILD_DIR)\ $? +@@ -66,7 +66,7 @@ mzparser : $(MZP_SRCDIR)\\*.cpp + # -- zlib --------------------------------------------- + # zlib file compression + #------------------------------------------------------ +-ZLIB_SRCDIR=$(BUILD_SRC)\zlib-1.2.11 ++ZLIB_SRCDIR=$(BUILD_SRC)\zlib-1.3.2 + + zlib : $(ZLIB_SRCDIR)\\*.c + cl $(CFLAGS) $(CDEFS) /I"$(BUILD_INC)" /c /Fo$(BUILD_DIR)\ $? +diff --git a/MSToolkit/MSToolkitLite.vcxproj b/MSToolkit/MSToolkitLite.vcxproj +index c8edd0c..d0fe4b4 100644 +--- a/MSToolkit/MSToolkitLite.vcxproj ++++ b/MSToolkit/MSToolkitLite.vcxproj +@@ -103,7 +103,7 @@ + + + Full +- ./include;./src/expat-2.2.9/lib;./src/zlib-1.2.11;%(AdditionalIncludeDirectories) ++ ./include;./src/expat-2.8.5/lib;./src/zlib-1.3.2;%(AdditionalIncludeDirectories) + WIN32;WIN64;_WIN64;_DEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;COMPILED_FROM_DSP;_NOSQLITE;XML_STATIC;%(PreprocessorDefinitions) + Default + MultiThreadedDebugDLL +@@ -129,7 +129,7 @@ + + MaxSpeed + true +- ./include;./src/expat-2.2.9/lib;./src/zlib-1.2.11;%(AdditionalIncludeDirectories) ++ ./include;./src/expat-2.8.5/lib;./src/zlib-1.3.2;%(AdditionalIncludeDirectories) + WIN32;NDEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;COMPILED_FROM_DSP;_NOSQLITE;XML_STATIC;%(PreprocessorDefinitions) + MultiThreadedDLL + false +@@ -155,7 +155,7 @@ + + MaxSpeed + true +- ./include;./src/expat-2.2.9/lib;./src/zlib-1.2.11;%(AdditionalIncludeDirectories) ++ ./include;./src/expat-2.8.5/lib;./src/zlib-1.3.2;%(AdditionalIncludeDirectories) + WIN32;WIN64;_WIN64;NDEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;COMPILED_FROM_DSP;_NOSQLITE;XML_STATIC;%(PreprocessorDefinitions) + MultiThreadedDLL + false +@@ -178,11 +178,12 @@ + + + +- +- +- +- +- ++ ++ ++ ++ ++ ++ + + + +@@ -204,25 +205,25 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + +@@ -310,57 +311,57 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +diff --git a/MSToolkit/MSToolkitLite.vcxproj.filters b/MSToolkit/MSToolkitLite.vcxproj.filters +index 1f4d2c2..3a6dba2 100644 +--- a/MSToolkit/MSToolkitLite.vcxproj.filters ++++ b/MSToolkit/MSToolkitLite.vcxproj.filters +@@ -27,19 +27,22 @@ + + + +- ++ + Source Files\expat + +- ++ + Source Files\expat + +- ++ + Source Files\expat + +- ++ + Source Files\expat + +- ++ ++ Source Files\expat ++ ++ + Source Files\expat + + +@@ -105,55 +108,55 @@ + + Source Files\mzParser + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + +- ++ + Source Files\zLib + + + + +- ++ + Header Files + + +@@ -209,157 +212,157 @@ + + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + +- ++ + Header Files + + +diff --git a/MSToolkit/Makefile b/MSToolkit/Makefile +index f199bb1..e6244e8 100644 +--- a/MSToolkit/Makefile ++++ b/MSToolkit/Makefile +@@ -52,7 +52,7 @@ realclean : expat-realclean zlib-realclean mstoolkit-realclean mzparser-realclea + # + # http://expat.sourceforge.net/ + # +-EXPAT_VER := expat-2.2.9 ++EXPAT_VER := expat-2.8.5 + EXPAT_SRC := $(BUILD_SRC)/$(EXPAT_VER) + EXPAT_DST := include/expat_config.h $(BUILD_DIR)/libexpat.a + EXPAT_LIC := $(BUILD_LIC)/$(EXPAT_VER)/COPYING +@@ -62,7 +62,7 @@ EXPAT_LIC := $(BUILD_LIC)/$(EXPAT_VER)/COPYING + expat : $(EXPAT_DST) $(EXPAT_LIC) + + $(EXPAT_DST) : | $(BUILD_DIR)/ $(EXPAT_SRC)/ +- cd $(BUILD_SRC); tar -xzf $(EXPAT_VER).tar.gz ++ cd $(BUILD_SRC); tar -xJf $(EXPAT_VER).tar.xz + cd $(EXPAT_SRC); ./configure --prefix=$(BUILD_DIR) \ + --enable-shared=yes --enable-static=yes \ + --without-xmlwf --without-examples --without-tests \ +@@ -92,7 +92,7 @@ expat-realclean : expat-clean + # + # http://zlib.net/ + # +-ZLIB_VER := zlib-1.2.11 ++ZLIB_VER := zlib-1.3.2 + ZLIB_SRC := $(BUILD_SRC)/$(ZLIB_VER) + ZLIB_SRCB := $(wildcard $(ZLIB_SRC)/*.c) + ZLIB_DST := $(BUILD_DIR)/libz.a +@@ -103,7 +103,7 @@ ZLIB_LIC := $(BUILD_LIC)/$(ZLIB_VER)/README + zlib : $(ZLIB_DST) $(ZLIB_LIC) + + $(ZLIB_DST) : | $(BUILD_DIR)/ $(ZLIB_SRC)/ +- cd $(BUILD_SRC); unzip -o zlib1211.zip ++ cd $(BUILD_SRC); tar -xzf $(ZLIB_VER).tar.gz + + ifeq ($(OS),Windows_NT) + cd $(ZLIB_SRC); make -f win32/Makefile.gcc; cp libz.a $(BUILD_DIR) +@@ -215,7 +215,7 @@ lib : expat zlib mstoolkit mzparser mzimltools + ifneq ($(UNAME_S),Darwin) + ar -M +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# the absolute path of a file +abs() { echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"; } +# native paths for the params file (Git Bash on Windows) +native() { if command -v cygpath > /dev/null; then cygpath -m "$1"; else echo "$1"; fi; } +python=${PYTHON:-python3} + +old=$(abs "$1") +new=$(abs "$2") +work=$3 +openms=$(cd "$4" && pwd) +data=$openms/src/tests/topp/THIRDPARTY +examples=$openms/share/OpenMS/examples + +searches=( + "c3|$data/CometAdapter_3.mzML|$data/CometAdapter_3.fasta" + "c6|$data/CometAdapter_6_in.mzML|$data/CometAdapter_3.fasta" + "bsa|$examples/FRACTIONS/BSA1_F1.mzML|$examples/TOPPAS/data/BSA_Identification/18Protein_SoCe_Tr_detergents_trace_target_decoy.fasta" + "spectra|$data/spectra_comet.mzML|$data/proteins.fasta" +) + +mkdir -p "$work" +cd "$work" +"$old" -p > /dev/null # writes comet.params.new +status=0 +for search in "${searches[@]}"; do + IFS='|' read -r name mzml fasta <<< "$search" + mkdir -p "in/$name" + cp "$mzml" "in/$name/plain.mzML" + "$python" "$here/mzml-zlib.py" "$mzml" "in/$name/zlib.mzML" + gzip -c "$mzml" > "in/$name/gz.mzML.gz" + sed -e "s|^database_name = .*|database_name = $(native "$fasta")|" \ + -e 's|^decoy_search = .*|decoy_search = 1|' \ + -e 's|^num_threads = .*|num_threads = 4|' \ + -e 's|^output_sqtfile = .*|output_sqtfile = 1|' \ + -e 's|^output_txtfile = .*|output_txtfile = 1|' \ + -e 's|^output_mzidentmlfile = .*|output_mzidentmlfile = 1|' \ + -e 's|^output_percolatorfile = .*|output_percolatorfile = 1|' \ + comet.params.new > "in/$name/comet.params" + for variant in plain zlib gz; do + input=$(cd "in/$name" && ls "$variant".mzML*) + for bin in old new; do + exe=$old + [ $bin = new ] && exe=$new + out=out/$bin/$name-$variant + mkdir -p "$out" + # the same relative paths for both binaries, so that their outputs are comparable + (cd "$out" && "$exe" -P"../../../in/$name/comet.params" -Nr "../../../in/$name/$input" > log.txt 2>&1) \ + || { echo "FAIL $bin $name-$variant"; status=1; } + # leave out what changes from run to run: dates, times, absolute paths + for f in "$out"/r.*; do + [ -e "$f" ] || continue # no output at all + case $f in *.norm) continue ;; esac + sed -E -e 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+//g' -e 's/(date|creationDate)="[^"]*"//g' \ + -e 's#[0-9]{2}/[0-9]{2}/[0-9]{4}, [0-9]{2}:[0-9]{2}:[0-9]{2} [AP]M##g' \ + -e 's/[A-Z][a-z]{2} [A-Z][a-z]{2} +[0-9]+ [0-9:]+ [0-9]{4}//g' \ + -e 's#[^ "<>]*[/\\]out[/\\](old|new)[/\\]#out/BIN/#g' \ + "$f" > "$f.norm" + done + done + # the same output files from both, and the same content in each + if ! diff <(cd "out/old/$name-$variant" && ls r.*.norm 2> /dev/null) \ + <(cd "out/new/$name-$variant" && ls r.*.norm 2> /dev/null) > /dev/null; then + echo "DIFF $name-$variant output files" + status=1 + fi + for f in out/old/"$name-$variant"/r.*.norm; do + g=out/new/$name-$variant/$(basename "$f") + if cmp -s "$f" "$g"; then + echo "same $name-$variant $(basename "${f%.norm}") ($(wc -l < "$f") lines)" + else + echo "DIFF $name-$variant $(basename "${f%.norm}")" + diff "$f" "$g" | head -n 10 || true + status=1 + fi + done + done + # and the encodings of the spectra must not change the results + for variant in zlib gz; do + if cmp -s "out/new/$name-plain/r.txt.norm" "out/new/$name-$variant/r.txt.norm"; then + echo "same $name plain and $variant input (txt)" + else + echo "DIFF $name plain and $variant input (txt)" + status=1 + fi + done +done +exit $status diff --git a/recipes/compare/maracluster.sh b/recipes/compare/maracluster.sh new file mode 100755 index 0000000..3cb9a7d --- /dev/null +++ b/recipes/compare/maracluster.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Runs two MaRaCluster binaries as OpenMS's MaRaClusterAdapter runs it (index in +# one thread, batch, consensus) on the adapter's test spectra and compares the +# clusters and the consensus spectra. The spectra are given as they are +# (uncompressed), with zlib-compressed binary arrays, and gzipped, so that the +# runs go through zlib. +# +# The batch step alone would convert the spectra in one OpenMP thread per file, +# and opening several files at once crashes MaRaCluster's Windows build, the +# release binary as the rebuilt one (see ../README.md); the index step converts +# them in one thread, and batch reuses them. With REPEAT=, the index and batch +# steps of both binaries run n more times on the uncompressed spectra, and each +# of these runs has to succeed as well. +# +# usage: [REPEAT=] maracluster.sh +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# the absolute path of a file +abs() { echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"; } +# native paths for the file lists (Git Bash on Windows) +native() { if command -v cygpath > /dev/null; then cygpath -m "$1"; else echo "$1"; fi; } +python=${PYTHON:-python3} + +# cluster : the index and batch steps, in the folder with files.txt, +# with the parameters of MaRaClusterAdapter's test 1 +cluster() { + OMP_NUM_THREADS=1 "$1" index -b files.txt -f res -p 20ppm > index.log 2>&1 && + "$1" batch -b files.txt -f res -p 20ppm -t -10 -c -10 > batch.log 2>&1 +} + +old=$(abs "$1") +new=$(abs "$2") +work=$3 +data=$(cd "$4/src/tests/topp/THIRDPARTY" && pwd) + +mkdir -p "$work/in" +cd "$work" +for i in 1 2; do + cp "$data/MaRaClusterAdapter_1_in_$i.mzML" "in/plain_$i.mzML" + "$python" "$here/mzml-zlib.py" "$data/MaRaClusterAdapter_1_in_$i.mzML" "in/zlib_$i.mzML" + gzip -c "$data/MaRaClusterAdapter_1_in_$i.mzML" > "in/gz_$i.mzML.gz" +done + +status=0 +for variant in plain zlib gz; do + for bin in old new; do + exe=$old + [ $bin = new ] && exe=$new + out=out/$bin/$variant + mkdir -p "$out" + for f in in/"$variant"_1.mzML* in/"$variant"_2.mzML*; do native "$PWD/$f"; done > "$out/files.txt" + (cd "$out" && cluster "$exe" \ + && "$exe" consensus -l res/MaRaCluster.clusters_p10.tsv -f res -o consensus.mzML -M 1 > consensus.log 2>&1) \ + || { echo "FAIL $bin $variant"; tail -n 5 "$out"/*.log; status=1; continue; } + # leave out what changes from run to run: checksums, paths, the time, ProteoWizard's version + sed -E -e '/fileChecksum|MS:1000569|completion time|sourceFile |location=/d' \ + -e 's/pwiz_3\.0\.[0-9]+/pwiz_3.0/g' -e 's/version="3\.0\.[0-9]+"/version="3.0"/g' \ + "$out/consensus.part1.mzML" > "$out/consensus.norm" + sed -E -e 's#^[^[:blank:]]*[/\\]in[/\\]#in/#' "$out/res/MaRaCluster.clusters_p10.tsv" > "$out/clusters.norm" + done + for f in clusters.norm consensus.norm; do + if cmp -s "out/old/$variant/$f" "out/new/$variant/$f"; then + echo "same $variant ${f%.norm} ($(wc -l < "out/old/$variant/$f") lines)" + else + echo "DIFF $variant ${f%.norm}" + diff "out/old/$variant/$f" "out/new/$variant/$f" | head -n 10 || true + status=1 + fi + done +done +# and the encodings of the spectra must not change the clusters +for variant in zlib gz; do + if diff -q <(sed -E "s#in/${variant}_([12])\.mzML(\.gz)?#in/plain_\1.mzML#" "out/new/$variant/clusters.norm") \ + "out/new/plain/clusters.norm" > /dev/null; then + echo "same plain and $variant input (clusters)" + else + echo "DIFF plain and $variant input (clusters)" + status=1 + fi +done + +repeat=${REPEAT:-0} +if [ "$repeat" -gt 0 ]; then + failed_old=0 failed_new=0 + for ((i = 1; i <= repeat; i++)); do + for bin in old new; do + exe=$old + [ $bin = new ] && exe=$new + out=out/repeat/$bin-$i + mkdir -p "$out" + cp "out/$bin/plain/files.txt" "$out/" + if (cd "$out" && cluster "$exe"); then + rm -r "$out" + else + echo "FAIL $bin repeat $i (exit status $?)" + tail -n 5 "$out"/*.log + if [ $bin = old ]; then failed_old=$((failed_old + 1)); else failed_new=$((failed_new + 1)); fi + fi + done + done + echo "old: $failed_old of $repeat repeated runs failed" + echo "new: $failed_new of $repeat repeated runs failed" + [ $failed_old -eq 0 ] && [ $failed_new -eq 0 ] || status=1 +fi +exit $status diff --git a/recipes/compare/mzml-zlib.py b/recipes/compare/mzml-zlib.py new file mode 100755 index 0000000..fef8ea7 --- /dev/null +++ b/recipes/compare/mzml-zlib.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Writes a copy of an (indexed) mzML whose binary data arrays are zlib-compressed +(MS:1000576 "no compression" becomes MS:1000574 "zlib compression"). The spectra +stay the same; the index offsets are recomputed. + +usage: mzml-zlib.py +""" +import base64 +import re +import sys +import zlib + +src, dst = sys.argv[1], sys.argv[2] +data = open(src, "rb").read().decode("latin-1") + + +def compress(match): + """Returns the binaryDataArray element of the match with its data zlib-compressed, + if it was uncompressed.""" + array = match.group(0) + if 'accession="MS:1000576"' not in array: + return array + array = re.sub(r'', + '', array) + array = re.sub(r"([^<]*)", + lambda m: "" + base64.b64encode(zlib.compress(base64.b64decode(m.group(1)))).decode() + "", + array) + length = len(re.search(r"([^<]*)", array).group(1)) + return re.sub(r'encodedLength="\d+"', 'encodedLength="%d"' % length, array, count=1) + + +head_end = data.index("") + len("") +head, mzml = data[:head_end], data[head_end:mzml_end] +mzml = re.sub(r"", compress, mzml, flags=re.S) +if "MS:1000576" in mzml: + sys.exit("%s: an uncompressed array is left" % src) + +out = head + mzml + "\n" +if "]*?id="([^"]*)"' % kind, mzml)] + if items: + indices.append('\t\n' % kind + + "".join('\t\t%d\n' % (i, len(head) + p) for p, i in items) + + "\t\n") + out += ('\n' % len(indices) + "".join(indices) + "\n" + + "%d\n" % (len(head) + len(mzml) + 1) + + "0\n\n") +open(dst, "wb").write(out.encode("latin-1")) diff --git a/recipes/fetch-sources.sh b/recipes/fetch-sources.sh new file mode 100755 index 0000000..a1b6c0c --- /dev/null +++ b/recipes/fetch-sources.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Fetches the sources named in versions.env into and checks them: +# the tarballs against their SHA-256, the git sources against the pinned +# commits (and the commits against their upstream tags). +# +# usage: fetch-sources.sh ... +# zlib, expat the release tarballs +# comet, maracluster the engine sources +# pwiz ProteoWizard without its vendor APIs, like its pwiz-src-without-tv tarball +# pwiz-with-vendor-apis ProteoWizard with its vendor APIs, like its pwiz-src-without-t tarball +# openms-testdata the OpenMS test data that the scripts in compare/ use +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR source=versions.env +. "$here/versions.env" + +dest=$1 +shift +mkdir -p "$dest" +cd "$dest" + +# sha256 : prints its SHA-256 (macOS has shasum, not sha256sum) +sha256() { + if command -v sha256sum > /dev/null; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi +} + +# download : fetches the file into the current folder and checks its SHA-256 +download() { + local file=${1##*/} + curl -fsSL --retry 5 --retry-delay 10 --retry-all-errors -o "$file" "$1" + local got + got=$(sha256 "$file") + if [ "$got" != "$2" ]; then + echo "$file: SHA-256 is $got, expected $2" >&2 + exit 1 + fi + echo "$file: SHA-256 $got as expected" +} + +# init_repo [as-committed (default) | platform]: an empty repository with +# as origin; its checkouts keep the line ends as committed, or use the platform's +init_repo() { + git init -q "$2" + if [ "${3:-as-committed}" = as-committed ]; then + # also on Windows, so that the patches apply and indexed mzML files keep their offsets + git -C "$2" config core.autocrlf false + fi + git -C "$2" remote add origin "$1" +} + +# checkout [ [...]]: a shallow checkout of ; +# with a tag, the tag must name that commit +checkout() { + if [ -n "${3:-}" ]; then + local refs tagged + refs=$(git -C "$1" ls-remote --tags origin "refs/tags/$3") + # an annotated tag is listed twice; its "^{}" line names the commit + tagged=$(printf '%s\n' "$refs" | awk -v t="refs/tags/$3^{}" '$2 == t { print $1 }') + [ -n "$tagged" ] || tagged=$(printf '%s\n' "$refs" | awk -v t="refs/tags/$3" '$2 == t { print $1 }') + if [ "$tagged" != "$2" ]; then + echo "$1: tag $3 is at ${tagged:-nothing}, expected $2" >&2 + exit 1 + fi + fi + git -C "$1" fetch -q --depth 1 "${@:4}" origin "$2" + git -C "$1" -c advice.detachedHead=false checkout -q FETCH_HEAD + test "$(git -C "$1" rev-parse HEAD)" = "$2" + echo "$1: $2${3:+ ($3)}" +} + +# ProteoWizard's source tarballs leave out the example data, some of the .NET +# applications and libraries, the tests (Jamroot.jam: .pwiz-src-exclusions, .l, +# .no-t), and pwiz-src-without-tv the vendor APIs (.no-v). The tests have to go: those +# of the vendor readers cannot be built without the vendor APIs, and b2 then +# skips everything that depends on them, among it the "libraries" target. +pwiz() { # + init_repo "$PWIZ_REPO" pwiz platform # CRLF for the batch files on Windows, as ProteoWizard's own builds have them + git -C pwiz sparse-checkout init --no-cone + { + echo '/*' + echo '!/example_data/' + # of libraries/, the tarballs have only what the libraries build needs + for f in arrow/ 7zz readme.txt expat-*.tar.bz2 fftw-*.tar.bz2 msvc-2005-2008-runtime.tar.bz2; do + echo "!/libraries/$f" + done + for f in BiblioSpec Skyline Bumbershoot Shared/BiblioSpec Shared/Crawdad Shared/ProteomeDb \ + Shared/Lib/Microsoft.Diagnostics.Runtime Shared/Lib/MSAmanda Shared/Lib/DotNetZip \ + Shared/Lib/NHibernate Shared/Lib/npgsql Shared/Lib/x86 Shared/Lib/x64 'Shared/Lib/mysql.*' \ + 'Shared/Lib/zlib.*' 'Shared/Lib/*.pdb' 'Shared/Lib/grpc*' 'Shared/Lib/log4net*' \ + 'Shared/Lib/MathNet.Numerics*'; do + echo "!/pwiz_tools/$f" + done + echo '!*Test*.data*' + echo '!*Test.?pp' + echo '!*TestData.?pp' + if [ "$1" = no ]; then + echo '!/libraries/msparser_*.7z' + echo '!/pwiz_aux/msrc/utility/vendor_api/*/' + echo '!/pwiz_aux/msrc/utility/vendor_api_*.7z' + fi + } > pwiz/.git/info/sparse-checkout + checkout pwiz "$PWIZ_COMMIT" "" --filter=blob:none +} + +# the OpenMS test data that recipes/compare runs the engines on +openms_testdata() { + init_repo "$OPENMS_REPO" openms + git -C openms sparse-checkout init --no-cone + { + for f in CometAdapter_3.mzML CometAdapter_3.fasta CometAdapter_6_in.mzML spectra_comet.mzML proteins.fasta \ + MaRaClusterAdapter_1_in_1.mzML MaRaClusterAdapter_1_in_2.mzML; do + echo "/src/tests/topp/THIRDPARTY/$f" + done + echo /share/OpenMS/examples/FRACTIONS/BSA1_F1.mzML + echo /share/OpenMS/examples/TOPPAS/data/BSA_Identification/18Protein_SoCe_Tr_detergents_trace_target_decoy.fasta + } > openms/.git/info/sparse-checkout + checkout openms "$OPENMS_TESTDATA_COMMIT" "" --filter=blob:none +} + +for source in "$@"; do + case $source in + zlib) download "$ZLIB_URL" "$ZLIB_SHA256" ;; + expat) download "$EXPAT_URL" "$EXPAT_SHA256" ;; + comet) init_repo "$COMET_REPO" comet && checkout comet "$COMET_COMMIT" "$COMET_TAG" ;; + maracluster) init_repo "$MARACLUSTER_REPO" maracluster && checkout maracluster "$MARACLUSTER_COMMIT" "$MARACLUSTER_TAG" ;; + pwiz) pwiz no ;; + pwiz-with-vendor-apis) pwiz yes ;; + openms-testdata) openms_testdata ;; + *) echo "unknown source: $source" >&2; exit 1 ;; + esac +done diff --git a/recipes/maracluster/prepare.sh b/recipes/maracluster/prepare.sh new file mode 100755 index 0000000..4a73d30 --- /dev/null +++ b/recipes/maracluster/prepare.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Prepares a MaRaCluster build with its own builder scripts (admin/builders) +# that uses zlib 1.3.2 instead of ProteoWizard's zlib 1.2.3, in ProteoWizard, +# in Boost and in MaRaCluster itself. +# +# The patch makes the builders take the ProteoWizard source tree that is already +# in /tools/proteowizard (instead of the newest one from ProteoWizard's +# TeamCity server) and build it with --zlib-src pointing to zlib 1.3.2 in its +# libraries folder. This script puts the pinned ProteoWizard tree and zlib there, +# and removes ProteoWizard's zlib 1.2.3, so that nothing can use it. +# +# usage: prepare.sh +set -euo pipefail +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source-path=SCRIPTDIR source=../versions.env +. "$here/../versions.env" + +maracluster=$1 +pwiz=$2 +downloads=$(cd "$3" && pwd) +build=$4 + +test "$(git -C "$maracluster" rev-parse HEAD)" = "$MARACLUSTER_COMMIT" +test "$(git -C "$pwiz" rev-parse HEAD)" = "$PWIZ_COMMIT" +git -C "$maracluster" apply "$here/pwiz-zlib-$ZLIB_VERSION.patch" # written for this version + +mkdir -p "$build/tools" +mv "$pwiz" "$build/tools/proteowizard" +libraries=$build/tools/proteowizard/libraries +rm "$libraries"/zlib-1.2.3.tar.bz2 +tar -xzf "$downloads/zlib-$ZLIB_VERSION.tar.gz" -C "$libraries" +ls -d "$libraries"/zlib* + +# ProteoWizard compiles zlib's sources without zlib's configure, which leaves +# zconf.h without : gzlib.c, gzread.c and gzwrite.c then call lseek, +# read, write and close undeclared, an error for clang and a warning for GCC. +# Configure zlib as its own build does (Windows declares them in ). +if [ "${OS:-}" != Windows_NT ]; then + (cd "$libraries/zlib-$ZLIB_VERSION" && ./configure > /dev/null) + if grep -n 'HAVE_UNISTD_H-0' "$libraries/zlib-$ZLIB_VERSION/zconf.h"; then + echo "zlib's configure did not find unistd.h" >&2 + exit 1 + fi +fi + +# cmd.exe needs CRLF line ends to find the labels that the batch files call, +# as a checkout on Windows would give them +if [ "${OS:-}" = Windows_NT ]; then + for f in "$maracluster"/admin/builders/*.bat; do + sed -i -e 's/\r$//' -e 's/$/\r/' "$f" + done +fi diff --git a/recipes/maracluster/pwiz-zlib-1.3.2.patch b/recipes/maracluster/pwiz-zlib-1.3.2.patch new file mode 100644 index 0000000..8d0f3f3 --- /dev/null +++ b/recipes/maracluster/pwiz-zlib-1.3.2.patch @@ -0,0 +1,68 @@ +diff --git a/admin/builders/_urls_and_file_names_.bat b/admin/builders/_urls_and_file_names_.bat +index d014e13..42dc035 100644 +--- a/admin/builders/_urls_and_file_names_.bat ++++ b/admin/builders/_urls_and_file_names_.bat +@@ -23,7 +23,8 @@ set PWIZ_URL=https://mc-tca-01.s3.us-west-2.amazonaws.com/ProteoWizard/bt81/%PWI + + ::: remember to update the boost and zlib version when ProteoWizard updates their versions + set BOOST_BASE=boost_1_86_0 +-set ZLIB_BASE=zlib-1.2.3 ++::: OpenMS THIRDPARTY: zlib 1.3.2, which the build expects in %PWIZ_DIR%\libraries ++set ZLIB_BASE=zlib-1.3.2 + + ::: Boost asio library (note the additional %%% before %20 to prevent interpretation as a variable) + set BOOST_ASIO_BASE=boost_asio_1_30_2 +diff --git a/admin/builders/install_proteowizard.bat b/admin/builders/install_proteowizard.bat +index f9365ab..ac4434e 100644 +--- a/admin/builders/install_proteowizard.bat ++++ b/admin/builders/install_proteowizard.bat +@@ -20,7 +20,9 @@ if not exist "%PWIZ_DIR%\lib" ( + + cd /D "%PWIZ_DIR%" + echo Starting Proteowizard and Boost build, this can take a while... ++ ::: OpenMS THIRDPARTY: zlib 1.3.2 in place of ProteoWizard's zlib 1.2.3, for ProteoWizard, Boost and MaRaCluster + call quickbuild.bat address-model=%ADDRESS_MODEL% -j4 --toolset=msvc-%MSVC_VER%.0 --i-agree-to-the-vendor-licenses ^ ++ --zlib-src="%PWIZ_DIR%\libraries\%ZLIB_BASE%" ^ + pwiz/data/common//pwiz_data_common ^ + pwiz/data/identdata//pwiz_data_identdata ^ + pwiz/data/identdata//pwiz_data_identdata_version ^ +diff --git a/admin/builders/install_proteowizard.sh b/admin/builders/install_proteowizard.sh +index 0bc2823..338bcd7 100755 +--- a/admin/builders/install_proteowizard.sh ++++ b/admin/builders/install_proteowizard.sh +@@ -3,6 +3,8 @@ tools_dir=$1 + # change directory to the /tools directory + cd ${tools_dir} + ++# OpenMS THIRDPARTY: build the ProteoWizard source tree in proteowizard/ if there is one ++if [ ! -d proteowizard ]; then + echo "Download source code for ProteoWizard from their TeamCity server" + wget --no-check-certificate --no-verbose -O bt81.xml https://proteowizard.sourceforge.io/releases/bt81.xml + # without-tv: without tests and vendor reader +@@ -15,6 +17,7 @@ if [ ! -f ${tools_dir}/${FILE_NAME} ]; then + mkdir proteowizard + tar xf ${FILE_NAME} --directory proteowizard + fi ++fi + + cd proteowizard + +@@ -25,7 +28,8 @@ echo "Building ProteoWizard and Boost, this may take some time.." + + # if you have more than 4GB of memory available, you could try to use more than 2 cores to speed things up + # add -d2 flag to get verbose output with compiler and linker commands +-./quickbuild.sh ${toolset} -j2 --prefix=../ \ ++# OpenMS THIRDPARTY: zlib 1.3.2 in place of ProteoWizard's zlib 1.2.3, for ProteoWizard, Boost and MaRaCluster ++./quickbuild.sh ${toolset} -j2 --prefix=../ --zlib-src="$(pwd)/libraries/zlib-1.3.2" \ + pwiz/data/common//pwiz_data_common \ + pwiz/data/identdata//pwiz_data_identdata \ + pwiz/data/identdata//pwiz_data_identdata_version \ +@@ -67,7 +71,7 @@ echo "Updating Proteowizard and Boost libraries and include files." + find build-*/ -type f | grep -i libboost_regex-.*\.a$ | xargs -I{} cp {} ../lib + find build-*/ -type f | grep -i libboost_program_options-.*\.a$ | xargs -I{} cp {} ../lib + +-rsync -ap --include "*/" --include "*.h" --include "*.hpp" --exclude "*" libraries/zlib-1.2.3/ ../include ++rsync -ap --include "*/" --include "*.h" --include "*.hpp" --exclude "*" libraries/zlib-1.3.2/ ../include + rsync -ap --include "*/" --include "*.h" --include "*.hpp" --exclude "*" libraries/boost_aux/boost/ ../include/boost + rsync -ap --include "*/" --include '*.ipp' --exclude '*' libraries/boost_1_86_0/boost/ ../include/boost + diff --git a/recipes/versions.env b/recipes/versions.env new file mode 100644 index 0000000..b9b3f24 --- /dev/null +++ b/recipes/versions.env @@ -0,0 +1,32 @@ +# Sources of the engines that this repository rebuilds (see README.md). +# Read by the scripts in this folder and by .github/workflows/rebuild-engines.yml. + +# zlib and expat that replace the copies the engines bundle +ZLIB_VERSION=1.3.2 +ZLIB_URL=https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz +ZLIB_SHA256=bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16 +EXPAT_VERSION=2.8.5 +EXPAT_URL=https://github.com/libexpat/libexpat/releases/download/R_2_8_5/expat-2.8.5.tar.xz +EXPAT_SHA256=1e727b8933ec51a77a9a9d9afcf8e688bce45d907c13e36ab7393fe36e703182 + +# Comet 2025.01 rev. 1: its MSToolkit bundles zlib 1.2.11 and expat 2.2.9 +COMET_REPO=https://github.com/UWPR/Comet.git +COMET_TAG=v2025.01.1 +COMET_COMMIT=4181df6987a4b997dfded79b07f154682ff7838d + +# MaRaCluster 1.04.1: links ProteoWizard, which bundles zlib 1.2.3 +MARACLUSTER_REPO=https://github.com/statisticalbiotechnology/maracluster.git +MARACLUSTER_TAG=rel-1-04-1 +MARACLUSTER_COMMIT=c122929692d731766385a5fce36acbfab17b807a + +# ProteoWizard as the rel-1-04-1 release build used it: that build (26 June 2025) +# took the newest ProteoWizard source from TeamCity, which was master at this commit. +PWIZ_REPO=https://github.com/ProteoWizard/pwiz.git +PWIZ_COMMIT=9ba731735533a808ad9b983a6789af31b9971bcd + +# Verification (compare/): the rebuilt engines must give the same results as the +# upstream-built binaries that this repository held before (THIRDPARTY at this +# commit), on this OpenMS revision's test data. +BASELINE_COMMIT=fd8c89f407c1ecae9328ec71bb452195a7799641 +OPENMS_REPO=https://github.com/OpenMS/OpenMS.git +OPENMS_TESTDATA_COMMIT=0baa9a5594314910eea65dcaf555cc6f59517a97