From 791e866d2f5b8fc65f90f63036f0411291332bd7 Mon Sep 17 00:00:00 2001 From: Marc Suchard Date: Wed, 14 Jul 2021 06:37:12 -0700 Subject: [PATCH 001/421] update linux build to statically link libtdl --- libhmsbeagle/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libhmsbeagle/CMakeLists.txt b/libhmsbeagle/CMakeLists.txt index ff05770d..8715c353 100644 --- a/libhmsbeagle/CMakeLists.txt +++ b/libhmsbeagle/CMakeLists.txt @@ -76,6 +76,10 @@ add_library(hmsbeagle SHARED set_target_properties(hmsbeagle PROPERTIES PUBLIC_HEADER "beagle.h;platform.h") +if(NOT WIN32 AND NOT APPLE) + target_link_libraries(hmsbeagle ${CMAKE_DL_LIBS}) +endif(NOT WIN32 AND NOT APPLE) + add_subdirectory(CPU) add_subdirectory(GPU) if(BUILD_JNI) From 341fddbbe3fe7e40ea6f4a261c03b8f1848267c9 Mon Sep 17 00:00:00 2001 From: Jonathan Chang Date: Sun, 18 Jul 2021 23:44:58 -0700 Subject: [PATCH 002/421] Fix macOS rpath for non-default prefixes --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f4df1495..402822d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ if(APPLE) endif() # Seems necessary from macOS 10.15 - set(CMAKE_INSTALL_RPATH "/usr/local/lib") + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") endif(APPLE) if (BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH) From 1d4b21d062999c303d0d8b3e1e306c3d5e81d708 Mon Sep 17 00:00:00 2001 From: Marc Suchard Date: Mon, 24 Jan 2022 09:49:08 -0800 Subject: [PATCH 003/421] change file headers to MIT --- examples/standalone/epochtest/COPYING | 674 ------------------ examples/standalone/epochtest/LICENSE | 21 + java/beagle/Beagle.java | 16 +- libhmsbeagle/BeagleImpl.h | 74 +- libhmsbeagle/CPU/AVXDefinitions.h | 16 +- libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.h | 80 +-- libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.hpp | 38 +- libhmsbeagle/CPU/BeagleCPU4StateImpl.h | 34 +- libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp | 316 ++++---- libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h | 74 +- libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp | 16 +- libhmsbeagle/CPU/BeagleCPUAVXImpl.h | 30 +- libhmsbeagle/CPU/BeagleCPUAVXImpl.hpp | 44 +- libhmsbeagle/CPU/BeagleCPUImpl.h | 16 +- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 16 +- libhmsbeagle/CPU/BeagleCPUSSEImpl.h | 28 +- libhmsbeagle/CPU/BeagleCPUSSEImpl.hpp | 50 +- libhmsbeagle/CPU/SSEDefinitions.h | 16 +- libhmsbeagle/GPU/BeagleGPUImpl.h | 125 ++-- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 52 +- libhmsbeagle/GPU/GPUImplDefs.h | 16 +- libhmsbeagle/GPU/GPUImplHelper.cpp | 16 +- libhmsbeagle/GPU/GPUImplHelper.h | 16 +- libhmsbeagle/GPU/GPUInterface.h | 68 +- libhmsbeagle/GPU/GPUInterfaceCUDA.cpp | 16 +- libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp | 16 +- libhmsbeagle/GPU/KernelLauncher.cpp | 356 +++++---- libhmsbeagle/GPU/KernelLauncher.h | 84 +-- libhmsbeagle/GPU/KernelResource.h | 26 +- libhmsbeagle/GPU/kernels/kernels4.cu | 158 ++-- libhmsbeagle/GPU/kernels/kernelsAll.cu | 62 +- libhmsbeagle/GPU/kernels/kernelsX.cu | 44 +- libhmsbeagle/beagle.cpp | 16 +- libhmsbeagle/beagle.h | 104 ++- libhmsbeagle/benchmark/BeagleBenchmark.cpp | 154 ++-- libhmsbeagle/benchmark/BeagleBenchmark.h | 30 +- libhmsbeagle/platform.h | 20 +- .../COPYING.LESSER.rtf | 169 ----- .../installer/COPYING.LESSER.rtf | 169 ----- .../beagle-package.pmdoc/index.xml | 106 --- 40 files changed, 967 insertions(+), 2415 deletions(-) delete mode 100644 examples/standalone/epochtest/COPYING create mode 100644 examples/standalone/epochtest/LICENSE delete mode 100644 project/beagle-vs-2012-installer/COPYING.LESSER.rtf delete mode 100644 project/beagle-vs-2017/installer/COPYING.LESSER.rtf delete mode 100644 project/beagle-xcode/beagle-package.pmdoc/index.xml diff --git a/examples/standalone/epochtest/COPYING b/examples/standalone/epochtest/COPYING deleted file mode 100644 index 94a9ed02..00000000 --- a/examples/standalone/epochtest/COPYING +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/examples/standalone/epochtest/LICENSE b/examples/standalone/epochtest/LICENSE new file mode 100644 index 00000000..12741b8c --- /dev/null +++ b/examples/standalone/epochtest/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Phylogenetic Likelihood Working Group + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/java/beagle/Beagle.java b/java/beagle/Beagle.java index 9bfb765c..043f529a 100644 --- a/java/beagle/Beagle.java +++ b/java/beagle/Beagle.java @@ -5,19 +5,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @brief This file documents the API as well as header for the * Broad-platform Evolutionary Analysis General Likelihood Evaluator diff --git a/libhmsbeagle/BeagleImpl.h b/libhmsbeagle/BeagleImpl.h index 5f640216..ab7a1071 100644 --- a/libhmsbeagle/BeagleImpl.h +++ b/libhmsbeagle/BeagleImpl.h @@ -6,20 +6,10 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . - * * @author Andrew Rambaut * @author Marc Suchard * @author Daniel Ayres @@ -42,23 +32,23 @@ class BeagleImpl { public: virtual ~BeagleImpl(){} - + virtual int createInstance(int tipCount, int partialsBufferCount, int compactBufferCount, int stateCount, int patternCount, int eigenBufferCount, - int matrixBufferCount, + int matrixBufferCount, int categoryCount, int scaleBufferCount, int resourceNumber, int pluginResourceNumber, long preferenceFlags, long requirementFlags) = 0; - + virtual int getInstanceDetails(BeagleInstanceDetails* returnInfo) = 0; - + virtual int setCPUThreadCount(int threadCount) = 0; virtual int setTipStates(int tipIndex, @@ -66,7 +56,7 @@ class BeagleImpl virtual int setTipPartials(int tipIndex, const double* inPartials) = 0; - + virtual int setPartials(int bufferIndex, const double* inPartials) = 0; @@ -77,28 +67,28 @@ class BeagleImpl virtual int getPartials(int bufferIndex, int scaleIndex, double* outPartials) = 0; - + virtual int setEigenDecomposition(int eigenIndex, const double* inEigenVectors, const double* inInverseEigenVectors, const double* inEigenValues) = 0; - + virtual int setStateFrequencies(int stateFrequenciesIndex, - const double* inStateFrequencies) = 0; - + const double* inStateFrequencies) = 0; + virtual int setCategoryWeights(int categoryWeightsIndex, const double* inCategoryWeights) = 0; - + virtual int setPatternWeights(const double* inPatternWeights) = 0; virtual int setPatternPartitions(int partitionCount, const int* inPatternPartitions) = 0; - + virtual int setCategoryRates(const double* inCategoryRates) = 0; virtual int setCategoryRatesWithIndex(int categoryRatesIndex, const double* inCategoryRates) = 0; - + virtual int setTransitionMatrix(int matrixIndex, const double* inMatrix, double paddedValue) = 0; @@ -109,8 +99,8 @@ class BeagleImpl virtual int setTransitionMatrices(const int* matrixIndices, const double* inMatrices, const double* paddedValues, - int count) = 0; - + int count) = 0; + virtual int getTransitionMatrix(int matrixIndex, double* outMatrix) = 0; @@ -149,7 +139,7 @@ class BeagleImpl const int* secondDerivativeIndices, const double* edgeLengths, int count) = 0; - + virtual int updatePartials(const int* operations, int operationCount, int cumulativeScalingIndex) = 0; @@ -166,7 +156,7 @@ class BeagleImpl virtual int waitForPartials(const int* destinationPartials, int destinationPartialsCount) = 0; - + virtual int accumulateScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex) = 0; @@ -175,7 +165,7 @@ class BeagleImpl int count, int cumulativeScaleIndex, int partitionIndex) = 0; - + virtual int removeScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex) = 0; @@ -184,18 +174,18 @@ class BeagleImpl int count, int cumulativeScaleIndex, int partitionIndex) = 0; - - virtual int resetScaleFactors(int cumulativeScalingIndex) = 0; + + virtual int resetScaleFactors(int cumulativeScalingIndex) = 0; virtual int resetScaleFactorsByPartition(int cumulativeScaleIndex, - int partitionIndex) = 0; - + int partitionIndex) = 0; + virtual int copyScaleFactors(int destScalingIndex, - int srcScalingIndex) = 0; - + int srcScalingIndex) = 0; + virtual int getScaleFactors(int srcScalingIndex, - double* scaleFactors) = 0; - + double* scaleFactors) = 0; + virtual int calculateRootLogLikelihoods(const int* bufferIndices, const int* categoryWeightsIndices, const int* stateFrequenciesIndices, @@ -263,14 +253,14 @@ class BeagleImpl double* outSumFirstDerivative, double* outSumSecondDerivativeByPartition, double* outSumSecondDerivative) = 0; - + virtual int getLogLikelihood(double* outSumLogLikelihood) = 0; virtual int getDerivatives(double* outSumFirstDerivative, double* outSumSecondDerivative) = 0; virtual int getSiteLogLikelihoods(double* outLogLikelihoods) = 0; - + virtual int getSiteDerivatives(double* outFirstDerivatives, double* outSecondDerivatives) = 0; //protected: @@ -293,9 +283,9 @@ class BeagleImplFactory { long preferenceFlags, long requirementFlags, int* errorCode) = 0; // pure virtual - + virtual const char* getName() = 0; // pure virtual - + virtual const long getFlags() = 0; // pure virtual }; diff --git a/libhmsbeagle/CPU/AVXDefinitions.h b/libhmsbeagle/CPU/AVXDefinitions.h index 92e070bf..1a72b61c 100644 --- a/libhmsbeagle/CPU/AVXDefinitions.h +++ b/libhmsbeagle/CPU/AVXDefinitions.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ diff --git a/libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.h b/libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.h index 1ae5954b..1b40a4ec 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.h +++ b/libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ @@ -48,11 +38,11 @@ namespace cpu { BEAGLE_CPU_TEMPLATE class BeagleCPU4StateAVXImpl : public BeagleCPU4StateImpl {}; - + BEAGLE_CPU_4_AVX_TEMPLATE class BeagleCPU4StateAVXImpl : public BeagleCPU4StateImpl { - + protected: using BeagleCPUImpl::kTipCount; using BeagleCPUImpl::gPartials; @@ -70,56 +60,56 @@ class BeagleCPU4StateAVXImpl : public BeagleCPU4StateImp using BeagleCPUImpl::realtypeMin; using BeagleCPUImpl::outLogLikelihoodsTmp; using BeagleCPUImpl::gPatternWeights; - -public: + +public: virtual const char* getName(); - + virtual const long getFlags(); - + protected: - virtual int getPaddedPatternsModulus(); - + virtual int getPaddedPatternsModulus(); + private: - + virtual void calcStatesStates(float* destP, const int* states1, const float* matrices1, const int* states2, const float* matrices2); - + virtual void calcStatesPartials(float* destP, const int* states1, const float* __restrict matrices1, const float* __restrict partials2, const float* __restrict matrices2); - + virtual void calcStatesPartialsFixedScaling(float* destP, const int* states1, const float* __restrict matrices1, const float* __restrict partials2, const float* __restrict matrices2, const float* __restrict scaleFactors); - + virtual void calcPartialsPartials(float* __restrict destP, const float* __restrict partials1, const float* __restrict matrices1, const float* __restrict partials2, const float* __restrict matrices2); - + virtual void calcPartialsPartialsFixedScaling(float* __restrict destP, const float* __restrict child0Partials, const float* __restrict child0TransMat, const float* __restrict child1Partials, const float* __restrict child1TransMat, const float* __restrict scaleFactors); - + virtual void calcPartialsPartialsAutoScaling(float* __restrict destP, const float* __restrict partials1, const float* __restrict matrices1, const float* __restrict partials2, const float* __restrict matrices2, int* activateScaling); - + virtual int calcEdgeLogLikelihoods(const int parentBufferIndex, const int childBufferIndex, const int probabilityIndex, @@ -127,13 +117,13 @@ class BeagleCPU4StateAVXImpl : public BeagleCPU4StateImp const int stateFrequenciesIndex, const int scalingFactorsIndex, double* outSumLogLikelihood); - + }; - + BEAGLE_CPU_4_AVX_TEMPLATE class BeagleCPU4StateAVXImpl : public BeagleCPU4StateImpl { - + protected: using BeagleCPUImpl::kTipCount; using BeagleCPUImpl::gPartials; @@ -151,56 +141,56 @@ class BeagleCPU4StateAVXImpl : public BeagleCPU4StateIm using BeagleCPUImpl::realtypeMin; using BeagleCPUImpl::outLogLikelihoodsTmp; using BeagleCPUImpl::gPatternWeights; - + public: virtual const char* getName(); - + virtual const long getFlags(); - + protected: virtual int getPaddedPatternsModulus(); - + private: - + virtual void calcStatesStates(double* destP, const int* states1, const double* matrices1, const int* states2, const double* matrices2); - + virtual void calcStatesPartials(double* destP, const int* states1, const double* __restrict matrices1, const double* __restrict partials2, const double* __restrict matrices2); - + virtual void calcStatesPartialsFixedScaling(double* destP, const int* states1, const double* __restrict matrices1, const double* __restrict partials2, const double* __restrict matrices2, const double* __restrict scaleFactors); - + virtual void calcPartialsPartials(double* __restrict destP, const double* __restrict partials1, const double* __restrict matrices1, const double* __restrict partials2, const double* __restrict matrices2); - + virtual void calcPartialsPartialsFixedScaling(double* __restrict destP, const double* __restrict child0Partials, const double* __restrict child0TransMat, const double* __restrict child1Partials, const double* __restrict child1TransMat, const double* __restrict scaleFactors); - + virtual void calcPartialsPartialsAutoScaling(double* __restrict destP, const double* __restrict partials1, const double* __restrict matrices1, const double* __restrict partials2, const double* __restrict matrices2, int* activateScaling); - + virtual int calcEdgeLogLikelihoods(const int parentBufferIndex, const int childBufferIndex, const int probabilityIndex, @@ -208,10 +198,10 @@ class BeagleCPU4StateAVXImpl : public BeagleCPU4StateIm const int stateFrequenciesIndex, const int scalingFactorsIndex, double* outSumLogLikelihood); - + }; - - + + BEAGLE_CPU_FACTORY_TEMPLATE class BeagleCPU4StateAVXImplFactory : public BeagleImplFactory { public: diff --git a/libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.hpp b/libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.hpp index 592b2f17..7b7f3cb8 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPU4StateAVXImpl.hpp @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ @@ -109,7 +99,7 @@ inline const char* getBeagleCPU4StateAVXName(){ return "CPU-4State-AVX-D template<> inline const char* getBeagleCPU4StateAVXName(){ return "CPU-4State-AVX-Single"; }; - + /* * Calculates partial likelihoods at a node when both children have states. */ @@ -349,7 +339,7 @@ void BeagleCPU4StateAVXImpl::calcPartialsPartials(doubl // fprintf(stderr,"APM\n"); for (int k = 0; k < kPatternCount; k++) { - + # if 1 && !defined(_WIN32) __builtin_prefetch (&partials_q[v+64]); __builtin_prefetch (&partials_r[v+64]); @@ -462,7 +452,7 @@ void BeagleCPU4StateAVXImpl::calcPartialsPartialsFixedS __builtin_prefetch (&partials_r[v+64]); // __builtin_prefetch (destPvec+32,1,0); # endif - + // Prefetch scale factor // const V_Real scaleFactor = VEC_LOAD_SCALAR(scaleFactors + k); // Option below appears faster, why? @@ -507,7 +497,7 @@ void BeagleCPU4StateAVXImpl::calcPartialsPartialsFixedS } } - + BEAGLE_CPU_4_AVX_TEMPLATE void BeagleCPU4StateAVXImpl::calcPartialsPartialsAutoScaling(float* destP, const float* partials_q, @@ -538,7 +528,7 @@ void BeagleCPU4StateAVXImpl::calcPartialsPartialsAutoSc matrices_r, activateScaling); } - + BEAGLE_CPU_4_AVX_TEMPLATE int BeagleCPU4StateAVXImpl::calcEdgeLogLikelihoods(const int parIndex, const int childIndex, @@ -677,7 +667,7 @@ int BeagleCPU4StateAVXImpl::calcEdgeLogLikelihoods(cons if (*outSumLogLikelihood != *outSumLogLikelihood) returnCode = BEAGLE_ERROR_FLOATING_POINT; - + return returnCode; } @@ -688,7 +678,7 @@ int BeagleCPU4StateAVXImpl::getPaddedPatternsModulus() { // return 4; // For single-precision, can operate on 4 patterns at a time // TODO Vectorize final log operations over patterns } - + BEAGLE_CPU_4_AVX_TEMPLATE int BeagleCPU4StateAVXImpl::getPaddedPatternsModulus() { // return 2; // For double-precision, can operate on 2 patterns at a time @@ -705,7 +695,7 @@ const char* BeagleCPU4StateAVXImpl::getName() { return getBeagleCPU4StateAVXName(); } - + BEAGLE_CPU_4_AVX_TEMPLATE const long BeagleCPU4StateAVXImpl::getFlags() { return BEAGLE_FLAG_COMPUTATION_SYNCH | @@ -740,7 +730,7 @@ BeagleImpl* BeagleCPU4StateAVXImplFactory::createImp int categoryCount, int scaleBufferCount, int resourceNumber, - int pluginResourceNumber, + int pluginResourceNumber, long preferenceFlags, long requirementFlags, int* errorCode) { @@ -791,7 +781,7 @@ const long BeagleCPU4StateAVXImplFactory::getFlags() { BEAGLE_FLAG_SCALERS_LOG | BEAGLE_FLAG_SCALERS_RAW | BEAGLE_FLAG_EIGEN_COMPLEX | BEAGLE_FLAG_EIGEN_REAL| BEAGLE_FLAG_INVEVEC_STANDARD | BEAGLE_FLAG_INVEVEC_TRANSPOSED | - BEAGLE_FLAG_FRAMEWORK_CPU; + BEAGLE_FLAG_FRAMEWORK_CPU; } template <> @@ -805,7 +795,7 @@ const long BeagleCPU4StateAVXImplFactory::getFlags() { BEAGLE_FLAG_SCALERS_LOG | BEAGLE_FLAG_SCALERS_RAW | BEAGLE_FLAG_EIGEN_COMPLEX | BEAGLE_FLAG_EIGEN_REAL | BEAGLE_FLAG_INVEVEC_STANDARD | BEAGLE_FLAG_INVEVEC_TRANSPOSED | - BEAGLE_FLAG_FRAMEWORK_CPU; + BEAGLE_FLAG_FRAMEWORK_CPU; } diff --git a/libhmsbeagle/CPU/BeagleCPU4StateImpl.h b/libhmsbeagle/CPU/BeagleCPU4StateImpl.h index a099c4f8..56c4ce8e 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateImpl.h +++ b/libhmsbeagle/CPU/BeagleCPU4StateImpl.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Andrew Rambaut * @author Marc Suchard @@ -87,7 +77,7 @@ class BeagleCPU4StateImpl : public BeagleCPUImpl { const REALTYPE* matrices2, int startPattern, int endPattern); - + virtual void calcStatesPartials(REALTYPE* destP, const int* states1, const REALTYPE* matrices1, @@ -95,7 +85,7 @@ class BeagleCPU4StateImpl : public BeagleCPUImpl { const REALTYPE* matrices2, int startPattern, int endPattern); - + virtual void calcPartialsPartials(REALTYPE* destP, const REALTYPE* partials1, const REALTYPE* matrices1, @@ -170,14 +160,14 @@ class BeagleCPU4StateImpl : public BeagleCPUImpl { const int* partitionIndices, int partitionCount, double* outSumLogLikelihoodByPartition); - + virtual int calcRootLogLikelihoodsMulti(const int* bufferIndices, const int* categoryWeightsIndices, const int* stateFrequenciesIndices, const int* scaleBufferIndices, int count, - double* outSumLogLikelihood); - + double* outSumLogLikelihood); + virtual int calcEdgeLogLikelihoods(const int parentBufferIndex, const int childBufferIndex, const int probabilityIndex, @@ -195,7 +185,7 @@ class BeagleCPU4StateImpl : public BeagleCPUImpl { const int* partitionIndices, int partitionCount, double* outSumLogLikelihoodByPartition); - + virtual void calcStatesStatesFixedScaling(REALTYPE *destP, const int *child0States, const REALTYPE *child0TransMat, @@ -222,15 +212,15 @@ class BeagleCPU4StateImpl : public BeagleCPUImpl { const REALTYPE *scaleFactors, int startPattern, int endPattern); - + virtual void calcPartialsPartialsAutoScaling(REALTYPE *destP, const REALTYPE *child0Partials, const REALTYPE *child0TransMat, const REALTYPE *child1Partials, const REALTYPE *child1TransMat, int *activateScaling); - - + + inline int integrateOutStatesAndScale(const REALTYPE* integrationTmp, const int stateFrequenciesIndex, const int scalingFactorsIndex, diff --git a/libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp b/libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp index 00141917..e6988194 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp @@ -7,19 +7,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Andrew Rambaut * @author Marc Suchard @@ -215,13 +205,13 @@ void BeagleCPU4StateImpl::calcStatesStates(REALTYPE* destP, const int state1 = states1[k]; const int state2 = states2[k]; - destP[v ] = matrices1[w + state1] * + destP[v ] = matrices1[w + state1] * matrices2[w + state2]; - destP[v + 1] = matrices1[w + OFFSET*1 + state1] * + destP[v + 1] = matrices1[w + OFFSET*1 + state1] * matrices2[w + OFFSET*1 + state2]; - destP[v + 2] = matrices1[w + OFFSET*2 + state1] * + destP[v + 2] = matrices1[w + OFFSET*2 + state1] * matrices2[w + OFFSET*2 + state2]; - destP[v + 3] = matrices1[w + OFFSET*3 + state1] * + destP[v + 3] = matrices1[w + OFFSET*3 + state1] * matrices2[w + OFFSET*3 + state2]; v += 4; } @@ -246,20 +236,20 @@ void BeagleCPU4StateImpl::calcStatesStatesFixedScaling(REALT } int w = l*4*OFFSET; - + for (int k = startPattern; k < endPattern; k++) { - + const int state1 = states1[k]; const int state2 = states2[k]; const REALTYPE scaleFactor = scaleFactors[k]; - - destP[v ] = matrices1[w + state1] * + + destP[v ] = matrices1[w + state1] * matrices2[w + state2] / scaleFactor; - destP[v + 1] = matrices1[w + OFFSET*1 + state1] * + destP[v + 1] = matrices1[w + OFFSET*1 + state1] * matrices2[w + OFFSET*1 + state2] / scaleFactor; - destP[v + 2] = matrices1[w + OFFSET*2 + state1] * + destP[v + 2] = matrices1[w + OFFSET*2 + state1] * matrices2[w + OFFSET*2 + state2] / scaleFactor; - destP[v + 3] = matrices1[w + OFFSET*3 + state1] * + destP[v + 3] = matrices1[w + OFFSET*3 + state1] * matrices2[w + OFFSET*3 + state2] / scaleFactor; v += 4; } @@ -286,22 +276,22 @@ void BeagleCPU4StateImpl::calcStatesPartials(REALTYPE* destP } int w = l*4*OFFSET; - + PREFETCH_MATRIX(2,matrices2,w); - + for (int k = startPattern; k < endPattern; k++) { - + const int state1 = states1[k]; - + PREFETCH_PARTIALS(2,partials2,u); - + DO_INTEGRATION(2); // defines sum20, sum21, sum22, sum23; - + destP[u ] = matrices1[w + state1] * sum20; destP[u + 1] = matrices1[w + OFFSET*1 + state1] * sum21; destP[u + 2] = matrices1[w + OFFSET*2 + state1] * sum22; destP[u + 3] = matrices1[w + OFFSET*3 + state1] * sum23; - + u += 4; } } @@ -325,26 +315,26 @@ void BeagleCPU4StateImpl::calcStatesPartialsFixedScaling(REA } int w = l*4*OFFSET; - + PREFETCH_MATRIX(2,matrices2,w); - + for (int k = startPattern; k < endPattern; k++) { - + const int state1 = states1[k]; const REALTYPE scaleFactor = scaleFactors[k]; - + PREFETCH_PARTIALS(2,partials2,u); - + DO_INTEGRATION(2); // defines sum20, sum21, sum22, sum23 - + destP[u ] = matrices1[w + state1] * sum20 / scaleFactor; destP[u + 1] = matrices1[w + OFFSET*1 + state1] * sum21 / scaleFactor; destP[u + 2] = matrices1[w + OFFSET*2 + state1] * sum22 / scaleFactor; destP[u + 3] = matrices1[w + OFFSET*3 + state1] * sum23 / scaleFactor; - - u += 4; + + u += 4; } - } + } } BEAGLE_CPU_TEMPLATE @@ -355,8 +345,8 @@ void BeagleCPU4StateImpl::calcPartialsPartials(REALTYPE* des const REALTYPE* matrices2, int startPattern, int endPattern) { - - + + #pragma omp parallel for num_threads(kCategoryCount) for (int l = 0; l < kCategoryCount; l++) { int u = l*4*kPaddedPatternCount; @@ -364,16 +354,16 @@ void BeagleCPU4StateImpl::calcPartialsPartials(REALTYPE* des u += 4*startPattern; } int w = l*4*OFFSET; - - PREFETCH_MATRIX(1,matrices1,w); + + PREFETCH_MATRIX(1,matrices1,w); PREFETCH_MATRIX(2,matrices2,w); - for (int k = startPattern; k < endPattern; k++) { + for (int k = startPattern; k < endPattern; k++) { PREFETCH_PARTIALS(1,partials1,u); PREFETCH_PARTIALS(2,partials2,u); - + DO_INTEGRATION(1); // defines sum10, sum11, sum12, sum13 DO_INTEGRATION(2); // defines sum20, sum21, sum22, sum23 - + // Final results destP[u ] = sum10 * sum20; destP[u + 1] = sum11 * sum21; @@ -477,28 +467,28 @@ void BeagleCPU4StateImpl::calcPartialsPartialsAutoScaling(RE const REALTYPE* partials2, const REALTYPE* matrices2, int* activateScaling) { - - + + #pragma omp parallel for num_threads(kCategoryCount) for (int l = 0; l < kCategoryCount; l++) { int u = l*4*kPaddedPatternCount; int w = l*4*OFFSET; - - PREFETCH_MATRIX(1,matrices1,w); + + PREFETCH_MATRIX(1,matrices1,w); PREFETCH_MATRIX(2,matrices2,w); - for (int k = 0; k < kPatternCount; k++) { + for (int k = 0; k < kPatternCount; k++) { PREFETCH_PARTIALS(1,partials1,u); PREFETCH_PARTIALS(2,partials2,u); - + DO_INTEGRATION(1); // defines sum10, sum11, sum12, sum13 DO_INTEGRATION(2); // defines sum20, sum21, sum22, sum23 - + // Final results destP[u ] = sum10 * sum20; destP[u + 1] = sum11 * sum21; destP[u + 2] = sum12 * sum22; destP[u + 3] = sum13 * sum23; - + if (*activateScaling == 0) { int expTmp; int expMax; @@ -517,13 +507,13 @@ void BeagleCPU4StateImpl::calcPartialsPartialsAutoScaling(RE *activateScaling = 1; } } - + u += 4; - + } } } - + BEAGLE_CPU_TEMPLATE void BeagleCPU4StateImpl::calcPartialsPartialsFixedScaling(REALTYPE* destP, @@ -543,30 +533,30 @@ void BeagleCPU4StateImpl::calcPartialsPartialsFixedScaling(R } int w = l*4*OFFSET; - + PREFETCH_MATRIX(1,matrices1,w); PREFETCH_MATRIX(2,matrices2,w); - + for (int k = startPattern; k < endPattern; k++) { - + // Prefetch scale factor const REALTYPE scaleFactor = scaleFactors[k]; - + PREFETCH_PARTIALS(1,partials1,u); PREFETCH_PARTIALS(2,partials2,u); - + DO_INTEGRATION(1); // defines sum10, sum11, sum12, sum13 DO_INTEGRATION(2); // defines sum20, sum21, sum22, sum23 - + // Final results destP[u ] = sum10 * sum20 / scaleFactor; destP[u + 1] = sum11 * sum21 / scaleFactor; destP[u + 2] = sum12 * sum22 / scaleFactor; destP[u + 3] = sum13 * sum23 / scaleFactor; - + u += 4; } - } + } } BEAGLE_CPU_TEMPLATE @@ -574,15 +564,15 @@ int inline BeagleCPU4StateImpl::integrateOutStatesAndScale(c const int stateFrequenciesIndex, const int scalingFactorsIndex, double* outSumLogLikelihood) { - + int returnCode = BEAGLE_SUCCESS; - + REALTYPE freq0, freq1, freq2, freq3; - freq0 = gStateFrequencies[stateFrequenciesIndex][0]; + freq0 = gStateFrequencies[stateFrequenciesIndex][0]; freq1 = gStateFrequencies[stateFrequenciesIndex][1]; freq2 = gStateFrequencies[stateFrequenciesIndex][2]; freq3 = gStateFrequencies[stateFrequenciesIndex][3]; - + int u = 0; for(int k = 0; k < kPatternCount; k++) { REALTYPE sumOverI = @@ -590,11 +580,11 @@ int inline BeagleCPU4StateImpl::integrateOutStatesAndScale(c freq1 * integrationTmp[u + 1] + freq2 * integrationTmp[u + 2] + freq3 * integrationTmp[u + 3]; - + u += 4; - + outLogLikelihoodsTmp[k] = log(sumOverI); - } + } if (scalingFactorsIndex != BEAGLE_OP_NONE) { const REALTYPE* scalingFactors = gScaleBuffers[scalingFactorsIndex]; @@ -602,15 +592,15 @@ int inline BeagleCPU4StateImpl::integrateOutStatesAndScale(c outLogLikelihoodsTmp[k] += scalingFactors[k]; } } - - *outSumLogLikelihood = 0.0; + + *outSumLogLikelihood = 0.0; for(int k=0; k < kPatternCount; k++) { *outSumLogLikelihood += outLogLikelihoodsTmp[k] * gPatternWeights[k]; - } - + } + if (*outSumLogLikelihood != *outSumLogLikelihood) returnCode = BEAGLE_ERROR_FLOATING_POINT; - + return returnCode; } @@ -622,7 +612,7 @@ void inline BeagleCPU4StateImpl::integrateOutStatesAndScaleB const int* partitionIndices, int partitionCount, double* outSumLogLikelihoodByPartition) { - + for (int p = 0; p < partitionCount; p++) { int pIndex = partitionIndices[p]; @@ -633,7 +623,7 @@ void inline BeagleCPU4StateImpl::integrateOutStatesAndScaleB const int scalingFactorsIndex = cumulativeScaleIndices[p]; REALTYPE freq0, freq1, freq2, freq3; - freq0 = gStateFrequencies[stateFrequenciesIndex][0]; + freq0 = gStateFrequencies[stateFrequenciesIndex][0]; freq1 = gStateFrequencies[stateFrequenciesIndex][1]; freq2 = gStateFrequencies[stateFrequenciesIndex][2]; freq3 = gStateFrequencies[stateFrequenciesIndex][3]; @@ -645,11 +635,11 @@ void inline BeagleCPU4StateImpl::integrateOutStatesAndScaleB freq1 * integrationTmp[u + 1] + freq2 * integrationTmp[u + 2] + freq3 * integrationTmp[u + 3]; - + u += 4; - + outLogLikelihoodsTmp[k] = log(sumOverI); - } + } if (scalingFactorsIndex != BEAGLE_OP_NONE) { const REALTYPE* scalingFactors = gScaleBuffers[scalingFactorsIndex]; @@ -657,13 +647,13 @@ void inline BeagleCPU4StateImpl::integrateOutStatesAndScaleB outLogLikelihoodsTmp[k] += scalingFactors[k]; } } - + outSumLogLikelihoodByPartition[p] = 0.0; for(int k=startPattern; k < endPattern; k++) { outSumLogLikelihoodByPartition[p] += outLogLikelihoodsTmp[k] * gPatternWeights[k]; } } - + } #define FAST_MAX(x,y) (x > y ? x : y) @@ -680,7 +670,7 @@ void BeagleCPU4StateImpl::rescalePartials(REALTYPE* destP, bool useLogScalars = kFlags & BEAGLE_FLAG_SCALERS_LOG; for (int k = 0; k < kPatternCount; k++) { - REALTYPE max = 0; + REALTYPE max = 0; const int patternOffset = k * 4; for (int l = 0; l < kCategoryCount; l++) { int offset = l * kPaddedPatternCount * 4 + patternOffset; @@ -738,7 +728,7 @@ void BeagleCPU4StateImpl::rescalePartialsByPartition( int endPattern = gPatternPartitionsStartPatterns[partitionIndex + 1]; for (int k = startPattern; k < endPattern; k++) { - REALTYPE max = 0; + REALTYPE max = 0; const int patternOffset = k * 4; for (int l = 0; l < kCategoryCount; l++) { int offset = l * kPaddedPatternCount * 4 + patternOffset; @@ -792,73 +782,73 @@ int BeagleCPU4StateImpl::calcEdgeLogLikelihoods(const int pa const int scalingFactorsIndex, double* outSumLogLikelihood) { // TODO: implement derivatives for calculateEdgeLnL - + assert(parIndex >= kTipCount); - + const REALTYPE* partialsParent = gPartials[parIndex]; const REALTYPE* transMatrix = gTransitionMatrices[probIndex]; const REALTYPE* wt = gCategoryWeights[categoryWeightsIndex]; - + memset(integrationTmp, 0, (kPatternCount * kStateCount)*sizeof(REALTYPE)); - + if (childIndex < kTipCount && gTipStates[childIndex]) { // Integrate against a state at the child - - const int* statesChild = gTipStates[childIndex]; + + const int* statesChild = gTipStates[childIndex]; int v = 0; // Index for parent partials int w = 0; for(int l = 0; l < kCategoryCount; l++) { int u = 0; // Index in resulting product-partials (summed over categories) const REALTYPE weight = wt[l]; for(int k = 0; k < kPatternCount; k++) { - - const int stateChild = statesChild[k]; - - integrationTmp[u ] += transMatrix[w + stateChild] * partialsParent[v ] * weight; + + const int stateChild = statesChild[k]; + + integrationTmp[u ] += transMatrix[w + stateChild] * partialsParent[v ] * weight; integrationTmp[u + 1] += transMatrix[w + OFFSET*1 + stateChild] * partialsParent[v + 1] * weight; integrationTmp[u + 2] += transMatrix[w + OFFSET*2 + stateChild] * partialsParent[v + 2] * weight; integrationTmp[u + 3] += transMatrix[w + OFFSET*3 + stateChild] * partialsParent[v + 3] * weight; - + u += 4; - v += 4; + v += 4; } w += OFFSET*4; if (kExtraPatterns) v += 4 * kExtraPatterns; } - + } else { // Integrate against a partial at the child - + const REALTYPE* partialsChild = gPartials[childIndex]; #if 0// int v = 0; #endif int w = 0; - for(int l = 0; l < kCategoryCount; l++) { + for(int l = 0; l < kCategoryCount; l++) { int u = 0; #if 1// int v = l*kPaddedPatternCount*4; #endif const REALTYPE weight = wt[l]; - + PREFETCH_MATRIX(1,transMatrix,w); - - for(int k = 0; k < kPatternCount; k++) { - + + for(int k = 0; k < kPatternCount; k++) { + const REALTYPE* partials1 = partialsChild; - + PREFETCH_PARTIALS(1,partials1,v); - + DO_INTEGRATION(1); - + integrationTmp[u ] += sum10 * partialsParent[v ] * weight; integrationTmp[u + 1] += sum11 * partialsParent[v + 1] * weight; integrationTmp[u + 2] += sum12 * partialsParent[v + 2] * weight; integrationTmp[u + 3] += sum13 * partialsParent[v + 3] * weight; - + u += 4; v += 4; - } + } w += OFFSET*4; #if 0// if (kExtraPatterns) @@ -884,7 +874,7 @@ void BeagleCPU4StateImpl::calcEdgeLogLikelihoodsByPartition( for (int p = 0; p < partitionCount; p++) { int pIndex = partitionIndices[p]; - + int startPattern = gPatternPartitionsStartPatterns[pIndex]; int endPattern = gPatternPartitionsStartPatterns[pIndex + 1]; @@ -896,30 +886,30 @@ void BeagleCPU4StateImpl::calcEdgeLogLikelihoodsByPartition( const int categoryWeightsIndex = categoryWeightsIndices[p]; assert(parIndex >= kTipCount); - + const REALTYPE* partialsParent = gPartials[parIndex]; const REALTYPE* transMatrix = gTransitionMatrices[probIndex]; const REALTYPE* wt = gCategoryWeights[categoryWeightsIndex]; - + if (childIndex < kTipCount && gTipStates[childIndex]) { // Integrate against a state at the child - - const int* statesChild = gTipStates[childIndex]; + + const int* statesChild = gTipStates[childIndex]; int v = startPattern * 4; // Index for parent partials int w = 0; for(int l = 0; l < kCategoryCount; l++) { int u = startPattern * 4; // Index in resulting product-partials (summed over categories) const REALTYPE weight = wt[l]; for(int k = startPattern; k < endPattern; k++) { - - const int stateChild = statesChild[k]; - - integrationTmp[u ] += transMatrix[w + stateChild] * partialsParent[v ] * weight; + + const int stateChild = statesChild[k]; + + integrationTmp[u ] += transMatrix[w + stateChild] * partialsParent[v ] * weight; integrationTmp[u + 1] += transMatrix[w + OFFSET*1 + stateChild] * partialsParent[v + 1] * weight; integrationTmp[u + 2] += transMatrix[w + OFFSET*2 + stateChild] * partialsParent[v + 2] * weight; integrationTmp[u + 3] += transMatrix[w + OFFSET*3 + stateChild] * partialsParent[v + 3] * weight; - + u += 4; - v += 4; + v += 4; } w += OFFSET*4; if (kExtraPatterns) @@ -932,31 +922,31 @@ void BeagleCPU4StateImpl::calcEdgeLogLikelihoodsByPartition( int v = 0; #endif int w = 0; - for(int l = 0; l < kCategoryCount; l++) { + for(int l = 0; l < kCategoryCount; l++) { int u = startPattern * 4; #if 1// int v = l*kPaddedPatternCount*4 + startPattern * 4; #endif const REALTYPE weight = wt[l]; - + PREFETCH_MATRIX(1,transMatrix,w); - - for(int k = startPattern; k < endPattern; k++) { - + + for(int k = startPattern; k < endPattern; k++) { + const REALTYPE* partials1 = partialsChild; - + PREFETCH_PARTIALS(1,partials1,v); - + DO_INTEGRATION(1); - + integrationTmp[u ] += sum10 * partialsParent[v ] * weight; integrationTmp[u + 1] += sum11 * partialsParent[v + 1] * weight; integrationTmp[u + 2] += sum12 * partialsParent[v + 2] * weight; integrationTmp[u + 3] += sum13 * partialsParent[v + 3] * weight; - + u += 4; v += 4; - } + } w += OFFSET*4; #if 0// if (kExtraPatterns) @@ -999,13 +989,13 @@ int BeagleCPU4StateImpl::calcRootLogLikelihoods(const int bu integrationTmp[u + 1] += rootPartials[v + 1] * wtl; integrationTmp[u + 2] += rootPartials[v + 2] * wtl; integrationTmp[u + 3] += rootPartials[v + 3] * wtl; - + u += 4; v += 4; } v += 4 * kExtraPatterns; } - + return integrateOutStatesAndScale(integrationTmp, stateFrequenciesIndex, scalingFactorsIndex, outSumLogLikelihood); } @@ -1276,14 +1266,14 @@ void BeagleCPU4StateImpl::calcRootLogLikelihoodsByPartition( for (int p = 0; p < partitionCount; p++) { int pIndex = partitionIndices[p]; - + int startPattern = gPatternPartitionsStartPatterns[pIndex]; int endPattern = gPatternPartitionsStartPatterns[pIndex + 1]; const REALTYPE* rootPartials = gPartials[bufferIndices[p]]; assert(rootPartials); const REALTYPE* wt = gCategoryWeights[categoryWeightsIndices[p]]; - + int v = startPattern * 4; const REALTYPE wt0 = wt[0]; for (int k = startPattern; k < endPattern; k++) { @@ -1302,7 +1292,7 @@ void BeagleCPU4StateImpl::calcRootLogLikelihoodsByPartition( integrationTmp[u + 1] += rootPartials[v + 1] * wtl; integrationTmp[u + 2] += rootPartials[v + 2] * wtl; integrationTmp[u + 3] += rootPartials[v + 3] * wtl; - + u += 4; v += 4; } @@ -1319,12 +1309,12 @@ int BeagleCPU4StateImpl::calcRootLogLikelihoodsMulti(const i const int* scaleBufferIndices, int count, double* outSumLogLikelihood) { - + int returnCode = BEAGLE_SUCCESS; - + std::vector indexMaxScale(kPatternCount); std::vector maxScaleFactor(kPatternCount); - + for (int subsetIndex = 0 ; subsetIndex < count; ++subsetIndex ) { const int rootPartialIndex = bufferIndices[subsetIndex]; const REALTYPE* rootPartials = gPartials[rootPartialIndex]; @@ -1332,7 +1322,7 @@ int BeagleCPU4StateImpl::calcRootLogLikelihoodsMulti(const i const REALTYPE* wt = gCategoryWeights[categoryWeightsIndices[subsetIndex]]; int u = 0; int v = 0; - + const REALTYPE wt0 = wt[0]; for (int k = 0; k < kPatternCount; k++) { integrationTmp[v ] = rootPartials[v ] * wt0; @@ -1349,88 +1339,88 @@ int BeagleCPU4StateImpl::calcRootLogLikelihoodsMulti(const i integrationTmp[u + 1] += rootPartials[v + 1] * wtl; integrationTmp[u + 2] += rootPartials[v + 2] * wtl; integrationTmp[u + 3] += rootPartials[v + 3] * wtl; - + u += 4; v += 4; } v += 4 * kExtraPatterns; } - + REALTYPE freq0, freq1, freq2, freq3; - freq0 = frequencies[0]; + freq0 = frequencies[0]; freq1 = frequencies[1]; freq2 = frequencies[2]; freq3 = frequencies[3]; - + u = 0; for (int k = 0; k < kPatternCount; k++) { - REALTYPE sum = + REALTYPE sum = freq0 * integrationTmp[u ] + freq1 * integrationTmp[u + 1] + freq2 * integrationTmp[u + 2] + freq3 * integrationTmp[u + 3]; - - u += 4; - + + u += 4; + // TODO: allow only some subsets to have scale indices if (scaleBufferIndices[0] != BEAGLE_OP_NONE || (kFlags & BEAGLE_FLAG_SCALING_ALWAYS)) { int cumulativeScalingFactorIndex; if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) - cumulativeScalingFactorIndex = rootPartialIndex - kTipCount; + cumulativeScalingFactorIndex = rootPartialIndex - kTipCount; else cumulativeScalingFactorIndex = scaleBufferIndices[subsetIndex]; - + const REALTYPE* cumulativeScaleFactors = gScaleBuffers[cumulativeScalingFactorIndex]; - + if (subsetIndex == 0) { indexMaxScale[k] = 0; maxScaleFactor[k] = cumulativeScaleFactors[k]; for (int j = 1; j < count; j++) { REALTYPE tmpScaleFactor; if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) - tmpScaleFactor = gScaleBuffers[bufferIndices[j] - kTipCount][k]; + tmpScaleFactor = gScaleBuffers[bufferIndices[j] - kTipCount][k]; else tmpScaleFactor = gScaleBuffers[scaleBufferIndices[j]][k]; - + if (tmpScaleFactor > maxScaleFactor[k]) { indexMaxScale[k] = j; maxScaleFactor[k] = tmpScaleFactor; } } } - + if (subsetIndex != indexMaxScale[k]) sum *= exp((REALTYPE)(cumulativeScaleFactors[k] - maxScaleFactor[k])); } - + if (subsetIndex == 0) { outLogLikelihoodsTmp[k] = sum; } else if (subsetIndex == count - 1) { REALTYPE tmpSum = outLogLikelihoodsTmp[k] + sum; - + outLogLikelihoodsTmp[k] = log(tmpSum); } else { outLogLikelihoodsTmp[k] += sum; } } } - + if (scaleBufferIndices[0] != BEAGLE_OP_NONE || (kFlags & BEAGLE_FLAG_SCALING_ALWAYS)) { for(int i=0; i::getName() { @@ -1511,7 +1501,7 @@ const long BeagleCPU4StateImplFactory::getFlags() { BEAGLE_FLAG_INVEVEC_STANDARD | BEAGLE_FLAG_INVEVEC_TRANSPOSED | BEAGLE_FLAG_PREORDER_TRANSPOSE_MANUAL | BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO | BEAGLE_FLAG_FRAMEWORK_CPU; - + if (DOUBLE_PRECISION) flags |= BEAGLE_FLAG_PRECISION_DOUBLE; else diff --git a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h index 98bbceaa..d1881311 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h +++ b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ @@ -48,11 +38,11 @@ namespace cpu { BEAGLE_CPU_TEMPLATE class BeagleCPU4StateSSEImpl : public BeagleCPU4StateImpl {}; - + BEAGLE_CPU_4_SSE_TEMPLATE class BeagleCPU4StateSSEImpl : public BeagleCPU4StateImpl { - + protected: using BeagleCPUImpl::kTipCount; using BeagleCPUImpl::gPartials; @@ -71,56 +61,56 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateImp using BeagleCPUImpl::outLogLikelihoodsTmp; using BeagleCPUImpl::gPatternWeights; using BeagleCPUImpl::gPatternPartitionsStartPatterns; - -public: + +public: virtual const char* getName(); - + virtual const long getFlags(); - + protected: - virtual int getPaddedPatternsModulus(); - + virtual int getPaddedPatternsModulus(); + private: - + virtual void calcStatesStates(float* destP, const int* states1, const float* matrices1, const int* states2, const float* matrices2); - + virtual void calcStatesPartials(float* destP, const int* states1, const float* __restrict matrices1, const float* __restrict partials2, const float* __restrict matrices2); - + virtual void calcStatesPartialsFixedScaling(float* destP, const int* states1, const float* __restrict matrices1, const float* __restrict partials2, const float* __restrict matrices2, const float* __restrict scaleFactors); - + virtual void calcPartialsPartials(float* __restrict destP, const float* __restrict partials1, const float* __restrict matrices1, const float* __restrict partials2, const float* __restrict matrices2); - + virtual void calcPartialsPartialsFixedScaling(float* __restrict destP, const float* __restrict child0Partials, const float* __restrict child0TransMat, const float* __restrict child1Partials, const float* __restrict child1TransMat, const float* __restrict scaleFactors); - + virtual void calcPartialsPartialsAutoScaling(float* __restrict destP, const float* __restrict partials1, const float* __restrict matrices1, const float* __restrict partials2, const float* __restrict matrices2, int* activateScaling); - + virtual int calcEdgeLogLikelihoods(const int parentBufferIndex, const int childBufferIndex, const int probabilityIndex, @@ -140,11 +130,11 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateImp double* outSumLogLikelihoodByPartition); }; - + BEAGLE_CPU_4_SSE_TEMPLATE class BeagleCPU4StateSSEImpl : public BeagleCPU4StateImpl { - + protected: using BeagleCPUImpl::kTipCount; using BeagleCPUImpl::gPartials; @@ -169,9 +159,9 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm public: virtual const char* getName(); - + virtual const long getFlags(); - + protected: virtual int getPaddedPatternsModulus(); @@ -195,7 +185,7 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm void accumulateDerivativesImpl(double* outDerivatives, double* outSumDerivatives, double* outSumSquaredDerivatives); - + virtual void calcStatesStates(double* destP, const int* states1, const double* matrices1, @@ -203,7 +193,7 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm const double* matrices2, int startPattern, int endPattern); - + virtual void calcStatesPartials(double* destP, const int* states1, const double* __restrict matrices1, @@ -211,7 +201,7 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm const double* __restrict matrices2, int startPattern, int endPattern); - + virtual void calcStatesPartialsFixedScaling(double* destP, const int* states1, const double* __restrict matrices1, @@ -220,7 +210,7 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm const double* __restrict scaleFactors, int startPattern, int endPattern); - + virtual void calcPartialsPartials(double* __restrict destP, const double* __restrict partials1, const double* __restrict matrices1, @@ -281,7 +271,7 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm double *outDerivatives, double *outSumDerivatives, double *outSumSquaredDerivatives); - + virtual void calcPartialsPartialsFixedScaling(double* __restrict destP, const double* __restrict child0Partials, const double* __restrict child0TransMat, @@ -290,14 +280,14 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm const double* __restrict scaleFactors, int startPattern, int endPattern); - + virtual void calcPartialsPartialsAutoScaling(double* __restrict destP, const double* __restrict partials1, const double* __restrict matrices1, const double* __restrict partials2, const double* __restrict matrices2, int* activateScaling); - + virtual int calcEdgeLogLikelihoods(const int parentBufferIndex, const int childBufferIndex, const int probabilityIndex, @@ -315,10 +305,10 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm const int* partitionIndices, int partitionCount, double* outSumLogLikelihoodByPartition); - + }; - - + + BEAGLE_CPU_FACTORY_TEMPLATE class BeagleCPU4StateSSEImplFactory : public BeagleImplFactory { public: diff --git a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp index 3207e314..ab342175 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author David Swofford diff --git a/libhmsbeagle/CPU/BeagleCPUAVXImpl.h b/libhmsbeagle/CPU/BeagleCPUAVXImpl.h index b4d21805..ee320999 100644 --- a/libhmsbeagle/CPU/BeagleCPUAVXImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUAVXImpl.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ @@ -79,7 +69,7 @@ class BeagleCPUAVXImpl : public BeagleCPUImpl : public BeagleCPUImpl : public BeagleCPUImpl : public BeagleCPUImpl { @@ -153,7 +143,7 @@ class BeagleCPUAVXImpl : public BeagleCPUImpl : public BeagleCPUImpl : public BeagleCPUImpl. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ @@ -344,7 +334,7 @@ void BeagleCPUAVXImpl::calcPartialsPartialsFixedScaling( // } } - + BEAGLE_CPU_AVX_TEMPLATE void BeagleCPUAVXImpl::calcPartialsPartialsAutoScaling(double* destP, const double* partials_q, @@ -375,7 +365,7 @@ void BeagleCPUAVXImpl::calcPartialsPartialsAutoScaling(do // matrices_r, // activateScaling); //} - + BEAGLE_CPU_AVX_TEMPLATE int BeagleCPUAVXImpl::calcEdgeLogLikelihoods(const int parIndex, const int childIndex, @@ -511,7 +501,7 @@ return BeagleCPUImpl::calcEdgeLogLikelihoods( // for (int i = 0; i < kPatternCount; i++) { // *outSumLogLikelihood += outLogLikelihoodsTmp[i] * gPatternWeights[i]; // } -// +// // if (!(*outSumLogLikelihood - *outSumLogLikelihood == 0.0)) // returnCode = BEAGLE_ERROR_FLOATING_POINT; // @@ -522,7 +512,7 @@ BEAGLE_CPU_AVX_TEMPLATE int BeagleCPUAVXImpl::getPaddedPatternsModulus() { return 1; // We currently do not vectorize across patterns } - + BEAGLE_CPU_AVX_TEMPLATE const char* BeagleCPUAVXImpl::getName() { return getBeagleCPUAVXName(); @@ -532,7 +522,7 @@ BEAGLE_CPU_AVX_TEMPLATE const char* BeagleCPUAVXImpl::getName() { return getBeagleCPUAVXName(); } - + BEAGLE_CPU_AVX_TEMPLATE const long BeagleCPUAVXImpl::getFlags() { return BEAGLE_FLAG_COMPUTATION_SYNCH | @@ -566,19 +556,19 @@ BeagleImpl* BeagleCPUAVXImplFactory::createImpl(int int categoryCount, int scaleBufferCount, int resourceNumber, - int pluginResourceNumber, + int pluginResourceNumber, long preferenceFlags, long requirementFlags, int* errorCode) { if (!CPUSupportsAVX()) return NULL; - + if (stateCount & 1) { // is odd BeagleCPUAVXImpl* impl = new BeagleCPUAVXImpl(); - - + + try { if (impl->createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, patternCount, eigenBufferCount, matrixBufferCount, @@ -591,8 +581,8 @@ BeagleImpl* BeagleCPUAVXImplFactory::createImpl(int delete impl; throw; } - - delete impl; + + delete impl; } else { BeagleCPUAVXImpl* impl = new BeagleCPUAVXImpl(); @@ -613,7 +603,7 @@ BeagleImpl* BeagleCPUAVXImplFactory::createImpl(int delete impl; } - + return NULL; } @@ -634,7 +624,7 @@ const long BeagleCPUAVXImplFactory::getFlags() { BEAGLE_FLAG_EIGEN_COMPLEX | BEAGLE_FLAG_EIGEN_REAL | BEAGLE_FLAG_INVEVEC_STANDARD | BEAGLE_FLAG_INVEVEC_TRANSPOSED | BEAGLE_FLAG_PREORDER_TRANSPOSE_MANUAL | BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO | - BEAGLE_FLAG_FRAMEWORK_CPU; + BEAGLE_FLAG_FRAMEWORK_CPU; } template <> @@ -649,7 +639,7 @@ const long BeagleCPUAVXImplFactory::getFlags() { BEAGLE_FLAG_EIGEN_COMPLEX | BEAGLE_FLAG_EIGEN_REAL | BEAGLE_FLAG_INVEVEC_STANDARD | BEAGLE_FLAG_INVEVEC_TRANSPOSED | BEAGLE_FLAG_PREORDER_TRANSPOSE_MANUAL | BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO | - BEAGLE_FLAG_FRAMEWORK_CPU; + BEAGLE_FLAG_FRAMEWORK_CPU; } // Code to save: diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index f06c4852..8efb86b1 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Andrew Rambaut * @author Marc Suchard diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 5281bc76..8ff9d0a5 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Andrew Rambaut * @author Marc Suchard diff --git a/libhmsbeagle/CPU/BeagleCPUSSEImpl.h b/libhmsbeagle/CPU/BeagleCPUSSEImpl.h index 7cefb781..0ef8f4be 100644 --- a/libhmsbeagle/CPU/BeagleCPUSSEImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUSSEImpl.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ @@ -91,7 +81,7 @@ class BeagleCPUSSEImpl : public BeagleCPUImpl : public BeagleCPUImpl : public BeagleCPUImpl : public BeagleCPUImpl { @@ -165,7 +155,7 @@ class BeagleCPUSSEImpl : public BeagleCPUImpl : public BeagleCPUImpl : public BeagleCPUImpl. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author David Swofford @@ -302,9 +292,9 @@ void BeagleCPUSSEImpl::calcPartialsPartials(double* __res BEAGLE_CPU_SSE_TEMPLATE void BeagleCPUSSEImpl::calcPartialsPartialsFixedScaling(double* __restrict destP, - const double* __restrict partials1, - const double* __restrict matrices1, - const double* __restrict partials2, + const double* __restrict partials1, + const double* __restrict matrices1, + const double* __restrict partials2, const double* __restrict matrices2, const double* __restrict scaleFactors, int startPattern, @@ -358,7 +348,7 @@ void BeagleCPUSSEImpl::calcPartialsPartialsFixedScaling(d } } - + BEAGLE_CPU_SSE_TEMPLATE void BeagleCPUSSEImpl::calcPartialsPartialsAutoScaling(double* destP, const double* partials_q, @@ -389,7 +379,7 @@ void BeagleCPUSSEImpl::calcPartialsPartialsAutoScaling(do // matrices_r, // activateScaling); //} - + BEAGLE_CPU_SSE_TEMPLATE int BeagleCPUSSEImpl::calcEdgeLogLikelihoods(const int parIndex, const int childIndex, @@ -525,7 +515,7 @@ return BeagleCPUImpl::calcEdgeLogLikelihoods( // for (int i = 0; i < kPatternCount; i++) { // *outSumLogLikelihood += outLogLikelihoodsTmp[i] * gPatternWeights[i]; // } -// +// // if (!(*outSumLogLikelihood - *outSumLogLikelihood == 0.0)) // returnCode = BEAGLE_ERROR_FLOATING_POINT; // @@ -536,7 +526,7 @@ BEAGLE_CPU_SSE_TEMPLATE int BeagleCPUSSEImpl::getPaddedPatternsModulus() { return 1; // We currently do not vectorize across patterns } - + BEAGLE_CPU_SSE_TEMPLATE const char* BeagleCPUSSEImpl::getName() { return getBeagleCPUSSEName(); @@ -546,7 +536,7 @@ BEAGLE_CPU_SSE_TEMPLATE const char* BeagleCPUSSEImpl::getName() { return getBeagleCPUSSEName(); } - + BEAGLE_CPU_SSE_TEMPLATE const long BeagleCPUSSEImpl::getFlags() { return BEAGLE_FLAG_COMPUTATION_SYNCH | @@ -587,17 +577,17 @@ BeagleImpl* BeagleCPUSSEImplFactory::createImpl(int if (!CPUSupportsSSE()) return NULL; - + if (stateCount & 1) { // is odd BeagleCPUSSEImpl* impl = new BeagleCPUSSEImpl(); - - + + try { if (impl->createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, patternCount, eigenBufferCount, matrixBufferCount, - categoryCount,scaleBufferCount, resourceNumber, - pluginResourceNumber, + categoryCount,scaleBufferCount, resourceNumber, + pluginResourceNumber, preferenceFlags, requirementFlags) == 0) return impl; } @@ -607,8 +597,8 @@ BeagleImpl* BeagleCPUSSEImplFactory::createImpl(int delete impl; throw; } - - delete impl; + + delete impl; } else { BeagleCPUSSEImpl* impl = new BeagleCPUSSEImpl(); @@ -618,7 +608,7 @@ BeagleImpl* BeagleCPUSSEImplFactory::createImpl(int if (impl->createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, patternCount, eigenBufferCount, matrixBufferCount, categoryCount,scaleBufferCount, resourceNumber, - pluginResourceNumber, + pluginResourceNumber, preferenceFlags, requirementFlags) == 0) return impl; } @@ -631,7 +621,7 @@ BeagleImpl* BeagleCPUSSEImplFactory::createImpl(int delete impl; } - + return NULL; } diff --git a/libhmsbeagle/CPU/SSEDefinitions.h b/libhmsbeagle/CPU/SSEDefinitions.h index 5e0e771c..0b5cc44f 100644 --- a/libhmsbeagle/CPU/SSEDefinitions.h +++ b/libhmsbeagle/CPU/SSEDefinitions.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 53130862..16e32f08 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -5,21 +5,10 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . - * - * * @brief GPU implementation header * * @author Marc Suchard @@ -66,11 +55,11 @@ class BeagleGPUImpl : public BeagleImpl { private: GPUInterface* gpu; KernelLauncher* kernels; - + int kInitialized; - + long kFlags; - + int kTipCount; int kPartialsBufferCount; int kCompactBufferCount; @@ -79,17 +68,17 @@ class BeagleGPUImpl : public BeagleImpl { int kEigenDecompCount; int kMatrixCount; int kCategoryCount; - + int kTipPartialsBufferCount; int kInternalPartialsBufferCount; int kBufferCount; int kScaleBufferCount; - + int kPaddedStateCount; int kPaddedPatternCount; // total # of patterns with padding so that kPaddedPatternCount // * kPaddedStateCount is a multiple of 16 int kSumSitesBlockCount; - + int kPartialsSize; int kMatrixSize; int kEigenValuesSize; @@ -99,53 +88,53 @@ class BeagleGPUImpl : public BeagleImpl { int kLastTipPartialsBufferIndex; int kResultPaddedPatterns; - + GPUPtr dIntegrationTmp; GPUPtr dOutFirstDeriv; GPUPtr dOutSecondDeriv; GPUPtr dPartialsTmp; GPUPtr dFirstDerivTmp; GPUPtr dSecondDerivTmp; - + GPUPtr dSumLogLikelihood; GPUPtr dSumFirstDeriv; GPUPtr dSumSecondDeriv; GPUPtr dMultipleDerivatives; GPUPtr dMultipleDerivativeSum; - - GPUPtr dPatternWeights; - + + GPUPtr dPatternWeights; + GPUPtr dBranchLengths; - + GPUPtr dDistanceQueue; - + GPUPtr dPtrQueue; GPUPtr dDerivativeQueue; - + GPUPtr dMaxScalingFactors; GPUPtr dIndexMaxScalingFactors; - + GPUPtr dAccumulatedScalingFactors; - + GPUPtr* dEigenValues; GPUPtr* dEvec; GPUPtr* dIevc; - + GPUPtr* dWeights; - GPUPtr* dFrequencies; + GPUPtr* dFrequencies; GPUPtr* dScalingFactors; - + GPUPtr* dStates; - + GPUPtr* dPartials; GPUPtr* dMatrices; - + GPUPtr* dCompactBuffers; GPUPtr* dTipPartialsBuffers; - + bool kUsingMultiGrid; bool kDerivBuffersInitialised; int kMultipleDerivativesLength; // TODO Change to length (max node count used) @@ -201,32 +190,32 @@ class BeagleGPUImpl : public BeagleImpl { unsigned int* hPtrQueue; unsigned int* hDerivativeQueue; - + double** hCategoryRates; // Can keep in double-precision Real* hPatternWeightsCache; - + Real* hDistanceQueue; - + Real* hWeightsCache; Real* hFrequenciesCache; Real* hLogLikelihoodsCache; Real* hPartialsCache; int* hStatesCache; Real* hMatrixCache; - + int* hRescalingTrigger; GPUPtr dRescalingTrigger; - + GPUPtr* dScalingFactorsMaster; - + int* hStreamIndices; -public: +public: BeagleGPUImpl(); - + virtual ~BeagleGPUImpl(); - + int createInstance(int tipCount, int partialsBufferCount, int compactBufferCount, @@ -240,7 +229,7 @@ class BeagleGPUImpl : public BeagleImpl { int pluginResourceNumber, long preferenceFlags, long requirementFlags); - + int getInstanceDetails(BeagleInstanceDetails* retunInfo); int setCPUThreadCount(int threadCount); @@ -254,47 +243,47 @@ class BeagleGPUImpl : public BeagleImpl { int setRootPrePartials(const int* bufferIndices, const int* stateFrequenciesIndices, int count); - + int setPartials(int bufferIndex, const double* inPartials); - + int getPartials(int bufferIndex, int scaleIndex, double* outPartials); - + int setEigenDecomposition(int eigenIndex, const double* inEigenVectors, const double* inInverseEigenVectors, const double* inEigenValues); - + int setStateFrequencies(int stateFrequenciesIndex, - const double* inStateFrequencies); - + const double* inStateFrequencies); + int setCategoryWeights(int categoryWeightsIndex, const double* inCategoryWeights); - + int setPatternWeights(const double* inPatternWeights); int setPatternPartitions(int partitionCount, const int* inPatternPartitions); - + int setCategoryRates(const double* inCategoryRates); int setCategoryRatesWithIndex(int categoryRatesIndex, const double* inCategoryRates); - + int setTransitionMatrix(int matrixIndex, const double* inMatrix, double paddedValue); int setDifferentialMatrix(int matrixIndex, const double* inMatrix); - + int setTransitionMatrices(const int* matrixIndices, const double* inMatrices, const double* paddedValues, - int count); - + int count); + int getTransitionMatrix(int matrixIndex, double* outMatrix); @@ -333,7 +322,7 @@ class BeagleGPUImpl : public BeagleImpl { const int* secondDerivativeIndices, const double* edgeLengths, int count); - + int updatePartials(const int* operations, int operationCount, int cumulativeScalingIndex); @@ -374,7 +363,7 @@ class BeagleGPUImpl : public BeagleImpl { int waitForPartials(const int* destinationPartials, int destinationPartialsCount); - + int accumulateScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex); @@ -383,7 +372,7 @@ class BeagleGPUImpl : public BeagleImpl { int count, int cumulativeScalingIndex, int partitionIndex); - + int removeScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex); @@ -392,17 +381,17 @@ class BeagleGPUImpl : public BeagleImpl { int count, int cumulativeScalingIndex, int partitionIndex); - + int resetScaleFactors(int cumulativeScalingIndex); int resetScaleFactorsByPartition(int cumulativeScalingIndex, int partitionIndex); - + int copyScaleFactors(int destScalingIndex, int srcScalingIndex); - + int getScaleFactors(int srcScalingIndex, - double* scaleFactors); - + double* scaleFactors); + int calculateRootLogLikelihoods(const int* bufferIndices, const int* categoryWeightsIndices, const int* stateFrequenciesIndices, @@ -419,7 +408,7 @@ class BeagleGPUImpl : public BeagleImpl { int count, double* outSumLogLikelihoodByPartition, double* outSumLogLikelihood); - + int calculateEdgeLogLikelihoods(const int* parentBufferIndices, const int* childBufferIndices, const int* probabilityIndices, @@ -466,7 +455,7 @@ class BeagleGPUImpl : public BeagleImpl { double* outSumSecondDerivative); int getSiteLogLikelihoods(double* outLogLikelihoods); - + int getSiteDerivatives(double* outFirstDerivatives, double* outSecondDerivatives); diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 82875ee7..1cc10d80 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -7,19 +7,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Andrew Rambaut @@ -726,7 +716,7 @@ int BeagleGPUImpl::createInstance(int tipCount, const int distanceQueueLength = std::max( kMatrixCount * kCategoryCount * 2, // for transition matrices kMatrixCount + kCategoryCount); // for cross-products - + dDistanceQueue = gpu->AllocateMemory(distanceQueueLength * sizeof(Real)); hDistanceQueue = (Real*) gpu->MallocHost(distanceQueueLength * sizeof(Real)); checkHostMemory(hDistanceQueue); @@ -754,13 +744,13 @@ int BeagleGPUImpl::createInstance(int tipCount, kUsingMultiGrid = false; - if (kPaddedStateCount == 4 && + if (kPaddedStateCount == 4 && (kDeviceType == BEAGLE_FLAG_PROCESSOR_CPU || #ifdef FW_OPENCL kDeviceCode == BEAGLE_OPENCL_DEVICE_AMD_GPU || -#endif - kPaddedPatternCount < BEAGLE_MULTI_GRID_MAX || - kFlags & BEAGLE_FLAG_PARALLELOPS_GRID) && +#endif + kPaddedPatternCount < BEAGLE_MULTI_GRID_MAX || + kFlags & BEAGLE_FLAG_PARALLELOPS_GRID) && !(kFlags & BEAGLE_FLAG_PARALLELOPS_STREAMS)) { kUsingMultiGrid = true; allocateMultiGridBuffers(); @@ -2225,19 +2215,19 @@ int BeagleGPUImpl::calculateEdgeDerivative(const int *postBu #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUImpl::calculateEdgeDerivative\n"); #endif - + if (dOutFirstDeriv == (GPUPtr)NULL) { dOutFirstDeriv = gpu->AllocateMemory(kPaddedPatternCount * kBufferCount * 2 * sizeof(Real)); } int returnCode = BEAGLE_ERROR_GENERAL; - + returnCode = calcEdgeFirstDerivatives(postBufferIndices, preBufferIndices, firstDerivativeIndices, &categoryWeightsIndex, cumulativeScaleIndices, count, outFirstDerivative, NULL, NULL); if (outDiagonalSecondDerivative != NULL) { - int diagonalSecondDerivativeReturnCode = + int diagonalSecondDerivativeReturnCode = calcEdgeFirstDerivatives(postBufferIndices, preBufferIndices, secondDerivativeIndices, &categoryWeightsIndex, cumulativeScaleIndices, count, @@ -3051,8 +3041,8 @@ int BeagleGPUImpl::accumulateScaleFactors(const int* scaling gpu->MemcpyHostToDevice(dPtrQueue, hPtrQueue, sizeof(unsigned int) * count); kernels->AccumulateFactorsAutoScaling(dScalingFactors[0], dPtrQueue, dAccumulatedScalingFactors, count, kPaddedPatternCount, kScaleBufferSize); - - } else { + + } else { for(int n = 0; n < count; n++) { hPtrQueue[n] = scalingIndices[n] * kScaleBufferSize; } @@ -4500,13 +4490,13 @@ int BeagleGPUImpl::calcCrossProducts(const int *postBufferIn } gpu->MemcpyHostToDevice(dDerivativeQueue, hDerivativeQueue, sizeof(unsigned int) * 2 * totalCount); - + const double* categoryRates = hCategoryRates[0]; // TODO parameterize index const GPUPtr categoryWeights = dWeights[0]; int lengthCount = 0; - for (int i = 0; i < totalCount; i++) { - hDistanceQueue[lengthCount++] = (Real) edgeLengths[i]; + for (int i = 0; i < totalCount; i++) { + hDistanceQueue[lengthCount++] = (Real) edgeLengths[i]; } for (int i = 0; i < kCategoryCount; i++) { hDistanceQueue[lengthCount++] = (Real) categoryRates[i]; @@ -4525,14 +4515,14 @@ int BeagleGPUImpl::calcCrossProducts(const int *postBufferIn dMultipleDerivatives, dStatesOrigin, dPartialsOrigin, - dDistanceQueue, + dDistanceQueue, dDerivativeQueue, - categoryWeights, dPatternWeights, - 0, statesTipsCount, totalCount, + categoryWeights, dPatternWeights, + 0, statesTipsCount, totalCount, kPaddedPatternCount, kCategoryCount, accumulate, nodeBlocks, patternBlocks); - accumulate = true; + accumulate = true; } kernels->PartialsPartialsCrossProducts( @@ -4558,7 +4548,7 @@ int BeagleGPUImpl::calcCrossProducts(const int *postBufferIn } for (int i = 0; i < kStateCount; ++i) { - beagleMemCpy(outCrossProducts + i * kStateCount, + beagleMemCpy(outCrossProducts + i * kStateCount, hTmp.data() + i * kPaddedStateCount, kStateCount); } diff --git a/libhmsbeagle/GPU/GPUImplDefs.h b/libhmsbeagle/GPU/GPUImplDefs.h index 86eeb18c..5a88619c 100644 --- a/libhmsbeagle/GPU/GPUImplDefs.h +++ b/libhmsbeagle/GPU/GPUImplDefs.h @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Daniel Ayres diff --git a/libhmsbeagle/GPU/GPUImplHelper.cpp b/libhmsbeagle/GPU/GPUImplHelper.cpp index 886080d6..6e505700 100644 --- a/libhmsbeagle/GPU/GPUImplHelper.cpp +++ b/libhmsbeagle/GPU/GPUImplHelper.cpp @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Daniel Ayres diff --git a/libhmsbeagle/GPU/GPUImplHelper.h b/libhmsbeagle/GPU/GPUImplHelper.h index 7dc97c12..67406f6b 100644 --- a/libhmsbeagle/GPU/GPUImplHelper.h +++ b/libhmsbeagle/GPU/GPUImplHelper.h @@ -5,19 +5,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Daniel Ayres diff --git a/libhmsbeagle/GPU/GPUInterface.h b/libhmsbeagle/GPU/GPUInterface.h index 5c4490c1..cc8a3e13 100644 --- a/libhmsbeagle/GPU/GPUInterface.h +++ b/libhmsbeagle/GPU/GPUInterface.h @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Dat Huynh @@ -82,7 +72,7 @@ class GPUInterface { CUevent cudaEvent; const char* GetCUDAErrorDescription(int errorCode); #elif defined(FW_OPENCL) - cl_device_id openClDeviceId; // compute device id + cl_device_id openClDeviceId; // compute device id cl_context openClContext; // compute context cl_command_queue* openClCommandQueues; // compute command queue cl_event* openClEvents; // compute events @@ -93,30 +83,30 @@ class GPUInterface { public: GPUInterface(); - + ~GPUInterface(); - + int Initialize(); int GetDeviceCount(); - void SetDevice(int deviceNumber, - int paddedStateCount, - int categoryCount, + void SetDevice(int deviceNumber, + int paddedStateCount, + int categoryCount, int patternCount, int unpaddedPatternCount, int tipCount, long flags); - + void ResizeStreamCount(int newStreamCount); void SynchronizeHost(); void SynchronizeDevice(); void SynchronizeDeviceWithIndex(int streamRecordIndex, int streamWaitIndex); - + GPUFunction GetFunction(const char* functionName); - + void LaunchKernel(GPUFunction deviceFunction, Dim3Int block, Dim3Int grid, @@ -134,13 +124,13 @@ class GPUInterface { ...); // parameters void* MallocHost(size_t memSize); - + void* CallocHost(size_t size, size_t length); - + void* AllocatePinnedHostMemory(size_t memSize, bool writeCombined, bool mapped); - + #ifdef FW_OPENCL void* MapMemory(GPUPtr dPtr, size_t memSize); @@ -150,7 +140,7 @@ class GPUInterface { #endif GPUPtr AllocateMemory(size_t memSize); - + GPUPtr AllocateRealMemory(size_t length); GPUPtr AllocateIntMemory(size_t length); @@ -158,7 +148,7 @@ class GPUInterface { GPUPtr CreateSubPointer(GPUPtr dPtr, size_t offset, size_t size); size_t AlignMemOffset(size_t offset); - + void MemsetShort(GPUPtr dest, unsigned short val, size_t count); @@ -170,28 +160,28 @@ class GPUInterface { void MemcpyDeviceToHost(void* dest, const GPUPtr src, size_t memSize); - + void MemcpyDeviceToDevice(GPUPtr dest, GPUPtr src, size_t memSize); void FreeHostMemory(void* hPtr); - + void FreePinnedHostMemory(void* hPtr); - + void FreeMemory(GPUPtr dPtr); GPUPtr GetDeviceHostPointer(void* hPtr); - + size_t GetAvailableMemory(); - + void GetDeviceName(int deviceNumber, char* deviceName, int nameLength); - + void GetDeviceDescription(int deviceNumber, char* deviceDescription); - + long GetDeviceTypeFlag(int deviceNumber); BeagleDeviceImplementationCodes GetDeviceImplementationCode(int deviceNumber); @@ -202,11 +192,11 @@ class GPUInterface { void PrintfDeviceVector(GPUPtr dPtr, int length, Real r) { PrintfDeviceVector(dPtr,length,-1, 0, r); } - + template void PrintfDeviceVector(GPUPtr dPtr, int length, double checkValue, Real r); - + template void PrintfDeviceVector(GPUPtr dPtr, int length, @@ -245,7 +235,7 @@ class GPUInterface { void PrintfDeviceInt(GPUPtr dPtr, int length); - + KernelResource* kernelResource; #ifdef BEAGLE_DEBUG_OPENCL_CORES @@ -253,11 +243,11 @@ class GPUInterface { void ReleaseDevice(int deviceNumber); #endif - + protected: void InitializeKernelResource(int paddedStateCount, bool doublePrecision); - + std::map* resourceMap; bool supportDoublePrecision; diff --git a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp index 570c2500..e77a5612 100644 --- a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp +++ b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Daniel Ayres diff --git a/libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp b/libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp index 25fdb62b..2b68493a 100644 --- a/libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp +++ b/libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Daniel Ayres */ diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index e42ecdf1..3f192223 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Daniel Ayres @@ -80,10 +70,10 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { kMatrixBlockSize = gpu->kernelResource->matrixBlockSize; kSumSitesBlockSize = SUM_SITES_BLOCK_SIZE; kFlags = gpu->kernelResource->flags; - + // Set up block/grid for transition matrices computation bgTransitionProbabilitiesBlock = Dim3Int(kMultiplyBlockSize, kMultiplyBlockSize); - bgTransitionProbabilitiesGrid = Dim3Int(kPaddedStateCount/kMultiplyBlockSize, + bgTransitionProbabilitiesGrid = Dim3Int(kPaddedStateCount/kMultiplyBlockSize, kPaddedStateCount/kMultiplyBlockSize); if(kPaddedStateCount % kMultiplyBlockSize != 0) { bgTransitionProbabilitiesGrid.x += 1; @@ -117,7 +107,7 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { if (!kCPUImplementation && (kPatternCount % kPatternBlockSize != 0)) { bgPeelingGrid.x += 1; } - } + } // Set up block/grid for likelihood computation if (kPaddedStateCount == 4) { @@ -176,8 +166,8 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgAccumulateGrid = Dim3Int(kPatternCount / kPatternBlockSize); if (kPatternCount % kPatternBlockSize != 0) - bgAccumulateGrid.x += 1; - + bgAccumulateGrid.x += 1; + // Set up block/grid for scaling partials if (kCPUImplementation) { bgScaleBlock = Dim3Int(kPatternBlockSize); @@ -185,28 +175,28 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { } else { if (kSlowReweighing) { bgScaleBlock = Dim3Int(kPaddedStateCount); - bgScaleGrid = Dim3Int(kPatternCount); + bgScaleGrid = Dim3Int(kPatternCount); } else { if (kPaddedStateCount == 4) { bgScaleBlock = Dim3Int(16, kMatrixBlockSize); bgScaleGrid = Dim3Int(kPatternCount / 4, kCategoryCount/kMatrixBlockSize); if (kPatternCount % 4 != 0) { - bgScaleGrid.x += 1; // + bgScaleGrid.x += 1; // fprintf(stderr,"PATTERNS SHOULD BE PADDED! Inform Marc, please.\n"); exit(-1); - } - } else { + } + } else { bgScaleBlock = Dim3Int(kPaddedStateCount, kMatrixBlockSize); bgScaleGrid = Dim3Int(kPatternCount, kCategoryCount/kMatrixBlockSize); } if (kCategoryCount % kMatrixBlockSize != 0) bgScaleGrid.y += 1; - if (bgScaleGrid.y > 1) { + if (bgScaleGrid.y > 1) { fprintf(stderr, "Not yet implemented! Try slow reweighing.\n"); exit(-1); - } + } } - } + } // Set up block for site likelihood accumulation if (kCPUImplementation) { @@ -233,7 +223,7 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgReorderPatternsBlock = Dim3Int(REORDER_BLOCK_SIZE_CPU); } else { bgReorderPatternsBlock = Dim3Int(kPaddedStateCount, REORDER_BLOCK_SIZE); - } + } bgReorderPatternsGrid = Dim3Int((kUnpaddedPatternCount + REORDER_BLOCK_SIZE - 1) / REORDER_BLOCK_SIZE, kCategoryCount); } @@ -259,9 +249,9 @@ void KernelLauncher::LoadKernels() { } fMatrixMulADBFirstDeriv = gpu->GetFunction("kernelMatrixMulADBFirstDeriv"); - + fMatrixMulADBSecondDeriv = gpu->GetFunction("kernelMatrixMulADBSecondDeriv"); - + if (kFlags & BEAGLE_FLAG_EIGEN_COMPLEX) { fMatrixMulADB = gpu->GetFunction("kernelMatrixMulADBComplex"); } else { @@ -269,11 +259,11 @@ void KernelLauncher::LoadKernels() { } fPartialsPartialsByPatternBlockCoherent = gpu->GetFunction( - "kernelPartialsPartialsNoScale"); - + "kernelPartialsPartialsNoScale"); + fPartialsPartialsByPatternBlockFixedScaling = gpu->GetFunction( "kernelPartialsPartialsFixedScale"); - + fPartialsPartialsByPatternBlockAutoScaling = gpu->GetFunction( "kernelPartialsPartialsAutoScale"); @@ -310,11 +300,11 @@ void KernelLauncher::LoadKernels() { fPartialsPartialsByPatternBlockCheckScaling = gpu->GetFunction( "kernelPartialsPartialsCheckScale"); - + fPartialsPartialsByPatternBlockFixedCheckScaling = gpu->GetFunction( "kernelPartialsPartialsFixedCheckScale"); } - + fStatesPartialsByPatternBlockCoherent = gpu->GetFunction( "kernelStatesPartialsNoScale"); @@ -340,16 +330,16 @@ void KernelLauncher::LoadKernels() { fStatesPartialsEdgeLikelihoodsSecondDeriv = gpu->GetFunction( "kernelStatesPartialsEdgeLikelihoodsSecondDeriv"); - + fIntegrateLikelihoodsDynamicScalingSecondDeriv = gpu->GetFunction( "kernelIntegrateLikelihoodsFixedScaleSecondDeriv"); - + if (kFlags & BEAGLE_FLAG_SCALING_AUTO) { fIntegrateLikelihoodsDynamicScaling = gpu->GetFunction("kernelIntegrateLikelihoodsAutoScaling"); } else { fIntegrateLikelihoodsDynamicScaling = gpu->GetFunction("kernelIntegrateLikelihoodsFixedScale"); } - + if (kFlags & BEAGLE_FLAG_SCALERS_LOG) { fAccumulateFactorsDynamicScaling = gpu->GetFunction("kernelAccumulateFactorsScalersLog"); fRemoveFactorsDynamicScaling = gpu->GetFunction("kernelRemoveFactorsScalersLog"); @@ -382,10 +372,10 @@ void KernelLauncher::LoadKernels() { fPartialsDynamicScaling = gpu->GetFunction( "kernelPartialsDynamicScalingSlow"); fPartialsDynamicScalingAccumulate = gpu->GetFunction( - "kernelPartialsDynamicScalingAccumulate"); // TODO Write kernel + "kernelPartialsDynamicScalingAccumulate"); // TODO Write kernel } } - + if (kPaddedStateCount == 4) { // TODO Temporary fPartialsDynamicScalingAccumulateDifference = gpu->GetFunction( "kernelPartialsDynamicScalingAccumulateDifference"); @@ -393,15 +383,15 @@ void KernelLauncher::LoadKernels() { fPartialsDynamicScalingAccumulateReciprocal = gpu->GetFunction( "kernelPartialsDynamicScalingAccumulateReciprocal"); } - + fIntegrateLikelihoods = gpu->GetFunction("kernelIntegrateLikelihoods"); - + fIntegrateLikelihoodsSecondDeriv = gpu->GetFunction("kernelIntegrateLikelihoodsSecondDeriv"); - + fIntegrateLikelihoodsMulti = gpu->GetFunction("kernelIntegrateLikelihoodsMulti"); - + fIntegrateLikelihoodsFixedScaleMulti = gpu->GetFunction("kernelIntegrateLikelihoodsFixedScaleMulti"); - + fSumSites1 = gpu->GetFunction("kernelSumSites1"); fSumSites2 = gpu->GetFunction("kernelSumSites2"); fSumSites3 = gpu->GetFunction("kernelSumSites3"); @@ -409,12 +399,12 @@ void KernelLauncher::LoadKernels() { fReorderPatterns = gpu->GetFunction("kernelReorderPatterns"); // partitioning and multi-op kernels - if (kPaddedStateCount == 4) { + if (kPaddedStateCount == 4) { fPartialsPartialsByPatternBlockCoherentMulti = gpu->GetFunction( "kernelPartialsPartialsNoScaleMulti"); fPartialsPartialsByPatternBlockCoherentPartition = gpu->GetFunction( - "kernelPartialsPartialsNoScalePartition"); + "kernelPartialsPartialsNoScalePartition"); fPartialsPartialsByPatternBlockFixedScalingMulti = gpu->GetFunction( "kernelPartialsPartialsFixedScaleMulti"); @@ -495,9 +485,9 @@ void KernelLauncher::LoadKernels() { "kernelPartialsDynamicScalingAccumulateByPartition"); } } - + fIntegrateLikelihoodsPartition = gpu->GetFunction("kernelIntegrateLikelihoodsPartition"); - + fSumSites1Partition = gpu->GetFunction("kernelSumSites1Partition"); } #endif // !FW_OPENCL_TESTING @@ -540,7 +530,7 @@ void KernelLauncher::ReorderPatterns(GPUPtr dPartials, dPartials, dStates, dStatesSort, dTipOffsets, dTipTypes, dPatternsNewOrder, dPatternWeights, dPatternWeightsSort, patternCount, paddedPatternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReorderPatterns\n"); #endif @@ -613,7 +603,7 @@ void KernelLauncher::GetTransitionProbabilitiesSquareMulti(GPUPtr dMatrices, bgTransitionProbabilitiesGrid.x *= totalMatrix; - // Transposed (interchanged Ievc and Evec) + // Transposed (interchanged Ievc and Evec) int parameterCountV = 6; int totalParameterCount = 9; gpu->LaunchKernel(fMatrixMulADBMulti, @@ -624,7 +614,7 @@ void KernelLauncher::GetTransitionProbabilitiesSquareMulti(GPUPtr dMatrices, totalMatrix); bgTransitionProbabilitiesGrid.x /= totalMatrix; // Reset value - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::GetTransitionProbabilitiesSquareMulti\n"); #endif @@ -643,7 +633,7 @@ void KernelLauncher::GetTransitionProbabilitiesSquare(GPUPtr dMatrices, bgTransitionProbabilitiesGrid.x *= totalMatrix; - // Transposed (interchanged Ievc and Evec) + // Transposed (interchanged Ievc and Evec) int parameterCountV = 6; int totalParameterCount = 9; gpu->LaunchKernel(fMatrixMulADB, @@ -655,7 +645,7 @@ void KernelLauncher::GetTransitionProbabilitiesSquare(GPUPtr dMatrices, bgTransitionProbabilitiesGrid.x /= totalMatrix; // Reset value - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::GetTransitionProbabilitiesSquare\n"); #endif @@ -671,10 +661,10 @@ void KernelLauncher::GetTransitionProbabilitiesSquareFirstDeriv(GPUPtr dMatrices #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::GetTransitionProbabilitiesSquareFirstDeriv\n"); #endif - + bgTransitionProbabilitiesGrid.x *= totalMatrix; - - // Transposed (interchanged Ievc and Evec) + + // Transposed (interchanged Ievc and Evec) int parameterCountV = 6; int totalParameterCount = 9; gpu->LaunchKernel(fMatrixMulADBFirstDeriv, @@ -683,10 +673,10 @@ void KernelLauncher::GetTransitionProbabilitiesSquareFirstDeriv(GPUPtr dMatrices dMatrices, dPtrQueue, dIevc, dEigenValues, dEvec, distanceQueue, kPaddedStateCount, kPaddedStateCount, totalMatrix); - + bgTransitionProbabilitiesGrid.x /= totalMatrix; // Reset value - - + + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::GetTransitionProbabilitiesSquareFirstDeriv\n"); #endif @@ -702,10 +692,10 @@ void KernelLauncher::GetTransitionProbabilitiesSquareSecondDeriv(GPUPtr dMatrice #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::GetTransitionProbabilitiesSquareSecondDeriv\n"); #endif - + bgTransitionProbabilitiesGrid.x *= totalMatrix; - - // Transposed (interchanged Ievc and Evec) + + // Transposed (interchanged Ievc and Evec) int parameterCountV = 6; int totalParameterCount = 9; gpu->LaunchKernel(fMatrixMulADBSecondDeriv, @@ -714,10 +704,10 @@ void KernelLauncher::GetTransitionProbabilitiesSquareSecondDeriv(GPUPtr dMatrice dMatrices, dPtrQueue, dIevc, dEigenValues, dEvec, distanceQueue, kPaddedStateCount, kPaddedStateCount, totalMatrix); - + bgTransitionProbabilitiesGrid.x /= totalMatrix; // Reset value - - + + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::GetTransitionProbabilitiesSquareSecondDeriv\n"); #endif @@ -802,7 +792,7 @@ void KernelLauncher::PartialsStatesCrossProducts(GPUPtr out, GPUPtr states0, GPUPtr partials, GPUPtr lengths, - GPUPtr instructions, + GPUPtr instructions, GPUPtr categoryWeights, GPUPtr patternWeights, unsigned int instructionOffset, @@ -828,14 +818,14 @@ void KernelLauncher::PartialsStatesCrossProducts(GPUPtr out, gpu->LaunchKernel(fPartialsStatesCrossProducts, block, grid, 7, 13, - out, states0, partials, lengths, instructions, + out, states0, partials, lengths, instructions, categoryWeights, patternWeights, - instructionOffset, + instructionOffset, patternCount, nodeCount, categoryCount, rateOffset, accumulate); gpu->SynchronizeDevice(); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsStatesCrossProducts\n"); @@ -845,7 +835,7 @@ void KernelLauncher::PartialsStatesCrossProducts(GPUPtr out, void KernelLauncher::PartialsPartialsCrossProducts(GPUPtr out, GPUPtr partials, GPUPtr lengths, - GPUPtr instructions, + GPUPtr instructions, GPUPtr categoryWeights, GPUPtr patternWeights, unsigned int instructionOffset, @@ -875,12 +865,12 @@ void KernelLauncher::PartialsPartialsCrossProducts(GPUPtr out, gpu->LaunchKernel(fPartialsPartialsCrossProducts, block, grid, 6, 12, - out, partials, lengths, instructions, + out, partials, lengths, instructions, categoryWeights, patternWeights, - instructionOffset, + instructionOffset, patternCount, nodeCount, categoryCount, rateOffset, accumulate); - - gpu->SynchronizeDevice(); + + gpu->SynchronizeDevice(); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsPartialsCrossProducts\n"); @@ -1025,7 +1015,7 @@ void KernelLauncher::PartialsPartialsPruningDynamicCheckScaling(GPUPtr partials1 bgPeelingBlock, bgPeelingGrid, 6, 7, partials1, partials2, partials3, matrices1, matrices2, dRescalingTrigger, - patternCount); + patternCount); gpu->SynchronizeDevice(); // printf("hRescalingTrigger (no factors) %d\n", *hRescalingTrigger); @@ -1037,13 +1027,13 @@ void KernelLauncher::PartialsPartialsPruningDynamicCheckScaling(GPUPtr partials1 dScalingFactors[writeScalingIndex] = gpu->AllocateMemory(patternCount * sizeReal); dScalingFactorsMaster[writeScalingIndex] = dScalingFactors[writeScalingIndex]; } - + if (dScalingFactors[cumulativeScalingIndex] != dScalingFactorsMaster[cumulativeScalingIndex]) { gpu->MemcpyDeviceToDevice(dScalingFactorsMaster[cumulativeScalingIndex], dScalingFactors[cumulativeScalingIndex], sizeReal *patternCount); gpu->SynchronizeDevice(); dScalingFactors[cumulativeScalingIndex] = dScalingFactorsMaster[cumulativeScalingIndex]; } - + gpu->LaunchKernel(fPartialsDynamicScalingAccumulateReciprocal, bgScaleBlock, bgScaleGrid, 3, 4, @@ -1052,31 +1042,31 @@ void KernelLauncher::PartialsPartialsPruningDynamicCheckScaling(GPUPtr partials1 } } else { *hRescalingTrigger = 0; - // Compute partials with known rescalings + // Compute partials with known rescalings gpu->LaunchKernel(fPartialsPartialsByPatternBlockFixedCheckScaling, bgPeelingBlock, bgPeelingGrid, 7, 8, partials1, partials2, partials3, matrices1, matrices2, dScalingFactors[readScalingIndex], dRescalingTrigger, - patternCount); - + patternCount); + gpu->SynchronizeDevice(); // printf("hRescalingTrigger (existing factors) %d\n", *hRescalingTrigger); if (*hRescalingTrigger) { // check if any partials need rescaling if (dScalingFactors[writeScalingIndex] != dScalingFactorsMaster[writeScalingIndex]) dScalingFactors[writeScalingIndex] = dScalingFactorsMaster[writeScalingIndex]; - + if (dScalingFactors[writeScalingIndex] == 0) { dScalingFactors[writeScalingIndex] = gpu->AllocateRealMemory(patternCount); dScalingFactorsMaster[writeScalingIndex] = dScalingFactors[writeScalingIndex]; } - + if (dScalingFactors[cumulativeScalingIndex] != dScalingFactorsMaster[cumulativeScalingIndex]) { gpu->MemcpyDeviceToDevice(dScalingFactorsMaster[cumulativeScalingIndex], dScalingFactors[cumulativeScalingIndex], sizeReal * patternCount); gpu->SynchronizeDevice(); dScalingFactors[cumulativeScalingIndex] = dScalingFactorsMaster[cumulativeScalingIndex]; } - + gpu->LaunchKernel(fPartialsDynamicScalingAccumulateDifference, bgScaleBlock, bgScaleGrid, 4, 5, @@ -1086,11 +1076,11 @@ void KernelLauncher::PartialsPartialsPruningDynamicCheckScaling(GPUPtr partials1 dScalingFactors[writeScalingIndex] = dScalingFactors[readScalingIndex]; } } - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsPartialsPruningDynamicCheckScaling\n"); #endif - + } void KernelLauncher::PartialsPartialsPruningMulti(GPUPtr partials, @@ -1163,9 +1153,9 @@ void KernelLauncher::PartialsPartialsPruningDynamicScaling(GPUPtr partials1, bgPeelingBlock, bgPeelingGrid, 6, 7, partials1, partials2, partials3, matrices1, matrices2, scalingFactors, - patternCount); + patternCount); } else if (doRescaling != 0) { - // Compute partials without any rescaling + // Compute partials without any rescaling if (endPattern != 0) { gpu->LaunchKernelConcurrent(fPartialsPartialsByPatternBlockCoherentPartition, @@ -1195,7 +1185,7 @@ void KernelLauncher::PartialsPartialsPruningDynamicScaling(GPUPtr partials1, startPattern, endPattern); } } - + } else { // Compute partials with known rescalings if (endPattern != 0) { @@ -1213,16 +1203,16 @@ void KernelLauncher::PartialsPartialsPruningDynamicScaling(GPUPtr partials1, 6, 7, partials1, partials2, partials3, matrices1, matrices2, scalingFactors, - patternCount); + patternCount); } } bgPeelingGrid.x = tmpGridx; - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsPartialsPruningDynamicScaling\n"); #endif - + } @@ -1253,7 +1243,7 @@ void KernelLauncher::StatesPartialsPruningMulti(GPUPtr states, bgPeelingBlock, bgPeelingGrid, 5, 7, states, partials, matrices, scalingFactors, ptrOffsets, - gridStartOp, patternCount); + gridStartOp, patternCount); } bgPeelingGrid.x = tmpGridx; @@ -1307,7 +1297,7 @@ void KernelLauncher::StatesPartialsPruningDynamicScaling(GPUPtr states1, int tmpGridx = bgPeelingGrid.x; bgPeelingGrid.x = (launchPatternCount + blockPatternCount - 1) / blockPatternCount; } - + if (doRescaling != 0) { // Compute partials without any rescaling if (endPattern != 0) { @@ -1326,7 +1316,7 @@ void KernelLauncher::StatesPartialsPruningDynamicScaling(GPUPtr states1, states1, partials2, partials3, matrices1, matrices2, patternCount); } - + // Rescale partials and save scaling factors if (doRescaling > 0) { if (endPattern == 0 ) { @@ -1350,7 +1340,7 @@ void KernelLauncher::StatesPartialsPruningDynamicScaling(GPUPtr states1, } } - } else { + } else { // Compute partials with known rescalings #ifdef BEAGLE_FILL_4_STATE_SCALAR_SP @@ -1404,11 +1394,11 @@ void KernelLauncher::StatesPartialsPruningDynamicScaling(GPUPtr states1, bgPeelingGrid.x = gridX; } #endif - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr,"\t\tLeaving KernelLauncher::StatesPartialsPruningDynamicScaling\n"); #endif - + } void KernelLauncher::StatesStatesPruningMulti(GPUPtr states, @@ -1440,7 +1430,7 @@ void KernelLauncher::StatesStatesPruningMulti(GPUPtr states, states, partials, matrices, scalingFactors, ptrOffsets, gridStartOp, patternCount); } - + bgPeelingGrid.x = tmpGridx; #ifdef BEAGLE_DEBUG_FLOW @@ -1494,7 +1484,7 @@ void KernelLauncher::StatesStatesPruningDynamicScaling(GPUPtr states1, } if (doRescaling != 0) { - + // Compute partials without any rescaling if (endPattern != 0) { gpu->LaunchKernelConcurrent(fStatesStatesByPatternBlockCoherentPartition, @@ -1512,7 +1502,7 @@ void KernelLauncher::StatesStatesPruningDynamicScaling(GPUPtr states1, patternCount); } - // Rescale partials and save scaling factors + // Rescale partials and save scaling factors if (doRescaling > 0) { if (endPattern == 0 ) { KernelLauncher::RescalePartials(partials3, scalingFactors, cumulativeScaling, @@ -1534,9 +1524,9 @@ void KernelLauncher::StatesStatesPruningDynamicScaling(GPUPtr states1, , streamIndex, -1, startPattern, endPattern); } } - + } else { - + // Compute partials with known rescalings #ifdef BEAGLE_FILL_4_STATE_SCALAR_SS if (kPaddedStateCount == 4) { @@ -1588,7 +1578,7 @@ void KernelLauncher::StatesStatesPruningDynamicScaling(GPUPtr states1, bgPeelingGrid.x = gridX; } #endif - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::StatesStatesPruningDynamicScaling\n"); #endif @@ -1604,12 +1594,12 @@ void KernelLauncher::IntegrateLikelihoodsDynamicScaling(GPUPtr dResult, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::IntegrateLikelihoodsDynamicScaling\n"); #endif - + gpu->LaunchKernel(fIntegrateLikelihoodsDynamicScaling, bgLikelihoodBlock, bgLikelihoodGrid, 5, 7, dResult, dRootPartials, dWeights, dFrequencies, dRootScalingFactors, - categoryCount,patternCount); + categoryCount,patternCount); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::IntegrateLikelihoodsDynamicScaling\n"); #endif @@ -1629,7 +1619,7 @@ void KernelLauncher::IntegrateLikelihoodsDynamicScalingPartition(GPUPtr dResult, #endif Dim3Int bgLikelihoodPartitionGrid = Dim3Int(gridSize); - + gpu->LaunchKernel(fIntegrateLikelihoodsDynamicScalingPartition, bgLikelihoodBlock, bgLikelihoodPartitionGrid, 6, 8, @@ -1655,14 +1645,14 @@ void KernelLauncher::IntegrateLikelihoodsDynamicScalingSecondDeriv(GPUPtr dResul #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::IntegrateLikelihoodsDynamicScalingSecondDeriv\n"); #endif - + gpu->LaunchKernel(fIntegrateLikelihoodsDynamicScalingSecondDeriv, bgLikelihoodBlock, bgLikelihoodGrid, 9, 11, dResult, dFirstDerivResult, dSecondDerivResult, dRootPartials, dRootFirstDeriv, dRootSecondDeriv, dWeights, dFrequencies, dRootScalingFactors, - categoryCount, patternCount); + categoryCount, patternCount); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::IntegrateLikelihoodsDynamicScalingSecondDeriv\n"); #endif @@ -1731,7 +1721,7 @@ void KernelLauncher::PartialsPartialsEdgeLikelihoodsSecondDeriv(GPUPtr dPartials #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr,"\t\tEntering KernelLauncher::PartialsPartialsEdgeLikelihoodsSecondDeriv\n"); #endif - + gpu->LaunchKernel(fPartialsPartialsEdgeLikelihoodsSecondDeriv, bgPeelingBlock, bgPeelingGrid, 8, 9, @@ -1739,11 +1729,11 @@ void KernelLauncher::PartialsPartialsEdgeLikelihoodsSecondDeriv(GPUPtr dPartials dParentPartials, dChildParials, dTransMatrix, dFirstDerivMatrix, dSecondDerivMatrix, patternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsPartialsEdgeLikelihoodsSecondDeriv\n"); #endif - + } void KernelLauncher::StatesPartialsEdgeLikelihoods(GPUPtr dPartialsTmp, @@ -1755,7 +1745,7 @@ void KernelLauncher::StatesPartialsEdgeLikelihoods(GPUPtr dPartialsTmp, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr,"\t\tEntering KernelLauncher::StatesPartialsEdgeLikelihoods\n"); #endif - + #ifdef FW_OPENCL // fix for Apple CPU OpenCL limitations size_t blockX = bgPeelingBlock.x; @@ -1773,7 +1763,7 @@ void KernelLauncher::StatesPartialsEdgeLikelihoods(GPUPtr dPartialsTmp, bgPeelingBlock, bgPeelingGrid, 4, 5, dPartialsTmp, dParentPartials, dChildStates, dTransMatrix, - patternCount); + patternCount); #ifdef FW_OPENCL // restore values if used fix for Apple CPU OpenCL limitations @@ -1782,11 +1772,11 @@ void KernelLauncher::StatesPartialsEdgeLikelihoods(GPUPtr dPartialsTmp, bgPeelingGrid.x = gridX; } #endif - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::StatesPartialsEdgeLikelihoods\n"); #endif - + } void KernelLauncher::StatesPartialsEdgeLikelihoodsByPartition(GPUPtr dPartialsTmp, @@ -1843,7 +1833,7 @@ void KernelLauncher::StatesPartialsEdgeLikelihoodsSecondDeriv(GPUPtr dPartialsTm AppleCPUImplementation = true; } #endif - + gpu->LaunchKernel(fStatesPartialsEdgeLikelihoodsSecondDeriv, bgPeelingBlock, bgPeelingGrid, 8, 9, @@ -1851,11 +1841,11 @@ void KernelLauncher::StatesPartialsEdgeLikelihoodsSecondDeriv(GPUPtr dPartialsTm dParentPartials, dChildStates, dTransMatrix, dFirstDerivMatrix, dSecondDerivMatrix, patternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::StatesPartialsEdgeLikelihoodsSecondDeriv\n"); #endif - + #ifdef FW_OPENCL // restore values if used fix for Apple CPU OpenCL limitations if (AppleCPUImplementation) { @@ -1874,7 +1864,7 @@ void KernelLauncher::AccumulateFactorsDynamicScaling(GPUPtr dScalingFactors, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::AccumulateFactorsDynamicScaling\n"); #endif - + int parameterCountV = 3; int totalParameterCount = 5; gpu->LaunchKernel(fAccumulateFactorsDynamicScaling, @@ -1882,11 +1872,11 @@ void KernelLauncher::AccumulateFactorsDynamicScaling(GPUPtr dScalingFactors, parameterCountV, totalParameterCount, dScalingFactors, dNodePtrQueue, dRootScalingFactors, nodeCount, patternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::AccumulateFactorsDynamicScaling\n"); #endif - + } @@ -1904,8 +1894,8 @@ void KernelLauncher::AccumulateFactorsDynamicScalingByPartition(GPUPtr dScalingF Dim3Int bgAccumulatePartitionGrid = Dim3Int(partitionPatternCount / kPatternBlockSize); if (partitionPatternCount % kPatternBlockSize != 0) - bgAccumulatePartitionGrid.x += 1; - + bgAccumulatePartitionGrid.x += 1; + int parameterCountV = 3; int totalParameterCount = 6; gpu->LaunchKernel(fAccumulateFactorsDynamicScalingByPartition, @@ -1913,11 +1903,11 @@ void KernelLauncher::AccumulateFactorsDynamicScalingByPartition(GPUPtr dScalingF parameterCountV, totalParameterCount, dScalingFactors, dNodePtrQueue, dRootScalingFactors, nodeCount, startPattern, endPattern); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::AccumulateFactorsDynamicScalingByPartition\n"); #endif - + } void KernelLauncher::AccumulateFactorsAutoScaling(GPUPtr dScalingFactors, @@ -1929,7 +1919,7 @@ void KernelLauncher::AccumulateFactorsAutoScaling(GPUPtr dScalingFactors, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::AccumulateFactorsAutoScaling\n"); #endif - + int parameterCountV = 3; int totalParameterCount = 6; gpu->LaunchKernel(fAccumulateFactorsAutoScaling, @@ -1941,7 +1931,7 @@ void KernelLauncher::AccumulateFactorsAutoScaling(GPUPtr dScalingFactors, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::AccumulateFactorsAutoScaling\n"); #endif - + } @@ -1952,8 +1942,8 @@ void KernelLauncher::RemoveFactorsDynamicScaling(GPUPtr dScalingFactors, unsigned int patternCount) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::RemoveFactorsDynamicScaling\n"); -#endif - +#endif + int parameterCountV = 3; int totalParameterCount = 5; gpu->LaunchKernel(fRemoveFactorsDynamicScaling, @@ -1964,7 +1954,7 @@ void KernelLauncher::RemoveFactorsDynamicScaling(GPUPtr dScalingFactors, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::RemoveFactorsDynamicScaling\n"); -#endif +#endif } @@ -1976,13 +1966,13 @@ void KernelLauncher::RemoveFactorsDynamicScalingByPartition(GPUPtr dScalingFacto int endPattern) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::RemoveFactorsDynamicScalingByPartition\n"); -#endif - +#endif + int partitionPatternCount = endPattern - startPattern; Dim3Int bgAccumulatePartitionGrid = Dim3Int(partitionPatternCount / kPatternBlockSize); if (partitionPatternCount % kPatternBlockSize != 0) - bgAccumulatePartitionGrid.x += 1; + bgAccumulatePartitionGrid.x += 1; int parameterCountV = 3; int totalParameterCount = 6; @@ -1994,7 +1984,7 @@ void KernelLauncher::RemoveFactorsDynamicScalingByPartition(GPUPtr dScalingFacto #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::RemoveFactorsDynamicScalingByPartition\n"); -#endif +#endif } @@ -2003,13 +1993,13 @@ void KernelLauncher::ResetFactorsDynamicScalingByPartition(GPUPtr dScalingFactor int endPattern) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::ResetFactorsDynamicScalingByPartition\n"); -#endif - +#endif + int partitionPatternCount = endPattern - startPattern; Dim3Int bgAccumulatePartitionGrid = Dim3Int(partitionPatternCount / kPatternBlockSize); if (partitionPatternCount % kPatternBlockSize != 0) - bgAccumulatePartitionGrid.x += 1; + bgAccumulatePartitionGrid.x += 1; int parameterCountV = 1; int totalParameterCount = 3; @@ -2021,23 +2011,23 @@ void KernelLauncher::ResetFactorsDynamicScalingByPartition(GPUPtr dScalingFactor #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ResetFactorsDynamicScalingByPartition\n"); -#endif +#endif } void KernelLauncher::RescalePartials(GPUPtr partials3, GPUPtr scalingFactors, - GPUPtr cumulativeScaling, + GPUPtr cumulativeScaling, unsigned int patternCount, unsigned int categoryCount, unsigned int fillWithOnes, int streamIndex, int waitIndex) { - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::RescalePartials\n"); -#endif - +#endif + // TODO: remove fillWithOnes and leave it up to client? // printf("RESCALE ON STREAM %d\n",streamIndex ); // Rescale partials and save scaling factors @@ -2059,17 +2049,17 @@ void KernelLauncher::RescalePartials(GPUPtr partials3, exit(0); } } - + // TODO: Totally incoherent for kPaddedStateCount == 4 - + if (cumulativeScaling != 0) { - - if (kSlowReweighing) { + + if (kSlowReweighing) { fprintf(stderr,"Simultaneous slow reweighing and accumulation is not yet implemented.\n"); exit(-1); - // TODO: add support for accumulate scaling as you rescale for SLOW_REWEIGHING + // TODO: add support for accumulate scaling as you rescale for SLOW_REWEIGHING } - + int parameterCountV = 3; int totalParameterCount = 4; gpu->LaunchKernelConcurrent(fPartialsDynamicScalingAccumulate, @@ -2088,7 +2078,7 @@ void KernelLauncher::RescalePartials(GPUPtr partials3, partials3, scalingFactors, categoryCount); } - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::RescalePartials\n"); #endif @@ -2096,7 +2086,7 @@ void KernelLauncher::RescalePartials(GPUPtr partials3, void KernelLauncher::RescalePartialsByPartition(GPUPtr partials3, GPUPtr scalingFactors, - GPUPtr cumulativeScaling, + GPUPtr cumulativeScaling, unsigned int patternCount, unsigned int categoryCount, unsigned int fillWithOnes, @@ -2104,11 +2094,11 @@ void KernelLauncher::RescalePartialsByPartition(GPUPtr partials3, int waitIndex, int startPattern, int endPattern) { - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::RescalePartialsByPartition\n"); -#endif - +#endif + int partitionPatternCount = endPattern - startPattern; Dim3Int bgScalePartitionGrid = bgScaleGrid; @@ -2116,7 +2106,7 @@ void KernelLauncher::RescalePartialsByPartition(GPUPtr partials3, if (kCPUImplementation) { bgScalePartitionGrid.x = partitionPatternCount/kPatternBlockSize; if (partitionPatternCount % kPatternBlockSize != 0) - bgScalePartitionGrid.x += 1; + bgScalePartitionGrid.x += 1; } else { if (kSlowReweighing) { fprintf(stderr,"Slow reweighing and partitioning not yet implemented\n"); @@ -2125,16 +2115,16 @@ void KernelLauncher::RescalePartialsByPartition(GPUPtr partials3, if (kPaddedStateCount == 4) { bgScalePartitionGrid.x = partitionPatternCount / 4; if (partitionPatternCount % 4 != 0) { - bgScalePartitionGrid.x += 1; // + bgScalePartitionGrid.x += 1; // // fprintf(stderr,"PATTERNS SHOULD BE PADDED! Inform Marc, please.\n"); // exit(-1); - } - } else { + } + } else { fprintf(stderr,"Partitioning and state count != 4 not implemented\n"); exit(-1); } } - } + } // printf("p %d, s %2d, e %2d, bgScaleGrid.x %2d, bgScalePartitionGrid.x %2d\n", partials3, startPattern, endPattern, bgScaleGrid.x, bgScalePartitionGrid.x ); @@ -2144,8 +2134,8 @@ void KernelLauncher::RescalePartialsByPartition(GPUPtr partials3, exit(0); } } - - // TODO: Totally incoherent for kPaddedStateCount == 4 + + // TODO: Totally incoherent for kPaddedStateCount == 4 if (cumulativeScaling != 0) { int parameterCountV = 3; int totalParameterCount = 7; @@ -2180,7 +2170,7 @@ void KernelLauncher::IntegrateLikelihoods(GPUPtr dResult, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr,"\t\tEntering KernelLauncher::IntegrateLikelihoods\n"); #endif - + int parameterCountV = 4; int totalParameterCount = 6; gpu->LaunchKernel(fIntegrateLikelihoods, @@ -2188,11 +2178,11 @@ void KernelLauncher::IntegrateLikelihoods(GPUPtr dResult, parameterCountV, totalParameterCount, dResult, dRootPartials, dWeights, dFrequencies, categoryCount, patternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::IntegrateLikelihoods\n"); #endif - + } void KernelLauncher::IntegrateLikelihoodsPartition(GPUPtr dResult, @@ -2208,7 +2198,7 @@ void KernelLauncher::IntegrateLikelihoodsPartition(GPUPtr dResult, #endif Dim3Int bgLikelihoodPartitionGrid = Dim3Int(gridSize); - + int parameterCountV = 5; int totalParameterCount = 7; gpu->LaunchKernel(fIntegrateLikelihoodsPartition, @@ -2216,11 +2206,11 @@ void KernelLauncher::IntegrateLikelihoodsPartition(GPUPtr dResult, parameterCountV, totalParameterCount, dResult, dRootPartials, dWeights, dFrequencies, dPtrOffsets, categoryCount, patternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::IntegrateLikelihoodsPartition\n"); #endif - + } @@ -2237,7 +2227,7 @@ void KernelLauncher::IntegrateLikelihoodsSecondDeriv(GPUPtr dResult, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr,"\t\tEntering KernelLauncher::IntegrateLikelihoodsSecondDeriv\n"); #endif - + int parameterCountV = 8; int totalParameterCount = 10; gpu->LaunchKernel(fIntegrateLikelihoodsSecondDeriv, @@ -2247,11 +2237,11 @@ void KernelLauncher::IntegrateLikelihoodsSecondDeriv(GPUPtr dResult, dRootPartials, dRootFirstDeriv, dRootSecondDeriv, dWeights, dFrequencies, categoryCount, patternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::IntegrateLikelihoodsSecondDeriv\n"); #endif - + } @@ -2265,7 +2255,7 @@ void KernelLauncher::IntegrateLikelihoodsMulti(GPUPtr dResult, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr,"\t\tEntering KernelLauncher::IntegrateLikelihoodsNoLog\n"); #endif - + int parameterCountV = 4; int totalParameterCount = 7; gpu->LaunchKernel(fIntegrateLikelihoodsMulti, @@ -2273,11 +2263,11 @@ void KernelLauncher::IntegrateLikelihoodsMulti(GPUPtr dResult, parameterCountV, totalParameterCount, dResult, dRootPartials, dWeights, dFrequencies, categoryCount, patternCount, takeLog); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::IntegrateLikelihoodsNoLog\n"); #endif - + } void KernelLauncher::IntegrateLikelihoodsFixedScaleMulti(GPUPtr dResult, @@ -2295,13 +2285,13 @@ void KernelLauncher::IntegrateLikelihoodsFixedScaleMulti(GPUPtr dResult, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::IntegrateLikelihoodsFixedScaleMulti\n"); #endif - + gpu->LaunchKernel(fIntegrateLikelihoodsFixedScaleMulti, bgLikelihoodBlock, bgLikelihoodGrid, 8, 12, dResult, dRootPartials, dWeights, dFrequencies, dScalingFactors, dPtrQueue, dMaxScalingFactors, dIndexMaxScalingFactors, - categoryCount, patternCount, subsetCount, subsetIndex); + categoryCount, patternCount, subsetCount, subsetIndex); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::IntegrateLikelihoodsFixedScaleMulti\n"); #endif @@ -2322,11 +2312,11 @@ void KernelLauncher::SumSites1(GPUPtr dArray1, parameterCountV, totalParameterCount, dArray1, dSum1, dPatternWeights, patternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::SumSites1\n"); #endif - + } void KernelLauncher::SumSites1Partition(GPUPtr dArray1, @@ -2348,11 +2338,11 @@ void KernelLauncher::SumSites1Partition(GPUPtr dArray1, parameterCountV, totalParameterCount, dArray1, dSum1, dPatternWeights, startPattern, endPattern); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::SumSites1Partition\n"); #endif - + } void KernelLauncher::SumSites2(GPUPtr dArray1, @@ -2372,11 +2362,11 @@ void KernelLauncher::SumSites2(GPUPtr dArray1, parameterCountV, totalParameterCount, dArray1, dSum1, dArray2, dSum2, dPatternWeights, patternCount); - + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::SumSites2\n"); #endif - + } void KernelLauncher::SumSites3(GPUPtr dArray1, @@ -2390,19 +2380,19 @@ void KernelLauncher::SumSites3(GPUPtr dArray1, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::SumSites3\n"); #endif - + int parameterCountV = 7; int totalParameterCount = 8; gpu->LaunchKernel(fSumSites3, bgSumSitesBlock, bgSumSitesGrid, parameterCountV, totalParameterCount, dArray1, dSum1, dArray2, dSum2, dArray3, dSum3, dPatternWeights, - patternCount); - + patternCount); + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::SumSites3\n"); #endif - + } }; // namespace diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index b5bc6e51..1de16627 100644 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @brief GPU kernel launcher * @@ -44,7 +34,7 @@ class KernelLauncher { private: GPUInterface* gpu; - + GPUFunction fMatrixConvolution; GPUFunction fMatrixTranspose; GPUFunction fMatrixMulADBMulti; @@ -87,7 +77,7 @@ class KernelLauncher { GPUFunction fStatesPartialsEdgeLikelihoods; GPUFunction fStatesPartialsEdgeLikelihoodsByPartition; GPUFunction fStatesPartialsEdgeLikelihoodsSecondDeriv; - + GPUFunction fIntegrateLikelihoodsDynamicScaling; GPUFunction fIntegrateLikelihoodsDynamicScalingPartition; GPUFunction fIntegrateLikelihoodsDynamicScalingSecondDeriv; @@ -117,7 +107,7 @@ class KernelLauncher { GPUFunction fSumSites3; GPUFunction fReorderPatterns; - + Dim3Int bgTransitionProbabilitiesBlock; Dim3Int bgTransitionProbabilitiesGrid; Dim3Int bgPeelingBlock; @@ -137,26 +127,26 @@ class KernelLauncher { Dim3Int bgMultiNodeSumBlock; Dim3Int bgMultiNodeSumGrid; - + unsigned int kPaddedStateCount; unsigned int kCategoryCount; unsigned int kPatternCount; unsigned int kUnpaddedPatternCount; unsigned int kPatternBlockSize; unsigned int kMatrixBlockSize; - unsigned int kSlowReweighing; + unsigned int kSlowReweighing; unsigned int kMultiplyBlockSize; unsigned int kSumSitesBlockSize; long kFlags; bool kCPUImplementation; bool kAppleCPUImplementation; - + public: KernelLauncher(GPUInterface* inGpu); - + ~KernelLauncher(); - + // void SetupPartitioningKernelGrid(unsigned int partitionBlockCount); // Kernel links @@ -203,8 +193,8 @@ class KernelLauncher { GPUPtr dIevc, GPUPtr dEigenValues, GPUPtr distanceQueue, - unsigned int totalMatrix); - + unsigned int totalMatrix); + void GetTransitionProbabilitiesSquareSecondDeriv(GPUPtr dMatrices, GPUPtr dPtrQueue, GPUPtr dEvec, @@ -253,7 +243,7 @@ class KernelLauncher { int doRescaling, int streamIndex, int waitIndex); - + void StatesPartialsPruningMulti(GPUPtr states, GPUPtr partials, GPUPtr matrices, @@ -278,7 +268,7 @@ class KernelLauncher { int doRescaling, int streamIndex, int waitIndex); - + void StatesStatesPruningMulti(GPUPtr states, GPUPtr partials, GPUPtr matrices, @@ -350,7 +340,7 @@ class KernelLauncher { GPUPtr partials, GPUPtr lengths, GPUPtr instructions, - GPUPtr categoryWeights, + GPUPtr categoryWeights, GPUPtr patternWeights, unsigned int instructionOffset, unsigned int nodeCount, @@ -365,10 +355,10 @@ class KernelLauncher { GPUPtr partials, GPUPtr lengths, GPUPtr instructions, - GPUPtr categoryWeights, + GPUPtr categoryWeights, GPUPtr patternWeights, unsigned int instructionOffset, - unsigned int nodeCount, + unsigned int nodeCount, unsigned int rateOffset, unsigned int patternCount, unsigned int categoryCount, @@ -407,7 +397,7 @@ class KernelLauncher { unsigned int patternCount, unsigned int categoryCount, int gridSize); - + void IntegrateLikelihoodsAutoScaling(GPUPtr dResult, GPUPtr dRootPartials, GPUPtr dWeights, @@ -415,7 +405,7 @@ class KernelLauncher { GPUPtr dRootScalingFactors, unsigned int patternCount, unsigned int categoryCount); - + void IntegrateLikelihoodsDynamicScalingSecondDeriv(GPUPtr dResult, GPUPtr dFirstDerivResult, GPUPtr dSecondDerivResult, @@ -427,7 +417,7 @@ class KernelLauncher { GPUPtr dRootScalingFactors, unsigned int patternCount, unsigned int categoryCount); - + void PartialsPartialsEdgeLikelihoods(GPUPtr dPartialsTmp, GPUPtr dParentPartials, GPUPtr dChildParials, @@ -441,7 +431,7 @@ class KernelLauncher { GPUPtr dPtrOffsets, unsigned int patternCount, int gridSize); - + void PartialsPartialsEdgeLikelihoodsSecondDeriv(GPUPtr dPartialsTmp, GPUPtr dFirstDerivTmp, GPUPtr dSecondDerivTmp, @@ -452,8 +442,8 @@ class KernelLauncher { GPUPtr dSecondDerivMatrix, unsigned int patternCount, unsigned int categoryCount); - - + + void StatesPartialsEdgeLikelihoods(GPUPtr dPartialsTmp, GPUPtr dParentPartials, GPUPtr dChildStates, @@ -468,7 +458,7 @@ class KernelLauncher { GPUPtr dPtrOffsets, unsigned int patternCount, int gridSize); - + void StatesPartialsEdgeLikelihoodsSecondDeriv(GPUPtr dPartialsTmp, GPUPtr dFirstDerivTmp, GPUPtr dSecondDerivTmp, @@ -479,7 +469,7 @@ class KernelLauncher { GPUPtr dSecondDerivMatrix, unsigned int patternCount, unsigned int categoryCount); - + void AccumulateFactorsDynamicScaling(GPUPtr dScalingFactors, GPUPtr dNodePtrQueue, GPUPtr dRootScalingFactors, @@ -499,12 +489,12 @@ class KernelLauncher { unsigned int nodeCount, unsigned int patternCount, unsigned int scaleBufferSize); - + void RemoveFactorsDynamicScaling(GPUPtr dScalingFactors, GPUPtr dNodePtrQueue, GPUPtr dRootScalingFactors, unsigned int nodeCount, - unsigned int patternCount); + unsigned int patternCount); void RemoveFactorsDynamicScalingByPartition(GPUPtr dScalingFactors, GPUPtr dNodePtrQueue, @@ -512,7 +502,7 @@ class KernelLauncher { unsigned int nodeCount, int startPattern, int endPattern); - + void ResetFactorsDynamicScalingByPartition(GPUPtr dScalingFactors, int startPattern, int endPattern); @@ -543,7 +533,7 @@ class KernelLauncher { GPUPtr dFrequencies, unsigned int patternCount, unsigned int categoryCount); - + void IntegrateLikelihoodsPartition(GPUPtr dResult, GPUPtr dRootPartials, GPUPtr dWeights, @@ -563,7 +553,7 @@ class KernelLauncher { GPUPtr dFrequencies, unsigned int patternCount, unsigned int categoryCount); - + void IntegrateLikelihoodsMulti(GPUPtr dResult, GPUPtr dRootPartials, GPUPtr dWeights, @@ -571,7 +561,7 @@ class KernelLauncher { unsigned int patternCount, unsigned int categoryCount, unsigned int takeLog); - + void IntegrateLikelihoodsFixedScaleMulti(GPUPtr dResult, GPUPtr dRootPartials, GPUPtr dWeights, @@ -584,7 +574,7 @@ class KernelLauncher { unsigned int categoryCount, unsigned int subsetCount, unsigned int subsetIndex); - + void SumSites1(GPUPtr dArray1, GPUPtr dSum1, GPUPtr dPatternWeights, @@ -596,14 +586,14 @@ class KernelLauncher { int startPattern, int endPattern, int blockCount); - + void SumSites2(GPUPtr dArray1, GPUPtr dSum1, GPUPtr dArray2, GPUPtr dSum2, GPUPtr dPatternWeights, unsigned int patternCount); - + void SumSites3(GPUPtr dArray1, GPUPtr dSum1, GPUPtr dArray2, @@ -612,9 +602,9 @@ class KernelLauncher { GPUPtr dSum3, GPUPtr dPatternWeights, unsigned int patternCount); - + void SetupKernelBlocksAndGrids(); - + protected: void LoadKernels(); diff --git a/libhmsbeagle/GPU/KernelResource.h b/libhmsbeagle/GPU/KernelResource.h index 477ef8ac..cc6b5c87 100644 --- a/libhmsbeagle/GPU/KernelResource.h +++ b/libhmsbeagle/GPU/KernelResource.h @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard */ @@ -29,7 +19,7 @@ class KernelResource { public: KernelResource(); - + KernelResource( int inPaddedStateCount, char* inKernelString, @@ -43,12 +33,12 @@ class KernelResource { int inUnpaddedPatternCount, long inFlags ); - + KernelResource(const KernelResource& krIn, char* inKernelCode); - + virtual ~KernelResource(); - + int paddedStateCount; int categoryCount; int patternCount; @@ -62,7 +52,7 @@ class KernelResource { int slowReweighing; int multiplyBlockSize; long flags; - + KernelResource* copy(); }; diff --git a/libhmsbeagle/GPU/kernels/kernels4.cu b/libhmsbeagle/GPU/kernels/kernels4.cu index 63be43b7..fbd1803a 100644 --- a/libhmsbeagle/GPU/kernels/kernels4.cu +++ b/libhmsbeagle/GPU/kernels/kernels4.cu @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Daniel Ayres @@ -29,7 +19,7 @@ #include "libhmsbeagle/GPU/GPUImplDefs.h" #include "libhmsbeagle/GPU/kernels/kernelsAll.cu" // This file includes the non-state-count specific kernels extern "C" { -#endif +#endif #define multBy4(x) ((x) << 2) #define multBy16(x) ((x) << 4) @@ -46,14 +36,14 @@ int deltaPartialsByMatrix = matrix * PADDED_STATE_COUNT * endPattern;\ int deltaMatrix = matrix * PADDED_STATE_COUNT * PADDED_STATE_COUNT;\ int deltaPartials = deltaPartialsByMatrix + deltaPartialsByState; - + #define DETERMINE_INDICES_4_MULTI_1_CPU()\ int opIndexPtr = (gridStartOp + KW_GROUP_ID_0) * 8;\ int startPat = ptrOffsets[opIndexPtr ];\ int endPattern = ptrOffsets[opIndexPtr + 1];\ int patIdx = KW_LOCAL_ID_0;\ - int pattern = startPat + patIdx; - + int pattern = startPat + patIdx; + #define DETERMINE_INDICES_4_MULTI_2_CPU()\ int matrix = KW_GROUP_ID_1;\ int deltaPartialsByState = pattern * PADDED_STATE_COUNT;\ @@ -75,15 +65,15 @@ int deltaPartialsByState = pattern * PADDED_STATE_COUNT;\ int deltaPartialsByMatrix = matrix * PADDED_STATE_COUNT * totalPatterns;\ int deltaMatrix = matrix * PADDED_STATE_COUNT * PADDED_STATE_COUNT;\ - int deltaPartials = deltaPartialsByMatrix + deltaPartialsByState; + int deltaPartials = deltaPartialsByMatrix + deltaPartialsByState; #define DETERMINE_INDICES_4_EDGEPART_1_CPU()\ int opIndexPtr = KW_GROUP_ID_0 * 5;\ int startPat = ptrOffsets[opIndexPtr ];\ int endPattern = ptrOffsets[opIndexPtr + 1];\ int patIdx = KW_LOCAL_ID_0;\ - int pattern = startPat + patIdx; - + int pattern = startPat + patIdx; + #define SUM_PARTIALS_PARTIALS_4_CPU()\ REAL sum1[PADDED_STATE_COUNT];\ REAL sum2[PADDED_STATE_COUNT];\ @@ -503,7 +493,7 @@ sMatrix2[tx] = matrix2[tx];\ }\ KW_LOCAL_FENCE; - + #define LOAD_MATRIX_4_MULTI_GPU()\ matrix1 += x2;\ matrix2 += x2;\ @@ -815,7 +805,7 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsNoScaleMulti(KW_GLOBAL_VAR REAL* KW_ DETERMINE_INDICES_4_MULTI_2_CPU(); const KW_GLOBAL_VAR REAL* KW_RESTRICT partials1 = partials + ptrOffsets[opIndexPtr + 2]; const KW_GLOBAL_VAR REAL* KW_RESTRICT partials2 = partials + ptrOffsets[opIndexPtr + 3]; - DETERMINE_INDICES_4_MULTI_3_CPU(); + DETERMINE_INDICES_4_MULTI_3_CPU(); SUM_PARTIALS_PARTIALS_4_CPU(); for(int i = 0; i < PADDED_STATE_COUNT; i++) { partials3[deltaPartials + i] = sum1[i] * sum2[i]; @@ -1243,7 +1233,7 @@ KW_GLOBAL_KERNEL void kernelStatesStatesNoScaleMulti(KW_GLOBAL_VAR int* KW_RESTR #ifdef FW_OPENCL_CPU // CPU/MIC implementation DETERMINE_INDICES_4_MULTI_1_CPU(); if (pattern < endPattern) { - DETERMINE_INDICES_4_MULTI_2_CPU(); + DETERMINE_INDICES_4_MULTI_2_CPU(); const KW_GLOBAL_VAR int* KW_RESTRICT states1 = states + ptrOffsets[opIndexPtr + 2]; const KW_GLOBAL_VAR int* KW_RESTRICT states2 = states + ptrOffsets[opIndexPtr + 3]; DETERMINE_INDICES_4_MULTI_3_CPU(); @@ -1315,7 +1305,7 @@ KW_GLOBAL_KERNEL void kernelStatesStatesFixedScaleMulti( #ifdef FW_OPENCL_CPU // CPU/MIC implementation DETERMINE_INDICES_4_MULTI_1_CPU(); if (pattern < endPattern) { - DETERMINE_INDICES_4_MULTI_2_CPU(); + DETERMINE_INDICES_4_MULTI_2_CPU(); const KW_GLOBAL_VAR int* KW_RESTRICT states1 = states + ptrOffsets[opIndexPtr + 2]; const KW_GLOBAL_VAR int* KW_RESTRICT states2 = states + ptrOffsets[opIndexPtr + 3]; DETERMINE_INDICES_4_MULTI_3_CPU(); @@ -1534,7 +1524,7 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulate(KW_GLOBAL_VAR REAL* FIND_MAX_PARTIALS_MATRIX_4_GPU(); if (matrixMax[pat] == 0) matrixMax[pat] = 1.0; - scalingFactors[pattern] = matrixMax[pat]; + scalingFactors[pattern] = matrixMax[pat]; #ifdef CUDA atomicAdd(&cumulativeScaling[pattern], log(matrixMax[pat])); #else @@ -1775,7 +1765,7 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsMulti(KW_GLOBAL_VAR REAL* KW_RES dResult[pattern] = sumTotal; else if (takeLog == 1) dResult[pattern] = log(dResult[pattern] + sumTotal); - else + else dResult[pattern] += sumTotal; #else // GPU implementation DETERMINE_INTEGRATE_INDICES_4_GPU(); @@ -1785,7 +1775,7 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsMulti(KW_GLOBAL_VAR REAL* KW_RES dResult[pattern] = sum[pat][state]; else if (takeLog == 1) dResult[pattern] = log(dResult[pattern] + sum[pat][state]); - else + else dResult[pattern] += sum[pat][state]; } #endif // FW_OPENCL_CPU @@ -1818,7 +1808,7 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsFixedScaleMulti(KW_GLOBAL_VAR RE } } dIndexMaxScalingFactors[pattern] = indexMaxScalingFactor; - dMaxScalingFactors[pattern] = maxScalingFactor; + dMaxScalingFactors[pattern] = maxScalingFactor; if (indexMaxScalingFactor != 0) sumTotal *= exp((REAL)(cumulativeScalingFactor - maxScalingFactor)); dResult[pattern] = sumTotal; @@ -1845,7 +1835,7 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsFixedScaleMulti(KW_GLOBAL_VAR RE } } dIndexMaxScalingFactors[pattern] = indexMaxScalingFactor; - dMaxScalingFactors[pattern] = maxScalingFactor; + dMaxScalingFactors[pattern] = maxScalingFactor; if (indexMaxScalingFactor != 0) sum[pat][state] *= exp((REAL)(cumulativeScalingFactor - maxScalingFactor)); if (state == 0) @@ -2119,7 +2109,7 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsCheckScale(KW_GLOBAL_VAR REAL* parti int patIdx16pat4 = multBy16(patIdx) | (tx & 0xC); int y = deltaPartialsByState + deltaPartialsByMatrix; - + KW_GLOBAL_VAR REAL* matrix1 = matrices1 + x2; // Points to *this* matrix KW_GLOBAL_VAR REAL* matrix2 = matrices2 + x2; @@ -2168,21 +2158,21 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsCheckScale(KW_GLOBAL_VAR REAL* parti i = (i + 1) & 0x3; sum1 += sMatrix1[multBy4(i) | state] * sPartials1[patIdx16pat4 | i]; sum2 += sMatrix2[multBy4(i) | state] * sPartials2[patIdx16pat4 | i]; - + REAL tmpPartial = sum1 * sum2; - + partials3[u] = tmpPartial; if (tmpPartial < SCALING_THRESHOLD_LOWER || tmpPartial > SCALING_THRESHOLD_UPPER) *dRescalingTrigger = 1; - + // union {float f; long l;} fl; // fl.f = sum1 * sum2;; // // partials3[u] = fl.f; -// +// // int expTmp = ((fl.l >> 23) & 0x000000ff) - 0x7e; -// +// // if (abs(expTmp) > SCALING_EXPONENT_THRESHOLD) // *dRescalingTrigger = 1; } @@ -2257,9 +2247,9 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsFixedCheckScale(KW_GLOBAL_VAR REAL* i = (i + 1) & 0x3; sum1 += sMatrix1[i * 4 + state] * sPartials1[patIdx * 16 + pat * 4 + i]; sum2 += sMatrix2[i * 4 + state] * sPartials2[patIdx * 16 + pat * 4 + i]; - + REAL tmpPartial = sum1 * sum2 * fixedScalingFactors[patIdx * 4 + pat]; - + partials3[u] = tmpPartial; if (tmpPartial < SCALING_THRESHOLD_LOWER || tmpPartial > SCALING_THRESHOLD_UPPER) @@ -2285,7 +2275,7 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsAutoScale(KW_GLOBAL_VAR REAL* partia int patIdx16pat4 = multBy16(patIdx) | (tx & 0xC); int y = deltaPartialsByState + deltaPartialsByMatrix; int myIdx = multBy16(patIdx) + tx; // threadId in block - + KW_GLOBAL_VAR REAL* matrix1 = matrices1 + x2; // Points to *this* matrix KW_GLOBAL_VAR REAL* matrix2 = matrices2 + x2; @@ -2332,13 +2322,13 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsAutoScale(KW_GLOBAL_VAR REAL* partia i = (i + 1) & 0x3; sum1 += sMatrix1[multBy4(i) | state] * sPartials1[patIdx16pat4 | i]; sum2 += sMatrix2[multBy4(i) | state] * sPartials2[patIdx16pat4 | i]; - + REAL tmpPartial = sum1 * sum2; int expTmp; - REAL sigTmp = frexp(tmpPartial, &expTmp); + REAL sigTmp = frexp(tmpPartial, &expTmp); KW_LOCAL_FENCE; - + if (pattern < endPattern) { if (abs(expTmp) > SCALING_EXPONENT_THRESHOLD) { // now using sPartials2 to hold scaling trigger boolean @@ -2348,28 +2338,28 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsAutoScale(KW_GLOBAL_VAR REAL* partia sPartials2[patIdx16pat4] = 0; sPartials1[myIdx] = 0; } - } - + } + KW_LOCAL_FENCE; - + int scalingActive = sPartials2[patIdx16pat4]; - + if (scalingActive) { // now using sPartials1 to store max unscaled partials3 sPartials1[myIdx] = tmpPartial; } - + KW_LOCAL_FENCE; - + // Unrolled parallel max-reduction if (scalingActive && state < 2) { REAL compare = sPartials1[myIdx + 2]; if (compare > sPartials1[myIdx]) sPartials1[myIdx] = compare; } - + KW_LOCAL_FENCE; - + if (scalingActive && state < 1) { REAL maxPartial = sPartials1[myIdx + 1]; if (maxPartial < sPartials1[myIdx]) @@ -2380,10 +2370,10 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsAutoScale(KW_GLOBAL_VAR REAL* partia } KW_LOCAL_FENCE; - - if (scalingActive) + + if (scalingActive) partials3[u] = ldexp(sigTmp, expTmp - sPartials1[patIdx16pat4]); - + if ((myIdx < PATTERN_BLOCK_SIZE * 4) && (myIdx + (KW_GROUP_ID_0 * PATTERN_BLOCK_SIZE * 4) < endPattern)) scalingFactors[(KW_GROUP_ID_0 * PATTERN_BLOCK_SIZE * 4) + (matrix * endPattern) + myIdx] = sPartials1[multBy4(myIdx)]; } @@ -2394,16 +2384,16 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateReciprocal(KW_GLOBAL KW_GLOBAL_VAR REAL* cumulativeScaling, int matrixCount) { int tx = KW_LOCAL_ID_0; - + int state = tx & 0x3; int pat = tx >> 2; - + int patIdx = KW_GROUP_ID_0; - + int pattern = (patIdx << 2) + pat; int matrix = KW_LOCAL_ID_1; // TODO: Assumes matrixCount < MATRIX_BLOCK_SIZ - + // Patterns are always padded, so no reading/writing past end possible // Find start of patternBlock for thread-block int partialsOffset = (matrix * KW_NUM_GROUPS_0 + patIdx) << 4; //* 16; @@ -2412,14 +2402,14 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateReciprocal(KW_GLOBAL KW_LOCAL_MEM REAL storedPartials[MATRIX_BLOCK_SIZE][16]; KW_LOCAL_MEM REAL matrixMax[4]; - + if (matrix < matrixCount) - partials[matrix][tx] = allPartials[partialsOffset + tx]; + partials[matrix][tx] = allPartials[partialsOffset + tx]; storedPartials[matrix][tx] = partials[matrix][tx]; - + KW_LOCAL_FENCE; - + // Unrolled parallel max-reduction if (state < 2) { REAL compare1 = partials[matrix][tx]; @@ -2428,7 +2418,7 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateReciprocal(KW_GLOBAL partials[matrix][tx] = compare2; } KW_LOCAL_FENCE; - + if (state < 1) { REAL compare1 = partials[matrix][tx]; REAL compare2 = partials[matrix][tx + 1]; @@ -2436,7 +2426,7 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateReciprocal(KW_GLOBAL partials[matrix][tx] = compare2; } KW_LOCAL_FENCE; - + // Could also parallel-reduce here. if (state == 0 && matrix == 0) { matrixMax[pat] = 0; @@ -2445,11 +2435,11 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateReciprocal(KW_GLOBAL if (partials[m][tx] > matrixMax[pat]) matrixMax[pat] = partials[m][tx]; } - + if (matrixMax[pat] == 0) matrixMax[pat] = 1.0; - - scalingFactors[pattern] = 1/matrixMax[pat]; + + scalingFactors[pattern] = 1/matrixMax[pat]; cumulativeScaling[pattern] += log(matrixMax[pat]); } @@ -2457,7 +2447,7 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateReciprocal(KW_GLOBAL if (matrix < matrixCount) allPartials[partialsOffset + tx] = storedPartials[matrix][tx] / matrixMax[pat]; - + } KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateDifference(KW_GLOBAL_VAR REAL* allPartials, @@ -2466,16 +2456,16 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateDifference(KW_GLOBAL KW_GLOBAL_VAR REAL* cumulativeScaling, int matrixCount) { int tx = KW_LOCAL_ID_0; - + int state = tx & 0x3; int pat = tx >> 2; - + int patIdx = KW_GROUP_ID_0; - + int pattern = (patIdx << 2) + pat; int matrix = KW_LOCAL_ID_1; // TODO: Assumes matrixCount < MATRIX_BLOCK_SIZ - + // Patterns are always padded, so no reading/writing past end possible // Find start of patternBlock for thread-block int partialsOffset = (matrix * KW_NUM_GROUPS_0 + patIdx) << 4; //* 16; @@ -2484,14 +2474,14 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateDifference(KW_GLOBAL KW_LOCAL_MEM REAL storedPartials[MATRIX_BLOCK_SIZE][16]; KW_LOCAL_MEM REAL matrixMax[4]; - + if (matrix < matrixCount) - partials[matrix][tx] = allPartials[partialsOffset + tx]; + partials[matrix][tx] = allPartials[partialsOffset + tx]; storedPartials[matrix][tx] = partials[matrix][tx]; - + KW_LOCAL_FENCE; - + // Unrolled parallel max-reduction if (state < 2) { REAL compare1 = partials[matrix][tx]; @@ -2500,7 +2490,7 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateDifference(KW_GLOBAL partials[matrix][tx] = compare2; } KW_LOCAL_FENCE; - + if (state < 1) { REAL compare1 = partials[matrix][tx]; REAL compare2 = partials[matrix][tx + 1]; @@ -2508,7 +2498,7 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateDifference(KW_GLOBAL partials[matrix][tx] = compare2; } KW_LOCAL_FENCE; - + // Could also parallel-reduce here. if (state == 0 && matrix == 0) { matrixMax[pat] = 0; @@ -2517,12 +2507,12 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateDifference(KW_GLOBAL if (partials[m][tx] > matrixMax[pat]) matrixMax[pat] = partials[m][tx]; } - + if (matrixMax[pat] == 0) matrixMax[pat] = 1.0; - + REAL currentFactors = existingScalingFactors[pattern]; - scalingFactors[pattern] = 1/matrixMax[pat] * currentFactors; + scalingFactors[pattern] = 1/matrixMax[pat] * currentFactors; cumulativeScaling[pattern] += (log(matrixMax[pat])); } @@ -2530,7 +2520,7 @@ KW_GLOBAL_KERNEL void kernelPartialsDynamicScalingAccumulateDifference(KW_GLOBAL if (matrix < matrixCount) allPartials[partialsOffset + tx] = storedPartials[matrix][tx] / matrixMax[pat]; - + } KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsAutoScaling(KW_GLOBAL_VAR REAL* dResult, @@ -2543,9 +2533,9 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsAutoScaling(KW_GLOBAL_VAR REAL* int state = KW_LOCAL_ID_0; int pat = KW_LOCAL_ID_1; int pattern = KW_GROUP_ID_0 * LIKE_PATTERN_BLOCK_SIZE + KW_LOCAL_ID_1; - + KW_LOCAL_MEM REAL stateFreq[4]; - + // TODO: Currently assumes MATRIX_BLOCK_SIZE >= matrixCount KW_LOCAL_MEM REAL matrixProp[MATRIX_BLOCK_SIZE]; KW_LOCAL_MEM REAL sum[LIKE_PATTERN_BLOCK_SIZE][4]; @@ -2555,9 +2545,9 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsAutoScaling(KW_GLOBAL_VAR REAL* if (pat == 0) { stateFreq[state] = dFrequencies[state]; } - + sum[pat][state] = 0; - + // TODO: Assumes matrixCount < LIKE_PATTERN_BLOCK_SIZE * 4 if (pat * 4 + state < matrixCount) { matrixProp[pat * 4 + state] = dWeights[pat * 4 + state]; diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 2522c6c3..107467fb 100644 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Daniel Ayres @@ -27,8 +17,8 @@ #include #include #include - extern "C" { -#elif defined(FW_OPENCL) + extern "C" { +#elif defined(FW_OPENCL) #ifdef DOUBLE_PRECISION #pragma OPENCL EXTENSION cl_khr_fp64: enable #endif @@ -73,10 +63,10 @@ KW_GLOBAL_KERNEL void kernelReorderPatterns( KW_GLOBAL_VAR REAL* KW_GLOBAL_VAR REAL* KW_RESTRICT dPatternWeightsSort, int patternCount, int paddedPatternCount) { -#ifdef FW_OPENCL_CPU +#ifdef FW_OPENCL_CPU int state = 0; int pattern = KW_LOCAL_ID_0 + KW_GROUP_ID_0 * KW_LOCAL_SIZE_0; -#else +#else int state = KW_LOCAL_ID_0; int pattern = KW_LOCAL_ID_1 + KW_GROUP_ID_0 * KW_LOCAL_SIZE_1; #endif @@ -97,10 +87,10 @@ KW_GLOBAL_KERNEL void kernelReorderPatterns( KW_GLOBAL_VAR REAL* const KW_GLOBAL_VAR REAL* KW_RESTRICT partialOriginal = dPartials + dTipOffsets[tip]; KW_GLOBAL_VAR REAL* KW_RESTRICT partialSorted = dPartials + dTipOffsets[tip+tipCount]; -#ifdef FW_OPENCL_CPU +#ifdef FW_OPENCL_CPU for (int i=0; i < stateCount; i++) { partialSorted[sortIndex+i] = partialOriginal[originIndex+i]; - } + } #else sortIndex += state; originIndex += state; @@ -128,7 +118,7 @@ KW_GLOBAL_KERNEL void kernelMatrixMulADBMulti(KW_GLOBAL_VAR REAL* dMatrices, int length, int wB, int totalMatrix) { - + int wMatrix = KW_GROUP_ID_0 % totalMatrix; int offIndex = wMatrix * 3; @@ -337,7 +327,7 @@ KW_GLOBAL_KERNEL void kernelMatrixMulADB(KW_GLOBAL_VAR REAL* dMatrices, PADDED_STATE_COUNT * ty + tx] = Csub; } } - + KW_GLOBAL_KERNEL void kernelMatrixMulADBFirstDeriv(KW_GLOBAL_VAR REAL* dMatrices, KW_GLOBAL_VAR unsigned int* listC, KW_GLOBAL_VAR REAL* A, @@ -380,7 +370,7 @@ KW_GLOBAL_KERNEL void kernelMatrixMulADBFirstDeriv(KW_GLOBAL_VAR REAL* dMatrices distanceLength = distanceQueue[wMatrix]; distanceRate = distanceQueue[wMatrix + totalMatrix]; #endif - + KW_LOCAL_FENCE; const int EDGE = PADDED_STATE_COUNT - (BLOCKS - 1) * MULTIPLY_BLOCK_SIZE; @@ -626,7 +616,7 @@ KW_GLOBAL_KERNEL void kernelMatrixMulADBSecondDeriv(KW_GLOBAL_VAR REAL* dMatrice PADDED_STATE_COUNT * ty + tx] = CSecondDerivSub; } } - + KW_GLOBAL_KERNEL void kernelMatrixConvolution(KW_GLOBAL_VAR REAL* dMatrices, KW_GLOBAL_VAR unsigned int* list, int totalMatrixCount @@ -643,7 +633,7 @@ KW_GLOBAL_KERNEL void kernelMatrixConvolution(KW_GLOBAL_VAR REAL* dMatrices, int ty = KW_LOCAL_ID_1; int BLOCKS = KW_NUM_GROUPS_1; - + #ifdef CUDA KW_LOCAL_MEM REAL* A; KW_LOCAL_MEM REAL* B; @@ -786,7 +776,7 @@ KW_GLOBAL_KERNEL void kernelMatrixTranspose(KW_GLOBAL_VAR REAL* dMatrices, } KW_LOCAL_FENCE; - + if (row < PADDED_STATE_COUNT && col < PADDED_STATE_COUNT) { C[PADDED_STATE_COUNT * rowOffset + colOffset + PADDED_STATE_COUNT * ty + tx] = As[tx][ty]; @@ -1011,7 +1001,7 @@ KW_GLOBAL_KERNEL void kernelMatrixMulADBComplexMulti(KW_GLOBAL_VAR REAL* dMatric } #endif } - + KW_GLOBAL_KERNEL void kernelMatrixMulADBComplex(KW_GLOBAL_VAR REAL* dMatrices, KW_GLOBAL_VAR unsigned int* listC, @@ -1240,13 +1230,13 @@ KW_GLOBAL_KERNEL void kernelMatrixMulADBComplex(KW_GLOBAL_VAR REAL* dMatrices, } #endif } - + KW_GLOBAL_KERNEL void kernelSumSites1(KW_GLOBAL_VAR REAL* dArray, KW_GLOBAL_VAR REAL* dSum, KW_GLOBAL_VAR REAL* dPatternWeights, int patternCount) { #ifdef FW_OPENCL_CPU - + REAL sum = 0; int pattern = KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE; @@ -1263,7 +1253,7 @@ KW_GLOBAL_KERNEL void kernelSumSites1(KW_GLOBAL_VAR REAL* dArray, dSum[KW_GROUP_ID_0] = sum; #else - + KW_LOCAL_MEM REAL sum[SUM_SITES_BLOCK_SIZE]; int tx = KW_LOCAL_ID_0; @@ -1294,7 +1284,7 @@ KW_GLOBAL_KERNEL void kernelSumSites1Partition(KW_GLOBAL_VAR REAL* dArray, int startPattern, int endPattern) { #ifdef FW_OPENCL_CPU - + REAL sum = 0; int pattern = startPattern + KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE; @@ -1311,7 +1301,7 @@ KW_GLOBAL_KERNEL void kernelSumSites1Partition(KW_GLOBAL_VAR REAL* dArray, dSum[KW_GROUP_ID_0] = sum; #else - + KW_LOCAL_MEM REAL sum[SUM_SITES_BLOCK_SIZE]; int tx = KW_LOCAL_ID_0; @@ -1340,13 +1330,13 @@ KW_GLOBAL_KERNEL void kernelSumSites1Partition(KW_GLOBAL_VAR REAL* dArray, // KW_GLOBAL_VAR REAL* dSum, // KW_GLOBAL_VAR REAL* dPatternWeights, // KW_GLOBAL_VAR unsigned int* dPtrOffsets) { - + // int opIndexPtr = KW_GROUP_ID_0 * 2; // int startPattern = dPtrOffsets[opIndexPtr ]; // int endPattern = dPtrOffsets[opIndexPtr + 1]; // #ifdef FW_OPENCL_CPU - + // REAL sum = 0; // int pattern = startPattern + KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE; @@ -1359,7 +1349,7 @@ KW_GLOBAL_KERNEL void kernelSumSites1Partition(KW_GLOBAL_VAR REAL* dArray, // dSum[KW_GROUP_ID_0] = sum; // #else - + // KW_LOCAL_MEM REAL sum[SUM_SITES_BLOCK_SIZE]; // int tx = KW_LOCAL_ID_0; @@ -1392,7 +1382,7 @@ KW_GLOBAL_KERNEL void kernelSumSites2(KW_GLOBAL_VAR REAL* dArray1, int patternCount) { #ifdef FW_OPENCL_CPU - + REAL sum1 = 0, sum2 = 0; int pattern = KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE; @@ -1455,7 +1445,7 @@ KW_GLOBAL_KERNEL void kernelSumSites3(KW_GLOBAL_VAR REAL* dArray1, int patternCount) { #ifdef FW_OPENCL_CPU - + REAL sum1 = 0, sum2 = 0, sum3 = 0; int pattern = KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE; diff --git a/libhmsbeagle/GPU/kernels/kernelsX.cu b/libhmsbeagle/GPU/kernels/kernelsX.cu index 62c58f71..f94d757f 100644 --- a/libhmsbeagle/GPU/kernels/kernelsX.cu +++ b/libhmsbeagle/GPU/kernels/kernelsX.cu @@ -4,19 +4,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Marc Suchard * @author Daniel Ayres @@ -25,7 +15,7 @@ #ifdef CUDA #include "libhmsbeagle/GPU/GPUImplDefs.h" #include "libhmsbeagle/GPU/kernels/kernelsAll.cu" // This file includes the non-state-count specific kernels - extern "C" { + extern "C" { #endif /////////////////////////////////////////////////////////////////////////////// @@ -535,7 +525,7 @@ KW_GLOBAL_KERNEL void kernelStatesStatesNoScale(KW_GLOBAL_VAR int* KW_RESTRICT s int state1 = states1[pattern]; int state2 = states2[pattern]; KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix + state1 * PADDED_STATE_COUNT; - KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix + state2 * PADDED_STATE_COUNT; + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix + state2 * PADDED_STATE_COUNT; if (state1 < PADDED_STATE_COUNT && state2 < PADDED_STATE_COUNT) { partials3[u] = matrix1[state] * matrix2[state]; } else if (state1 < PADDED_STATE_COUNT) { @@ -550,7 +540,7 @@ KW_GLOBAL_KERNEL void kernelStatesStatesNoScale(KW_GLOBAL_VAR int* KW_RESTRICT s int state1 = states1[pattern]; int state2 = states2[pattern]; KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix + state1 * PADDED_STATE_COUNT; - KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix + state2 * PADDED_STATE_COUNT; + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix + state2 * PADDED_STATE_COUNT; if (pattern < totalPatterns) { if (state1 < PADDED_STATE_COUNT && state2 < PADDED_STATE_COUNT) { partials3[u] = matrix1[state] * matrix2[state]; @@ -1131,7 +1121,7 @@ KW_GLOBAL_KERNEL void kernelStatesPartialsEdgeLikelihoodsSecondDeriv(KW_GLOBAL_V if (pattern < totalPatterns) { dPartialsTmp[u] = sum1 * sPartials2[patIdx][state]; dFirstDerivTmp[u] = sumFirstDeriv * sPartials2[patIdx][state]; - dSecondDerivTmp[u] = sumSecondDeriv * sPartials2[patIdx][state]; + dSecondDerivTmp[u] = sumSecondDeriv * sPartials2[patIdx][state]; } #endif // FW_OPENCL_CPU } @@ -1281,27 +1271,27 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsAutoScale(KW_GLOBAL_VAR REAL* partia sPartials1[patIdx][0] = 0; } } - + KW_LOCAL_FENCE; - + int scalingActive = sPartials2[patIdx][0]; - + if (scalingActive) { // now using sPartials1 to store max unscaled partials3 sPartials1[patIdx][state] = tmpPartial; } - + KW_LOCAL_FENCE; - + // Unrolled parallel max-reduction if (scalingActive && state < 2) { REAL compare = sPartials1[patIdx][state + 2]; if (compare > sPartials1[patIdx][state]) sPartials1[patIdx][state] = compare; } - + KW_LOCAL_FENCE; - + if (scalingActive && state < 1) { REAL maxPartial = sPartials1[patIdx][1]; if (maxPartial < sPartials1[patIdx][0]) @@ -1312,7 +1302,7 @@ KW_GLOBAL_KERNEL void kernelPartialsPartialsAutoScale(KW_GLOBAL_VAR REAL* partia } KW_LOCAL_FENCE; - + if (scalingActive) partials3[u] = ldexp(sigTmp, expTmp - sPartials1[patIdx][0]); @@ -1356,14 +1346,14 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsAutoScaling(KW_GLOBAL_VAR REAL* int u = state + pattern * PADDED_STATE_COUNT; int delta = patternCount * PADDED_STATE_COUNT; - + short maxScaleFactor = matrixScalers[0]; for(int r = 1; r < matrixCount; r++) { int tmpFactor = matrixScalers[r]; if (tmpFactor > maxScaleFactor) maxScaleFactor = tmpFactor; } - + for(int r = 0; r < matrixCount; r++) { int tmpFactor = matrixScalers[r]; if (tmpFactor != maxScaleFactor) { diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index bd28ec39..4e91d709 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Andrew Rambaut * @author Marc Suchard diff --git a/libhmsbeagle/beagle.h b/libhmsbeagle/beagle.h index 8ceaafc6..d055d256 100644 --- a/libhmsbeagle/beagle.h +++ b/libhmsbeagle/beagle.h @@ -5,25 +5,15 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @brief This file documents the API as well as header for the * Broad-platform Evolutionary Analysis General Likelihood Evaluator * * KEY CONCEPTS - * + * * The key to BEAGLE performance lies in delivering fine-scale * parallelization while minimizing data transfer and memory copy overhead. * To accomplish this, the library lacks the concept of data structure for @@ -42,9 +32,9 @@ * calls can be asynchronous, allowing the calling program to implement * other coarse-scale parallelization schemes such as evaluating * independent genes or running concurrent Markov chains. - * + * * USAGE - * + * * To use the library, a client program first creates an instance of BEAGLE * by calling beagleCreateInstance; multiple instances per client are * possible and encouraged. All additional functions are called with a @@ -61,7 +51,7 @@ * can then be specified, including the equilibrium state frequencies, the * rates for one or more substitution rate categories and their weights, * and finally, the eigen decomposition for the substitution process. - * + * * In order to calculate the likelihood of a particular tree, the client * program then specifies a series of integration operations that * correspond to steps in Felsenstein’s algorithm. Finite-time transition @@ -77,7 +67,7 @@ * software to keep track of these dependencies. The final step in * evaluating the phylogenetic model is done using an API call that yields * a single log likelihood for the model given the data. - * + * * Aspects of the BEAGLE API design support both maximum likelihood (ML) * and Bayesian phylogenetic tree inference. For ML inference, API calls * can calculate first and second derivatives of the likelihood with @@ -102,7 +92,7 @@ * @author Marc Suchard * @author David Swofford * @author Derrick Zwickl - * + * */ #ifndef __beagle__ @@ -142,39 +132,39 @@ enum BeagleReturnCodes { enum BeagleFlags { BEAGLE_FLAG_PRECISION_SINGLE = 1 << 0, /**< Single precision computation */ BEAGLE_FLAG_PRECISION_DOUBLE = 1 << 1, /**< Double precision computation */ - + BEAGLE_FLAG_COMPUTATION_SYNCH = 1 << 2, /**< Synchronous computation (blocking) */ BEAGLE_FLAG_COMPUTATION_ASYNCH = 1 << 3, /**< Asynchronous computation (non-blocking) */ - + BEAGLE_FLAG_EIGEN_REAL = 1 << 4, /**< Real eigenvalue computation */ BEAGLE_FLAG_EIGEN_COMPLEX = 1 << 5, /**< Complex eigenvalue computation */ - + BEAGLE_FLAG_SCALING_MANUAL = 1 << 6, /**< Manual scaling */ BEAGLE_FLAG_SCALING_AUTO = 1 << 7, /**< Auto-scaling on (deprecated, may not work correctly) */ BEAGLE_FLAG_SCALING_ALWAYS = 1 << 8, /**< Scale at every updatePartials (deprecated, may not work correctly) */ BEAGLE_FLAG_SCALING_DYNAMIC = 1 << 25, /**< Manual scaling with dynamic checking (deprecated, may not work correctly) */ - + BEAGLE_FLAG_SCALERS_RAW = 1 << 9, /**< Save raw scalers */ BEAGLE_FLAG_SCALERS_LOG = 1 << 10, /**< Save log scalers */ - + BEAGLE_FLAG_INVEVEC_STANDARD = 1 << 20, /**< Inverse eigen vectors passed to BEAGLE have not been transposed */ BEAGLE_FLAG_INVEVEC_TRANSPOSED = 1 << 21, /**< Inverse eigen vectors passed to BEAGLE have been transposed */ - + BEAGLE_FLAG_VECTOR_SSE = 1 << 11, /**< SSE computation */ BEAGLE_FLAG_VECTOR_AVX = 1 << 24, /**< AVX computation */ BEAGLE_FLAG_VECTOR_NONE = 1 << 12, /**< No vector computation */ - + BEAGLE_FLAG_THREADING_CPP = 1 << 30, /**< C++11 threading */ BEAGLE_FLAG_THREADING_OPENMP = 1 << 13, /**< OpenMP threading */ BEAGLE_FLAG_THREADING_NONE = 1 << 14, /**< No threading (default) */ - + BEAGLE_FLAG_PROCESSOR_CPU = 1 << 15, /**< Use CPU as main processor */ BEAGLE_FLAG_PROCESSOR_GPU = 1 << 16, /**< Use GPU as main processor */ BEAGLE_FLAG_PROCESSOR_FPGA = 1 << 17, /**< Use FPGA as main processor */ BEAGLE_FLAG_PROCESSOR_CELL = 1 << 18, /**< Use Cell as main processor */ BEAGLE_FLAG_PROCESSOR_PHI = 1 << 19, /**< Use Intel Phi as main processor */ BEAGLE_FLAG_PROCESSOR_OTHER = 1 << 26, /**< Use other type of processor */ - + BEAGLE_FLAG_FRAMEWORK_CUDA = 1 << 22, /**< Use CUDA implementation with GPU resources */ BEAGLE_FLAG_FRAMEWORK_OPENCL = 1 << 23, /**< Use OpenCL implementation with GPU resources */ BEAGLE_FLAG_FRAMEWORK_CPU = 1 << 27, /**< Use CPU implementation */ @@ -318,7 +308,7 @@ BEAGLE_DLLEXPORT BeagleResourceList* beagleGetResourceList(void); * benchmark times and CPU performance ratios for each resource. Resources are benchmarked * with the given analysis parameters and the array is ordered from fastest to slowest. * If there is an error the function returns NULL. - * + * * @param tipCount Number of tip data elements (input) * @param compactBufferCount Number of compact state representation tips (input) * @param stateCount Number of states in the continuous-time Markov chain (input) @@ -331,14 +321,14 @@ BEAGLE_DLLEXPORT BeagleResourceList* beagleGetResourceList(void); * see BeagleFlags (input) * @param requirementFlags Bit-flags indicating required implementation characteristics, * see BeagleFlags (input) - * @param eigenModelCount Number of full-alignment rate matrix eigen-decomposition models (input) + * @param eigenModelCount Number of full-alignment rate matrix eigen-decomposition models (input) * @param partitionCount Number of partitions (input) * @param calculateDerivatives Indicates if calculation of derivatives are required (input) * @param benchmarkFlags Bit-flags indicating benchmarking preferences (input) * - * @return An ordered (fastest to slowest) list of hardware resources available to the library as a + * @return An ordered (fastest to slowest) list of hardware resources available to the library as a * BeagleBenchmarkedResourceList for the specified analysis parameters - * + * */ BEAGLE_DLLEXPORT BeagleBenchmarkedResourceList* beagleGetBenchmarkedResourceList(int tipCount, int compactBufferCount, @@ -428,7 +418,7 @@ BEAGLE_DLLEXPORT int beagleFinalize(void); * implementation. It should only be called after beagleCreateInstance and requires the * BEAGLE_FLAG_THREADING_CPP flag to be set. It has no effect on GPU-based * implementations. It has no effect with the default BEAGLE_FLAG_THREADING_NONE setting. - * If BEAGLE_FLAG_THREADING_CPP is set and this function is not called BEAGLE will use + * If BEAGLE_FLAG_THREADING_CPP is set and this function is not called BEAGLE will use * a heuristic to set an appropriate number of threads. * * @param instance Instance number (input) @@ -497,7 +487,7 @@ BEAGLE_DLLEXPORT int beagleSetRootPrePartials(const int instance, * @brief Set an instance partials buffer * * This function copies an array of partials into an instance buffer. The inPartials array should - * be stateCount * patternCount * categoryCount in length. + * be stateCount * patternCount * categoryCount in length. * * @param instance Instance number in which to set a partialsBuffer (input) * @param bufferIndex Index of destination partialsBuffer (input) @@ -560,8 +550,8 @@ BEAGLE_DLLEXPORT int beagleSetEigenDecomposition(int instance, */ BEAGLE_DLLEXPORT int beagleSetStateFrequencies(int instance, int stateFrequenciesIndex, - const double* inStateFrequencies); - + const double* inStateFrequencies); + /** * @brief Set a category weights buffer * @@ -617,7 +607,7 @@ BEAGLE_DLLEXPORT int beagleSetCategoryRatesWithIndex(int instance, */ BEAGLE_DLLEXPORT int beagleSetPatternWeights(int instance, const double* inPatternWeights); - + /** * @brief Set pattern partition assignments * @@ -660,11 +650,11 @@ BEAGLE_DLLEXPORT int beagleSetPatternPartitions(int instance, * This function convolves two lists of transition probability matrices. * * @param instance Instance number (input) - * @param firstIndices List of indices of the first transition probability matrices + * @param firstIndices List of indices of the first transition probability matrices * to convolve (input) * @param secondIndices List of indices of the second transition probability matrices * to convolve (input) - * @param resultIndices List of indices of resulting transition probability matrices + * @param resultIndices List of indices of resulting transition probability matrices * (input) * @param matrixCount Length of lists */ @@ -739,7 +729,7 @@ BEAGLE_DLLEXPORT int beagleUpdateTransitionMatrices(int instance, int count); /** - * @brief Calculate a list of transition probability matrices with + * @brief Calculate a list of transition probability matrices with * each category using a different eigen decompsition * * This function calculates a list of transition probabilities matrices and their first and @@ -757,7 +747,7 @@ BEAGLE_DLLEXPORT int beagleUpdateTransitionMatrices(int instance, * (input, NULL implies no calculation) * @param edgeLengths List of edge lengths with which to perform calculations (input) * @param count Length of lists (except for eigenIndices list) - * + * * @return error code */ BEAGLE_DLLEXPORT int beagleUpdateTransitionMatricesWithModelCategories(int instance, @@ -850,7 +840,7 @@ BEAGLE_DLLEXPORT int beagleSetDifferentialMatrix(int instance, BEAGLE_DLLEXPORT int beagleGetTransitionMatrix(int instance, int matrixIndex, double* outMatrix); - + /** * @brief Set multiple transition matrices * @@ -873,7 +863,7 @@ BEAGLE_DLLEXPORT int beagleSetTransitionMatrices(int instance, const double* paddedValues, int count); - + /** * @brief A list of integer indices which specify a partial likelihoods operation. */ @@ -1104,7 +1094,7 @@ BEAGLE_DLLEXPORT int beagleResetScaleFactorsByPartition(int instance, BEAGLE_DLLEXPORT int beagleCopyScaleFactors(int instance, int destScalingIndex, int srcScalingIndex); - + /** * @brief Get scale factors * @@ -1114,10 +1104,10 @@ BEAGLE_DLLEXPORT int beagleCopyScaleFactors(int instance, * @param srcScalingIndex Source scaleBuffer (input) * @param outScaleFactors Pointer to which to receive scaleFactors (output) */ -BEAGLE_DLLEXPORT int beagleGetScaleFactors(int instance, +BEAGLE_DLLEXPORT int beagleGetScaleFactors(int instance, int srcScalingIndex, - double* outScaleFactors); - + double* outScaleFactors); + /** * @brief Calculate site log likelihoods at a root node * @@ -1164,8 +1154,8 @@ BEAGLE_DLLEXPORT int beagleCalculateRootLogLikelihoods(int instance, * @param cumulativeScaleIndices List of scaleBuffers containing accumulated factors to apply to * each partialsBuffer (input). There should be one index for each * of bufferIndices - * @param partitionIndices List of partition indices indicating which sites in each - * partialsBuffer should be used (input). There should be one + * @param partitionIndices List of partition indices indicating which sites in each + * partialsBuffer should be used (input). There should be one * index for each of bufferIndices * @param partitionCount Number of distinct partitionIndices (input) * @param count Number of sets of partitions to integrate across (input) @@ -1287,7 +1277,7 @@ BEAGLE_DLLEXPORT int beagleCalculateEdgeLogLikelihoods(int instance, double* outSumSecondDerivative); /** - * @brief Calculate multiple site log likelihoods and derivatives along an edge with + * @brief Calculate multiple site log likelihoods and derivatives along an edge with * per partition buffers * * This function integrates lists of partials at a parent and child node with respect @@ -1307,8 +1297,8 @@ BEAGLE_DLLEXPORT int beagleCalculateEdgeLogLikelihoods(int instance, * @param cumulativeScaleIndices List of scaleBuffers containing accumulated factors to apply to * each partialsBuffer (input). There should be one index for each * of parentBufferIndices - * @param partitionIndices List of partition indices indicating which sites in each - * partialsBuffer should be used (input). There should be one + * @param partitionIndices List of partition indices indicating which sites in each + * partialsBuffer should be used (input). There should be one * index for each of parentBufferIndices * @param partitionCount Number of distinct partitionIndices (input) * @param count Number of sets of partitions to integrate across (input) @@ -1378,12 +1368,12 @@ BEAGLE_DLLEXPORT int beagleGetLogLikelihood(int instance, BEAGLE_DLLEXPORT int beagleGetDerivatives(int instance, double* outSumFirstDerivative, double* outSumSecondDerivative); - + /** * @brief Get site log likelihoods for last beagleCalculateRootLogLikelihoods or * beagleCalculateEdgeLogLikelihoods call * - * This function returns the log likelihoods for each site + * This function returns the log likelihoods for each site * * @param instance Instance number (input) * @param outLogLikelihoods Pointer to destination for resulting log likelihoods (output) @@ -1396,7 +1386,7 @@ BEAGLE_DLLEXPORT int beagleGetSiteLogLikelihoods(int instance, /** * @brief Get site derivatives for last beagleCalculateEdgeLogLikelihoods call * - * This function returns the derivatives for each site + * This function returns the derivatives for each site * * @param instance Instance number (input) * @param outFirstDerivatives Pointer to destination for resulting first derivatives (output) @@ -1406,8 +1396,8 @@ BEAGLE_DLLEXPORT int beagleGetSiteLogLikelihoods(int instance, */ BEAGLE_DLLEXPORT int beagleGetSiteDerivatives(int instance, double* outFirstDerivatives, - double* outSecondDerivatives); - + double* outSecondDerivatives); + /* using C calling conventions so that C programs can successfully link the beagle library * (closing brace) */ diff --git a/libhmsbeagle/benchmark/BeagleBenchmark.cpp b/libhmsbeagle/benchmark/BeagleBenchmark.cpp index d14055d0..61d27a97 100644 --- a/libhmsbeagle/benchmark/BeagleBenchmark.cpp +++ b/libhmsbeagle/benchmark/BeagleBenchmark.cpp @@ -6,19 +6,9 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * * @author Daniel Ayres * Based on synthetictest.cpp by Daniel Ayres, Aaron Darling. @@ -33,7 +23,7 @@ namespace benchmark { int gettimeofday(struct timeval *tp,void * tz) { FILETIME f; ULARGE_INTEGER ifreq; - LONGLONG res; + LONGLONG res; GetSystemTimeAsFileTime(&f); ifreq.HighPart = f.dwHighDateTime; ifreq.LowPart = f.dwLowDateTime; @@ -70,7 +60,7 @@ void gt_srand(unsigned int *seed, unsigned int newSeed) double* getRandomTipPartials( int nsites, int stateCount, unsigned int *seed ) { - double *partials = (double*) calloc(sizeof(double), nsites * stateCount); + double *partials = (double*) calloc(sizeof(double), nsites * stateCount); for( int i=0; i 1) { for (int i=0; i < partitionCount; i++) { beagleSetCategoryRatesWithIndex(instance, i, &rates[0]); @@ -218,20 +208,20 @@ int benchmarkResource(int resource, beagleSetCategoryRates(instance, &rates[0]); } - + double* patternWeights = (double*) malloc(sizeof(double) * nsites); - + for (int i = 0; i < nsites; i++) { patternWeights[i] = gt_rand(&seed) / (double) GT_RAND_MAX; - } + } beagleSetPatternWeights(instance, patternWeights); - + int* patternPartitions; double* partitionLogLs; double* partitionD1; double* partitionD2; - + if (partitionCount > 1) { partitionLogLs = (double*) malloc(sizeof(double) * partitionCount); partitionD1 = (double*) malloc(sizeof(double) * partitionCount); @@ -243,7 +233,7 @@ int benchmarkResource(int resource, if (sitePartition > partitionCount - 1) sitePartition = partitionCount - 1; patternPartitions[i] = sitePartition; - } + } } @@ -253,7 +243,7 @@ int benchmarkResource(int resource, #else double freqs[stateCount]; #endif - + // create an array containing site category weights #ifdef _WIN32 std::vector weights(rateCategoryCount); @@ -264,19 +254,19 @@ int benchmarkResource(int resource, for (int eigenIndex=0; eigenIndex < eigenCount; eigenIndex++) { for (int i = 0; i < rateCategoryCount; i++) { weights[i] = gt_rand(&seed) / (double) GT_RAND_MAX; - } - + } + beagleSetCategoryWeights(instance, eigenIndex, &weights[0]); } - + double* eval; eval = (double*)malloc(sizeof(double)*stateCount); double* evec = (double*)malloc(sizeof(double)*stateCount*stateCount); double* ivec = (double*)malloc(sizeof(double)*stateCount*stateCount); - + for (int eigenIndex=0; eigenIndex < modelCount; eigenIndex++) { if ((stateCount & (stateCount-1)) == 0) { - + for (int i=0; i(stateCount, stateCount); + + double** qmat=New2DArray(stateCount, stateCount); double* relNucRates = new double[(stateCount * stateCount - stateCount) / 2]; - + int rnum=0; for(int i=0;i(stateCount, stateCount);//eigenvecs double** teigvecs=New2DArray(stateCount, stateCount);//temp eigenvecs - double** inveigvecs=New2DArray(stateCount, stateCount);//inv eigenvecs + double** inveigvecs=New2DArray(stateCount, stateCount);//inv eigenvecs int* iwork=new int[stateCount]; double* work=new double[stateCount]; - + EigenRealGeneral(stateCount, qmat, eval, eigvalsimag, eigvecs, iwork, work); memcpy(*teigvecs, *eigvecs, stateCount*stateCount*sizeof(double)); InvertMatrix(teigvecs, stateCount, work, iwork, inveigvecs); - + for(int x=0;x 1 && i==0) { + if (partitionCount > 1 && i==0) { if (beagleSetPatternPartitions(instance, partitionCount, patternPartitions) != BEAGLE_SUCCESS) { return BEAGLE_ERROR_GENERAL; } @@ -545,19 +535,19 @@ int benchmarkResource(int resource, } int scalingFactorsCount = internalCount; - + for (int eigenIndex=0; eigenIndex < eigenCount; eigenIndex++) { if (manualScaling && !(i % rescaleFrequency)) { beagleResetScaleFactors(instance, cumulativeScalingFactorIndices[eigenIndex]); - + beagleAccumulateScaleFactors(instance, &scalingFactorsIndices[eigenIndex*internalCount], scalingFactorsCount, cumulativeScalingFactorIndices[eigenIndex]); - } + } } - + // calculate the site likelihoods at the root node if (!unrooted) { if (partitionCount > 1) { @@ -602,7 +592,7 @@ int benchmarkResource(int resource, (calcderivs ? &deriv1 : NULL), (calcderivs ? partitionD2 : NULL), (calcderivs ? &deriv2 : NULL)); - } else { + } else { beagleCalculateEdgeLogLikelihoods(instance, // instance rootIndices,// bufferIndices lastTipIndices, @@ -621,12 +611,12 @@ int benchmarkResource(int resource, } // end timing! gettimeofday(&time5,NULL); - + if (i == 0 || getTimeDiff(time0, time5) < bestTimeTotal) { bestTimeTotal = getTimeDiff(time0, time5); - } + } } - + // fprintf(stdout, "logL = %.5f \n", logL); // fprintf(stdout, "time = %.5f \n\n", getTimeDiff(time0, time5)); diff --git a/libhmsbeagle/benchmark/BeagleBenchmark.h b/libhmsbeagle/benchmark/BeagleBenchmark.h index 64279ad1..90409176 100644 --- a/libhmsbeagle/benchmark/BeagleBenchmark.h +++ b/libhmsbeagle/benchmark/BeagleBenchmark.h @@ -6,20 +6,10 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . - * * @author Daniel Ayres */ @@ -46,7 +36,7 @@ #ifdef _WIN32 //From January 1, 1601 (UTC). to January 1,1970 - #define FACTOR 0x19db1ded53e8000 + #define FACTOR 0x19db1ded53e8000 #include #include #else @@ -56,11 +46,11 @@ namespace beagle { namespace benchmark { -int benchmarkResource(int resource, - int stateCount, - int ntaxa, - int nsites, - bool manualScaling, +int benchmarkResource(int resource, + int stateCount, + int ntaxa, + int nsites, + bool manualScaling, int rateCategoryCount, int nreps, int compactTipCount, @@ -81,4 +71,4 @@ int benchmarkResource(int resource, } // namespace benchmark -} // namespace beagle \ No newline at end of file +} // namespace beagle diff --git a/libhmsbeagle/platform.h b/libhmsbeagle/platform.h index 721313d8..b5d08c5b 100644 --- a/libhmsbeagle/platform.h +++ b/libhmsbeagle/platform.h @@ -6,20 +6,10 @@ * * This file is part of BEAGLE. * - * BEAGLE is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * Use of this source code is governed by an MIT-style + * license that can be found in the LICENSE file or at + * https://opensource.org/licenses/MIT. * - * BEAGLE is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with BEAGLE. If not, see - * . - * * @author Aaron Darling */ @@ -29,9 +19,9 @@ #ifdef _WIN32 // needed to export library symbols #ifdef _EXPORTING -#define BEAGLE_DLLEXPORT __declspec(dllexport) +#define BEAGLE_DLLEXPORT __declspec(dllexport) #else -#define BEAGLE_DLLEXPORT __declspec(dllimport) +#define BEAGLE_DLLEXPORT __declspec(dllimport) #endif /* // automatically include the appropriate beagle library diff --git a/project/beagle-vs-2012-installer/COPYING.LESSER.rtf b/project/beagle-vs-2012-installer/COPYING.LESSER.rtf deleted file mode 100644 index db3d1b16..00000000 --- a/project/beagle-vs-2012-installer/COPYING.LESSER.rtf +++ /dev/null @@ -1,169 +0,0 @@ -{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\fswiss\fprq2\fcharset0 Arial;}} -{\*\generator Msftedit 5.41.21.2500;}\viewkind4\uc1\pard\lang1033\f0\fs18\tab\tab GNU LESSER GENERAL PUBLIC LICENSE\par - Version 3, 29 June 2007\par -\par - Copyright (C) 2007 Free Software Foundation, Inc. \par - Everyone is permitted to copy and distribute verbatim copies\par - of this license document, but changing it is not allowed.\par -\par -\par - This version of the GNU Lesser General Public License incorporates\par -the terms and conditions of version 3 of the GNU General Public\par -License, supplemented by the additional permissions listed below.\par -\par - 0. Additional Definitions.\par -\par - As used herein, "this License" refers to version 3 of the GNU Lesser\par -General Public License, and the "GNU GPL" refers to version 3 of the GNU\par -General Public License.\par -\par - "The Library" refers to a covered work governed by this License,\par -other than an Application or a Combined Work as defined below.\par -\par - An "Application" is any work that makes use of an interface provided\par -by the Library, but which is not otherwise based on the Library.\par -Defining a subclass of a class defined by the Library is deemed a mode\par -of using an interface provided by the Library.\par -\par - A "Combined Work" is a work produced by combining or linking an\par -Application with the Library. The particular version of the Library\par -with which the Combined Work was made is also called the "Linked\par -Version".\par -\par - The "Minimal Corresponding Source" for a Combined Work means the\par -Corresponding Source for the Combined Work, excluding any source code\par -for portions of the Combined Work that, considered in isolation, are\par -based on the Application, and not on the Linked Version.\par -\par - The "Corresponding Application Code" for a Combined Work means the\par -object code and/or source code for the Application, including any data\par -and utility programs needed for reproducing the Combined Work from the\par -Application, but excluding the System Libraries of the Combined Work.\par -\par - 1. Exception to Section 3 of the GNU GPL.\par -\par - You may convey a covered work under sections 3 and 4 of this License\par -without being bound by section 3 of the GNU GPL.\par -\par - 2. Conveying Modified Versions.\par -\par - If you modify a copy of the Library, and, in your modifications, a\par -facility refers to a function or data to be supplied by an Application\par -that uses the facility (other than as an argument passed when the\par -facility is invoked), then you may convey a copy of the modified\par -version:\par -\par - a) under this License, provided that you make a good faith effort to\par - ensure that, in the event an Application does not supply the\par - function or data, the facility still operates, and performs\par - whatever part of its purpose remains meaningful, or\par -\par - b) under the GNU GPL, with none of the additional permissions of\par - this License applicable to that copy.\par -\par - 3. Object Code Incorporating Material from Library Header Files.\par -\par - The object code form of an Application may incorporate material from\par -a header file that is part of the Library. You may convey such object\par -code under terms of your choice, provided that, if the incorporated\par -material is not limited to numerical parameters, data structure\par -layouts and accessors, or small macros, inline functions and templates\par -(ten or fewer lines in length), you do both of the following:\par -\par - a) Give prominent notice with each copy of the object code that the\par - Library is used in it and that the Library and its use are\par - covered by this License.\par -\par - b) Accompany the object code with a copy of the GNU GPL and this license\par - document.\par -\par - 4. Combined Works.\par -\par - You may convey a Combined Work under terms of your choice that,\par -taken together, effectively do not restrict modification of the\par -portions of the Library contained in the Combined Work and reverse\par -engineering for debugging such modifications, if you also do each of\par -the following:\par -\par - a) Give prominent notice with each copy of the Combined Work that\par - the Library is used in it and that the Library and its use are\par - covered by this License.\par -\par - b) Accompany the Combined Work with a copy of the GNU GPL and this license\par - document.\par -\par - c) For a Combined Work that displays copyright notices during\par - execution, include the copyright notice for the Library among\par - these notices, as well as a reference directing the user to the\par - copies of the GNU GPL and this license document.\par -\par - d) Do one of the following:\par -\par - 0) Convey the Minimal Corresponding Source under the terms of this\par - License, and the Corresponding Application Code in a form\par - suitable for, and under terms that permit, the user to\par - recombine or relink the Application with a modified version of\par - the Linked Version to produce a modified Combined Work, in the\par - manner specified by section 6 of the GNU GPL for conveying\par - Corresponding Source.\par -\par - 1) Use a suitable shared library mechanism for linking with the\par - Library. A suitable mechanism is one that (a) uses at run time\par - a copy of the Library already present on the user's computer\par - system, and (b) will operate properly with a modified version\par - of the Library that is interface-compatible with the Linked\par - Version.\par -\par - e) Provide Installation Information, but only if you would otherwise\par - be required to provide such information under section 6 of the\par - GNU GPL, and only to the extent that such information is\par - necessary to install and execute a modified version of the\par - Combined Work produced by recombining or relinking the\par - Application with a modified version of the Linked Version. (If\par - you use option 4d0, the Installation Information must accompany\par - the Minimal Corresponding Source and Corresponding Application\par - Code. If you use option 4d1, you must provide the Installation\par - Information in the manner specified by section 6 of the GNU GPL\par - for conveying Corresponding Source.)\par -\par - 5. Combined Libraries.\par -\par - You may place library facilities that are a work based on the\par -Library side by side in a single library together with other library\par -facilities that are not Applications and are not covered by this\par -License, and convey such a combined library under terms of your\par -choice, if you do both of the following:\par -\par - a) Accompany the combined library with a copy of the same work based\par - on the Library, uncombined with any other library facilities,\par - conveyed under the terms of this License.\par -\par - b) Give prominent notice with the combined library that part of it\par - is a work based on the Library, and explaining where to find the\par - accompanying uncombined form of the same work.\par -\par - 6. Revised Versions of the GNU Lesser General Public License.\par -\par - The Free Software Foundation may publish revised and/or new versions\par -of the GNU Lesser General Public License from time to time. Such new\par -versions will be similar in spirit to the present version, but may\par -differ in detail to address new problems or concerns.\par -\par - Each version is given a distinguishing version number. If the\par -Library as you received it specifies that a certain numbered version\par -of the GNU Lesser General Public License "or any later version"\par -applies to it, you have the option of following the terms and\par -conditions either of that published version or of any later version\par -published by the Free Software Foundation. If the Library as you\par -received it does not specify a version number of the GNU Lesser\par -General Public License, you may choose any version of the GNU Lesser\par -General Public License ever published by the Free Software Foundation.\par -\par - If the Library as you received it specifies that a proxy can decide\par -whether future versions of the GNU Lesser General Public License shall\par -apply, that proxy's public statement of acceptance of any version is\par -permanent authorization for you to choose that version for the\par -Library.\par -\par -} - \ No newline at end of file diff --git a/project/beagle-vs-2017/installer/COPYING.LESSER.rtf b/project/beagle-vs-2017/installer/COPYING.LESSER.rtf deleted file mode 100644 index db3d1b16..00000000 --- a/project/beagle-vs-2017/installer/COPYING.LESSER.rtf +++ /dev/null @@ -1,169 +0,0 @@ -{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0\fswiss\fprq2\fcharset0 Arial;}} -{\*\generator Msftedit 5.41.21.2500;}\viewkind4\uc1\pard\lang1033\f0\fs18\tab\tab GNU LESSER GENERAL PUBLIC LICENSE\par - Version 3, 29 June 2007\par -\par - Copyright (C) 2007 Free Software Foundation, Inc. \par - Everyone is permitted to copy and distribute verbatim copies\par - of this license document, but changing it is not allowed.\par -\par -\par - This version of the GNU Lesser General Public License incorporates\par -the terms and conditions of version 3 of the GNU General Public\par -License, supplemented by the additional permissions listed below.\par -\par - 0. Additional Definitions.\par -\par - As used herein, "this License" refers to version 3 of the GNU Lesser\par -General Public License, and the "GNU GPL" refers to version 3 of the GNU\par -General Public License.\par -\par - "The Library" refers to a covered work governed by this License,\par -other than an Application or a Combined Work as defined below.\par -\par - An "Application" is any work that makes use of an interface provided\par -by the Library, but which is not otherwise based on the Library.\par -Defining a subclass of a class defined by the Library is deemed a mode\par -of using an interface provided by the Library.\par -\par - A "Combined Work" is a work produced by combining or linking an\par -Application with the Library. The particular version of the Library\par -with which the Combined Work was made is also called the "Linked\par -Version".\par -\par - The "Minimal Corresponding Source" for a Combined Work means the\par -Corresponding Source for the Combined Work, excluding any source code\par -for portions of the Combined Work that, considered in isolation, are\par -based on the Application, and not on the Linked Version.\par -\par - The "Corresponding Application Code" for a Combined Work means the\par -object code and/or source code for the Application, including any data\par -and utility programs needed for reproducing the Combined Work from the\par -Application, but excluding the System Libraries of the Combined Work.\par -\par - 1. Exception to Section 3 of the GNU GPL.\par -\par - You may convey a covered work under sections 3 and 4 of this License\par -without being bound by section 3 of the GNU GPL.\par -\par - 2. Conveying Modified Versions.\par -\par - If you modify a copy of the Library, and, in your modifications, a\par -facility refers to a function or data to be supplied by an Application\par -that uses the facility (other than as an argument passed when the\par -facility is invoked), then you may convey a copy of the modified\par -version:\par -\par - a) under this License, provided that you make a good faith effort to\par - ensure that, in the event an Application does not supply the\par - function or data, the facility still operates, and performs\par - whatever part of its purpose remains meaningful, or\par -\par - b) under the GNU GPL, with none of the additional permissions of\par - this License applicable to that copy.\par -\par - 3. Object Code Incorporating Material from Library Header Files.\par -\par - The object code form of an Application may incorporate material from\par -a header file that is part of the Library. You may convey such object\par -code under terms of your choice, provided that, if the incorporated\par -material is not limited to numerical parameters, data structure\par -layouts and accessors, or small macros, inline functions and templates\par -(ten or fewer lines in length), you do both of the following:\par -\par - a) Give prominent notice with each copy of the object code that the\par - Library is used in it and that the Library and its use are\par - covered by this License.\par -\par - b) Accompany the object code with a copy of the GNU GPL and this license\par - document.\par -\par - 4. Combined Works.\par -\par - You may convey a Combined Work under terms of your choice that,\par -taken together, effectively do not restrict modification of the\par -portions of the Library contained in the Combined Work and reverse\par -engineering for debugging such modifications, if you also do each of\par -the following:\par -\par - a) Give prominent notice with each copy of the Combined Work that\par - the Library is used in it and that the Library and its use are\par - covered by this License.\par -\par - b) Accompany the Combined Work with a copy of the GNU GPL and this license\par - document.\par -\par - c) For a Combined Work that displays copyright notices during\par - execution, include the copyright notice for the Library among\par - these notices, as well as a reference directing the user to the\par - copies of the GNU GPL and this license document.\par -\par - d) Do one of the following:\par -\par - 0) Convey the Minimal Corresponding Source under the terms of this\par - License, and the Corresponding Application Code in a form\par - suitable for, and under terms that permit, the user to\par - recombine or relink the Application with a modified version of\par - the Linked Version to produce a modified Combined Work, in the\par - manner specified by section 6 of the GNU GPL for conveying\par - Corresponding Source.\par -\par - 1) Use a suitable shared library mechanism for linking with the\par - Library. A suitable mechanism is one that (a) uses at run time\par - a copy of the Library already present on the user's computer\par - system, and (b) will operate properly with a modified version\par - of the Library that is interface-compatible with the Linked\par - Version.\par -\par - e) Provide Installation Information, but only if you would otherwise\par - be required to provide such information under section 6 of the\par - GNU GPL, and only to the extent that such information is\par - necessary to install and execute a modified version of the\par - Combined Work produced by recombining or relinking the\par - Application with a modified version of the Linked Version. (If\par - you use option 4d0, the Installation Information must accompany\par - the Minimal Corresponding Source and Corresponding Application\par - Code. If you use option 4d1, you must provide the Installation\par - Information in the manner specified by section 6 of the GNU GPL\par - for conveying Corresponding Source.)\par -\par - 5. Combined Libraries.\par -\par - You may place library facilities that are a work based on the\par -Library side by side in a single library together with other library\par -facilities that are not Applications and are not covered by this\par -License, and convey such a combined library under terms of your\par -choice, if you do both of the following:\par -\par - a) Accompany the combined library with a copy of the same work based\par - on the Library, uncombined with any other library facilities,\par - conveyed under the terms of this License.\par -\par - b) Give prominent notice with the combined library that part of it\par - is a work based on the Library, and explaining where to find the\par - accompanying uncombined form of the same work.\par -\par - 6. Revised Versions of the GNU Lesser General Public License.\par -\par - The Free Software Foundation may publish revised and/or new versions\par -of the GNU Lesser General Public License from time to time. Such new\par -versions will be similar in spirit to the present version, but may\par -differ in detail to address new problems or concerns.\par -\par - Each version is given a distinguishing version number. If the\par -Library as you received it specifies that a certain numbered version\par -of the GNU Lesser General Public License "or any later version"\par -applies to it, you have the option of following the terms and\par -conditions either of that published version or of any later version\par -published by the Free Software Foundation. If the Library as you\par -received it does not specify a version number of the GNU Lesser\par -General Public License, you may choose any version of the GNU Lesser\par -General Public License ever published by the Free Software Foundation.\par -\par - If the Library as you received it specifies that a proxy can decide\par -whether future versions of the GNU Lesser General Public License shall\par -apply, that proxy's public statement of acceptance of any version is\par -permanent authorization for you to choose that version for the\par -Library.\par -\par -} - \ No newline at end of file diff --git a/project/beagle-xcode/beagle-package.pmdoc/index.xml b/project/beagle-xcode/beagle-package.pmdoc/index.xml deleted file mode 100644 index 5ef7a4c6..00000000 --- a/project/beagle-xcode/beagle-package.pmdoc/index.xml +++ /dev/null @@ -1,106 +0,0 @@ -BEAGLE v2.1.2/Users/ayres/developer/hmsbeagle/project/beagle-xcode/BEAGLE-2.1.2.pkghms.beagle-libbeagle-icon.png\ -Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.\ -This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below.\ -\pard\pardeftab720\sl540 - -\b\fs36 \cf2 0. Additional Definitions.\ -\pard\pardeftab720\sl360\sa280 - -\b0\fs28 \cf2 As used herein, \'93this License\'94 refers to version 3 of the GNU Lesser General Public License, and the \'93GNU GPL\'94 refers to version 3 of the GNU General Public License.\ -\'93The Library\'94 refers to a covered work governed by this License, other than an Application or a Combined Work as defined below.\ -An \'93Application\'94 is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library.\ -A \'93Combined Work\'94 is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the \'93Linked Version\'94.\ -The \'93Minimal Corresponding Source\'94 for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version.\ -The \'93Corresponding Application Code\'94 for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work.\ -\pard\pardeftab720\sl540 - -\b\fs36 \cf2 1. Exception to Section 3 of the GNU GPL.\ -\pard\pardeftab720\sl360\sa280 - -\b0\fs28 \cf2 You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL.\ -\pard\pardeftab720\sl540 - -\b\fs36 \cf2 2. Conveying Modified Versions.\ -\pard\pardeftab720\sl360\sa280 - -\b0\fs28 \cf2 If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version:\ -\pard\tx220\tx720\pardeftab720\li720\fi-720\sl360 -\ls1\ilvl0\cf2 {\listtext \uc0\u9642 }a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or\ -{\listtext \uc0\u9642 }b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy.\ -\pard\pardeftab720\sl540 - -\b\fs36 \cf2 3. Object Code Incorporating Material from Library Header Files.\ -\pard\pardeftab720\sl360\sa280 - -\b0\fs28 \cf2 The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following:\ -\pard\tx220\tx720\pardeftab720\li720\fi-720\sl360 -\ls2\ilvl0\cf2 {\listtext \uc0\u9642 }a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License.\ -{\listtext \uc0\u9642 }b) Accompany the object code with a copy of the GNU GPL and this license document.\ -\pard\pardeftab720\sl540 - -\b\fs36 \cf2 4. Combined Works.\ -\pard\pardeftab720\sl360\sa280 - -\b0\fs28 \cf2 You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following:\ -\pard\tx220\tx720\pardeftab720\li720\fi-720\sl360 -\ls3\ilvl0\cf2 {\listtext \uc0\u9642 }a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License.\ -{\listtext \uc0\u9642 }b) Accompany the Combined Work with a copy of the GNU GPL and this license document.\ -{\listtext \uc0\u9642 }c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document.\ -{\listtext \uc0\u9642 }d) Do one of the following:\ -\pard\tx940\tx1440\pardeftab720\li1440\fi-1440\sl360 -\ls3\ilvl1\cf2 {\listtext \uc0\u9642 }0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.\ -{\listtext \uc0\u9642 }1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version.\ -\pard\tx220\tx720\pardeftab720\li720\fi-720\sl360 -\ls3\ilvl0\cf2 {\listtext \uc0\u9642 }e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.)\ -\pard\pardeftab720\sl540 - -\b\fs36 \cf2 5. Combined Libraries.\ -\pard\pardeftab720\sl360\sa280 - -\b0\fs28 \cf2 You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following:\ -\pard\tx220\tx720\pardeftab720\li720\fi-720\sl360 -\ls4\ilvl0\cf2 {\listtext \uc0\u9642 }a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License.\ -{\listtext \uc0\u9642 }b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work.\ -\pard\pardeftab720\sl540 - -\b\fs36 \cf2 6. Revised Versions of the GNU Lesser General Public License.\ -\pard\pardeftab720\sl360\sa280 - -\b0\fs28 \cf2 The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.\ -Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License \'93or any later version\'94 applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation.\ -If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library.}]]>01libhmsbeagle.xml02libhmsbeagle.xml03libhmsbeagle.xml04libhmsbeagle.xml05libhmsbeagle.xml06libhmsbeagle.xml07libhmsbeagle.xml08libhmsbeagle.xmlproperties.titleproperties.customizeOptionproperties.anywhereDomainproperties.systemDomain \ No newline at end of file From 23a6fd21cffb7825ba52400ff2cd3fa23d6cfa34 Mon Sep 17 00:00:00 2001 From: "Marc A. Suchard" Date: Sun, 17 Jul 2022 06:39:39 -0700 Subject: [PATCH 004/421] update locations for 4.0 files --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 96e02005..991fc40f 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ The paper describing the algorithms used for calculating likelihoods of sequence ### Current binary installers -* [BEAGLE v4.0.0 for macOS universal](https://github.com/beagle-dev/beagle-lib/releases/download/v4.0.0/BEAGLE.v4.0.0.pkg) -* [BEAGLE v4.0.0 for Windows 64-bit](https://github.com/beagle-dev/beagle-lib/releases/download/v4.0.0/BEAGLE.v4.0.0.msi) +* [BEAGLE v4.0.0 for macOS universal](https://github.com/beagle-dev/beagle-lib/releases/download/v4.0.0/BEAGLE-4.0.0-Darwin-x86-ARM.pkg) +* [BEAGLE v4.0.0 for Windows 64-bit](https://github.com/beagle-dev/beagle-lib/releases/download/v4.0.0/BEAGLE-v4.0.0-win64.msi) ### Previous binary installers From 6513447e4262e5043e3cfb1bc1d88dd3e1946459 Mon Sep 17 00:00:00 2001 From: "Marc A. Suchard" Date: Sun, 17 Jul 2022 06:44:43 -0700 Subject: [PATCH 005/421] typo in filename --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 991fc40f..cfbebe57 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ The paper describing the algorithms used for calculating likelihoods of sequence ### Current binary installers * [BEAGLE v4.0.0 for macOS universal](https://github.com/beagle-dev/beagle-lib/releases/download/v4.0.0/BEAGLE-4.0.0-Darwin-x86-ARM.pkg) -* [BEAGLE v4.0.0 for Windows 64-bit](https://github.com/beagle-dev/beagle-lib/releases/download/v4.0.0/BEAGLE-v4.0.0-win64.msi) +* [BEAGLE v4.0.0 for Windows 64-bit](https://github.com/beagle-dev/beagle-lib/releases/download/v4.0.0/BEAGLE-4.0.0-win64.msi) ### Previous binary installers From 038d7ea49a0cb0d8efdc6a34beaff29c17f7880a Mon Sep 17 00:00:00 2001 From: Thanasi Bakis Date: Sat, 30 Jul 2022 16:25:44 -0700 Subject: [PATCH 006/421] Reinstate versioning for libhmsbeagle --- CMakeLists.txt | 2 ++ libhmsbeagle/CMakeLists.txt | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 402822d6..3f408162 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,8 @@ set(BEAGLE_VERSION_MAJOR "4") set(BEAGLE_VERSION_MINOR "0") set(BEAGLE_VERSION_PATCH "0") +set(GENERIC_API_VERSION "1") + set(BEAGLE_VERSION "${BEAGLE_VERSION_MAJOR}.${BEAGLE_VERSION_MINOR}.${BEAGLE_VERSION_PATCH}") set(BEAGLE_PLUGIN_VERSION "${BEAGLE_VERSION_MAJOR}${BEAGLE_VERSION_MINOR}") diff --git a/libhmsbeagle/CMakeLists.txt b/libhmsbeagle/CMakeLists.txt index 8715c353..bded606e 100644 --- a/libhmsbeagle/CMakeLists.txt +++ b/libhmsbeagle/CMakeLists.txt @@ -74,7 +74,9 @@ add_library(hmsbeagle SHARED ${BEAGLE_PLUGIN_SOURCE} ) -set_target_properties(hmsbeagle PROPERTIES PUBLIC_HEADER "beagle.h;platform.h") +set_target_properties(hmsbeagle PROPERTIES + PUBLIC_HEADER "beagle.h;platform.h" + VERSION ${GENERIC_API_VERSION}) if(NOT WIN32 AND NOT APPLE) target_link_libraries(hmsbeagle ${CMAKE_DL_LIBS}) From 1b31012fb1a7823d51f15ecfc9ad6892fd25469b Mon Sep 17 00:00:00 2001 From: Marc Suchard Date: Wed, 12 Oct 2022 12:05:20 -0700 Subject: [PATCH 007/421] patch for OpenCL on Penguin --- libhmsbeagle/GPU/CMakeLists.txt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/libhmsbeagle/GPU/CMakeLists.txt b/libhmsbeagle/GPU/CMakeLists.txt index 22b3c1ef..282288bf 100644 --- a/libhmsbeagle/GPU/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMakeLists.txt @@ -1,13 +1,18 @@ find_package(OpenCL) -if(OpenCL_FOUND AND BUILD_OPENCL) - message(STATUS "OpenCL Includes: ${OpenCL_INCLUDE_DIRS}") - message(STATUS "OpenCL Libraries: ${OpenCL_LIBRARIES}") - add_subdirectory("CMake_OpenCL") -endif(OpenCL_FOUND AND BUILD_OPENCL) +if(BUILD_OPENCL) + set(OpenCL_INCLUDE_DIRS /opt/rocm/opencl/include) + set(OpenCL_LIBRARIES /opt/rocm/opencl/lib/libOpenCL.so) + if(OpenCL_FOUND OR (OpenCL_INCLUDE_DIRS AND OpenCL_LIBRARIES)) + message(STATUS "OpenCL Includes: ${OpenCL_INCLUDE_DIRS}") + message(STATUS "OpenCL Libraries: ${OpenCL_LIBRARIES}") + add_subdirectory("CMake_OpenCL") + endif(OpenCL_FOUND OR (OpenCL_INCLUDE_DIRS AND OpenCL_LIBRARIES)) +endif(BUILD_OPENCL) find_package(CUDA) if(CUDA_FOUND AND BUILD_CUDA) message(STATUS "CUDA Includes: ${CUDA_INCLUDE_DIRS}") add_subdirectory("CMake_CUDA") endif(CUDA_FOUND AND BUILD_CUDA) + From 2883ced7533ece1d30ec404939a6cb097d54daba Mon Sep 17 00:00:00 2001 From: Marc Suchard Date: Mon, 12 Dec 2022 08:51:57 -0800 Subject: [PATCH 008/421] add option to disable SSE build --- CMakeLists.txt | 1 + examples/CMakeLists.txt | 12 ++++++++++-- libhmsbeagle/CPU/CMakeLists.txt | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f408162..0c29cd35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,7 @@ set(BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH true) option(BUILD_OPENCL "Build beagle with OpenCL library" ON) option(BUILD_CUDA "Build beagle with CUDA library" ON) option(BUILD_JNI "Build beagle with JNI library" ON) +option(BUILD_SSE "Build beagle with SSE library" ON) # Old config.h settings diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index ea341822..981ccaf1 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -30,16 +30,24 @@ add_executable(synthetictest target_link_libraries(hmctest hmsbeagle - hmsbeagle-cpu-sse +# hmsbeagle-cpu-sse hmsbeagle-cpu ${CMAKE_DL_LIBS}) target_link_libraries(synthetictest hmsbeagle - hmsbeagle-cpu-sse +# hmsbeagle-cpu-sse hmsbeagle-cpu ${CMAKE_DL_LIBS}) +if(BUILD_SSE) + target_link_libraries(hmctest + hmsbeagle-cpu-sse) + + target_link_libraries(synthetictest + hmsbeagle-cpu-sse) +endif(BUILD_SSE) + add_test(hmctest hmctest) #target_link_libraries(hmctest5 hmsbeagle ${CMAKE_DL_LIBS}) diff --git a/libhmsbeagle/CPU/CMakeLists.txt b/libhmsbeagle/CPU/CMakeLists.txt index b3f16345..b4490095 100644 --- a/libhmsbeagle/CPU/CMakeLists.txt +++ b/libhmsbeagle/CPU/CMakeLists.txt @@ -25,6 +25,7 @@ SET_TARGET_PROPERTIES(hmsbeagle-cpu SUFFIX "${BEAGLE_PLUGIN_SUFFIX}" ) +if(BUILD_SSE) add_library(hmsbeagle-cpu-sse SHARED BeagleCPU4StateSSEImpl.h BeagleCPU4StateSSEImpl.hpp @@ -55,3 +56,4 @@ SET_TARGET_PROPERTIES(hmsbeagle-cpu-sse SOVERSION "${BEAGLE_PLUGIN_VERSION_EXTENDED}" SUFFIX "${BEAGLE_PLUGIN_SUFFIX}" ) +endif(BUILD_SSE) From 2f5c1f30145247cf1d9260fe47cf45c80c2cd8ea Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 26 Jan 2023 16:49:21 -0600 Subject: [PATCH 009/421] add option to turn off sse build --- CMakeLists.txt | 1 + libhmsbeagle/CPU/CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 024492de..a60f618d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ option(BUILD_OPENCL "Build beagle with OpenCL library" ON) option(BUILD_CUDA "Build beagle with CUDA library" ON) option(BUILD_JNI "Build beagle with JNI library" ON) option(BUILD_OPENMP "Build beagle with OpenMP library" ON) +option(BUILD_SSE "Build beagle with SSE" ON) # Old config.h settings diff --git a/libhmsbeagle/CPU/CMakeLists.txt b/libhmsbeagle/CPU/CMakeLists.txt index b3f16345..b4490095 100644 --- a/libhmsbeagle/CPU/CMakeLists.txt +++ b/libhmsbeagle/CPU/CMakeLists.txt @@ -25,6 +25,7 @@ SET_TARGET_PROPERTIES(hmsbeagle-cpu SUFFIX "${BEAGLE_PLUGIN_SUFFIX}" ) +if(BUILD_SSE) add_library(hmsbeagle-cpu-sse SHARED BeagleCPU4StateSSEImpl.h BeagleCPU4StateSSEImpl.hpp @@ -55,3 +56,4 @@ SET_TARGET_PROPERTIES(hmsbeagle-cpu-sse SOVERSION "${BEAGLE_PLUGIN_VERSION_EXTENDED}" SUFFIX "${BEAGLE_PLUGIN_SUFFIX}" ) +endif(BUILD_SSE) From 3bdb30bd645e15983f8c8cf952564813e306ad83 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 26 Jan 2023 16:54:28 -0600 Subject: [PATCH 010/421] examples sse change --- examples/CMakeLists.txt | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9f240ef6..993be41e 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -30,21 +30,29 @@ add_executable(synthetictest target_link_libraries(hmctest hmsbeagle - hmsbeagle-cpu-sse hmsbeagle-cpu ${CMAKE_DL_LIBS}) +if(BUILD_SSE) + target_link_libraries(hmctest hmsbeagle-cpu-sse) +endif(BUILD_SSE) + target_link_libraries(hmctest5 hmsbeagle - hmsbeagle-cpu-sse hmsbeagle-cpu ${CMAKE_DL_LIBS}) +if(BUILD_SSE) + target_link_libraries(hmctest5 hmsbeagle-cpu-sse) +endif(BUILD_SSE) + target_link_libraries(synthetictest hmsbeagle - hmsbeagle-cpu-sse hmsbeagle-cpu ${CMAKE_DL_LIBS}) +if(BUILD_SSE) + target_link_libraries(synthetictest hmsbeagle-cpu-sse) +endif(BUILD_SSE) add_test(hmctest hmctest) From 345cb5d6b201a80329b67fe1502a6c41264316bd Mon Sep 17 00:00:00 2001 From: gkarthik Date: Fri, 3 Mar 2023 14:00:25 -0800 Subject: [PATCH 011/421] Added BEAGLE_DEBUG_SYNCH option to cmake --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a60f618d..beed393c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ option(BUILD_SSE "Build beagle with SSE" ON) OPTION(BEAGLE_BENCHMARK "Print runtimes" OFF) OPTION(BEAGLE_DEBUG_FLOW "Print flow" OFF) OPTION(BEAGLE_DEBUG_VALUES "Print debug values" OFF) +OPTION(BEAGLE_DEBUG_SYNCH "Synchronize threads on GPU for timing" OFF) IF(BEAGLE_BENCHMARK) ADD_DEFINITIONS(-DBEAGLE_BENCHMARK) @@ -35,6 +36,10 @@ IF(BEAGLE_DEBUG_VALUES) ADD_DEFINITIONS(-DBEAGLE_DEBUG_VALUES) ENDIF(BEAGLE_DEBUG_VALUES) +IF(BEAGLE_DEBUG_SYNCH) + ADD_DEFINITIONS(-DBEAGLE_DEBUG_SYNCH) +ENDIF(BEAGLE_DEBUG_SYNCH) + add_definitions( -DPACKAGE_BUGREPORT="beagle-dev@googlegroups.com" -DPACKAGE_NAME="${BEAGLE_PACKAGE_NAME}" From 31b4d9ef2b08f8af187f985f3a10e2844ff7d775 Mon Sep 17 00:00:00 2001 From: gkarthik Date: Fri, 3 Mar 2023 14:52:41 -0800 Subject: [PATCH 012/421] Removed old gpu->Synchronize() --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 03749244..331d9d03 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -3022,7 +3022,7 @@ int BeagleGPUImpl::upPrePartials(bool byPartition, // #ifdef BEAGLE_DEBUG_SYNCH - gpu->Synchronize(); + gpu->SynchronizeHost(); #endif return BEAGLE_SUCCESS; } From 28dba4a1c6cf36992f6e2f0fc8ca8821111010b4 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Sat, 11 Mar 2023 16:46:26 -0500 Subject: [PATCH 013/421] Generate and install *.pc and *.cmake files to find hmsbeagle-1 --- CMakeLists.txt | 3 +++ libhmsbeagle/CMakeLists.txt | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c29cd35..21110afe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,9 @@ set(GENERIC_API_VERSION "1") set(BEAGLE_VERSION "${BEAGLE_VERSION_MAJOR}.${BEAGLE_VERSION_MINOR}.${BEAGLE_VERSION_PATCH}") set(BEAGLE_PLUGIN_VERSION "${BEAGLE_VERSION_MAJOR}${BEAGLE_VERSION_MINOR}") +message(STATUS "BEAGLE_VERSION = ${BEAGLE_VERSION}") +message(STATUS "BEAGLE_PLUGIN_VERSION = ${BEAGLE_PLUGIN_VERSION}") + set(BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH true) option(BUILD_OPENCL "Build beagle with OpenCL library" ON) diff --git a/libhmsbeagle/CMakeLists.txt b/libhmsbeagle/CMakeLists.txt index bded606e..4669be8d 100644 --- a/libhmsbeagle/CMakeLists.txt +++ b/libhmsbeagle/CMakeLists.txt @@ -88,12 +88,43 @@ if(BUILD_JNI) add_subdirectory(JNI) endif(BUILD_JNI) +set(PKG "hmsbeagle-${GENERIC_API_VERSION}") + install(TARGETS hmsbeagle + EXPORT ${PKG} LIBRARY DESTINATION ${BEAGLE_INSTALL_DIR} PUBLIC_HEADER DESTINATION ${BEAGLE_HEADER_DIR} COMPONENT main + ) + +include(CMakePackageConfigHelpers) + +write_basic_package_version_file( + ${PKG}ConfigVersion.cmake + VERSION ${BEAGLE_VERSION} + COMPATIBILITY AnyNewerVersion ) +install(EXPORT ${PKG} + FILE ${PKG}Config.cmake + NAMESPACE ${PKG}-${BEAGLE_VERSION}:: + DESTINATION lib/cmake/${PKG} + ) + +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/hmsbeagle-1ConfigVersion.cmake" + DESTINATION lib/cmake/${PKG} + ) + +configure_file( + ${PKG}.pc.in + ${PKG}.pc + @ONLY +) + +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${PKG}.pc" + DESTINATION lib/pkg-config +) + if(WIN32) SET_TARGET_PROPERTIES(hmsbeagle PROPERTIES From e0a384c3b0f11b7fe318aae58d64e81a4e118562 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Sat, 11 Mar 2023 16:48:22 -0500 Subject: [PATCH 014/421] Remove old non-templated *.pc file. --- hmsbeagle-1.pc | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 hmsbeagle-1.pc diff --git a/hmsbeagle-1.pc b/hmsbeagle-1.pc deleted file mode 100644 index 9cae501a..00000000 --- a/hmsbeagle-1.pc +++ /dev/null @@ -1,11 +0,0 @@ -prefix=/usr/local -exec_prefix=${prefix} -libdir=${exec_prefix}/lib -includedir=${prefix}/include - -Name: hmsbeagle -Description: A library to calculate phylogenetic likelihoods. -Requires: -Version: 4.0.0 -Libs: -L${libdir} -lhmsbeagle -Cflags: -I${includedir}/libhmsbeagle-1 From 6c79eedefa26bf5c571e9043785c99ca1b4f9d83 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Sat, 11 Mar 2023 17:20:30 -0500 Subject: [PATCH 015/421] Add missing file. --- libhmsbeagle/hmsbeagle-1.pc.in | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 libhmsbeagle/hmsbeagle-1.pc.in diff --git a/libhmsbeagle/hmsbeagle-1.pc.in b/libhmsbeagle/hmsbeagle-1.pc.in new file mode 100644 index 00000000..ae514889 --- /dev/null +++ b/libhmsbeagle/hmsbeagle-1.pc.in @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: hmsbeagle +Description: A library to calculate phylogenetic likelihoods. +Requires: +Version: @BEAGLE_VERSION@ +Libs: -L"${libdir}" -lhmsbeagle +Cflags: -I"${includedir}/libhmsbeagle-1" From cec9314eca576f1b416a7eb311e0626174ac4591 Mon Sep 17 00:00:00 2001 From: "Marc A. Suchard" Date: Sun, 19 Mar 2023 06:08:14 -0700 Subject: [PATCH 016/421] how did these get hardcoded? --- libhmsbeagle/GPU/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/CMakeLists.txt b/libhmsbeagle/GPU/CMakeLists.txt index 282288bf..fca46f34 100644 --- a/libhmsbeagle/GPU/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMakeLists.txt @@ -1,8 +1,8 @@ find_package(OpenCL) if(BUILD_OPENCL) - set(OpenCL_INCLUDE_DIRS /opt/rocm/opencl/include) - set(OpenCL_LIBRARIES /opt/rocm/opencl/lib/libOpenCL.so) +# set(OpenCL_INCLUDE_DIRS /opt/rocm/opencl/include) +# set(OpenCL_LIBRARIES /opt/rocm/opencl/lib/libOpenCL.so) if(OpenCL_FOUND OR (OpenCL_INCLUDE_DIRS AND OpenCL_LIBRARIES)) message(STATUS "OpenCL Includes: ${OpenCL_INCLUDE_DIRS}") message(STATUS "OpenCL Libraries: ${OpenCL_LIBRARIES}") From ba1604885f696b57dd52779dad726eca79a40c24 Mon Sep 17 00:00:00 2001 From: gkarthik Date: Tue, 11 Apr 2023 09:29:30 -0700 Subject: [PATCH 017/421] Removed fprintf statements leftover from debugging --- libhmsbeagle/GPU/KernelLauncher.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 3f192223..92d8fb7d 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -285,17 +285,12 @@ void KernelLauncher::LoadKernels() { fMultipleNodeSiteSquaredReduction = gpu->GetFunction( "kernelMultipleNodeSiteSquaredReduction"); - - fprintf(stderr, "A\n"); - fPartialsPartialsCrossProducts = gpu->GetFunction( "kernelPartialsPartialsCrossProducts"); fPartialsStatesCrossProducts = gpu->GetFunction( "kernelPartialsStatesCrossProducts"); - fprintf(stderr, "B\n"); - if (kPaddedStateCount == 4) { // TODO Temporary hack until kernels are written fPartialsPartialsByPatternBlockCheckScaling = gpu->GetFunction( "kernelPartialsPartialsCheckScale"); From 7560750a7c89f18ce8af2ebc1474dd53cb19489e Mon Sep 17 00:00:00 2001 From: gkarthik Date: Wed, 28 Jun 2023 15:34:58 -0700 Subject: [PATCH 018/421] Measure energy consumption for each kernel. --- examples/CMakeLists.txt | 6 ++++++ libhmsbeagle/JNI/CMakeLists.txt | 7 ++++++- libhmsbeagle/benchmark/BenchmarkHelper.h | 26 ++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 993be41e..95962be7 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -27,10 +27,14 @@ add_executable(synthetictest #add_executable(complextest # complextest/complextest.cpp) +if(BEAGLE_BENCHMARK) + list(APPEND BENCHMARK_LIBS nvidia-ml) # Needed for nvml.h +endif() target_link_libraries(hmctest hmsbeagle hmsbeagle-cpu + ${BENCHMARK_LIBS} ${CMAKE_DL_LIBS}) if(BUILD_SSE) @@ -40,6 +44,7 @@ endif(BUILD_SSE) target_link_libraries(hmctest5 hmsbeagle hmsbeagle-cpu + ${BENCHMARK_LIBS} ${CMAKE_DL_LIBS}) if(BUILD_SSE) target_link_libraries(hmctest5 hmsbeagle-cpu-sse) @@ -49,6 +54,7 @@ endif(BUILD_SSE) target_link_libraries(synthetictest hmsbeagle hmsbeagle-cpu + ${BENCHMARK_LIBS} ${CMAKE_DL_LIBS}) if(BUILD_SSE) target_link_libraries(synthetictest hmsbeagle-cpu-sse) diff --git a/libhmsbeagle/JNI/CMakeLists.txt b/libhmsbeagle/JNI/CMakeLists.txt index 72578278..bd3f61b8 100644 --- a/libhmsbeagle/JNI/CMakeLists.txt +++ b/libhmsbeagle/JNI/CMakeLists.txt @@ -6,7 +6,12 @@ add_library(hmsbeagle-jni SHARED beagle_BeagleJNIWrapper.h ) -target_link_libraries(hmsbeagle-jni hmsbeagle) + +if(BEAGLE_BENCHMARK) + target_link_libraries(hmsbeagle-jni hmsbeagle nvidia-ml) +else() + target_link_libraries(hmsbeagle-jni hmsbeagle) +endif() if(APPLE) set_target_properties(hmsbeagle-jni PROPERTIES PREFIX "lib" SUFFIX ".jnilib") diff --git a/libhmsbeagle/benchmark/BenchmarkHelper.h b/libhmsbeagle/benchmark/BenchmarkHelper.h index f6c01b5c..1d932f18 100644 --- a/libhmsbeagle/benchmark/BenchmarkHelper.h +++ b/libhmsbeagle/benchmark/BenchmarkHelper.h @@ -17,17 +17,31 @@ double debugGetTime() { #include #include #include +#include std::map, int>> benchmarkDuration; +std::map energyConsumption; +nvmlReturn_t statusInit = nvmlInit(); + #define DEBUG_CREATE_TIME() // Note: Calling DEBUG_FINALIZE_TIME() in beagle_library_finalize() will lead to memory errors #define DEBUG_FINALIZE_TIME() std::string funcName(__func__);\ std::map, int>>::iterator it;\ -std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime " << std::endl;\ +std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ +}\ +nvmlReturn_t statusShutdown = nvmlShutdown();\ +std::map::iterator it2;\ +std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tEnergy (mJ)" << std::endl;\ +for (it2 = energyConsumption.begin(); it2 != energyConsumption.end(); it2++) {\ + std::cerr << it2->first << "\t" << it2->second << std::endl;\ } -#define DEBUG_START_TIME() auto start = std::chrono::high_resolution_clock::now(); +#define DEBUG_START_TIME() auto start = std::chrono::high_resolution_clock::now(); \ +nvmlDevice_t handle;\ +nvmlReturn_t status = nvmlDeviceGetHandleByIndex(0, &handle);\ +unsigned long long int energyStart;\ +status = nvmlDeviceGetTotalEnergyConsumption(handle, &energyStart); #define DEBUG_END_TIME() auto end = std::chrono::high_resolution_clock::now();\ std::string key = __func__;\ std::map, int>>::iterator it = benchmarkDuration.find(key);\ @@ -36,6 +50,14 @@ if(it != benchmarkDuration.end()){\ it->second.first += end-start;\ } else {\ benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key, std::pair, int>(end - start, 1)));\ +} \ +std::map::iterator it2 = energyConsumption.find(key); \ +unsigned long long int energyEnd;\ +status = nvmlDeviceGetTotalEnergyConsumption(handle, &energyEnd);\ +if(it2 != energyConsumption.end()){\ + it2->second += energyEnd - energyStart;\ +} else {\ + energyConsumption.insert(energyConsumption.end(), std::pair(key, energyEnd - energyStart));\ } #else #define DEBUG_CREATE_TIME() From 1dfc07ca02e5ca714b03ec1102899145a43939b4 Mon Sep 17 00:00:00 2001 From: gkarthik Date: Thu, 29 Jun 2023 12:41:41 -0500 Subject: [PATCH 019/421] Linking shared library and path to header files in top-level CmakeLists.txt --- examples/CMakeLists.txt | 2 +- libhmsbeagle/CMakeLists.txt | 23 +++++++++++++++++++++++ libhmsbeagle/JNI/CMakeLists.txt | 7 +------ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 95962be7..0654fd11 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -28,7 +28,7 @@ add_executable(synthetictest #add_executable(complextest # complextest/complextest.cpp) if(BEAGLE_BENCHMARK) - list(APPEND BENCHMARK_LIBS nvidia-ml) # Needed for nvml.h +# list(APPEND BENCHMARK_LIBS nvidia-ml) # Needed for nvml.h endif() target_link_libraries(hmctest diff --git a/libhmsbeagle/CMakeLists.txt b/libhmsbeagle/CMakeLists.txt index 4201f3d9..8ee48fd5 100644 --- a/libhmsbeagle/CMakeLists.txt +++ b/libhmsbeagle/CMakeLists.txt @@ -81,6 +81,29 @@ if(NOT WIN32) endif(NOT APPLE) endif(NOT WIN32) + +if(BEAGLE_BENCHMARK) + find_library(NVIDIA_ML_LIBRARY + NAMES nvidia-ml + PATHS /usr/lib/x86_64-linux-gnu + ) + target_link_libraries(hmsbeagle nvidia-ml) + + if(NOT NVIDIA_ML_LIBRARY) + message(FATAL_ERROR "Nvidia ML library not found") + else() + message(STATUS "Nvidia ML library found at ${NVIDIA_ML_LIBRARY}") + endif() + + + find_package(CUDA) + if(CUDA_FOUND AND BUILD_CUDA) + message(STATUS "CUDA Includes: ${CUDA_INCLUDE_DIRS}") + target_include_directories(hmsbeagle PRIVATE ${CUDA_INCLUDE_DIRS}) + endif(CUDA_FOUND AND BUILD_CUDA) + +endif() + set_target_properties(hmsbeagle PROPERTIES PUBLIC_HEADER "beagle.h;platform.h") add_subdirectory(CPU) diff --git a/libhmsbeagle/JNI/CMakeLists.txt b/libhmsbeagle/JNI/CMakeLists.txt index bd3f61b8..72578278 100644 --- a/libhmsbeagle/JNI/CMakeLists.txt +++ b/libhmsbeagle/JNI/CMakeLists.txt @@ -6,12 +6,7 @@ add_library(hmsbeagle-jni SHARED beagle_BeagleJNIWrapper.h ) - -if(BEAGLE_BENCHMARK) - target_link_libraries(hmsbeagle-jni hmsbeagle nvidia-ml) -else() - target_link_libraries(hmsbeagle-jni hmsbeagle) -endif() +target_link_libraries(hmsbeagle-jni hmsbeagle) if(APPLE) set_target_properties(hmsbeagle-jni PROPERTIES PREFIX "lib" SUFFIX ".jnilib") From 789c61f89429cb25a1f04bcbfd5c1c4255b9a432 Mon Sep 17 00:00:00 2001 From: gkarthik Date: Thu, 29 Jun 2023 21:20:17 -0500 Subject: [PATCH 020/421] Separated DEBUG_TIME and DEBUG_ENERGY. Added new flag BEAGLE_BENCHMARK_ENERGY to measure energy consumption. --- CMakeLists.txt | 5 ++ libhmsbeagle/GPU/KernelLauncher.cpp | 16 ++-- libhmsbeagle/beagle.cpp | 100 ++++++++++++++++++++++- libhmsbeagle/benchmark/BenchmarkHelper.h | 64 +++++++++------ 4 files changed, 151 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index beed393c..a93de987 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,11 +23,16 @@ OPTION(BEAGLE_BENCHMARK "Print runtimes" OFF) OPTION(BEAGLE_DEBUG_FLOW "Print flow" OFF) OPTION(BEAGLE_DEBUG_VALUES "Print debug values" OFF) OPTION(BEAGLE_DEBUG_SYNCH "Synchronize threads on GPU for timing" OFF) +OPTION(BEAGLE_BENCHMARK_ENERGY "Print energy consumption" OFF) IF(BEAGLE_BENCHMARK) ADD_DEFINITIONS(-DBEAGLE_BENCHMARK) ENDIF(BEAGLE_BENCHMARK) +IF(BEAGLE_BENCHMARK_ENERGY) + ADD_DEFINITIONS(-DBEAGLE_BENCHMARK_ENERGY) +ENDIF(BEAGLE_BENCHMARK_ENERGY) + IF(BEAGLE_DEBUG_FLOW) ADD_DEFINITIONS(-DBEAGLE_DEBUG_FLOW) ENDIF(BEAGLE_DEBUG_FLOW) diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 3a81bfe3..00128408 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -34,7 +34,6 @@ #include "libhmsbeagle/beagle.h" #include "libhmsbeagle/GPU/GPUImplDefs.h" #include "libhmsbeagle/GPU/KernelLauncher.h" -#include "libhmsbeagle/benchmark/BenchmarkHelper.h" /**************CODE***********/ @@ -53,7 +52,6 @@ KernelLauncher::KernelLauncher(GPUInterface* inGpu) { } KernelLauncher::~KernelLauncher() { - DEBUG_FINALIZE_TIME(); } void KernelLauncher::SetupKernelBlocksAndGrids() { @@ -750,7 +748,6 @@ void KernelLauncher::PartialsStatesEdgeFirstDerivatives(GPUPtr out, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::PartialsStatesEdgeFirstDerivatives\n"); #endif -DEBUG_START_TIME(); unsigned int saved = bgDerivativeGrid.y; bgDerivativeGrid.y = nodeCount; @@ -765,7 +762,6 @@ DEBUG_START_TIME(); } bgDerivativeGrid.y = saved; -DEBUG_END_TIME(); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsStatesEdgeFirstDerivatives\n"); #endif @@ -785,7 +781,7 @@ void KernelLauncher::PartialsPartialsEdgeFirstDerivatives(GPUPtr out, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::PartialsPartialsEdgeFirstDerivatives\n"); #endif -DEBUG_START_TIME(); + unsigned int saved = bgDerivativeGrid.y; bgDerivativeGrid.y = nodeCount; @@ -804,7 +800,7 @@ DEBUG_START_TIME(); } bgDerivativeGrid.y = saved; -DEBUG_END_TIME(); + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsPartialsEdgeFirstDerivatives\n"); #endif @@ -970,14 +966,14 @@ void KernelLauncher::PartialsStatesGrowing(GPUPtr partials1, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::PartialsStatesGrowing\n"); #endif -DEBUG_START_TIME(); + gpu->LaunchKernel(fPartialsStatesGrowing, bgPeelingBlock, bgPeelingGrid, 5, 6, partials1, states2, partials3, matrices1, matrices2, patternCount); gpu->SynchronizeDevice(); -DEBUG_END_TIME(); + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsStatesGrowing\n"); #endif @@ -994,14 +990,14 @@ void KernelLauncher::PartialsPartialsGrowing(GPUPtr partials1, #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::PartialsPartialsGrowing\n"); #endif -DEBUG_START_TIME(); + gpu->LaunchKernel(fPartialsPartialsGrowing, bgPeelingBlock, bgPeelingGrid, 5, 6, partials1, partials2, partials3, matrices1, matrices2, patternCount); gpu->SynchronizeDevice(); -DEBUG_END_TIME(); + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::PartialsPartialsGrowing\n"); #endif diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 20c0e388..62cde6c1 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -63,6 +63,9 @@ typedef std::pair > RsrcImpl; typedef std::list RsrcImplList; typedef std::list RsrcBenchPairList; +//std::map, int>> benchmarkDuration; +//std::map energyConsumption; + // #define BEAGLE_DEBUG_LOAD #define BEAGLE_PREORDER @@ -636,8 +639,10 @@ int beagleCreateInstance(int tipCount, BeagleInstanceDetails* returnInfo) { DEBUG_CREATE_TIME(); try { - if (instances == NULL) + if (instances == NULL) { + DEBUG_CREATE_ENERGY(); instances = new std::vector; + } if (rsrcList == NULL) beagleGetResourceList(); @@ -734,6 +739,7 @@ int beagleCreateInstance(int tipCount, int beagleFinalizeInstance(int instance) { DEBUG_FINALIZE_TIME(); + DEBUG_FINALIZE_ENERGY(); // TODO: Figure out why this doesn't work on L248. try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -756,11 +762,13 @@ int beagleFinalizeInstance(int instance) { int beagleSetCPUThreadCount(int instance, int threadCount) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setCPUThreadCount(threadCount); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -768,12 +776,14 @@ int beagleSetTipStates(int instance, int tipIndex, const int* inStates) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setTipStates(tipIndex, inStates); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } catch (std::bad_alloc &) { @@ -791,12 +801,14 @@ int beagleSetTipPartials(int instance, int tipIndex, const double* inPartials) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setTipPartials(tipIndex, inPartials); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } catch (std::bad_alloc &) { @@ -814,12 +826,14 @@ int beagleSetPartials(int instance, int bufferIndex, const double* inPartials) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setPartials(bufferIndex, inPartials); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } catch (std::bad_alloc &) { @@ -838,6 +852,7 @@ int beagleSetRootPrePartials(const int instance, const int *stateFrequenciesIndices, int count){ DEBUG_START_TIME(); + DEBUG_START_ENERGY(); try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -846,6 +861,7 @@ int beagleSetRootPrePartials(const int instance, stateFrequenciesIndices, count); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } catch (std::bad_alloc &) { @@ -861,12 +877,14 @@ int beagleSetRootPrePartials(const int instance, int beagleGetPartials(int instance, int bufferIndex, int scaleIndex, double* outPartials) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->getPartials(bufferIndex, scaleIndex, outPartials); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } catch (std::bad_alloc &) { @@ -886,6 +904,7 @@ int beagleSetEigenDecomposition(int instance, const double* inInverseEigenVectors, const double* inEigenValues) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -893,6 +912,7 @@ int beagleSetEigenDecomposition(int instance, int returnValue = beagleInstance->setEigenDecomposition(eigenIndex, inEigenVectors, inInverseEigenVectors, inEigenValues); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } catch (std::bad_alloc &) { @@ -910,11 +930,13 @@ int beagleSetStateFrequencies(int instance, int stateFrequenciesIndex, const double* inStateFrequencies) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setStateFrequencies(stateFrequenciesIndex, inStateFrequencies); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -922,22 +944,26 @@ int beagleSetCategoryWeights(int instance, int categoryWeightsIndex, const double* inCategoryWeights) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setCategoryWeights(categoryWeightsIndex, inCategoryWeights); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } int beagleSetPatternWeights(int instance, const double* inPatternWeights) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setPatternWeights(inPatternWeights); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -945,23 +971,27 @@ int beagleSetPatternPartitions(int instance, int partitionCount, const int* inPatternPartitions) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setPatternPartitions(partitionCount, inPatternPartitions); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } int beagleSetCategoryRates(int instance, const double* inCategoryRates) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setCategoryRates(inCategoryRates); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -979,11 +1009,13 @@ int beagleSetCategoryRatesWithIndex(int instance, int categoryRatesIndex, const double* inCategoryRates) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setCategoryRatesWithIndex(categoryRatesIndex, inCategoryRates); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -992,12 +1024,14 @@ int beagleSetTransitionMatrix(int instance, const double* inMatrix, double paddedValue) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setTransitionMatrix(matrixIndex, inMatrix, paddedValue); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1015,12 +1049,14 @@ int beagleSetDifferentialMatrix(int instance, int matrixIndex, const double* inMatrix) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setDifferentialMatrix(matrixIndex, inMatrix); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -1030,12 +1066,14 @@ int beagleSetTransitionMatrices(int instance, const double* paddedValues, int count) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->setTransitionMatrices(matrixIndices, inMatrices, paddedValues, count); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1053,11 +1091,13 @@ int beagleGetTransitionMatrix(int instance, int matrixIndex, double* outMatrix) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->getTransitionMatrix(matrixIndex,outMatrix); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -1067,6 +1107,7 @@ int beagleConvolveTransitionMatrices(int instance, const int* resultIndices, const int matrixCount) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -1075,6 +1116,7 @@ int beagleConvolveTransitionMatrices(int instance, int returnValue = beagleInstance->convolveTransitionMatrices(firstIndices, secondIndices, resultIndices, matrixCount); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } } @@ -1085,6 +1127,7 @@ int beagleAddTransitionMatrices(int instance, const int* resultIndices, int matrixCount) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -1094,6 +1137,7 @@ int beagleAddTransitionMatrices(int instance, firstIndices, secondIndices, resultIndices, matrixCount); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } } @@ -1103,6 +1147,7 @@ int beagleTransposeTransitionMatrices(int instance, const int* resultIndices, int matrixCount) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -1111,6 +1156,7 @@ int beagleTransposeTransitionMatrices(int instance, int returnValue = beagleInstance->transposeTransitionMatrices( inputIndices, resultIndices, matrixCount); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } } @@ -1123,6 +1169,7 @@ int beagleUpdateTransitionMatrices(int instance, const double* edgeLengths, int count) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -1131,6 +1178,7 @@ int beagleUpdateTransitionMatrices(int instance, firstDerivativeIndices, secondDerivativeIndices, edgeLengths, count); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1152,6 +1200,7 @@ int beagleUpdateTransitionMatricesWithModelCategories(int instance, const double* edgeLengths, int count) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -1160,6 +1209,7 @@ int beagleUpdateTransitionMatricesWithModelCategories(int instance, firstDerivativeIndices, secondDerivativeIndices, edgeLengths, count); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1183,6 +1233,7 @@ int beagleUpdateTransitionMatricesWithMultipleModels(int instance, const double* edgeLengths, int count) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; @@ -1190,6 +1241,7 @@ int beagleUpdateTransitionMatricesWithMultipleModels(int instance, probabilityIndices, firstDerivativeIndices, secondDerivativeIndices, edgeLengths, count); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -1199,12 +1251,14 @@ int beagleUpdatePartials(const int instance, int operationCount, int cumulativeScalingIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->updatePartials((const int*)operations, operationCount, cumulativeScalingIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1223,12 +1277,14 @@ int beagleUpdatePrePartials(const int instance, int operationCount, int cumulativeScalingIndex){ DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->updatePrePartials((const int *) operations, operationCount, cumulativeScalingIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -1236,11 +1292,13 @@ int beagleUpdatePartialsByPartition(const int instance, const BeagleOperationByPartition* operations, int operationCount) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->updatePartialsByPartition((const int*)operations, operationCount); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -1248,11 +1306,13 @@ int beagleUpdatePrePartialsByPartition(const int instance, const BeagleOperationByPartition* operations, int operationCount) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->updatePrePartialsByPartition((const int*)operations, operationCount); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -1260,6 +1320,7 @@ int beagleWaitForPartials(const int instance, const int* destinationPartials, int destinationPartialsCount) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -1267,6 +1328,7 @@ int beagleWaitForPartials(const int instance, int returnValue = beagleInstance->waitForPartials(destinationPartials, destinationPartialsCount); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1285,12 +1347,14 @@ int beagleAccumulateScaleFactors(int instance, int count, int cumulativeScalingIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->accumulateScaleFactors(scalingIndices, count, cumulativeScalingIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1310,12 +1374,14 @@ int beagleAccumulateScaleFactorsByPartition(int instance, int cumulativeScalingIndex, int partitionIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->accumulateScaleFactorsByPartition(scalingIndices, count, cumulativeScalingIndex, partitionIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1334,12 +1400,14 @@ int beagleRemoveScaleFactors(int instance, int count, int cumulativeScalingIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->removeScaleFactors(scalingIndices, count, cumulativeScalingIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1359,12 +1427,14 @@ int beagleRemoveScaleFactorsByPartition(int instance, int cumulativeScalingIndex, int partitionIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->removeScaleFactorsByPartition(scalingIndices, count, cumulativeScalingIndex, partitionIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1381,12 +1451,14 @@ int beagleRemoveScaleFactorsByPartition(int instance, int beagleResetScaleFactors(int instance, int cumulativeScalingIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->resetScaleFactors(cumulativeScalingIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1404,12 +1476,14 @@ int beagleResetScaleFactorsByPartition(int instance, int cumulativeScalingIndex, int partitionIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->resetScaleFactorsByPartition(cumulativeScalingIndex, partitionIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1427,12 +1501,14 @@ int beagleCopyScaleFactors(int instance, int destScalingIndex, int srcScalingIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->copyScaleFactors(destScalingIndex, srcScalingIndex); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1450,12 +1526,14 @@ int beagleGetScaleFactors(int instance, int srcScalingIndex, double* scaleFactors) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->getScaleFactors(srcScalingIndex, scaleFactors); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; // } // catch (std::bad_alloc &) { @@ -1477,6 +1555,7 @@ int beagleCalculateRootLogLikelihoods(int instance, int count, double* outSumLogLikelihood) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -1487,6 +1566,7 @@ int beagleCalculateRootLogLikelihoods(int instance, count, outSumLogLikelihood); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION union {double f; long l;} dfp; @@ -1521,6 +1601,7 @@ int beagleCalculateRootLogLikelihoodsByPartition(int instance, double* outSumLogLikelihoodByPartition, double* outSumLogLikelihood) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -1535,6 +1616,7 @@ int beagleCalculateRootLogLikelihoodsByPartition(int instance, outSumLogLikelihoodByPartition, outSumLogLikelihood); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION union {double f; long l;} dfp; @@ -1578,6 +1660,7 @@ int beagleCalculateEdgeLogLikelihoods(int instance, double* outSumFirstDerivative, double* outSumSecondDerivative) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -1591,6 +1674,7 @@ int beagleCalculateEdgeLogLikelihoods(int instance, outSumLogLikelihood, outSumFirstDerivative, outSumSecondDerivative); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION union {double f; long l;} dfp; @@ -1637,6 +1721,7 @@ int beagleCalculateEdgeLogLikelihoodsByPartition(int instance, double* outSumSecondDerivativeByPartition, double* outSumSecondDerivative) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); // try { beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) @@ -1660,6 +1745,7 @@ int beagleCalculateEdgeLogLikelihoodsByPartition(int instance, outSumSecondDerivativeByPartition, outSumSecondDerivative); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION union {double f; long l;} dfp; @@ -1696,11 +1782,13 @@ int beagleCalculateEdgeLogLikelihoodsByPartition(int instance, int beagleGetLogLikelihood(int instance, double* outSumLogLikelihood) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->getLogLikelihood(outSumLogLikelihood); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION union {double f; long l;} dfp; @@ -1716,12 +1804,14 @@ int beagleGetDerivatives(int instance, double* outSumFirstDerivative, double* outSumSecondDerivative) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->getDerivatives(outSumFirstDerivative, outSumSecondDerivative); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION union {double f; long l;} dfp; @@ -1739,11 +1829,13 @@ int beagleGetDerivatives(int instance, int beagleGetSiteLogLikelihoods(int instance, double* outLogLikelihoods) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->getSiteLogLikelihoods(outLogLikelihoods); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION union {double f; long l;} dfp; @@ -1761,11 +1853,13 @@ int beagleGetSiteDerivatives(int instance, double* outFirstDerivatives, double* outSecondDerivatives) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; int returnValue = beagleInstance->getSiteDerivatives(outFirstDerivatives, outSecondDerivatives); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION union {double f; long l;} dfp; @@ -1792,6 +1886,7 @@ int beagleCalculateEdgeDerivatives(int instance, double *outSumDerivatives, double *outSumSquaredDerivatives) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -1810,6 +1905,7 @@ int beagleCalculateEdgeDerivatives(int instance, outSumSquaredDerivatives); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } @@ -1824,6 +1920,7 @@ int beagleCalculateCrossProductDerivative(int instance, double *outSumDerivatives, double *outSumSquaredDerivatives) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -1840,6 +1937,7 @@ int beagleCalculateCrossProductDerivative(int instance, outSumSquaredDerivatives); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); return returnValue; } diff --git a/libhmsbeagle/benchmark/BenchmarkHelper.h b/libhmsbeagle/benchmark/BenchmarkHelper.h index 1d932f18..79de9e04 100644 --- a/libhmsbeagle/benchmark/BenchmarkHelper.h +++ b/libhmsbeagle/benchmark/BenchmarkHelper.h @@ -20,28 +20,15 @@ double debugGetTime() { #include std::map, int>> benchmarkDuration; -std::map energyConsumption; -nvmlReturn_t statusInit = nvmlInit(); #define DEBUG_CREATE_TIME() // Note: Calling DEBUG_FINALIZE_TIME() in beagle_library_finalize() will lead to memory errors -#define DEBUG_FINALIZE_TIME() std::string funcName(__func__);\ -std::map, int>>::iterator it;\ +#define DEBUG_FINALIZE_TIME() std::map, int>>::iterator it;\ std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ -}\ -nvmlReturn_t statusShutdown = nvmlShutdown();\ -std::map::iterator it2;\ -std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tEnergy (mJ)" << std::endl;\ -for (it2 = energyConsumption.begin(); it2 != energyConsumption.end(); it2++) {\ - std::cerr << it2->first << "\t" << it2->second << std::endl;\ } -#define DEBUG_START_TIME() auto start = std::chrono::high_resolution_clock::now(); \ -nvmlDevice_t handle;\ -nvmlReturn_t status = nvmlDeviceGetHandleByIndex(0, &handle);\ -unsigned long long int energyStart;\ -status = nvmlDeviceGetTotalEnergyConsumption(handle, &energyStart); +#define DEBUG_START_TIME() auto start = std::chrono::high_resolution_clock::now(); #define DEBUG_END_TIME() auto end = std::chrono::high_resolution_clock::now();\ std::string key = __func__;\ std::map, int>>::iterator it = benchmarkDuration.find(key);\ @@ -50,18 +37,49 @@ if(it != benchmarkDuration.end()){\ it->second.first += end-start;\ } else {\ benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key, std::pair, int>(end - start, 1)));\ -} \ -std::map::iterator it2 = energyConsumption.find(key); \ -unsigned long long int energyEnd;\ -status = nvmlDeviceGetTotalEnergyConsumption(handle, &energyEnd);\ -if(it2 != energyConsumption.end()){\ - it2->second += energyEnd - energyStart;\ -} else {\ - energyConsumption.insert(energyConsumption.end(), std::pair(key, energyEnd - energyStart));\ } #else #define DEBUG_CREATE_TIME() #define DEBUG_START_TIME() #define DEBUG_END_TIME() #define DEBUG_FINALIZE_TIME() +#endif + +#ifdef BEAGLE_BENCHMARK_ENERGY +#include +#include +#include +#include +#include + +std::map> energyConsumption; + +//TODO: Add checks on nvmlReturn_t objects + +#define DEBUG_CREATE_ENERGY() nvmlReturn_t statusInit = nvmlInit(); +#define DEBUG_FINALIZE_ENERGY() nvmlReturn_t statusShutdown = nvmlShutdown();\ +std::map>::iterator it2;\ +std::cerr << "\nBENCHMARKS ENERGY: " << __FILE__ << "\n" << "Function\tEnergy (mJ)\tnTimes" << std::endl;\ +for (it2 = energyConsumption.begin(); it2 != energyConsumption.end(); it2++) {\ + std::cerr << it2->first << "\t" << it2->second.first << "\t" << it2->second.second << std::endl;\ +} +#define DEBUG_START_ENERGY() nvmlDevice_t handle;\ +nvmlReturn_t status = nvmlDeviceGetHandleByIndex(0, &handle);\ +unsigned long long int energyStart;\ +status = nvmlDeviceGetTotalEnergyConsumption(handle, &energyStart); +#define DEBUG_END_ENERGY() unsigned long long int energyEnd;\ +status = nvmlDeviceGetTotalEnergyConsumption(handle, &energyEnd);\ +std::string energyKey = __func__;\ +std::map>::iterator it2 = energyConsumption.find(energyKey);\ +if(it2 != energyConsumption.end()){\ + it2->second.first += energyEnd - energyStart;\ + it2->second.second += 1;\ +} else {\ + energyConsumption.insert(energyConsumption.end(), std::pair>(energyKey, std::pair(energyEnd - energyStart, 1)));\ +} +#else +#define DEBUG_CREATE_ENERGY() +#define DEBUG_FINALIZE_ENERGY() +#define DEBUG_START_ENERGY() +#define DEBUG_END_ENERGY() #endif \ No newline at end of file From 9524a2f091e553261ece72075da12407bd18429c Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 7 Aug 2023 14:52:31 -0500 Subject: [PATCH 021/421] fix seg fault from pre fetch 4x6 (tapped) matrix without transpose in 4 state SSE impl --- libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp index 3207e314..80c798a3 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp @@ -104,7 +104,7 @@ #define SSE_PREFETCH_PRE_MATRICES(src_m1, src_m2, dest_vu_m1, dest_vu_m2) \ const double *m1 = (src_m1); \ const double *m2 = (src_m2); \ - for (int i = 0; i < OFFSET; i++, m1+=OFFSET, m2++) { \ + for (int i = 0; i < 4; i++, m1+=OFFSET, m2++) { \ dest_vu_m1[i][0].x[0] = m1[0]; \ dest_vu_m1[i][0].x[1] = m1[1]; \ dest_vu_m2[i][0].x[0] = m2[0*OFFSET]; \ From 876ffd99af6a9c42aa2c7af0fa6e1d72eb0a5427 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 7 Aug 2023 15:50:20 -0500 Subject: [PATCH 022/421] fetch full padded matrix for calcPrePartialsStates with missing data --- libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp index 80c798a3..de07460c 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp @@ -113,6 +113,12 @@ dest_vu_m1[i][1].x[1] = m1[3]; \ dest_vu_m2[i][1].x[0] = m2[2*OFFSET]; \ dest_vu_m2[i][1].x[1] = m2[3*OFFSET]; \ + } \ + for (int i = 4; i < OFFSET; i++, m2++) { \ + dest_vu_m2[i][0].x[0] = m2[0*OFFSET]; \ + dest_vu_m2[i][0].x[1] = m2[1*OFFSET]; \ + dest_vu_m2[i][1].x[0] = m2[2*OFFSET]; \ + dest_vu_m2[i][1].x[1] = m2[3*OFFSET]; \ } #define SSE_PREFETCH_MATRIX(src_m1, dest_vu_m1) \ From 21b49bffc2f8fd628624e2d4568382f6214948a6 Mon Sep 17 00:00:00 2001 From: gkarthik Date: Fri, 18 Aug 2023 09:27:18 -0700 Subject: [PATCH 023/421] Removed header files for energy from time benchmarks. --- libhmsbeagle/CMakeLists.txt | 2 +- libhmsbeagle/benchmark/BenchmarkHelper.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/libhmsbeagle/CMakeLists.txt b/libhmsbeagle/CMakeLists.txt index 8ee48fd5..e7b5ef73 100644 --- a/libhmsbeagle/CMakeLists.txt +++ b/libhmsbeagle/CMakeLists.txt @@ -82,7 +82,7 @@ if(NOT WIN32) endif(NOT WIN32) -if(BEAGLE_BENCHMARK) +if(BEAGLE_BENCHMARK_ENERGY) find_library(NVIDIA_ML_LIBRARY NAMES nvidia-ml PATHS /usr/lib/x86_64-linux-gnu diff --git a/libhmsbeagle/benchmark/BenchmarkHelper.h b/libhmsbeagle/benchmark/BenchmarkHelper.h index 79de9e04..e27880a4 100644 --- a/libhmsbeagle/benchmark/BenchmarkHelper.h +++ b/libhmsbeagle/benchmark/BenchmarkHelper.h @@ -17,7 +17,6 @@ double debugGetTime() { #include #include #include -#include std::map, int>> benchmarkDuration; From 458fe439c0a1c4aab52de80fd88c0860732c82ba Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 10 Oct 2023 15:15:00 +0300 Subject: [PATCH 024/421] Update sse2neon.h to latest for better support for Linux ARM64 * Use sse2neon.h from https://github.com/DLTcollab/sse2neon/blob/de2817727c72fc2f4ce9f54e2db6e40ce0548414/sse2neon.h This is current master from 10.10.2023 * _mm_shuffle_pd is provided by newer sse2neon.h https://github.com/DLTcollab/sse2neon/blob/de2817727c72fc2f4ce9f54e2db6e40ce0548414/sse2neon.h#L5118 * aarch64 does not support cpuid Signed-off-by: Martin Tzvetanov Grigorov --- .github/workflows/ci.yaml | 59 + libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp | 2 + libhmsbeagle/CPU/SSEDefinitions.h | 30 +- libhmsbeagle/CPU/sse2neon.h | 12210 ++++++++++++++-------- 4 files changed, 7628 insertions(+), 4673 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..c6afdcaf --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,59 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build: + name: Build on Linux x86_64 + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake file + + - name: Build + run: | + mkdir build + cd build + cmake .. + make -j + file libhmsbeagle/libhmsbeagle.so.* | grep x86-64 + + build-aarch64: + name: Build on Linux aarch64 + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build + uses: uraimo/run-on-arch-action@v2 + with: + arch: aarch64 + distro: ubuntu20.04 + githubToken: ${{ github.token }} + dockerRunArgs: | + --volume "${PWD}:/beagle-lib" + install: | + apt-get update -q -y + apt-get install -q -y cmake gcc g++ openjdk-11-jdk file + run: | + export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-arm64 + cd /beagle-lib + mkdir build + cd build + cmake .. + make -j + file libhmsbeagle/libhmsbeagle.so.* | grep aarch64 diff --git a/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp b/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp index 63e5217d..ad3e0008 100644 --- a/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp +++ b/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp @@ -116,6 +116,8 @@ bool check_sse2() if (edx & bit_SSE2) return true; return false; +#elif defined(__aarch64__) + return false; #else // HAVE_CPUID.H // Determine if cpuid supported: unsigned int res; diff --git a/libhmsbeagle/CPU/SSEDefinitions.h b/libhmsbeagle/CPU/SSEDefinitions.h index 0b5cc44f..8e60a943 100644 --- a/libhmsbeagle/CPU/SSEDefinitions.h +++ b/libhmsbeagle/CPU/SSEDefinitions.h @@ -23,40 +23,12 @@ #define DLS_USE_SSE2 #if defined(DLS_USE_SSE2) -# if defined(__ARM64_ARCH_8__) +# if defined(__aarch64__) # include "libhmsbeagle/CPU/sse2neon.h" # define _MM_SHUFFLE2(fp1,fp0) (((fp1) << 1) | (fp0)) # define VEC_SHUFFLE0(a,b) _mm_shuffle_pd(a, b, _MM_SHUFFLE2(0,0)) // vreinterpretq_f64_m128d(a) # define VEC_SHUFFLE1(a,b) _mm_shuffle_pd(a, b, _MM_SHUFFLE2(1,1)) // vreinterpretq_f64_m128d(a) -# if __has_builtin(__builtin_shufflevector) -# define _mm_shuffle_pd(a,b,imm) \ - __extension__({ \ - float64x2_t _input1 = vreinterpretq_f64_m128(a); \ - float64x2_t _input2 = vreinterpretq_f64_m128(b); \ - float64x2_t _shuf = __builtin_shufflevector( \ - _input1, _input2, (imm) & (0x1), ((imm) >> 1) & 0x1); \ - vreinterpretq_m128_f32(_shuf); \ - }) -# else -# error "Need to implement NEON translation of _mm_shuffle_pd" -# endif - - static inline __m128 _mm_div_pd(__m128 a, __m128 b) { - return vreinterpretq_m128_f64( - vdivq_f64(vreinterpretq_f64_m128(a), vreinterpretq_f64_m128(b))); - } - - static inline void _mm_store_sd(double* a, __m128 b) { - const auto _b = vreinterpretq_f64_m128(b); - a[0] = _b[0]; - } - static inline __m128 _mm_dp_pd(__m128 lhs, __m128 rhs, const int) { - auto const product = vmulq_f64(vreinterpretq_f64_m128d(lhs), - vreinterpretq_f64_m128d(rhs)); - auto const sum = product[0] + product[1]; // TODO Almost surely an more efficient way - return vreinterpretq_m128d_f64(vdupq_n_f64(sum)); - } # else # if !defined(DLS_MACOS) # include diff --git a/libhmsbeagle/CPU/sse2neon.h b/libhmsbeagle/CPU/sse2neon.h index 4e02e80d..8d605973 100644 --- a/libhmsbeagle/CPU/sse2neon.h +++ b/libhmsbeagle/CPU/sse2neon.h @@ -1,13 +1,9 @@ #ifndef SSE2NEON_H #define SSE2NEON_H -// Source: https://github.com/DLTcollab/sse2neon (05-Jan-2021) - // This header file provides a simple API translation layer // between SSE intrinsics to their corresponding Arm/Aarch64 NEON versions // -// This header file does not yet translate all of the SSE intrinsics. -// // Contributors to this work are: // John W. Ratcliff // Brandon Rowlett @@ -15,8 +11,8 @@ // Eric van Beurden // Alexander Potylitsin // Hasindu Gamaarachchi -// Jim Huang -// Mark Cheng +// Jim Huang +// Mark Cheng // Malcolm James MacLeod // Devin Hussey (easyaspi314) // Sebastian Pop @@ -24,7 +20,13 @@ // Danila Kutenin // François Turban (JishinMaster) // Pei-Hsuan Hung -// Yang-Hao Yuan +// Yang-Hao Yuan +// Syoyo Fujita +// Brecht Van Lommel +// Jonathan Hue +// Cuda Chen +// Aymen Qader +// Anthony Roberts /* * sse2neon is freely redistributable under the MIT License. @@ -50,33 +52,160 @@ /* Tunable configurations */ -/* Enable precise implementation of _mm_min_ps and _mm_max_ps +/* Enable precise implementation of math operations * This would slow down the computation a bit, but gives consistent result with - * x86 SSE2. (e.g. would solve a hole or NaN pixel in the rendering result) + * x86 SSE. (e.g. would solve a hole or NaN pixel in the rendering result) */ +/* _mm_min|max_ps|ss|pd|sd */ #ifndef SSE2NEON_PRECISE_MINMAX #define SSE2NEON_PRECISE_MINMAX (0) #endif +/* _mm_rcp_ps and _mm_div_ps */ +#ifndef SSE2NEON_PRECISE_DIV +#define SSE2NEON_PRECISE_DIV (0) +#endif +/* _mm_sqrt_ps and _mm_rsqrt_ps */ +#ifndef SSE2NEON_PRECISE_SQRT +#define SSE2NEON_PRECISE_SQRT (0) +#endif +/* _mm_dp_pd */ +#ifndef SSE2NEON_PRECISE_DP +#define SSE2NEON_PRECISE_DP (0) +#endif + +/* Enable inclusion of windows.h on MSVC platforms + * This makes _mm_clflush functional on windows, as there is no builtin. + */ +#ifndef SSE2NEON_INCLUDE_WINDOWS_H +#define SSE2NEON_INCLUDE_WINDOWS_H (0) +#endif +/* compiler specific definitions */ #if defined(__GNUC__) || defined(__clang__) #pragma push_macro("FORCE_INLINE") #pragma push_macro("ALIGN_STRUCT") #define FORCE_INLINE static inline __attribute__((always_inline)) #define ALIGN_STRUCT(x) __attribute__((aligned(x))) -#else -#error "Macro name collisions may happen with unsupported compiler." -#ifdef FORCE_INLINE -#undef FORCE_INLINE +#define _sse2neon_likely(x) __builtin_expect(!!(x), 1) +#define _sse2neon_unlikely(x) __builtin_expect(!!(x), 0) +#elif defined(_MSC_VER) +#if _MSVC_TRADITIONAL +#error Using the traditional MSVC preprocessor is not supported! Use /Zc:preprocessor instead. #endif +#ifndef FORCE_INLINE #define FORCE_INLINE static inline +#endif #ifndef ALIGN_STRUCT #define ALIGN_STRUCT(x) __declspec(align(x)) #endif +#define _sse2neon_likely(x) (x) +#define _sse2neon_unlikely(x) (x) +#else +#pragma message("Macro name collisions may happen with unsupported compilers.") +#endif + +/* C language does not allow initializing a variable with a function call. */ +#ifdef __cplusplus +#define _sse2neon_const static const +#else +#define _sse2neon_const const #endif #include #include +#if defined(_WIN32) +/* Definitions for _mm_{malloc,free} are provided by + * from both MinGW-w64 and MSVC. + */ +#define SSE2NEON_ALLOC_DEFINED +#endif + +/* If using MSVC */ +#ifdef _MSC_VER +#include +#if SSE2NEON_INCLUDE_WINDOWS_H +#include +#include +#endif + +#if !defined(__cplusplus) +#error sse2neon only supports C++ compilation with this compiler +#endif + +#ifdef SSE2NEON_ALLOC_DEFINED +#include +#endif + +#if (defined(_M_AMD64) || defined(__x86_64__)) || \ + (defined(_M_ARM64) || defined(__arm64__)) +#define SSE2NEON_HAS_BITSCAN64 +#endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define _sse2neon_define0(type, s, body) \ + __extension__({ \ + type _a = (s); \ + body \ + }) +#define _sse2neon_define1(type, s, body) \ + __extension__({ \ + type _a = (s); \ + body \ + }) +#define _sse2neon_define2(type, a, b, body) \ + __extension__({ \ + type _a = (a), _b = (b); \ + body \ + }) +#define _sse2neon_return(ret) (ret) +#else +#define _sse2neon_define0(type, a, body) [=](type _a) { body }(a) +#define _sse2neon_define1(type, a, body) [](type _a) { body }(a) +#define _sse2neon_define2(type, a, b, body) \ + [](type _a, type _b) { body }((a), (b)) +#define _sse2neon_return(ret) return ret +#endif + +#define _sse2neon_init(...) \ + { \ + __VA_ARGS__ \ + } + +/* Compiler barrier */ +#if defined(_MSC_VER) +#define SSE2NEON_BARRIER() _ReadWriteBarrier() +#else +#define SSE2NEON_BARRIER() \ + do { \ + __asm__ __volatile__("" ::: "memory"); \ + (void) 0; \ + } while (0) +#endif + +/* Memory barriers + * __atomic_thread_fence does not include a compiler barrier; instead, + * the barrier is part of __atomic_load/__atomic_store's "volatile-like" + * semantics. + */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +#include +#endif + +FORCE_INLINE void _sse2neon_smp_mb(void) +{ + SSE2NEON_BARRIER(); +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) + atomic_thread_fence(memory_order_seq_cst); +#elif defined(__GNUC__) || defined(__clang__) + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#else /* MSVC */ + __dmb(_ARM64_BARRIER_ISH); +#endif +} + /* Architecture-specific build options */ /* FIXME: #pragma GCC push_options is only available on GCC */ #if defined(__GNUC__) @@ -88,32 +217,79 @@ #if !defined(__ARM_NEON) || !defined(__ARM_NEON__) #error "You must enable NEON instructions (e.g. -mfpu=neon) to use SSE2NEON." #endif +#if !defined(__clang__) #pragma GCC push_options #pragma GCC target("fpu=neon") -#elif defined(__aarch64__) +#endif +#elif defined(__aarch64__) || defined(_M_ARM64) +#if !defined(__clang__) && !defined(_MSC_VER) #pragma GCC push_options #pragma GCC target("+simd") +#endif +#elif __ARM_ARCH == 8 +#if !defined(__ARM_NEON) || !defined(__ARM_NEON__) +#error \ + "You must enable NEON instructions (e.g. -mfpu=neon-fp-armv8) to use SSE2NEON." +#endif +#if !defined(__clang__) && !defined(_MSC_VER) +#pragma GCC push_options +#endif #else #error "Unsupported target. Must be either ARMv7-A+NEON or ARMv8-A." #endif #endif #include +#if (!defined(__aarch64__) && !defined(_M_ARM64)) && (__ARM_ARCH == 8) +#if defined __has_include && __has_include() +#include +#endif +#endif + +/* Apple Silicon cache lines are double of what is commonly used by Intel, AMD + * and other Arm microarchitectures use. + * From sysctl -a on Apple M1: + * hw.cachelinesize: 128 + */ +#if defined(__APPLE__) && (defined(__aarch64__) || defined(__arm64__)) +#define SSE2NEON_CACHELINE_SIZE 128 +#else +#define SSE2NEON_CACHELINE_SIZE 64 +#endif -/* Rounding functions require either Aarch64 instructions or libm failback */ -#if !defined(__aarch64__) +/* Rounding functions require either Aarch64 instructions or libm fallback */ +#if !defined(__aarch64__) && !defined(_M_ARM64) #include #endif +/* On ARMv7, some registers, such as PMUSERENR and PMCCNTR, are read-only + * or even not accessible in user mode. + * To write or access to these registers in user mode, + * we have to perform syscall instead. + */ +#if (!defined(__aarch64__) && !defined(_M_ARM64)) +#include +#endif + /* "__has_builtin" can be used to query support for built-in functions * provided by gcc/clang and other compilers that support it. */ #ifndef __has_builtin /* GCC prior to 10 or non-clang compilers */ /* Compatibility with gcc <= 9 */ -#if __GNUC__ <= 9 +#if defined(__GNUC__) && (__GNUC__ <= 9) #define __has_builtin(x) HAS##x #define HAS__builtin_popcount 1 #define HAS__builtin_popcountll 1 + +// __builtin_shuffle introduced in GCC 4.7.0 +#if (__GNUC__ >= 5) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 7)) +#define HAS__builtin_shuffle 1 +#else +#define HAS__builtin_shuffle 0 +#endif + +#define HAS__builtin_shufflevector 0 +#define HAS__builtin_nontemporal_store 0 #else #define __has_builtin(x) 0 #endif @@ -130,6 +306,26 @@ #define _MM_SHUFFLE(fp3, fp2, fp1, fp0) \ (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | ((fp0))) +#if __has_builtin(__builtin_shufflevector) +#define _sse2neon_shuffle(type, a, b, ...) \ + __builtin_shufflevector(a, b, __VA_ARGS__) +#elif __has_builtin(__builtin_shuffle) +#define _sse2neon_shuffle(type, a, b, ...) \ + __extension__({ \ + type tmp = {__VA_ARGS__}; \ + __builtin_shuffle(a, b, tmp); \ + }) +#endif + +#ifdef _sse2neon_shuffle +#define vshuffle_s16(a, b, ...) _sse2neon_shuffle(int16x4_t, a, b, __VA_ARGS__) +#define vshuffleq_s16(a, b, ...) _sse2neon_shuffle(int16x8_t, a, b, __VA_ARGS__) +#define vshuffle_s32(a, b, ...) _sse2neon_shuffle(int32x2_t, a, b, __VA_ARGS__) +#define vshuffleq_s32(a, b, ...) _sse2neon_shuffle(int32x4_t, a, b, __VA_ARGS__) +#define vshuffle_s64(a, b, ...) _sse2neon_shuffle(int64x1_t, a, b, __VA_ARGS__) +#define vshuffleq_s64(a, b, ...) _sse2neon_shuffle(int64x2_t, a, b, __VA_ARGS__) +#endif + /* Rounding mode macros. */ #define _MM_FROUND_TO_NEAREST_INT 0x00 #define _MM_FROUND_TO_NEG_INF 0x01 @@ -137,6 +333,25 @@ #define _MM_FROUND_TO_ZERO 0x03 #define _MM_FROUND_CUR_DIRECTION 0x04 #define _MM_FROUND_NO_EXC 0x08 +#define _MM_FROUND_RAISE_EXC 0x00 +#define _MM_FROUND_NINT (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_FLOOR (_MM_FROUND_TO_NEG_INF | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_CEIL (_MM_FROUND_TO_POS_INF | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_TRUNC (_MM_FROUND_TO_ZERO | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_RINT (_MM_FROUND_CUR_DIRECTION | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_NEARBYINT (_MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC) +#define _MM_ROUND_NEAREST 0x0000 +#define _MM_ROUND_DOWN 0x2000 +#define _MM_ROUND_UP 0x4000 +#define _MM_ROUND_TOWARD_ZERO 0x6000 +/* Flush zero mode macros. */ +#define _MM_FLUSH_ZERO_MASK 0x8000 +#define _MM_FLUSH_ZERO_ON 0x8000 +#define _MM_FLUSH_ZERO_OFF 0x0000 +/* Denormals are zeros mode macros. */ +#define _MM_DENORMALS_ZERO_MASK 0x0040 +#define _MM_DENORMALS_ZERO_ON 0x0040 +#define _MM_DENORMALS_ZERO_OFF 0x0000 /* indicate immediate constant argument in a given range */ #define __constrange(a, b) const @@ -152,13 +367,23 @@ typedef float32x4_t __m128; /* 128-bit vector containing 4 floats */ // On ARM 32-bit architecture, the float64x2_t is not supported. // The data type __m128d should be represented in a different way for related // intrinsic conversion. -#if defined(__aarch64__) +#if defined(__aarch64__) || defined(_M_ARM64) typedef float64x2_t __m128d; /* 128-bit vector containing 2 doubles */ #else typedef float32x4_t __m128d; #endif typedef int64x2_t __m128i; /* 128-bit vector containing integers */ +// __int64 is defined in the Intrinsics Guide which maps to different datatype +// in different data model +#if !(defined(_WIN32) || defined(_WIN64) || defined(__int64)) +#if (defined(__x86_64__) || defined(__i386__)) +#define __int64 long long +#else +#define __int64 int64_t +#endif +#endif + /* type-safe casting between types */ #define vreinterpretq_m128_f16(x) vreinterpretq_f32_f16(x) @@ -199,6 +424,9 @@ typedef int64x2_t __m128i; /* 128-bit vector containing integers */ #define vreinterpretq_m128i_u32(x) vreinterpretq_s64_u32(x) #define vreinterpretq_m128i_u64(x) vreinterpretq_s64_u64(x) +#define vreinterpretq_f32_m128i(x) vreinterpretq_f32_s64(x) +#define vreinterpretq_f64_m128i(x) vreinterpretq_f64_s64(x) + #define vreinterpretq_s8_m128i(x) vreinterpretq_s8_s64(x) #define vreinterpretq_s16_m128i(x) vreinterpretq_s16_s64(x) #define vreinterpretq_s32_m128i(x) vreinterpretq_s32_s64(x) @@ -235,36 +463,44 @@ typedef int64x2_t __m128i; /* 128-bit vector containing integers */ #define vreinterpret_f32_m64(x) vreinterpret_f32_s64(x) -#if defined(__aarch64__) +#if defined(__aarch64__) || defined(_M_ARM64) #define vreinterpretq_m128d_s32(x) vreinterpretq_f64_s32(x) #define vreinterpretq_m128d_s64(x) vreinterpretq_f64_s64(x) +#define vreinterpretq_m128d_u64(x) vreinterpretq_f64_u64(x) + #define vreinterpretq_m128d_f32(x) vreinterpretq_f64_f32(x) #define vreinterpretq_m128d_f64(x) (x) #define vreinterpretq_s64_m128d(x) vreinterpretq_s64_f64(x) +#define vreinterpretq_u32_m128d(x) vreinterpretq_u32_f64(x) +#define vreinterpretq_u64_m128d(x) vreinterpretq_u64_f64(x) + #define vreinterpretq_f64_m128d(x) (x) #define vreinterpretq_f32_m128d(x) vreinterpretq_f32_f64(x) #else #define vreinterpretq_m128d_s32(x) vreinterpretq_f32_s32(x) #define vreinterpretq_m128d_s64(x) vreinterpretq_f32_s64(x) + +#define vreinterpretq_m128d_u32(x) vreinterpretq_f32_u32(x) #define vreinterpretq_m128d_u64(x) vreinterpretq_f32_u64(x) #define vreinterpretq_m128d_f32(x) (x) #define vreinterpretq_s64_m128d(x) vreinterpretq_s64_f32(x) +#define vreinterpretq_u32_m128d(x) vreinterpretq_u32_f32(x) #define vreinterpretq_u64_m128d(x) vreinterpretq_u64_f32(x) #define vreinterpretq_f32_m128d(x) (x) #endif // A struct is defined in this header file called 'SIMDVec' which can be used -// by applications which attempt to access the contents of an _m128 struct +// by applications which attempt to access the contents of an __m128 struct // directly. It is important to note that accessing the __m128 struct directly // is bad coding practice by Microsoft: @see: -// https://msdn.microsoft.com/en-us/library/ayeb3ayc.aspx +// https://learn.microsoft.com/en-us/cpp/cpp/m128 // // However, some legacy source code may try to access the contents of an __m128 // struct directly so the developer can use the SIMDVec as an alias for it. Any @@ -300,12 +536,49 @@ typedef union ALIGN_STRUCT(16) SIMDVec { #define vreinterpretq_nth_u32_m128i(x, n) (((SIMDVec *) &x)->m128_u32[n]) #define vreinterpretq_nth_u8_m128i(x, n) (((SIMDVec *) &x)->m128_u8[n]) +/* SSE macros */ +#define _MM_GET_FLUSH_ZERO_MODE _sse2neon_mm_get_flush_zero_mode +#define _MM_SET_FLUSH_ZERO_MODE _sse2neon_mm_set_flush_zero_mode +#define _MM_GET_DENORMALS_ZERO_MODE _sse2neon_mm_get_denormals_zero_mode +#define _MM_SET_DENORMALS_ZERO_MODE _sse2neon_mm_set_denormals_zero_mode + +// Function declaration +// SSE +FORCE_INLINE unsigned int _MM_GET_ROUNDING_MODE(void); +FORCE_INLINE __m128 _mm_move_ss(__m128, __m128); +FORCE_INLINE __m128 _mm_or_ps(__m128, __m128); +FORCE_INLINE __m128 _mm_set_ps1(float); +FORCE_INLINE __m128 _mm_setzero_ps(void); +// SSE2 +FORCE_INLINE __m128i _mm_and_si128(__m128i, __m128i); +FORCE_INLINE __m128i _mm_castps_si128(__m128); +FORCE_INLINE __m128i _mm_cmpeq_epi32(__m128i, __m128i); +FORCE_INLINE __m128i _mm_cvtps_epi32(__m128); +FORCE_INLINE __m128d _mm_move_sd(__m128d, __m128d); +FORCE_INLINE __m128i _mm_or_si128(__m128i, __m128i); +FORCE_INLINE __m128i _mm_set_epi32(int, int, int, int); +FORCE_INLINE __m128i _mm_set_epi64x(int64_t, int64_t); +FORCE_INLINE __m128d _mm_set_pd(double, double); +FORCE_INLINE __m128i _mm_set1_epi32(int); +FORCE_INLINE __m128i _mm_setzero_si128(void); +// SSE4.1 +FORCE_INLINE __m128d _mm_ceil_pd(__m128d); +FORCE_INLINE __m128 _mm_ceil_ps(__m128); +FORCE_INLINE __m128d _mm_floor_pd(__m128d); +FORCE_INLINE __m128 _mm_floor_ps(__m128); +FORCE_INLINE __m128d _mm_round_pd(__m128d, int); +FORCE_INLINE __m128 _mm_round_ps(__m128, int); +// SSE4.2 +FORCE_INLINE uint32_t _mm_crc32_u8(uint32_t, uint8_t); + /* Backwards compatibility for compilers with lack of specific type support */ // Older gcc does not define vld1q_u8_x4 type -#if defined(__GNUC__) && !defined(__clang__) -#if __GNUC__ <= 9 -FORCE_INLINE uint8x16x4_t vld1q_u8_x4(const uint8_t *p) +#if defined(__GNUC__) && !defined(__clang__) && \ + ((__GNUC__ <= 13 && defined(__arm__)) || \ + (__GNUC__ == 10 && __GNUC_MINOR__ < 3 && defined(__aarch64__)) || \ + (__GNUC__ <= 9 && defined(__aarch64__))) +FORCE_INLINE uint8x16x4_t _sse2neon_vld1q_u8_x4(const uint8_t *p) { uint8x16x4_t ret; ret.val[0] = vld1q_u8(p + 0); @@ -314,7 +587,63 @@ FORCE_INLINE uint8x16x4_t vld1q_u8_x4(const uint8_t *p) ret.val[3] = vld1q_u8(p + 48); return ret; } +#else +// Wraps vld1q_u8_x4 +FORCE_INLINE uint8x16x4_t _sse2neon_vld1q_u8_x4(const uint8_t *p) +{ + return vld1q_u8_x4(p); +} +#endif + +#if !defined(__aarch64__) && !defined(_M_ARM64) +/* emulate vaddv u8 variant */ +FORCE_INLINE uint8_t _sse2neon_vaddv_u8(uint8x8_t v8) +{ + const uint64x1_t v1 = vpaddl_u32(vpaddl_u16(vpaddl_u8(v8))); + return vget_lane_u8(vreinterpret_u8_u64(v1), 0); +} +#else +// Wraps vaddv_u8 +FORCE_INLINE uint8_t _sse2neon_vaddv_u8(uint8x8_t v8) +{ + return vaddv_u8(v8); +} +#endif + +#if !defined(__aarch64__) && !defined(_M_ARM64) +/* emulate vaddvq u8 variant */ +FORCE_INLINE uint8_t _sse2neon_vaddvq_u8(uint8x16_t a) +{ + uint8x8_t tmp = vpadd_u8(vget_low_u8(a), vget_high_u8(a)); + uint8_t res = 0; + for (int i = 0; i < 8; ++i) + res += tmp[i]; + return res; +} +#else +// Wraps vaddvq_u8 +FORCE_INLINE uint8_t _sse2neon_vaddvq_u8(uint8x16_t a) +{ + return vaddvq_u8(a); +} #endif + +#if !defined(__aarch64__) && !defined(_M_ARM64) +/* emulate vaddvq u16 variant */ +FORCE_INLINE uint16_t _sse2neon_vaddvq_u16(uint16x8_t a) +{ + uint32x4_t m = vpaddlq_u16(a); + uint64x2_t n = vpaddlq_u32(m); + uint64x1_t o = vget_low_u64(n) + vget_high_u64(n); + + return vget_lane_u32((uint32x2_t) o, 0); +} +#else +// Wraps vaddvq_u16 +FORCE_INLINE uint16_t _sse2neon_vaddvq_u16(uint16x8_t a) +{ + return vaddvq_u16(a); +} #endif /* Function Naming Conventions @@ -329,7 +658,7 @@ FORCE_INLINE uint8x16x4_t vld1q_u8_x4(const uint8_t *p) * This last part, , is a little complicated. It identifies the * content of the input values, and can be set to any of the following values: * + ps - vectors contain floats (ps stands for packed single-precision) - * + pd - vectors cantain doubles (pd stands for packed double-precision) + * + pd - vectors contain doubles (pd stands for packed double-precision) * + epi8/epi16/epi32/epi64 - vectors contain 8-bit/16-bit/32-bit/64-bit * signed integers * + epu8/epu16/epu32/epu64 - vectors contain 8-bit/16-bit/32-bit/64-bit @@ -351,2653 +680,2271 @@ FORCE_INLINE uint8x16x4_t vld1q_u8_x4(const uint8_t *p) * 4, 5, 12, 13, 6, 7, 14, 15); * // Shuffle packed 8-bit integers * __m128i v_out = _mm_shuffle_epi8(v_in, v_perm); // pshufb - * - * Data (Number, Binary, Byte Index): - +------+------+-------------+------+------+-------------+ - | 1 | 2 | 3 | 4 | Number - +------+------+------+------+------+------+------+------+ - | 0000 | 0001 | 0000 | 0010 | 0000 | 0011 | 0000 | 0100 | Binary - +------+------+------+------+------+------+------+------+ - | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | Index - +------+------+------+------+------+------+------+------+ - - +------+------+------+------+------+------+------+------+ - | 5 | 6 | 7 | 8 | Number - +------+------+------+------+------+------+------+------+ - | 0000 | 0101 | 0000 | 0110 | 0000 | 0111 | 0000 | 1000 | Binary - +------+------+------+------+------+------+------+------+ - | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | Index - +------+------+------+------+------+------+------+------+ - * Index (Byte Index): - +------+------+------+------+------+------+------+------+ - | 1 | 0 | 2 | 3 | 8 | 9 | 10 | 11 | - +------+------+------+------+------+------+------+------+ - - +------+------+------+------+------+------+------+------+ - | 4 | 5 | 12 | 13 | 6 | 7 | 14 | 15 | - +------+------+------+------+------+------+------+------+ - * Result: - +------+------+------+------+------+------+------+------+ - | 1 | 0 | 2 | 3 | 8 | 9 | 10 | 11 | Index - +------+------+------+------+------+------+------+------+ - | 0001 | 0000 | 0000 | 0010 | 0000 | 0101 | 0000 | 0110 | Binary - +------+------+------+------+------+------+------+------+ - | 256 | 2 | 5 | 6 | Number - +------+------+------+------+------+------+------+------+ - - +------+------+------+------+------+------+------+------+ - | 4 | 5 | 12 | 13 | 6 | 7 | 14 | 15 | Index - +------+------+------+------+------+------+------+------+ - | 0000 | 0011 | 0000 | 0111 | 0000 | 0100 | 0000 | 1000 | Binary - +------+------+------+------+------+------+------+------+ - | 3 | 7 | 4 | 8 | Number - +------+------+------+------+------+------+-------------+ */ -/* Set/get methods */ - -/* Constants for use with _mm_prefetch. */ +/* Constants for use with _mm_prefetch. */ enum _mm_hint { - _MM_HINT_NTA = 0, /* load data to L1 and L2 cache, mark it as NTA */ - _MM_HINT_T0 = 1, /* load data to L1 and L2 cache */ - _MM_HINT_T1 = 2, /* load data to L2 cache only */ - _MM_HINT_T2 = 3, /* load data to L2 cache only, mark it as NTA */ - _MM_HINT_ENTA = 4, /* exclusive version of _MM_HINT_NTA */ - _MM_HINT_ET0 = 5, /* exclusive version of _MM_HINT_T0 */ - _MM_HINT_ET1 = 6, /* exclusive version of _MM_HINT_T1 */ - _MM_HINT_ET2 = 7 /* exclusive version of _MM_HINT_T2 */ + _MM_HINT_NTA = 0, /* load data to L1 and L2 cache, mark it as NTA */ + _MM_HINT_T0 = 1, /* load data to L1 and L2 cache */ + _MM_HINT_T1 = 2, /* load data to L2 cache only */ + _MM_HINT_T2 = 3, /* load data to L2 cache only, mark it as NTA */ }; -// Loads one cache line of data from address p to a location closer to the -// processor. https://msdn.microsoft.com/en-us/library/84szxsww(v=vs.100).aspx -FORCE_INLINE void _mm_prefetch(const void *p, int i) +// The bit field mapping to the FPCR(floating-point control register) +typedef struct { + uint16_t res0; + uint8_t res1 : 6; + uint8_t bit22 : 1; + uint8_t bit23 : 1; + uint8_t bit24 : 1; + uint8_t res2 : 7; +#if defined(__aarch64__) || defined(_M_ARM64) + uint32_t res3; +#endif +} fpcr_bitfield; + +// Takes the upper 64 bits of a and places it in the low end of the result +// Takes the lower 64 bits of b and places it into the high end of the result. +FORCE_INLINE __m128 _mm_shuffle_ps_1032(__m128 a, __m128 b) { - (void) i; - __builtin_prefetch(p); + float32x2_t a32 = vget_high_f32(vreinterpretq_f32_m128(a)); + float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_f32(vcombine_f32(a32, b10)); } -// Copy the lower single-precision (32-bit) floating-point element of a to dst. -// -// dst[31:0] := a[31:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_f32 -FORCE_INLINE float _mm_cvtss_f32(__m128 a) +// takes the lower two 32-bit values from a and swaps them and places in high +// end of result takes the higher two 32 bit values from b and swaps them and +// places in low end of result. +FORCE_INLINE __m128 _mm_shuffle_ps_2301(__m128 a, __m128 b) { - return vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + float32x2_t a01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(a))); + float32x2_t b23 = vrev64_f32(vget_high_f32(vreinterpretq_f32_m128(b))); + return vreinterpretq_m128_f32(vcombine_f32(a01, b23)); } -// Convert the lower single-precision (32-bit) floating-point element in a to a -// 32-bit integer, and store the result in dst. -// -// dst[31:0] := Convert_FP32_To_Int32(a[31:0]) -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_si32 -#define _mm_cvtss_si32(a) _mm_cvt_ss2si(a) - -// Convert the lower single-precision (32-bit) floating-point element in a to a -// 64-bit integer, and store the result in dst. -// -// dst[63:0] := Convert_FP32_To_Int64(a[31:0]) -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_si64 -FORCE_INLINE int _mm_cvtss_si64(__m128 a) +FORCE_INLINE __m128 _mm_shuffle_ps_0321(__m128 a, __m128 b) { -#if defined(__aarch64__) - return vgetq_lane_s64( - vreinterpretq_s64_s32(vcvtnq_s32_f32(vreinterpretq_f32_m128(a))), 0); -#else - float32_t data = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); - float32_t diff = data - floor(data); - if (diff > 0.5) - return (int64_t) ceil(data); - if (diff == 0.5) { - int64_t f = (int64_t) floor(data); - int64_t c = (int64_t) ceil(data); - return c & 1 ? f : c; - } - return (int64_t) floor(data); -#endif + float32x2_t a21 = vget_high_f32( + vextq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 3)); + float32x2_t b03 = vget_low_f32( + vextq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b), 3)); + return vreinterpretq_m128_f32(vcombine_f32(a21, b03)); } -// Convert packed single-precision (32-bit) floating-point elements in a to -// packed 32-bit integers with truncation, and store the results in dst. -// -// FOR j := 0 to 1 -// i := 32*j -// dst[i+31:i] := Convert_FP32_To_Int32_Truncate(a[i+31:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_ps2pi -FORCE_INLINE __m64 _mm_cvtt_ps2pi(__m128 a) +FORCE_INLINE __m128 _mm_shuffle_ps_2103(__m128 a, __m128 b) { - return vreinterpret_m64_s32( - vget_low_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)))); + float32x2_t a03 = vget_low_f32( + vextq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 3)); + float32x2_t b21 = vget_high_f32( + vextq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b), 3)); + return vreinterpretq_m128_f32(vcombine_f32(a03, b21)); } -// Convert the lower single-precision (32-bit) floating-point element in a to a -// 32-bit integer with truncation, and store the result in dst. -// -// dst[31:0] := Convert_FP32_To_Int32_Truncate(a[31:0]) -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_ss2si -FORCE_INLINE int _mm_cvtt_ss2si(__m128 a) +FORCE_INLINE __m128 _mm_shuffle_ps_1010(__m128 a, __m128 b) { - return vgetq_lane_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)), 0); + float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(a)); + float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_f32(vcombine_f32(a10, b10)); } -// Convert packed single-precision (32-bit) floating-point elements in a to -// packed 32-bit integers with truncation, and store the results in dst. -// -// FOR j := 0 to 1 -// i := 32*j -// dst[i+31:i] := Convert_FP32_To_Int32_Truncate(a[i+31:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttps_pi32 -#define _mm_cvttps_pi32(a) _mm_cvtt_ps2pi(a) - -// Convert the lower single-precision (32-bit) floating-point element in a to a -// 32-bit integer with truncation, and store the result in dst. -// -// dst[31:0] := Convert_FP32_To_Int32_Truncate(a[31:0]) -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_si32 -#define _mm_cvttss_si32(a) _mm_cvtt_ss2si(a) - -// Convert the lower single-precision (32-bit) floating-point element in a to a -// 64-bit integer with truncation, and store the result in dst. -// -// dst[63:0] := Convert_FP32_To_Int64_Truncate(a[31:0]) -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_si64 -FORCE_INLINE int64_t _mm_cvttss_si64(__m128 a) +FORCE_INLINE __m128 _mm_shuffle_ps_1001(__m128 a, __m128 b) { - return vgetq_lane_s64( - vmovl_s32(vget_low_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)))), 0); + float32x2_t a01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(a))); + float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_f32(vcombine_f32(a01, b10)); } -// Sets the 128-bit value to zero -// https://msdn.microsoft.com/en-us/library/vstudio/ys7dw0kh(v=vs.100).aspx -FORCE_INLINE __m128i _mm_setzero_si128(void) +FORCE_INLINE __m128 _mm_shuffle_ps_0101(__m128 a, __m128 b) { - return vreinterpretq_m128i_s32(vdupq_n_s32(0)); + float32x2_t a01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(a))); + float32x2_t b01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(b))); + return vreinterpretq_m128_f32(vcombine_f32(a01, b01)); } -// Clears the four single-precision, floating-point values. -// https://msdn.microsoft.com/en-us/library/vstudio/tk1t2tbz(v=vs.100).aspx -FORCE_INLINE __m128 _mm_setzero_ps(void) +// keeps the low 64 bits of b in the low and puts the high 64 bits of a in the +// high +FORCE_INLINE __m128 _mm_shuffle_ps_3210(__m128 a, __m128 b) { - return vreinterpretq_m128_f32(vdupq_n_f32(0)); + float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(a)); + float32x2_t b32 = vget_high_f32(vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_f32(vcombine_f32(a10, b32)); } -// Return vector of type __m128d with all elements set to zero. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setzero_pd -FORCE_INLINE __m128d _mm_setzero_pd(void) +FORCE_INLINE __m128 _mm_shuffle_ps_0011(__m128 a, __m128 b) { -#if defined(__aarch64__) - return vreinterpretq_m128d_f64(vdupq_n_f64(0)); -#else - return vreinterpretq_m128d_f32(vdupq_n_f32(0)); -#endif + float32x2_t a11 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(a)), 1); + float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); + return vreinterpretq_m128_f32(vcombine_f32(a11, b00)); } -// Sets the four single-precision, floating-point values to w. -// -// r0 := r1 := r2 := r3 := w -// -// https://msdn.microsoft.com/en-us/library/vstudio/2x1se8ha(v=vs.100).aspx -FORCE_INLINE __m128 _mm_set1_ps(float _w) +FORCE_INLINE __m128 _mm_shuffle_ps_0022(__m128 a, __m128 b) { - return vreinterpretq_m128_f32(vdupq_n_f32(_w)); + float32x2_t a22 = + vdup_lane_f32(vget_high_f32(vreinterpretq_f32_m128(a)), 0); + float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); + return vreinterpretq_m128_f32(vcombine_f32(a22, b00)); } -// Sets the four single-precision, floating-point values to w. -// https://msdn.microsoft.com/en-us/library/vstudio/2x1se8ha(v=vs.100).aspx -FORCE_INLINE __m128 _mm_set_ps1(float _w) +FORCE_INLINE __m128 _mm_shuffle_ps_2200(__m128 a, __m128 b) { - return vreinterpretq_m128_f32(vdupq_n_f32(_w)); + float32x2_t a00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(a)), 0); + float32x2_t b22 = + vdup_lane_f32(vget_high_f32(vreinterpretq_f32_m128(b)), 0); + return vreinterpretq_m128_f32(vcombine_f32(a00, b22)); } -// Sets the four single-precision, floating-point values to the four inputs. -// https://msdn.microsoft.com/en-us/library/vstudio/afh0zf75(v=vs.100).aspx -FORCE_INLINE __m128 _mm_set_ps(float w, float z, float y, float x) +FORCE_INLINE __m128 _mm_shuffle_ps_3202(__m128 a, __m128 b) { - float ALIGN_STRUCT(16) data[4] = {x, y, z, w}; - return vreinterpretq_m128_f32(vld1q_f32(data)); + float32_t a0 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + float32x2_t a22 = + vdup_lane_f32(vget_high_f32(vreinterpretq_f32_m128(a)), 0); + float32x2_t a02 = vset_lane_f32(a0, a22, 1); /* TODO: use vzip ?*/ + float32x2_t b32 = vget_high_f32(vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_f32(vcombine_f32(a02, b32)); } -// Copy single-precision (32-bit) floating-point element a to the lower element -// of dst, and zero the upper 3 elements. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_ss -FORCE_INLINE __m128 _mm_set_ss(float a) +FORCE_INLINE __m128 _mm_shuffle_ps_1133(__m128 a, __m128 b) { - float ALIGN_STRUCT(16) data[4] = {a, 0, 0, 0}; - return vreinterpretq_m128_f32(vld1q_f32(data)); + float32x2_t a33 = + vdup_lane_f32(vget_high_f32(vreinterpretq_f32_m128(a)), 1); + float32x2_t b11 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 1); + return vreinterpretq_m128_f32(vcombine_f32(a33, b11)); } -// Sets the four single-precision, floating-point values to the four inputs in -// reverse order. -// https://msdn.microsoft.com/en-us/library/vstudio/d2172ct3(v=vs.100).aspx -FORCE_INLINE __m128 _mm_setr_ps(float w, float z, float y, float x) +FORCE_INLINE __m128 _mm_shuffle_ps_2010(__m128 a, __m128 b) { - float ALIGN_STRUCT(16) data[4] = {w, z, y, x}; - return vreinterpretq_m128_f32(vld1q_f32(data)); + float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(a)); + float32_t b2 = vgetq_lane_f32(vreinterpretq_f32_m128(b), 2); + float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); + float32x2_t b20 = vset_lane_f32(b2, b00, 1); + return vreinterpretq_m128_f32(vcombine_f32(a10, b20)); } -// Sets the 8 signed 16-bit integer values in reverse order. -// -// Return Value -// r0 := w0 -// r1 := w1 -// ... -// r7 := w7 -FORCE_INLINE __m128i _mm_setr_epi16(short w0, - short w1, - short w2, - short w3, - short w4, - short w5, - short w6, - short w7) +FORCE_INLINE __m128 _mm_shuffle_ps_2001(__m128 a, __m128 b) { - int16_t ALIGN_STRUCT(16) data[8] = {w0, w1, w2, w3, w4, w5, w6, w7}; - return vreinterpretq_m128i_s16(vld1q_s16((int16_t *) data)); + float32x2_t a01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(a))); + float32_t b2 = vgetq_lane_f32(b, 2); + float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); + float32x2_t b20 = vset_lane_f32(b2, b00, 1); + return vreinterpretq_m128_f32(vcombine_f32(a01, b20)); } -// Sets the 4 signed 32-bit integer values in reverse order -// https://technet.microsoft.com/en-us/library/security/27yb3ee5(v=vs.90).aspx -FORCE_INLINE __m128i _mm_setr_epi32(int i3, int i2, int i1, int i0) +FORCE_INLINE __m128 _mm_shuffle_ps_2032(__m128 a, __m128 b) { - int32_t ALIGN_STRUCT(16) data[4] = {i3, i2, i1, i0}; - return vreinterpretq_m128i_s32(vld1q_s32(data)); + float32x2_t a32 = vget_high_f32(vreinterpretq_f32_m128(a)); + float32_t b2 = vgetq_lane_f32(b, 2); + float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); + float32x2_t b20 = vset_lane_f32(b2, b00, 1); + return vreinterpretq_m128_f32(vcombine_f32(a32, b20)); } -// Set packed 64-bit integers in dst with the supplied values in reverse order. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setr_epi64 -FORCE_INLINE __m128i _mm_setr_epi64(__m64 e1, __m64 e0) +// For MSVC, we check only if it is ARM64, as every single ARM64 processor +// supported by WoA has crypto extensions. If this changes in the future, +// this can be verified via the runtime-only method of: +// IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) +#if (defined(_M_ARM64) && !defined(__clang__)) || \ + (defined(__ARM_FEATURE_CRYPTO) && \ + (defined(__aarch64__) || __has_builtin(__builtin_arm_crypto_vmullp64))) +// Wraps vmull_p64 +FORCE_INLINE uint64x2_t _sse2neon_vmull_p64(uint64x1_t _a, uint64x1_t _b) { - return vreinterpretq_m128i_s64(vcombine_s64(e1, e0)); + poly64_t a = vget_lane_p64(vreinterpret_p64_u64(_a), 0); + poly64_t b = vget_lane_p64(vreinterpret_p64_u64(_b), 0); +#if defined(_MSC_VER) + __n64 a1 = {a}, b1 = {b}; + return vreinterpretq_u64_p128(vmull_p64(a1, b1)); +#else + return vreinterpretq_u64_p128(vmull_p64(a, b)); +#endif } - -// Sets the 16 signed 8-bit integer values to b. +#else // ARMv7 polyfill +// ARMv7/some A64 lacks vmull_p64, but it has vmull_p8. // -// r0 := b -// r1 := b -// ... -// r15 := b +// vmull_p8 calculates 8 8-bit->16-bit polynomial multiplies, but we need a +// 64-bit->128-bit polynomial multiply. // -// https://msdn.microsoft.com/en-us/library/6e14xhyf(v=vs.100).aspx -FORCE_INLINE __m128i _mm_set1_epi8(signed char w) +// It needs some work and is somewhat slow, but it is still faster than all +// known scalar methods. +// +// Algorithm adapted to C from +// https://www.workofard.com/2017/07/ghash-for-low-end-cores/, which is adapted +// from "Fast Software Polynomial Multiplication on ARM Processors Using the +// NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and Ricardo Dahab +// (https://hal.inria.fr/hal-01506572) +static uint64x2_t _sse2neon_vmull_p64(uint64x1_t _a, uint64x1_t _b) { - return vreinterpretq_m128i_s8(vdupq_n_s8(w)); -} + poly8x8_t a = vreinterpret_p8_u64(_a); + poly8x8_t b = vreinterpret_p8_u64(_b); -// Broadcast double-precision (64-bit) floating-point value a to all elements of -// dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set1_pd -FORCE_INLINE __m128d _mm_set1_pd(double d) -{ + // Masks + uint8x16_t k48_32 = vcombine_u8(vcreate_u8(0x0000ffffffffffff), + vcreate_u8(0x00000000ffffffff)); + uint8x16_t k16_00 = vcombine_u8(vcreate_u8(0x000000000000ffff), + vcreate_u8(0x0000000000000000)); + + // Do the multiplies, rotating with vext to get all combinations + uint8x16_t d = vreinterpretq_u8_p16(vmull_p8(a, b)); // D = A0 * B0 + uint8x16_t e = + vreinterpretq_u8_p16(vmull_p8(a, vext_p8(b, b, 1))); // E = A0 * B1 + uint8x16_t f = + vreinterpretq_u8_p16(vmull_p8(vext_p8(a, a, 1), b)); // F = A1 * B0 + uint8x16_t g = + vreinterpretq_u8_p16(vmull_p8(a, vext_p8(b, b, 2))); // G = A0 * B2 + uint8x16_t h = + vreinterpretq_u8_p16(vmull_p8(vext_p8(a, a, 2), b)); // H = A2 * B0 + uint8x16_t i = + vreinterpretq_u8_p16(vmull_p8(a, vext_p8(b, b, 3))); // I = A0 * B3 + uint8x16_t j = + vreinterpretq_u8_p16(vmull_p8(vext_p8(a, a, 3), b)); // J = A3 * B0 + uint8x16_t k = + vreinterpretq_u8_p16(vmull_p8(a, vext_p8(b, b, 4))); // L = A0 * B4 + + // Add cross products + uint8x16_t l = veorq_u8(e, f); // L = E + F + uint8x16_t m = veorq_u8(g, h); // M = G + H + uint8x16_t n = veorq_u8(i, j); // N = I + J + + // Interleave. Using vzip1 and vzip2 prevents Clang from emitting TBL + // instructions. #if defined(__aarch64__) - return vreinterpretq_m128d_f64(vdupq_n_f64(d)); + uint8x16_t lm_p0 = vreinterpretq_u8_u64( + vzip1q_u64(vreinterpretq_u64_u8(l), vreinterpretq_u64_u8(m))); + uint8x16_t lm_p1 = vreinterpretq_u8_u64( + vzip2q_u64(vreinterpretq_u64_u8(l), vreinterpretq_u64_u8(m))); + uint8x16_t nk_p0 = vreinterpretq_u8_u64( + vzip1q_u64(vreinterpretq_u64_u8(n), vreinterpretq_u64_u8(k))); + uint8x16_t nk_p1 = vreinterpretq_u8_u64( + vzip2q_u64(vreinterpretq_u64_u8(n), vreinterpretq_u64_u8(k))); #else - return vreinterpretq_m128d_s64(vdupq_n_s64(*(int64_t *) &d)); + uint8x16_t lm_p0 = vcombine_u8(vget_low_u8(l), vget_low_u8(m)); + uint8x16_t lm_p1 = vcombine_u8(vget_high_u8(l), vget_high_u8(m)); + uint8x16_t nk_p0 = vcombine_u8(vget_low_u8(n), vget_low_u8(k)); + uint8x16_t nk_p1 = vcombine_u8(vget_high_u8(n), vget_high_u8(k)); +#endif + // t0 = (L) (P0 + P1) << 8 + // t1 = (M) (P2 + P3) << 16 + uint8x16_t t0t1_tmp = veorq_u8(lm_p0, lm_p1); + uint8x16_t t0t1_h = vandq_u8(lm_p1, k48_32); + uint8x16_t t0t1_l = veorq_u8(t0t1_tmp, t0t1_h); + + // t2 = (N) (P4 + P5) << 24 + // t3 = (K) (P6 + P7) << 32 + uint8x16_t t2t3_tmp = veorq_u8(nk_p0, nk_p1); + uint8x16_t t2t3_h = vandq_u8(nk_p1, k16_00); + uint8x16_t t2t3_l = veorq_u8(t2t3_tmp, t2t3_h); + + // De-interleave +#if defined(__aarch64__) + uint8x16_t t0 = vreinterpretq_u8_u64( + vuzp1q_u64(vreinterpretq_u64_u8(t0t1_l), vreinterpretq_u64_u8(t0t1_h))); + uint8x16_t t1 = vreinterpretq_u8_u64( + vuzp2q_u64(vreinterpretq_u64_u8(t0t1_l), vreinterpretq_u64_u8(t0t1_h))); + uint8x16_t t2 = vreinterpretq_u8_u64( + vuzp1q_u64(vreinterpretq_u64_u8(t2t3_l), vreinterpretq_u64_u8(t2t3_h))); + uint8x16_t t3 = vreinterpretq_u8_u64( + vuzp2q_u64(vreinterpretq_u64_u8(t2t3_l), vreinterpretq_u64_u8(t2t3_h))); +#else + uint8x16_t t1 = vcombine_u8(vget_high_u8(t0t1_l), vget_high_u8(t0t1_h)); + uint8x16_t t0 = vcombine_u8(vget_low_u8(t0t1_l), vget_low_u8(t0t1_h)); + uint8x16_t t3 = vcombine_u8(vget_high_u8(t2t3_l), vget_high_u8(t2t3_h)); + uint8x16_t t2 = vcombine_u8(vget_low_u8(t2t3_l), vget_low_u8(t2t3_h)); #endif + // Shift the cross products + uint8x16_t t0_shift = vextq_u8(t0, t0, 15); // t0 << 8 + uint8x16_t t1_shift = vextq_u8(t1, t1, 14); // t1 << 16 + uint8x16_t t2_shift = vextq_u8(t2, t2, 13); // t2 << 24 + uint8x16_t t3_shift = vextq_u8(t3, t3, 12); // t3 << 32 + + // Accumulate the products + uint8x16_t cross1 = veorq_u8(t0_shift, t1_shift); + uint8x16_t cross2 = veorq_u8(t2_shift, t3_shift); + uint8x16_t mix = veorq_u8(d, cross1); + uint8x16_t r = veorq_u8(mix, cross2); + return vreinterpretq_u64_u8(r); } +#endif // ARMv7 polyfill -// Sets the 8 signed 16-bit integer values to w. -// -// r0 := w -// r1 := w -// ... -// r7 := w -// -// https://msdn.microsoft.com/en-us/library/k0ya3x0e(v=vs.90).aspx -FORCE_INLINE __m128i _mm_set1_epi16(short w) +// C equivalent: +// __m128i _mm_shuffle_epi32_default(__m128i a, +// __constrange(0, 255) int imm) { +// __m128i ret; +// ret[0] = a[imm & 0x3]; ret[1] = a[(imm >> 2) & 0x3]; +// ret[2] = a[(imm >> 4) & 0x03]; ret[3] = a[(imm >> 6) & 0x03]; +// return ret; +// } +#define _mm_shuffle_epi32_default(a, imm) \ + vreinterpretq_m128i_s32(vsetq_lane_s32( \ + vgetq_lane_s32(vreinterpretq_s32_m128i(a), ((imm) >> 6) & 0x3), \ + vsetq_lane_s32( \ + vgetq_lane_s32(vreinterpretq_s32_m128i(a), ((imm) >> 4) & 0x3), \ + vsetq_lane_s32(vgetq_lane_s32(vreinterpretq_s32_m128i(a), \ + ((imm) >> 2) & 0x3), \ + vmovq_n_s32(vgetq_lane_s32( \ + vreinterpretq_s32_m128i(a), (imm) & (0x3))), \ + 1), \ + 2), \ + 3)) + +// Takes the upper 64 bits of a and places it in the low end of the result +// Takes the lower 64 bits of a and places it into the high end of the result. +FORCE_INLINE __m128i _mm_shuffle_epi_1032(__m128i a) { - return vreinterpretq_m128i_s16(vdupq_n_s16(w)); + int32x2_t a32 = vget_high_s32(vreinterpretq_s32_m128i(a)); + int32x2_t a10 = vget_low_s32(vreinterpretq_s32_m128i(a)); + return vreinterpretq_m128i_s32(vcombine_s32(a32, a10)); } -// Sets the 16 signed 8-bit integer values. -// https://msdn.microsoft.com/en-us/library/x0cx8zd3(v=vs.90).aspx -FORCE_INLINE __m128i _mm_set_epi8(signed char b15, - signed char b14, - signed char b13, - signed char b12, - signed char b11, - signed char b10, - signed char b9, - signed char b8, - signed char b7, - signed char b6, - signed char b5, - signed char b4, - signed char b3, - signed char b2, - signed char b1, - signed char b0) +// takes the lower two 32-bit values from a and swaps them and places in low end +// of result takes the higher two 32 bit values from a and swaps them and places +// in high end of result. +FORCE_INLINE __m128i _mm_shuffle_epi_2301(__m128i a) { - int8_t ALIGN_STRUCT(16) - data[16] = {(int8_t) b0, (int8_t) b1, (int8_t) b2, (int8_t) b3, - (int8_t) b4, (int8_t) b5, (int8_t) b6, (int8_t) b7, - (int8_t) b8, (int8_t) b9, (int8_t) b10, (int8_t) b11, - (int8_t) b12, (int8_t) b13, (int8_t) b14, (int8_t) b15}; - return (__m128i) vld1q_s8(data); + int32x2_t a01 = vrev64_s32(vget_low_s32(vreinterpretq_s32_m128i(a))); + int32x2_t a23 = vrev64_s32(vget_high_s32(vreinterpretq_s32_m128i(a))); + return vreinterpretq_m128i_s32(vcombine_s32(a01, a23)); } -// Sets the 8 signed 16-bit integer values. -// https://msdn.microsoft.com/en-au/library/3e0fek84(v=vs.90).aspx -FORCE_INLINE __m128i _mm_set_epi16(short i7, - short i6, - short i5, - short i4, - short i3, - short i2, - short i1, - short i0) +// rotates the least significant 32 bits into the most significant 32 bits, and +// shifts the rest down +FORCE_INLINE __m128i _mm_shuffle_epi_0321(__m128i a) { - int16_t ALIGN_STRUCT(16) data[8] = {i0, i1, i2, i3, i4, i5, i6, i7}; - return vreinterpretq_m128i_s16(vld1q_s16(data)); + return vreinterpretq_m128i_s32( + vextq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(a), 1)); } -// Sets the 16 signed 8-bit integer values in reverse order. -// https://msdn.microsoft.com/en-us/library/2khb9c7k(v=vs.90).aspx -FORCE_INLINE __m128i _mm_setr_epi8(signed char b0, - signed char b1, - signed char b2, - signed char b3, - signed char b4, - signed char b5, - signed char b6, - signed char b7, - signed char b8, - signed char b9, - signed char b10, - signed char b11, - signed char b12, - signed char b13, - signed char b14, - signed char b15) +// rotates the most significant 32 bits into the least significant 32 bits, and +// shifts the rest up +FORCE_INLINE __m128i _mm_shuffle_epi_2103(__m128i a) { - int8_t ALIGN_STRUCT(16) - data[16] = {(int8_t) b0, (int8_t) b1, (int8_t) b2, (int8_t) b3, - (int8_t) b4, (int8_t) b5, (int8_t) b6, (int8_t) b7, - (int8_t) b8, (int8_t) b9, (int8_t) b10, (int8_t) b11, - (int8_t) b12, (int8_t) b13, (int8_t) b14, (int8_t) b15}; - return (__m128i) vld1q_s8(data); + return vreinterpretq_m128i_s32( + vextq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(a), 3)); } -// Sets the 4 signed 32-bit integer values to i. -// -// r0 := i -// r1 := i -// r2 := i -// r3 := I -// -// https://msdn.microsoft.com/en-us/library/vstudio/h4xscxat(v=vs.100).aspx -FORCE_INLINE __m128i _mm_set1_epi32(int _i) +// gets the lower 64 bits of a, and places it in the upper 64 bits +// gets the lower 64 bits of a and places it in the lower 64 bits +FORCE_INLINE __m128i _mm_shuffle_epi_1010(__m128i a) { - return vreinterpretq_m128i_s32(vdupq_n_s32(_i)); + int32x2_t a10 = vget_low_s32(vreinterpretq_s32_m128i(a)); + return vreinterpretq_m128i_s32(vcombine_s32(a10, a10)); } -// Sets the 2 signed 64-bit integer values to i. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/whtfzhzk(v=vs.100) -FORCE_INLINE __m128i _mm_set1_epi64(__m64 _i) +// gets the lower 64 bits of a, swaps the 0 and 1 elements, and places it in the +// lower 64 bits gets the lower 64 bits of a, and places it in the upper 64 bits +FORCE_INLINE __m128i _mm_shuffle_epi_1001(__m128i a) { - return vreinterpretq_m128i_s64(vdupq_n_s64((int64_t) _i)); + int32x2_t a01 = vrev64_s32(vget_low_s32(vreinterpretq_s32_m128i(a))); + int32x2_t a10 = vget_low_s32(vreinterpretq_s32_m128i(a)); + return vreinterpretq_m128i_s32(vcombine_s32(a01, a10)); } -// Sets the 2 signed 64-bit integer values to i. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set1_epi64x -FORCE_INLINE __m128i _mm_set1_epi64x(int64_t _i) +// gets the lower 64 bits of a, swaps the 0 and 1 elements and places it in the +// upper 64 bits gets the lower 64 bits of a, swaps the 0 and 1 elements, and +// places it in the lower 64 bits +FORCE_INLINE __m128i _mm_shuffle_epi_0101(__m128i a) { - return vreinterpretq_m128i_s64(vdupq_n_s64(_i)); + int32x2_t a01 = vrev64_s32(vget_low_s32(vreinterpretq_s32_m128i(a))); + return vreinterpretq_m128i_s32(vcombine_s32(a01, a01)); } -// Sets the 4 signed 32-bit integer values. -// https://msdn.microsoft.com/en-us/library/vstudio/019beekt(v=vs.100).aspx -FORCE_INLINE __m128i _mm_set_epi32(int i3, int i2, int i1, int i0) +FORCE_INLINE __m128i _mm_shuffle_epi_2211(__m128i a) { - int32_t ALIGN_STRUCT(16) data[4] = {i0, i1, i2, i3}; - return vreinterpretq_m128i_s32(vld1q_s32(data)); + int32x2_t a11 = vdup_lane_s32(vget_low_s32(vreinterpretq_s32_m128i(a)), 1); + int32x2_t a22 = vdup_lane_s32(vget_high_s32(vreinterpretq_s32_m128i(a)), 0); + return vreinterpretq_m128i_s32(vcombine_s32(a11, a22)); } -// Returns the __m128i structure with its two 64-bit integer values -// initialized to the values of the two 64-bit integers passed in. -// https://msdn.microsoft.com/en-us/library/dk2sdw0h(v=vs.120).aspx -FORCE_INLINE __m128i _mm_set_epi64x(int64_t i1, int64_t i2) +FORCE_INLINE __m128i _mm_shuffle_epi_0122(__m128i a) { - int64_t ALIGN_STRUCT(16) data[2] = {i2, i1}; - return vreinterpretq_m128i_s64(vld1q_s64(data)); + int32x2_t a22 = vdup_lane_s32(vget_high_s32(vreinterpretq_s32_m128i(a)), 0); + int32x2_t a01 = vrev64_s32(vget_low_s32(vreinterpretq_s32_m128i(a))); + return vreinterpretq_m128i_s32(vcombine_s32(a22, a01)); } -// Returns the __m128i structure with its two 64-bit integer values -// initialized to the values of the two 64-bit integers passed in. -// https://msdn.microsoft.com/en-us/library/dk2sdw0h(v=vs.120).aspx -FORCE_INLINE __m128i _mm_set_epi64(__m64 i1, __m64 i2) +FORCE_INLINE __m128i _mm_shuffle_epi_3332(__m128i a) { - return _mm_set_epi64x((int64_t) i1, (int64_t) i2); + int32x2_t a32 = vget_high_s32(vreinterpretq_s32_m128i(a)); + int32x2_t a33 = vdup_lane_s32(vget_high_s32(vreinterpretq_s32_m128i(a)), 1); + return vreinterpretq_m128i_s32(vcombine_s32(a32, a33)); } -// Set packed double-precision (64-bit) floating-point elements in dst with the -// supplied values. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_pd -FORCE_INLINE __m128d _mm_set_pd(double e1, double e0) -{ - double ALIGN_STRUCT(16) data[2] = {e0, e1}; -#if defined(__aarch64__) - return vreinterpretq_m128d_f64(vld1q_f64((float64_t *) data)); +#if defined(__aarch64__) || defined(_M_ARM64) +#define _mm_shuffle_epi32_splat(a, imm) \ + vreinterpretq_m128i_s32(vdupq_laneq_s32(vreinterpretq_s32_m128i(a), (imm))) #else - return vreinterpretq_m128d_f32(vld1q_f32((float32_t *) data)); +#define _mm_shuffle_epi32_splat(a, imm) \ + vreinterpretq_m128i_s32( \ + vdupq_n_s32(vgetq_lane_s32(vreinterpretq_s32_m128i(a), (imm)))) #endif -} -// Stores four single-precision, floating-point values. -// https://msdn.microsoft.com/en-us/library/vstudio/s3h4ay6y(v=vs.100).aspx -FORCE_INLINE void _mm_store_ps(float *p, __m128 a) +// NEON does not support a general purpose permute intrinsic. +// Shuffle single-precision (32-bit) floating-point elements in a using the +// control in imm8, and store the results in dst. +// +// C equivalent: +// __m128 _mm_shuffle_ps_default(__m128 a, __m128 b, +// __constrange(0, 255) int imm) { +// __m128 ret; +// ret[0] = a[imm & 0x3]; ret[1] = a[(imm >> 2) & 0x3]; +// ret[2] = b[(imm >> 4) & 0x03]; ret[3] = b[(imm >> 6) & 0x03]; +// return ret; +// } +// +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shuffle_ps +#define _mm_shuffle_ps_default(a, b, imm) \ + vreinterpretq_m128_f32(vsetq_lane_f32( \ + vgetq_lane_f32(vreinterpretq_f32_m128(b), ((imm) >> 6) & 0x3), \ + vsetq_lane_f32( \ + vgetq_lane_f32(vreinterpretq_f32_m128(b), ((imm) >> 4) & 0x3), \ + vsetq_lane_f32( \ + vgetq_lane_f32(vreinterpretq_f32_m128(a), ((imm) >> 2) & 0x3), \ + vmovq_n_f32( \ + vgetq_lane_f32(vreinterpretq_f32_m128(a), (imm) & (0x3))), \ + 1), \ + 2), \ + 3)) + +// Shuffle 16-bit integers in the low 64 bits of a using the control in imm8. +// Store the results in the low 64 bits of dst, with the high 64 bits being +// copied from a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shufflelo_epi16 +#define _mm_shufflelo_epi16_function(a, imm) \ + _sse2neon_define1( \ + __m128i, a, int16x8_t ret = vreinterpretq_s16_m128i(_a); \ + int16x4_t lowBits = vget_low_s16(ret); \ + ret = vsetq_lane_s16(vget_lane_s16(lowBits, (imm) & (0x3)), ret, 0); \ + ret = vsetq_lane_s16(vget_lane_s16(lowBits, ((imm) >> 2) & 0x3), ret, \ + 1); \ + ret = vsetq_lane_s16(vget_lane_s16(lowBits, ((imm) >> 4) & 0x3), ret, \ + 2); \ + ret = vsetq_lane_s16(vget_lane_s16(lowBits, ((imm) >> 6) & 0x3), ret, \ + 3); \ + _sse2neon_return(vreinterpretq_m128i_s16(ret));) + +// Shuffle 16-bit integers in the high 64 bits of a using the control in imm8. +// Store the results in the high 64 bits of dst, with the low 64 bits being +// copied from a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shufflehi_epi16 +#define _mm_shufflehi_epi16_function(a, imm) \ + _sse2neon_define1( \ + __m128i, a, int16x8_t ret = vreinterpretq_s16_m128i(_a); \ + int16x4_t highBits = vget_high_s16(ret); \ + ret = vsetq_lane_s16(vget_lane_s16(highBits, (imm) & (0x3)), ret, 4); \ + ret = vsetq_lane_s16(vget_lane_s16(highBits, ((imm) >> 2) & 0x3), ret, \ + 5); \ + ret = vsetq_lane_s16(vget_lane_s16(highBits, ((imm) >> 4) & 0x3), ret, \ + 6); \ + ret = vsetq_lane_s16(vget_lane_s16(highBits, ((imm) >> 6) & 0x3), ret, \ + 7); \ + _sse2neon_return(vreinterpretq_m128i_s16(ret));) + +/* MMX */ + +//_mm_empty is a no-op on arm +FORCE_INLINE void _mm_empty(void) {} + +/* SSE */ + +// Add packed single-precision (32-bit) floating-point elements in a and b, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_ps +FORCE_INLINE __m128 _mm_add_ps(__m128 a, __m128 b) { - vst1q_f32(p, vreinterpretq_f32_m128(a)); + return vreinterpretq_m128_f32( + vaddq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } -// Stores four single-precision, floating-point values. -// https://msdn.microsoft.com/en-us/library/44e30x22(v=vs.100).aspx -FORCE_INLINE void _mm_storeu_ps(float *p, __m128 a) +// Add the lower single-precision (32-bit) floating-point element in a and b, +// store the result in the lower element of dst, and copy the upper 3 packed +// elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_ss +FORCE_INLINE __m128 _mm_add_ss(__m128 a, __m128 b) { - vst1q_f32(p, vreinterpretq_f32_m128(a)); + float32_t b0 = vgetq_lane_f32(vreinterpretq_f32_m128(b), 0); + float32x4_t value = vsetq_lane_f32(b0, vdupq_n_f32(0), 0); + // the upper values in the result must be the remnants of . + return vreinterpretq_m128_f32(vaddq_f32(a, value)); } -// Stores four 32-bit integer values as (as a __m128i value) at the address p. -// https://msdn.microsoft.com/en-us/library/vstudio/edk11s13(v=vs.100).aspx -FORCE_INLINE void _mm_store_si128(__m128i *p, __m128i a) +// Compute the bitwise AND of packed single-precision (32-bit) floating-point +// elements in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_and_ps +FORCE_INLINE __m128 _mm_and_ps(__m128 a, __m128 b) { - vst1q_s32((int32_t *) p, vreinterpretq_s32_m128i(a)); + return vreinterpretq_m128_s32( + vandq_s32(vreinterpretq_s32_m128(a), vreinterpretq_s32_m128(b))); } -// Stores four 32-bit integer values as (as a __m128i value) at the address p. -// https://msdn.microsoft.com/en-us/library/vstudio/edk11s13(v=vs.100).aspx -FORCE_INLINE void _mm_storeu_si128(__m128i *p, __m128i a) +// Compute the bitwise NOT of packed single-precision (32-bit) floating-point +// elements in a and then AND with b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_andnot_ps +FORCE_INLINE __m128 _mm_andnot_ps(__m128 a, __m128 b) { - vst1q_s32((int32_t *) p, vreinterpretq_s32_m128i(a)); + return vreinterpretq_m128_s32( + vbicq_s32(vreinterpretq_s32_m128(b), + vreinterpretq_s32_m128(a))); // *NOTE* argument swap } -// Stores the lower single - precision, floating - point value. -// https://msdn.microsoft.com/en-us/library/tzz10fbx(v=vs.100).aspx -FORCE_INLINE void _mm_store_ss(float *p, __m128 a) +// Average packed unsigned 16-bit integers in a and b, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_avg_pu16 +FORCE_INLINE __m64 _mm_avg_pu16(__m64 a, __m64 b) { - vst1q_lane_f32(p, vreinterpretq_f32_m128(a), 0); + return vreinterpret_m64_u16( + vrhadd_u16(vreinterpret_u16_m64(a), vreinterpret_u16_m64(b))); } -// Store 128-bits (composed of 2 packed double-precision (64-bit) floating-point -// elements) from a into memory. mem_addr must be aligned on a 16-byte boundary -// or a general-protection exception may be generated. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_pd -FORCE_INLINE void _mm_store_pd(double *mem_addr, __m128d a) +// Average packed unsigned 8-bit integers in a and b, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_avg_pu8 +FORCE_INLINE __m64 _mm_avg_pu8(__m64 a, __m64 b) { -#if defined(__aarch64__) - vst1q_f64((float64_t *) mem_addr, vreinterpretq_f64_m128d(a)); -#else - vst1q_f32((float32_t *) mem_addr, vreinterpretq_f32_m128d(a)); -#endif + return vreinterpret_m64_u8( + vrhadd_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); } -// Store the lower double-precision (64-bit) floating-point element from a into -// 2 contiguous elements in memory. mem_addr must be aligned on a 16-byte -// boundary or a general-protection exception may be generated. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_pd1 -FORCE_INLINE void _mm_store_pd1(double *mem_addr, __m128d a) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for equality, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_ps +FORCE_INLINE __m128 _mm_cmpeq_ps(__m128 a, __m128 b) { -#if defined(__aarch64__) - float64x1_t a_low = vget_low_f64(vreinterpretq_f64_m128d(a)); - vst1q_f64((float64_t *) mem_addr, - vreinterpretq_f64_m128d(vcombine_f64(a_low, a_low))); -#else - float32x2_t a_low = vget_low_f32(vreinterpretq_f32_m128d(a)); - vst1q_f32((float32_t *) mem_addr, - vreinterpretq_f32_m128d(vcombine_f32(a_low, a_low))); -#endif + return vreinterpretq_m128_u32( + vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } -// Store the lower double-precision (64-bit) floating-point element from a into -// 2 contiguous elements in memory. mem_addr must be aligned on a 16-byte -// boundary or a general-protection exception may be generated. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#expand=9,526,5601&text=_mm_store1_pd -#define _mm_store1_pd _mm_store_pd1 - -// Store 128-bits (composed of 2 packed double-precision (64-bit) floating-point -// elements) from a into memory. mem_addr does not need to be aligned on any -// particular boundary. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_pd -FORCE_INLINE void _mm_storeu_pd(double *mem_addr, __m128d a) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for equality, store the result in the lower element of dst, and copy the +// upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_ss +FORCE_INLINE __m128 _mm_cmpeq_ss(__m128 a, __m128 b) { - _mm_store_pd(mem_addr, a); + return _mm_move_ss(a, _mm_cmpeq_ps(a, b)); } -// Reads the lower 64 bits of b and stores them into the lower 64 bits of a. -// https://msdn.microsoft.com/en-us/library/hhwf428f%28v=vs.90%29.aspx -FORCE_INLINE void _mm_storel_epi64(__m128i *a, __m128i b) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for greater-than-or-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpge_ps +FORCE_INLINE __m128 _mm_cmpge_ps(__m128 a, __m128 b) { - uint64x1_t hi = vget_high_u64(vreinterpretq_u64_m128i(*a)); - uint64x1_t lo = vget_low_u64(vreinterpretq_u64_m128i(b)); - *a = vreinterpretq_m128i_u64(vcombine_u64(lo, hi)); + return vreinterpretq_m128_u32( + vcgeq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } -// Stores the lower two single-precision floating point values of a to the -// address p. -// -// *p0 := a0 -// *p1 := a1 -// -// https://msdn.microsoft.com/en-us/library/h54t98ks(v=vs.90).aspx -FORCE_INLINE void _mm_storel_pi(__m64 *p, __m128 a) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for greater-than-or-equal, store the result in the lower element of dst, +// and copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpge_ss +FORCE_INLINE __m128 _mm_cmpge_ss(__m128 a, __m128 b) { - *p = vreinterpret_m64_f32(vget_low_f32(a)); + return _mm_move_ss(a, _mm_cmpge_ps(a, b)); } -// Stores the upper two single-precision, floating-point values of a to the -// address p. -// -// *p0 := a2 -// *p1 := a3 -// -// https://msdn.microsoft.com/en-us/library/a7525fs8(v%3dvs.90).aspx -FORCE_INLINE void _mm_storeh_pi(__m64 *p, __m128 a) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for greater-than, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_ps +FORCE_INLINE __m128 _mm_cmpgt_ps(__m128 a, __m128 b) { - *p = vreinterpret_m64_f32(vget_high_f32(a)); + return vreinterpretq_m128_u32( + vcgtq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } -// Loads a single single-precision, floating-point value, copying it into all -// four words -// https://msdn.microsoft.com/en-us/library/vstudio/5cdkf716(v=vs.100).aspx -FORCE_INLINE __m128 _mm_load1_ps(const float *p) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for greater-than, store the result in the lower element of dst, and copy +// the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_ss +FORCE_INLINE __m128 _mm_cmpgt_ss(__m128 a, __m128 b) { - return vreinterpretq_m128_f32(vld1q_dup_f32(p)); + return _mm_move_ss(a, _mm_cmpgt_ps(a, b)); } -// Load a single-precision (32-bit) floating-point element from memory into all -// elements of dst. -// -// dst[31:0] := MEM[mem_addr+31:mem_addr] -// dst[63:32] := MEM[mem_addr+31:mem_addr] -// dst[95:64] := MEM[mem_addr+31:mem_addr] -// dst[127:96] := MEM[mem_addr+31:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_ps1 -#define _mm_load_ps1 _mm_load1_ps - -// Sets the lower two single-precision, floating-point values with 64 -// bits of data loaded from the address p; the upper two values are passed -// through from a. -// -// Return Value -// r0 := *p0 -// r1 := *p1 -// r2 := a2 -// r3 := a3 -// -// https://msdn.microsoft.com/en-us/library/s57cyak2(v=vs.100).aspx -FORCE_INLINE __m128 _mm_loadl_pi(__m128 a, __m64 const *p) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for less-than-or-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmple_ps +FORCE_INLINE __m128 _mm_cmple_ps(__m128 a, __m128 b) { - return vreinterpretq_m128_f32( - vcombine_f32(vld1_f32((const float32_t *) p), vget_high_f32(a))); + return vreinterpretq_m128_u32( + vcleq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } -// Load 4 single-precision (32-bit) floating-point elements from memory into dst -// in reverse order. mem_addr must be aligned on a 16-byte boundary or a -// general-protection exception may be generated. -// -// dst[31:0] := MEM[mem_addr+127:mem_addr+96] -// dst[63:32] := MEM[mem_addr+95:mem_addr+64] -// dst[95:64] := MEM[mem_addr+63:mem_addr+32] -// dst[127:96] := MEM[mem_addr+31:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadr_ps -FORCE_INLINE __m128 _mm_loadr_ps(const float *p) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for less-than-or-equal, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmple_ss +FORCE_INLINE __m128 _mm_cmple_ss(__m128 a, __m128 b) { - float32x4_t v = vrev64q_f32(vld1q_f32(p)); - return vreinterpretq_m128_f32(vextq_f32(v, v, 2)); + return _mm_move_ss(a, _mm_cmple_ps(a, b)); } -// Sets the upper two single-precision, floating-point values with 64 -// bits of data loaded from the address p; the lower two values are passed -// through from a. -// -// r0 := a0 -// r1 := a1 -// r2 := *p0 -// r3 := *p1 -// -// https://msdn.microsoft.com/en-us/library/w92wta0x(v%3dvs.100).aspx -FORCE_INLINE __m128 _mm_loadh_pi(__m128 a, __m64 const *p) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for less-than, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_ps +FORCE_INLINE __m128 _mm_cmplt_ps(__m128 a, __m128 b) { - return vreinterpretq_m128_f32( - vcombine_f32(vget_low_f32(a), vld1_f32((const float32_t *) p))); + return vreinterpretq_m128_u32( + vcltq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } -// Loads four single-precision, floating-point values. -// https://msdn.microsoft.com/en-us/library/vstudio/zzd50xxt(v=vs.100).aspx -FORCE_INLINE __m128 _mm_load_ps(const float *p) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for less-than, store the result in the lower element of dst, and copy the +// upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_ss +FORCE_INLINE __m128 _mm_cmplt_ss(__m128 a, __m128 b) { - return vreinterpretq_m128_f32(vld1q_f32(p)); + return _mm_move_ss(a, _mm_cmplt_ps(a, b)); } -// Loads four single-precision, floating-point values. -// https://msdn.microsoft.com/en-us/library/x1b16s7z%28v=vs.90%29.aspx -FORCE_INLINE __m128 _mm_loadu_ps(const float *p) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for not-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpneq_ps +FORCE_INLINE __m128 _mm_cmpneq_ps(__m128 a, __m128 b) { - // for neon, alignment doesn't matter, so _mm_load_ps and _mm_loadu_ps are - // equivalent for neon - return vreinterpretq_m128_f32(vld1q_f32(p)); + return vreinterpretq_m128_u32(vmvnq_u32( + vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)))); } -// Load unaligned 16-bit integer from memory into the first element of dst. -// -// dst[15:0] := MEM[mem_addr+15:mem_addr] -// dst[MAX:16] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_si16 -FORCE_INLINE __m128i _mm_loadu_si16(const void *p) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for not-equal, store the result in the lower element of dst, and copy the +// upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpneq_ss +FORCE_INLINE __m128 _mm_cmpneq_ss(__m128 a, __m128 b) { - return vreinterpretq_m128i_s16( - vsetq_lane_s16(*(const int16_t *) p, vdupq_n_s16(0), 0)); + return _mm_move_ss(a, _mm_cmpneq_ps(a, b)); } -// Load unaligned 64-bit integer from memory into the first element of dst. -// -// dst[63:0] := MEM[mem_addr+63:mem_addr] -// dst[MAX:64] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_si64 -FORCE_INLINE __m128i _mm_loadu_si64(const void *p) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for not-greater-than-or-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnge_ps +FORCE_INLINE __m128 _mm_cmpnge_ps(__m128 a, __m128 b) { - return vreinterpretq_m128i_s64( - vcombine_s64(vld1_s64((const int64_t *) p), vdup_n_s64(0))); + return vreinterpretq_m128_u32(vmvnq_u32( + vcgeq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)))); } -// Load a double-precision (64-bit) floating-point element from memory into the -// lower of dst, and zero the upper element. mem_addr does not need to be -// aligned on any particular boundary. -// -// dst[63:0] := MEM[mem_addr+63:mem_addr] -// dst[127:64] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_sd -FORCE_INLINE __m128d _mm_load_sd(const double *p) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for not-greater-than-or-equal, store the result in the lower element of +// dst, and copy the upper 3 packed elements from a to the upper elements of +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnge_ss +FORCE_INLINE __m128 _mm_cmpnge_ss(__m128 a, __m128 b) { -#if defined(__aarch64__) - return vreinterpretq_m128d_f64(vsetq_lane_f64(*p, vdupq_n_f64(0), 0)); -#else - const float *fp = (const float *) p; - float ALIGN_STRUCT(16) data[4] = {fp[0], fp[1], 0, 0}; - return vreinterpretq_m128d_f32(vld1q_f32(data)); -#endif + return _mm_move_ss(a, _mm_cmpnge_ps(a, b)); } -// Loads two double-precision from 16-byte aligned memory, floating-point -// values. -// -// dst[127:0] := MEM[mem_addr+127:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_pd -FORCE_INLINE __m128d _mm_load_pd(const double *p) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for not-greater-than, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpngt_ps +FORCE_INLINE __m128 _mm_cmpngt_ps(__m128 a, __m128 b) { -#if defined(__aarch64__) - return vreinterpretq_m128d_f64(vld1q_f64(p)); -#else - const float *fp = (const float *) p; - float ALIGN_STRUCT(16) data[4] = {fp[0], fp[1], fp[2], fp[3]}; - return vreinterpretq_m128d_f32(vld1q_f32(data)); -#endif + return vreinterpretq_m128_u32(vmvnq_u32( + vcgtq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)))); } -// Loads two double-precision from unaligned memory, floating-point values. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_pd -FORCE_INLINE __m128d _mm_loadu_pd(const double *p) -{ - return _mm_load_pd(p); -} - -// Loads an single - precision, floating - point value into the low word and -// clears the upper three words. -// https://msdn.microsoft.com/en-us/library/548bb9h4%28v=vs.90%29.aspx -FORCE_INLINE __m128 _mm_load_ss(const float *p) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for not-greater-than, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpngt_ss +FORCE_INLINE __m128 _mm_cmpngt_ss(__m128 a, __m128 b) { - return vreinterpretq_m128_f32(vsetq_lane_f32(*p, vdupq_n_f32(0), 0)); + return _mm_move_ss(a, _mm_cmpngt_ps(a, b)); } -FORCE_INLINE __m128i _mm_loadl_epi64(__m128i const *p) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for not-less-than-or-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnle_ps +FORCE_INLINE __m128 _mm_cmpnle_ps(__m128 a, __m128 b) { - /* Load the lower 64 bits of the value pointed to by p into the - * lower 64 bits of the result, zeroing the upper 64 bits of the result. - */ - return vreinterpretq_m128i_s32( - vcombine_s32(vld1_s32((int32_t const *) p), vcreate_s32(0))); + return vreinterpretq_m128_u32(vmvnq_u32( + vcleq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)))); } -// Load a double-precision (64-bit) floating-point element from memory into the -// lower element of dst, and copy the upper element from a to dst. mem_addr does -// not need to be aligned on any particular boundary. -// -// dst[63:0] := MEM[mem_addr+63:mem_addr] -// dst[127:64] := a[127:64] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadl_pd -FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double *p) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for not-less-than-or-equal, store the result in the lower element of dst, +// and copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnle_ss +FORCE_INLINE __m128 _mm_cmpnle_ss(__m128 a, __m128 b) { -#if defined(__aarch64__) - return vreinterpretq_m128d_f64( - vcombine_f64(vld1_f64(p), vget_high_f64(vreinterpretq_f64_m128d(a)))); -#else - return vreinterpretq_m128d_f32( - vcombine_f32(vld1_f32((const float *) p), - vget_high_f32(vreinterpretq_f32_m128d(a)))); -#endif + return _mm_move_ss(a, _mm_cmpnle_ps(a, b)); } -// Load 2 double-precision (64-bit) floating-point elements from memory into dst -// in reverse order. mem_addr must be aligned on a 16-byte boundary or a -// general-protection exception may be generated. -// -// dst[63:0] := MEM[mem_addr+127:mem_addr+64] -// dst[127:64] := MEM[mem_addr+63:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadr_pd -FORCE_INLINE __m128d _mm_loadr_pd(const double *p) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// for not-less-than, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnlt_ps +FORCE_INLINE __m128 _mm_cmpnlt_ps(__m128 a, __m128 b) { -#if defined(__aarch64__) - float64x2_t v = vld1q_f64(p); - return vreinterpretq_m128d_f64(vextq_f64(v, v, 1)); -#else - int64x2_t v = vld1q_s64((const int64_t *) p); - return vreinterpretq_m128d_s64(vextq_s64(v, v, 1)); -#endif + return vreinterpretq_m128_u32(vmvnq_u32( + vcltq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)))); } -// Sets the low word to the single-precision, floating-point value of b -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/35hdzazd(v=vs.100) -FORCE_INLINE __m128 _mm_move_ss(__m128 a, __m128 b) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b for not-less-than, store the result in the lower element of dst, and copy +// the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnlt_ss +FORCE_INLINE __m128 _mm_cmpnlt_ss(__m128 a, __m128 b) { - return vreinterpretq_m128_f32( - vsetq_lane_f32(vgetq_lane_f32(vreinterpretq_f32_m128(b), 0), - vreinterpretq_f32_m128(a), 0)); + return _mm_move_ss(a, _mm_cmpnlt_ps(a, b)); } -// Move the lower double-precision (64-bit) floating-point element from b to the -// lower element of dst, and copy the upper element from a to the upper element -// of dst. +// Compare packed single-precision (32-bit) floating-point elements in a and b +// to see if neither is NaN, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpord_ps // -// dst[63:0] := b[63:0] -// dst[127:64] := a[127:64] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_move_sd -FORCE_INLINE __m128d _mm_move_sd(__m128d a, __m128d b) +// See also: +// http://stackoverflow.com/questions/8627331/what-does-ordered-unordered-comparison-mean +// http://stackoverflow.com/questions/29349621/neon-isnanval-intrinsics +FORCE_INLINE __m128 _mm_cmpord_ps(__m128 a, __m128 b) { - return vreinterpretq_m128d_f32( - vcombine_f32(vget_low_f32(vreinterpretq_f32_m128d(b)), - vget_high_f32(vreinterpretq_f32_m128d(a)))); + // Note: NEON does not have ordered compare builtin + // Need to compare a eq a and b eq b to check for NaN + // Do AND of results to get final + uint32x4_t ceqaa = + vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); + uint32x4_t ceqbb = + vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_u32(vandq_u32(ceqaa, ceqbb)); } -// Copy the lower 64-bit integer in a to the lower element of dst, and zero the -// upper element. -// -// dst[63:0] := a[63:0] -// dst[127:64] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_move_epi64 -FORCE_INLINE __m128i _mm_move_epi64(__m128i a) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b to see if neither is NaN, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpord_ss +FORCE_INLINE __m128 _mm_cmpord_ss(__m128 a, __m128 b) { - return vreinterpretq_m128i_s64( - vsetq_lane_s64(0, vreinterpretq_s64_m128i(a), 1)); + return _mm_move_ss(a, _mm_cmpord_ps(a, b)); } -// Return vector of type __m128 with undefined elements. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_undefined_ps -FORCE_INLINE __m128 _mm_undefined_ps(void) +// Compare packed single-precision (32-bit) floating-point elements in a and b +// to see if either is NaN, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpunord_ps +FORCE_INLINE __m128 _mm_cmpunord_ps(__m128 a, __m128 b) { - __m128 a; - return a; + uint32x4_t f32a = + vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); + uint32x4_t f32b = + vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_u32(vmvnq_u32(vandq_u32(f32a, f32b))); } -/* Logic/Binary operations */ - -// Computes the bitwise AND-NOT of the four single-precision, floating-point -// values of a and b. -// -// r0 := ~a0 & b0 -// r1 := ~a1 & b1 -// r2 := ~a2 & b2 -// r3 := ~a3 & b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/68h7wd02(v=vs.100).aspx -FORCE_INLINE __m128 _mm_andnot_ps(__m128 a, __m128 b) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b to see if either is NaN, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpunord_ss +FORCE_INLINE __m128 _mm_cmpunord_ss(__m128 a, __m128 b) { - return vreinterpretq_m128_s32( - vbicq_s32(vreinterpretq_s32_m128(b), - vreinterpretq_s32_m128(a))); // *NOTE* argument swap + return _mm_move_ss(a, _mm_cmpunord_ps(a, b)); } -// Compute the bitwise NOT of packed double-precision (64-bit) floating-point -// elements in a and then AND with b, and store the results in dst. -// -// FOR j := 0 to 1 -// i := j*64 -// dst[i+63:i] := ((NOT a[i+63:i]) AND b[i+63:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_andnot_pd -FORCE_INLINE __m128d _mm_andnot_pd(__m128d a, __m128d b) +// Compare the lower single-precision (32-bit) floating-point element in a and b +// for equality, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comieq_ss +FORCE_INLINE int _mm_comieq_ss(__m128 a, __m128 b) { - // *NOTE* argument swap - return vreinterpretq_m128d_s64( - vbicq_s64(vreinterpretq_s64_m128d(b), vreinterpretq_s64_m128d(a))); + uint32x4_t a_eq_b = + vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); + return vgetq_lane_u32(a_eq_b, 0) & 0x1; } -// Computes the bitwise AND of the 128-bit value in b and the bitwise NOT of the -// 128-bit value in a. -// -// r := (~a) & b -// -// https://msdn.microsoft.com/en-us/library/vstudio/1beaceh8(v=vs.100).aspx -FORCE_INLINE __m128i _mm_andnot_si128(__m128i a, __m128i b) +// Compare the lower single-precision (32-bit) floating-point element in a and b +// for greater-than-or-equal, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comige_ss +FORCE_INLINE int _mm_comige_ss(__m128 a, __m128 b) { - return vreinterpretq_m128i_s32( - vbicq_s32(vreinterpretq_s32_m128i(b), - vreinterpretq_s32_m128i(a))); // *NOTE* argument swap + uint32x4_t a_ge_b = + vcgeq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); + return vgetq_lane_u32(a_ge_b, 0) & 0x1; } -// Computes the bitwise AND of the 128-bit value in a and the 128-bit value in -// b. -// -// r := a & b -// -// https://msdn.microsoft.com/en-us/library/vstudio/6d1txsa8(v=vs.100).aspx -FORCE_INLINE __m128i _mm_and_si128(__m128i a, __m128i b) +// Compare the lower single-precision (32-bit) floating-point element in a and b +// for greater-than, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comigt_ss +FORCE_INLINE int _mm_comigt_ss(__m128 a, __m128 b) { - return vreinterpretq_m128i_s32( - vandq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); + uint32x4_t a_gt_b = + vcgtq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); + return vgetq_lane_u32(a_gt_b, 0) & 0x1; } -// Computes the bitwise AND of the four single-precision, floating-point values -// of a and b. -// -// r0 := a0 & b0 -// r1 := a1 & b1 -// r2 := a2 & b2 -// r3 := a3 & b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/73ck1xc5(v=vs.100).aspx -FORCE_INLINE __m128 _mm_and_ps(__m128 a, __m128 b) +// Compare the lower single-precision (32-bit) floating-point element in a and b +// for less-than-or-equal, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comile_ss +FORCE_INLINE int _mm_comile_ss(__m128 a, __m128 b) { - return vreinterpretq_m128_s32( - vandq_s32(vreinterpretq_s32_m128(a), vreinterpretq_s32_m128(b))); + uint32x4_t a_le_b = + vcleq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); + return vgetq_lane_u32(a_le_b, 0) & 0x1; } -// Compute the bitwise AND of packed double-precision (64-bit) floating-point -// elements in a and b, and store the results in dst. -// -// FOR j := 0 to 1 -// i := j*64 -// dst[i+63:i] := a[i+63:i] AND b[i+63:i] -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_and_pd -FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) +// Compare the lower single-precision (32-bit) floating-point element in a and b +// for less-than, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comilt_ss +FORCE_INLINE int _mm_comilt_ss(__m128 a, __m128 b) { - return vreinterpretq_m128d_s64( - vandq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); + uint32x4_t a_lt_b = + vcltq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); + return vgetq_lane_u32(a_lt_b, 0) & 0x1; } -// Computes the bitwise OR of the four single-precision, floating-point values -// of a and b. -// https://msdn.microsoft.com/en-us/library/vstudio/7ctdsyy0(v=vs.100).aspx -FORCE_INLINE __m128 _mm_or_ps(__m128 a, __m128 b) +// Compare the lower single-precision (32-bit) floating-point element in a and b +// for not-equal, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comineq_ss +FORCE_INLINE int _mm_comineq_ss(__m128 a, __m128 b) { - return vreinterpretq_m128_s32( - vorrq_s32(vreinterpretq_s32_m128(a), vreinterpretq_s32_m128(b))); + return !_mm_comieq_ss(a, b); } -// Computes bitwise EXOR (exclusive-or) of the four single-precision, -// floating-point values of a and b. -// https://msdn.microsoft.com/en-us/library/ss6k3wk8(v=vs.100).aspx -FORCE_INLINE __m128 _mm_xor_ps(__m128 a, __m128 b) +// Convert packed signed 32-bit integers in b to packed single-precision +// (32-bit) floating-point elements, store the results in the lower 2 elements +// of dst, and copy the upper 2 packed elements from a to the upper elements of +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvt_pi2ps +FORCE_INLINE __m128 _mm_cvt_pi2ps(__m128 a, __m64 b) { - return vreinterpretq_m128_s32( - veorq_s32(vreinterpretq_s32_m128(a), vreinterpretq_s32_m128(b))); + return vreinterpretq_m128_f32( + vcombine_f32(vcvt_f32_s32(vreinterpret_s32_m64(b)), + vget_high_f32(vreinterpretq_f32_m128(a)))); } -// Compute the bitwise XOR of packed double-precision (64-bit) floating-point -// elements in a and b, and store the results in dst. -// -// FOR j := 0 to 1 -// i := j*64 -// dst[i+63:i] := a[i+63:i] XOR b[i+63:i] -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_xor_pd -FORCE_INLINE __m128d _mm_xor_pd(__m128d a, __m128d b) +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvt_ps2pi +FORCE_INLINE __m64 _mm_cvt_ps2pi(__m128 a) { - return vreinterpretq_m128d_s64( - veorq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); +#if (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ARM_FEATURE_DIRECTED_ROUNDING) + return vreinterpret_m64_s32( + vget_low_s32(vcvtnq_s32_f32(vrndiq_f32(vreinterpretq_f32_m128(a))))); +#else + return vreinterpret_m64_s32(vcvt_s32_f32(vget_low_f32( + vreinterpretq_f32_m128(_mm_round_ps(a, _MM_FROUND_CUR_DIRECTION))))); +#endif } -// Compute the bitwise OR of packed double-precision (64-bit) floating-point -// elements in a and b, and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=mm_or_pd -FORCE_INLINE __m128d _mm_or_pd(__m128d a, __m128d b) +// Convert the signed 32-bit integer b to a single-precision (32-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvt_si2ss +FORCE_INLINE __m128 _mm_cvt_si2ss(__m128 a, int b) { - return vreinterpretq_m128d_s64( - vorrq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); + return vreinterpretq_m128_f32( + vsetq_lane_f32((float) b, vreinterpretq_f32_m128(a), 0)); } -// Computes the bitwise OR of the 128-bit value in a and the 128-bit value in b. -// -// r := a | b -// -// https://msdn.microsoft.com/en-us/library/vstudio/ew8ty0db(v=vs.100).aspx -FORCE_INLINE __m128i _mm_or_si128(__m128i a, __m128i b) +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 32-bit integer, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvt_ss2si +FORCE_INLINE int _mm_cvt_ss2si(__m128 a) { - return vreinterpretq_m128i_s32( - vorrq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +#if (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ARM_FEATURE_DIRECTED_ROUNDING) + return vgetq_lane_s32(vcvtnq_s32_f32(vrndiq_f32(vreinterpretq_f32_m128(a))), + 0); +#else + float32_t data = vgetq_lane_f32( + vreinterpretq_f32_m128(_mm_round_ps(a, _MM_FROUND_CUR_DIRECTION)), 0); + return (int32_t) data; +#endif } -// Computes the bitwise XOR of the 128-bit value in a and the 128-bit value in -// b. https://msdn.microsoft.com/en-us/library/fzt08www(v=vs.100).aspx -FORCE_INLINE __m128i _mm_xor_si128(__m128i a, __m128i b) +// Convert packed 16-bit integers in a to packed single-precision (32-bit) +// floating-point elements, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpi16_ps +FORCE_INLINE __m128 _mm_cvtpi16_ps(__m64 a) { - return vreinterpretq_m128i_s32( - veorq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); + return vreinterpretq_m128_f32( + vcvtq_f32_s32(vmovl_s16(vreinterpret_s16_m64(a)))); } -// Duplicate odd-indexed single-precision (32-bit) floating-point elements -// from a, and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movehdup_ps -FORCE_INLINE __m128 _mm_movehdup_ps(__m128 a) +// Convert packed 32-bit integers in b to packed single-precision (32-bit) +// floating-point elements, store the results in the lower 2 elements of dst, +// and copy the upper 2 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpi32_ps +FORCE_INLINE __m128 _mm_cvtpi32_ps(__m128 a, __m64 b) { -#if __has_builtin(__builtin_shufflevector) - return vreinterpretq_m128_f32(__builtin_shufflevector( - vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 1, 1, 3, 3)); -#else - float32_t a1 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 1); - float32_t a3 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 3); - float ALIGN_STRUCT(16) data[4] = {a1, a1, a3, a3}; - return vreinterpretq_m128_f32(vld1q_f32(data)); -#endif + return vreinterpretq_m128_f32( + vcombine_f32(vcvt_f32_s32(vreinterpret_s32_m64(b)), + vget_high_f32(vreinterpretq_f32_m128(a)))); } -// Duplicate even-indexed single-precision (32-bit) floating-point elements -// from a, and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_moveldup_ps -FORCE_INLINE __m128 _mm_moveldup_ps(__m128 a) +// Convert packed signed 32-bit integers in a to packed single-precision +// (32-bit) floating-point elements, store the results in the lower 2 elements +// of dst, then convert the packed signed 32-bit integers in b to +// single-precision (32-bit) floating-point element, and store the results in +// the upper 2 elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpi32x2_ps +FORCE_INLINE __m128 _mm_cvtpi32x2_ps(__m64 a, __m64 b) { -#if __has_builtin(__builtin_shufflevector) - return vreinterpretq_m128_f32(__builtin_shufflevector( - vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 0, 0, 2, 2)); -#else - float32_t a0 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); - float32_t a2 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 2); - float ALIGN_STRUCT(16) data[4] = {a0, a0, a2, a2}; - return vreinterpretq_m128_f32(vld1q_f32(data)); -#endif + return vreinterpretq_m128_f32(vcvtq_f32_s32( + vcombine_s32(vreinterpret_s32_m64(a), vreinterpret_s32_m64(b)))); } -// Moves the upper two values of B into the lower two values of A. -// -// r3 := a3 -// r2 := a2 -// r1 := b3 -// r0 := b2 -FORCE_INLINE __m128 _mm_movehl_ps(__m128 __A, __m128 __B) -{ - float32x2_t a32 = vget_high_f32(vreinterpretq_f32_m128(__A)); - float32x2_t b32 = vget_high_f32(vreinterpretq_f32_m128(__B)); - return vreinterpretq_m128_f32(vcombine_f32(b32, a32)); +// Convert the lower packed 8-bit integers in a to packed single-precision +// (32-bit) floating-point elements, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpi8_ps +FORCE_INLINE __m128 _mm_cvtpi8_ps(__m64 a) +{ + return vreinterpretq_m128_f32(vcvtq_f32_s32( + vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_m64(a)))))); } -// Moves the lower two values of B into the upper two values of A. -// -// r3 := b1 -// r2 := b0 -// r1 := a1 -// r0 := a0 -FORCE_INLINE __m128 _mm_movelh_ps(__m128 __A, __m128 __B) +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 16-bit integers, and store the results in dst. Note: this intrinsic +// will generate 0x7FFF, rather than 0x8000, for input values between 0x7FFF and +// 0x7FFFFFFF. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_pi16 +FORCE_INLINE __m64 _mm_cvtps_pi16(__m128 a) { - float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(__A)); - float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(__B)); - return vreinterpretq_m128_f32(vcombine_f32(a10, b10)); + return vreinterpret_m64_s16( + vqmovn_s32(vreinterpretq_s32_m128i(_mm_cvtps_epi32(a)))); } -// Compute the absolute value of packed signed 32-bit integers in a, and store -// the unsigned results in dst. -// -// FOR j := 0 to 3 -// i := j*32 -// dst[i+31:i] := ABS(a[i+31:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_epi32 -FORCE_INLINE __m128i _mm_abs_epi32(__m128i a) +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_pi32 +#define _mm_cvtps_pi32(a) _mm_cvt_ps2pi(a) + +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 8-bit integers, and store the results in lower 4 elements of dst. +// Note: this intrinsic will generate 0x7F, rather than 0x80, for input values +// between 0x7F and 0x7FFFFFFF. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_pi8 +FORCE_INLINE __m64 _mm_cvtps_pi8(__m128 a) { - return vreinterpretq_m128i_s32(vabsq_s32(vreinterpretq_s32_m128i(a))); + return vreinterpret_m64_s8(vqmovn_s16( + vcombine_s16(vreinterpret_s16_m64(_mm_cvtps_pi16(a)), vdup_n_s16(0)))); } -// Compute the absolute value of packed signed 16-bit integers in a, and store -// the unsigned results in dst. -// -// FOR j := 0 to 7 -// i := j*16 -// dst[i+15:i] := ABS(a[i+15:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_epi16 -FORCE_INLINE __m128i _mm_abs_epi16(__m128i a) +// Convert packed unsigned 16-bit integers in a to packed single-precision +// (32-bit) floating-point elements, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpu16_ps +FORCE_INLINE __m128 _mm_cvtpu16_ps(__m64 a) { - return vreinterpretq_m128i_s16(vabsq_s16(vreinterpretq_s16_m128i(a))); + return vreinterpretq_m128_f32( + vcvtq_f32_u32(vmovl_u16(vreinterpret_u16_m64(a)))); } -// Compute the absolute value of packed signed 8-bit integers in a, and store -// the unsigned results in dst. -// -// FOR j := 0 to 15 -// i := j*8 -// dst[i+7:i] := ABS(a[i+7:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_epi8 -FORCE_INLINE __m128i _mm_abs_epi8(__m128i a) +// Convert the lower packed unsigned 8-bit integers in a to packed +// single-precision (32-bit) floating-point elements, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpu8_ps +FORCE_INLINE __m128 _mm_cvtpu8_ps(__m64 a) { - return vreinterpretq_m128i_s8(vabsq_s8(vreinterpretq_s8_m128i(a))); + return vreinterpretq_m128_f32(vcvtq_f32_u32( + vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_m64(a)))))); } -// Compute the absolute value of packed signed 32-bit integers in a, and store -// the unsigned results in dst. -// -// FOR j := 0 to 1 -// i := j*32 -// dst[i+31:i] := ABS(a[i+31:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_pi32 -FORCE_INLINE __m64 _mm_abs_pi32(__m64 a) +// Convert the signed 32-bit integer b to a single-precision (32-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi32_ss +#define _mm_cvtsi32_ss(a, b) _mm_cvt_si2ss(a, b) + +// Convert the signed 64-bit integer b to a single-precision (32-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi64_ss +FORCE_INLINE __m128 _mm_cvtsi64_ss(__m128 a, int64_t b) { - return vreinterpret_m64_s32(vabs_s32(vreinterpret_s32_m64(a))); + return vreinterpretq_m128_f32( + vsetq_lane_f32((float) b, vreinterpretq_f32_m128(a), 0)); } -// Compute the absolute value of packed signed 16-bit integers in a, and store -// the unsigned results in dst. -// -// FOR j := 0 to 3 -// i := j*16 -// dst[i+15:i] := ABS(a[i+15:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_pi16 -FORCE_INLINE __m64 _mm_abs_pi16(__m64 a) +// Copy the lower single-precision (32-bit) floating-point element of a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtss_f32 +FORCE_INLINE float _mm_cvtss_f32(__m128 a) { - return vreinterpret_m64_s16(vabs_s16(vreinterpret_s16_m64(a))); + return vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); } -// Compute the absolute value of packed signed 8-bit integers in a, and store -// the unsigned results in dst. -// -// FOR j := 0 to 7 -// i := j*8 -// dst[i+7:i] := ABS(a[i+7:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_abs_pi8 -FORCE_INLINE __m64 _mm_abs_pi8(__m64 a) +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 32-bit integer, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtss_si32 +#define _mm_cvtss_si32(a) _mm_cvt_ss2si(a) + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 64-bit integer, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtss_si64 +FORCE_INLINE int64_t _mm_cvtss_si64(__m128 a) { - return vreinterpret_m64_s8(vabs_s8(vreinterpret_s8_m64(a))); +#if (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ARM_FEATURE_DIRECTED_ROUNDING) + return (int64_t) vgetq_lane_f32(vrndiq_f32(vreinterpretq_f32_m128(a)), 0); +#else + float32_t data = vgetq_lane_f32( + vreinterpretq_f32_m128(_mm_round_ps(a, _MM_FROUND_CUR_DIRECTION)), 0); + return (int64_t) data; +#endif } -// Takes the upper 64 bits of a and places it in the low end of the result -// Takes the lower 64 bits of b and places it into the high end of the result. -FORCE_INLINE __m128 _mm_shuffle_ps_1032(__m128 a, __m128 b) +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers with truncation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtt_ps2pi +FORCE_INLINE __m64 _mm_cvtt_ps2pi(__m128 a) { - float32x2_t a32 = vget_high_f32(vreinterpretq_f32_m128(a)); - float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(b)); - return vreinterpretq_m128_f32(vcombine_f32(a32, b10)); + return vreinterpret_m64_s32( + vget_low_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)))); } -// takes the lower two 32-bit values from a and swaps them and places in high -// end of result takes the higher two 32 bit values from b and swaps them and -// places in low end of result. -FORCE_INLINE __m128 _mm_shuffle_ps_2301(__m128 a, __m128 b) +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 32-bit integer with truncation, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtt_ss2si +FORCE_INLINE int _mm_cvtt_ss2si(__m128 a) { - float32x2_t a01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(a))); - float32x2_t b23 = vrev64_f32(vget_high_f32(vreinterpretq_f32_m128(b))); - return vreinterpretq_m128_f32(vcombine_f32(a01, b23)); + return vgetq_lane_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)), 0); } -FORCE_INLINE __m128 _mm_shuffle_ps_0321(__m128 a, __m128 b) +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers with truncation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttps_pi32 +#define _mm_cvttps_pi32(a) _mm_cvtt_ps2pi(a) + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 32-bit integer with truncation, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttss_si32 +#define _mm_cvttss_si32(a) _mm_cvtt_ss2si(a) + +// Convert the lower single-precision (32-bit) floating-point element in a to a +// 64-bit integer with truncation, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttss_si64 +FORCE_INLINE int64_t _mm_cvttss_si64(__m128 a) { - float32x2_t a21 = vget_high_f32( - vextq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 3)); - float32x2_t b03 = vget_low_f32( - vextq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b), 3)); - return vreinterpretq_m128_f32(vcombine_f32(a21, b03)); + return (int64_t) vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); } -FORCE_INLINE __m128 _mm_shuffle_ps_2103(__m128 a, __m128 b) +// Divide packed single-precision (32-bit) floating-point elements in a by +// packed elements in b, and store the results in dst. +// Due to ARMv7-A NEON's lack of a precise division intrinsic, we implement +// division by multiplying a by b's reciprocal before using the Newton-Raphson +// method to approximate the results. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_div_ps +FORCE_INLINE __m128 _mm_div_ps(__m128 a, __m128 b) { - float32x2_t a03 = vget_low_f32( - vextq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 3)); - float32x2_t b21 = vget_high_f32( - vextq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b), 3)); - return vreinterpretq_m128_f32(vcombine_f32(a03, b21)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32( + vdivq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +#else + float32x4_t recip = vrecpeq_f32(vreinterpretq_f32_m128(b)); + recip = vmulq_f32(recip, vrecpsq_f32(recip, vreinterpretq_f32_m128(b))); + // Additional Netwon-Raphson iteration for accuracy + recip = vmulq_f32(recip, vrecpsq_f32(recip, vreinterpretq_f32_m128(b))); + return vreinterpretq_m128_f32(vmulq_f32(vreinterpretq_f32_m128(a), recip)); +#endif } -FORCE_INLINE __m128 _mm_shuffle_ps_1010(__m128 a, __m128 b) +// Divide the lower single-precision (32-bit) floating-point element in a by the +// lower single-precision (32-bit) floating-point element in b, store the result +// in the lower element of dst, and copy the upper 3 packed elements from a to +// the upper elements of dst. +// Warning: ARMv7-A does not produce the same result compared to Intel and not +// IEEE-compliant. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_div_ss +FORCE_INLINE __m128 _mm_div_ss(__m128 a, __m128 b) { - float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(a)); - float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(b)); - return vreinterpretq_m128_f32(vcombine_f32(a10, b10)); + float32_t value = + vgetq_lane_f32(vreinterpretq_f32_m128(_mm_div_ps(a, b)), 0); + return vreinterpretq_m128_f32( + vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); } -FORCE_INLINE __m128 _mm_shuffle_ps_1001(__m128 a, __m128 b) +// Extract a 16-bit integer from a, selected with imm8, and store the result in +// the lower element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_extract_pi16 +#define _mm_extract_pi16(a, imm) \ + (int32_t) vget_lane_u16(vreinterpret_u16_m64(a), (imm)) + +// Free aligned memory that was allocated with _mm_malloc. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_free +#if !defined(SSE2NEON_ALLOC_DEFINED) +FORCE_INLINE void _mm_free(void *addr) { - float32x2_t a01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(a))); - float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(b)); - return vreinterpretq_m128_f32(vcombine_f32(a01, b10)); + free(addr); } +#endif -FORCE_INLINE __m128 _mm_shuffle_ps_0101(__m128 a, __m128 b) +FORCE_INLINE uint64_t _sse2neon_get_fpcr(void) { - float32x2_t a01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(a))); - float32x2_t b01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(b))); - return vreinterpretq_m128_f32(vcombine_f32(a01, b01)); + uint64_t value; +#if defined(_MSC_VER) + value = _ReadStatusReg(ARM64_FPCR); +#else + __asm__ __volatile__("mrs %0, FPCR" : "=r"(value)); /* read */ +#endif + return value; } -// keeps the low 64 bits of b in the low and puts the high 64 bits of a in the -// high -FORCE_INLINE __m128 _mm_shuffle_ps_3210(__m128 a, __m128 b) +FORCE_INLINE void _sse2neon_set_fpcr(uint64_t value) { - float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(a)); - float32x2_t b32 = vget_high_f32(vreinterpretq_f32_m128(b)); - return vreinterpretq_m128_f32(vcombine_f32(a10, b32)); +#if defined(_MSC_VER) + _WriteStatusReg(ARM64_FPCR, value); +#else + __asm__ __volatile__("msr FPCR, %0" ::"r"(value)); /* write */ +#endif } -FORCE_INLINE __m128 _mm_shuffle_ps_0011(__m128 a, __m128 b) +// Macro: Get the flush zero bits from the MXCSR control and status register. +// The flush zero may contain any of the following flags: _MM_FLUSH_ZERO_ON or +// _MM_FLUSH_ZERO_OFF +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_GET_FLUSH_ZERO_MODE +FORCE_INLINE unsigned int _sse2neon_mm_get_flush_zero_mode(void) { - float32x2_t a11 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(a)), 1); - float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); - return vreinterpretq_m128_f32(vcombine_f32(a11, b00)); + union { + fpcr_bitfield field; +#if defined(__aarch64__) || defined(_M_ARM64) + uint64_t value; +#else + uint32_t value; +#endif + } r; + +#if defined(__aarch64__) || defined(_M_ARM64) + r.value = _sse2neon_get_fpcr(); +#else + __asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ +#endif + + return r.field.bit24 ? _MM_FLUSH_ZERO_ON : _MM_FLUSH_ZERO_OFF; } -FORCE_INLINE __m128 _mm_shuffle_ps_0022(__m128 a, __m128 b) +// Macro: Get the rounding mode bits from the MXCSR control and status register. +// The rounding mode may contain any of the following flags: _MM_ROUND_NEAREST, +// _MM_ROUND_DOWN, _MM_ROUND_UP, _MM_ROUND_TOWARD_ZERO +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_GET_ROUNDING_MODE +FORCE_INLINE unsigned int _MM_GET_ROUNDING_MODE(void) { - float32x2_t a22 = - vdup_lane_f32(vget_high_f32(vreinterpretq_f32_m128(a)), 0); - float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); - return vreinterpretq_m128_f32(vcombine_f32(a22, b00)); + union { + fpcr_bitfield field; +#if defined(__aarch64__) || defined(_M_ARM64) + uint64_t value; +#else + uint32_t value; +#endif + } r; + +#if defined(__aarch64__) || defined(_M_ARM64) + r.value = _sse2neon_get_fpcr(); +#else + __asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ +#endif + + if (r.field.bit22) { + return r.field.bit23 ? _MM_ROUND_TOWARD_ZERO : _MM_ROUND_UP; + } else { + return r.field.bit23 ? _MM_ROUND_DOWN : _MM_ROUND_NEAREST; + } } -FORCE_INLINE __m128 _mm_shuffle_ps_2200(__m128 a, __m128 b) +// Copy a to dst, and insert the 16-bit integer i into dst at the location +// specified by imm8. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_insert_pi16 +#define _mm_insert_pi16(a, b, imm) \ + vreinterpret_m64_s16(vset_lane_s16((b), vreinterpret_s16_m64(a), (imm))) + +// Load 128-bits (composed of 4 packed single-precision (32-bit) floating-point +// elements) from memory into dst. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_ps +FORCE_INLINE __m128 _mm_load_ps(const float *p) { - float32x2_t a00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(a)), 0); - float32x2_t b22 = - vdup_lane_f32(vget_high_f32(vreinterpretq_f32_m128(b)), 0); - return vreinterpretq_m128_f32(vcombine_f32(a00, b22)); + return vreinterpretq_m128_f32(vld1q_f32(p)); } -FORCE_INLINE __m128 _mm_shuffle_ps_3202(__m128 a, __m128 b) +// Load a single-precision (32-bit) floating-point element from memory into all +// elements of dst. +// +// dst[31:0] := MEM[mem_addr+31:mem_addr] +// dst[63:32] := MEM[mem_addr+31:mem_addr] +// dst[95:64] := MEM[mem_addr+31:mem_addr] +// dst[127:96] := MEM[mem_addr+31:mem_addr] +// +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_ps1 +#define _mm_load_ps1 _mm_load1_ps + +// Load a single-precision (32-bit) floating-point element from memory into the +// lower of dst, and zero the upper 3 elements. mem_addr does not need to be +// aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_ss +FORCE_INLINE __m128 _mm_load_ss(const float *p) { - float32_t a0 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); - float32x2_t a22 = - vdup_lane_f32(vget_high_f32(vreinterpretq_f32_m128(a)), 0); - float32x2_t a02 = vset_lane_f32(a0, a22, 1); /* TODO: use vzip ?*/ - float32x2_t b32 = vget_high_f32(vreinterpretq_f32_m128(b)); - return vreinterpretq_m128_f32(vcombine_f32(a02, b32)); + return vreinterpretq_m128_f32(vsetq_lane_f32(*p, vdupq_n_f32(0), 0)); } -FORCE_INLINE __m128 _mm_shuffle_ps_1133(__m128 a, __m128 b) +// Load a single-precision (32-bit) floating-point element from memory into all +// elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load1_ps +FORCE_INLINE __m128 _mm_load1_ps(const float *p) { - float32x2_t a33 = - vdup_lane_f32(vget_high_f32(vreinterpretq_f32_m128(a)), 1); - float32x2_t b11 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 1); - return vreinterpretq_m128_f32(vcombine_f32(a33, b11)); + return vreinterpretq_m128_f32(vld1q_dup_f32(p)); } -FORCE_INLINE __m128 _mm_shuffle_ps_2010(__m128 a, __m128 b) +// Load 2 single-precision (32-bit) floating-point elements from memory into the +// upper 2 elements of dst, and copy the lower 2 elements from a to dst. +// mem_addr does not need to be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadh_pi +FORCE_INLINE __m128 _mm_loadh_pi(__m128 a, __m64 const *p) { - float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(a)); - float32_t b2 = vgetq_lane_f32(vreinterpretq_f32_m128(b), 2); - float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); - float32x2_t b20 = vset_lane_f32(b2, b00, 1); - return vreinterpretq_m128_f32(vcombine_f32(a10, b20)); + return vreinterpretq_m128_f32( + vcombine_f32(vget_low_f32(a), vld1_f32((const float32_t *) p))); } -FORCE_INLINE __m128 _mm_shuffle_ps_2001(__m128 a, __m128 b) +// Load 2 single-precision (32-bit) floating-point elements from memory into the +// lower 2 elements of dst, and copy the upper 2 elements from a to dst. +// mem_addr does not need to be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadl_pi +FORCE_INLINE __m128 _mm_loadl_pi(__m128 a, __m64 const *p) { - float32x2_t a01 = vrev64_f32(vget_low_f32(vreinterpretq_f32_m128(a))); - float32_t b2 = vgetq_lane_f32(b, 2); - float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); - float32x2_t b20 = vset_lane_f32(b2, b00, 1); - return vreinterpretq_m128_f32(vcombine_f32(a01, b20)); + return vreinterpretq_m128_f32( + vcombine_f32(vld1_f32((const float32_t *) p), vget_high_f32(a))); } -FORCE_INLINE __m128 _mm_shuffle_ps_2032(__m128 a, __m128 b) +// Load 4 single-precision (32-bit) floating-point elements from memory into dst +// in reverse order. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadr_ps +FORCE_INLINE __m128 _mm_loadr_ps(const float *p) { - float32x2_t a32 = vget_high_f32(vreinterpretq_f32_m128(a)); - float32_t b2 = vgetq_lane_f32(b, 2); - float32x2_t b00 = vdup_lane_f32(vget_low_f32(vreinterpretq_f32_m128(b)), 0); - float32x2_t b20 = vset_lane_f32(b2, b00, 1); - return vreinterpretq_m128_f32(vcombine_f32(a32, b20)); + float32x4_t v = vrev64q_f32(vld1q_f32(p)); + return vreinterpretq_m128_f32(vextq_f32(v, v, 2)); } -// NEON does not support a general purpose permute intrinsic -// Selects four specific single-precision, floating-point values from a and b, -// based on the mask i. -// -// C equivalent: -// __m128 _mm_shuffle_ps_default(__m128 a, __m128 b, -// __constrange(0, 255) int imm) { -// __m128 ret; -// ret[0] = a[imm & 0x3]; ret[1] = a[(imm >> 2) & 0x3]; -// ret[2] = b[(imm >> 4) & 0x03]; ret[3] = b[(imm >> 6) & 0x03]; -// return ret; -// } -// -// https://msdn.microsoft.com/en-us/library/vstudio/5f0858x0(v=vs.100).aspx -#define _mm_shuffle_ps_default(a, b, imm) \ - __extension__({ \ - float32x4_t ret; \ - ret = vmovq_n_f32( \ - vgetq_lane_f32(vreinterpretq_f32_m128(a), (imm) & (0x3))); \ - ret = vsetq_lane_f32( \ - vgetq_lane_f32(vreinterpretq_f32_m128(a), ((imm) >> 2) & 0x3), \ - ret, 1); \ - ret = vsetq_lane_f32( \ - vgetq_lane_f32(vreinterpretq_f32_m128(b), ((imm) >> 4) & 0x3), \ - ret, 2); \ - ret = vsetq_lane_f32( \ - vgetq_lane_f32(vreinterpretq_f32_m128(b), ((imm) >> 6) & 0x3), \ - ret, 3); \ - vreinterpretq_m128_f32(ret); \ - }) +// Load 128-bits (composed of 4 packed single-precision (32-bit) floating-point +// elements) from memory into dst. mem_addr does not need to be aligned on any +// particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadu_ps +FORCE_INLINE __m128 _mm_loadu_ps(const float *p) +{ + // for neon, alignment doesn't matter, so _mm_load_ps and _mm_loadu_ps are + // equivalent for neon + return vreinterpretq_m128_f32(vld1q_f32(p)); +} -// FORCE_INLINE __m128 _mm_shuffle_ps(__m128 a, __m128 b, __constrange(0,255) -// int imm) -#if __has_builtin(__builtin_shufflevector) -#define _mm_shuffle_ps(a, b, imm) \ - __extension__({ \ - float32x4_t _input1 = vreinterpretq_f32_m128(a); \ - float32x4_t _input2 = vreinterpretq_f32_m128(b); \ - float32x4_t _shuf = __builtin_shufflevector( \ - _input1, _input2, (imm) & (0x3), ((imm) >> 2) & 0x3, \ - (((imm) >> 4) & 0x3) + 4, (((imm) >> 6) & 0x3) + 4); \ - vreinterpretq_m128_f32(_shuf); \ - }) -#else // generic -#define _mm_shuffle_ps(a, b, imm) \ - __extension__({ \ - __m128 ret; \ - switch (imm) { \ - case _MM_SHUFFLE(1, 0, 3, 2): \ - ret = _mm_shuffle_ps_1032((a), (b)); \ - break; \ - case _MM_SHUFFLE(2, 3, 0, 1): \ - ret = _mm_shuffle_ps_2301((a), (b)); \ - break; \ - case _MM_SHUFFLE(0, 3, 2, 1): \ - ret = _mm_shuffle_ps_0321((a), (b)); \ - break; \ - case _MM_SHUFFLE(2, 1, 0, 3): \ - ret = _mm_shuffle_ps_2103((a), (b)); \ - break; \ - case _MM_SHUFFLE(1, 0, 1, 0): \ - ret = _mm_movelh_ps((a), (b)); \ - break; \ - case _MM_SHUFFLE(1, 0, 0, 1): \ - ret = _mm_shuffle_ps_1001((a), (b)); \ - break; \ - case _MM_SHUFFLE(0, 1, 0, 1): \ - ret = _mm_shuffle_ps_0101((a), (b)); \ - break; \ - case _MM_SHUFFLE(3, 2, 1, 0): \ - ret = _mm_shuffle_ps_3210((a), (b)); \ - break; \ - case _MM_SHUFFLE(0, 0, 1, 1): \ - ret = _mm_shuffle_ps_0011((a), (b)); \ - break; \ - case _MM_SHUFFLE(0, 0, 2, 2): \ - ret = _mm_shuffle_ps_0022((a), (b)); \ - break; \ - case _MM_SHUFFLE(2, 2, 0, 0): \ - ret = _mm_shuffle_ps_2200((a), (b)); \ - break; \ - case _MM_SHUFFLE(3, 2, 0, 2): \ - ret = _mm_shuffle_ps_3202((a), (b)); \ - break; \ - case _MM_SHUFFLE(3, 2, 3, 2): \ - ret = _mm_movehl_ps((b), (a)); \ - break; \ - case _MM_SHUFFLE(1, 1, 3, 3): \ - ret = _mm_shuffle_ps_1133((a), (b)); \ - break; \ - case _MM_SHUFFLE(2, 0, 1, 0): \ - ret = _mm_shuffle_ps_2010((a), (b)); \ - break; \ - case _MM_SHUFFLE(2, 0, 0, 1): \ - ret = _mm_shuffle_ps_2001((a), (b)); \ - break; \ - case _MM_SHUFFLE(2, 0, 3, 2): \ - ret = _mm_shuffle_ps_2032((a), (b)); \ - break; \ - default: \ - ret = _mm_shuffle_ps_default((a), (b), (imm)); \ - break; \ - } \ - ret; \ - }) +// Load unaligned 16-bit integer from memory into the first element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadu_si16 +FORCE_INLINE __m128i _mm_loadu_si16(const void *p) +{ + return vreinterpretq_m128i_s16( + vsetq_lane_s16(*(const int16_t *) p, vdupq_n_s16(0), 0)); +} + +// Load unaligned 64-bit integer from memory into the first element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadu_si64 +FORCE_INLINE __m128i _mm_loadu_si64(const void *p) +{ + return vreinterpretq_m128i_s64( + vcombine_s64(vld1_s64((const int64_t *) p), vdup_n_s64(0))); +} + +// Allocate size bytes of memory, aligned to the alignment specified in align, +// and return a pointer to the allocated memory. _mm_free should be used to free +// memory that is allocated with _mm_malloc. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_malloc +#if !defined(SSE2NEON_ALLOC_DEFINED) +FORCE_INLINE void *_mm_malloc(size_t size, size_t align) +{ + void *ptr; + if (align == 1) + return malloc(size); + if (align == 2 || (sizeof(void *) == 8 && align == 4)) + align = sizeof(void *); + if (!posix_memalign(&ptr, align, size)) + return ptr; + return NULL; +} #endif -// Takes the upper 64 bits of a and places it in the low end of the result -// Takes the lower 64 bits of a and places it into the high end of the result. -FORCE_INLINE __m128i _mm_shuffle_epi_1032(__m128i a) +// Conditionally store 8-bit integer elements from a into memory using mask +// (elements are not stored when the highest bit is not set in the corresponding +// element) and a non-temporal memory hint. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maskmove_si64 +FORCE_INLINE void _mm_maskmove_si64(__m64 a, __m64 mask, char *mem_addr) { - int32x2_t a32 = vget_high_s32(vreinterpretq_s32_m128i(a)); - int32x2_t a10 = vget_low_s32(vreinterpretq_s32_m128i(a)); - return vreinterpretq_m128i_s32(vcombine_s32(a32, a10)); + int8x8_t shr_mask = vshr_n_s8(vreinterpret_s8_m64(mask), 7); + __m128 b = _mm_load_ps((const float *) mem_addr); + int8x8_t masked = + vbsl_s8(vreinterpret_u8_s8(shr_mask), vreinterpret_s8_m64(a), + vreinterpret_s8_u64(vget_low_u64(vreinterpretq_u64_m128(b)))); + vst1_s8((int8_t *) mem_addr, masked); } -// takes the lower two 32-bit values from a and swaps them and places in low end -// of result takes the higher two 32 bit values from a and swaps them and places -// in high end of result. -FORCE_INLINE __m128i _mm_shuffle_epi_2301(__m128i a) +// Conditionally store 8-bit integer elements from a into memory using mask +// (elements are not stored when the highest bit is not set in the corresponding +// element) and a non-temporal memory hint. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_maskmovq +#define _m_maskmovq(a, mask, mem_addr) _mm_maskmove_si64(a, mask, mem_addr) + +// Compare packed signed 16-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_pi16 +FORCE_INLINE __m64 _mm_max_pi16(__m64 a, __m64 b) { - int32x2_t a01 = vrev64_s32(vget_low_s32(vreinterpretq_s32_m128i(a))); - int32x2_t a23 = vrev64_s32(vget_high_s32(vreinterpretq_s32_m128i(a))); - return vreinterpretq_m128i_s32(vcombine_s32(a01, a23)); + return vreinterpret_m64_s16( + vmax_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); } -// rotates the least significant 32 bits into the most signficant 32 bits, and -// shifts the rest down -FORCE_INLINE __m128i _mm_shuffle_epi_0321(__m128i a) +// Compare packed single-precision (32-bit) floating-point elements in a and b, +// and store packed maximum values in dst. dst does not follow the IEEE Standard +// for Floating-Point Arithmetic (IEEE 754) maximum value when inputs are NaN or +// signed-zero values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_ps +FORCE_INLINE __m128 _mm_max_ps(__m128 a, __m128 b) { - return vreinterpretq_m128i_s32( - vextq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(a), 1)); +#if SSE2NEON_PRECISE_MINMAX + float32x4_t _a = vreinterpretq_f32_m128(a); + float32x4_t _b = vreinterpretq_f32_m128(b); + return vreinterpretq_m128_f32(vbslq_f32(vcgtq_f32(_a, _b), _a, _b)); +#else + return vreinterpretq_m128_f32( + vmaxq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +#endif } -// rotates the most significant 32 bits into the least signficant 32 bits, and -// shifts the rest up -FORCE_INLINE __m128i _mm_shuffle_epi_2103(__m128i a) +// Compare packed unsigned 8-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_pu8 +FORCE_INLINE __m64 _mm_max_pu8(__m64 a, __m64 b) { - return vreinterpretq_m128i_s32( - vextq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(a), 3)); + return vreinterpret_m64_u8( + vmax_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); } -// gets the lower 64 bits of a, and places it in the upper 64 bits -// gets the lower 64 bits of a and places it in the lower 64 bits -FORCE_INLINE __m128i _mm_shuffle_epi_1010(__m128i a) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b, store the maximum value in the lower element of dst, and copy the upper 3 +// packed elements from a to the upper element of dst. dst does not follow the +// IEEE Standard for Floating-Point Arithmetic (IEEE 754) maximum value when +// inputs are NaN or signed-zero values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_ss +FORCE_INLINE __m128 _mm_max_ss(__m128 a, __m128 b) { - int32x2_t a10 = vget_low_s32(vreinterpretq_s32_m128i(a)); - return vreinterpretq_m128i_s32(vcombine_s32(a10, a10)); + float32_t value = vgetq_lane_f32(_mm_max_ps(a, b), 0); + return vreinterpretq_m128_f32( + vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); } -// gets the lower 64 bits of a, swaps the 0 and 1 elements, and places it in the -// lower 64 bits gets the lower 64 bits of a, and places it in the upper 64 bits -FORCE_INLINE __m128i _mm_shuffle_epi_1001(__m128i a) +// Compare packed signed 16-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_pi16 +FORCE_INLINE __m64 _mm_min_pi16(__m64 a, __m64 b) { - int32x2_t a01 = vrev64_s32(vget_low_s32(vreinterpretq_s32_m128i(a))); - int32x2_t a10 = vget_low_s32(vreinterpretq_s32_m128i(a)); - return vreinterpretq_m128i_s32(vcombine_s32(a01, a10)); + return vreinterpret_m64_s16( + vmin_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); } -// gets the lower 64 bits of a, swaps the 0 and 1 elements and places it in the -// upper 64 bits gets the lower 64 bits of a, swaps the 0 and 1 elements, and -// places it in the lower 64 bits -FORCE_INLINE __m128i _mm_shuffle_epi_0101(__m128i a) +// Compare packed single-precision (32-bit) floating-point elements in a and b, +// and store packed minimum values in dst. dst does not follow the IEEE Standard +// for Floating-Point Arithmetic (IEEE 754) minimum value when inputs are NaN or +// signed-zero values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_ps +FORCE_INLINE __m128 _mm_min_ps(__m128 a, __m128 b) { - int32x2_t a01 = vrev64_s32(vget_low_s32(vreinterpretq_s32_m128i(a))); - return vreinterpretq_m128i_s32(vcombine_s32(a01, a01)); +#if SSE2NEON_PRECISE_MINMAX + float32x4_t _a = vreinterpretq_f32_m128(a); + float32x4_t _b = vreinterpretq_f32_m128(b); + return vreinterpretq_m128_f32(vbslq_f32(vcltq_f32(_a, _b), _a, _b)); +#else + return vreinterpretq_m128_f32( + vminq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +#endif } -FORCE_INLINE __m128i _mm_shuffle_epi_2211(__m128i a) +// Compare packed unsigned 8-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_pu8 +FORCE_INLINE __m64 _mm_min_pu8(__m64 a, __m64 b) { - int32x2_t a11 = vdup_lane_s32(vget_low_s32(vreinterpretq_s32_m128i(a)), 1); - int32x2_t a22 = vdup_lane_s32(vget_high_s32(vreinterpretq_s32_m128i(a)), 0); - return vreinterpretq_m128i_s32(vcombine_s32(a11, a22)); + return vreinterpret_m64_u8( + vmin_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); } -FORCE_INLINE __m128i _mm_shuffle_epi_0122(__m128i a) +// Compare the lower single-precision (32-bit) floating-point elements in a and +// b, store the minimum value in the lower element of dst, and copy the upper 3 +// packed elements from a to the upper element of dst. dst does not follow the +// IEEE Standard for Floating-Point Arithmetic (IEEE 754) minimum value when +// inputs are NaN or signed-zero values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_ss +FORCE_INLINE __m128 _mm_min_ss(__m128 a, __m128 b) { - int32x2_t a22 = vdup_lane_s32(vget_high_s32(vreinterpretq_s32_m128i(a)), 0); - int32x2_t a01 = vrev64_s32(vget_low_s32(vreinterpretq_s32_m128i(a))); - return vreinterpretq_m128i_s32(vcombine_s32(a22, a01)); + float32_t value = vgetq_lane_f32(_mm_min_ps(a, b), 0); + return vreinterpretq_m128_f32( + vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); } -FORCE_INLINE __m128i _mm_shuffle_epi_3332(__m128i a) +// Move the lower single-precision (32-bit) floating-point element from b to the +// lower element of dst, and copy the upper 3 packed elements from a to the +// upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_move_ss +FORCE_INLINE __m128 _mm_move_ss(__m128 a, __m128 b) { - int32x2_t a32 = vget_high_s32(vreinterpretq_s32_m128i(a)); - int32x2_t a33 = vdup_lane_s32(vget_high_s32(vreinterpretq_s32_m128i(a)), 1); - return vreinterpretq_m128i_s32(vcombine_s32(a32, a33)); + return vreinterpretq_m128_f32( + vsetq_lane_f32(vgetq_lane_f32(vreinterpretq_f32_m128(b), 0), + vreinterpretq_f32_m128(a), 0)); } -// Shuffle packed 8-bit integers in a according to shuffle control mask in the -// corresponding 8-bit element of b, and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_shuffle_epi8 -FORCE_INLINE __m128i _mm_shuffle_epi8(__m128i a, __m128i b) +// Move the upper 2 single-precision (32-bit) floating-point elements from b to +// the lower 2 elements of dst, and copy the upper 2 elements from a to the +// upper 2 elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movehl_ps +FORCE_INLINE __m128 _mm_movehl_ps(__m128 a, __m128 b) { - int8x16_t tbl = vreinterpretq_s8_m128i(a); // input a - uint8x16_t idx = vreinterpretq_u8_m128i(b); // input b - uint8x16_t idx_masked = - vandq_u8(idx, vdupq_n_u8(0x8F)); // avoid using meaningless bits -#if defined(__aarch64__) - return vreinterpretq_m128i_s8(vqtbl1q_s8(tbl, idx_masked)); -#elif defined(__GNUC__) - int8x16_t ret; - // %e and %f represent the even and odd D registers - // respectively. - __asm__ __volatile__( - "vtbl.8 %e[ret], {%e[tbl], %f[tbl]}, %e[idx]\n" - "vtbl.8 %f[ret], {%e[tbl], %f[tbl]}, %f[idx]\n" - : [ret] "=&w"(ret) - : [tbl] "w"(tbl), [idx] "w"(idx_masked)); - return vreinterpretq_m128i_s8(ret); +#if defined(aarch64__) + return vreinterpretq_m128_u64( + vzip2q_u64(vreinterpretq_u64_m128(b), vreinterpretq_u64_m128(a))); #else - // use this line if testing on aarch64 - int8x8x2_t a_split = {vget_low_s8(tbl), vget_high_s8(tbl)}; - return vreinterpretq_m128i_s8( - vcombine_s8(vtbl2_s8(a_split, vget_low_u8(idx_masked)), - vtbl2_s8(a_split, vget_high_u8(idx_masked)))); + float32x2_t a32 = vget_high_f32(vreinterpretq_f32_m128(a)); + float32x2_t b32 = vget_high_f32(vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_f32(vcombine_f32(b32, a32)); #endif } -// C equivalent: -// __m128i _mm_shuffle_epi32_default(__m128i a, -// __constrange(0, 255) int imm) { -// __m128i ret; -// ret[0] = a[imm & 0x3]; ret[1] = a[(imm >> 2) & 0x3]; -// ret[2] = a[(imm >> 4) & 0x03]; ret[3] = a[(imm >> 6) & 0x03]; -// return ret; -// } -#define _mm_shuffle_epi32_default(a, imm) \ - __extension__({ \ - int32x4_t ret; \ - ret = vmovq_n_s32( \ - vgetq_lane_s32(vreinterpretq_s32_m128i(a), (imm) & (0x3))); \ - ret = vsetq_lane_s32( \ - vgetq_lane_s32(vreinterpretq_s32_m128i(a), ((imm) >> 2) & 0x3), \ - ret, 1); \ - ret = vsetq_lane_s32( \ - vgetq_lane_s32(vreinterpretq_s32_m128i(a), ((imm) >> 4) & 0x3), \ - ret, 2); \ - ret = vsetq_lane_s32( \ - vgetq_lane_s32(vreinterpretq_s32_m128i(a), ((imm) >> 6) & 0x3), \ - ret, 3); \ - vreinterpretq_m128i_s32(ret); \ - }) +// Move the lower 2 single-precision (32-bit) floating-point elements from b to +// the upper 2 elements of dst, and copy the lower 2 elements from a to the +// lower 2 elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movelh_ps +FORCE_INLINE __m128 _mm_movelh_ps(__m128 __A, __m128 __B) +{ + float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(__A)); + float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(__B)); + return vreinterpretq_m128_f32(vcombine_f32(a10, b10)); +} -// FORCE_INLINE __m128i _mm_shuffle_epi32_splat(__m128i a, __constrange(0,255) -// int imm) -#if defined(__aarch64__) -#define _mm_shuffle_epi32_splat(a, imm) \ - __extension__({ \ - vreinterpretq_m128i_s32( \ - vdupq_laneq_s32(vreinterpretq_s32_m128i(a), (imm))); \ - }) +// Create mask from the most significant bit of each 8-bit element in a, and +// store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movemask_pi8 +FORCE_INLINE int _mm_movemask_pi8(__m64 a) +{ + uint8x8_t input = vreinterpret_u8_m64(a); +#if defined(__aarch64__) || defined(_M_ARM64) + static const int8_t shift[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint8x8_t tmp = vshr_n_u8(input, 7); + return vaddv_u8(vshl_u8(tmp, vld1_s8(shift))); #else -#define _mm_shuffle_epi32_splat(a, imm) \ - __extension__({ \ - vreinterpretq_m128i_s32( \ - vdupq_n_s32(vgetq_lane_s32(vreinterpretq_s32_m128i(a), (imm)))); \ - }) + // Refer the implementation of `_mm_movemask_epi8` + uint16x4_t high_bits = vreinterpret_u16_u8(vshr_n_u8(input, 7)); + uint32x2_t paired16 = + vreinterpret_u32_u16(vsra_n_u16(high_bits, high_bits, 7)); + uint8x8_t paired32 = + vreinterpret_u8_u32(vsra_n_u32(paired16, paired16, 14)); + return vget_lane_u8(paired32, 0) | ((int) vget_lane_u8(paired32, 4) << 4); #endif +} -// Shuffles the 4 signed or unsigned 32-bit integers in a as specified by imm. -// https://msdn.microsoft.com/en-us/library/56f67xbk%28v=vs.90%29.aspx -// FORCE_INLINE __m128i _mm_shuffle_epi32(__m128i a, -// __constrange(0,255) int imm) -#if __has_builtin(__builtin_shufflevector) -#define _mm_shuffle_epi32(a, imm) \ - __extension__({ \ - int32x4_t _input = vreinterpretq_s32_m128i(a); \ - int32x4_t _shuf = __builtin_shufflevector( \ - _input, _input, (imm) & (0x3), ((imm) >> 2) & 0x3, \ - ((imm) >> 4) & 0x3, ((imm) >> 6) & 0x3); \ - vreinterpretq_m128i_s32(_shuf); \ - }) -#else // generic -#define _mm_shuffle_epi32(a, imm) \ - __extension__({ \ - __m128i ret; \ - switch (imm) { \ - case _MM_SHUFFLE(1, 0, 3, 2): \ - ret = _mm_shuffle_epi_1032((a)); \ - break; \ - case _MM_SHUFFLE(2, 3, 0, 1): \ - ret = _mm_shuffle_epi_2301((a)); \ - break; \ - case _MM_SHUFFLE(0, 3, 2, 1): \ - ret = _mm_shuffle_epi_0321((a)); \ - break; \ - case _MM_SHUFFLE(2, 1, 0, 3): \ - ret = _mm_shuffle_epi_2103((a)); \ - break; \ - case _MM_SHUFFLE(1, 0, 1, 0): \ - ret = _mm_shuffle_epi_1010((a)); \ - break; \ - case _MM_SHUFFLE(1, 0, 0, 1): \ - ret = _mm_shuffle_epi_1001((a)); \ - break; \ - case _MM_SHUFFLE(0, 1, 0, 1): \ - ret = _mm_shuffle_epi_0101((a)); \ - break; \ - case _MM_SHUFFLE(2, 2, 1, 1): \ - ret = _mm_shuffle_epi_2211((a)); \ - break; \ - case _MM_SHUFFLE(0, 1, 2, 2): \ - ret = _mm_shuffle_epi_0122((a)); \ - break; \ - case _MM_SHUFFLE(3, 3, 3, 2): \ - ret = _mm_shuffle_epi_3332((a)); \ - break; \ - case _MM_SHUFFLE(0, 0, 0, 0): \ - ret = _mm_shuffle_epi32_splat((a), 0); \ - break; \ - case _MM_SHUFFLE(1, 1, 1, 1): \ - ret = _mm_shuffle_epi32_splat((a), 1); \ - break; \ - case _MM_SHUFFLE(2, 2, 2, 2): \ - ret = _mm_shuffle_epi32_splat((a), 2); \ - break; \ - case _MM_SHUFFLE(3, 3, 3, 3): \ - ret = _mm_shuffle_epi32_splat((a), 3); \ - break; \ - default: \ - ret = _mm_shuffle_epi32_default((a), (imm)); \ - break; \ - } \ - ret; \ - }) +// Set each bit of mask dst based on the most significant bit of the +// corresponding packed single-precision (32-bit) floating-point element in a. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movemask_ps +FORCE_INLINE int _mm_movemask_ps(__m128 a) +{ + uint32x4_t input = vreinterpretq_u32_m128(a); +#if defined(__aarch64__) || defined(_M_ARM64) + static const int32_t shift[4] = {0, 1, 2, 3}; + uint32x4_t tmp = vshrq_n_u32(input, 31); + return vaddvq_u32(vshlq_u32(tmp, vld1q_s32(shift))); +#else + // Uses the exact same method as _mm_movemask_epi8, see that for details. + // Shift out everything but the sign bits with a 32-bit unsigned shift + // right. + uint64x2_t high_bits = vreinterpretq_u64_u32(vshrq_n_u32(input, 31)); + // Merge the two pairs together with a 64-bit unsigned shift right + add. + uint8x16_t paired = + vreinterpretq_u8_u64(vsraq_n_u64(high_bits, high_bits, 31)); + // Extract the result. + return vgetq_lane_u8(paired, 0) | (vgetq_lane_u8(paired, 8) << 2); #endif +} -// Shuffles the lower 4 signed or unsigned 16-bit integers in a as specified -// by imm. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/y41dkk37(v=vs.100) -// FORCE_INLINE __m128i _mm_shufflelo_epi16_function(__m128i a, -// __constrange(0,255) int -// imm) -#define _mm_shufflelo_epi16_function(a, imm) \ - __extension__({ \ - int16x8_t ret = vreinterpretq_s16_m128i(a); \ - int16x4_t lowBits = vget_low_s16(ret); \ - ret = vsetq_lane_s16(vget_lane_s16(lowBits, (imm) & (0x3)), ret, 0); \ - ret = vsetq_lane_s16(vget_lane_s16(lowBits, ((imm) >> 2) & 0x3), ret, \ - 1); \ - ret = vsetq_lane_s16(vget_lane_s16(lowBits, ((imm) >> 4) & 0x3), ret, \ - 2); \ - ret = vsetq_lane_s16(vget_lane_s16(lowBits, ((imm) >> 6) & 0x3), ret, \ - 3); \ - vreinterpretq_m128i_s16(ret); \ - }) +// Multiply packed single-precision (32-bit) floating-point elements in a and b, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mul_ps +FORCE_INLINE __m128 _mm_mul_ps(__m128 a, __m128 b) +{ + return vreinterpretq_m128_f32( + vmulq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +} -// FORCE_INLINE __m128i _mm_shufflelo_epi16(__m128i a, -// __constrange(0,255) int imm) -#if __has_builtin(__builtin_shufflevector) -#define _mm_shufflelo_epi16(a, imm) \ - __extension__({ \ - int16x8_t _input = vreinterpretq_s16_m128i(a); \ - int16x8_t _shuf = __builtin_shufflevector( \ - _input, _input, ((imm) & (0x3)), (((imm) >> 2) & 0x3), \ - (((imm) >> 4) & 0x3), (((imm) >> 6) & 0x3), 4, 5, 6, 7); \ - vreinterpretq_m128i_s16(_shuf); \ - }) -#else // generic -#define _mm_shufflelo_epi16(a, imm) _mm_shufflelo_epi16_function((a), (imm)) -#endif +// Multiply the lower single-precision (32-bit) floating-point element in a and +// b, store the result in the lower element of dst, and copy the upper 3 packed +// elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mul_ss +FORCE_INLINE __m128 _mm_mul_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_mul_ps(a, b)); +} -// Shuffles the upper 4 signed or unsigned 16-bit integers in a as specified -// by imm. -// https://msdn.microsoft.com/en-us/library/13ywktbs(v=vs.100).aspx -// FORCE_INLINE __m128i _mm_shufflehi_epi16_function(__m128i a, -// __constrange(0,255) int -// imm) -#define _mm_shufflehi_epi16_function(a, imm) \ - __extension__({ \ - int16x8_t ret = vreinterpretq_s16_m128i(a); \ - int16x4_t highBits = vget_high_s16(ret); \ - ret = vsetq_lane_s16(vget_lane_s16(highBits, (imm) & (0x3)), ret, 4); \ - ret = vsetq_lane_s16(vget_lane_s16(highBits, ((imm) >> 2) & 0x3), ret, \ - 5); \ - ret = vsetq_lane_s16(vget_lane_s16(highBits, ((imm) >> 4) & 0x3), ret, \ - 6); \ - ret = vsetq_lane_s16(vget_lane_s16(highBits, ((imm) >> 6) & 0x3), ret, \ - 7); \ - vreinterpretq_m128i_s16(ret); \ - }) +// Multiply the packed unsigned 16-bit integers in a and b, producing +// intermediate 32-bit integers, and store the high 16 bits of the intermediate +// integers in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mulhi_pu16 +FORCE_INLINE __m64 _mm_mulhi_pu16(__m64 a, __m64 b) +{ + return vreinterpret_m64_u16(vshrn_n_u32( + vmull_u16(vreinterpret_u16_m64(a), vreinterpret_u16_m64(b)), 16)); +} -// FORCE_INLINE __m128i _mm_shufflehi_epi16(__m128i a, -// __constrange(0,255) int imm) -#if __has_builtin(__builtin_shufflevector) -#define _mm_shufflehi_epi16(a, imm) \ - __extension__({ \ - int16x8_t _input = vreinterpretq_s16_m128i(a); \ - int16x8_t _shuf = __builtin_shufflevector( \ - _input, _input, 0, 1, 2, 3, ((imm) & (0x3)) + 4, \ - (((imm) >> 2) & 0x3) + 4, (((imm) >> 4) & 0x3) + 4, \ - (((imm) >> 6) & 0x3) + 4); \ - vreinterpretq_m128i_s16(_shuf); \ - }) -#else // generic -#define _mm_shufflehi_epi16(a, imm) _mm_shufflehi_epi16_function((a), (imm)) -#endif +// Compute the bitwise OR of packed single-precision (32-bit) floating-point +// elements in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_or_ps +FORCE_INLINE __m128 _mm_or_ps(__m128 a, __m128 b) +{ + return vreinterpretq_m128_s32( + vorrq_s32(vreinterpretq_s32_m128(a), vreinterpretq_s32_m128(b))); +} -// Blend packed 16-bit integers from a and b using control mask imm8, and store -// the results in dst. -// -// FOR j := 0 to 7 -// i := j*16 -// IF imm8[j] -// dst[i+15:i] := b[i+15:i] -// ELSE -// dst[i+15:i] := a[i+15:i] -// FI -// ENDFOR -// FORCE_INLINE __m128i _mm_blend_epi16(__m128i a, __m128i b, -// __constrange(0,255) int imm) -#define _mm_blend_epi16(a, b, imm) \ - __extension__({ \ - const uint16_t _mask[8] = {((imm) & (1 << 0)) ? 0xFFFF : 0x0000, \ - ((imm) & (1 << 1)) ? 0xFFFF : 0x0000, \ - ((imm) & (1 << 2)) ? 0xFFFF : 0x0000, \ - ((imm) & (1 << 3)) ? 0xFFFF : 0x0000, \ - ((imm) & (1 << 4)) ? 0xFFFF : 0x0000, \ - ((imm) & (1 << 5)) ? 0xFFFF : 0x0000, \ - ((imm) & (1 << 6)) ? 0xFFFF : 0x0000, \ - ((imm) & (1 << 7)) ? 0xFFFF : 0x0000}; \ - uint16x8_t _mask_vec = vld1q_u16(_mask); \ - uint16x8_t _a = vreinterpretq_u16_m128i(a); \ - uint16x8_t _b = vreinterpretq_u16_m128i(b); \ - vreinterpretq_m128i_u16(vbslq_u16(_mask_vec, _b, _a)); \ - }) +// Average packed unsigned 8-bit integers in a and b, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pavgb +#define _m_pavgb(a, b) _mm_avg_pu8(a, b) -// Blend packed 8-bit integers from a and b using mask, and store the results in +// Average packed unsigned 16-bit integers in a and b, and store the results in // dst. -// -// FOR j := 0 to 15 -// i := j*8 -// IF mask[i+7] -// dst[i+7:i] := b[i+7:i] -// ELSE -// dst[i+7:i] := a[i+7:i] -// FI -// ENDFOR -FORCE_INLINE __m128i _mm_blendv_epi8(__m128i _a, __m128i _b, __m128i _mask) -{ - // Use a signed shift right to create a mask with the sign bit - uint8x16_t mask = - vreinterpretq_u8_s8(vshrq_n_s8(vreinterpretq_s8_m128i(_mask), 7)); - uint8x16_t a = vreinterpretq_u8_m128i(_a); - uint8x16_t b = vreinterpretq_u8_m128i(_b); - return vreinterpretq_m128i_u8(vbslq_u8(mask, b, a)); -} +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pavgw +#define _m_pavgw(a, b) _mm_avg_pu16(a, b) -/* Shifts */ +// Extract a 16-bit integer from a, selected with imm8, and store the result in +// the lower element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pextrw +#define _m_pextrw(a, imm) _mm_extract_pi16(a, imm) +// Copy a to dst, and insert the 16-bit integer i into dst at the location +// specified by imm8. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=m_pinsrw +#define _m_pinsrw(a, i, imm) _mm_insert_pi16(a, i, imm) -// Shift packed 16-bit integers in a right by imm while shifting in sign -// bits, and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi16 -FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int imm) +// Compare packed signed 16-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pmaxsw +#define _m_pmaxsw(a, b) _mm_max_pi16(a, b) + +// Compare packed unsigned 8-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pmaxub +#define _m_pmaxub(a, b) _mm_max_pu8(a, b) + +// Compare packed signed 16-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pminsw +#define _m_pminsw(a, b) _mm_min_pi16(a, b) + +// Compare packed unsigned 8-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pminub +#define _m_pminub(a, b) _mm_min_pu8(a, b) + +// Create mask from the most significant bit of each 8-bit element in a, and +// store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pmovmskb +#define _m_pmovmskb(a) _mm_movemask_pi8(a) + +// Multiply the packed unsigned 16-bit integers in a and b, producing +// intermediate 32-bit integers, and store the high 16 bits of the intermediate +// integers in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pmulhuw +#define _m_pmulhuw(a, b) _mm_mulhi_pu16(a, b) + +// Fetch the line of data from memory that contains address p to a location in +// the cache hierarchy specified by the locality hint i. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_prefetch +FORCE_INLINE void _mm_prefetch(char const *p, int i) { - const int count = (imm & ~15) ? 15 : imm; - return (__m128i) vshlq_s16((int16x8_t) a, vdupq_n_s16(-count)); + (void) i; +#if defined(_MSC_VER) + switch (i) { + case _MM_HINT_NTA: + __prefetch2(p, 1); + break; + case _MM_HINT_T0: + __prefetch2(p, 0); + break; + case _MM_HINT_T1: + __prefetch2(p, 2); + break; + case _MM_HINT_T2: + __prefetch2(p, 4); + break; + } +#else + switch (i) { + case _MM_HINT_NTA: + __builtin_prefetch(p, 0, 0); + break; + case _MM_HINT_T0: + __builtin_prefetch(p, 0, 3); + break; + case _MM_HINT_T1: + __builtin_prefetch(p, 0, 2); + break; + case _MM_HINT_T2: + __builtin_prefetch(p, 0, 1); + break; + } +#endif } -// Shifts the 8 signed or unsigned 16-bit integers in a left by count bits while -// shifting in zeros. -// -// r0 := a0 << count -// r1 := a1 << count -// ... -// r7 := a7 << count -// -// https://msdn.microsoft.com/en-us/library/es73bcsy(v=vs.90).aspx -#define _mm_slli_epi16(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 15) { \ - ret = _mm_setzero_si128(); \ - } else { \ - ret = vreinterpretq_m128i_s16( \ - vshlq_n_s16(vreinterpretq_s16_m128i(a), (imm))); \ - } \ - ret; \ - }) +// Compute the absolute differences of packed unsigned 8-bit integers in a and +// b, then horizontally sum each consecutive 8 differences to produce four +// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low +// 16 bits of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=m_psadbw +#define _m_psadbw(a, b) _mm_sad_pu8(a, b) -// Shifts the 4 signed or unsigned 32-bit integers in a left by count bits while -// shifting in zeros. : -// https://msdn.microsoft.com/en-us/library/z2k3bbtb%28v=vs.90%29.aspx -// FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, __constrange(0,255) int imm) -FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, int imm) +// Shuffle 16-bit integers in a using the control in imm8, and store the results +// in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_m_pshufw +#define _m_pshufw(a, imm) _mm_shuffle_pi16(a, imm) + +// Compute the approximate reciprocal of packed single-precision (32-bit) +// floating-point elements in a, and store the results in dst. The maximum +// relative error for this approximation is less than 1.5*2^-12. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_rcp_ps +FORCE_INLINE __m128 _mm_rcp_ps(__m128 in) { - if (imm <= 0) /* TODO: add constant range macro: [0, 255] */ - return a; - if (imm > 31) /* TODO: add unlikely macro */ - return _mm_setzero_si128(); - return vreinterpretq_m128i_s32( - vshlq_s32(vreinterpretq_s32_m128i(a), vdupq_n_s32(imm))); + float32x4_t recip = vrecpeq_f32(vreinterpretq_f32_m128(in)); + recip = vmulq_f32(recip, vrecpsq_f32(recip, vreinterpretq_f32_m128(in))); + return vreinterpretq_m128_f32(recip); } -// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and -// store the results in dst. -FORCE_INLINE __m128i _mm_slli_epi64(__m128i a, int imm) +// Compute the approximate reciprocal of the lower single-precision (32-bit) +// floating-point element in a, store the result in the lower element of dst, +// and copy the upper 3 packed elements from a to the upper elements of dst. The +// maximum relative error for this approximation is less than 1.5*2^-12. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_rcp_ss +FORCE_INLINE __m128 _mm_rcp_ss(__m128 a) { - if (imm <= 0) /* TODO: add constant range macro: [0, 255] */ - return a; - if (imm > 63) /* TODO: add unlikely macro */ - return _mm_setzero_si128(); - return vreinterpretq_m128i_s64( - vshlq_s64(vreinterpretq_s64_m128i(a), vdupq_n_s64(imm))); + return _mm_move_ss(a, _mm_rcp_ps(a)); } -// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and -// store the results in dst. -// -// FOR j := 0 to 7 -// i := j*16 -// IF imm8[7:0] > 15 -// dst[i+15:i] := 0 -// ELSE -// dst[i+15:i] := ZeroExtend16(a[i+15:i] >> imm8[7:0]) -// FI -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi16 -#define _mm_srli_epi16(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) == 0) { \ - ret = a; \ - } else if (0 < (imm) && (imm) < 16) { \ - ret = vreinterpretq_m128i_u16( \ - vshlq_u16(vreinterpretq_u16_m128i(a), vdupq_n_s16(-imm))); \ - } else { \ - ret = _mm_setzero_si128(); \ - } \ - ret; \ - }) - -// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and -// store the results in dst. -// -// FOR j := 0 to 3 -// i := j*32 -// IF imm8[7:0] > 31 -// dst[i+31:i] := 0 -// ELSE -// dst[i+31:i] := ZeroExtend32(a[i+31:i] >> imm8[7:0]) -// FI -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi32 -// FORCE_INLINE __m128i _mm_srli_epi32(__m128i a, __constrange(0,255) int imm) -#define _mm_srli_epi32(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) == 0) { \ - ret = a; \ - } else if (0 < (imm) && (imm) < 32) { \ - ret = vreinterpretq_m128i_u32( \ - vshlq_u32(vreinterpretq_u32_m128i(a), vdupq_n_s32(-imm))); \ - } else { \ - ret = _mm_setzero_si128(); \ - } \ - ret; \ - }) +// Compute the approximate reciprocal square root of packed single-precision +// (32-bit) floating-point elements in a, and store the results in dst. The +// maximum relative error for this approximation is less than 1.5*2^-12. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_rsqrt_ps +FORCE_INLINE __m128 _mm_rsqrt_ps(__m128 in) +{ + float32x4_t out = vrsqrteq_f32(vreinterpretq_f32_m128(in)); -// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and -// store the results in dst. -// -// FOR j := 0 to 1 -// i := j*64 -// IF imm8[7:0] > 63 -// dst[i+63:i] := 0 -// ELSE -// dst[i+63:i] := ZeroExtend64(a[i+63:i] >> imm8[7:0]) -// FI -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srli_epi64 -#define _mm_srli_epi64(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) == 0) { \ - ret = a; \ - } else if (0 < (imm) && (imm) < 64) { \ - ret = vreinterpretq_m128i_u64( \ - vshlq_u64(vreinterpretq_u64_m128i(a), vdupq_n_s64(-imm))); \ - } else { \ - ret = _mm_setzero_si128(); \ - } \ - ret; \ - }) + // Generate masks for detecting whether input has any 0.0f/-0.0f + // (which becomes positive/negative infinity by IEEE-754 arithmetic rules). + const uint32x4_t pos_inf = vdupq_n_u32(0x7F800000); + const uint32x4_t neg_inf = vdupq_n_u32(0xFF800000); + const uint32x4_t has_pos_zero = + vceqq_u32(pos_inf, vreinterpretq_u32_f32(out)); + const uint32x4_t has_neg_zero = + vceqq_u32(neg_inf, vreinterpretq_u32_f32(out)); -// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, -// and store the results in dst. -// -// FOR j := 0 to 3 -// i := j*32 -// IF imm8[7:0] > 31 -// dst[i+31:i] := (a[i+31] ? 0xFFFFFFFF : 0x0) -// ELSE -// dst[i+31:i] := SignExtend32(a[i+31:i] >> imm8[7:0]) -// FI -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_srai_epi32 -// FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, __constrange(0,255) int imm) -#define _mm_srai_epi32(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) == 0) { \ - ret = a; \ - } else if (0 < (imm) && (imm) < 32) { \ - ret = vreinterpretq_m128i_s32( \ - vshlq_s32(vreinterpretq_s32_m128i(a), vdupq_n_s32(-imm))); \ - } else { \ - ret = vreinterpretq_m128i_s32( \ - vshrq_n_s32(vreinterpretq_s32_m128i(a), 31)); \ - } \ - ret; \ - }) + out = vmulq_f32( + out, vrsqrtsq_f32(vmulq_f32(vreinterpretq_f32_m128(in), out), out)); -// Shifts the 128 - bit value in a right by imm bytes while shifting in -// zeros.imm must be an immediate. -// -// r := srl(a, imm*8) -// -// https://msdn.microsoft.com/en-us/library/305w28yz(v=vs.100).aspx -// FORCE_INLINE _mm_srli_si128(__m128i a, __constrange(0,255) int imm) -#define _mm_srli_si128(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 15) { \ - ret = _mm_setzero_si128(); \ - } else { \ - ret = vreinterpretq_m128i_s8( \ - vextq_s8(vreinterpretq_s8_m128i(a), vdupq_n_s8(0), (imm))); \ - } \ - ret; \ - }) + // Set output vector element to infinity/negative-infinity if + // the corresponding input vector element is 0.0f/-0.0f. + out = vbslq_f32(has_pos_zero, (float32x4_t) pos_inf, out); + out = vbslq_f32(has_neg_zero, (float32x4_t) neg_inf, out); -// Shifts the 128-bit value in a left by imm bytes while shifting in zeros. imm -// must be an immediate. -// -// r := a << (imm * 8) -// -// https://msdn.microsoft.com/en-us/library/34d3k2kt(v=vs.100).aspx -// FORCE_INLINE __m128i _mm_slli_si128(__m128i a, __constrange(0,255) int imm) -#define _mm_slli_si128(a, imm) \ - __extension__({ \ - __m128i ret; \ - if ((imm) <= 0) { \ - ret = a; \ - } else if ((imm) > 15) { \ - ret = _mm_setzero_si128(); \ - } else { \ - ret = vreinterpretq_m128i_s8(vextq_s8( \ - vdupq_n_s8(0), vreinterpretq_s8_m128i(a), 16 - (imm))); \ - } \ - ret; \ - }) + return vreinterpretq_m128_f32(out); +} -// Shifts the 8 signed or unsigned 16-bit integers in a left by count bits while -// shifting in zeros. -// -// r0 := a0 << count -// r1 := a1 << count -// ... -// r7 := a7 << count -// -// https://msdn.microsoft.com/en-us/library/c79w388h(v%3dvs.90).aspx -FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) +// Compute the approximate reciprocal square root of the lower single-precision +// (32-bit) floating-point element in a, store the result in the lower element +// of dst, and copy the upper 3 packed elements from a to the upper elements of +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_rsqrt_ss +FORCE_INLINE __m128 _mm_rsqrt_ss(__m128 in) { - uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); - if (c > 15) - return _mm_setzero_si128(); + return vsetq_lane_f32(vgetq_lane_f32(_mm_rsqrt_ps(in), 0), in, 0); +} - int16x8_t vc = vdupq_n_s16((int16_t) c); - return vreinterpretq_m128i_s16(vshlq_s16(vreinterpretq_s16_m128i(a), vc)); +// Compute the absolute differences of packed unsigned 8-bit integers in a and +// b, then horizontally sum each consecutive 8 differences to produce four +// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low +// 16 bits of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sad_pu8 +FORCE_INLINE __m64 _mm_sad_pu8(__m64 a, __m64 b) +{ + uint64x1_t t = vpaddl_u32(vpaddl_u16( + vpaddl_u8(vabd_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))))); + return vreinterpret_m64_u16( + vset_lane_u16((int) vget_lane_u64(t, 0), vdup_n_u16(0), 0)); } -// Shifts the 4 signed or unsigned 32-bit integers in a left by count bits while -// shifting in zeros. -// -// r0 := a0 << count -// r1 := a1 << count -// r2 := a2 << count -// r3 := a3 << count -// -// https://msdn.microsoft.com/en-us/library/6fe5a6s9(v%3dvs.90).aspx -FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) +// Macro: Set the flush zero bits of the MXCSR control and status register to +// the value in unsigned 32-bit integer a. The flush zero may contain any of the +// following flags: _MM_FLUSH_ZERO_ON or _MM_FLUSH_ZERO_OFF +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_SET_FLUSH_ZERO_MODE +FORCE_INLINE void _sse2neon_mm_set_flush_zero_mode(unsigned int flag) { - uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); - if (c > 31) - return _mm_setzero_si128(); + // AArch32 Advanced SIMD arithmetic always uses the Flush-to-zero setting, + // regardless of the value of the FZ bit. + union { + fpcr_bitfield field; +#if defined(__aarch64__) || defined(_M_ARM64) + uint64_t value; +#else + uint32_t value; +#endif + } r; - int32x4_t vc = vdupq_n_s32((int32_t) c); - return vreinterpretq_m128i_s32(vshlq_s32(vreinterpretq_s32_m128i(a), vc)); +#if defined(__aarch64__) || defined(_M_ARM64) + r.value = _sse2neon_get_fpcr(); +#else + __asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ +#endif + + r.field.bit24 = (flag & _MM_FLUSH_ZERO_MASK) == _MM_FLUSH_ZERO_ON; + +#if defined(__aarch64__) || defined(_M_ARM64) + _sse2neon_set_fpcr(r.value); +#else + __asm__ __volatile__("vmsr FPSCR, %0" ::"r"(r)); /* write */ +#endif } -// Shifts the 2 signed or unsigned 64-bit integers in a left by count bits while -// shifting in zeros. -// -// r0 := a0 << count -// r1 := a1 << count -// -// https://msdn.microsoft.com/en-us/library/6ta9dffd(v%3dvs.90).aspx -FORCE_INLINE __m128i _mm_sll_epi64(__m128i a, __m128i count) +// Set packed single-precision (32-bit) floating-point elements in dst with the +// supplied values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_ps +FORCE_INLINE __m128 _mm_set_ps(float w, float z, float y, float x) { - uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); - if (c > 63) - return _mm_setzero_si128(); + float ALIGN_STRUCT(16) data[4] = {x, y, z, w}; + return vreinterpretq_m128_f32(vld1q_f32(data)); +} - int64x2_t vc = vdupq_n_s64((int64_t) c); - return vreinterpretq_m128i_s64(vshlq_s64(vreinterpretq_s64_m128i(a), vc)); +// Broadcast single-precision (32-bit) floating-point value a to all elements of +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_ps1 +FORCE_INLINE __m128 _mm_set_ps1(float _w) +{ + return vreinterpretq_m128_f32(vdupq_n_f32(_w)); } -// Shifts the 8 signed or unsigned 16-bit integers in a right by count bits -// while shifting in zeros. -// -// r0 := srl(a0, count) -// r1 := srl(a1, count) -// ... -// r7 := srl(a7, count) -// -// https://msdn.microsoft.com/en-us/library/wd5ax830(v%3dvs.90).aspx -FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) +// Macro: Set the rounding mode bits of the MXCSR control and status register to +// the value in unsigned 32-bit integer a. The rounding mode may contain any of +// the following flags: _MM_ROUND_NEAREST, _MM_ROUND_DOWN, _MM_ROUND_UP, +// _MM_ROUND_TOWARD_ZERO +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_MM_SET_ROUNDING_MODE +FORCE_INLINE void _MM_SET_ROUNDING_MODE(int rounding) { - uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); - if (c > 15) - return _mm_setzero_si128(); + union { + fpcr_bitfield field; +#if defined(__aarch64__) || defined(_M_ARM64) + uint64_t value; +#else + uint32_t value; +#endif + } r; - int16x8_t vc = vdupq_n_s16(-(int16_t) c); - return vreinterpretq_m128i_u16(vshlq_u16(vreinterpretq_u16_m128i(a), vc)); +#if defined(__aarch64__) || defined(_M_ARM64) + r.value = _sse2neon_get_fpcr(); +#else + __asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ +#endif + + switch (rounding) { + case _MM_ROUND_TOWARD_ZERO: + r.field.bit22 = 1; + r.field.bit23 = 1; + break; + case _MM_ROUND_DOWN: + r.field.bit22 = 0; + r.field.bit23 = 1; + break; + case _MM_ROUND_UP: + r.field.bit22 = 1; + r.field.bit23 = 0; + break; + default: //_MM_ROUND_NEAREST + r.field.bit22 = 0; + r.field.bit23 = 0; + } + +#if defined(__aarch64__) || defined(_M_ARM64) + _sse2neon_set_fpcr(r.value); +#else + __asm__ __volatile__("vmsr FPSCR, %0" ::"r"(r)); /* write */ +#endif } -// Shifts the 4 signed or unsigned 32-bit integers in a right by count bits -// while shifting in zeros. -// -// r0 := srl(a0, count) -// r1 := srl(a1, count) -// r2 := srl(a2, count) -// r3 := srl(a3, count) -// -// https://msdn.microsoft.com/en-us/library/a9cbttf4(v%3dvs.90).aspx -FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) +// Copy single-precision (32-bit) floating-point element a to the lower element +// of dst, and zero the upper 3 elements. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_ss +FORCE_INLINE __m128 _mm_set_ss(float a) { - uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); - if (c > 31) - return _mm_setzero_si128(); + return vreinterpretq_m128_f32(vsetq_lane_f32(a, vdupq_n_f32(0), 0)); +} - int32x4_t vc = vdupq_n_s32(-(int32_t) c); - return vreinterpretq_m128i_u32(vshlq_u32(vreinterpretq_u32_m128i(a), vc)); +// Broadcast single-precision (32-bit) floating-point value a to all elements of +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set1_ps +FORCE_INLINE __m128 _mm_set1_ps(float _w) +{ + return vreinterpretq_m128_f32(vdupq_n_f32(_w)); } -// Shifts the 2 signed or unsigned 64-bit integers in a right by count bits -// while shifting in zeros. -// -// r0 := srl(a0, count) -// r1 := srl(a1, count) -// -// https://msdn.microsoft.com/en-us/library/yf6cf9k8(v%3dvs.90).aspx -FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) +// Set the MXCSR control and status register with the value in unsigned 32-bit +// integer a. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setcsr +// FIXME: _mm_setcsr() implementation supports changing the rounding mode only. +FORCE_INLINE void _mm_setcsr(unsigned int a) { - uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); - if (c > 63) - return _mm_setzero_si128(); + _MM_SET_ROUNDING_MODE(a); +} - int64x2_t vc = vdupq_n_s64(-(int64_t) c); - return vreinterpretq_m128i_u64(vshlq_u64(vreinterpretq_u64_m128i(a), vc)); +// Get the unsigned 32-bit value of the MXCSR control and status register. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_getcsr +// FIXME: _mm_getcsr() implementation supports reading the rounding mode only. +FORCE_INLINE unsigned int _mm_getcsr(void) +{ + return _MM_GET_ROUNDING_MODE(); } -// NEON does not provide a version of this function. -// Creates a 16-bit mask from the most significant bits of the 16 signed or -// unsigned 8-bit integers in a and zero extends the upper bits. -// https://msdn.microsoft.com/en-us/library/vstudio/s090c8fk(v=vs.100).aspx -FORCE_INLINE int _mm_movemask_epi8(__m128i a) +// Set packed single-precision (32-bit) floating-point elements in dst with the +// supplied values in reverse order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setr_ps +FORCE_INLINE __m128 _mm_setr_ps(float w, float z, float y, float x) { -#if defined(__aarch64__) - uint8x16_t input = vreinterpretq_u8_m128i(a); - const int8_t ALIGN_STRUCT(16) - xr[16] = {-7, -6, -5, -4, -3, -2, -1, 0, -7, -6, -5, -4, -3, -2, -1, 0}; - const uint8x16_t mask_and = vdupq_n_u8(0x80); - const int8x16_t mask_shift = vld1q_s8(xr); - const uint8x16_t mask_result = - vshlq_u8(vandq_u8(input, mask_and), mask_shift); - uint8x8_t lo = vget_low_u8(mask_result); - uint8x8_t hi = vget_high_u8(mask_result); + float ALIGN_STRUCT(16) data[4] = {w, z, y, x}; + return vreinterpretq_m128_f32(vld1q_f32(data)); +} + +// Return vector of type __m128 with all elements set to zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setzero_ps +FORCE_INLINE __m128 _mm_setzero_ps(void) +{ + return vreinterpretq_m128_f32(vdupq_n_f32(0)); +} - return vaddv_u8(lo) + (vaddv_u8(hi) << 8); +// Shuffle 16-bit integers in a using the control in imm8, and store the results +// in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shuffle_pi16 +#ifdef _sse2neon_shuffle +#define _mm_shuffle_pi16(a, imm) \ + vreinterpret_m64_s16(vshuffle_s16( \ + vreinterpret_s16_m64(a), vreinterpret_s16_m64(a), (imm & 0x3), \ + ((imm >> 2) & 0x3), ((imm >> 4) & 0x3), ((imm >> 6) & 0x3))) #else - // Use increasingly wide shifts+adds to collect the sign bits - // together. - // Since the widening shifts would be rather confusing to follow in little - // endian, everything will be illustrated in big endian order instead. This - // has a different result - the bits would actually be reversed on a big - // endian machine. +#define _mm_shuffle_pi16(a, imm) \ + _sse2neon_define1( \ + __m64, a, int16x4_t ret; \ + ret = vmov_n_s16( \ + vget_lane_s16(vreinterpret_s16_m64(_a), (imm) & (0x3))); \ + ret = vset_lane_s16( \ + vget_lane_s16(vreinterpret_s16_m64(_a), ((imm) >> 2) & 0x3), ret, \ + 1); \ + ret = vset_lane_s16( \ + vget_lane_s16(vreinterpret_s16_m64(_a), ((imm) >> 4) & 0x3), ret, \ + 2); \ + ret = vset_lane_s16( \ + vget_lane_s16(vreinterpret_s16_m64(_a), ((imm) >> 6) & 0x3), ret, \ + 3); \ + _sse2neon_return(vreinterpret_m64_s16(ret));) +#endif - // Starting input (only half the elements are shown): - // 89 ff 1d c0 00 10 99 33 - uint8x16_t input = vreinterpretq_u8_m128i(a); - - // Shift out everything but the sign bits with an unsigned shift right. - // - // Bytes of the vector:: - // 89 ff 1d c0 00 10 99 33 - // \ \ \ \ \ \ \ \ high_bits = (uint16x4_t)(input >> 7) - // | | | | | | | | - // 01 01 00 01 00 00 01 00 - // - // Bits of first important lane(s): - // 10001001 (89) - // \______ - // | - // 00000001 (01) - uint16x8_t high_bits = vreinterpretq_u16_u8(vshrq_n_u8(input, 7)); - - // Merge the even lanes together with a 16-bit unsigned shift right + add. - // 'xx' represents garbage data which will be ignored in the final result. - // In the important bytes, the add functions like a binary OR. - // - // 01 01 00 01 00 00 01 00 - // \_ | \_ | \_ | \_ | paired16 = (uint32x4_t)(input + (input >> 7)) - // \| \| \| \| - // xx 03 xx 01 xx 00 xx 02 - // - // 00000001 00000001 (01 01) - // \_______ | - // \| - // xxxxxxxx xxxxxx11 (xx 03) - uint32x4_t paired16 = - vreinterpretq_u32_u16(vsraq_n_u16(high_bits, high_bits, 7)); - - // Repeat with a wider 32-bit shift + add. - // xx 03 xx 01 xx 00 xx 02 - // \____ | \____ | paired32 = (uint64x1_t)(paired16 + (paired16 >> - // 14)) - // \| \| - // xx xx xx 0d xx xx xx 02 - // - // 00000011 00000001 (03 01) - // \\_____ || - // '----.\|| - // xxxxxxxx xxxx1101 (xx 0d) - uint64x2_t paired32 = - vreinterpretq_u64_u32(vsraq_n_u32(paired16, paired16, 14)); - - // Last, an even wider 64-bit shift + add to get our result in the low 8 bit - // lanes. xx xx xx 0d xx xx xx 02 - // \_________ | paired64 = (uint8x8_t)(paired32 + (paired32 >> - // 28)) - // \| - // xx xx xx xx xx xx xx d2 - // - // 00001101 00000010 (0d 02) - // \ \___ | | - // '---. \| | - // xxxxxxxx 11010010 (xx d2) - uint8x16_t paired64 = - vreinterpretq_u8_u64(vsraq_n_u64(paired32, paired32, 28)); - - // Extract the low 8 bits from each 64-bit lane with 2 8-bit extracts. - // xx xx xx xx xx xx xx d2 - // || return paired64[0] - // d2 - // Note: Little endian would return the correct value 4b (01001011) instead. - return vgetq_lane_u8(paired64, 0) | ((int) vgetq_lane_u8(paired64, 8) << 8); -#endif +// Perform a serializing operation on all store-to-memory instructions that were +// issued prior to this instruction. Guarantees that every store instruction +// that precedes, in program order, is globally visible before any store +// instruction which follows the fence in program order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sfence +FORCE_INLINE void _mm_sfence(void) +{ + _sse2neon_smp_mb(); } -// Copy the lower 64-bit integer in a to dst. -// -// dst[63:0] := a[63:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movepi64_pi64 -FORCE_INLINE __m64 _mm_movepi64_pi64(__m128i a) +// Perform a serializing operation on all load-from-memory and store-to-memory +// instructions that were issued prior to this instruction. Guarantees that +// every memory access that precedes, in program order, the memory fence +// instruction is globally visible before any memory instruction which follows +// the fence in program order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mfence +FORCE_INLINE void _mm_mfence(void) { - return vreinterpret_m64_s64(vget_low_s64(vreinterpretq_s64_m128i(a))); + _sse2neon_smp_mb(); } -// Copy the 64-bit integer a to the lower element of dst, and zero the upper -// element. -// -// dst[63:0] := a[63:0] -// dst[127:64] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_movpi64_epi64 -FORCE_INLINE __m128i _mm_movpi64_epi64(__m64 a) +// Perform a serializing operation on all load-from-memory instructions that +// were issued prior to this instruction. Guarantees that every load instruction +// that precedes, in program order, is globally visible before any load +// instruction which follows the fence in program order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_lfence +FORCE_INLINE void _mm_lfence(void) { - return vreinterpretq_m128i_s64( - vcombine_s64(vreinterpret_s64_m64(a), vdup_n_s64(0))); + _sse2neon_smp_mb(); } -// NEON does not provide this method -// Creates a 4-bit mask from the most significant bits of the four -// single-precision, floating-point values. -// https://msdn.microsoft.com/en-us/library/vstudio/4490ys29(v=vs.100).aspx -FORCE_INLINE int _mm_movemask_ps(__m128 a) +// FORCE_INLINE __m128 _mm_shuffle_ps(__m128 a, __m128 b, __constrange(0,255) +// int imm) +#ifdef _sse2neon_shuffle +#define _mm_shuffle_ps(a, b, imm) \ + __extension__({ \ + float32x4_t _input1 = vreinterpretq_f32_m128(a); \ + float32x4_t _input2 = vreinterpretq_f32_m128(b); \ + float32x4_t _shuf = \ + vshuffleq_s32(_input1, _input2, (imm) & (0x3), ((imm) >> 2) & 0x3, \ + (((imm) >> 4) & 0x3) + 4, (((imm) >> 6) & 0x3) + 4); \ + vreinterpretq_m128_f32(_shuf); \ + }) +#else // generic +#define _mm_shuffle_ps(a, b, imm) \ + _sse2neon_define2( \ + __m128, a, b, __m128 ret; switch (imm) { \ + case _MM_SHUFFLE(1, 0, 3, 2): \ + ret = _mm_shuffle_ps_1032(_a, _b); \ + break; \ + case _MM_SHUFFLE(2, 3, 0, 1): \ + ret = _mm_shuffle_ps_2301(_a, _b); \ + break; \ + case _MM_SHUFFLE(0, 3, 2, 1): \ + ret = _mm_shuffle_ps_0321(_a, _b); \ + break; \ + case _MM_SHUFFLE(2, 1, 0, 3): \ + ret = _mm_shuffle_ps_2103(_a, _b); \ + break; \ + case _MM_SHUFFLE(1, 0, 1, 0): \ + ret = _mm_movelh_ps(_a, _b); \ + break; \ + case _MM_SHUFFLE(1, 0, 0, 1): \ + ret = _mm_shuffle_ps_1001(_a, _b); \ + break; \ + case _MM_SHUFFLE(0, 1, 0, 1): \ + ret = _mm_shuffle_ps_0101(_a, _b); \ + break; \ + case _MM_SHUFFLE(3, 2, 1, 0): \ + ret = _mm_shuffle_ps_3210(_a, _b); \ + break; \ + case _MM_SHUFFLE(0, 0, 1, 1): \ + ret = _mm_shuffle_ps_0011(_a, _b); \ + break; \ + case _MM_SHUFFLE(0, 0, 2, 2): \ + ret = _mm_shuffle_ps_0022(_a, _b); \ + break; \ + case _MM_SHUFFLE(2, 2, 0, 0): \ + ret = _mm_shuffle_ps_2200(_a, _b); \ + break; \ + case _MM_SHUFFLE(3, 2, 0, 2): \ + ret = _mm_shuffle_ps_3202(_a, _b); \ + break; \ + case _MM_SHUFFLE(3, 2, 3, 2): \ + ret = _mm_movehl_ps(_b, _a); \ + break; \ + case _MM_SHUFFLE(1, 1, 3, 3): \ + ret = _mm_shuffle_ps_1133(_a, _b); \ + break; \ + case _MM_SHUFFLE(2, 0, 1, 0): \ + ret = _mm_shuffle_ps_2010(_a, _b); \ + break; \ + case _MM_SHUFFLE(2, 0, 0, 1): \ + ret = _mm_shuffle_ps_2001(_a, _b); \ + break; \ + case _MM_SHUFFLE(2, 0, 3, 2): \ + ret = _mm_shuffle_ps_2032(_a, _b); \ + break; \ + default: \ + ret = _mm_shuffle_ps_default(_a, _b, (imm)); \ + break; \ + } _sse2neon_return(ret);) +#endif + +// Compute the square root of packed single-precision (32-bit) floating-point +// elements in a, and store the results in dst. +// Due to ARMv7-A NEON's lack of a precise square root intrinsic, we implement +// square root by multiplying input in with its reciprocal square root before +// using the Newton-Raphson method to approximate the results. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_ps +FORCE_INLINE __m128 _mm_sqrt_ps(__m128 in) { - uint32x4_t input = vreinterpretq_u32_m128(a); -#if defined(__aarch64__) - static const int32x4_t shift = {0, 1, 2, 3}; - uint32x4_t tmp = vshrq_n_u32(input, 31); - return vaddvq_u32(vshlq_u32(tmp, shift)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32(vsqrtq_f32(vreinterpretq_f32_m128(in))); #else - // Uses the exact same method as _mm_movemask_epi8, see that for details. - // Shift out everything but the sign bits with a 32-bit unsigned shift - // right. - uint64x2_t high_bits = vreinterpretq_u64_u32(vshrq_n_u32(input, 31)); - // Merge the two pairs together with a 64-bit unsigned shift right + add. - uint8x16_t paired = - vreinterpretq_u8_u64(vsraq_n_u64(high_bits, high_bits, 31)); - // Extract the result. - return vgetq_lane_u8(paired, 0) | (vgetq_lane_u8(paired, 8) << 2); + float32x4_t recip = vrsqrteq_f32(vreinterpretq_f32_m128(in)); + + // Test for vrsqrteq_f32(0) -> positive infinity case. + // Change to zero, so that s * 1/sqrt(s) result is zero too. + const uint32x4_t pos_inf = vdupq_n_u32(0x7F800000); + const uint32x4_t div_by_zero = + vceqq_u32(pos_inf, vreinterpretq_u32_f32(recip)); + recip = vreinterpretq_f32_u32( + vandq_u32(vmvnq_u32(div_by_zero), vreinterpretq_u32_f32(recip))); + + recip = vmulq_f32( + vrsqrtsq_f32(vmulq_f32(recip, recip), vreinterpretq_f32_m128(in)), + recip); + // Additional Netwon-Raphson iteration for accuracy + recip = vmulq_f32( + vrsqrtsq_f32(vmulq_f32(recip, recip), vreinterpretq_f32_m128(in)), + recip); + + // sqrt(s) = s * 1/sqrt(s) + return vreinterpretq_m128_f32(vmulq_f32(vreinterpretq_f32_m128(in), recip)); #endif } -// Compute the bitwise NOT of a and then AND with a 128-bit vector containing -// all 1's, and return 1 if the result is zero, otherwise return 0. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_all_ones -FORCE_INLINE int _mm_test_all_ones(__m128i a) +// Compute the square root of the lower single-precision (32-bit) floating-point +// element in a, store the result in the lower element of dst, and copy the +// upper 3 packed elements from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_ss +FORCE_INLINE __m128 _mm_sqrt_ss(__m128 in) { - return (uint64_t)(vgetq_lane_s64(a, 0) & vgetq_lane_s64(a, 1)) == - ~(uint64_t) 0; + float32_t value = + vgetq_lane_f32(vreinterpretq_f32_m128(_mm_sqrt_ps(in)), 0); + return vreinterpretq_m128_f32( + vsetq_lane_f32(value, vreinterpretq_f32_m128(in), 0)); } -// Compute the bitwise AND of 128 bits (representing integer data) in a and -// mask, and return 1 if the result is zero, otherwise return 0. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_test_all_zeros -FORCE_INLINE int _mm_test_all_zeros(__m128i a, __m128i mask) +// Store 128-bits (composed of 4 packed single-precision (32-bit) floating-point +// elements) from a into memory. mem_addr must be aligned on a 16-byte boundary +// or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_ps +FORCE_INLINE void _mm_store_ps(float *p, __m128 a) { - int64x2_t a_and_mask = - vandq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(mask)); - return (vgetq_lane_s64(a_and_mask, 0) | vgetq_lane_s64(a_and_mask, 1)) ? 0 - : 1; + vst1q_f32(p, vreinterpretq_f32_m128(a)); } -/* Math operations */ - -// Subtracts the four single-precision, floating-point values of a and b. -// -// r0 := a0 - b0 -// r1 := a1 - b1 -// r2 := a2 - b2 -// r3 := a3 - b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/1zad2k61(v=vs.100).aspx -FORCE_INLINE __m128 _mm_sub_ps(__m128 a, __m128 b) +// Store the lower single-precision (32-bit) floating-point element from a into +// 4 contiguous elements in memory. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_ps1 +FORCE_INLINE void _mm_store_ps1(float *p, __m128 a) { - return vreinterpretq_m128_f32( - vsubq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + float32_t a0 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + vst1q_f32(p, vdupq_n_f32(a0)); } -// Subtract the lower single-precision (32-bit) floating-point element in b from -// the lower single-precision (32-bit) floating-point element in a, store the -// result in the lower element of dst, and copy the upper 3 packed elements from -// a to the upper elements of dst. -// -// dst[31:0] := a[31:0] - b[31:0] -// dst[127:32] := a[127:32] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sub_ss -FORCE_INLINE __m128 _mm_sub_ss(__m128 a, __m128 b) +// Store the lower single-precision (32-bit) floating-point element from a into +// memory. mem_addr does not need to be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_ss +FORCE_INLINE void _mm_store_ss(float *p, __m128 a) { - return _mm_move_ss(a, _mm_sub_ps(a, b)); + vst1q_lane_f32(p, vreinterpretq_f32_m128(a), 0); } -// Subtract 2 packed 64-bit integers in b from 2 packed 64-bit integers in a, -// and store the results in dst. -// r0 := a0 - b0 -// r1 := a1 - b1 -FORCE_INLINE __m128i _mm_sub_epi64(__m128i a, __m128i b) +// Store the lower single-precision (32-bit) floating-point element from a into +// 4 contiguous elements in memory. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store1_ps +#define _mm_store1_ps _mm_store_ps1 + +// Store the upper 2 single-precision (32-bit) floating-point elements from a +// into memory. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeh_pi +FORCE_INLINE void _mm_storeh_pi(__m64 *p, __m128 a) { - return vreinterpretq_m128i_s64( - vsubq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b))); + *p = vreinterpret_m64_f32(vget_high_f32(a)); } -// Subtracts the 4 signed or unsigned 32-bit integers of b from the 4 signed or -// unsigned 32-bit integers of a. -// -// r0 := a0 - b0 -// r1 := a1 - b1 -// r2 := a2 - b2 -// r3 := a3 - b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/fhh866h0(v=vs.100).aspx -FORCE_INLINE __m128i _mm_sub_epi32(__m128i a, __m128i b) +// Store the lower 2 single-precision (32-bit) floating-point elements from a +// into memory. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storel_pi +FORCE_INLINE void _mm_storel_pi(__m64 *p, __m128 a) { - return vreinterpretq_m128i_s32( - vsubq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); + *p = vreinterpret_m64_f32(vget_low_f32(a)); } -FORCE_INLINE __m128i _mm_sub_epi16(__m128i a, __m128i b) +// Store 4 single-precision (32-bit) floating-point elements from a into memory +// in reverse order. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storer_ps +FORCE_INLINE void _mm_storer_ps(float *p, __m128 a) { - return vreinterpretq_m128i_s16( - vsubq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); + float32x4_t tmp = vrev64q_f32(vreinterpretq_f32_m128(a)); + float32x4_t rev = vextq_f32(tmp, tmp, 2); + vst1q_f32(p, rev); } -FORCE_INLINE __m128i _mm_sub_epi8(__m128i a, __m128i b) +// Store 128-bits (composed of 4 packed single-precision (32-bit) floating-point +// elements) from a into memory. mem_addr does not need to be aligned on any +// particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeu_ps +FORCE_INLINE void _mm_storeu_ps(float *p, __m128 a) { - return vreinterpretq_m128i_s8( - vsubq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); + vst1q_f32(p, vreinterpretq_f32_m128(a)); } -// Subtract 64-bit integer b from 64-bit integer a, and store the result in dst. -// -// dst[63:0] := a[63:0] - b[63:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sub_si64 -FORCE_INLINE __m64 _mm_sub_si64(__m64 a, __m64 b) +// Stores 16-bits of integer data a at the address p. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeu_si16 +FORCE_INLINE void _mm_storeu_si16(void *p, __m128i a) { - return vreinterpret_m64_s64( - vsub_s64(vreinterpret_s64_m64(a), vreinterpret_s64_m64(b))); + vst1q_lane_s16((int16_t *) p, vreinterpretq_s16_m128i(a), 0); } -// Subtracts the 8 unsigned 16-bit integers of bfrom the 8 unsigned 16-bit -// integers of a and saturates.. -// https://technet.microsoft.com/en-us/subscriptions/index/f44y0s19(v=vs.90).aspx -FORCE_INLINE __m128i _mm_subs_epu16(__m128i a, __m128i b) +// Stores 64-bits of integer data a at the address p. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeu_si64 +FORCE_INLINE void _mm_storeu_si64(void *p, __m128i a) { - return vreinterpretq_m128i_u16( - vqsubq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); + vst1q_lane_s64((int64_t *) p, vreinterpretq_s64_m128i(a), 0); } -// Subtracts the 16 unsigned 8-bit integers of b from the 16 unsigned 8-bit -// integers of a and saturates. -// -// r0 := UnsignedSaturate(a0 - b0) -// r1 := UnsignedSaturate(a1 - b1) -// ... -// r15 := UnsignedSaturate(a15 - b15) -// -// https://technet.microsoft.com/en-us/subscriptions/yadkxc18(v=vs.90) -FORCE_INLINE __m128i _mm_subs_epu8(__m128i a, __m128i b) +// Store 64-bits of integer data from a into memory using a non-temporal memory +// hint. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_stream_pi +FORCE_INLINE void _mm_stream_pi(__m64 *p, __m64 a) { - return vreinterpretq_m128i_u8( - vqsubq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); + vst1_s64((int64_t *) p, vreinterpret_s64_m64(a)); } -// Subtracts the 16 signed 8-bit integers of b from the 16 signed 8-bit integers -// of a and saturates. -// -// r0 := SignedSaturate(a0 - b0) -// r1 := SignedSaturate(a1 - b1) -// ... -// r15 := SignedSaturate(a15 - b15) -// -// https://technet.microsoft.com/en-us/subscriptions/by7kzks1(v=vs.90) -FORCE_INLINE __m128i _mm_subs_epi8(__m128i a, __m128i b) +// Store 128-bits (composed of 4 packed single-precision (32-bit) floating- +// point elements) from a into memory using a non-temporal memory hint. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_stream_ps +FORCE_INLINE void _mm_stream_ps(float *p, __m128 a) { - return vreinterpretq_m128i_s8( - vqsubq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +#if __has_builtin(__builtin_nontemporal_store) + __builtin_nontemporal_store(a, (float32x4_t *) p); +#else + vst1q_f32(p, vreinterpretq_f32_m128(a)); +#endif } -// Subtracts the 8 signed 16-bit integers of b from the 8 signed 16-bit integers -// of a and saturates. -// -// r0 := SignedSaturate(a0 - b0) -// r1 := SignedSaturate(a1 - b1) -// ... -// r7 := SignedSaturate(a7 - b7) -// -// https://technet.microsoft.com/en-us/subscriptions/3247z5b8(v=vs.90) -FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) +// Subtract packed single-precision (32-bit) floating-point elements in b from +// packed single-precision (32-bit) floating-point elements in a, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_ps +FORCE_INLINE __m128 _mm_sub_ps(__m128 a, __m128 b) { - return vreinterpretq_m128i_s16( - vqsubq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); + return vreinterpretq_m128_f32( + vsubq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); } -FORCE_INLINE __m128i _mm_adds_epu16(__m128i a, __m128i b) +// Subtract the lower single-precision (32-bit) floating-point element in b from +// the lower single-precision (32-bit) floating-point element in a, store the +// result in the lower element of dst, and copy the upper 3 packed elements from +// a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_ss +FORCE_INLINE __m128 _mm_sub_ss(__m128 a, __m128 b) { - return vreinterpretq_m128i_u16( - vqaddq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); + return _mm_move_ss(a, _mm_sub_ps(a, b)); } -// Negate packed 8-bit integers in a when the corresponding signed -// 8-bit integer in b is negative, and store the results in dst. -// Element in dst are zeroed out when the corresponding element -// in b is zero. -// -// for i in 0..15 -// if b[i] < 0 -// r[i] := -a[i] -// else if b[i] == 0 -// r[i] := 0 -// else -// r[i] := a[i] -// fi -// done -FORCE_INLINE __m128i _mm_sign_epi8(__m128i _a, __m128i _b) -{ - int8x16_t a = vreinterpretq_s8_m128i(_a); - int8x16_t b = vreinterpretq_s8_m128i(_b); +// Macro: Transpose the 4x4 matrix formed by the 4 rows of single-precision +// (32-bit) floating-point elements in row0, row1, row2, and row3, and store the +// transposed matrix in these vectors (row0 now contains column 0, etc.). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=MM_TRANSPOSE4_PS +#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ + do { \ + float32x4x2_t ROW01 = vtrnq_f32(row0, row1); \ + float32x4x2_t ROW23 = vtrnq_f32(row2, row3); \ + row0 = vcombine_f32(vget_low_f32(ROW01.val[0]), \ + vget_low_f32(ROW23.val[0])); \ + row1 = vcombine_f32(vget_low_f32(ROW01.val[1]), \ + vget_low_f32(ROW23.val[1])); \ + row2 = vcombine_f32(vget_high_f32(ROW01.val[0]), \ + vget_high_f32(ROW23.val[0])); \ + row3 = vcombine_f32(vget_high_f32(ROW01.val[1]), \ + vget_high_f32(ROW23.val[1])); \ + } while (0) - // signed shift right: faster than vclt - // (b < 0) ? 0xFF : 0 - uint8x16_t ltMask = vreinterpretq_u8_s8(vshrq_n_s8(b, 7)); +// according to the documentation, these intrinsics behave the same as the +// non-'u' versions. We'll just alias them here. +#define _mm_ucomieq_ss _mm_comieq_ss +#define _mm_ucomige_ss _mm_comige_ss +#define _mm_ucomigt_ss _mm_comigt_ss +#define _mm_ucomile_ss _mm_comile_ss +#define _mm_ucomilt_ss _mm_comilt_ss +#define _mm_ucomineq_ss _mm_comineq_ss - // (b == 0) ? 0xFF : 0 -#if defined(__aarch64__) - int8x16_t zeroMask = vreinterpretq_s8_u8(vceqzq_s8(b)); -#else - int8x16_t zeroMask = vreinterpretq_s8_u8(vceqq_s8(b, vdupq_n_s8(0))); -#endif - - // bitwise select either a or nagative 'a' (vnegq_s8(a) return nagative 'a') - // based on ltMask - int8x16_t masked = vbslq_s8(ltMask, vnegq_s8(a), a); - // res = masked & (~zeroMask) - int8x16_t res = vbicq_s8(masked, zeroMask); - - return vreinterpretq_m128i_s8(res); -} - -// Negate packed 16-bit integers in a when the corresponding signed -// 16-bit integer in b is negative, and store the results in dst. -// Element in dst are zeroed out when the corresponding element -// in b is zero. -// -// for i in 0..7 -// if b[i] < 0 -// r[i] := -a[i] -// else if b[i] == 0 -// r[i] := 0 -// else -// r[i] := a[i] -// fi -// done -FORCE_INLINE __m128i _mm_sign_epi16(__m128i _a, __m128i _b) +// Return vector of type __m128i with undefined elements. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=mm_undefined_si128 +FORCE_INLINE __m128i _mm_undefined_si128(void) { - int16x8_t a = vreinterpretq_s16_m128i(_a); - int16x8_t b = vreinterpretq_s16_m128i(_b); - - // signed shift right: faster than vclt - // (b < 0) ? 0xFFFF : 0 - uint16x8_t ltMask = vreinterpretq_u16_s16(vshrq_n_s16(b, 15)); - // (b == 0) ? 0xFFFF : 0 -#if defined(__aarch64__) - int16x8_t zeroMask = vreinterpretq_s16_u16(vceqzq_s16(b)); -#else - int16x8_t zeroMask = vreinterpretq_s16_u16(vceqq_s16(b, vdupq_n_s16(0))); +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" #endif - - // bitwise select either a or negative 'a' (vnegq_s16(a) equals to negative - // 'a') based on ltMask - int16x8_t masked = vbslq_s16(ltMask, vnegq_s16(a), a); - // res = masked & (~zeroMask) - int16x8_t res = vbicq_s16(masked, zeroMask); - return vreinterpretq_m128i_s16(res); -} - -// Negate packed 32-bit integers in a when the corresponding signed -// 32-bit integer in b is negative, and store the results in dst. -// Element in dst are zeroed out when the corresponding element -// in b is zero. -// -// for i in 0..3 -// if b[i] < 0 -// r[i] := -a[i] -// else if b[i] == 0 -// r[i] := 0 -// else -// r[i] := a[i] -// fi -// done -FORCE_INLINE __m128i _mm_sign_epi32(__m128i _a, __m128i _b) -{ - int32x4_t a = vreinterpretq_s32_m128i(_a); - int32x4_t b = vreinterpretq_s32_m128i(_b); - - // signed shift right: faster than vclt - // (b < 0) ? 0xFFFFFFFF : 0 - uint32x4_t ltMask = vreinterpretq_u32_s32(vshrq_n_s32(b, 31)); - - // (b == 0) ? 0xFFFFFFFF : 0 -#if defined(__aarch64__) - int32x4_t zeroMask = vreinterpretq_s32_u32(vceqzq_s32(b)); -#else - int32x4_t zeroMask = vreinterpretq_s32_u32(vceqq_s32(b, vdupq_n_s32(0))); + __m128i a; +#if defined(_MSC_VER) + a = _mm_setzero_si128(); +#endif + return a; +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop #endif - - // bitwise select either a or negative 'a' (vnegq_s32(a) equals to negative - // 'a') based on ltMask - int32x4_t masked = vbslq_s32(ltMask, vnegq_s32(a), a); - // res = masked & (~zeroMask) - int32x4_t res = vbicq_s32(masked, zeroMask); - return vreinterpretq_m128i_s32(res); } -// Negate packed 16-bit integers in a when the corresponding signed 16-bit -// integer in b is negative, and store the results in dst. Element in dst are -// zeroed out when the corresponding element in b is zero. -// -// FOR j := 0 to 3 -// i := j*16 -// IF b[i+15:i] < 0 -// dst[i+15:i] := -(a[i+15:i]) -// ELSE IF b[i+15:i] == 0 -// dst[i+15:i] := 0 -// ELSE -// dst[i+15:i] := a[i+15:i] -// FI -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sign_pi16 -FORCE_INLINE __m64 _mm_sign_pi16(__m64 _a, __m64 _b) +// Return vector of type __m128 with undefined elements. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_undefined_ps +FORCE_INLINE __m128 _mm_undefined_ps(void) { - int16x4_t a = vreinterpret_s16_m64(_a); - int16x4_t b = vreinterpret_s16_m64(_b); - - // signed shift right: faster than vclt - // (b < 0) ? 0xFFFF : 0 - uint16x4_t ltMask = vreinterpret_u16_s16(vshr_n_s16(b, 15)); - - // (b == 0) ? 0xFFFF : 0 -#if defined(__aarch64__) - int16x4_t zeroMask = vreinterpret_s16_u16(vceqz_s16(b)); -#else - int16x4_t zeroMask = vreinterpret_s16_u16(vceq_s16(b, vdup_n_s16(0))); +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif + __m128 a; +#if defined(_MSC_VER) + a = _mm_setzero_ps(); +#endif + return a; +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop #endif - - // bitwise select either a or nagative 'a' (vneg_s16(a) return nagative 'a') - // based on ltMask - int16x4_t masked = vbsl_s16(ltMask, vneg_s16(a), a); - // res = masked & (~zeroMask) - int16x4_t res = vbic_s16(masked, zeroMask); - - return vreinterpret_m64_s16(res); } -// Negate packed 32-bit integers in a when the corresponding signed 32-bit -// integer in b is negative, and store the results in dst. Element in dst are -// zeroed out when the corresponding element in b is zero. -// -// FOR j := 0 to 1 -// i := j*32 -// IF b[i+31:i] < 0 -// dst[i+31:i] := -(a[i+31:i]) -// ELSE IF b[i+31:i] == 0 -// dst[i+31:i] := 0 -// ELSE -// dst[i+31:i] := a[i+31:i] -// FI -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sign_pi32 -FORCE_INLINE __m64 _mm_sign_pi32(__m64 _a, __m64 _b) +// Unpack and interleave single-precision (32-bit) floating-point elements from +// the high half a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpackhi_ps +FORCE_INLINE __m128 _mm_unpackhi_ps(__m128 a, __m128 b) { - int32x2_t a = vreinterpret_s32_m64(_a); - int32x2_t b = vreinterpret_s32_m64(_b); - - // signed shift right: faster than vclt - // (b < 0) ? 0xFFFFFFFF : 0 - uint32x2_t ltMask = vreinterpret_u32_s32(vshr_n_s32(b, 31)); - - // (b == 0) ? 0xFFFFFFFF : 0 -#if defined(__aarch64__) - int32x2_t zeroMask = vreinterpret_s32_u32(vceqz_s32(b)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32( + vzip2q_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); #else - int32x2_t zeroMask = vreinterpret_s32_u32(vceq_s32(b, vdup_n_s32(0))); + float32x2_t a1 = vget_high_f32(vreinterpretq_f32_m128(a)); + float32x2_t b1 = vget_high_f32(vreinterpretq_f32_m128(b)); + float32x2x2_t result = vzip_f32(a1, b1); + return vreinterpretq_m128_f32(vcombine_f32(result.val[0], result.val[1])); #endif - - // bitwise select either a or nagative 'a' (vneg_s32(a) return nagative 'a') - // based on ltMask - int32x2_t masked = vbsl_s32(ltMask, vneg_s32(a), a); - // res = masked & (~zeroMask) - int32x2_t res = vbic_s32(masked, zeroMask); - - return vreinterpret_m64_s32(res); } -// Negate packed 8-bit integers in a when the corresponding signed 8-bit integer -// in b is negative, and store the results in dst. Element in dst are zeroed out -// when the corresponding element in b is zero. -// -// FOR j := 0 to 7 -// i := j*8 -// IF b[i+7:i] < 0 -// dst[i+7:i] := -(a[i+7:i]) -// ELSE IF b[i+7:i] == 0 -// dst[i+7:i] := 0 -// ELSE -// dst[i+7:i] := a[i+7:i] -// FI -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sign_pi8 -FORCE_INLINE __m64 _mm_sign_pi8(__m64 _a, __m64 _b) +// Unpack and interleave single-precision (32-bit) floating-point elements from +// the low half of a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpacklo_ps +FORCE_INLINE __m128 _mm_unpacklo_ps(__m128 a, __m128 b) { - int8x8_t a = vreinterpret_s8_m64(_a); - int8x8_t b = vreinterpret_s8_m64(_b); - - // signed shift right: faster than vclt - // (b < 0) ? 0xFF : 0 - uint8x8_t ltMask = vreinterpret_u8_s8(vshr_n_s8(b, 7)); - - // (b == 0) ? 0xFF : 0 -#if defined(__aarch64__) - int8x8_t zeroMask = vreinterpret_s8_u8(vceqz_s8(b)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32( + vzip1q_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); #else - int8x8_t zeroMask = vreinterpret_s8_u8(vceq_s8(b, vdup_n_s8(0))); + float32x2_t a1 = vget_low_f32(vreinterpretq_f32_m128(a)); + float32x2_t b1 = vget_low_f32(vreinterpretq_f32_m128(b)); + float32x2x2_t result = vzip_f32(a1, b1); + return vreinterpretq_m128_f32(vcombine_f32(result.val[0], result.val[1])); #endif - - // bitwise select either a or nagative 'a' (vneg_s8(a) return nagative 'a') - // based on ltMask - int8x8_t masked = vbsl_s8(ltMask, vneg_s8(a), a); - // res = masked & (~zeroMask) - int8x8_t res = vbic_s8(masked, zeroMask); - - return vreinterpret_m64_s8(res); } -// Average packed unsigned 16-bit integers in a and b, and store the results in -// dst. -// -// FOR j := 0 to 3 -// i := j*16 -// dst[i+15:i] := (a[i+15:i] + b[i+15:i] + 1) >> 1 -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_avg_pu16 -FORCE_INLINE __m64 _mm_avg_pu16(__m64 a, __m64 b) +// Compute the bitwise XOR of packed single-precision (32-bit) floating-point +// elements in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_xor_ps +FORCE_INLINE __m128 _mm_xor_ps(__m128 a, __m128 b) { - return vreinterpret_m64_u16( - vrhadd_u16(vreinterpret_u16_m64(a), vreinterpret_u16_m64(b))); + return vreinterpretq_m128_s32( + veorq_s32(vreinterpretq_s32_m128(a), vreinterpretq_s32_m128(b))); } -// Average packed unsigned 8-bit integers in a and b, and store the results in -// dst. -// -// FOR j := 0 to 7 -// i := j*8 -// dst[i+7:i] := (a[i+7:i] + b[i+7:i] + 1) >> 1 -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_avg_pu8 -FORCE_INLINE __m64 _mm_avg_pu8(__m64 a, __m64 b) +/* SSE2 */ + +// Add packed 16-bit integers in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_epi16 +FORCE_INLINE __m128i _mm_add_epi16(__m128i a, __m128i b) { - return vreinterpret_m64_u8( - vrhadd_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); + return vreinterpretq_m128i_s16( + vaddq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } -// Average packed unsigned 8-bit integers in a and b, and store the results in -// dst. -// -// FOR j := 0 to 7 -// i := j*8 -// dst[i+7:i] := (a[i+7:i] + b[i+7:i] + 1) >> 1 -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pavgb -#define _m_pavgb(a, b) _mm_avg_pu8(a, b) - -// Average packed unsigned 16-bit integers in a and b, and store the results in -// dst. -// -// FOR j := 0 to 3 -// i := j*16 -// dst[i+15:i] := (a[i+15:i] + b[i+15:i] + 1) >> 1 -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pavgw -#define _m_pavgw(a, b) _mm_avg_pu16(a, b) - -// Computes the average of the 16 unsigned 8-bit integers in a and the 16 -// unsigned 8-bit integers in b and rounds. -// -// r0 := (a0 + b0) / 2 -// r1 := (a1 + b1) / 2 -// ... -// r15 := (a15 + b15) / 2 -// -// https://msdn.microsoft.com/en-us/library/vstudio/8zwh554a(v%3dvs.90).aspx -FORCE_INLINE __m128i _mm_avg_epu8(__m128i a, __m128i b) +// Add packed 32-bit integers in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_epi32 +FORCE_INLINE __m128i _mm_add_epi32(__m128i a, __m128i b) { - return vreinterpretq_m128i_u8( - vrhaddq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); + return vreinterpretq_m128i_s32( + vaddq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } -// Computes the average of the 8 unsigned 16-bit integers in a and the 8 -// unsigned 16-bit integers in b and rounds. -// -// r0 := (a0 + b0) / 2 -// r1 := (a1 + b1) / 2 -// ... -// r7 := (a7 + b7) / 2 -// -// https://msdn.microsoft.com/en-us/library/vstudio/y13ca3c8(v=vs.90).aspx -FORCE_INLINE __m128i _mm_avg_epu16(__m128i a, __m128i b) +// Add packed 64-bit integers in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_epi64 +FORCE_INLINE __m128i _mm_add_epi64(__m128i a, __m128i b) { - return (__m128i) vrhaddq_u16(vreinterpretq_u16_m128i(a), - vreinterpretq_u16_m128i(b)); + return vreinterpretq_m128i_s64( + vaddq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b))); } -// Adds the four single-precision, floating-point values of a and b. -// -// r0 := a0 + b0 -// r1 := a1 + b1 -// r2 := a2 + b2 -// r3 := a3 + b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/c9848chc(v=vs.100).aspx -FORCE_INLINE __m128 _mm_add_ps(__m128 a, __m128 b) +// Add packed 8-bit integers in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_epi8 +FORCE_INLINE __m128i _mm_add_epi8(__m128i a, __m128i b) { - return vreinterpretq_m128_f32( - vaddq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + return vreinterpretq_m128i_s8( + vaddq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } // Add packed double-precision (64-bit) floating-point elements in a and b, and // store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_pd +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_pd FORCE_INLINE __m128d _mm_add_pd(__m128d a, __m128d b) { -#if defined(__aarch64__) +#if defined(__aarch64__) || defined(_M_ARM64) return vreinterpretq_m128d_f64( vaddq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); #else @@ -3010,2570 +2957,4298 @@ FORCE_INLINE __m128d _mm_add_pd(__m128d a, __m128d b) #endif } +// Add the lower double-precision (64-bit) floating-point element in a and b, +// store the result in the lower element of dst, and copy the upper element from +// a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_sd +FORCE_INLINE __m128d _mm_add_sd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_add_pd(a, b)); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] + db[0]; + c[1] = da[1]; + return vld1q_f32((float32_t *) c); +#endif +} + // Add 64-bit integers a and b, and store the result in dst. -// -// dst[63:0] := a[63:0] + b[63:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_add_si64 +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_si64 FORCE_INLINE __m64 _mm_add_si64(__m64 a, __m64 b) { return vreinterpret_m64_s64( vadd_s64(vreinterpret_s64_m64(a), vreinterpret_s64_m64(b))); } -// adds the scalar single-precision floating point values of a and b. -// https://msdn.microsoft.com/en-us/library/be94x2y6(v=vs.100).aspx -FORCE_INLINE __m128 _mm_add_ss(__m128 a, __m128 b) +// Add packed signed 16-bit integers in a and b using saturation, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_adds_epi16 +FORCE_INLINE __m128i _mm_adds_epi16(__m128i a, __m128i b) { - float32_t b0 = vgetq_lane_f32(vreinterpretq_f32_m128(b), 0); - float32x4_t value = vsetq_lane_f32(b0, vdupq_n_f32(0), 0); - // the upper values in the result must be the remnants of . - return vreinterpretq_m128_f32(vaddq_f32(a, value)); + return vreinterpretq_m128i_s16( + vqaddq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } -// Adds the 4 signed or unsigned 64-bit integers in a to the 4 signed or -// unsigned 32-bit integers in b. -// https://msdn.microsoft.com/en-us/library/vstudio/09xs4fkk(v=vs.100).aspx -FORCE_INLINE __m128i _mm_add_epi64(__m128i a, __m128i b) +// Add packed signed 8-bit integers in a and b using saturation, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_adds_epi8 +FORCE_INLINE __m128i _mm_adds_epi8(__m128i a, __m128i b) { - return vreinterpretq_m128i_s64( - vaddq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b))); + return vreinterpretq_m128i_s8( + vqaddq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } -// Adds the 4 signed or unsigned 32-bit integers in a to the 4 signed or -// unsigned 32-bit integers in b. -// -// r0 := a0 + b0 -// r1 := a1 + b1 -// r2 := a2 + b2 -// r3 := a3 + b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/09xs4fkk(v=vs.100).aspx -FORCE_INLINE __m128i _mm_add_epi32(__m128i a, __m128i b) +// Add packed unsigned 16-bit integers in a and b using saturation, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_adds_epu16 +FORCE_INLINE __m128i _mm_adds_epu16(__m128i a, __m128i b) { - return vreinterpretq_m128i_s32( - vaddq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); -} + return vreinterpretq_m128i_u16( + vqaddq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); +} -// Adds the 8 signed or unsigned 16-bit integers in a to the 8 signed or -// unsigned 16-bit integers in b. -// https://msdn.microsoft.com/en-us/library/fceha5k4(v=vs.100).aspx -FORCE_INLINE __m128i _mm_add_epi16(__m128i a, __m128i b) +// Add packed unsigned 8-bit integers in a and b using saturation, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_adds_epu8 +FORCE_INLINE __m128i _mm_adds_epu8(__m128i a, __m128i b) { - return vreinterpretq_m128i_s16( - vaddq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); + return vreinterpretq_m128i_u8( + vqaddq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); } -// Adds the 16 signed or unsigned 8-bit integers in a to the 16 signed or -// unsigned 8-bit integers in b. -// https://technet.microsoft.com/en-us/subscriptions/yc7tcyzs(v=vs.90) -FORCE_INLINE __m128i _mm_add_epi8(__m128i a, __m128i b) +// Compute the bitwise AND of packed double-precision (64-bit) floating-point +// elements in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_and_pd +FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) { - return vreinterpretq_m128i_s8( - vaddq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); + return vreinterpretq_m128d_s64( + vandq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); } -// Adds the 8 signed 16-bit integers in a to the 8 signed 16-bit integers in b -// and saturates. -// -// r0 := SignedSaturate(a0 + b0) -// r1 := SignedSaturate(a1 + b1) -// ... -// r7 := SignedSaturate(a7 + b7) -// -// https://msdn.microsoft.com/en-us/library/1a306ef8(v=vs.100).aspx -FORCE_INLINE __m128i _mm_adds_epi16(__m128i a, __m128i b) +// Compute the bitwise AND of 128 bits (representing integer data) in a and b, +// and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_and_si128 +FORCE_INLINE __m128i _mm_and_si128(__m128i a, __m128i b) { - return vreinterpretq_m128i_s16( - vqaddq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); + return vreinterpretq_m128i_s32( + vandq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } -// Add packed signed 8-bit integers in a and b using saturation, and store the -// results in dst. -// -// FOR j := 0 to 15 -// i := j*8 -// dst[i+7:i] := Saturate8( a[i+7:i] + b[i+7:i] ) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_adds_epi8 -FORCE_INLINE __m128i _mm_adds_epi8(__m128i a, __m128i b) +// Compute the bitwise NOT of packed double-precision (64-bit) floating-point +// elements in a and then AND with b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_andnot_pd +FORCE_INLINE __m128d _mm_andnot_pd(__m128d a, __m128d b) { - return vreinterpretq_m128i_s8( - vqaddq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); + // *NOTE* argument swap + return vreinterpretq_m128d_s64( + vbicq_s64(vreinterpretq_s64_m128d(b), vreinterpretq_s64_m128d(a))); } -// Adds the 16 unsigned 8-bit integers in a to the 16 unsigned 8-bit integers in -// b and saturates.. -// https://msdn.microsoft.com/en-us/library/9hahyddy(v=vs.100).aspx -FORCE_INLINE __m128i _mm_adds_epu8(__m128i a, __m128i b) +// Compute the bitwise NOT of 128 bits (representing integer data) in a and then +// AND with b, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_andnot_si128 +FORCE_INLINE __m128i _mm_andnot_si128(__m128i a, __m128i b) { - return vreinterpretq_m128i_u8( - vqaddq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); + return vreinterpretq_m128i_s32( + vbicq_s32(vreinterpretq_s32_m128i(b), + vreinterpretq_s32_m128i(a))); // *NOTE* argument swap } -// Multiplies the 8 signed or unsigned 16-bit integers from a by the 8 signed or -// unsigned 16-bit integers from b. -// -// r0 := (a0 * b0)[15:0] -// r1 := (a1 * b1)[15:0] -// ... -// r7 := (a7 * b7)[15:0] -// -// https://msdn.microsoft.com/en-us/library/vstudio/9ks1472s(v=vs.100).aspx -FORCE_INLINE __m128i _mm_mullo_epi16(__m128i a, __m128i b) +// Average packed unsigned 16-bit integers in a and b, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_avg_epu16 +FORCE_INLINE __m128i _mm_avg_epu16(__m128i a, __m128i b) { - return vreinterpretq_m128i_s16( - vmulq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); + return (__m128i) vrhaddq_u16(vreinterpretq_u16_m128i(a), + vreinterpretq_u16_m128i(b)); } -// Multiplies the 4 signed or unsigned 32-bit integers from a by the 4 signed or -// unsigned 32-bit integers from b. -// https://msdn.microsoft.com/en-us/library/vstudio/bb531409(v=vs.100).aspx -FORCE_INLINE __m128i _mm_mullo_epi32(__m128i a, __m128i b) +// Average packed unsigned 8-bit integers in a and b, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_avg_epu8 +FORCE_INLINE __m128i _mm_avg_epu8(__m128i a, __m128i b) { - return vreinterpretq_m128i_s32( - vmulq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); + return vreinterpretq_m128i_u8( + vrhaddq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); } -// Multiply the packed unsigned 16-bit integers in a and b, producing -// intermediate 32-bit integers, and store the high 16 bits of the intermediate -// integers in dst. -// -// FOR j := 0 to 3 -// i := j*16 -// tmp[31:0] := a[i+15:i] * b[i+15:i] -// dst[i+15:i] := tmp[31:16] -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_pmulhuw -#define _m_pmulhuw(a, b) _mm_mulhi_pu16(a, b) +// Shift a left by imm8 bytes while shifting in zeros, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_bslli_si128 +#define _mm_bslli_si128(a, imm) _mm_slli_si128(a, imm) -// Multiplies the four single-precision, floating-point values of a and b. -// -// r0 := a0 * b0 -// r1 := a1 * b1 -// r2 := a2 * b2 -// r3 := a3 * b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/22kbk6t9(v=vs.100).aspx -FORCE_INLINE __m128 _mm_mul_ps(__m128 a, __m128 b) +// Shift a right by imm8 bytes while shifting in zeros, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_bsrli_si128 +#define _mm_bsrli_si128(a, imm) _mm_srli_si128(a, imm) + +// Cast vector of type __m128d to type __m128. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_castpd_ps +FORCE_INLINE __m128 _mm_castpd_ps(__m128d a) { - return vreinterpretq_m128_f32( - vmulq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + return vreinterpretq_m128_s64(vreinterpretq_s64_m128d(a)); } -// Multiply packed double-precision (64-bit) floating-point elements in a and b, -// and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mul_pd -FORCE_INLINE __m128d _mm_mul_pd(__m128d a, __m128d b) +// Cast vector of type __m128d to type __m128i. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_castpd_si128 +FORCE_INLINE __m128i _mm_castpd_si128(__m128d a) { -#if defined(__aarch64__) - return vreinterpretq_m128d_f64( - vmulq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); -#else - double *da = (double *) &a; - double *db = (double *) &b; - double c[2]; - c[0] = da[0] * db[0]; - c[1] = da[1] * db[1]; - return vld1q_f32((float32_t *) c); -#endif + return vreinterpretq_m128i_s64(vreinterpretq_s64_m128d(a)); } -// Multiply the lower single-precision (32-bit) floating-point element in a and -// b, store the result in the lower element of dst, and copy the upper 3 packed -// elements from a to the upper elements of dst. -// -// dst[31:0] := a[31:0] * b[31:0] -// dst[127:32] := a[127:32] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mul_ss -FORCE_INLINE __m128 _mm_mul_ss(__m128 a, __m128 b) +// Cast vector of type __m128 to type __m128d. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_castps_pd +FORCE_INLINE __m128d _mm_castps_pd(__m128 a) { - return _mm_move_ss(a, _mm_mul_ps(a, b)); + return vreinterpretq_m128d_s32(vreinterpretq_s32_m128(a)); } -// Multiply the low unsigned 32-bit integers from each packed 64-bit element in -// a and b, and store the unsigned 64-bit results in dst. -// -// r0 := (a0 & 0xFFFFFFFF) * (b0 & 0xFFFFFFFF) -// r1 := (a2 & 0xFFFFFFFF) * (b2 & 0xFFFFFFFF) -FORCE_INLINE __m128i _mm_mul_epu32(__m128i a, __m128i b) +// Cast vector of type __m128 to type __m128i. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_castps_si128 +FORCE_INLINE __m128i _mm_castps_si128(__m128 a) { - // vmull_u32 upcasts instead of masking, so we downcast. - uint32x2_t a_lo = vmovn_u64(vreinterpretq_u64_m128i(a)); - uint32x2_t b_lo = vmovn_u64(vreinterpretq_u64_m128i(b)); - return vreinterpretq_m128i_u64(vmull_u32(a_lo, b_lo)); + return vreinterpretq_m128i_s32(vreinterpretq_s32_m128(a)); } -// Multiply the low unsigned 32-bit integers from a and b, and store the -// unsigned 64-bit result in dst. -// -// dst[63:0] := a[31:0] * b[31:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mul_su32 -FORCE_INLINE __m64 _mm_mul_su32(__m64 a, __m64 b) +// Cast vector of type __m128i to type __m128d. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_castsi128_pd +FORCE_INLINE __m128d _mm_castsi128_pd(__m128i a) { - return vreinterpret_m64_u64(vget_low_u64( - vmull_u32(vreinterpret_u32_m64(a), vreinterpret_u32_m64(b)))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vreinterpretq_f64_m128i(a)); +#else + return vreinterpretq_m128d_f32(vreinterpretq_f32_m128i(a)); +#endif } -// Multiply the low signed 32-bit integers from each packed 64-bit element in -// a and b, and store the signed 64-bit results in dst. -// -// r0 := (int64_t)(int32_t)a0 * (int64_t)(int32_t)b0 -// r1 := (int64_t)(int32_t)a2 * (int64_t)(int32_t)b2 -FORCE_INLINE __m128i _mm_mul_epi32(__m128i a, __m128i b) +// Cast vector of type __m128i to type __m128. This intrinsic is only used for +// compilation and does not generate any instructions, thus it has zero latency. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_castsi128_ps +FORCE_INLINE __m128 _mm_castsi128_ps(__m128i a) { - // vmull_s32 upcasts instead of masking, so we downcast. - int32x2_t a_lo = vmovn_s64(vreinterpretq_s64_m128i(a)); - int32x2_t b_lo = vmovn_s64(vreinterpretq_s64_m128i(b)); - return vreinterpretq_m128i_s64(vmull_s32(a_lo, b_lo)); + return vreinterpretq_m128_s32(vreinterpretq_s32_m128i(a)); } -// Multiplies the 8 signed 16-bit integers from a by the 8 signed 16-bit -// integers from b. -// -// r0 := (a0 * b0) + (a1 * b1) -// r1 := (a2 * b2) + (a3 * b3) -// r2 := (a4 * b4) + (a5 * b5) -// r3 := (a6 * b6) + (a7 * b7) -// https://msdn.microsoft.com/en-us/library/yht36sa6(v=vs.90).aspx -FORCE_INLINE __m128i _mm_madd_epi16(__m128i a, __m128i b) +// Invalidate and flush the cache line that contains p from all levels of the +// cache hierarchy. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_clflush +#if defined(__APPLE__) +#include +#endif +FORCE_INLINE void _mm_clflush(void const *p) { - int32x4_t low = vmull_s16(vget_low_s16(vreinterpretq_s16_m128i(a)), - vget_low_s16(vreinterpretq_s16_m128i(b))); - int32x4_t high = vmull_s16(vget_high_s16(vreinterpretq_s16_m128i(a)), - vget_high_s16(vreinterpretq_s16_m128i(b))); - - int32x2_t low_sum = vpadd_s32(vget_low_s32(low), vget_high_s32(low)); - int32x2_t high_sum = vpadd_s32(vget_low_s32(high), vget_high_s32(high)); + (void) p; - return vreinterpretq_m128i_s32(vcombine_s32(low_sum, high_sum)); + /* sys_icache_invalidate is supported since macOS 10.5. + * However, it does not work on non-jailbroken iOS devices, although the + * compilation is successful. + */ +#if defined(__APPLE__) + sys_icache_invalidate((void *) (uintptr_t) p, SSE2NEON_CACHELINE_SIZE); +#elif defined(__GNUC__) || defined(__clang__) + uintptr_t ptr = (uintptr_t) p; + __builtin___clear_cache((char *) ptr, + (char *) ptr + SSE2NEON_CACHELINE_SIZE); +#elif (_MSC_VER) && SSE2NEON_INCLUDE_WINDOWS_H + FlushInstructionCache(GetCurrentProcess(), p, SSE2NEON_CACHELINE_SIZE); +#endif } -// Multiply packed signed 16-bit integers in a and b, producing intermediate -// signed 32-bit integers. Shift right by 15 bits while rounding up, and store -// the packed 16-bit integers in dst. -// -// r0 := Round(((int32_t)a0 * (int32_t)b0) >> 15) -// r1 := Round(((int32_t)a1 * (int32_t)b1) >> 15) -// r2 := Round(((int32_t)a2 * (int32_t)b2) >> 15) -// ... -// r7 := Round(((int32_t)a7 * (int32_t)b7) >> 15) -FORCE_INLINE __m128i _mm_mulhrs_epi16(__m128i a, __m128i b) +// Compare packed 16-bit integers in a and b for equality, and store the results +// in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_epi16 +FORCE_INLINE __m128i _mm_cmpeq_epi16(__m128i a, __m128i b) { - // Has issues due to saturation - // return vreinterpretq_m128i_s16(vqrdmulhq_s16(a, b)); - - // Multiply - int32x4_t mul_lo = vmull_s16(vget_low_s16(vreinterpretq_s16_m128i(a)), - vget_low_s16(vreinterpretq_s16_m128i(b))); - int32x4_t mul_hi = vmull_s16(vget_high_s16(vreinterpretq_s16_m128i(a)), - vget_high_s16(vreinterpretq_s16_m128i(b))); + return vreinterpretq_m128i_u16( + vceqq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +} - // Rounding narrowing shift right - // narrow = (int16_t)((mul + 16384) >> 15); - int16x4_t narrow_lo = vrshrn_n_s32(mul_lo, 15); - int16x4_t narrow_hi = vrshrn_n_s32(mul_hi, 15); +// Compare packed 32-bit integers in a and b for equality, and store the results +// in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_epi32 +FORCE_INLINE __m128i _mm_cmpeq_epi32(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u32( + vceqq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +} - // Join together - return vreinterpretq_m128i_s16(vcombine_s16(narrow_lo, narrow_hi)); +// Compare packed 8-bit integers in a and b for equality, and store the results +// in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_epi8 +FORCE_INLINE __m128i _mm_cmpeq_epi8(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u8( + vceqq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } -// Vertically multiply each unsigned 8-bit integer from a with the corresponding -// signed 8-bit integer from b, producing intermediate signed 16-bit integers. -// Horizontally add adjacent pairs of intermediate signed 16-bit integers, -// and pack the saturated results in dst. -// -// FOR j := 0 to 7 -// i := j*16 -// dst[i+15:i] := Saturate_To_Int16( a[i+15:i+8]*b[i+15:i+8] + -// a[i+7:i]*b[i+7:i] ) -// ENDFOR -FORCE_INLINE __m128i _mm_maddubs_epi16(__m128i _a, __m128i _b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for equality, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_pd +FORCE_INLINE __m128d _mm_cmpeq_pd(__m128d a, __m128d b) { -#if defined(__aarch64__) - uint8x16_t a = vreinterpretq_u8_m128i(_a); - int8x16_t b = vreinterpretq_s8_m128i(_b); - int16x8_t tl = vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(a))), - vmovl_s8(vget_low_s8(b))); - int16x8_t th = vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(a))), - vmovl_s8(vget_high_s8(b))); - return vreinterpretq_m128i_s16( - vqaddq_s16(vuzp1q_s16(tl, th), vuzp2q_s16(tl, th))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64( + vceqq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); #else - // This would be much simpler if x86 would choose to zero extend OR sign - // extend, not both. This could probably be optimized better. - uint16x8_t a = vreinterpretq_u16_m128i(_a); - int16x8_t b = vreinterpretq_s16_m128i(_b); - - // Zero extend a - int16x8_t a_odd = vreinterpretq_s16_u16(vshrq_n_u16(a, 8)); - int16x8_t a_even = vreinterpretq_s16_u16(vbicq_u16(a, vdupq_n_u16(0xff00))); - - // Sign extend by shifting left then shifting right. - int16x8_t b_even = vshrq_n_s16(vshlq_n_s16(b, 8), 8); - int16x8_t b_odd = vshrq_n_s16(b, 8); + // (a == b) -> (a_lo == b_lo) && (a_hi == b_hi) + uint32x4_t cmp = + vceqq_u32(vreinterpretq_u32_m128d(a), vreinterpretq_u32_m128d(b)); + uint32x4_t swapped = vrev64q_u32(cmp); + return vreinterpretq_m128d_u32(vandq_u32(cmp, swapped)); +#endif +} - // multiply - int16x8_t prod1 = vmulq_s16(a_even, b_even); - int16x8_t prod2 = vmulq_s16(a_odd, b_odd); +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for equality, store the result in the lower element of dst, and copy the +// upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpeq_sd +FORCE_INLINE __m128d _mm_cmpeq_sd(__m128d a, __m128d b) +{ + return _mm_move_sd(a, _mm_cmpeq_pd(a, b)); +} - // saturated add - return vreinterpretq_m128i_s16(vqaddq_s16(prod1, prod2)); +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for greater-than-or-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpge_pd +FORCE_INLINE __m128d _mm_cmpge_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64( + vcgeq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) >= (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = (*(double *) &a1) >= (*(double *) &b1) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); #endif } -// Computes the fused multiple add product of 32-bit floating point numbers. -// -// Return Value -// Multiplies A and B, and adds C to the temporary result before returning it. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_fmadd -FORCE_INLINE __m128 _mm_fmadd_ps(__m128 a, __m128 b, __m128 c) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for greater-than-or-equal, store the result in the lower element of dst, +// and copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpge_sd +FORCE_INLINE __m128d _mm_cmpge_sd(__m128d a, __m128d b) { -#if defined(__aarch64__) - return vreinterpretq_m128_f32(vfmaq_f32(vreinterpretq_f32_m128(c), - vreinterpretq_f32_m128(b), - vreinterpretq_f32_m128(a))); +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_cmpge_pd(a, b)); #else - return _mm_add_ps(_mm_mul_ps(a, b), c); + // expand "_mm_cmpge_pd()" to reduce unnecessary operations + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) >= (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = a1; + + return vreinterpretq_m128d_u64(vld1q_u64(d)); #endif } -// Alternatively add and subtract packed single-precision (32-bit) -// floating-point elements in a to/from packed elements in b, and store the -// results in dst. -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=addsub_ps -FORCE_INLINE __m128 _mm_addsub_ps(__m128 a, __m128 b) +// Compare packed signed 16-bit integers in a and b for greater-than, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_epi16 +FORCE_INLINE __m128i _mm_cmpgt_epi16(__m128i a, __m128i b) { - __m128 mask = {-1.0f, 1.0f, -1.0f, 1.0f}; - return _mm_fmadd_ps(b, mask, a); + return vreinterpretq_m128i_u16( + vcgtq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } -// Compute the absolute differences of packed unsigned 8-bit integers in a and -// b, then horizontally sum each consecutive 8 differences to produce two -// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low -// 16 bits of 64-bit elements in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sad_epu8 -FORCE_INLINE __m128i _mm_sad_epu8(__m128i a, __m128i b) +// Compare packed signed 32-bit integers in a and b for greater-than, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_epi32 +FORCE_INLINE __m128i _mm_cmpgt_epi32(__m128i a, __m128i b) { - uint16x8_t t = vpaddlq_u8(vabdq_u8((uint8x16_t) a, (uint8x16_t) b)); - uint16_t r0 = t[0] + t[1] + t[2] + t[3]; - uint16_t r4 = t[4] + t[5] + t[6] + t[7]; - uint16x8_t r = vsetq_lane_u16(r0, vdupq_n_u16(0), 0); - return (__m128i) vsetq_lane_u16(r4, r, 4); + return vreinterpretq_m128i_u32( + vcgtq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } -// Compute the absolute differences of packed unsigned 8-bit integers in a and -// b, then horizontally sum each consecutive 8 differences to produce four -// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low -// 16 bits of dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_sad_pu8 -FORCE_INLINE __m64 _mm_sad_pu8(__m64 a, __m64 b) +// Compare packed signed 8-bit integers in a and b for greater-than, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_epi8 +FORCE_INLINE __m128i _mm_cmpgt_epi8(__m128i a, __m128i b) { - uint16x4_t t = - vpaddl_u8(vabd_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); - uint16_t r0 = t[0] + t[1] + t[2] + t[3]; - return vreinterpret_m64_u16(vset_lane_u16(r0, vdup_n_u16(0), 0)); + return vreinterpretq_m128i_u8( + vcgtq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } -// Compute the absolute differences of packed unsigned 8-bit integers in a and -// b, then horizontally sum each consecutive 8 differences to produce four -// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low -// 16 bits of dst. -// -// FOR j := 0 to 7 -// i := j*8 -// tmp[i+7:i] := ABS(a[i+7:i] - b[i+7:i]) -// ENDFOR -// dst[15:0] := tmp[7:0] + tmp[15:8] + tmp[23:16] + tmp[31:24] + tmp[39:32] + -// tmp[47:40] + tmp[55:48] + tmp[63:56] dst[63:16] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_m_psadbw -#define _m_psadbw(a, b) _mm_sad_pu8(a, b) - -// Divides the four single-precision, floating-point values of a and b. -// -// r0 := a0 / b0 -// r1 := a1 / b1 -// r2 := a2 / b2 -// r3 := a3 / b3 -// -// https://msdn.microsoft.com/en-us/library/edaw8147(v=vs.100).aspx -FORCE_INLINE __m128 _mm_div_ps(__m128 a, __m128 b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for greater-than, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_pd +FORCE_INLINE __m128d _mm_cmpgt_pd(__m128d a, __m128d b) { -#if defined(__aarch64__) - return vreinterpretq_m128_f32( - vdivq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64( + vcgtq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); #else - float32x4_t recip0 = vrecpeq_f32(vreinterpretq_f32_m128(b)); - float32x4_t recip1 = - vmulq_f32(recip0, vrecpsq_f32(recip0, vreinterpretq_f32_m128(b))); - return vreinterpretq_m128_f32(vmulq_f32(vreinterpretq_f32_m128(a), recip1)); + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) > (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = (*(double *) &a1) > (*(double *) &b1) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); #endif } -// Divides the scalar single-precision floating point value of a by b. -// https://msdn.microsoft.com/en-us/library/4y73xa49(v=vs.100).aspx -FORCE_INLINE __m128 _mm_div_ss(__m128 a, __m128 b) -{ - float32_t value = - vgetq_lane_f32(vreinterpretq_f32_m128(_mm_div_ps(a, b)), 0); - return vreinterpretq_m128_f32( - vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); -} - -// Compute the approximate reciprocal of packed single-precision (32-bit) -// floating-point elements in a, and store the results in dst. The maximum -// relative error for this approximation is less than 1.5*2^-12. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rcp_ps -FORCE_INLINE __m128 _mm_rcp_ps(__m128 in) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for greater-than, store the result in the lower element of dst, and copy +// the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpgt_sd +FORCE_INLINE __m128d _mm_cmpgt_sd(__m128d a, __m128d b) { -#if defined(__aarch64__) - return vreinterpretq_m128_f32( - vdivq_f32(vdupq_n_f32(1.0f), vreinterpretq_f32_m128(in))); +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_cmpgt_pd(a, b)); #else - float32x4_t recip = vrecpeq_f32(vreinterpretq_f32_m128(in)); - recip = vmulq_f32(recip, vrecpsq_f32(recip, vreinterpretq_f32_m128(in))); - return vreinterpretq_m128_f32(recip); + // expand "_mm_cmpge_pd()" to reduce unnecessary operations + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) > (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = a1; + + return vreinterpretq_m128d_u64(vld1q_u64(d)); #endif } -// Compute the approximate reciprocal of the lower single-precision (32-bit) -// floating-point element in a, store the result in the lower element of dst, -// and copy the upper 3 packed elements from a to the upper elements of dst. The -// maximum relative error for this approximation is less than 1.5*2^-12. -// -// dst[31:0] := (1.0 / a[31:0]) -// dst[127:32] := a[127:32] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rcp_ss -FORCE_INLINE __m128 _mm_rcp_ss(__m128 a) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for less-than-or-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmple_pd +FORCE_INLINE __m128d _mm_cmple_pd(__m128d a, __m128d b) { - return _mm_move_ss(a, _mm_rcp_ps(a)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64( + vcleq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) <= (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = (*(double *) &a1) <= (*(double *) &b1) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Computes the approximations of square roots of the four single-precision, -// floating-point values of a. First computes reciprocal square roots and then -// reciprocals of the four values. -// -// r0 := sqrt(a0) -// r1 := sqrt(a1) -// r2 := sqrt(a2) -// r3 := sqrt(a3) -// -// https://msdn.microsoft.com/en-us/library/vstudio/8z67bwwk(v=vs.100).aspx -FORCE_INLINE __m128 _mm_sqrt_ps(__m128 in) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for less-than-or-equal, store the result in the lower element of dst, and +// copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmple_sd +FORCE_INLINE __m128d _mm_cmple_sd(__m128d a, __m128d b) { -#if defined(__aarch64__) - return vreinterpretq_m128_f32(vsqrtq_f32(vreinterpretq_f32_m128(in))); +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_cmple_pd(a, b)); #else - float32x4_t recipsq = vrsqrteq_f32(vreinterpretq_f32_m128(in)); - float32x4_t sq = vrecpeq_f32(recipsq); - // ??? use step versions of both sqrt and recip for better accuracy? - return vreinterpretq_m128_f32(sq); + // expand "_mm_cmpge_pd()" to reduce unnecessary operations + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) <= (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = a1; + + return vreinterpretq_m128d_u64(vld1q_u64(d)); #endif } -// Computes the approximation of the square root of the scalar single-precision -// floating point value of in. -// https://msdn.microsoft.com/en-us/library/ahfsc22d(v=vs.100).aspx -FORCE_INLINE __m128 _mm_sqrt_ss(__m128 in) +// Compare packed signed 16-bit integers in a and b for less-than, and store the +// results in dst. Note: This intrinsic emits the pcmpgtw instruction with the +// order of the operands switched. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_epi16 +FORCE_INLINE __m128i _mm_cmplt_epi16(__m128i a, __m128i b) { - float32_t value = - vgetq_lane_f32(vreinterpretq_f32_m128(_mm_sqrt_ps(in)), 0); - return vreinterpretq_m128_f32( - vsetq_lane_f32(value, vreinterpretq_f32_m128(in), 0)); + return vreinterpretq_m128i_u16( + vcltq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } -// Computes the approximations of the reciprocal square roots of the four -// single-precision floating point values of in. -// https://msdn.microsoft.com/en-us/library/22hfsh53(v=vs.100).aspx -FORCE_INLINE __m128 _mm_rsqrt_ps(__m128 in) +// Compare packed signed 32-bit integers in a and b for less-than, and store the +// results in dst. Note: This intrinsic emits the pcmpgtd instruction with the +// order of the operands switched. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_epi32 +FORCE_INLINE __m128i _mm_cmplt_epi32(__m128i a, __m128i b) { - return vreinterpretq_m128_f32(vrsqrteq_f32(vreinterpretq_f32_m128(in))); + return vreinterpretq_m128i_u32( + vcltq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } -// Compute the approximate reciprocal square root of the lower single-precision -// (32-bit) floating-point element in a, store the result in the lower element -// of dst, and copy the upper 3 packed elements from a to the upper elements of -// dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_rsqrt_ss -FORCE_INLINE __m128 _mm_rsqrt_ss(__m128 in) +// Compare packed signed 8-bit integers in a and b for less-than, and store the +// results in dst. Note: This intrinsic emits the pcmpgtb instruction with the +// order of the operands switched. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_epi8 +FORCE_INLINE __m128i _mm_cmplt_epi8(__m128i a, __m128i b) { - return vsetq_lane_f32(vgetq_lane_f32(_mm_rsqrt_ps(in), 0), in, 0); + return vreinterpretq_m128i_u8( + vcltq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } -// Compare packed signed 16-bit integers in a and b, and store packed maximum -// values in dst. -// -// FOR j := 0 to 3 -// i := j*16 -// dst[i+15:i] := MAX(a[i+15:i], b[i+15:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pi16 -FORCE_INLINE __m64 _mm_max_pi16(__m64 a, __m64 b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for less-than, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_pd +FORCE_INLINE __m128d _mm_cmplt_pd(__m128d a, __m128d b) { - return vreinterpret_m64_s16( - vmax_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64( + vcltq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) < (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = (*(double *) &a1) < (*(double *) &b1) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Compare packed signed 16-bit integers in a and b, and store packed maximum -// values in dst. -// -// FOR j := 0 to 3 -// i := j*16 -// dst[i+15:i] := MAX(a[i+15:i], b[i+15:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pi16 -#define _m_pmaxsw(a, b) _mm_max_pi16(a, b) - -// Computes the maximums of the four single-precision, floating-point values of -// a and b. -// https://msdn.microsoft.com/en-us/library/vstudio/ff5d607a(v=vs.100).aspx -FORCE_INLINE __m128 _mm_max_ps(__m128 a, __m128 b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for less-than, store the result in the lower element of dst, and copy the +// upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmplt_sd +FORCE_INLINE __m128d _mm_cmplt_sd(__m128d a, __m128d b) { -#if SSE2NEON_PRECISE_MINMAX - float32x4_t _a = vreinterpretq_f32_m128(a); - float32x4_t _b = vreinterpretq_f32_m128(b); - return vbslq_f32(vcltq_f32(_b, _a), _a, _b); +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_cmplt_pd(a, b)); #else - return vreinterpretq_m128_f32( - vmaxq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) < (*(double *) &b0) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = a1; + + return vreinterpretq_m128d_u64(vld1q_u64(d)); #endif } -// Compare packed unsigned 8-bit integers in a and b, and store packed maximum -// values in dst. -// -// FOR j := 0 to 7 -// i := j*8 -// dst[i+7:i] := MAX(a[i+7:i], b[i+7:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pu8 -FORCE_INLINE __m64 _mm_max_pu8(__m64 a, __m64 b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for not-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpneq_pd +FORCE_INLINE __m128d _mm_cmpneq_pd(__m128d a, __m128d b) { - return vreinterpret_m64_u8( - vmax_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_s32(vmvnq_s32(vreinterpretq_s32_u64( + vceqq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))))); +#else + // (a == b) -> (a_lo == b_lo) && (a_hi == b_hi) + uint32x4_t cmp = + vceqq_u32(vreinterpretq_u32_m128d(a), vreinterpretq_u32_m128d(b)); + uint32x4_t swapped = vrev64q_u32(cmp); + return vreinterpretq_m128d_u32(vmvnq_u32(vandq_u32(cmp, swapped))); +#endif } -// Compare packed unsigned 8-bit integers in a and b, and store packed maximum -// values in dst. -// -// FOR j := 0 to 7 -// i := j*8 -// dst[i+7:i] := MAX(a[i+7:i], b[i+7:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_pu8 -#define _m_pmaxub(a, b) _mm_max_pu8(a, b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for not-equal, store the result in the lower element of dst, and copy the +// upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpneq_sd +FORCE_INLINE __m128d _mm_cmpneq_sd(__m128d a, __m128d b) +{ + return _mm_move_sd(a, _mm_cmpneq_pd(a, b)); +} -// Compare packed signed 16-bit integers in a and b, and store packed minimum -// values in dst. -// -// FOR j := 0 to 3 -// i := j*16 -// dst[i+15:i] := MIN(a[i+15:i], b[i+15:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_pi16 -FORCE_INLINE __m64 _mm_min_pi16(__m64 a, __m64 b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for not-greater-than-or-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnge_pd +FORCE_INLINE __m128d _mm_cmpnge_pd(__m128d a, __m128d b) { - return vreinterpret_m64_s16( - vmin_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64(veorq_u64( + vcgeq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b)), + vdupq_n_u64(UINT64_MAX))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = + !((*(double *) &a0) >= (*(double *) &b0)) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = + !((*(double *) &a1) >= (*(double *) &b1)) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Compare packed signed 16-bit integers in a and b, and store packed minimum -// values in dst. -// -// FOR j := 0 to 3 -// i := j*16 -// dst[i+15:i] := MIN(a[i+15:i], b[i+15:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_pi16 -#define _m_pminsw(a, b) _mm_min_pi16(a, b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for not-greater-than-or-equal, store the result in the lower element of +// dst, and copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnge_sd +FORCE_INLINE __m128d _mm_cmpnge_sd(__m128d a, __m128d b) +{ + return _mm_move_sd(a, _mm_cmpnge_pd(a, b)); +} -// Computes the minima of the four single-precision, floating-point values of a -// and b. -// https://msdn.microsoft.com/en-us/library/vstudio/wh13kadz(v=vs.100).aspx -FORCE_INLINE __m128 _mm_min_ps(__m128 a, __m128 b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for not-greater-than, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_cmpngt_pd +FORCE_INLINE __m128d _mm_cmpngt_pd(__m128d a, __m128d b) { -#if SSE2NEON_PRECISE_MINMAX - float32x4_t _a = vreinterpretq_f32_m128(a); - float32x4_t _b = vreinterpretq_f32_m128(b); - return vbslq_f32(vcltq_f32(_a, _b), _a, _b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64(veorq_u64( + vcgtq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b)), + vdupq_n_u64(UINT64_MAX))); #else - return vreinterpretq_m128_f32( - vminq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = + !((*(double *) &a0) > (*(double *) &b0)) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = + !((*(double *) &a1) > (*(double *) &b1)) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); #endif } -// Compare packed unsigned 8-bit integers in a and b, and store packed minimum -// values in dst. -// -// FOR j := 0 to 7 -// i := j*8 -// dst[i+7:i] := MIN(a[i+7:i], b[i+7:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_pu8 -FORCE_INLINE __m64 _mm_min_pu8(__m64 a, __m64 b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for not-greater-than, store the result in the lower element of dst, and +// copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpngt_sd +FORCE_INLINE __m128d _mm_cmpngt_sd(__m128d a, __m128d b) { - return vreinterpret_m64_u8( - vmin_u8(vreinterpret_u8_m64(a), vreinterpret_u8_m64(b))); + return _mm_move_sd(a, _mm_cmpngt_pd(a, b)); } -// Compare packed unsigned 8-bit integers in a and b, and store packed minimum -// values in dst. -// -// FOR j := 0 to 7 -// i := j*8 -// dst[i+7:i] := MIN(a[i+7:i], b[i+7:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_pu8 -#define _m_pminub(a, b) _mm_min_pu8(a, b) - -// Computes the maximum of the two lower scalar single-precision floating point -// values of a and b. -// https://msdn.microsoft.com/en-us/library/s6db5esz(v=vs.100).aspx -FORCE_INLINE __m128 _mm_max_ss(__m128 a, __m128 b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for not-less-than-or-equal, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnle_pd +FORCE_INLINE __m128d _mm_cmpnle_pd(__m128d a, __m128d b) { - float32_t value = vgetq_lane_f32(_mm_max_ps(a, b), 0); - return vreinterpretq_m128_f32( - vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64(veorq_u64( + vcleq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b)), + vdupq_n_u64(UINT64_MAX))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = + !((*(double *) &a0) <= (*(double *) &b0)) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = + !((*(double *) &a1) <= (*(double *) &b1)) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Computes the minimum of the two lower scalar single-precision floating point -// values of a and b. -// https://msdn.microsoft.com/en-us/library/0a9y7xaa(v=vs.100).aspx -FORCE_INLINE __m128 _mm_min_ss(__m128 a, __m128 b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for not-less-than-or-equal, store the result in the lower element of dst, +// and copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnle_sd +FORCE_INLINE __m128d _mm_cmpnle_sd(__m128d a, __m128d b) { - float32_t value = vgetq_lane_f32(_mm_min_ps(a, b), 0); - return vreinterpretq_m128_f32( - vsetq_lane_f32(value, vreinterpretq_f32_m128(a), 0)); + return _mm_move_sd(a, _mm_cmpnle_pd(a, b)); } -// Computes the pairwise maxima of the 16 unsigned 8-bit integers from a and the -// 16 unsigned 8-bit integers from b. -// https://msdn.microsoft.com/en-us/library/st6634za(v=vs.100).aspx -FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// for not-less-than, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnlt_pd +FORCE_INLINE __m128d _mm_cmpnlt_pd(__m128d a, __m128d b) { - return vreinterpretq_m128i_u8( - vmaxq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_u64(veorq_u64( + vcltq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b)), + vdupq_n_u64(UINT64_MAX))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = + !((*(double *) &a0) < (*(double *) &b0)) ? ~UINT64_C(0) : UINT64_C(0); + d[1] = + !((*(double *) &a1) < (*(double *) &b1)) ? ~UINT64_C(0) : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Computes the pairwise minima of the 16 unsigned 8-bit integers from a and the -// 16 unsigned 8-bit integers from b. -// https://msdn.microsoft.com/ko-kr/library/17k8cf58(v=vs.100).aspxx -FORCE_INLINE __m128i _mm_min_epu8(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b for not-less-than, store the result in the lower element of dst, and copy +// the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpnlt_sd +FORCE_INLINE __m128d _mm_cmpnlt_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_u8( - vminq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); + return _mm_move_sd(a, _mm_cmpnlt_pd(a, b)); } -// Computes the pairwise minima of the 8 signed 16-bit integers from a and the 8 -// signed 16-bit integers from b. -// https://msdn.microsoft.com/en-us/library/vstudio/6te997ew(v=vs.100).aspx -FORCE_INLINE __m128i _mm_min_epi16(__m128i a, __m128i b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// to see if neither is NaN, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpord_pd +FORCE_INLINE __m128d _mm_cmpord_pd(__m128d a, __m128d b) { - return vreinterpretq_m128i_s16( - vminq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + // Excluding NaNs, any two floating point numbers can be compared. + uint64x2_t not_nan_a = + vceqq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(a)); + uint64x2_t not_nan_b = + vceqq_f64(vreinterpretq_f64_m128d(b), vreinterpretq_f64_m128d(b)); + return vreinterpretq_m128d_u64(vandq_u64(not_nan_a, not_nan_b)); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = ((*(double *) &a0) == (*(double *) &a0) && + (*(double *) &b0) == (*(double *) &b0)) + ? ~UINT64_C(0) + : UINT64_C(0); + d[1] = ((*(double *) &a1) == (*(double *) &a1) && + (*(double *) &b1) == (*(double *) &b1)) + ? ~UINT64_C(0) + : UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Compare packed signed 8-bit integers in a and b, and store packed maximum -// values in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epi8 -FORCE_INLINE __m128i _mm_max_epi8(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b to see if neither is NaN, store the result in the lower element of dst, and +// copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpord_sd +FORCE_INLINE __m128d _mm_cmpord_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_s8( - vmaxq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_cmpord_pd(a, b)); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t d[2]; + d[0] = ((*(double *) &a0) == (*(double *) &a0) && + (*(double *) &b0) == (*(double *) &b0)) + ? ~UINT64_C(0) + : UINT64_C(0); + d[1] = a1; + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Compare packed unsigned 16-bit integers in a and b, and store packed maximum -// values in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epu16 -FORCE_INLINE __m128i _mm_max_epu16(__m128i a, __m128i b) +// Compare packed double-precision (64-bit) floating-point elements in a and b +// to see if either is NaN, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpunord_pd +FORCE_INLINE __m128d _mm_cmpunord_pd(__m128d a, __m128d b) { - return vreinterpretq_m128i_u16( - vmaxq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + // Two NaNs are not equal in comparison operation. + uint64x2_t not_nan_a = + vceqq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(a)); + uint64x2_t not_nan_b = + vceqq_f64(vreinterpretq_f64_m128d(b), vreinterpretq_f64_m128d(b)); + return vreinterpretq_m128d_s32( + vmvnq_s32(vreinterpretq_s32_u64(vandq_u64(not_nan_a, not_nan_b)))); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = ((*(double *) &a0) == (*(double *) &a0) && + (*(double *) &b0) == (*(double *) &b0)) + ? UINT64_C(0) + : ~UINT64_C(0); + d[1] = ((*(double *) &a1) == (*(double *) &a1) && + (*(double *) &b1) == (*(double *) &b1)) + ? UINT64_C(0) + : ~UINT64_C(0); + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Compare packed signed 8-bit integers in a and b, and store packed minimum -// values in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_epi8 -FORCE_INLINE __m128i _mm_min_epi8(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b to see if either is NaN, store the result in the lower element of dst, and +// copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpunord_sd +FORCE_INLINE __m128d _mm_cmpunord_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_s8( - vminq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_cmpunord_pd(a, b)); +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t d[2]; + d[0] = ((*(double *) &a0) == (*(double *) &a0) && + (*(double *) &b0) == (*(double *) &b0)) + ? UINT64_C(0) + : ~UINT64_C(0); + d[1] = a1; + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Compare packed unsigned 16-bit integers in a and b, and store packed minimum -// values in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_min_epu16 -FORCE_INLINE __m128i _mm_min_epu16(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point element in a and b +// for greater-than-or-equal, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comige_sd +FORCE_INLINE int _mm_comige_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_u16( - vminq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vgetq_lane_u64(vcgeq_f64(a, b), 0) & 0x1; +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + + return (*(double *) &a0 >= *(double *) &b0); +#endif } -// Computes the pairwise maxima of the 8 signed 16-bit integers from a and the 8 -// signed 16-bit integers from b. -// https://msdn.microsoft.com/en-us/LIBRary/3x060h7c(v=vs.100).aspx -FORCE_INLINE __m128i _mm_max_epi16(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point element in a and b +// for greater-than, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comigt_sd +FORCE_INLINE int _mm_comigt_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_s16( - vmaxq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vgetq_lane_u64(vcgtq_f64(a, b), 0) & 0x1; +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + + return (*(double *) &a0 > *(double *) &b0); +#endif } -// epi versions of min/max -// Computes the pariwise maximums of the four signed 32-bit integer values of a -// and b. -// -// A 128-bit parameter that can be defined with the following equations: -// r0 := (a0 > b0) ? a0 : b0 -// r1 := (a1 > b1) ? a1 : b1 -// r2 := (a2 > b2) ? a2 : b2 -// r3 := (a3 > b3) ? a3 : b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/bb514055(v=vs.100).aspx -FORCE_INLINE __m128i _mm_max_epi32(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point element in a and b +// for less-than-or-equal, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comile_sd +FORCE_INLINE int _mm_comile_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_s32( - vmaxq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vgetq_lane_u64(vcleq_f64(a, b), 0) & 0x1; +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + + return (*(double *) &a0 <= *(double *) &b0); +#endif } -// Computes the pariwise minima of the four signed 32-bit integer values of a -// and b. -// -// A 128-bit parameter that can be defined with the following equations: -// r0 := (a0 < b0) ? a0 : b0 -// r1 := (a1 < b1) ? a1 : b1 -// r2 := (a2 < b2) ? a2 : b2 -// r3 := (a3 < b3) ? a3 : b3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/bb531476(v=vs.100).aspx -FORCE_INLINE __m128i _mm_min_epi32(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point element in a and b +// for less-than, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comilt_sd +FORCE_INLINE int _mm_comilt_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_s32( - vminq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vgetq_lane_u64(vcltq_f64(a, b), 0) & 0x1; +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + + return (*(double *) &a0 < *(double *) &b0); +#endif } -// Compare packed unsigned 32-bit integers in a and b, and store packed maximum -// values in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epu32 -FORCE_INLINE __m128i _mm_max_epu32(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point element in a and b +// for equality, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comieq_sd +FORCE_INLINE int _mm_comieq_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_u32( - vmaxq_u32(vreinterpretq_u32_m128i(a), vreinterpretq_u32_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vgetq_lane_u64(vceqq_f64(a, b), 0) & 0x1; +#else + uint32x4_t a_not_nan = + vceqq_u32(vreinterpretq_u32_m128d(a), vreinterpretq_u32_m128d(a)); + uint32x4_t b_not_nan = + vceqq_u32(vreinterpretq_u32_m128d(b), vreinterpretq_u32_m128d(b)); + uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); + uint32x4_t a_eq_b = + vceqq_u32(vreinterpretq_u32_m128d(a), vreinterpretq_u32_m128d(b)); + uint64x2_t and_results = vandq_u64(vreinterpretq_u64_u32(a_and_b_not_nan), + vreinterpretq_u64_u32(a_eq_b)); + return vgetq_lane_u64(and_results, 0) & 0x1; +#endif } -// Compare packed unsigned 32-bit integers in a and b, and store packed minimum -// values in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_max_epu32 -FORCE_INLINE __m128i _mm_min_epu32(__m128i a, __m128i b) +// Compare the lower double-precision (64-bit) floating-point element in a and b +// for not-equal, and return the boolean result (0 or 1). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_comineq_sd +FORCE_INLINE int _mm_comineq_sd(__m128d a, __m128d b) { - return vreinterpretq_m128i_u32( - vminq_u32(vreinterpretq_u32_m128i(a), vreinterpretq_u32_m128i(b))); + return !_mm_comieq_sd(a, b); } -// Multiply the packed unsigned 16-bit integers in a and b, producing -// intermediate 32-bit integers, and store the high 16 bits of the intermediate -// integers in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mulhi_pu16 -FORCE_INLINE __m64 _mm_mulhi_pu16(__m64 a, __m64 b) +// Convert packed signed 32-bit integers in a to packed double-precision +// (64-bit) floating-point elements, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepi32_pd +FORCE_INLINE __m128d _mm_cvtepi32_pd(__m128i a) { - return vreinterpret_m64_u16(vshrn_n_u32( - vmull_u16(vreinterpret_u16_m64(a), vreinterpret_u16_m64(b)), 16)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vcvtq_f64_s64(vmovl_s32(vget_low_s32(vreinterpretq_s32_m128i(a))))); +#else + double a0 = (double) vgetq_lane_s32(vreinterpretq_s32_m128i(a), 0); + double a1 = (double) vgetq_lane_s32(vreinterpretq_s32_m128i(a), 1); + return _mm_set_pd(a1, a0); +#endif } -// Multiplies the 8 signed 16-bit integers from a by the 8 signed 16-bit -// integers from b. -// -// r0 := (a0 * b0)[31:16] -// r1 := (a1 * b1)[31:16] -// ... -// r7 := (a7 * b7)[31:16] -// -// https://msdn.microsoft.com/en-us/library/vstudio/59hddw1d(v=vs.100).aspx -FORCE_INLINE __m128i _mm_mulhi_epi16(__m128i a, __m128i b) +// Convert packed signed 32-bit integers in a to packed single-precision +// (32-bit) floating-point elements, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepi32_ps +FORCE_INLINE __m128 _mm_cvtepi32_ps(__m128i a) { - /* FIXME: issue with large values because of result saturation */ - // int16x8_t ret = vqdmulhq_s16(vreinterpretq_s16_m128i(a), - // vreinterpretq_s16_m128i(b)); /* =2*a*b */ return - // vreinterpretq_m128i_s16(vshrq_n_s16(ret, 1)); - int16x4_t a3210 = vget_low_s16(vreinterpretq_s16_m128i(a)); - int16x4_t b3210 = vget_low_s16(vreinterpretq_s16_m128i(b)); - int32x4_t ab3210 = vmull_s16(a3210, b3210); /* 3333222211110000 */ - int16x4_t a7654 = vget_high_s16(vreinterpretq_s16_m128i(a)); - int16x4_t b7654 = vget_high_s16(vreinterpretq_s16_m128i(b)); - int32x4_t ab7654 = vmull_s16(a7654, b7654); /* 7777666655554444 */ - uint16x8x2_t r = - vuzpq_u16(vreinterpretq_u16_s32(ab3210), vreinterpretq_u16_s32(ab7654)); - return vreinterpretq_m128i_u16(r.val[1]); + return vreinterpretq_m128_f32(vcvtq_f32_s32(vreinterpretq_s32_m128i(a))); } -// Multiply the packed unsigned 16-bit integers in a and b, producing -// intermediate 32-bit integers, and store the high 16 bits of the intermediate -// integers in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_mulhi_epu16 -FORCE_INLINE __m128i _mm_mulhi_epu16(__m128i a, __m128i b) +// Convert packed double-precision (64-bit) floating-point elements in a to +// packed 32-bit integers, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpd_epi32 +FORCE_INLINE __m128i _mm_cvtpd_epi32(__m128d a) { - uint16x4_t a3210 = vget_low_u16(vreinterpretq_u16_m128i(a)); - uint16x4_t b3210 = vget_low_u16(vreinterpretq_u16_m128i(b)); - uint32x4_t ab3210 = vmull_u16(a3210, b3210); -#if defined(__aarch64__) - uint32x4_t ab7654 = - vmull_high_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b)); - uint16x8_t r = vuzp2q_u16(vreinterpretq_u16_u32(ab3210), - vreinterpretq_u16_u32(ab7654)); - return vreinterpretq_m128i_u16(r); +// vrnd32xq_f64 not supported on clang +#if defined(__ARM_FEATURE_FRINT) && !defined(__clang__) + float64x2_t rounded = vrnd32xq_f64(vreinterpretq_f64_m128d(a)); + int64x2_t integers = vcvtq_s64_f64(rounded); + return vreinterpretq_m128i_s32( + vcombine_s32(vmovn_s64(integers), vdup_n_s32(0))); #else - uint16x4_t a7654 = vget_high_u16(vreinterpretq_u16_m128i(a)); - uint16x4_t b7654 = vget_high_u16(vreinterpretq_u16_m128i(b)); - uint32x4_t ab7654 = vmull_u16(a7654, b7654); - uint16x8x2_t r = - vuzpq_u16(vreinterpretq_u16_u32(ab3210), vreinterpretq_u16_u32(ab7654)); - return vreinterpretq_m128i_u16(r.val[1]); + __m128d rnd = _mm_round_pd(a, _MM_FROUND_CUR_DIRECTION); + double d0 = ((double *) &rnd)[0]; + double d1 = ((double *) &rnd)[1]; + return _mm_set_epi32(0, 0, (int32_t) d1, (int32_t) d0); #endif } -// Computes pairwise add of each argument as single-precision, floating-point -// values a and b. -// https://msdn.microsoft.com/en-us/library/yd9wecaa.aspx -FORCE_INLINE __m128 _mm_hadd_ps(__m128 a, __m128 b) +// Convert packed double-precision (64-bit) floating-point elements in a to +// packed 32-bit integers, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpd_pi32 +FORCE_INLINE __m64 _mm_cvtpd_pi32(__m128d a) { -#if defined(__aarch64__) - return vreinterpretq_m128_f32( - vpaddq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); -#else - float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(a)); - float32x2_t a32 = vget_high_f32(vreinterpretq_f32_m128(a)); - float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(b)); - float32x2_t b32 = vget_high_f32(vreinterpretq_f32_m128(b)); - return vreinterpretq_m128_f32( - vcombine_f32(vpadd_f32(a10, a32), vpadd_f32(b10, b32))); -#endif + __m128d rnd = _mm_round_pd(a, _MM_FROUND_CUR_DIRECTION); + double d0 = ((double *) &rnd)[0]; + double d1 = ((double *) &rnd)[1]; + int32_t ALIGN_STRUCT(16) data[2] = {(int32_t) d0, (int32_t) d1}; + return vreinterpret_m64_s32(vld1_s32(data)); } -// Computes pairwise add of each argument as a 16-bit signed or unsigned integer -// values a and b. -FORCE_INLINE __m128i _mm_hadd_epi16(__m128i _a, __m128i _b) +// Convert packed double-precision (64-bit) floating-point elements in a to +// packed single-precision (32-bit) floating-point elements, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpd_ps +FORCE_INLINE __m128 _mm_cvtpd_ps(__m128d a) { - int16x8_t a = vreinterpretq_s16_m128i(_a); - int16x8_t b = vreinterpretq_s16_m128i(_b); -#if defined(__aarch64__) - return vreinterpretq_m128i_s16(vpaddq_s16(a, b)); +#if defined(__aarch64__) || defined(_M_ARM64) + float32x2_t tmp = vcvt_f32_f64(vreinterpretq_f64_m128d(a)); + return vreinterpretq_m128_f32(vcombine_f32(tmp, vdup_n_f32(0))); #else - return vreinterpretq_m128i_s16( - vcombine_s16(vpadd_s16(vget_low_s16(a), vget_high_s16(a)), - vpadd_s16(vget_low_s16(b), vget_high_s16(b)))); + float a0 = (float) ((double *) &a)[0]; + float a1 = (float) ((double *) &a)[1]; + return _mm_set_ps(0, 0, a1, a0); #endif } -// Horizontally substract adjacent pairs of single-precision (32-bit) -// floating-point elements in a and b, and pack the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hsub_ps -FORCE_INLINE __m128 _mm_hsub_ps(__m128 _a, __m128 _b) +// Convert packed signed 32-bit integers in a to packed double-precision +// (64-bit) floating-point elements, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtpi32_pd +FORCE_INLINE __m128d _mm_cvtpi32_pd(__m64 a) { -#if defined(__aarch64__) - return vreinterpretq_m128_f32(vsubq_f32( - vuzp1q_f32(vreinterpretq_f32_m128(_a), vreinterpretq_f32_m128(_b)), - vuzp2q_f32(vreinterpretq_f32_m128(_a), vreinterpretq_f32_m128(_b)))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vcvtq_f64_s64(vmovl_s32(vreinterpret_s32_m64(a)))); #else - float32x4x2_t c = - vuzpq_f32(vreinterpretq_f32_m128(_a), vreinterpretq_f32_m128(_b)); - return vreinterpretq_m128_f32(vsubq_f32(c.val[0], c.val[1])); + double a0 = (double) vget_lane_s32(vreinterpret_s32_m64(a), 0); + double a1 = (double) vget_lane_s32(vreinterpret_s32_m64(a), 1); + return _mm_set_pd(a1, a0); #endif } -// Horizontally add adjacent pairs of 16-bit integers in a and b, and pack the -// signed 16-bit results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hadd_pi16 -FORCE_INLINE __m64 _mm_hadd_pi16(__m64 a, __m64 b) +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_epi32 +// *NOTE*. The default rounding mode on SSE is 'round to even', which ARMv7-A +// does not support! It is supported on ARMv8-A however. +FORCE_INLINE __m128i _mm_cvtps_epi32(__m128 a) { - return vreinterpret_m64_s16( - vpadd_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); +#if defined(__ARM_FEATURE_FRINT) + return vreinterpretq_m128i_s32(vcvtq_s32_f32(vrnd32xq_f32(a))); +#elif (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ARM_FEATURE_DIRECTED_ROUNDING) + switch (_MM_GET_ROUNDING_MODE()) { + case _MM_ROUND_NEAREST: + return vreinterpretq_m128i_s32(vcvtnq_s32_f32(a)); + case _MM_ROUND_DOWN: + return vreinterpretq_m128i_s32(vcvtmq_s32_f32(a)); + case _MM_ROUND_UP: + return vreinterpretq_m128i_s32(vcvtpq_s32_f32(a)); + default: // _MM_ROUND_TOWARD_ZERO + return vreinterpretq_m128i_s32(vcvtq_s32_f32(a)); + } +#else + float *f = (float *) &a; + switch (_MM_GET_ROUNDING_MODE()) { + case _MM_ROUND_NEAREST: { + uint32x4_t signmask = vdupq_n_u32(0x80000000); + float32x4_t half = vbslq_f32(signmask, vreinterpretq_f32_m128(a), + vdupq_n_f32(0.5f)); /* +/- 0.5 */ + int32x4_t r_normal = vcvtq_s32_f32(vaddq_f32( + vreinterpretq_f32_m128(a), half)); /* round to integer: [a + 0.5]*/ + int32x4_t r_trunc = vcvtq_s32_f32( + vreinterpretq_f32_m128(a)); /* truncate to integer: [a] */ + int32x4_t plusone = vreinterpretq_s32_u32(vshrq_n_u32( + vreinterpretq_u32_s32(vnegq_s32(r_trunc)), 31)); /* 1 or 0 */ + int32x4_t r_even = vbicq_s32(vaddq_s32(r_trunc, plusone), + vdupq_n_s32(1)); /* ([a] + {0,1}) & ~1 */ + float32x4_t delta = vsubq_f32( + vreinterpretq_f32_m128(a), + vcvtq_f32_s32(r_trunc)); /* compute delta: delta = (a - [a]) */ + uint32x4_t is_delta_half = + vceqq_f32(delta, half); /* delta == +/- 0.5 */ + return vreinterpretq_m128i_s32( + vbslq_s32(is_delta_half, r_even, r_normal)); + } + case _MM_ROUND_DOWN: + return _mm_set_epi32(floorf(f[3]), floorf(f[2]), floorf(f[1]), + floorf(f[0])); + case _MM_ROUND_UP: + return _mm_set_epi32(ceilf(f[3]), ceilf(f[2]), ceilf(f[1]), + ceilf(f[0])); + default: // _MM_ROUND_TOWARD_ZERO + return _mm_set_epi32((int32_t) f[3], (int32_t) f[2], (int32_t) f[1], + (int32_t) f[0]); + } +#endif } -// Horizontally add adjacent pairs of 32-bit integers in a and b, and pack the -// signed 32-bit results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hadd_pi32 -FORCE_INLINE __m64 _mm_hadd_pi32(__m64 a, __m64 b) +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed double-precision (64-bit) floating-point elements, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_pd +FORCE_INLINE __m128d _mm_cvtps_pd(__m128 a) { - return vreinterpret_m64_s32( - vpadd_s32(vreinterpret_s32_m64(a), vreinterpret_s32_m64(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vcvt_f64_f32(vget_low_f32(vreinterpretq_f32_m128(a)))); +#else + double a0 = (double) vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + double a1 = (double) vgetq_lane_f32(vreinterpretq_f32_m128(a), 1); + return _mm_set_pd(a1, a0); +#endif } -// Computes pairwise difference of each argument as a 16-bit signed or unsigned -// integer values a and b. -FORCE_INLINE __m128i _mm_hsub_epi16(__m128i _a, __m128i _b) +// Copy the lower double-precision (64-bit) floating-point element of a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsd_f64 +FORCE_INLINE double _mm_cvtsd_f64(__m128d a) { - int32x4_t a = vreinterpretq_s32_m128i(_a); - int32x4_t b = vreinterpretq_s32_m128i(_b); - // Interleave using vshrn/vmovn - // [a0|a2|a4|a6|b0|b2|b4|b6] - // [a1|a3|a5|a7|b1|b3|b5|b7] - int16x8_t ab0246 = vcombine_s16(vmovn_s32(a), vmovn_s32(b)); - int16x8_t ab1357 = vcombine_s16(vshrn_n_s32(a, 16), vshrn_n_s32(b, 16)); - // Subtract - return vreinterpretq_m128i_s16(vsubq_s16(ab0246, ab1357)); +#if defined(__aarch64__) || defined(_M_ARM64) + return (double) vgetq_lane_f64(vreinterpretq_f64_m128d(a), 0); +#else + return ((double *) &a)[0]; +#endif } -// Computes saturated pairwise sub of each argument as a 16-bit signed -// integer values a and b. -FORCE_INLINE __m128i _mm_hadds_epi16(__m128i _a, __m128i _b) +// Convert the lower double-precision (64-bit) floating-point element in a to a +// 32-bit integer, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsd_si32 +FORCE_INLINE int32_t _mm_cvtsd_si32(__m128d a) { -#if defined(__aarch64__) - int16x8_t a = vreinterpretq_s16_m128i(_a); - int16x8_t b = vreinterpretq_s16_m128i(_b); - return vreinterpretq_s64_s16( - vqaddq_s16(vuzp1q_s16(a, b), vuzp2q_s16(a, b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return (int32_t) vgetq_lane_f64(vrndiq_f64(vreinterpretq_f64_m128d(a)), 0); #else - int32x4_t a = vreinterpretq_s32_m128i(_a); - int32x4_t b = vreinterpretq_s32_m128i(_b); - // Interleave using vshrn/vmovn - // [a0|a2|a4|a6|b0|b2|b4|b6] - // [a1|a3|a5|a7|b1|b3|b5|b7] - int16x8_t ab0246 = vcombine_s16(vmovn_s32(a), vmovn_s32(b)); - int16x8_t ab1357 = vcombine_s16(vshrn_n_s32(a, 16), vshrn_n_s32(b, 16)); - // Saturated add - return vreinterpretq_m128i_s16(vqaddq_s16(ab0246, ab1357)); + __m128d rnd = _mm_round_pd(a, _MM_FROUND_CUR_DIRECTION); + double ret = ((double *) &rnd)[0]; + return (int32_t) ret; #endif } -// Computes saturated pairwise difference of each argument as a 16-bit signed -// integer values a and b. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_hsubs_epi16 -FORCE_INLINE __m128i _mm_hsubs_epi16(__m128i _a, __m128i _b) +// Convert the lower double-precision (64-bit) floating-point element in a to a +// 64-bit integer, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsd_si64 +FORCE_INLINE int64_t _mm_cvtsd_si64(__m128d a) { -#if defined(__aarch64__) - int16x8_t a = vreinterpretq_s16_m128i(_a); - int16x8_t b = vreinterpretq_s16_m128i(_b); - return vreinterpretq_s64_s16( - vqsubq_s16(vuzp1q_s16(a, b), vuzp2q_s16(a, b))); +#if defined(__aarch64__) || defined(_M_ARM64) + return (int64_t) vgetq_lane_f64(vrndiq_f64(vreinterpretq_f64_m128d(a)), 0); #else - int32x4_t a = vreinterpretq_s32_m128i(_a); - int32x4_t b = vreinterpretq_s32_m128i(_b); - // Interleave using vshrn/vmovn - // [a0|a2|a4|a6|b0|b2|b4|b6] - // [a1|a3|a5|a7|b1|b3|b5|b7] - int16x8_t ab0246 = vcombine_s16(vmovn_s32(a), vmovn_s32(b)); - int16x8_t ab1357 = vcombine_s16(vshrn_n_s32(a, 16), vshrn_n_s32(b, 16)); - // Saturated subtract - return vreinterpretq_m128i_s16(vqsubq_s16(ab0246, ab1357)); + __m128d rnd = _mm_round_pd(a, _MM_FROUND_CUR_DIRECTION); + double ret = ((double *) &rnd)[0]; + return (int64_t) ret; #endif } -// Computes pairwise add of each argument as a 32-bit signed or unsigned integer -// values a and b. -FORCE_INLINE __m128i _mm_hadd_epi32(__m128i _a, __m128i _b) -{ - int32x4_t a = vreinterpretq_s32_m128i(_a); - int32x4_t b = vreinterpretq_s32_m128i(_b); - return vreinterpretq_m128i_s32( - vcombine_s32(vpadd_s32(vget_low_s32(a), vget_high_s32(a)), - vpadd_s32(vget_low_s32(b), vget_high_s32(b)))); +// Convert the lower double-precision (64-bit) floating-point element in a to a +// 64-bit integer, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsd_si64x +#define _mm_cvtsd_si64x _mm_cvtsd_si64 + +// Convert the lower double-precision (64-bit) floating-point element in b to a +// single-precision (32-bit) floating-point element, store the result in the +// lower element of dst, and copy the upper 3 packed elements from a to the +// upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsd_ss +FORCE_INLINE __m128 _mm_cvtsd_ss(__m128 a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32(vsetq_lane_f32( + vget_lane_f32(vcvt_f32_f64(vreinterpretq_f64_m128d(b)), 0), + vreinterpretq_f32_m128(a), 0)); +#else + return vreinterpretq_m128_f32(vsetq_lane_f32((float) ((double *) &b)[0], + vreinterpretq_f32_m128(a), 0)); +#endif } -// Computes pairwise difference of each argument as a 32-bit signed or unsigned -// integer values a and b. -FORCE_INLINE __m128i _mm_hsub_epi32(__m128i _a, __m128i _b) +// Copy the lower 32-bit integer in a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi128_si32 +FORCE_INLINE int _mm_cvtsi128_si32(__m128i a) { - int64x2_t a = vreinterpretq_s64_m128i(_a); - int64x2_t b = vreinterpretq_s64_m128i(_b); - // Interleave using vshrn/vmovn - // [a0|a2|b0|b2] - // [a1|a2|b1|b3] - int32x4_t ab02 = vcombine_s32(vmovn_s64(a), vmovn_s64(b)); - int32x4_t ab13 = vcombine_s32(vshrn_n_s64(a, 32), vshrn_n_s64(b, 32)); - // Subtract - return vreinterpretq_m128i_s32(vsubq_s32(ab02, ab13)); + return vgetq_lane_s32(vreinterpretq_s32_m128i(a), 0); } -// Kahan summation for accurate summation of floating-point numbers. -// http://blog.zachbjornson.com/2019/08/11/fast-float-summation.html -FORCE_INLINE void sse2neon_kadd_f32(float *sum, float *c, float y) +// Copy the lower 64-bit integer in a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi128_si64 +FORCE_INLINE int64_t _mm_cvtsi128_si64(__m128i a) { - y -= *c; - float t = *sum + y; - *c = (t - *sum) - y; - *sum = t; + return vgetq_lane_s64(vreinterpretq_s64_m128i(a), 0); } -// Conditionally multiply the packed single-precision (32-bit) floating-point -// elements in a and b using the high 4 bits in imm8, sum the four products, -// and conditionally store the sum in dst using the low 4 bits of imm. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_dp_ps -FORCE_INLINE __m128 _mm_dp_ps(__m128 a, __m128 b, const int imm) +// Copy the lower 64-bit integer in a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi128_si64x +#define _mm_cvtsi128_si64x(a) _mm_cvtsi128_si64(a) + +// Convert the signed 32-bit integer b to a double-precision (64-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi32_sd +FORCE_INLINE __m128d _mm_cvtsi32_sd(__m128d a, int32_t b) { -#if defined(__aarch64__) - /* shortcuts */ - if (imm == 0xFF) { - return _mm_set1_ps(vaddvq_f32(_mm_mul_ps(a, b))); - } - if (imm == 0x7F) { - float32x4_t m = _mm_mul_ps(a, b); - m[3] = 0; - return _mm_set1_ps(vaddvq_f32(m)); - } +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vsetq_lane_f64((double) b, vreinterpretq_f64_m128d(a), 0)); +#else + double bf = (double) b; + return vreinterpretq_m128d_s64( + vsetq_lane_s64(*(int64_t *) &bf, vreinterpretq_s64_m128d(a), 0)); #endif - - float s = 0, c = 0; - float32x4_t f32a = vreinterpretq_f32_m128(a); - float32x4_t f32b = vreinterpretq_f32_m128(b); - - /* To improve the accuracy of floating-point summation, Kahan algorithm - * is used for each operation. - */ - if (imm & (1 << 4)) - sse2neon_kadd_f32(&s, &c, f32a[0] * f32b[0]); - if (imm & (1 << 5)) - sse2neon_kadd_f32(&s, &c, f32a[1] * f32b[1]); - if (imm & (1 << 6)) - sse2neon_kadd_f32(&s, &c, f32a[2] * f32b[2]); - if (imm & (1 << 7)) - sse2neon_kadd_f32(&s, &c, f32a[3] * f32b[3]); - s += c; - - float32x4_t res = { - (imm & 0x1) ? s : 0, - (imm & 0x2) ? s : 0, - (imm & 0x4) ? s : 0, - (imm & 0x8) ? s : 0, - }; - return vreinterpretq_m128_f32(res); } -/* Compare operations */ +// Copy the lower 64-bit integer in a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi128_si64x +#define _mm_cvtsi128_si64x(a) _mm_cvtsi128_si64(a) -// Compares for less than -// https://msdn.microsoft.com/en-us/library/vstudio/f330yhc8(v=vs.100).aspx -FORCE_INLINE __m128 _mm_cmplt_ps(__m128 a, __m128 b) +// Copy 32-bit integer a to the lower elements of dst, and zero the upper +// elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi32_si128 +FORCE_INLINE __m128i _mm_cvtsi32_si128(int a) { - return vreinterpretq_m128_u32( - vcltq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + return vreinterpretq_m128i_s32(vsetq_lane_s32(a, vdupq_n_s32(0), 0)); } -// Compares for less than -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/fy94wye7(v=vs.100) -FORCE_INLINE __m128 _mm_cmplt_ss(__m128 a, __m128 b) +// Convert the signed 64-bit integer b to a double-precision (64-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi64_sd +FORCE_INLINE __m128d _mm_cvtsi64_sd(__m128d a, int64_t b) { - return _mm_move_ss(a, _mm_cmplt_ps(a, b)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vsetq_lane_f64((double) b, vreinterpretq_f64_m128d(a), 0)); +#else + double bf = (double) b; + return vreinterpretq_m128d_s64( + vsetq_lane_s64(*(int64_t *) &bf, vreinterpretq_s64_m128d(a), 0)); +#endif } -// Compares for greater than. -// -// r0 := (a0 > b0) ? 0xffffffff : 0x0 -// r1 := (a1 > b1) ? 0xffffffff : 0x0 -// r2 := (a2 > b2) ? 0xffffffff : 0x0 -// r3 := (a3 > b3) ? 0xffffffff : 0x0 -// -// https://msdn.microsoft.com/en-us/library/vstudio/11dy102s(v=vs.100).aspx -FORCE_INLINE __m128 _mm_cmpgt_ps(__m128 a, __m128 b) +// Copy 64-bit integer a to the lower element of dst, and zero the upper +// element. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi64_si128 +FORCE_INLINE __m128i _mm_cvtsi64_si128(int64_t a) { - return vreinterpretq_m128_u32( - vcgtq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + return vreinterpretq_m128i_s64(vsetq_lane_s64(a, vdupq_n_s64(0), 0)); } -// Compares for greater than. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/1xyyyy9e(v=vs.100) -FORCE_INLINE __m128 _mm_cmpgt_ss(__m128 a, __m128 b) -{ - return _mm_move_ss(a, _mm_cmpgt_ps(a, b)); -} +// Copy 64-bit integer a to the lower element of dst, and zero the upper +// element. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi64x_si128 +#define _mm_cvtsi64x_si128(a) _mm_cvtsi64_si128(a) -// Compares for greater than or equal. -// https://msdn.microsoft.com/en-us/library/vstudio/fs813y2t(v=vs.100).aspx -FORCE_INLINE __m128 _mm_cmpge_ps(__m128 a, __m128 b) +// Convert the signed 64-bit integer b to a double-precision (64-bit) +// floating-point element, store the result in the lower element of dst, and +// copy the upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtsi64x_sd +#define _mm_cvtsi64x_sd(a, b) _mm_cvtsi64_sd(a, b) + +// Convert the lower single-precision (32-bit) floating-point element in b to a +// double-precision (64-bit) floating-point element, store the result in the +// lower element of dst, and copy the upper element from a to the upper element +// of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtss_sd +FORCE_INLINE __m128d _mm_cvtss_sd(__m128d a, __m128 b) { - return vreinterpretq_m128_u32( - vcgeq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + double d = (double) vgetq_lane_f32(vreinterpretq_f32_m128(b), 0); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vsetq_lane_f64(d, vreinterpretq_f64_m128d(a), 0)); +#else + return vreinterpretq_m128d_s64( + vsetq_lane_s64(*(int64_t *) &d, vreinterpretq_s64_m128d(a), 0)); +#endif } -// Compares for greater than or equal. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/kesh3ddc(v=vs.100) -FORCE_INLINE __m128 _mm_cmpge_ss(__m128 a, __m128 b) +// Convert packed double-precision (64-bit) floating-point elements in a to +// packed 32-bit integers with truncation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttpd_epi32 +FORCE_INLINE __m128i _mm_cvttpd_epi32(__m128d a) { - return _mm_move_ss(a, _mm_cmpge_ps(a, b)); + double a0 = ((double *) &a)[0]; + double a1 = ((double *) &a)[1]; + return _mm_set_epi32(0, 0, (int32_t) a1, (int32_t) a0); } -// Compares for less than or equal. -// -// r0 := (a0 <= b0) ? 0xffffffff : 0x0 -// r1 := (a1 <= b1) ? 0xffffffff : 0x0 -// r2 := (a2 <= b2) ? 0xffffffff : 0x0 -// r3 := (a3 <= b3) ? 0xffffffff : 0x0 -// -// https://msdn.microsoft.com/en-us/library/vstudio/1s75w83z(v=vs.100).aspx -FORCE_INLINE __m128 _mm_cmple_ps(__m128 a, __m128 b) +// Convert packed double-precision (64-bit) floating-point elements in a to +// packed 32-bit integers with truncation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttpd_pi32 +FORCE_INLINE __m64 _mm_cvttpd_pi32(__m128d a) { - return vreinterpretq_m128_u32( - vcleq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + double a0 = ((double *) &a)[0]; + double a1 = ((double *) &a)[1]; + int32_t ALIGN_STRUCT(16) data[2] = {(int32_t) a0, (int32_t) a1}; + return vreinterpret_m64_s32(vld1_s32(data)); } -// Compares for less than or equal. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/a7x0hbhw(v=vs.100) -FORCE_INLINE __m128 _mm_cmple_ss(__m128 a, __m128 b) +// Convert packed single-precision (32-bit) floating-point elements in a to +// packed 32-bit integers with truncation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttps_epi32 +FORCE_INLINE __m128i _mm_cvttps_epi32(__m128 a) { - return _mm_move_ss(a, _mm_cmple_ps(a, b)); + return vreinterpretq_m128i_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a))); } -// Compares for equality. -// https://msdn.microsoft.com/en-us/library/vstudio/36aectz5(v=vs.100).aspx -FORCE_INLINE __m128 _mm_cmpeq_ps(__m128 a, __m128 b) +// Convert the lower double-precision (64-bit) floating-point element in a to a +// 32-bit integer with truncation, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttsd_si32 +FORCE_INLINE int32_t _mm_cvttsd_si32(__m128d a) { - return vreinterpretq_m128_u32( - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); + double ret = *((double *) &a); + return (int32_t) ret; } -// Compares for equality. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/k423z28e(v=vs.100) -FORCE_INLINE __m128 _mm_cmpeq_ss(__m128 a, __m128 b) +// Convert the lower double-precision (64-bit) floating-point element in a to a +// 64-bit integer with truncation, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttsd_si64 +FORCE_INLINE int64_t _mm_cvttsd_si64(__m128d a) { - return _mm_move_ss(a, _mm_cmpeq_ps(a, b)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vgetq_lane_s64(vcvtq_s64_f64(vreinterpretq_f64_m128d(a)), 0); +#else + double ret = *((double *) &a); + return (int64_t) ret; +#endif } -// Compares for inequality. -// https://msdn.microsoft.com/en-us/library/sf44thbx(v=vs.100).aspx -FORCE_INLINE __m128 _mm_cmpneq_ps(__m128 a, __m128 b) +// Convert the lower double-precision (64-bit) floating-point element in a to a +// 64-bit integer with truncation, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttsd_si64x +#define _mm_cvttsd_si64x(a) _mm_cvttsd_si64(a) + +// Divide packed double-precision (64-bit) floating-point elements in a by +// packed elements in b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_div_pd +FORCE_INLINE __m128d _mm_div_pd(__m128d a, __m128d b) { - return vreinterpretq_m128_u32(vmvnq_u32( - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vdivq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] / db[0]; + c[1] = da[1] / db[1]; + return vld1q_f32((float32_t *) c); +#endif } -// Compares for inequality. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/ekya8fh4(v=vs.100) -FORCE_INLINE __m128 _mm_cmpneq_ss(__m128 a, __m128 b) +// Divide the lower double-precision (64-bit) floating-point element in a by the +// lower double-precision (64-bit) floating-point element in b, store the result +// in the lower element of dst, and copy the upper element from a to the upper +// element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_div_sd +FORCE_INLINE __m128d _mm_div_sd(__m128d a, __m128d b) { - return _mm_move_ss(a, _mm_cmpneq_ps(a, b)); +#if defined(__aarch64__) || defined(_M_ARM64) + float64x2_t tmp = + vdivq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b)); + return vreinterpretq_m128d_f64( + vsetq_lane_f64(vgetq_lane_f64(vreinterpretq_f64_m128d(a), 1), tmp, 1)); +#else + return _mm_move_sd(a, _mm_div_pd(a, b)); +#endif } -// Compares for not greater than or equal. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/wsexys62(v=vs.100) -FORCE_INLINE __m128 _mm_cmpnge_ps(__m128 a, __m128 b) +// Extract a 16-bit integer from a, selected with imm8, and store the result in +// the lower element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_extract_epi16 +// FORCE_INLINE int _mm_extract_epi16(__m128i a, __constrange(0,8) int imm) +#define _mm_extract_epi16(a, imm) \ + vgetq_lane_u16(vreinterpretq_u16_m128i(a), (imm)) + +// Copy a to dst, and insert the 16-bit integer i into dst at the location +// specified by imm8. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_insert_epi16 +// FORCE_INLINE __m128i _mm_insert_epi16(__m128i a, int b, +// __constrange(0,8) int imm) +#define _mm_insert_epi16(a, b, imm) \ + vreinterpretq_m128i_s16( \ + vsetq_lane_s16((b), vreinterpretq_s16_m128i(a), (imm))) + +// Load 128-bits (composed of 2 packed double-precision (64-bit) floating-point +// elements) from memory into dst. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_pd +FORCE_INLINE __m128d _mm_load_pd(const double *p) { - return _mm_cmplt_ps(a, b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vld1q_f64(p)); +#else + const float *fp = (const float *) p; + float ALIGN_STRUCT(16) data[4] = {fp[0], fp[1], fp[2], fp[3]}; + return vreinterpretq_m128d_f32(vld1q_f32(data)); +#endif } -// Compares for not greater than or equal. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/fk2y80s8(v=vs.100) -FORCE_INLINE __m128 _mm_cmpnge_ss(__m128 a, __m128 b) +// Load a double-precision (64-bit) floating-point element from memory into both +// elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_pd1 +#define _mm_load_pd1 _mm_load1_pd + +// Load a double-precision (64-bit) floating-point element from memory into the +// lower of dst, and zero the upper element. mem_addr does not need to be +// aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_sd +FORCE_INLINE __m128d _mm_load_sd(const double *p) { - return _mm_cmplt_ss(a, b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vsetq_lane_f64(*p, vdupq_n_f64(0), 0)); +#else + const float *fp = (const float *) p; + float ALIGN_STRUCT(16) data[4] = {fp[0], fp[1], 0, 0}; + return vreinterpretq_m128d_f32(vld1q_f32(data)); +#endif } -// Compares for not greater than. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/d0xh7w0s(v=vs.100) -FORCE_INLINE __m128 _mm_cmpngt_ps(__m128 a, __m128 b) +// Load 128-bits of integer data from memory into dst. mem_addr must be aligned +// on a 16-byte boundary or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load_si128 +FORCE_INLINE __m128i _mm_load_si128(const __m128i *p) { - return _mm_cmple_ps(a, b); + return vreinterpretq_m128i_s32(vld1q_s32((const int32_t *) p)); } -// Compares for not greater than. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/z7x9ydwh(v=vs.100) -FORCE_INLINE __m128 _mm_cmpngt_ss(__m128 a, __m128 b) +// Load a double-precision (64-bit) floating-point element from memory into both +// elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_load1_pd +FORCE_INLINE __m128d _mm_load1_pd(const double *p) { - return _mm_cmple_ss(a, b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vld1q_dup_f64(p)); +#else + return vreinterpretq_m128d_s64(vdupq_n_s64(*(const int64_t *) p)); +#endif } -// Compares for not less than or equal. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/6a330kxw(v=vs.100) -FORCE_INLINE __m128 _mm_cmpnle_ps(__m128 a, __m128 b) +// Load a double-precision (64-bit) floating-point element from memory into the +// upper element of dst, and copy the lower element from a to dst. mem_addr does +// not need to be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadh_pd +FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double *p) { - return _mm_cmpgt_ps(a, b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vcombine_f64(vget_low_f64(vreinterpretq_f64_m128d(a)), vld1_f64(p))); +#else + return vreinterpretq_m128d_f32(vcombine_f32( + vget_low_f32(vreinterpretq_f32_m128d(a)), vld1_f32((const float *) p))); +#endif } -// Compares for not less than or equal. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/z7x9ydwh(v=vs.100) -FORCE_INLINE __m128 _mm_cmpnle_ss(__m128 a, __m128 b) +// Load 64-bit integer from memory into the first element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadl_epi64 +FORCE_INLINE __m128i _mm_loadl_epi64(__m128i const *p) { - return _mm_cmpgt_ss(a, b); + /* Load the lower 64 bits of the value pointed to by p into the + * lower 64 bits of the result, zeroing the upper 64 bits of the result. + */ + return vreinterpretq_m128i_s32( + vcombine_s32(vld1_s32((int32_t const *) p), vcreate_s32(0))); } -// Compares for not less than. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/4686bbdw(v=vs.100) -FORCE_INLINE __m128 _mm_cmpnlt_ps(__m128 a, __m128 b) +// Load a double-precision (64-bit) floating-point element from memory into the +// lower element of dst, and copy the upper element from a to dst. mem_addr does +// not need to be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadl_pd +FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double *p) { - return _mm_cmpge_ps(a, b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vcombine_f64(vld1_f64(p), vget_high_f64(vreinterpretq_f64_m128d(a)))); +#else + return vreinterpretq_m128d_f32( + vcombine_f32(vld1_f32((const float *) p), + vget_high_f32(vreinterpretq_f32_m128d(a)))); +#endif } -// Compares for not less than. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/56b9z2wf(v=vs.100) -FORCE_INLINE __m128 _mm_cmpnlt_ss(__m128 a, __m128 b) +// Load 2 double-precision (64-bit) floating-point elements from memory into dst +// in reverse order. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadr_pd +FORCE_INLINE __m128d _mm_loadr_pd(const double *p) { - return _mm_cmpge_ss(a, b); +#if defined(__aarch64__) || defined(_M_ARM64) + float64x2_t v = vld1q_f64(p); + return vreinterpretq_m128d_f64(vextq_f64(v, v, 1)); +#else + int64x2_t v = vld1q_s64((const int64_t *) p); + return vreinterpretq_m128d_s64(vextq_s64(v, v, 1)); +#endif } -// Compares the 16 signed or unsigned 8-bit integers in a and the 16 signed or -// unsigned 8-bit integers in b for equality. -// https://msdn.microsoft.com/en-us/library/windows/desktop/bz5xk21a(v=vs.90).aspx -FORCE_INLINE __m128i _mm_cmpeq_epi8(__m128i a, __m128i b) +// Loads two double-precision from unaligned memory, floating-point values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadu_pd +FORCE_INLINE __m128d _mm_loadu_pd(const double *p) { - return vreinterpretq_m128i_u8( - vceqq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); + return _mm_load_pd(p); } -// Compares the 8 signed or unsigned 16-bit integers in a and the 8 signed or -// unsigned 16-bit integers in b for equality. -// https://msdn.microsoft.com/en-us/library/2ay060te(v=vs.100).aspx -FORCE_INLINE __m128i _mm_cmpeq_epi16(__m128i a, __m128i b) +// Load 128-bits of integer data from memory into dst. mem_addr does not need to +// be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadu_si128 +FORCE_INLINE __m128i _mm_loadu_si128(const __m128i *p) { - return vreinterpretq_m128i_u16( - vceqq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); + return vreinterpretq_m128i_s32(vld1q_s32((const int32_t *) p)); } -// Compare packed 32-bit integers in a and b for equality, and store the results -// in dst -FORCE_INLINE __m128i _mm_cmpeq_epi32(__m128i a, __m128i b) +// Load unaligned 32-bit integer from memory into the first element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadu_si32 +FORCE_INLINE __m128i _mm_loadu_si32(const void *p) { - return vreinterpretq_m128i_u32( - vceqq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); + return vreinterpretq_m128i_s32( + vsetq_lane_s32(*(const int32_t *) p, vdupq_n_s32(0), 0)); } -// Compare packed 64-bit integers in a and b for equality, and store the results -// in dst -FORCE_INLINE __m128i _mm_cmpeq_epi64(__m128i a, __m128i b) +// Multiply packed signed 16-bit integers in a and b, producing intermediate +// signed 32-bit integers. Horizontally add adjacent pairs of intermediate +// 32-bit integers, and pack the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_madd_epi16 +FORCE_INLINE __m128i _mm_madd_epi16(__m128i a, __m128i b) { -#if defined(__aarch64__) - return vreinterpretq_m128i_u64( - vceqq_u64(vreinterpretq_u64_m128i(a), vreinterpretq_u64_m128i(b))); + int32x4_t low = vmull_s16(vget_low_s16(vreinterpretq_s16_m128i(a)), + vget_low_s16(vreinterpretq_s16_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) + int32x4_t high = + vmull_high_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b)); + + return vreinterpretq_m128i_s32(vpaddq_s32(low, high)); #else - // ARMv7 lacks vceqq_u64 - // (a == b) -> (a_lo == b_lo) && (a_hi == b_hi) - uint32x4_t cmp = - vceqq_u32(vreinterpretq_u32_m128i(a), vreinterpretq_u32_m128i(b)); - uint32x4_t swapped = vrev64q_u32(cmp); - return vreinterpretq_m128i_u32(vandq_u32(cmp, swapped)); + int32x4_t high = vmull_s16(vget_high_s16(vreinterpretq_s16_m128i(a)), + vget_high_s16(vreinterpretq_s16_m128i(b))); + + int32x2_t low_sum = vpadd_s32(vget_low_s32(low), vget_high_s32(low)); + int32x2_t high_sum = vpadd_s32(vget_low_s32(high), vget_high_s32(high)); + + return vreinterpretq_m128i_s32(vcombine_s32(low_sum, high_sum)); #endif } -// Compares the 16 signed 8-bit integers in a and the 16 signed 8-bit integers -// in b for lesser than. -// https://msdn.microsoft.com/en-us/library/windows/desktop/9s46csht(v=vs.90).aspx -FORCE_INLINE __m128i _mm_cmplt_epi8(__m128i a, __m128i b) +// Conditionally store 8-bit integer elements from a into memory using mask +// (elements are not stored when the highest bit is not set in the corresponding +// element) and a non-temporal memory hint. mem_addr does not need to be aligned +// on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maskmoveu_si128 +FORCE_INLINE void _mm_maskmoveu_si128(__m128i a, __m128i mask, char *mem_addr) { - return vreinterpretq_m128i_u8( - vcltq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); + int8x16_t shr_mask = vshrq_n_s8(vreinterpretq_s8_m128i(mask), 7); + __m128 b = _mm_load_ps((const float *) mem_addr); + int8x16_t masked = + vbslq_s8(vreinterpretq_u8_s8(shr_mask), vreinterpretq_s8_m128i(a), + vreinterpretq_s8_m128(b)); + vst1q_s8((int8_t *) mem_addr, masked); } -// Compares the 16 signed 8-bit integers in a and the 16 signed 8-bit integers -// in b for greater than. -// -// r0 := (a0 > b0) ? 0xff : 0x0 -// r1 := (a1 > b1) ? 0xff : 0x0 -// ... -// r15 := (a15 > b15) ? 0xff : 0x0 -// -// https://msdn.microsoft.com/zh-tw/library/wf45zt2b(v=vs.100).aspx -FORCE_INLINE __m128i _mm_cmpgt_epi8(__m128i a, __m128i b) +// Compare packed signed 16-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_epi16 +FORCE_INLINE __m128i _mm_max_epi16(__m128i a, __m128i b) { - return vreinterpretq_m128i_u8( - vcgtq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); + return vreinterpretq_m128i_s16( + vmaxq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } -// Compares the 8 signed 16-bit integers in a and the 8 signed 16-bit integers -// in b for less than. -// -// r0 := (a0 < b0) ? 0xffff : 0x0 -// r1 := (a1 < b1) ? 0xffff : 0x0 -// ... -// r7 := (a7 < b7) ? 0xffff : 0x0 -// -// https://technet.microsoft.com/en-us/library/t863edb2(v=vs.100).aspx -FORCE_INLINE __m128i _mm_cmplt_epi16(__m128i a, __m128i b) +// Compare packed unsigned 8-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_epu8 +FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) { - return vreinterpretq_m128i_u16( - vcltq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); + return vreinterpretq_m128i_u8( + vmaxq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); } -// Compares the 8 signed 16-bit integers in a and the 8 signed 16-bit integers -// in b for greater than. -// -// r0 := (a0 > b0) ? 0xffff : 0x0 -// r1 := (a1 > b1) ? 0xffff : 0x0 -// ... -// r7 := (a7 > b7) ? 0xffff : 0x0 -// -// https://technet.microsoft.com/en-us/library/xd43yfsa(v=vs.100).aspx -FORCE_INLINE __m128i _mm_cmpgt_epi16(__m128i a, __m128i b) +// Compare packed double-precision (64-bit) floating-point elements in a and b, +// and store packed maximum values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_pd +FORCE_INLINE __m128d _mm_max_pd(__m128d a, __m128d b) { - return vreinterpretq_m128i_u16( - vcgtq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) +#if SSE2NEON_PRECISE_MINMAX + float64x2_t _a = vreinterpretq_f64_m128d(a); + float64x2_t _b = vreinterpretq_f64_m128d(b); + return vreinterpretq_m128d_f64(vbslq_f64(vcgtq_f64(_a, _b), _a, _b)); +#else + return vreinterpretq_m128d_f64( + vmaxq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#endif +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) > (*(double *) &b0) ? a0 : b0; + d[1] = (*(double *) &a1) > (*(double *) &b1) ? a1 : b1; + + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b, store the maximum value in the lower element of dst, and copy the upper +// element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_sd +FORCE_INLINE __m128d _mm_max_sd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_max_pd(a, b)); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2] = {da[0] > db[0] ? da[0] : db[0], da[1]}; + return vreinterpretq_m128d_f32(vld1q_f32((float32_t *) c)); +#endif +} -// Compares the 4 signed 32-bit integers in a and the 4 signed 32-bit integers -// in b for less than. -// https://msdn.microsoft.com/en-us/library/vstudio/4ak0bf5d(v=vs.100).aspx -FORCE_INLINE __m128i _mm_cmplt_epi32(__m128i a, __m128i b) +// Compare packed signed 16-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_epi16 +FORCE_INLINE __m128i _mm_min_epi16(__m128i a, __m128i b) { - return vreinterpretq_m128i_u32( - vcltq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); + return vreinterpretq_m128i_s16( + vminq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); } -// Compares the 4 signed 32-bit integers in a and the 4 signed 32-bit integers -// in b for greater than. -// https://msdn.microsoft.com/en-us/library/vstudio/1s9f2z0y(v=vs.100).aspx -FORCE_INLINE __m128i _mm_cmpgt_epi32(__m128i a, __m128i b) +// Compare packed unsigned 8-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_epu8 +FORCE_INLINE __m128i _mm_min_epu8(__m128i a, __m128i b) { - return vreinterpretq_m128i_u32( - vcgtq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); + return vreinterpretq_m128i_u8( + vminq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); } -// Compares the 2 signed 64-bit integers in a and the 2 signed 64-bit integers -// in b for greater than. -FORCE_INLINE __m128i _mm_cmpgt_epi64(__m128i a, __m128i b) +// Compare packed double-precision (64-bit) floating-point elements in a and b, +// and store packed minimum values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_pd +FORCE_INLINE __m128d _mm_min_pd(__m128d a, __m128d b) { -#if defined(__aarch64__) - return vreinterpretq_m128i_u64( - vcgtq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b))); +#if defined(__aarch64__) || defined(_M_ARM64) +#if SSE2NEON_PRECISE_MINMAX + float64x2_t _a = vreinterpretq_f64_m128d(a); + float64x2_t _b = vreinterpretq_f64_m128d(b); + return vreinterpretq_m128d_f64(vbslq_f64(vcltq_f64(_a, _b), _a, _b)); #else - // ARMv7 lacks vcgtq_s64. - // This is based off of Clang's SSE2 polyfill: - // (a > b) -> ((a_hi > b_hi) || (a_lo > b_lo && a_hi == b_hi)) - - // Mask the sign bit out since we need a signed AND an unsigned comparison - // and it is ugly to try and split them. - int32x4_t mask = vreinterpretq_s32_s64(vdupq_n_s64(0x80000000ull)); - int32x4_t a_mask = veorq_s32(vreinterpretq_s32_m128i(a), mask); - int32x4_t b_mask = veorq_s32(vreinterpretq_s32_m128i(b), mask); - // Check if a > b - int64x2_t greater = vreinterpretq_s64_u32(vcgtq_s32(a_mask, b_mask)); - // Copy upper mask to lower mask - // a_hi > b_hi - int64x2_t gt_hi = vshrq_n_s64(greater, 63); - // Copy lower mask to upper mask - // a_lo > b_lo - int64x2_t gt_lo = vsliq_n_s64(greater, greater, 32); - // Compare for equality - int64x2_t equal = vreinterpretq_s64_u32(vceqq_s32(a_mask, b_mask)); - // Copy upper mask to lower mask - // a_hi == b_hi - int64x2_t eq_hi = vshrq_n_s64(equal, 63); - // a_hi > b_hi || (a_lo > b_lo && a_hi == b_hi) - int64x2_t ret = vorrq_s64(gt_hi, vandq_s64(gt_lo, eq_hi)); - return vreinterpretq_m128i_s64(ret); -#endif -} - -// Compares the four 32-bit floats in a and b to check if any values are NaN. -// Ordered compare between each value returns true for "orderable" and false for -// "not orderable" (NaN). -// https://msdn.microsoft.com/en-us/library/vstudio/0h9w00fx(v=vs.100).aspx see -// also: -// http://stackoverflow.com/questions/8627331/what-does-ordered-unordered-comparison-mean -// http://stackoverflow.com/questions/29349621/neon-isnanval-intrinsics -FORCE_INLINE __m128 _mm_cmpord_ps(__m128 a, __m128 b) -{ - // Note: NEON does not have ordered compare builtin - // Need to compare a eq a and b eq b to check for NaN - // Do AND of results to get final - uint32x4_t ceqaa = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); - uint32x4_t ceqbb = - vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); - return vreinterpretq_m128_u32(vandq_u32(ceqaa, ceqbb)); + return vreinterpretq_m128d_f64( + vminq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#endif +#else + uint64_t a0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(a)); + uint64_t a1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(a)); + uint64_t b0 = (uint64_t) vget_low_u64(vreinterpretq_u64_m128d(b)); + uint64_t b1 = (uint64_t) vget_high_u64(vreinterpretq_u64_m128d(b)); + uint64_t d[2]; + d[0] = (*(double *) &a0) < (*(double *) &b0) ? a0 : b0; + d[1] = (*(double *) &a1) < (*(double *) &b1) ? a1 : b1; + return vreinterpretq_m128d_u64(vld1q_u64(d)); +#endif } -// Compares for ordered. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/343t62da(v=vs.100) -FORCE_INLINE __m128 _mm_cmpord_ss(__m128 a, __m128 b) +// Compare the lower double-precision (64-bit) floating-point elements in a and +// b, store the minimum value in the lower element of dst, and copy the upper +// element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_sd +FORCE_INLINE __m128d _mm_min_sd(__m128d a, __m128d b) { - return _mm_move_ss(a, _mm_cmpord_ps(a, b)); +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_min_pd(a, b)); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2] = {da[0] < db[0] ? da[0] : db[0], da[1]}; + return vreinterpretq_m128d_f32(vld1q_f32((float32_t *) c)); +#endif } -// Compares for unordered. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/khy6fk1t(v=vs.100) -FORCE_INLINE __m128 _mm_cmpunord_ps(__m128 a, __m128 b) +// Copy the lower 64-bit integer in a to the lower element of dst, and zero the +// upper element. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_move_epi64 +FORCE_INLINE __m128i _mm_move_epi64(__m128i a) { - uint32x4_t f32a = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); - uint32x4_t f32b = - vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); - return vreinterpretq_m128_u32(vmvnq_u32(vandq_u32(f32a, f32b))); + return vreinterpretq_m128i_s64( + vsetq_lane_s64(0, vreinterpretq_s64_m128i(a), 1)); } -// Compares for unordered. -// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/2as2387b(v=vs.100) -FORCE_INLINE __m128 _mm_cmpunord_ss(__m128 a, __m128 b) +// Move the lower double-precision (64-bit) floating-point element from b to the +// lower element of dst, and copy the upper element from a to the upper element +// of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_move_sd +FORCE_INLINE __m128d _mm_move_sd(__m128d a, __m128d b) { - return _mm_move_ss(a, _mm_cmpunord_ps(a, b)); + return vreinterpretq_m128d_f32( + vcombine_f32(vget_low_f32(vreinterpretq_f32_m128d(b)), + vget_high_f32(vreinterpretq_f32_m128d(a)))); } -// Compares the lower single-precision floating point scalar values of a and b -// using a less than operation. : -// https://msdn.microsoft.com/en-us/library/2kwe606b(v=vs.90).aspx Important -// note!! The documentation on MSDN is incorrect! If either of the values is a -// NAN the docs say you will get a one, but in fact, it will return a zero!! -FORCE_INLINE int _mm_comilt_ss(__m128 a, __m128 b) +// Create mask from the most significant bit of each 8-bit element in a, and +// store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movemask_epi8 +FORCE_INLINE int _mm_movemask_epi8(__m128i a) { - uint32x4_t a_not_nan = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); - uint32x4_t b_not_nan = - vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); - uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); - uint32x4_t a_lt_b = - vcltq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); - return (vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_lt_b), 0) != 0) ? 1 : 0; + // Use increasingly wide shifts+adds to collect the sign bits + // together. + // Since the widening shifts would be rather confusing to follow in little + // endian, everything will be illustrated in big endian order instead. This + // has a different result - the bits would actually be reversed on a big + // endian machine. + + // Starting input (only half the elements are shown): + // 89 ff 1d c0 00 10 99 33 + uint8x16_t input = vreinterpretq_u8_m128i(a); + + // Shift out everything but the sign bits with an unsigned shift right. + // + // Bytes of the vector:: + // 89 ff 1d c0 00 10 99 33 + // \ \ \ \ \ \ \ \ high_bits = (uint16x4_t)(input >> 7) + // | | | | | | | | + // 01 01 00 01 00 00 01 00 + // + // Bits of first important lane(s): + // 10001001 (89) + // \______ + // | + // 00000001 (01) + uint16x8_t high_bits = vreinterpretq_u16_u8(vshrq_n_u8(input, 7)); + + // Merge the even lanes together with a 16-bit unsigned shift right + add. + // 'xx' represents garbage data which will be ignored in the final result. + // In the important bytes, the add functions like a binary OR. + // + // 01 01 00 01 00 00 01 00 + // \_ | \_ | \_ | \_ | paired16 = (uint32x4_t)(input + (input >> 7)) + // \| \| \| \| + // xx 03 xx 01 xx 00 xx 02 + // + // 00000001 00000001 (01 01) + // \_______ | + // \| + // xxxxxxxx xxxxxx11 (xx 03) + uint32x4_t paired16 = + vreinterpretq_u32_u16(vsraq_n_u16(high_bits, high_bits, 7)); + + // Repeat with a wider 32-bit shift + add. + // xx 03 xx 01 xx 00 xx 02 + // \____ | \____ | paired32 = (uint64x1_t)(paired16 + (paired16 >> + // 14)) + // \| \| + // xx xx xx 0d xx xx xx 02 + // + // 00000011 00000001 (03 01) + // \\_____ || + // '----.\|| + // xxxxxxxx xxxx1101 (xx 0d) + uint64x2_t paired32 = + vreinterpretq_u64_u32(vsraq_n_u32(paired16, paired16, 14)); + + // Last, an even wider 64-bit shift + add to get our result in the low 8 bit + // lanes. xx xx xx 0d xx xx xx 02 + // \_________ | paired64 = (uint8x8_t)(paired32 + (paired32 >> + // 28)) + // \| + // xx xx xx xx xx xx xx d2 + // + // 00001101 00000010 (0d 02) + // \ \___ | | + // '---. \| | + // xxxxxxxx 11010010 (xx d2) + uint8x16_t paired64 = + vreinterpretq_u8_u64(vsraq_n_u64(paired32, paired32, 28)); + + // Extract the low 8 bits from each 64-bit lane with 2 8-bit extracts. + // xx xx xx xx xx xx xx d2 + // || return paired64[0] + // d2 + // Note: Little endian would return the correct value 4b (01001011) instead. + return vgetq_lane_u8(paired64, 0) | ((int) vgetq_lane_u8(paired64, 8) << 8); } -// Compares the lower single-precision floating point scalar values of a and b -// using a greater than operation. : -// https://msdn.microsoft.com/en-us/library/b0738e0t(v=vs.100).aspx -FORCE_INLINE int _mm_comigt_ss(__m128 a, __m128 b) +// Set each bit of mask dst based on the most significant bit of the +// corresponding packed double-precision (64-bit) floating-point element in a. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movemask_pd +FORCE_INLINE int _mm_movemask_pd(__m128d a) { - // return vgetq_lane_u32(vcgtq_f32(vreinterpretq_f32_m128(a), - // vreinterpretq_f32_m128(b)), 0); - uint32x4_t a_not_nan = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); - uint32x4_t b_not_nan = - vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); - uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); - uint32x4_t a_gt_b = - vcgtq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); - return (vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_gt_b), 0) != 0) ? 1 : 0; + uint64x2_t input = vreinterpretq_u64_m128d(a); + uint64x2_t high_bits = vshrq_n_u64(input, 63); + return (int) (vgetq_lane_u64(high_bits, 0) | + (vgetq_lane_u64(high_bits, 1) << 1)); } -// Compares the lower single-precision floating point scalar values of a and b -// using a less than or equal operation. : -// https://msdn.microsoft.com/en-us/library/1w4t7c57(v=vs.90).aspx -FORCE_INLINE int _mm_comile_ss(__m128 a, __m128 b) +// Copy the lower 64-bit integer in a to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movepi64_pi64 +FORCE_INLINE __m64 _mm_movepi64_pi64(__m128i a) { - // return vgetq_lane_u32(vcleq_f32(vreinterpretq_f32_m128(a), - // vreinterpretq_f32_m128(b)), 0); - uint32x4_t a_not_nan = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); - uint32x4_t b_not_nan = - vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); - uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); - uint32x4_t a_le_b = - vcleq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); - return (vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_le_b), 0) != 0) ? 1 : 0; + return vreinterpret_m64_s64(vget_low_s64(vreinterpretq_s64_m128i(a))); } -// Compares the lower single-precision floating point scalar values of a and b -// using a greater than or equal operation. : -// https://msdn.microsoft.com/en-us/library/8t80des6(v=vs.100).aspx -FORCE_INLINE int _mm_comige_ss(__m128 a, __m128 b) +// Copy the 64-bit integer a to the lower element of dst, and zero the upper +// element. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movpi64_epi64 +FORCE_INLINE __m128i _mm_movpi64_epi64(__m64 a) { - // return vgetq_lane_u32(vcgeq_f32(vreinterpretq_f32_m128(a), - // vreinterpretq_f32_m128(b)), 0); - uint32x4_t a_not_nan = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); - uint32x4_t b_not_nan = - vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); - uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); - uint32x4_t a_ge_b = - vcgeq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); - return (vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_ge_b), 0) != 0) ? 1 : 0; + return vreinterpretq_m128i_s64( + vcombine_s64(vreinterpret_s64_m64(a), vdup_n_s64(0))); } -// Compares the lower single-precision floating point scalar values of a and b -// using an equality operation. : -// https://msdn.microsoft.com/en-us/library/93yx2h2b(v=vs.100).aspx -FORCE_INLINE int _mm_comieq_ss(__m128 a, __m128 b) +// Multiply the low unsigned 32-bit integers from each packed 64-bit element in +// a and b, and store the unsigned 64-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mul_epu32 +FORCE_INLINE __m128i _mm_mul_epu32(__m128i a, __m128i b) { - // return vgetq_lane_u32(vceqq_f32(vreinterpretq_f32_m128(a), - // vreinterpretq_f32_m128(b)), 0); - uint32x4_t a_not_nan = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); - uint32x4_t b_not_nan = - vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); - uint32x4_t a_and_b_not_nan = vandq_u32(a_not_nan, b_not_nan); - uint32x4_t a_eq_b = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b)); - return (vgetq_lane_u32(vandq_u32(a_and_b_not_nan, a_eq_b), 0) != 0) ? 1 : 0; + // vmull_u32 upcasts instead of masking, so we downcast. + uint32x2_t a_lo = vmovn_u64(vreinterpretq_u64_m128i(a)); + uint32x2_t b_lo = vmovn_u64(vreinterpretq_u64_m128i(b)); + return vreinterpretq_m128i_u64(vmull_u32(a_lo, b_lo)); } -// Compares the lower single-precision floating point scalar values of a and b -// using an inequality operation. : -// https://msdn.microsoft.com/en-us/library/bafh5e0a(v=vs.90).aspx -FORCE_INLINE int _mm_comineq_ss(__m128 a, __m128 b) +// Multiply packed double-precision (64-bit) floating-point elements in a and b, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mul_pd +FORCE_INLINE __m128d _mm_mul_pd(__m128d a, __m128d b) { - // return !vgetq_lane_u32(vceqq_f32(vreinterpretq_f32_m128(a), - // vreinterpretq_f32_m128(b)), 0); - uint32x4_t a_not_nan = - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a)); - uint32x4_t b_not_nan = - vceqq_f32(vreinterpretq_f32_m128(b), vreinterpretq_f32_m128(b)); - uint32x4_t a_or_b_nan = vmvnq_u32(vandq_u32(a_not_nan, b_not_nan)); - uint32x4_t a_neq_b = vmvnq_u32( - vceqq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); - return (vgetq_lane_u32(vorrq_u32(a_or_b_nan, a_neq_b), 0) != 0) ? 1 : 0; +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vmulq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] * db[0]; + c[1] = da[1] * db[1]; + return vld1q_f32((float32_t *) c); +#endif } -// according to the documentation, these intrinsics behave the same as the -// non-'u' versions. We'll just alias them here. -#define _mm_ucomilt_ss _mm_comilt_ss -#define _mm_ucomile_ss _mm_comile_ss -#define _mm_ucomigt_ss _mm_comigt_ss -#define _mm_ucomige_ss _mm_comige_ss -#define _mm_ucomieq_ss _mm_comieq_ss -#define _mm_ucomineq_ss _mm_comineq_ss +// Multiply the lower double-precision (64-bit) floating-point element in a and +// b, store the result in the lower element of dst, and copy the upper element +// from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=mm_mul_sd +FORCE_INLINE __m128d _mm_mul_sd(__m128d a, __m128d b) +{ + return _mm_move_sd(a, _mm_mul_pd(a, b)); +} -/* Conversions */ +// Multiply the low unsigned 32-bit integers from a and b, and store the +// unsigned 64-bit result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mul_su32 +FORCE_INLINE __m64 _mm_mul_su32(__m64 a, __m64 b) +{ + return vreinterpret_m64_u64(vget_low_u64( + vmull_u32(vreinterpret_u32_m64(a), vreinterpret_u32_m64(b)))); +} -// Convert packed signed 32-bit integers in b to packed single-precision -// (32-bit) floating-point elements, store the results in the lower 2 elements -// of dst, and copy the upper 2 packed elements from a to the upper elements of +// Multiply the packed signed 16-bit integers in a and b, producing intermediate +// 32-bit integers, and store the high 16 bits of the intermediate integers in // dst. -// -// dst[31:0] := Convert_Int32_To_FP32(b[31:0]) -// dst[63:32] := Convert_Int32_To_FP32(b[63:32]) -// dst[95:64] := a[95:64] -// dst[127:96] := a[127:96] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_pi2ps -FORCE_INLINE __m128 _mm_cvt_pi2ps(__m128 a, __m64 b) +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mulhi_epi16 +FORCE_INLINE __m128i _mm_mulhi_epi16(__m128i a, __m128i b) { - return vreinterpretq_m128_f32( - vcombine_f32(vcvt_f32_s32(vreinterpret_s32_m64(b)), - vget_high_f32(vreinterpretq_f32_m128(a)))); + /* FIXME: issue with large values because of result saturation */ + // int16x8_t ret = vqdmulhq_s16(vreinterpretq_s16_m128i(a), + // vreinterpretq_s16_m128i(b)); /* =2*a*b */ return + // vreinterpretq_m128i_s16(vshrq_n_s16(ret, 1)); + int16x4_t a3210 = vget_low_s16(vreinterpretq_s16_m128i(a)); + int16x4_t b3210 = vget_low_s16(vreinterpretq_s16_m128i(b)); + int32x4_t ab3210 = vmull_s16(a3210, b3210); /* 3333222211110000 */ + int16x4_t a7654 = vget_high_s16(vreinterpretq_s16_m128i(a)); + int16x4_t b7654 = vget_high_s16(vreinterpretq_s16_m128i(b)); + int32x4_t ab7654 = vmull_s16(a7654, b7654); /* 7777666655554444 */ + uint16x8x2_t r = + vuzpq_u16(vreinterpretq_u16_s32(ab3210), vreinterpretq_u16_s32(ab7654)); + return vreinterpretq_m128i_u16(r.val[1]); } -// Convert the signed 32-bit integer b to a single-precision (32-bit) -// floating-point element, store the result in the lower element of dst, and -// copy the upper 3 packed elements from a to the upper elements of dst. -// -// dst[31:0] := Convert_Int32_To_FP32(b[31:0]) -// dst[127:32] := a[127:32] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_si2ss -FORCE_INLINE __m128 _mm_cvt_si2ss(__m128 a, int b) +// Multiply the packed unsigned 16-bit integers in a and b, producing +// intermediate 32-bit integers, and store the high 16 bits of the intermediate +// integers in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mulhi_epu16 +FORCE_INLINE __m128i _mm_mulhi_epu16(__m128i a, __m128i b) { - return vreinterpretq_m128_f32( - vsetq_lane_f32((float) b, vreinterpretq_f32_m128(a), 0)); + uint16x4_t a3210 = vget_low_u16(vreinterpretq_u16_m128i(a)); + uint16x4_t b3210 = vget_low_u16(vreinterpretq_u16_m128i(b)); + uint32x4_t ab3210 = vmull_u16(a3210, b3210); +#if defined(__aarch64__) || defined(_M_ARM64) + uint32x4_t ab7654 = + vmull_high_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b)); + uint16x8_t r = vuzp2q_u16(vreinterpretq_u16_u32(ab3210), + vreinterpretq_u16_u32(ab7654)); + return vreinterpretq_m128i_u16(r); +#else + uint16x4_t a7654 = vget_high_u16(vreinterpretq_u16_m128i(a)); + uint16x4_t b7654 = vget_high_u16(vreinterpretq_u16_m128i(b)); + uint32x4_t ab7654 = vmull_u16(a7654, b7654); + uint16x8x2_t r = + vuzpq_u16(vreinterpretq_u16_u32(ab3210), vreinterpretq_u16_u32(ab7654)); + return vreinterpretq_m128i_u16(r.val[1]); +#endif } -// Convert the signed 32-bit integer b to a single-precision (32-bit) -// floating-point element, store the result in the lower element of dst, and -// copy the upper 3 packed elements from a to the upper elements of dst. -// -// dst[31:0] := Convert_Int32_To_FP32(b[31:0]) -// dst[127:32] := a[127:32] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi32_ss -#define _mm_cvtsi32_ss(a, b) _mm_cvt_si2ss(a, b) +// Multiply the packed 16-bit integers in a and b, producing intermediate 32-bit +// integers, and store the low 16 bits of the intermediate integers in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mullo_epi16 +FORCE_INLINE __m128i _mm_mullo_epi16(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s16( + vmulq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +} -// Convert the signed 64-bit integer b to a single-precision (32-bit) -// floating-point element, store the result in the lower element of dst, and -// copy the upper 3 packed elements from a to the upper elements of dst. -// -// dst[31:0] := Convert_Int64_To_FP32(b[63:0]) -// dst[127:32] := a[127:32] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi64_ss -FORCE_INLINE __m128 _mm_cvtsi64_ss(__m128 a, int64_t b) +// Compute the bitwise OR of packed double-precision (64-bit) floating-point +// elements in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=mm_or_pd +FORCE_INLINE __m128d _mm_or_pd(__m128d a, __m128d b) { - return vreinterpretq_m128_f32( - vsetq_lane_f32((float) b, vreinterpretq_f32_m128(a), 0)); + return vreinterpretq_m128d_s64( + vorrq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); } -// Convert the lower single-precision (32-bit) floating-point element in a to a -// 32-bit integer, and store the result in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_ss2si -FORCE_INLINE int _mm_cvt_ss2si(__m128 a) -{ -#if defined(__aarch64__) - return vgetq_lane_s32(vcvtnq_s32_f32(vreinterpretq_f32_m128(a)), 0); -#else - float32_t data = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); - float32_t diff = data - floor(data); - if (diff > 0.5) - return (int32_t) ceil(data); - if (diff == 0.5) { - int32_t f = (int32_t) floor(data); - int32_t c = (int32_t) ceil(data); - return c & 1 ? f : c; - } - return (int32_t) floor(data); -#endif +// Compute the bitwise OR of 128 bits (representing integer data) in a and b, +// and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_or_si128 +FORCE_INLINE __m128i _mm_or_si128(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s32( + vorrq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +} + +// Convert packed signed 16-bit integers from a and b to packed 8-bit integers +// using signed saturation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packs_epi16 +FORCE_INLINE __m128i _mm_packs_epi16(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s8( + vcombine_s8(vqmovn_s16(vreinterpretq_s16_m128i(a)), + vqmovn_s16(vreinterpretq_s16_m128i(b)))); +} + +// Convert packed signed 32-bit integers from a and b to packed 16-bit integers +// using signed saturation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packs_epi32 +FORCE_INLINE __m128i _mm_packs_epi32(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s16( + vcombine_s16(vqmovn_s32(vreinterpretq_s32_m128i(a)), + vqmovn_s32(vreinterpretq_s32_m128i(b)))); +} + +// Convert packed signed 16-bit integers from a and b to packed 8-bit integers +// using unsigned saturation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packus_epi16 +FORCE_INLINE __m128i _mm_packus_epi16(const __m128i a, const __m128i b) +{ + return vreinterpretq_m128i_u8( + vcombine_u8(vqmovun_s16(vreinterpretq_s16_m128i(a)), + vqmovun_s16(vreinterpretq_s16_m128i(b)))); +} + +// Pause the processor. This is typically used in spin-wait loops and depending +// on the x86 processor typical values are in the 40-100 cycle range. The +// 'yield' instruction isn't a good fit because it's effectively a nop on most +// Arm cores. Experience with several databases has shown has shown an 'isb' is +// a reasonable approximation. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_pause +FORCE_INLINE void _mm_pause(void) +{ +#if defined(_MSC_VER) + __isb(_ARM64_BARRIER_SY); +#else + __asm__ __volatile__("isb\n"); +#endif +} + +// Compute the absolute differences of packed unsigned 8-bit integers in a and +// b, then horizontally sum each consecutive 8 differences to produce two +// unsigned 16-bit integers, and pack these unsigned 16-bit integers in the low +// 16 bits of 64-bit elements in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sad_epu8 +FORCE_INLINE __m128i _mm_sad_epu8(__m128i a, __m128i b) +{ + uint16x8_t t = vpaddlq_u8(vabdq_u8((uint8x16_t) a, (uint8x16_t) b)); + return vreinterpretq_m128i_u64(vpaddlq_u32(vpaddlq_u16(t))); +} + +// Set packed 16-bit integers in dst with the supplied values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_epi16 +FORCE_INLINE __m128i _mm_set_epi16(short i7, + short i6, + short i5, + short i4, + short i3, + short i2, + short i1, + short i0) +{ + int16_t ALIGN_STRUCT(16) data[8] = {i0, i1, i2, i3, i4, i5, i6, i7}; + return vreinterpretq_m128i_s16(vld1q_s16(data)); +} + +// Set packed 32-bit integers in dst with the supplied values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_epi32 +FORCE_INLINE __m128i _mm_set_epi32(int i3, int i2, int i1, int i0) +{ + int32_t ALIGN_STRUCT(16) data[4] = {i0, i1, i2, i3}; + return vreinterpretq_m128i_s32(vld1q_s32(data)); +} + +// Set packed 64-bit integers in dst with the supplied values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_epi64 +FORCE_INLINE __m128i _mm_set_epi64(__m64 i1, __m64 i2) +{ + return _mm_set_epi64x(vget_lane_s64(i1, 0), vget_lane_s64(i2, 0)); +} + +// Set packed 64-bit integers in dst with the supplied values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_epi64x +FORCE_INLINE __m128i _mm_set_epi64x(int64_t i1, int64_t i2) +{ + return vreinterpretq_m128i_s64( + vcombine_s64(vcreate_s64(i2), vcreate_s64(i1))); +} + +// Set packed 8-bit integers in dst with the supplied values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_epi8 +FORCE_INLINE __m128i _mm_set_epi8(signed char b15, + signed char b14, + signed char b13, + signed char b12, + signed char b11, + signed char b10, + signed char b9, + signed char b8, + signed char b7, + signed char b6, + signed char b5, + signed char b4, + signed char b3, + signed char b2, + signed char b1, + signed char b0) +{ + int8_t ALIGN_STRUCT(16) + data[16] = {(int8_t) b0, (int8_t) b1, (int8_t) b2, (int8_t) b3, + (int8_t) b4, (int8_t) b5, (int8_t) b6, (int8_t) b7, + (int8_t) b8, (int8_t) b9, (int8_t) b10, (int8_t) b11, + (int8_t) b12, (int8_t) b13, (int8_t) b14, (int8_t) b15}; + return (__m128i) vld1q_s8(data); +} + +// Set packed double-precision (64-bit) floating-point elements in dst with the +// supplied values. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_pd +FORCE_INLINE __m128d _mm_set_pd(double e1, double e0) +{ + double ALIGN_STRUCT(16) data[2] = {e0, e1}; +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vld1q_f64((float64_t *) data)); +#else + return vreinterpretq_m128d_f32(vld1q_f32((float32_t *) data)); +#endif +} + +// Broadcast double-precision (64-bit) floating-point value a to all elements of +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_pd1 +#define _mm_set_pd1 _mm_set1_pd + +// Copy double-precision (64-bit) floating-point element a to the lower element +// of dst, and zero the upper element. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set_sd +FORCE_INLINE __m128d _mm_set_sd(double a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vsetq_lane_f64(a, vdupq_n_f64(0), 0)); +#else + return _mm_set_pd(0, a); +#endif +} + +// Broadcast 16-bit integer a to all elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set1_epi16 +FORCE_INLINE __m128i _mm_set1_epi16(short w) +{ + return vreinterpretq_m128i_s16(vdupq_n_s16(w)); +} + +// Broadcast 32-bit integer a to all elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set1_epi32 +FORCE_INLINE __m128i _mm_set1_epi32(int _i) +{ + return vreinterpretq_m128i_s32(vdupq_n_s32(_i)); +} + +// Broadcast 64-bit integer a to all elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set1_epi64 +FORCE_INLINE __m128i _mm_set1_epi64(__m64 _i) +{ + return vreinterpretq_m128i_s64(vdupq_lane_s64(_i, 0)); +} + +// Broadcast 64-bit integer a to all elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set1_epi64x +FORCE_INLINE __m128i _mm_set1_epi64x(int64_t _i) +{ + return vreinterpretq_m128i_s64(vdupq_n_s64(_i)); +} + +// Broadcast 8-bit integer a to all elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set1_epi8 +FORCE_INLINE __m128i _mm_set1_epi8(signed char w) +{ + return vreinterpretq_m128i_s8(vdupq_n_s8(w)); +} + +// Broadcast double-precision (64-bit) floating-point value a to all elements of +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_set1_pd +FORCE_INLINE __m128d _mm_set1_pd(double d) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vdupq_n_f64(d)); +#else + return vreinterpretq_m128d_s64(vdupq_n_s64(*(int64_t *) &d)); +#endif +} + +// Set packed 16-bit integers in dst with the supplied values in reverse order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setr_epi16 +FORCE_INLINE __m128i _mm_setr_epi16(short w0, + short w1, + short w2, + short w3, + short w4, + short w5, + short w6, + short w7) +{ + int16_t ALIGN_STRUCT(16) data[8] = {w0, w1, w2, w3, w4, w5, w6, w7}; + return vreinterpretq_m128i_s16(vld1q_s16((int16_t *) data)); +} + +// Set packed 32-bit integers in dst with the supplied values in reverse order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setr_epi32 +FORCE_INLINE __m128i _mm_setr_epi32(int i3, int i2, int i1, int i0) +{ + int32_t ALIGN_STRUCT(16) data[4] = {i3, i2, i1, i0}; + return vreinterpretq_m128i_s32(vld1q_s32(data)); +} + +// Set packed 64-bit integers in dst with the supplied values in reverse order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setr_epi64 +FORCE_INLINE __m128i _mm_setr_epi64(__m64 e1, __m64 e0) +{ + return vreinterpretq_m128i_s64(vcombine_s64(e1, e0)); +} + +// Set packed 8-bit integers in dst with the supplied values in reverse order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setr_epi8 +FORCE_INLINE __m128i _mm_setr_epi8(signed char b0, + signed char b1, + signed char b2, + signed char b3, + signed char b4, + signed char b5, + signed char b6, + signed char b7, + signed char b8, + signed char b9, + signed char b10, + signed char b11, + signed char b12, + signed char b13, + signed char b14, + signed char b15) +{ + int8_t ALIGN_STRUCT(16) + data[16] = {(int8_t) b0, (int8_t) b1, (int8_t) b2, (int8_t) b3, + (int8_t) b4, (int8_t) b5, (int8_t) b6, (int8_t) b7, + (int8_t) b8, (int8_t) b9, (int8_t) b10, (int8_t) b11, + (int8_t) b12, (int8_t) b13, (int8_t) b14, (int8_t) b15}; + return (__m128i) vld1q_s8(data); +} + +// Set packed double-precision (64-bit) floating-point elements in dst with the +// supplied values in reverse order. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setr_pd +FORCE_INLINE __m128d _mm_setr_pd(double e1, double e0) +{ + return _mm_set_pd(e0, e1); +} + +// Return vector of type __m128d with all elements set to zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setzero_pd +FORCE_INLINE __m128d _mm_setzero_pd(void) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vdupq_n_f64(0)); +#else + return vreinterpretq_m128d_f32(vdupq_n_f32(0)); +#endif +} + +// Return vector of type __m128i with all elements set to zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_setzero_si128 +FORCE_INLINE __m128i _mm_setzero_si128(void) +{ + return vreinterpretq_m128i_s32(vdupq_n_s32(0)); +} + +// Shuffle 32-bit integers in a using the control in imm8, and store the results +// in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shuffle_epi32 +// FORCE_INLINE __m128i _mm_shuffle_epi32(__m128i a, +// __constrange(0,255) int imm) +#if defined(_sse2neon_shuffle) +#define _mm_shuffle_epi32(a, imm) \ + __extension__({ \ + int32x4_t _input = vreinterpretq_s32_m128i(a); \ + int32x4_t _shuf = \ + vshuffleq_s32(_input, _input, (imm) & (0x3), ((imm) >> 2) & 0x3, \ + ((imm) >> 4) & 0x3, ((imm) >> 6) & 0x3); \ + vreinterpretq_m128i_s32(_shuf); \ + }) +#else // generic +#define _mm_shuffle_epi32(a, imm) \ + _sse2neon_define1( \ + __m128i, a, __m128i ret; switch (imm) { \ + case _MM_SHUFFLE(1, 0, 3, 2): \ + ret = _mm_shuffle_epi_1032(_a); \ + break; \ + case _MM_SHUFFLE(2, 3, 0, 1): \ + ret = _mm_shuffle_epi_2301(_a); \ + break; \ + case _MM_SHUFFLE(0, 3, 2, 1): \ + ret = _mm_shuffle_epi_0321(_a); \ + break; \ + case _MM_SHUFFLE(2, 1, 0, 3): \ + ret = _mm_shuffle_epi_2103(_a); \ + break; \ + case _MM_SHUFFLE(1, 0, 1, 0): \ + ret = _mm_shuffle_epi_1010(_a); \ + break; \ + case _MM_SHUFFLE(1, 0, 0, 1): \ + ret = _mm_shuffle_epi_1001(_a); \ + break; \ + case _MM_SHUFFLE(0, 1, 0, 1): \ + ret = _mm_shuffle_epi_0101(_a); \ + break; \ + case _MM_SHUFFLE(2, 2, 1, 1): \ + ret = _mm_shuffle_epi_2211(_a); \ + break; \ + case _MM_SHUFFLE(0, 1, 2, 2): \ + ret = _mm_shuffle_epi_0122(_a); \ + break; \ + case _MM_SHUFFLE(3, 3, 3, 2): \ + ret = _mm_shuffle_epi_3332(_a); \ + break; \ + case _MM_SHUFFLE(0, 0, 0, 0): \ + ret = _mm_shuffle_epi32_splat(_a, 0); \ + break; \ + case _MM_SHUFFLE(1, 1, 1, 1): \ + ret = _mm_shuffle_epi32_splat(_a, 1); \ + break; \ + case _MM_SHUFFLE(2, 2, 2, 2): \ + ret = _mm_shuffle_epi32_splat(_a, 2); \ + break; \ + case _MM_SHUFFLE(3, 3, 3, 3): \ + ret = _mm_shuffle_epi32_splat(_a, 3); \ + break; \ + default: \ + ret = _mm_shuffle_epi32_default(_a, (imm)); \ + break; \ + } _sse2neon_return(ret);) +#endif + +// Shuffle double-precision (64-bit) floating-point elements using the control +// in imm8, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shuffle_pd +#ifdef _sse2neon_shuffle +#define _mm_shuffle_pd(a, b, imm8) \ + vreinterpretq_m128d_s64( \ + vshuffleq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b), \ + imm8 & 0x1, ((imm8 & 0x2) >> 1) + 2)) +#else +#define _mm_shuffle_pd(a, b, imm8) \ + _mm_castsi128_pd(_mm_set_epi64x( \ + vgetq_lane_s64(vreinterpretq_s64_m128d(b), (imm8 & 0x2) >> 1), \ + vgetq_lane_s64(vreinterpretq_s64_m128d(a), imm8 & 0x1))) +#endif + +// FORCE_INLINE __m128i _mm_shufflehi_epi16(__m128i a, +// __constrange(0,255) int imm) +#if defined(_sse2neon_shuffle) +#define _mm_shufflehi_epi16(a, imm) \ + __extension__({ \ + int16x8_t _input = vreinterpretq_s16_m128i(a); \ + int16x8_t _shuf = \ + vshuffleq_s16(_input, _input, 0, 1, 2, 3, ((imm) & (0x3)) + 4, \ + (((imm) >> 2) & 0x3) + 4, (((imm) >> 4) & 0x3) + 4, \ + (((imm) >> 6) & 0x3) + 4); \ + vreinterpretq_m128i_s16(_shuf); \ + }) +#else // generic +#define _mm_shufflehi_epi16(a, imm) _mm_shufflehi_epi16_function((a), (imm)) +#endif + +// FORCE_INLINE __m128i _mm_shufflelo_epi16(__m128i a, +// __constrange(0,255) int imm) +#if defined(_sse2neon_shuffle) +#define _mm_shufflelo_epi16(a, imm) \ + __extension__({ \ + int16x8_t _input = vreinterpretq_s16_m128i(a); \ + int16x8_t _shuf = vshuffleq_s16( \ + _input, _input, ((imm) & (0x3)), (((imm) >> 2) & 0x3), \ + (((imm) >> 4) & 0x3), (((imm) >> 6) & 0x3), 4, 5, 6, 7); \ + vreinterpretq_m128i_s16(_shuf); \ + }) +#else // generic +#define _mm_shufflelo_epi16(a, imm) _mm_shufflelo_epi16_function((a), (imm)) +#endif + +// Shift packed 16-bit integers in a left by count while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sll_epi16 +FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) +{ + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (_sse2neon_unlikely(c & ~15)) + return _mm_setzero_si128(); + + int16x8_t vc = vdupq_n_s16((int16_t) c); + return vreinterpretq_m128i_s16(vshlq_s16(vreinterpretq_s16_m128i(a), vc)); +} + +// Shift packed 32-bit integers in a left by count while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sll_epi32 +FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) +{ + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (_sse2neon_unlikely(c & ~31)) + return _mm_setzero_si128(); + + int32x4_t vc = vdupq_n_s32((int32_t) c); + return vreinterpretq_m128i_s32(vshlq_s32(vreinterpretq_s32_m128i(a), vc)); +} + +// Shift packed 64-bit integers in a left by count while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sll_epi64 +FORCE_INLINE __m128i _mm_sll_epi64(__m128i a, __m128i count) +{ + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (_sse2neon_unlikely(c & ~63)) + return _mm_setzero_si128(); + + int64x2_t vc = vdupq_n_s64((int64_t) c); + return vreinterpretq_m128i_s64(vshlq_s64(vreinterpretq_s64_m128i(a), vc)); +} + +// Shift packed 16-bit integers in a left by imm8 while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_slli_epi16 +FORCE_INLINE __m128i _mm_slli_epi16(__m128i a, int imm) +{ + if (_sse2neon_unlikely(imm & ~15)) + return _mm_setzero_si128(); + return vreinterpretq_m128i_s16( + vshlq_s16(vreinterpretq_s16_m128i(a), vdupq_n_s16(imm))); +} + +// Shift packed 32-bit integers in a left by imm8 while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_slli_epi32 +FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, int imm) +{ + if (_sse2neon_unlikely(imm & ~31)) + return _mm_setzero_si128(); + return vreinterpretq_m128i_s32( + vshlq_s32(vreinterpretq_s32_m128i(a), vdupq_n_s32(imm))); +} + +// Shift packed 64-bit integers in a left by imm8 while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_slli_epi64 +FORCE_INLINE __m128i _mm_slli_epi64(__m128i a, int imm) +{ + if (_sse2neon_unlikely(imm & ~63)) + return _mm_setzero_si128(); + return vreinterpretq_m128i_s64( + vshlq_s64(vreinterpretq_s64_m128i(a), vdupq_n_s64(imm))); +} + +// Shift a left by imm8 bytes while shifting in zeros, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_slli_si128 +#define _mm_slli_si128(a, imm) \ + _sse2neon_define1( \ + __m128i, a, int8x16_t ret; \ + if (_sse2neon_unlikely(imm == 0)) ret = vreinterpretq_s8_m128i(_a); \ + else if (_sse2neon_unlikely((imm) & ~15)) ret = vdupq_n_s8(0); \ + else ret = vextq_s8(vdupq_n_s8(0), vreinterpretq_s8_m128i(_a), \ + ((imm <= 0 || imm > 15) ? 0 : (16 - imm))); \ + _sse2neon_return(vreinterpretq_m128i_s8(ret));) + +// Compute the square root of packed double-precision (64-bit) floating-point +// elements in a, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_pd +FORCE_INLINE __m128d _mm_sqrt_pd(__m128d a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vsqrtq_f64(vreinterpretq_f64_m128d(a))); +#else + double a0 = sqrt(((double *) &a)[0]); + double a1 = sqrt(((double *) &a)[1]); + return _mm_set_pd(a1, a0); +#endif +} + +// Compute the square root of the lower double-precision (64-bit) floating-point +// element in b, store the result in the lower element of dst, and copy the +// upper element from a to the upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_sd +FORCE_INLINE __m128d _mm_sqrt_sd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return _mm_move_sd(a, _mm_sqrt_pd(b)); +#else + return _mm_set_pd(((double *) &a)[1], sqrt(((double *) &b)[0])); +#endif +} + +// Shift packed 16-bit integers in a right by count while shifting in sign bits, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sra_epi16 +FORCE_INLINE __m128i _mm_sra_epi16(__m128i a, __m128i count) +{ + int64_t c = vgetq_lane_s64(count, 0); + if (_sse2neon_unlikely(c & ~15)) + return _mm_cmplt_epi16(a, _mm_setzero_si128()); + return vreinterpretq_m128i_s16( + vshlq_s16((int16x8_t) a, vdupq_n_s16((int) -c))); +} + +// Shift packed 32-bit integers in a right by count while shifting in sign bits, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sra_epi32 +FORCE_INLINE __m128i _mm_sra_epi32(__m128i a, __m128i count) +{ + int64_t c = vgetq_lane_s64(count, 0); + if (_sse2neon_unlikely(c & ~31)) + return _mm_cmplt_epi32(a, _mm_setzero_si128()); + return vreinterpretq_m128i_s32( + vshlq_s32((int32x4_t) a, vdupq_n_s32((int) -c))); +} + +// Shift packed 16-bit integers in a right by imm8 while shifting in sign +// bits, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srai_epi16 +FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int imm) +{ + const int count = (imm & ~15) ? 15 : imm; + return (__m128i) vshlq_s16((int16x8_t) a, vdupq_n_s16(-count)); +} + +// Shift packed 32-bit integers in a right by imm8 while shifting in sign bits, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srai_epi32 +// FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, __constrange(0,255) int imm) +#define _mm_srai_epi32(a, imm) \ + _sse2neon_define0( \ + __m128i, a, __m128i ret; if (_sse2neon_unlikely((imm) == 0)) { \ + ret = _a; \ + } else if (_sse2neon_likely(0 < (imm) && (imm) < 32)) { \ + ret = vreinterpretq_m128i_s32( \ + vshlq_s32(vreinterpretq_s32_m128i(_a), vdupq_n_s32(-(imm)))); \ + } else { \ + ret = vreinterpretq_m128i_s32( \ + vshrq_n_s32(vreinterpretq_s32_m128i(_a), 31)); \ + } _sse2neon_return(ret);) + +// Shift packed 16-bit integers in a right by count while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srl_epi16 +FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) +{ + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (_sse2neon_unlikely(c & ~15)) + return _mm_setzero_si128(); + + int16x8_t vc = vdupq_n_s16(-(int16_t) c); + return vreinterpretq_m128i_u16(vshlq_u16(vreinterpretq_u16_m128i(a), vc)); +} + +// Shift packed 32-bit integers in a right by count while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srl_epi32 +FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) +{ + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (_sse2neon_unlikely(c & ~31)) + return _mm_setzero_si128(); + + int32x4_t vc = vdupq_n_s32(-(int32_t) c); + return vreinterpretq_m128i_u32(vshlq_u32(vreinterpretq_u32_m128i(a), vc)); +} + +// Shift packed 64-bit integers in a right by count while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srl_epi64 +FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) +{ + uint64_t c = vreinterpretq_nth_u64_m128i(count, 0); + if (_sse2neon_unlikely(c & ~63)) + return _mm_setzero_si128(); + + int64x2_t vc = vdupq_n_s64(-(int64_t) c); + return vreinterpretq_m128i_u64(vshlq_u64(vreinterpretq_u64_m128i(a), vc)); +} + +// Shift packed 16-bit integers in a right by imm8 while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srli_epi16 +#define _mm_srli_epi16(a, imm) \ + _sse2neon_define0( \ + __m128i, a, __m128i ret; if (_sse2neon_unlikely((imm) & ~15)) { \ + ret = _mm_setzero_si128(); \ + } else { \ + ret = vreinterpretq_m128i_u16( \ + vshlq_u16(vreinterpretq_u16_m128i(_a), vdupq_n_s16(-(imm)))); \ + } _sse2neon_return(ret);) + +// Shift packed 32-bit integers in a right by imm8 while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srli_epi32 +// FORCE_INLINE __m128i _mm_srli_epi32(__m128i a, __constrange(0,255) int imm) +#define _mm_srli_epi32(a, imm) \ + _sse2neon_define0( \ + __m128i, a, __m128i ret; if (_sse2neon_unlikely((imm) & ~31)) { \ + ret = _mm_setzero_si128(); \ + } else { \ + ret = vreinterpretq_m128i_u32( \ + vshlq_u32(vreinterpretq_u32_m128i(_a), vdupq_n_s32(-(imm)))); \ + } _sse2neon_return(ret);) + +// Shift packed 64-bit integers in a right by imm8 while shifting in zeros, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srli_epi64 +#define _mm_srli_epi64(a, imm) \ + _sse2neon_define0( \ + __m128i, a, __m128i ret; if (_sse2neon_unlikely((imm) & ~63)) { \ + ret = _mm_setzero_si128(); \ + } else { \ + ret = vreinterpretq_m128i_u64( \ + vshlq_u64(vreinterpretq_u64_m128i(_a), vdupq_n_s64(-(imm)))); \ + } _sse2neon_return(ret);) + +// Shift a right by imm8 bytes while shifting in zeros, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_srli_si128 +#define _mm_srli_si128(a, imm) \ + _sse2neon_define1( \ + __m128i, a, int8x16_t ret; \ + if (_sse2neon_unlikely((imm) & ~15)) ret = vdupq_n_s8(0); \ + else ret = vextq_s8(vreinterpretq_s8_m128i(_a), vdupq_n_s8(0), \ + (imm > 15 ? 0 : imm)); \ + _sse2neon_return(vreinterpretq_m128i_s8(ret));) + +// Store 128-bits (composed of 2 packed double-precision (64-bit) floating-point +// elements) from a into memory. mem_addr must be aligned on a 16-byte boundary +// or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_pd +FORCE_INLINE void _mm_store_pd(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + vst1q_f64((float64_t *) mem_addr, vreinterpretq_f64_m128d(a)); +#else + vst1q_f32((float32_t *) mem_addr, vreinterpretq_f32_m128d(a)); +#endif +} + +// Store the lower double-precision (64-bit) floating-point element from a into +// 2 contiguous elements in memory. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_pd1 +FORCE_INLINE void _mm_store_pd1(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + float64x1_t a_low = vget_low_f64(vreinterpretq_f64_m128d(a)); + vst1q_f64((float64_t *) mem_addr, + vreinterpretq_f64_m128d(vcombine_f64(a_low, a_low))); +#else + float32x2_t a_low = vget_low_f32(vreinterpretq_f32_m128d(a)); + vst1q_f32((float32_t *) mem_addr, + vreinterpretq_f32_m128d(vcombine_f32(a_low, a_low))); +#endif +} + +// Store the lower double-precision (64-bit) floating-point element from a into +// memory. mem_addr does not need to be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=mm_store_sd +FORCE_INLINE void _mm_store_sd(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + vst1_f64((float64_t *) mem_addr, vget_low_f64(vreinterpretq_f64_m128d(a))); +#else + vst1_u64((uint64_t *) mem_addr, vget_low_u64(vreinterpretq_u64_m128d(a))); +#endif +} + +// Store 128-bits of integer data from a into memory. mem_addr must be aligned +// on a 16-byte boundary or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_store_si128 +FORCE_INLINE void _mm_store_si128(__m128i *p, __m128i a) +{ + vst1q_s32((int32_t *) p, vreinterpretq_s32_m128i(a)); +} + +// Store the lower double-precision (64-bit) floating-point element from a into +// 2 contiguous elements in memory. mem_addr must be aligned on a 16-byte +// boundary or a general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#expand=9,526,5601&text=_mm_store1_pd +#define _mm_store1_pd _mm_store_pd1 + +// Store the upper double-precision (64-bit) floating-point element from a into +// memory. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeh_pd +FORCE_INLINE void _mm_storeh_pd(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + vst1_f64((float64_t *) mem_addr, vget_high_f64(vreinterpretq_f64_m128d(a))); +#else + vst1_f32((float32_t *) mem_addr, vget_high_f32(vreinterpretq_f32_m128d(a))); +#endif +} + +// Store 64-bit integer from the first element of a into memory. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storel_epi64 +FORCE_INLINE void _mm_storel_epi64(__m128i *a, __m128i b) +{ + vst1_u64((uint64_t *) a, vget_low_u64(vreinterpretq_u64_m128i(b))); +} + +// Store the lower double-precision (64-bit) floating-point element from a into +// memory. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storel_pd +FORCE_INLINE void _mm_storel_pd(double *mem_addr, __m128d a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + vst1_f64((float64_t *) mem_addr, vget_low_f64(vreinterpretq_f64_m128d(a))); +#else + vst1_f32((float32_t *) mem_addr, vget_low_f32(vreinterpretq_f32_m128d(a))); +#endif +} + +// Store 2 double-precision (64-bit) floating-point elements from a into memory +// in reverse order. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storer_pd +FORCE_INLINE void _mm_storer_pd(double *mem_addr, __m128d a) +{ + float32x4_t f = vreinterpretq_f32_m128d(a); + _mm_store_pd(mem_addr, vreinterpretq_m128d_f32(vextq_f32(f, f, 2))); +} + +// Store 128-bits (composed of 2 packed double-precision (64-bit) floating-point +// elements) from a into memory. mem_addr does not need to be aligned on any +// particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeu_pd +FORCE_INLINE void _mm_storeu_pd(double *mem_addr, __m128d a) +{ + _mm_store_pd(mem_addr, a); +} + +// Store 128-bits of integer data from a into memory. mem_addr does not need to +// be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeu_si128 +FORCE_INLINE void _mm_storeu_si128(__m128i *p, __m128i a) +{ + vst1q_s32((int32_t *) p, vreinterpretq_s32_m128i(a)); +} + +// Store 32-bit integer from the first element of a into memory. mem_addr does +// not need to be aligned on any particular boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_storeu_si32 +FORCE_INLINE void _mm_storeu_si32(void *p, __m128i a) +{ + vst1q_lane_s32((int32_t *) p, vreinterpretq_s32_m128i(a), 0); +} + +// Store 128-bits (composed of 2 packed double-precision (64-bit) floating-point +// elements) from a into memory using a non-temporal memory hint. mem_addr must +// be aligned on a 16-byte boundary or a general-protection exception may be +// generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_stream_pd +FORCE_INLINE void _mm_stream_pd(double *p, __m128d a) +{ +#if __has_builtin(__builtin_nontemporal_store) + __builtin_nontemporal_store(a, (__m128d *) p); +#elif defined(__aarch64__) || defined(_M_ARM64) + vst1q_f64(p, vreinterpretq_f64_m128d(a)); +#else + vst1q_s64((int64_t *) p, vreinterpretq_s64_m128d(a)); +#endif +} + +// Store 128-bits of integer data from a into memory using a non-temporal memory +// hint. mem_addr must be aligned on a 16-byte boundary or a general-protection +// exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_stream_si128 +FORCE_INLINE void _mm_stream_si128(__m128i *p, __m128i a) +{ +#if __has_builtin(__builtin_nontemporal_store) + __builtin_nontemporal_store(a, p); +#else + vst1q_s64((int64_t *) p, vreinterpretq_s64_m128i(a)); +#endif +} + +// Store 32-bit integer a into memory using a non-temporal hint to minimize +// cache pollution. If the cache line containing address mem_addr is already in +// the cache, the cache will be updated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_stream_si32 +FORCE_INLINE void _mm_stream_si32(int *p, int a) +{ + vst1q_lane_s32((int32_t *) p, vdupq_n_s32(a), 0); +} + +// Store 64-bit integer a into memory using a non-temporal hint to minimize +// cache pollution. If the cache line containing address mem_addr is already in +// the cache, the cache will be updated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_stream_si64 +FORCE_INLINE void _mm_stream_si64(__int64 *p, __int64 a) +{ + vst1_s64((int64_t *) p, vdup_n_s64((int64_t) a)); +} + +// Subtract packed 16-bit integers in b from packed 16-bit integers in a, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_epi16 +FORCE_INLINE __m128i _mm_sub_epi16(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s16( + vsubq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +} + +// Subtract packed 32-bit integers in b from packed 32-bit integers in a, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_epi32 +FORCE_INLINE __m128i _mm_sub_epi32(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s32( + vsubq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +} + +// Subtract packed 64-bit integers in b from packed 64-bit integers in a, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_epi64 +FORCE_INLINE __m128i _mm_sub_epi64(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s64( + vsubq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b))); +} + +// Subtract packed 8-bit integers in b from packed 8-bit integers in a, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_epi8 +FORCE_INLINE __m128i _mm_sub_epi8(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s8( + vsubq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +} + +// Subtract packed double-precision (64-bit) floating-point elements in b from +// packed double-precision (64-bit) floating-point elements in a, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=mm_sub_pd +FORCE_INLINE __m128d _mm_sub_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vsubq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[2]; + c[0] = da[0] - db[0]; + c[1] = da[1] - db[1]; + return vld1q_f32((float32_t *) c); +#endif +} + +// Subtract the lower double-precision (64-bit) floating-point element in b from +// the lower double-precision (64-bit) floating-point element in a, store the +// result in the lower element of dst, and copy the upper element from a to the +// upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_sd +FORCE_INLINE __m128d _mm_sub_sd(__m128d a, __m128d b) +{ + return _mm_move_sd(a, _mm_sub_pd(a, b)); +} + +// Subtract 64-bit integer b from 64-bit integer a, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sub_si64 +FORCE_INLINE __m64 _mm_sub_si64(__m64 a, __m64 b) +{ + return vreinterpret_m64_s64( + vsub_s64(vreinterpret_s64_m64(a), vreinterpret_s64_m64(b))); +} + +// Subtract packed signed 16-bit integers in b from packed 16-bit integers in a +// using saturation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_subs_epi16 +FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s16( + vqsubq_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +} + +// Subtract packed signed 8-bit integers in b from packed 8-bit integers in a +// using saturation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_subs_epi8 +FORCE_INLINE __m128i _mm_subs_epi8(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s8( + vqsubq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +} + +// Subtract packed unsigned 16-bit integers in b from packed unsigned 16-bit +// integers in a using saturation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_subs_epu16 +FORCE_INLINE __m128i _mm_subs_epu16(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u16( + vqsubq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); +} + +// Subtract packed unsigned 8-bit integers in b from packed unsigned 8-bit +// integers in a using saturation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_subs_epu8 +FORCE_INLINE __m128i _mm_subs_epu8(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u8( + vqsubq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b))); +} + +#define _mm_ucomieq_sd _mm_comieq_sd +#define _mm_ucomige_sd _mm_comige_sd +#define _mm_ucomigt_sd _mm_comigt_sd +#define _mm_ucomile_sd _mm_comile_sd +#define _mm_ucomilt_sd _mm_comilt_sd +#define _mm_ucomineq_sd _mm_comineq_sd + +// Return vector of type __m128d with undefined elements. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_undefined_pd +FORCE_INLINE __m128d _mm_undefined_pd(void) +{ +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#endif + __m128d a; +#if defined(_MSC_VER) + a = _mm_setzero_pd(); +#endif + return a; +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif +} + +// Unpack and interleave 16-bit integers from the high half of a and b, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpackhi_epi16 +FORCE_INLINE __m128i _mm_unpackhi_epi16(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s16( + vzip2q_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +#else + int16x4_t a1 = vget_high_s16(vreinterpretq_s16_m128i(a)); + int16x4_t b1 = vget_high_s16(vreinterpretq_s16_m128i(b)); + int16x4x2_t result = vzip_s16(a1, b1); + return vreinterpretq_m128i_s16(vcombine_s16(result.val[0], result.val[1])); +#endif +} + +// Unpack and interleave 32-bit integers from the high half of a and b, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpackhi_epi32 +FORCE_INLINE __m128i _mm_unpackhi_epi32(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s32( + vzip2q_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +#else + int32x2_t a1 = vget_high_s32(vreinterpretq_s32_m128i(a)); + int32x2_t b1 = vget_high_s32(vreinterpretq_s32_m128i(b)); + int32x2x2_t result = vzip_s32(a1, b1); + return vreinterpretq_m128i_s32(vcombine_s32(result.val[0], result.val[1])); +#endif +} + +// Unpack and interleave 64-bit integers from the high half of a and b, and +// store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpackhi_epi64 +FORCE_INLINE __m128i _mm_unpackhi_epi64(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s64( + vzip2q_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b))); +#else + int64x1_t a_h = vget_high_s64(vreinterpretq_s64_m128i(a)); + int64x1_t b_h = vget_high_s64(vreinterpretq_s64_m128i(b)); + return vreinterpretq_m128i_s64(vcombine_s64(a_h, b_h)); +#endif +} + +// Unpack and interleave 8-bit integers from the high half of a and b, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpackhi_epi8 +FORCE_INLINE __m128i _mm_unpackhi_epi8(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s8( + vzip2q_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +#else + int8x8_t a1 = + vreinterpret_s8_s16(vget_high_s16(vreinterpretq_s16_m128i(a))); + int8x8_t b1 = + vreinterpret_s8_s16(vget_high_s16(vreinterpretq_s16_m128i(b))); + int8x8x2_t result = vzip_s8(a1, b1); + return vreinterpretq_m128i_s8(vcombine_s8(result.val[0], result.val[1])); +#endif +} + +// Unpack and interleave double-precision (64-bit) floating-point elements from +// the high half of a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpackhi_pd +FORCE_INLINE __m128d _mm_unpackhi_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vzip2q_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + return vreinterpretq_m128d_s64( + vcombine_s64(vget_high_s64(vreinterpretq_s64_m128d(a)), + vget_high_s64(vreinterpretq_s64_m128d(b)))); +#endif +} + +// Unpack and interleave 16-bit integers from the low half of a and b, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpacklo_epi16 +FORCE_INLINE __m128i _mm_unpacklo_epi16(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s16( + vzip1q_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); +#else + int16x4_t a1 = vget_low_s16(vreinterpretq_s16_m128i(a)); + int16x4_t b1 = vget_low_s16(vreinterpretq_s16_m128i(b)); + int16x4x2_t result = vzip_s16(a1, b1); + return vreinterpretq_m128i_s16(vcombine_s16(result.val[0], result.val[1])); +#endif +} + +// Unpack and interleave 32-bit integers from the low half of a and b, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpacklo_epi32 +FORCE_INLINE __m128i _mm_unpacklo_epi32(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s32( + vzip1q_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +#else + int32x2_t a1 = vget_low_s32(vreinterpretq_s32_m128i(a)); + int32x2_t b1 = vget_low_s32(vreinterpretq_s32_m128i(b)); + int32x2x2_t result = vzip_s32(a1, b1); + return vreinterpretq_m128i_s32(vcombine_s32(result.val[0], result.val[1])); +#endif +} + +// Unpack and interleave 64-bit integers from the low half of a and b, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpacklo_epi64 +FORCE_INLINE __m128i _mm_unpacklo_epi64(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s64( + vzip1q_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b))); +#else + int64x1_t a_l = vget_low_s64(vreinterpretq_s64_m128i(a)); + int64x1_t b_l = vget_low_s64(vreinterpretq_s64_m128i(b)); + return vreinterpretq_m128i_s64(vcombine_s64(a_l, b_l)); +#endif +} + +// Unpack and interleave 8-bit integers from the low half of a and b, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpacklo_epi8 +FORCE_INLINE __m128i _mm_unpacklo_epi8(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s8( + vzip1q_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +#else + int8x8_t a1 = vreinterpret_s8_s16(vget_low_s16(vreinterpretq_s16_m128i(a))); + int8x8_t b1 = vreinterpret_s8_s16(vget_low_s16(vreinterpretq_s16_m128i(b))); + int8x8x2_t result = vzip_s8(a1, b1); + return vreinterpretq_m128i_s8(vcombine_s8(result.val[0], result.val[1])); +#endif +} + +// Unpack and interleave double-precision (64-bit) floating-point elements from +// the low half of a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_unpacklo_pd +FORCE_INLINE __m128d _mm_unpacklo_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vzip1q_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + return vreinterpretq_m128d_s64( + vcombine_s64(vget_low_s64(vreinterpretq_s64_m128d(a)), + vget_low_s64(vreinterpretq_s64_m128d(b)))); +#endif +} + +// Compute the bitwise XOR of packed double-precision (64-bit) floating-point +// elements in a and b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_xor_pd +FORCE_INLINE __m128d _mm_xor_pd(__m128d a, __m128d b) +{ + return vreinterpretq_m128d_s64( + veorq_s64(vreinterpretq_s64_m128d(a), vreinterpretq_s64_m128d(b))); +} + +// Compute the bitwise XOR of 128 bits (representing integer data) in a and b, +// and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_xor_si128 +FORCE_INLINE __m128i _mm_xor_si128(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s32( + veorq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +} + +/* SSE3 */ + +// Alternatively add and subtract packed double-precision (64-bit) +// floating-point elements in a to/from packed elements in b, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_addsub_pd +FORCE_INLINE __m128d _mm_addsub_pd(__m128d a, __m128d b) +{ + _sse2neon_const __m128d mask = _mm_set_pd(1.0f, -1.0f); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vfmaq_f64(vreinterpretq_f64_m128d(a), + vreinterpretq_f64_m128d(b), + vreinterpretq_f64_m128d(mask))); +#else + return _mm_add_pd(_mm_mul_pd(b, mask), a); +#endif +} + +// Alternatively add and subtract packed single-precision (32-bit) +// floating-point elements in a to/from packed elements in b, and store the +// results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=addsub_ps +FORCE_INLINE __m128 _mm_addsub_ps(__m128 a, __m128 b) +{ + _sse2neon_const __m128 mask = _mm_setr_ps(-1.0f, 1.0f, -1.0f, 1.0f); +#if (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ARM_FEATURE_FMA) /* VFPv4+ */ + return vreinterpretq_m128_f32(vfmaq_f32(vreinterpretq_f32_m128(a), + vreinterpretq_f32_m128(mask), + vreinterpretq_f32_m128(b))); +#else + return _mm_add_ps(_mm_mul_ps(b, mask), a); +#endif +} + +// Horizontally add adjacent pairs of double-precision (64-bit) floating-point +// elements in a and b, and pack the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hadd_pd +FORCE_INLINE __m128d _mm_hadd_pd(__m128d a, __m128d b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vpaddq_f64(vreinterpretq_f64_m128d(a), vreinterpretq_f64_m128d(b))); +#else + double *da = (double *) &a; + double *db = (double *) &b; + double c[] = {da[0] + da[1], db[0] + db[1]}; + return vreinterpretq_m128d_u64(vld1q_u64((uint64_t *) c)); +#endif +} + +// Horizontally add adjacent pairs of single-precision (32-bit) floating-point +// elements in a and b, and pack the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hadd_ps +FORCE_INLINE __m128 _mm_hadd_ps(__m128 a, __m128 b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32( + vpaddq_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); +#else + float32x2_t a10 = vget_low_f32(vreinterpretq_f32_m128(a)); + float32x2_t a32 = vget_high_f32(vreinterpretq_f32_m128(a)); + float32x2_t b10 = vget_low_f32(vreinterpretq_f32_m128(b)); + float32x2_t b32 = vget_high_f32(vreinterpretq_f32_m128(b)); + return vreinterpretq_m128_f32( + vcombine_f32(vpadd_f32(a10, a32), vpadd_f32(b10, b32))); +#endif +} + +// Horizontally subtract adjacent pairs of double-precision (64-bit) +// floating-point elements in a and b, and pack the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hsub_pd +FORCE_INLINE __m128d _mm_hsub_pd(__m128d _a, __m128d _b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + float64x2_t a = vreinterpretq_f64_m128d(_a); + float64x2_t b = vreinterpretq_f64_m128d(_b); + return vreinterpretq_m128d_f64( + vsubq_f64(vuzp1q_f64(a, b), vuzp2q_f64(a, b))); +#else + double *da = (double *) &_a; + double *db = (double *) &_b; + double c[] = {da[0] - da[1], db[0] - db[1]}; + return vreinterpretq_m128d_u64(vld1q_u64((uint64_t *) c)); +#endif +} + +// Horizontally subtract adjacent pairs of single-precision (32-bit) +// floating-point elements in a and b, and pack the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hsub_ps +FORCE_INLINE __m128 _mm_hsub_ps(__m128 _a, __m128 _b) +{ + float32x4_t a = vreinterpretq_f32_m128(_a); + float32x4_t b = vreinterpretq_f32_m128(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32( + vsubq_f32(vuzp1q_f32(a, b), vuzp2q_f32(a, b))); +#else + float32x4x2_t c = vuzpq_f32(a, b); + return vreinterpretq_m128_f32(vsubq_f32(c.val[0], c.val[1])); +#endif +} + +// Load 128-bits of integer data from unaligned memory into dst. This intrinsic +// may perform better than _mm_loadu_si128 when the data crosses a cache line +// boundary. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_lddqu_si128 +#define _mm_lddqu_si128 _mm_loadu_si128 + +// Load a double-precision (64-bit) floating-point element from memory into both +// elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loaddup_pd +#define _mm_loaddup_pd _mm_load1_pd + +// Duplicate the low double-precision (64-bit) floating-point element from a, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movedup_pd +FORCE_INLINE __m128d _mm_movedup_pd(__m128d a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64( + vdupq_laneq_f64(vreinterpretq_f64_m128d(a), 0)); +#else + return vreinterpretq_m128d_u64( + vdupq_n_u64(vgetq_lane_u64(vreinterpretq_u64_m128d(a), 0))); +#endif +} + +// Duplicate odd-indexed single-precision (32-bit) floating-point elements +// from a, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_movehdup_ps +FORCE_INLINE __m128 _mm_movehdup_ps(__m128 a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32( + vtrn2q_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a))); +#elif defined(_sse2neon_shuffle) + return vreinterpretq_m128_f32(vshuffleq_s32( + vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 1, 1, 3, 3)); +#else + float32_t a1 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 1); + float32_t a3 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 3); + float ALIGN_STRUCT(16) data[4] = {a1, a1, a3, a3}; + return vreinterpretq_m128_f32(vld1q_f32(data)); +#endif +} + +// Duplicate even-indexed single-precision (32-bit) floating-point elements +// from a, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_moveldup_ps +FORCE_INLINE __m128 _mm_moveldup_ps(__m128 a) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128_f32( + vtrn1q_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a))); +#elif defined(_sse2neon_shuffle) + return vreinterpretq_m128_f32(vshuffleq_s32( + vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(a), 0, 0, 2, 2)); +#else + float32_t a0 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); + float32_t a2 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 2); + float ALIGN_STRUCT(16) data[4] = {a0, a0, a2, a2}; + return vreinterpretq_m128_f32(vld1q_f32(data)); +#endif +} + +/* SSSE3 */ + +// Compute the absolute value of packed signed 16-bit integers in a, and store +// the unsigned results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_abs_epi16 +FORCE_INLINE __m128i _mm_abs_epi16(__m128i a) +{ + return vreinterpretq_m128i_s16(vabsq_s16(vreinterpretq_s16_m128i(a))); +} + +// Compute the absolute value of packed signed 32-bit integers in a, and store +// the unsigned results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_abs_epi32 +FORCE_INLINE __m128i _mm_abs_epi32(__m128i a) +{ + return vreinterpretq_m128i_s32(vabsq_s32(vreinterpretq_s32_m128i(a))); +} + +// Compute the absolute value of packed signed 8-bit integers in a, and store +// the unsigned results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_abs_epi8 +FORCE_INLINE __m128i _mm_abs_epi8(__m128i a) +{ + return vreinterpretq_m128i_s8(vabsq_s8(vreinterpretq_s8_m128i(a))); +} + +// Compute the absolute value of packed signed 16-bit integers in a, and store +// the unsigned results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_abs_pi16 +FORCE_INLINE __m64 _mm_abs_pi16(__m64 a) +{ + return vreinterpret_m64_s16(vabs_s16(vreinterpret_s16_m64(a))); +} + +// Compute the absolute value of packed signed 32-bit integers in a, and store +// the unsigned results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_abs_pi32 +FORCE_INLINE __m64 _mm_abs_pi32(__m64 a) +{ + return vreinterpret_m64_s32(vabs_s32(vreinterpret_s32_m64(a))); +} + +// Compute the absolute value of packed signed 8-bit integers in a, and store +// the unsigned results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_abs_pi8 +FORCE_INLINE __m64 _mm_abs_pi8(__m64 a) +{ + return vreinterpret_m64_s8(vabs_s8(vreinterpret_s8_m64(a))); +} + +// Concatenate 16-byte blocks in a and b into a 32-byte temporary result, shift +// the result right by imm8 bytes, and store the low 16 bytes in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_alignr_epi8 +#if defined(__GNUC__) && !defined(__clang__) +#define _mm_alignr_epi8(a, b, imm) \ + __extension__({ \ + uint8x16_t _a = vreinterpretq_u8_m128i(a); \ + uint8x16_t _b = vreinterpretq_u8_m128i(b); \ + __m128i ret; \ + if (_sse2neon_unlikely((imm) & ~31)) \ + ret = vreinterpretq_m128i_u8(vdupq_n_u8(0)); \ + else if (imm >= 16) \ + ret = _mm_srli_si128(a, imm >= 16 ? imm - 16 : 0); \ + else \ + ret = \ + vreinterpretq_m128i_u8(vextq_u8(_b, _a, imm < 16 ? imm : 0)); \ + ret; \ + }) + +#else +#define _mm_alignr_epi8(a, b, imm) \ + _sse2neon_define2( \ + __m128i, a, b, uint8x16_t __a = vreinterpretq_u8_m128i(_a); \ + uint8x16_t __b = vreinterpretq_u8_m128i(_b); __m128i ret; \ + if (_sse2neon_unlikely((imm) & ~31)) ret = \ + vreinterpretq_m128i_u8(vdupq_n_u8(0)); \ + else if (imm >= 16) ret = \ + _mm_srli_si128(_a, imm >= 16 ? imm - 16 : 0); \ + else ret = \ + vreinterpretq_m128i_u8(vextq_u8(__b, __a, imm < 16 ? imm : 0)); \ + _sse2neon_return(ret);) + +#endif + +// Concatenate 8-byte blocks in a and b into a 16-byte temporary result, shift +// the result right by imm8 bytes, and store the low 8 bytes in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_alignr_pi8 +#define _mm_alignr_pi8(a, b, imm) \ + _sse2neon_define2( \ + __m64, a, b, __m64 ret; if (_sse2neon_unlikely((imm) >= 16)) { \ + ret = vreinterpret_m64_s8(vdup_n_s8(0)); \ + } else { \ + uint8x8_t tmp_low; \ + uint8x8_t tmp_high; \ + if ((imm) >= 8) { \ + const int idx = (imm) -8; \ + tmp_low = vreinterpret_u8_m64(_a); \ + tmp_high = vdup_n_u8(0); \ + ret = vreinterpret_m64_u8(vext_u8(tmp_low, tmp_high, idx)); \ + } else { \ + const int idx = (imm); \ + tmp_low = vreinterpret_u8_m64(_b); \ + tmp_high = vreinterpret_u8_m64(_a); \ + ret = vreinterpret_m64_u8(vext_u8(tmp_low, tmp_high, idx)); \ + } \ + } _sse2neon_return(ret);) + +// Horizontally add adjacent pairs of 16-bit integers in a and b, and pack the +// signed 16-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hadd_epi16 +FORCE_INLINE __m128i _mm_hadd_epi16(__m128i _a, __m128i _b) +{ + int16x8_t a = vreinterpretq_s16_m128i(_a); + int16x8_t b = vreinterpretq_s16_m128i(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s16(vpaddq_s16(a, b)); +#else + return vreinterpretq_m128i_s16( + vcombine_s16(vpadd_s16(vget_low_s16(a), vget_high_s16(a)), + vpadd_s16(vget_low_s16(b), vget_high_s16(b)))); +#endif +} + +// Horizontally add adjacent pairs of 32-bit integers in a and b, and pack the +// signed 32-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hadd_epi32 +FORCE_INLINE __m128i _mm_hadd_epi32(__m128i _a, __m128i _b) +{ + int32x4_t a = vreinterpretq_s32_m128i(_a); + int32x4_t b = vreinterpretq_s32_m128i(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s32(vpaddq_s32(a, b)); +#else + return vreinterpretq_m128i_s32( + vcombine_s32(vpadd_s32(vget_low_s32(a), vget_high_s32(a)), + vpadd_s32(vget_low_s32(b), vget_high_s32(b)))); +#endif +} + +// Horizontally add adjacent pairs of 16-bit integers in a and b, and pack the +// signed 16-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hadd_pi16 +FORCE_INLINE __m64 _mm_hadd_pi16(__m64 a, __m64 b) +{ + return vreinterpret_m64_s16( + vpadd_s16(vreinterpret_s16_m64(a), vreinterpret_s16_m64(b))); +} + +// Horizontally add adjacent pairs of 32-bit integers in a and b, and pack the +// signed 32-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hadd_pi32 +FORCE_INLINE __m64 _mm_hadd_pi32(__m64 a, __m64 b) +{ + return vreinterpret_m64_s32( + vpadd_s32(vreinterpret_s32_m64(a), vreinterpret_s32_m64(b))); +} + +// Horizontally add adjacent pairs of signed 16-bit integers in a and b using +// saturation, and pack the signed 16-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hadds_epi16 +FORCE_INLINE __m128i _mm_hadds_epi16(__m128i _a, __m128i _b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + int16x8_t a = vreinterpretq_s16_m128i(_a); + int16x8_t b = vreinterpretq_s16_m128i(_b); + return vreinterpretq_s64_s16( + vqaddq_s16(vuzp1q_s16(a, b), vuzp2q_s16(a, b))); +#else + int32x4_t a = vreinterpretq_s32_m128i(_a); + int32x4_t b = vreinterpretq_s32_m128i(_b); + // Interleave using vshrn/vmovn + // [a0|a2|a4|a6|b0|b2|b4|b6] + // [a1|a3|a5|a7|b1|b3|b5|b7] + int16x8_t ab0246 = vcombine_s16(vmovn_s32(a), vmovn_s32(b)); + int16x8_t ab1357 = vcombine_s16(vshrn_n_s32(a, 16), vshrn_n_s32(b, 16)); + // Saturated add + return vreinterpretq_m128i_s16(vqaddq_s16(ab0246, ab1357)); +#endif +} + +// Horizontally add adjacent pairs of signed 16-bit integers in a and b using +// saturation, and pack the signed 16-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hadds_pi16 +FORCE_INLINE __m64 _mm_hadds_pi16(__m64 _a, __m64 _b) +{ + int16x4_t a = vreinterpret_s16_m64(_a); + int16x4_t b = vreinterpret_s16_m64(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpret_s64_s16(vqadd_s16(vuzp1_s16(a, b), vuzp2_s16(a, b))); +#else + int16x4x2_t res = vuzp_s16(a, b); + return vreinterpret_s64_s16(vqadd_s16(res.val[0], res.val[1])); +#endif +} + +// Horizontally subtract adjacent pairs of 16-bit integers in a and b, and pack +// the signed 16-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hsub_epi16 +FORCE_INLINE __m128i _mm_hsub_epi16(__m128i _a, __m128i _b) +{ + int16x8_t a = vreinterpretq_s16_m128i(_a); + int16x8_t b = vreinterpretq_s16_m128i(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s16( + vsubq_s16(vuzp1q_s16(a, b), vuzp2q_s16(a, b))); +#else + int16x8x2_t c = vuzpq_s16(a, b); + return vreinterpretq_m128i_s16(vsubq_s16(c.val[0], c.val[1])); +#endif +} + +// Horizontally subtract adjacent pairs of 32-bit integers in a and b, and pack +// the signed 32-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hsub_epi32 +FORCE_INLINE __m128i _mm_hsub_epi32(__m128i _a, __m128i _b) +{ + int32x4_t a = vreinterpretq_s32_m128i(_a); + int32x4_t b = vreinterpretq_s32_m128i(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s32( + vsubq_s32(vuzp1q_s32(a, b), vuzp2q_s32(a, b))); +#else + int32x4x2_t c = vuzpq_s32(a, b); + return vreinterpretq_m128i_s32(vsubq_s32(c.val[0], c.val[1])); +#endif +} + +// Horizontally subtract adjacent pairs of 16-bit integers in a and b, and pack +// the signed 16-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hsub_pi16 +FORCE_INLINE __m64 _mm_hsub_pi16(__m64 _a, __m64 _b) +{ + int16x4_t a = vreinterpret_s16_m64(_a); + int16x4_t b = vreinterpret_s16_m64(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpret_m64_s16(vsub_s16(vuzp1_s16(a, b), vuzp2_s16(a, b))); +#else + int16x4x2_t c = vuzp_s16(a, b); + return vreinterpret_m64_s16(vsub_s16(c.val[0], c.val[1])); +#endif +} + +// Horizontally subtract adjacent pairs of 32-bit integers in a and b, and pack +// the signed 32-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=mm_hsub_pi32 +FORCE_INLINE __m64 _mm_hsub_pi32(__m64 _a, __m64 _b) +{ + int32x2_t a = vreinterpret_s32_m64(_a); + int32x2_t b = vreinterpret_s32_m64(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpret_m64_s32(vsub_s32(vuzp1_s32(a, b), vuzp2_s32(a, b))); +#else + int32x2x2_t c = vuzp_s32(a, b); + return vreinterpret_m64_s32(vsub_s32(c.val[0], c.val[1])); +#endif +} + +// Horizontally subtract adjacent pairs of signed 16-bit integers in a and b +// using saturation, and pack the signed 16-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hsubs_epi16 +FORCE_INLINE __m128i _mm_hsubs_epi16(__m128i _a, __m128i _b) +{ + int16x8_t a = vreinterpretq_s16_m128i(_a); + int16x8_t b = vreinterpretq_s16_m128i(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s16( + vqsubq_s16(vuzp1q_s16(a, b), vuzp2q_s16(a, b))); +#else + int16x8x2_t c = vuzpq_s16(a, b); + return vreinterpretq_m128i_s16(vqsubq_s16(c.val[0], c.val[1])); +#endif +} + +// Horizontally subtract adjacent pairs of signed 16-bit integers in a and b +// using saturation, and pack the signed 16-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_hsubs_pi16 +FORCE_INLINE __m64 _mm_hsubs_pi16(__m64 _a, __m64 _b) +{ + int16x4_t a = vreinterpret_s16_m64(_a); + int16x4_t b = vreinterpret_s16_m64(_b); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpret_m64_s16(vqsub_s16(vuzp1_s16(a, b), vuzp2_s16(a, b))); +#else + int16x4x2_t c = vuzp_s16(a, b); + return vreinterpret_m64_s16(vqsub_s16(c.val[0], c.val[1])); +#endif +} + +// Vertically multiply each unsigned 8-bit integer from a with the corresponding +// signed 8-bit integer from b, producing intermediate signed 16-bit integers. +// Horizontally add adjacent pairs of intermediate signed 16-bit integers, +// and pack the saturated results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maddubs_epi16 +FORCE_INLINE __m128i _mm_maddubs_epi16(__m128i _a, __m128i _b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + uint8x16_t a = vreinterpretq_u8_m128i(_a); + int8x16_t b = vreinterpretq_s8_m128i(_b); + int16x8_t tl = vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(a))), + vmovl_s8(vget_low_s8(b))); + int16x8_t th = vmulq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(a))), + vmovl_s8(vget_high_s8(b))); + return vreinterpretq_m128i_s16( + vqaddq_s16(vuzp1q_s16(tl, th), vuzp2q_s16(tl, th))); +#else + // This would be much simpler if x86 would choose to zero extend OR sign + // extend, not both. This could probably be optimized better. + uint16x8_t a = vreinterpretq_u16_m128i(_a); + int16x8_t b = vreinterpretq_s16_m128i(_b); + + // Zero extend a + int16x8_t a_odd = vreinterpretq_s16_u16(vshrq_n_u16(a, 8)); + int16x8_t a_even = vreinterpretq_s16_u16(vbicq_u16(a, vdupq_n_u16(0xff00))); + + // Sign extend by shifting left then shifting right. + int16x8_t b_even = vshrq_n_s16(vshlq_n_s16(b, 8), 8); + int16x8_t b_odd = vshrq_n_s16(b, 8); + + // multiply + int16x8_t prod1 = vmulq_s16(a_even, b_even); + int16x8_t prod2 = vmulq_s16(a_odd, b_odd); + + // saturated add + return vreinterpretq_m128i_s16(vqaddq_s16(prod1, prod2)); +#endif +} + +// Vertically multiply each unsigned 8-bit integer from a with the corresponding +// signed 8-bit integer from b, producing intermediate signed 16-bit integers. +// Horizontally add adjacent pairs of intermediate signed 16-bit integers, and +// pack the saturated results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_maddubs_pi16 +FORCE_INLINE __m64 _mm_maddubs_pi16(__m64 _a, __m64 _b) +{ + uint16x4_t a = vreinterpret_u16_m64(_a); + int16x4_t b = vreinterpret_s16_m64(_b); + + // Zero extend a + int16x4_t a_odd = vreinterpret_s16_u16(vshr_n_u16(a, 8)); + int16x4_t a_even = vreinterpret_s16_u16(vand_u16(a, vdup_n_u16(0xff))); + + // Sign extend by shifting left then shifting right. + int16x4_t b_even = vshr_n_s16(vshl_n_s16(b, 8), 8); + int16x4_t b_odd = vshr_n_s16(b, 8); + + // multiply + int16x4_t prod1 = vmul_s16(a_even, b_even); + int16x4_t prod2 = vmul_s16(a_odd, b_odd); + + // saturated add + return vreinterpret_m64_s16(vqadd_s16(prod1, prod2)); +} + +// Multiply packed signed 16-bit integers in a and b, producing intermediate +// signed 32-bit integers. Shift right by 15 bits while rounding up, and store +// the packed 16-bit integers in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mulhrs_epi16 +FORCE_INLINE __m128i _mm_mulhrs_epi16(__m128i a, __m128i b) +{ + // Has issues due to saturation + // return vreinterpretq_m128i_s16(vqrdmulhq_s16(a, b)); + + // Multiply + int32x4_t mul_lo = vmull_s16(vget_low_s16(vreinterpretq_s16_m128i(a)), + vget_low_s16(vreinterpretq_s16_m128i(b))); + int32x4_t mul_hi = vmull_s16(vget_high_s16(vreinterpretq_s16_m128i(a)), + vget_high_s16(vreinterpretq_s16_m128i(b))); + + // Rounding narrowing shift right + // narrow = (int16_t)((mul + 16384) >> 15); + int16x4_t narrow_lo = vrshrn_n_s32(mul_lo, 15); + int16x4_t narrow_hi = vrshrn_n_s32(mul_hi, 15); + + // Join together + return vreinterpretq_m128i_s16(vcombine_s16(narrow_lo, narrow_hi)); } -// Convert packed 16-bit integers in a to packed single-precision (32-bit) -// floating-point elements, and store the results in dst. -// -// FOR j := 0 to 3 -// i := j*16 -// m := j*32 -// dst[m+31:m] := Convert_Int16_To_FP32(a[i+15:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpi16_ps -FORCE_INLINE __m128 _mm_cvtpi16_ps(__m64 a) +// Multiply packed signed 16-bit integers in a and b, producing intermediate +// signed 32-bit integers. Truncate each intermediate integer to the 18 most +// significant bits, round by adding 1, and store bits [16:1] to dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mulhrs_pi16 +FORCE_INLINE __m64 _mm_mulhrs_pi16(__m64 a, __m64 b) { - return vreinterpretq_m128_f32( - vcvtq_f32_s32(vmovl_s16(vreinterpret_s16_m64(a)))); -} + int32x4_t mul_extend = + vmull_s16((vreinterpret_s16_m64(a)), (vreinterpret_s16_m64(b))); -// Convert packed 32-bit integers in b to packed single-precision (32-bit) -// floating-point elements, store the results in the lower 2 elements of dst, -// and copy the upper 2 packed elements from a to the upper elements of dst. -// -// dst[31:0] := Convert_Int32_To_FP32(b[31:0]) -// dst[63:32] := Convert_Int32_To_FP32(b[63:32]) -// dst[95:64] := a[95:64] -// dst[127:96] := a[127:96] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpi32_ps -FORCE_INLINE __m128 _mm_cvtpi32_ps(__m128 a, __m64 b) -{ - return vreinterpretq_m128_f32( - vcombine_f32(vcvt_f32_s32(vreinterpret_s32_m64(b)), - vget_high_f32(vreinterpretq_f32_m128(a)))); + // Rounding narrowing shift right + return vreinterpret_m64_s16(vrshrn_n_s32(mul_extend, 15)); } -// Convert packed signed 32-bit integers in a to packed single-precision -// (32-bit) floating-point elements, store the results in the lower 2 elements -// of dst, then covert the packed signed 32-bit integers in b to -// single-precision (32-bit) floating-point element, and store the results in -// the upper 2 elements of dst. -// -// dst[31:0] := Convert_Int32_To_FP32(a[31:0]) -// dst[63:32] := Convert_Int32_To_FP32(a[63:32]) -// dst[95:64] := Convert_Int32_To_FP32(b[31:0]) -// dst[127:96] := Convert_Int32_To_FP32(b[63:32]) -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpi32x2_ps -FORCE_INLINE __m128 _mm_cvtpi32x2_ps(__m64 a, __m64 b) +// Shuffle packed 8-bit integers in a according to shuffle control mask in the +// corresponding 8-bit element of b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shuffle_epi8 +FORCE_INLINE __m128i _mm_shuffle_epi8(__m128i a, __m128i b) { - return vreinterpretq_m128_f32(vcvtq_f32_s32( - vcombine_s32(vreinterpret_s32_m64(a), vreinterpret_s32_m64(b)))); + int8x16_t tbl = vreinterpretq_s8_m128i(a); // input a + uint8x16_t idx = vreinterpretq_u8_m128i(b); // input b + uint8x16_t idx_masked = + vandq_u8(idx, vdupq_n_u8(0x8F)); // avoid using meaningless bits +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_s8(vqtbl1q_s8(tbl, idx_masked)); +#elif defined(__GNUC__) + int8x16_t ret; + // %e and %f represent the even and odd D registers + // respectively. + __asm__ __volatile__( + "vtbl.8 %e[ret], {%e[tbl], %f[tbl]}, %e[idx]\n" + "vtbl.8 %f[ret], {%e[tbl], %f[tbl]}, %f[idx]\n" + : [ret] "=&w"(ret) + : [tbl] "w"(tbl), [idx] "w"(idx_masked)); + return vreinterpretq_m128i_s8(ret); +#else + // use this line if testing on aarch64 + int8x8x2_t a_split = {vget_low_s8(tbl), vget_high_s8(tbl)}; + return vreinterpretq_m128i_s8( + vcombine_s8(vtbl2_s8(a_split, vget_low_u8(idx_masked)), + vtbl2_s8(a_split, vget_high_u8(idx_masked)))); +#endif } -// Convert the lower packed 8-bit integers in a to packed single-precision -// (32-bit) floating-point elements, and store the results in dst. -// -// FOR j := 0 to 3 -// i := j*8 -// m := j*32 -// dst[m+31:m] := Convert_Int8_To_FP32(a[i+7:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpi8_ps -FORCE_INLINE __m128 _mm_cvtpi8_ps(__m64 a) +// Shuffle packed 8-bit integers in a according to shuffle control mask in the +// corresponding 8-bit element of b, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_shuffle_pi8 +FORCE_INLINE __m64 _mm_shuffle_pi8(__m64 a, __m64 b) { - return vreinterpretq_m128_f32(vcvtq_f32_s32( - vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_m64(a)))))); + const int8x8_t controlMask = + vand_s8(vreinterpret_s8_m64(b), vdup_n_s8((int8_t) (0x1 << 7 | 0x07))); + int8x8_t res = vtbl1_s8(vreinterpret_s8_m64(a), controlMask); + return vreinterpret_m64_s8(res); } -// Convert packed unsigned 16-bit integers in a to packed single-precision -// (32-bit) floating-point elements, and store the results in dst. -// -// FOR j := 0 to 3 -// i := j*16 -// m := j*32 -// dst[m+31:m] := Convert_UInt16_To_FP32(a[i+15:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpu16_ps -FORCE_INLINE __m128 _mm_cvtpu16_ps(__m64 a) +// Negate packed 16-bit integers in a when the corresponding signed +// 16-bit integer in b is negative, and store the results in dst. +// Element in dst are zeroed out when the corresponding element +// in b is zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sign_epi16 +FORCE_INLINE __m128i _mm_sign_epi16(__m128i _a, __m128i _b) { - return vreinterpretq_m128_f32( - vcvtq_f32_u32(vmovl_u16(vreinterpret_u16_m64(a)))); -} + int16x8_t a = vreinterpretq_s16_m128i(_a); + int16x8_t b = vreinterpretq_s16_m128i(_b); -// Convert the lower packed unsigned 8-bit integers in a to packed -// single-precision (32-bit) floating-point elements, and store the results in -// dst. -// -// FOR j := 0 to 3 -// i := j*8 -// m := j*32 -// dst[m+31:m] := Convert_UInt8_To_FP32(a[i+7:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpu8_ps -FORCE_INLINE __m128 _mm_cvtpu8_ps(__m64 a) -{ - return vreinterpretq_m128_f32(vcvtq_f32_u32( - vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_m64(a)))))); + // signed shift right: faster than vclt + // (b < 0) ? 0xFFFF : 0 + uint16x8_t ltMask = vreinterpretq_u16_s16(vshrq_n_s16(b, 15)); + // (b == 0) ? 0xFFFF : 0 +#if defined(__aarch64__) || defined(_M_ARM64) + int16x8_t zeroMask = vreinterpretq_s16_u16(vceqzq_s16(b)); +#else + int16x8_t zeroMask = vreinterpretq_s16_u16(vceqq_s16(b, vdupq_n_s16(0))); +#endif + + // bitwise select either a or negative 'a' (vnegq_s16(a) equals to negative + // 'a') based on ltMask + int16x8_t masked = vbslq_s16(ltMask, vnegq_s16(a), a); + // res = masked & (~zeroMask) + int16x8_t res = vbicq_s16(masked, zeroMask); + return vreinterpretq_m128i_s16(res); } -// Converts the four single-precision, floating-point values of a to signed -// 32-bit integer values using truncate. -// https://msdn.microsoft.com/en-us/library/vstudio/1h005y6x(v=vs.100).aspx -FORCE_INLINE __m128i _mm_cvttps_epi32(__m128 a) +// Negate packed 32-bit integers in a when the corresponding signed +// 32-bit integer in b is negative, and store the results in dst. +// Element in dst are zeroed out when the corresponding element +// in b is zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sign_epi32 +FORCE_INLINE __m128i _mm_sign_epi32(__m128i _a, __m128i _b) { - return vreinterpretq_m128i_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a))); + int32x4_t a = vreinterpretq_s32_m128i(_a); + int32x4_t b = vreinterpretq_s32_m128i(_b); + + // signed shift right: faster than vclt + // (b < 0) ? 0xFFFFFFFF : 0 + uint32x4_t ltMask = vreinterpretq_u32_s32(vshrq_n_s32(b, 31)); + + // (b == 0) ? 0xFFFFFFFF : 0 +#if defined(__aarch64__) || defined(_M_ARM64) + int32x4_t zeroMask = vreinterpretq_s32_u32(vceqzq_s32(b)); +#else + int32x4_t zeroMask = vreinterpretq_s32_u32(vceqq_s32(b, vdupq_n_s32(0))); +#endif + + // bitwise select either a or negative 'a' (vnegq_s32(a) equals to negative + // 'a') based on ltMask + int32x4_t masked = vbslq_s32(ltMask, vnegq_s32(a), a); + // res = masked & (~zeroMask) + int32x4_t res = vbicq_s32(masked, zeroMask); + return vreinterpretq_m128i_s32(res); } -// Convert the lower double-precision (64-bit) floating-point element in a to a -// 64-bit integer with truncation, and store the result in dst. -// -// dst[63:0] := Convert_FP64_To_Int64_Truncate(a[63:0]) -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttsd_si64 -FORCE_INLINE int64_t _mm_cvttsd_si64(__m128d a) +// Negate packed 8-bit integers in a when the corresponding signed +// 8-bit integer in b is negative, and store the results in dst. +// Element in dst are zeroed out when the corresponding element +// in b is zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sign_epi8 +FORCE_INLINE __m128i _mm_sign_epi8(__m128i _a, __m128i _b) { -#if defined(__aarch64__) - return vgetq_lane_s64(vcvtq_s64_f64(vreinterpretq_f64_m128d(a)), 0); + int8x16_t a = vreinterpretq_s8_m128i(_a); + int8x16_t b = vreinterpretq_s8_m128i(_b); + + // signed shift right: faster than vclt + // (b < 0) ? 0xFF : 0 + uint8x16_t ltMask = vreinterpretq_u8_s8(vshrq_n_s8(b, 7)); + + // (b == 0) ? 0xFF : 0 +#if defined(__aarch64__) || defined(_M_ARM64) + int8x16_t zeroMask = vreinterpretq_s8_u8(vceqzq_s8(b)); #else - double ret = *((double *) &a); - return (int64_t) ret; + int8x16_t zeroMask = vreinterpretq_s8_u8(vceqq_s8(b, vdupq_n_s8(0))); #endif -} -// Convert the lower double-precision (64-bit) floating-point element in a to a -// 64-bit integer with truncation, and store the result in dst. -// -// dst[63:0] := Convert_FP64_To_Int64_Truncate(a[63:0]) -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttsd_si64x -#define _mm_cvttsd_si64x(a) _mm_cvttsd_si64(a) + // bitwise select either a or negative 'a' (vnegq_s8(a) return negative 'a') + // based on ltMask + int8x16_t masked = vbslq_s8(ltMask, vnegq_s8(a), a); + // res = masked & (~zeroMask) + int8x16_t res = vbicq_s8(masked, zeroMask); -// Converts the four signed 32-bit integer values of a to single-precision, -// floating-point values -// https://msdn.microsoft.com/en-us/library/vstudio/36bwxcx5(v=vs.100).aspx -FORCE_INLINE __m128 _mm_cvtepi32_ps(__m128i a) -{ - return vreinterpretq_m128_f32(vcvtq_f32_s32(vreinterpretq_s32_m128i(a))); + return vreinterpretq_m128i_s8(res); } -// Converts the four unsigned 8-bit integers in the lower 16 bits to four -// unsigned 32-bit integers. -FORCE_INLINE __m128i _mm_cvtepu8_epi16(__m128i a) +// Negate packed 16-bit integers in a when the corresponding signed 16-bit +// integer in b is negative, and store the results in dst. Element in dst are +// zeroed out when the corresponding element in b is zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sign_pi16 +FORCE_INLINE __m64 _mm_sign_pi16(__m64 _a, __m64 _b) { - uint8x16_t u8x16 = vreinterpretq_u8_m128i(a); /* xxxx xxxx xxxx DCBA */ - uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0x0x 0x0x 0D0C 0B0A */ - return vreinterpretq_m128i_u16(u16x8); -} + int16x4_t a = vreinterpret_s16_m64(_a); + int16x4_t b = vreinterpret_s16_m64(_b); -// Converts the four unsigned 8-bit integers in the lower 32 bits to four -// unsigned 32-bit integers. -// https://msdn.microsoft.com/en-us/library/bb531467%28v=vs.100%29.aspx -FORCE_INLINE __m128i _mm_cvtepu8_epi32(__m128i a) -{ - uint8x16_t u8x16 = vreinterpretq_u8_m128i(a); /* xxxx xxxx xxxx DCBA */ - uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0x0x 0x0x 0D0C 0B0A */ - uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000D 000C 000B 000A */ - return vreinterpretq_m128i_u32(u32x4); -} + // signed shift right: faster than vclt + // (b < 0) ? 0xFFFF : 0 + uint16x4_t ltMask = vreinterpret_u16_s16(vshr_n_s16(b, 15)); -// Converts the two unsigned 8-bit integers in the lower 16 bits to two -// unsigned 64-bit integers. -FORCE_INLINE __m128i _mm_cvtepu8_epi64(__m128i a) -{ - uint8x16_t u8x16 = vreinterpretq_u8_m128i(a); /* xxxx xxxx xxxx xxBA */ - uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0x0x 0x0x 0x0x 0B0A */ - uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000x 000x 000B 000A */ - uint64x2_t u64x2 = vmovl_u32(vget_low_u32(u32x4)); /* 0000 000B 0000 000A */ - return vreinterpretq_m128i_u64(u64x2); -} + // (b == 0) ? 0xFFFF : 0 +#if defined(__aarch64__) || defined(_M_ARM64) + int16x4_t zeroMask = vreinterpret_s16_u16(vceqz_s16(b)); +#else + int16x4_t zeroMask = vreinterpret_s16_u16(vceq_s16(b, vdup_n_s16(0))); +#endif -// Converts the four unsigned 8-bit integers in the lower 16 bits to four -// unsigned 32-bit integers. -FORCE_INLINE __m128i _mm_cvtepi8_epi16(__m128i a) -{ - int8x16_t s8x16 = vreinterpretq_s8_m128i(a); /* xxxx xxxx xxxx DCBA */ - int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0D0C 0B0A */ - return vreinterpretq_m128i_s16(s16x8); -} + // bitwise select either a or negative 'a' (vneg_s16(a) return negative 'a') + // based on ltMask + int16x4_t masked = vbsl_s16(ltMask, vneg_s16(a), a); + // res = masked & (~zeroMask) + int16x4_t res = vbic_s16(masked, zeroMask); -// Converts the four unsigned 8-bit integers in the lower 32 bits to four -// unsigned 32-bit integers. -FORCE_INLINE __m128i _mm_cvtepi8_epi32(__m128i a) -{ - int8x16_t s8x16 = vreinterpretq_s8_m128i(a); /* xxxx xxxx xxxx DCBA */ - int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0D0C 0B0A */ - int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000D 000C 000B 000A */ - return vreinterpretq_m128i_s32(s32x4); + return vreinterpret_m64_s16(res); } -// Converts the two signed 8-bit integers in the lower 32 bits to four -// signed 64-bit integers. -FORCE_INLINE __m128i _mm_cvtepi8_epi64(__m128i a) +// Negate packed 32-bit integers in a when the corresponding signed 32-bit +// integer in b is negative, and store the results in dst. Element in dst are +// zeroed out when the corresponding element in b is zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sign_pi32 +FORCE_INLINE __m64 _mm_sign_pi32(__m64 _a, __m64 _b) { - int8x16_t s8x16 = vreinterpretq_s8_m128i(a); /* xxxx xxxx xxxx xxBA */ - int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0x0x 0B0A */ - int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000x 000x 000B 000A */ - int64x2_t s64x2 = vmovl_s32(vget_low_s32(s32x4)); /* 0000 000B 0000 000A */ - return vreinterpretq_m128i_s64(s64x2); -} + int32x2_t a = vreinterpret_s32_m64(_a); + int32x2_t b = vreinterpret_s32_m64(_b); -// Converts the four signed 16-bit integers in the lower 64 bits to four signed -// 32-bit integers. -FORCE_INLINE __m128i _mm_cvtepi16_epi32(__m128i a) -{ - return vreinterpretq_m128i_s32( - vmovl_s16(vget_low_s16(vreinterpretq_s16_m128i(a)))); -} + // signed shift right: faster than vclt + // (b < 0) ? 0xFFFFFFFF : 0 + uint32x2_t ltMask = vreinterpret_u32_s32(vshr_n_s32(b, 31)); -// Converts the two signed 16-bit integers in the lower 32 bits two signed -// 32-bit integers. -FORCE_INLINE __m128i _mm_cvtepi16_epi64(__m128i a) -{ - int16x8_t s16x8 = vreinterpretq_s16_m128i(a); /* xxxx xxxx xxxx 0B0A */ - int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000x 000x 000B 000A */ - int64x2_t s64x2 = vmovl_s32(vget_low_s32(s32x4)); /* 0000 000B 0000 000A */ - return vreinterpretq_m128i_s64(s64x2); -} + // (b == 0) ? 0xFFFFFFFF : 0 +#if defined(__aarch64__) || defined(_M_ARM64) + int32x2_t zeroMask = vreinterpret_s32_u32(vceqz_s32(b)); +#else + int32x2_t zeroMask = vreinterpret_s32_u32(vceq_s32(b, vdup_n_s32(0))); +#endif -// Converts the four unsigned 16-bit integers in the lower 64 bits to four -// unsigned 32-bit integers. -FORCE_INLINE __m128i _mm_cvtepu16_epi32(__m128i a) -{ - return vreinterpretq_m128i_u32( - vmovl_u16(vget_low_u16(vreinterpretq_u16_m128i(a)))); -} + // bitwise select either a or negative 'a' (vneg_s32(a) return negative 'a') + // based on ltMask + int32x2_t masked = vbsl_s32(ltMask, vneg_s32(a), a); + // res = masked & (~zeroMask) + int32x2_t res = vbic_s32(masked, zeroMask); -// Converts the two unsigned 16-bit integers in the lower 32 bits to two -// unsigned 64-bit integers. -FORCE_INLINE __m128i _mm_cvtepu16_epi64(__m128i a) -{ - uint16x8_t u16x8 = vreinterpretq_u16_m128i(a); /* xxxx xxxx xxxx 0B0A */ - uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000x 000x 000B 000A */ - uint64x2_t u64x2 = vmovl_u32(vget_low_u32(u32x4)); /* 0000 000B 0000 000A */ - return vreinterpretq_m128i_u64(u64x2); + return vreinterpret_m64_s32(res); } -// Converts the two unsigned 32-bit integers in the lower 64 bits to two -// unsigned 64-bit integers. -FORCE_INLINE __m128i _mm_cvtepu32_epi64(__m128i a) +// Negate packed 8-bit integers in a when the corresponding signed 8-bit integer +// in b is negative, and store the results in dst. Element in dst are zeroed out +// when the corresponding element in b is zero. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sign_pi8 +FORCE_INLINE __m64 _mm_sign_pi8(__m64 _a, __m64 _b) { - return vreinterpretq_m128i_u64( - vmovl_u32(vget_low_u32(vreinterpretq_u32_m128i(a)))); -} + int8x8_t a = vreinterpret_s8_m64(_a); + int8x8_t b = vreinterpret_s8_m64(_b); -// Converts the two signed 32-bit integers in the lower 64 bits to two signed -// 64-bit integers. -FORCE_INLINE __m128i _mm_cvtepi32_epi64(__m128i a) -{ - return vreinterpretq_m128i_s64( - vmovl_s32(vget_low_s32(vreinterpretq_s32_m128i(a)))); -} + // signed shift right: faster than vclt + // (b < 0) ? 0xFF : 0 + uint8x8_t ltMask = vreinterpret_u8_s8(vshr_n_s8(b, 7)); -// Converts the four single-precision, floating-point values of a to signed -// 32-bit integer values. -// -// r0 := (int) a0 -// r1 := (int) a1 -// r2 := (int) a2 -// r3 := (int) a3 -// -// https://msdn.microsoft.com/en-us/library/vstudio/xdc42k5e(v=vs.100).aspx -// *NOTE*. The default rounding mode on SSE is 'round to even', which ARMv7-A -// does not support! It is supported on ARMv8-A however. -FORCE_INLINE __m128i _mm_cvtps_epi32(__m128 a) -{ -#if defined(__aarch64__) - return vreinterpretq_m128i_s32(vcvtnq_s32_f32(a)); -#else - uint32x4_t signmask = vdupq_n_u32(0x80000000); - float32x4_t half = vbslq_f32(signmask, vreinterpretq_f32_m128(a), - vdupq_n_f32(0.5f)); /* +/- 0.5 */ - int32x4_t r_normal = vcvtq_s32_f32(vaddq_f32( - vreinterpretq_f32_m128(a), half)); /* round to integer: [a + 0.5]*/ - int32x4_t r_trunc = - vcvtq_s32_f32(vreinterpretq_f32_m128(a)); /* truncate to integer: [a] */ - int32x4_t plusone = vreinterpretq_s32_u32(vshrq_n_u32( - vreinterpretq_u32_s32(vnegq_s32(r_trunc)), 31)); /* 1 or 0 */ - int32x4_t r_even = vbicq_s32(vaddq_s32(r_trunc, plusone), - vdupq_n_s32(1)); /* ([a] + {0,1}) & ~1 */ - float32x4_t delta = vsubq_f32( - vreinterpretq_f32_m128(a), - vcvtq_f32_s32(r_trunc)); /* compute delta: delta = (a - [a]) */ - uint32x4_t is_delta_half = vceqq_f32(delta, half); /* delta == +/- 0.5 */ - return vreinterpretq_m128i_s32(vbslq_s32(is_delta_half, r_even, r_normal)); + // (b == 0) ? 0xFF : 0 +#if defined(__aarch64__) || defined(_M_ARM64) + int8x8_t zeroMask = vreinterpret_s8_u8(vceqz_s8(b)); +#else + int8x8_t zeroMask = vreinterpret_s8_u8(vceq_s8(b, vdup_n_s8(0))); #endif + + // bitwise select either a or negative 'a' (vneg_s8(a) return negative 'a') + // based on ltMask + int8x8_t masked = vbsl_s8(ltMask, vneg_s8(a), a); + // res = masked & (~zeroMask) + int8x8_t res = vbic_s8(masked, zeroMask); + + return vreinterpret_m64_s8(res); } -// Copy the lower 32-bit integer in a to dst. -// -// dst[31:0] := a[31:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi128_si32 -FORCE_INLINE int _mm_cvtsi128_si32(__m128i a) +/* SSE4.1 */ + +// Blend packed 16-bit integers from a and b using control mask imm8, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_blend_epi16 +// FORCE_INLINE __m128i _mm_blend_epi16(__m128i a, __m128i b, +// __constrange(0,255) int imm) +#define _mm_blend_epi16(a, b, imm) \ + _sse2neon_define2( \ + __m128i, a, b, \ + const uint16_t _mask[8] = \ + _sse2neon_init(((imm) & (1 << 0)) ? (uint16_t) -1 : 0x0, \ + ((imm) & (1 << 1)) ? (uint16_t) -1 : 0x0, \ + ((imm) & (1 << 2)) ? (uint16_t) -1 : 0x0, \ + ((imm) & (1 << 3)) ? (uint16_t) -1 : 0x0, \ + ((imm) & (1 << 4)) ? (uint16_t) -1 : 0x0, \ + ((imm) & (1 << 5)) ? (uint16_t) -1 : 0x0, \ + ((imm) & (1 << 6)) ? (uint16_t) -1 : 0x0, \ + ((imm) & (1 << 7)) ? (uint16_t) -1 : 0x0); \ + uint16x8_t _mask_vec = vld1q_u16(_mask); \ + uint16x8_t __a = vreinterpretq_u16_m128i(_a); \ + uint16x8_t __b = vreinterpretq_u16_m128i(_b); _sse2neon_return( \ + vreinterpretq_m128i_u16(vbslq_u16(_mask_vec, __b, __a)));) + +// Blend packed double-precision (64-bit) floating-point elements from a and b +// using control mask imm8, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_blend_pd +#define _mm_blend_pd(a, b, imm) \ + _sse2neon_define2( \ + __m128d, a, b, \ + const uint64_t _mask[2] = \ + _sse2neon_init(((imm) & (1 << 0)) ? ~UINT64_C(0) : UINT64_C(0), \ + ((imm) & (1 << 1)) ? ~UINT64_C(0) : UINT64_C(0)); \ + uint64x2_t _mask_vec = vld1q_u64(_mask); \ + uint64x2_t __a = vreinterpretq_u64_m128d(_a); \ + uint64x2_t __b = vreinterpretq_u64_m128d(_b); _sse2neon_return( \ + vreinterpretq_m128d_u64(vbslq_u64(_mask_vec, __b, __a)));) + +// Blend packed single-precision (32-bit) floating-point elements from a and b +// using mask, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_blend_ps +FORCE_INLINE __m128 _mm_blend_ps(__m128 _a, __m128 _b, const char imm8) { - return vgetq_lane_s32(vreinterpretq_s32_m128i(a), 0); + const uint32_t ALIGN_STRUCT(16) + data[4] = {((imm8) & (1 << 0)) ? UINT32_MAX : 0, + ((imm8) & (1 << 1)) ? UINT32_MAX : 0, + ((imm8) & (1 << 2)) ? UINT32_MAX : 0, + ((imm8) & (1 << 3)) ? UINT32_MAX : 0}; + uint32x4_t mask = vld1q_u32(data); + float32x4_t a = vreinterpretq_f32_m128(_a); + float32x4_t b = vreinterpretq_f32_m128(_b); + return vreinterpretq_m128_f32(vbslq_f32(mask, b, a)); } -// Copy the lower 64-bit integer in a to dst. -// -// dst[63:0] := a[63:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi128_si64 -FORCE_INLINE int64_t _mm_cvtsi128_si64(__m128i a) +// Blend packed 8-bit integers from a and b using mask, and store the results in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_blendv_epi8 +FORCE_INLINE __m128i _mm_blendv_epi8(__m128i _a, __m128i _b, __m128i _mask) { - return vgetq_lane_s64(vreinterpretq_s64_m128i(a), 0); + // Use a signed shift right to create a mask with the sign bit + uint8x16_t mask = + vreinterpretq_u8_s8(vshrq_n_s8(vreinterpretq_s8_m128i(_mask), 7)); + uint8x16_t a = vreinterpretq_u8_m128i(_a); + uint8x16_t b = vreinterpretq_u8_m128i(_b); + return vreinterpretq_m128i_u8(vbslq_u8(mask, b, a)); } -// Copy the lower 64-bit integer in a to dst. -// -// dst[63:0] := a[63:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsi128_si64x -#define _mm_cvtsi128_si64x(a) _mm_cvtsi128_si64(a) - -// Moves 32-bit integer a to the least significant 32 bits of an __m128 object, -// zero extending the upper bits. -// -// r0 := a -// r1 := 0x0 -// r2 := 0x0 -// r3 := 0x0 -// -// https://msdn.microsoft.com/en-us/library/ct3539ha%28v=vs.90%29.aspx -FORCE_INLINE __m128i _mm_cvtsi32_si128(int a) +// Blend packed double-precision (64-bit) floating-point elements from a and b +// using mask, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_blendv_pd +FORCE_INLINE __m128d _mm_blendv_pd(__m128d _a, __m128d _b, __m128d _mask) { - return vreinterpretq_m128i_s32(vsetq_lane_s32(a, vdupq_n_s32(0), 0)); + uint64x2_t mask = + vreinterpretq_u64_s64(vshrq_n_s64(vreinterpretq_s64_m128d(_mask), 63)); +#if defined(__aarch64__) || defined(_M_ARM64) + float64x2_t a = vreinterpretq_f64_m128d(_a); + float64x2_t b = vreinterpretq_f64_m128d(_b); + return vreinterpretq_m128d_f64(vbslq_f64(mask, b, a)); +#else + uint64x2_t a = vreinterpretq_u64_m128d(_a); + uint64x2_t b = vreinterpretq_u64_m128d(_b); + return vreinterpretq_m128d_u64(vbslq_u64(mask, b, a)); +#endif } -// Moves 64-bit integer a to the least significant 64 bits of an __m128 object, -// zero extending the upper bits. -// -// r0 := a -// r1 := 0x0 -FORCE_INLINE __m128i _mm_cvtsi64_si128(int64_t a) +// Blend packed single-precision (32-bit) floating-point elements from a and b +// using mask, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_blendv_ps +FORCE_INLINE __m128 _mm_blendv_ps(__m128 _a, __m128 _b, __m128 _mask) { - return vreinterpretq_m128i_s64(vsetq_lane_s64(a, vdupq_n_s64(0), 0)); + // Use a signed shift right to create a mask with the sign bit + uint32x4_t mask = + vreinterpretq_u32_s32(vshrq_n_s32(vreinterpretq_s32_m128(_mask), 31)); + float32x4_t a = vreinterpretq_f32_m128(_a); + float32x4_t b = vreinterpretq_f32_m128(_b); + return vreinterpretq_m128_f32(vbslq_f32(mask, b, a)); } -// Cast vector of type __m128 to type __m128d. This intrinsic is only used for -// compilation and does not generate any instructions, thus it has zero latency. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_castps_pd -FORCE_INLINE __m128d _mm_castps_pd(__m128 a) +// Round the packed double-precision (64-bit) floating-point elements in a up +// to an integer value, and store the results as packed double-precision +// floating-point elements in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ceil_pd +FORCE_INLINE __m128d _mm_ceil_pd(__m128d a) { - return vreinterpretq_m128d_s32(vreinterpretq_s32_m128(a)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vrndpq_f64(vreinterpretq_f64_m128d(a))); +#else + double *f = (double *) &a; + return _mm_set_pd(ceil(f[1]), ceil(f[0])); +#endif } -// Applies a type cast to reinterpret four 32-bit floating point values passed -// in as a 128-bit parameter as packed 32-bit integers. -// https://msdn.microsoft.com/en-us/library/bb514099.aspx -FORCE_INLINE __m128i _mm_castps_si128(__m128 a) +// Round the packed single-precision (32-bit) floating-point elements in a up to +// an integer value, and store the results as packed single-precision +// floating-point elements in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ceil_ps +FORCE_INLINE __m128 _mm_ceil_ps(__m128 a) { - return vreinterpretq_m128i_s32(vreinterpretq_s32_m128(a)); +#if (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ARM_FEATURE_DIRECTED_ROUNDING) + return vreinterpretq_m128_f32(vrndpq_f32(vreinterpretq_f32_m128(a))); +#else + float *f = (float *) &a; + return _mm_set_ps(ceilf(f[3]), ceilf(f[2]), ceilf(f[1]), ceilf(f[0])); +#endif } -// Applies a type cast to reinterpret four 32-bit integers passed in as a -// 128-bit parameter as packed 32-bit floating point values. -// https://msdn.microsoft.com/en-us/library/bb514029.aspx -FORCE_INLINE __m128 _mm_castsi128_ps(__m128i a) +// Round the lower double-precision (64-bit) floating-point element in b up to +// an integer value, store the result as a double-precision floating-point +// element in the lower element of dst, and copy the upper element from a to the +// upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ceil_sd +FORCE_INLINE __m128d _mm_ceil_sd(__m128d a, __m128d b) { - return vreinterpretq_m128_s32(vreinterpretq_s32_m128i(a)); + return _mm_move_sd(a, _mm_ceil_pd(b)); } -// Loads 128-bit value. : -// https://msdn.microsoft.com/en-us/library/atzzad1h(v=vs.80).aspx -FORCE_INLINE __m128i _mm_load_si128(const __m128i *p) +// Round the lower single-precision (32-bit) floating-point element in b up to +// an integer value, store the result as a single-precision floating-point +// element in the lower element of dst, and copy the upper 3 packed elements +// from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_ceil_ss +FORCE_INLINE __m128 _mm_ceil_ss(__m128 a, __m128 b) { - return vreinterpretq_m128i_s32(vld1q_s32((const int32_t *) p)); + return _mm_move_ss(a, _mm_ceil_ps(b)); } -// Load a double-precision (64-bit) floating-point element from memory into both -// elements of dst. -// -// dst[63:0] := MEM[mem_addr+63:mem_addr] -// dst[127:64] := MEM[mem_addr+63:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load1_pd -FORCE_INLINE __m128d _mm_load1_pd(const double *p) +// Compare packed 64-bit integers in a and b for equality, and store the results +// in dst +FORCE_INLINE __m128i _mm_cmpeq_epi64(__m128i a, __m128i b) { -#if defined(__aarch64__) - return vreinterpretq_m128d_f64(vld1q_dup_f64(p)); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_u64( + vceqq_u64(vreinterpretq_u64_m128i(a), vreinterpretq_u64_m128i(b))); #else - return vreinterpretq_m128d_s64(vdupq_n_s64(*(const int64_t *) p)); + // ARMv7 lacks vceqq_u64 + // (a == b) -> (a_lo == b_lo) && (a_hi == b_hi) + uint32x4_t cmp = + vceqq_u32(vreinterpretq_u32_m128i(a), vreinterpretq_u32_m128i(b)); + uint32x4_t swapped = vrev64q_u32(cmp); + return vreinterpretq_m128i_u32(vandq_u32(cmp, swapped)); #endif } -// Load a double-precision (64-bit) floating-point element from memory into the -// upper element of dst, and copy the lower element from a to dst. mem_addr does -// not need to be aligned on any particular boundary. -// -// dst[63:0] := a[63:0] -// dst[127:64] := MEM[mem_addr+63:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadh_pd -FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double *p) +// Sign extend packed 16-bit integers in a to packed 32-bit integers, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepi16_epi32 +FORCE_INLINE __m128i _mm_cvtepi16_epi32(__m128i a) { -#if defined(__aarch64__) - return vreinterpretq_m128d_f64( - vcombine_f64(vget_low_f64(vreinterpretq_f64_m128d(a)), vld1_f64(p))); -#else - return vreinterpretq_m128d_f32(vcombine_f32( - vget_low_f32(vreinterpretq_f32_m128d(a)), vld1_f32((const float *) p))); -#endif + return vreinterpretq_m128i_s32( + vmovl_s16(vget_low_s16(vreinterpretq_s16_m128i(a)))); } -// Load a double-precision (64-bit) floating-point element from memory into both -// elements of dst. -// -// dst[63:0] := MEM[mem_addr+63:mem_addr] -// dst[127:64] := MEM[mem_addr+63:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_load_pd1 -#define _mm_load_pd1 _mm_load1_pd - -// Load a double-precision (64-bit) floating-point element from memory into both -// elements of dst. -// -// dst[63:0] := MEM[mem_addr+63:mem_addr] -// dst[127:64] := MEM[mem_addr+63:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loaddup_pd -#define _mm_loaddup_pd _mm_load1_pd +// Sign extend packed 16-bit integers in a to packed 64-bit integers, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepi16_epi64 +FORCE_INLINE __m128i _mm_cvtepi16_epi64(__m128i a) +{ + int16x8_t s16x8 = vreinterpretq_s16_m128i(a); /* xxxx xxxx xxxx 0B0A */ + int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000x 000x 000B 000A */ + int64x2_t s64x2 = vmovl_s32(vget_low_s32(s32x4)); /* 0000 000B 0000 000A */ + return vreinterpretq_m128i_s64(s64x2); +} -// Loads 128-bit value. : -// https://msdn.microsoft.com/zh-cn/library/f4k12ae8(v=vs.90).aspx -FORCE_INLINE __m128i _mm_loadu_si128(const __m128i *p) +// Sign extend packed 32-bit integers in a to packed 64-bit integers, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepi32_epi64 +FORCE_INLINE __m128i _mm_cvtepi32_epi64(__m128i a) { - return vreinterpretq_m128i_s32(vld1q_s32((const int32_t *) p)); + return vreinterpretq_m128i_s64( + vmovl_s32(vget_low_s32(vreinterpretq_s32_m128i(a)))); } -// Load unaligned 32-bit integer from memory into the first element of dst. -// -// dst[31:0] := MEM[mem_addr+31:mem_addr] -// dst[MAX:32] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_loadu_si32 -FORCE_INLINE __m128i _mm_loadu_si32(const void *p) +// Sign extend packed 8-bit integers in a to packed 16-bit integers, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepi8_epi16 +FORCE_INLINE __m128i _mm_cvtepi8_epi16(__m128i a) { - return vreinterpretq_m128i_s32( - vsetq_lane_s32(*(const int32_t *) p, vdupq_n_s32(0), 0)); + int8x16_t s8x16 = vreinterpretq_s8_m128i(a); /* xxxx xxxx xxxx DCBA */ + int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0D0C 0B0A */ + return vreinterpretq_m128i_s16(s16x8); } -// Convert packed double-precision (64-bit) floating-point elements in a to -// packed single-precision (32-bit) floating-point elements, and store the -// results in dst. -// -// FOR j := 0 to 1 -// i := 32*j -// k := 64*j -// dst[i+31:i] := Convert_FP64_To_FP32(a[k+64:k]) -// ENDFOR -// dst[127:64] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtpd_ps -FORCE_INLINE __m128 _mm_cvtpd_ps(__m128d a) +// Sign extend packed 8-bit integers in a to packed 32-bit integers, and store +// the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepi8_epi32 +FORCE_INLINE __m128i _mm_cvtepi8_epi32(__m128i a) { -#if defined(__aarch64__) - float32x2_t tmp = vcvt_f32_f64(vreinterpretq_f64_m128d(a)); - return vreinterpretq_m128_f32(vcombine_f32(tmp, vdup_n_f32(0))); -#else - float a0 = (float) ((double *) &a)[0]; - float a1 = (float) ((double *) &a)[1]; - return _mm_set_ps(0, 0, a1, a0); -#endif + int8x16_t s8x16 = vreinterpretq_s8_m128i(a); /* xxxx xxxx xxxx DCBA */ + int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0D0C 0B0A */ + int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000D 000C 000B 000A */ + return vreinterpretq_m128i_s32(s32x4); } -// Copy the lower double-precision (64-bit) floating-point element of a to dst. -// -// dst[63:0] := a[63:0] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtsd_f64 -FORCE_INLINE double _mm_cvtsd_f64(__m128d a) +// Sign extend packed 8-bit integers in the low 8 bytes of a to packed 64-bit +// integers, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepi8_epi64 +FORCE_INLINE __m128i _mm_cvtepi8_epi64(__m128i a) { -#if defined(__aarch64__) - return (double) vgetq_lane_f64(vreinterpretq_f64_m128d(a), 0); -#else - return ((double *) &a)[0]; -#endif + int8x16_t s8x16 = vreinterpretq_s8_m128i(a); /* xxxx xxxx xxxx xxBA */ + int16x8_t s16x8 = vmovl_s8(vget_low_s8(s8x16)); /* 0x0x 0x0x 0x0x 0B0A */ + int32x4_t s32x4 = vmovl_s16(vget_low_s16(s16x8)); /* 000x 000x 000B 000A */ + int64x2_t s64x2 = vmovl_s32(vget_low_s32(s32x4)); /* 0000 000B 0000 000A */ + return vreinterpretq_m128i_s64(s64x2); } -// Convert packed single-precision (32-bit) floating-point elements in a to -// packed double-precision (64-bit) floating-point elements, and store the -// results in dst. -// -// FOR j := 0 to 1 -// i := 64*j -// k := 32*j -// dst[i+63:i] := Convert_FP32_To_FP64(a[k+31:k]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtps_pd -FORCE_INLINE __m128d _mm_cvtps_pd(__m128 a) +// Zero extend packed unsigned 16-bit integers in a to packed 32-bit integers, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepu16_epi32 +FORCE_INLINE __m128i _mm_cvtepu16_epi32(__m128i a) { -#if defined(__aarch64__) - return vreinterpretq_m128d_f64( - vcvt_f64_f32(vget_low_f32(vreinterpretq_f32_m128(a)))); -#else - double a0 = (double) vgetq_lane_f32(vreinterpretq_f32_m128(a), 0); - double a1 = (double) vgetq_lane_f32(vreinterpretq_f32_m128(a), 1); - return _mm_set_pd(a1, a0); -#endif + return vreinterpretq_m128i_u32( + vmovl_u16(vget_low_u16(vreinterpretq_u16_m128i(a)))); } -// Cast vector of type __m128d to type __m128i. This intrinsic is only used for -// compilation and does not generate any instructions, thus it has zero latency. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_castpd_si128 -FORCE_INLINE __m128i _mm_castpd_si128(__m128d a) +// Zero extend packed unsigned 16-bit integers in a to packed 64-bit integers, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepu16_epi64 +FORCE_INLINE __m128i _mm_cvtepu16_epi64(__m128i a) { - return vreinterpretq_m128i_s64(vreinterpretq_s64_m128d(a)); + uint16x8_t u16x8 = vreinterpretq_u16_m128i(a); /* xxxx xxxx xxxx 0B0A */ + uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000x 000x 000B 000A */ + uint64x2_t u64x2 = vmovl_u32(vget_low_u32(u32x4)); /* 0000 000B 0000 000A */ + return vreinterpretq_m128i_u64(u64x2); } -// Cast vector of type __m128d to type __m128. This intrinsic is only used for -// compilation and does not generate any instructions, thus it has zero latency. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_castpd_ps -FORCE_INLINE __m128 _mm_castpd_ps(__m128d a) +// Zero extend packed unsigned 32-bit integers in a to packed 64-bit integers, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepu32_epi64 +FORCE_INLINE __m128i _mm_cvtepu32_epi64(__m128i a) { - return vreinterpretq_m128_s64(vreinterpretq_s64_m128d(a)); + return vreinterpretq_m128i_u64( + vmovl_u32(vget_low_u32(vreinterpretq_u32_m128i(a)))); } -// Blend packed single-precision (32-bit) floating-point elements from a and b -// using mask, and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blendv_ps -FORCE_INLINE __m128 _mm_blendv_ps(__m128 a, __m128 b, __m128 mask) +// Zero extend packed unsigned 8-bit integers in a to packed 16-bit integers, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepu8_epi16 +FORCE_INLINE __m128i _mm_cvtepu8_epi16(__m128i a) { - return vreinterpretq_m128_f32(vbslq_f32(vreinterpretq_u32_m128(mask), - vreinterpretq_f32_m128(b), - vreinterpretq_f32_m128(a))); + uint8x16_t u8x16 = vreinterpretq_u8_m128i(a); /* xxxx xxxx HGFE DCBA */ + uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0H0G 0F0E 0D0C 0B0A */ + return vreinterpretq_m128i_u16(u16x8); } -// Blend packed double-precision (64-bit) floating-point elements from a and b -// using mask, and store the results in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_blendv_pd -FORCE_INLINE __m128d _mm_blendv_pd(__m128d _a, __m128d _b, __m128d _mask) +// Zero extend packed unsigned 8-bit integers in a to packed 32-bit integers, +// and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepu8_epi32 +FORCE_INLINE __m128i _mm_cvtepu8_epi32(__m128i a) { - uint64x2_t mask = - vreinterpretq_u64_s64(vshrq_n_s64(vreinterpretq_s64_m128d(_mask), 63)); -#if defined(__aarch64__) - float64x2_t a = vreinterpretq_f64_m128d(_a); - float64x2_t b = vreinterpretq_f64_m128d(_b); - return vreinterpretq_m128d_f64(vbslq_f64(mask, b, a)); -#else - uint64x2_t a = vreinterpretq_u64_m128d(_a); - uint64x2_t b = vreinterpretq_u64_m128d(_b); - return vreinterpretq_m128d_u64(vbslq_u64(mask, b, a)); -#endif + uint8x16_t u8x16 = vreinterpretq_u8_m128i(a); /* xxxx xxxx xxxx DCBA */ + uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0x0x 0x0x 0D0C 0B0A */ + uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000D 000C 000B 000A */ + return vreinterpretq_m128i_u32(u32x4); } -// Round the packed single-precision (32-bit) floating-point elements in a using -// the rounding parameter, and store the results as packed single-precision -// floating-point elements in dst. -// software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_ps -FORCE_INLINE __m128 _mm_round_ps(__m128 a, int rounding) +// Zero extend packed unsigned 8-bit integers in the low 8 bytes of a to packed +// 64-bit integers, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtepu8_epi64 +FORCE_INLINE __m128i _mm_cvtepu8_epi64(__m128i a) { -#if defined(__aarch64__) - switch (rounding) { - case (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC): - return vreinterpretq_m128_f32(vrndnq_f32(vreinterpretq_f32_m128(a))); - case (_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC): - return vreinterpretq_m128_f32(vrndmq_f32(vreinterpretq_f32_m128(a))); - case (_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC): - return vreinterpretq_m128_f32(vrndpq_f32(vreinterpretq_f32_m128(a))); - case (_MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC): - return vreinterpretq_m128_f32(vrndq_f32(vreinterpretq_f32_m128(a))); - default: //_MM_FROUND_CUR_DIRECTION - return vreinterpretq_m128_f32(vrndiq_f32(vreinterpretq_f32_m128(a))); - } -#else - float *v_float = (float *) &a; - __m128 zero, neg_inf, pos_inf; - - switch (rounding) { - case (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC): - return _mm_cvtepi32_ps(_mm_cvtps_epi32(a)); - case (_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC): - return (__m128){floorf(v_float[0]), floorf(v_float[1]), - floorf(v_float[2]), floorf(v_float[3])}; - case (_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC): - return (__m128){ceilf(v_float[0]), ceilf(v_float[1]), ceilf(v_float[2]), - ceilf(v_float[3])}; - case (_MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC): - zero = _mm_set_ps(0.0f, 0.0f, 0.0f, 0.0f); - neg_inf = _mm_set_ps(floorf(v_float[0]), floorf(v_float[1]), - floorf(v_float[2]), floorf(v_float[3])); - pos_inf = _mm_set_ps(ceilf(v_float[0]), ceilf(v_float[1]), - ceilf(v_float[2]), ceilf(v_float[3])); - return _mm_blendv_ps(pos_inf, neg_inf, _mm_cmple_ps(a, zero)); - default: //_MM_FROUND_CUR_DIRECTION - return (__m128){roundf(v_float[0]), roundf(v_float[1]), - roundf(v_float[2]), roundf(v_float[3])}; - } -#endif + uint8x16_t u8x16 = vreinterpretq_u8_m128i(a); /* xxxx xxxx xxxx xxBA */ + uint16x8_t u16x8 = vmovl_u8(vget_low_u8(u8x16)); /* 0x0x 0x0x 0x0x 0B0A */ + uint32x4_t u32x4 = vmovl_u16(vget_low_u16(u16x8)); /* 000x 000x 000B 000A */ + uint64x2_t u64x2 = vmovl_u32(vget_low_u32(u32x4)); /* 0000 000B 0000 000A */ + return vreinterpretq_m128i_u64(u64x2); } -// Convert packed single-precision (32-bit) floating-point elements in a to -// packed 32-bit integers, and store the results in dst. -// -// FOR j := 0 to 1 -// i := 32*j -// dst[i+31:i] := Convert_FP32_To_Int32(a[i+31:i]) -// ENDFOR -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvt_ps2pi -FORCE_INLINE __m64 _mm_cvt_ps2pi(__m128 a) +// Conditionally multiply the packed double-precision (64-bit) floating-point +// elements in a and b using the high 4 bits in imm8, sum the four products, and +// conditionally store the sum in dst using the low 4 bits of imm8. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_dp_pd +FORCE_INLINE __m128d _mm_dp_pd(__m128d a, __m128d b, const int imm) { -#if defined(__aarch64__) - return vreinterpret_m64_s32( - vget_low_s32(vcvtnq_s32_f32(vreinterpretq_f32_m128(a)))); + // Generate mask value from constant immediate bit value + const int64_t bit0Mask = imm & 0x01 ? UINT64_MAX : 0; + const int64_t bit1Mask = imm & 0x02 ? UINT64_MAX : 0; +#if !SSE2NEON_PRECISE_DP + const int64_t bit4Mask = imm & 0x10 ? UINT64_MAX : 0; + const int64_t bit5Mask = imm & 0x20 ? UINT64_MAX : 0; +#endif + // Conditional multiplication +#if !SSE2NEON_PRECISE_DP + __m128d mul = _mm_mul_pd(a, b); + const __m128d mulMask = + _mm_castsi128_pd(_mm_set_epi64x(bit5Mask, bit4Mask)); + __m128d tmp = _mm_and_pd(mul, mulMask); #else - return vreinterpret_m64_s32( - vcvt_s32_f32(vget_low_f32(vreinterpretq_f32_m128( - _mm_round_ps(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC))))); +#if defined(__aarch64__) || defined(_M_ARM64) + double d0 = (imm & 0x10) ? vgetq_lane_f64(vreinterpretq_f64_m128d(a), 0) * + vgetq_lane_f64(vreinterpretq_f64_m128d(b), 0) + : 0; + double d1 = (imm & 0x20) ? vgetq_lane_f64(vreinterpretq_f64_m128d(a), 1) * + vgetq_lane_f64(vreinterpretq_f64_m128d(b), 1) + : 0; +#else + double d0 = (imm & 0x10) ? ((double *) &a)[0] * ((double *) &b)[0] : 0; + double d1 = (imm & 0x20) ? ((double *) &a)[1] * ((double *) &b)[1] : 0; +#endif + __m128d tmp = _mm_set_pd(d1, d0); +#endif + // Sum the products +#if defined(__aarch64__) || defined(_M_ARM64) + double sum = vpaddd_f64(vreinterpretq_f64_m128d(tmp)); +#else + double sum = *((double *) &tmp) + *(((double *) &tmp) + 1); #endif + // Conditionally store the sum + const __m128d sumMask = + _mm_castsi128_pd(_mm_set_epi64x(bit1Mask, bit0Mask)); + __m128d res = _mm_and_pd(_mm_set_pd1(sum), sumMask); + return res; } -// Round the packed single-precision (32-bit) floating-point elements in a up to -// an integer value, and store the results as packed single-precision -// floating-point elements in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_ceil_ps -FORCE_INLINE __m128 _mm_ceil_ps(__m128 a) +// Conditionally multiply the packed single-precision (32-bit) floating-point +// elements in a and b using the high 4 bits in imm8, sum the four products, +// and conditionally store the sum in dst using the low 4 bits of imm. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_dp_ps +FORCE_INLINE __m128 _mm_dp_ps(__m128 a, __m128 b, const int imm) { - return _mm_round_ps(a, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC); -} + float32x4_t elementwise_prod = _mm_mul_ps(a, b); -// Round the packed single-precision (32-bit) floating-point elements in a down -// to an integer value, and store the results as packed single-precision -// floating-point elements in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_floor_ps -FORCE_INLINE __m128 _mm_floor_ps(__m128 a) -{ - return _mm_round_ps(a, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC); -} +#if defined(__aarch64__) || defined(_M_ARM64) + /* shortcuts */ + if (imm == 0xFF) { + return _mm_set1_ps(vaddvq_f32(elementwise_prod)); + } + if ((imm & 0x0F) == 0x0F) { + if (!(imm & (1 << 4))) + elementwise_prod = vsetq_lane_f32(0.0f, elementwise_prod, 0); + if (!(imm & (1 << 5))) + elementwise_prod = vsetq_lane_f32(0.0f, elementwise_prod, 1); + if (!(imm & (1 << 6))) + elementwise_prod = vsetq_lane_f32(0.0f, elementwise_prod, 2); + if (!(imm & (1 << 7))) + elementwise_prod = vsetq_lane_f32(0.0f, elementwise_prod, 3); + + return _mm_set1_ps(vaddvq_f32(elementwise_prod)); + } +#endif -// Load 128-bits of integer data from unaligned memory into dst. This intrinsic -// may perform better than _mm_loadu_si128 when the data crosses a cache line -// boundary. -// -// dst[127:0] := MEM[mem_addr+127:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_lddqu_si128 -#define _mm_lddqu_si128 _mm_loadu_si128 + float s = 0.0f; -/* Miscellaneous Operations */ + if (imm & (1 << 4)) + s += vgetq_lane_f32(elementwise_prod, 0); + if (imm & (1 << 5)) + s += vgetq_lane_f32(elementwise_prod, 1); + if (imm & (1 << 6)) + s += vgetq_lane_f32(elementwise_prod, 2); + if (imm & (1 << 7)) + s += vgetq_lane_f32(elementwise_prod, 3); -// Shifts the 8 signed 16-bit integers in a right by count bits while shifting -// in the sign bit. -// -// r0 := a0 >> count -// r1 := a1 >> count -// ... -// r7 := a7 >> count -// -// https://msdn.microsoft.com/en-us/library/3c9997dk(v%3dvs.90).aspx -FORCE_INLINE __m128i _mm_sra_epi16(__m128i a, __m128i count) -{ - int64_t c = (int64_t) vget_low_s64((int64x2_t) count); - if (c > 15) - return _mm_cmplt_epi16(a, _mm_setzero_si128()); - return vreinterpretq_m128i_s16(vshlq_s16((int16x8_t) a, vdupq_n_s16(-c))); + const float32_t res[4] = { + (imm & 0x1) ? s : 0.0f, + (imm & 0x2) ? s : 0.0f, + (imm & 0x4) ? s : 0.0f, + (imm & 0x8) ? s : 0.0f, + }; + return vreinterpretq_m128_f32(vld1q_f32(res)); } -// Shifts the 4 signed 32-bit integers in a right by count bits while shifting -// in the sign bit. -// -// r0 := a0 >> count -// r1 := a1 >> count -// r2 := a2 >> count -// r3 := a3 >> count -// -// https://msdn.microsoft.com/en-us/library/ce40009e(v%3dvs.100).aspx -FORCE_INLINE __m128i _mm_sra_epi32(__m128i a, __m128i count) -{ - int64_t c = (int64_t) vget_low_s64((int64x2_t) count); - if (c > 31) - return _mm_cmplt_epi32(a, _mm_setzero_si128()); - return vreinterpretq_m128i_s32(vshlq_s32((int32x4_t) a, vdupq_n_s32(-c))); -} +// Extract a 32-bit integer from a, selected with imm8, and store the result in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_extract_epi32 +// FORCE_INLINE int _mm_extract_epi32(__m128i a, __constrange(0,4) int imm) +#define _mm_extract_epi32(a, imm) \ + vgetq_lane_s32(vreinterpretq_s32_m128i(a), (imm)) -// Packs the 16 signed 16-bit integers from a and b into 8-bit integers and -// saturates. -// https://msdn.microsoft.com/en-us/library/k4y4f7w5%28v=vs.90%29.aspx -FORCE_INLINE __m128i _mm_packs_epi16(__m128i a, __m128i b) -{ - return vreinterpretq_m128i_s8( - vcombine_s8(vqmovn_s16(vreinterpretq_s16_m128i(a)), - vqmovn_s16(vreinterpretq_s16_m128i(b)))); -} +// Extract a 64-bit integer from a, selected with imm8, and store the result in +// dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_extract_epi64 +// FORCE_INLINE __int64 _mm_extract_epi64(__m128i a, __constrange(0,2) int imm) +#define _mm_extract_epi64(a, imm) \ + vgetq_lane_s64(vreinterpretq_s64_m128i(a), (imm)) -// Packs the 16 signed 16 - bit integers from a and b into 8 - bit unsigned -// integers and saturates. -// -// r0 := UnsignedSaturate(a0) -// r1 := UnsignedSaturate(a1) -// ... -// r7 := UnsignedSaturate(a7) -// r8 := UnsignedSaturate(b0) -// r9 := UnsignedSaturate(b1) -// ... -// r15 := UnsignedSaturate(b7) -// -// https://msdn.microsoft.com/en-us/library/07ad1wx4(v=vs.100).aspx -FORCE_INLINE __m128i _mm_packus_epi16(const __m128i a, const __m128i b) -{ - return vreinterpretq_m128i_u8( - vcombine_u8(vqmovun_s16(vreinterpretq_s16_m128i(a)), - vqmovun_s16(vreinterpretq_s16_m128i(b)))); -} +// Extract an 8-bit integer from a, selected with imm8, and store the result in +// the lower element of dst. FORCE_INLINE int _mm_extract_epi8(__m128i a, +// __constrange(0,16) int imm) +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_extract_epi8 +#define _mm_extract_epi8(a, imm) vgetq_lane_u8(vreinterpretq_u8_m128i(a), (imm)) -// Packs the 8 signed 32-bit integers from a and b into signed 16-bit integers -// and saturates. -// -// r0 := SignedSaturate(a0) -// r1 := SignedSaturate(a1) -// r2 := SignedSaturate(a2) -// r3 := SignedSaturate(a3) -// r4 := SignedSaturate(b0) -// r5 := SignedSaturate(b1) -// r6 := SignedSaturate(b2) -// r7 := SignedSaturate(b3) -// -// https://msdn.microsoft.com/en-us/library/393t56f9%28v=vs.90%29.aspx -FORCE_INLINE __m128i _mm_packs_epi32(__m128i a, __m128i b) -{ - return vreinterpretq_m128i_s16( - vcombine_s16(vqmovn_s32(vreinterpretq_s32_m128i(a)), - vqmovn_s32(vreinterpretq_s32_m128i(b)))); -} +// Extracts the selected single-precision (32-bit) floating-point from a. +// FORCE_INLINE int _mm_extract_ps(__m128 a, __constrange(0,4) int imm) +#define _mm_extract_ps(a, imm) vgetq_lane_s32(vreinterpretq_s32_m128(a), (imm)) -// Packs the 8 unsigned 32-bit integers from a and b into unsigned 16-bit -// integers and saturates. -// -// r0 := UnsignedSaturate(a0) -// r1 := UnsignedSaturate(a1) -// r2 := UnsignedSaturate(a2) -// r3 := UnsignedSaturate(a3) -// r4 := UnsignedSaturate(b0) -// r5 := UnsignedSaturate(b1) -// r6 := UnsignedSaturate(b2) -// r7 := UnsignedSaturate(b3) -FORCE_INLINE __m128i _mm_packus_epi32(__m128i a, __m128i b) +// Round the packed double-precision (64-bit) floating-point elements in a down +// to an integer value, and store the results as packed double-precision +// floating-point elements in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_floor_pd +FORCE_INLINE __m128d _mm_floor_pd(__m128d a) { - return vreinterpretq_m128i_u16( - vcombine_u16(vqmovun_s32(vreinterpretq_s32_m128i(a)), - vqmovun_s32(vreinterpretq_s32_m128i(b)))); +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128d_f64(vrndmq_f64(vreinterpretq_f64_m128d(a))); +#else + double *f = (double *) &a; + return _mm_set_pd(floor(f[1]), floor(f[0])); +#endif } -// Interleaves the lower 8 signed or unsigned 8-bit integers in a with the lower -// 8 signed or unsigned 8-bit integers in b. -// -// r0 := a0 -// r1 := b0 -// r2 := a1 -// r3 := b1 -// ... -// r14 := a7 -// r15 := b7 -// -// https://msdn.microsoft.com/en-us/library/xf7k860c%28v=vs.90%29.aspx -FORCE_INLINE __m128i _mm_unpacklo_epi8(__m128i a, __m128i b) +// Round the packed single-precision (32-bit) floating-point elements in a down +// to an integer value, and store the results as packed single-precision +// floating-point elements in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_floor_ps +FORCE_INLINE __m128 _mm_floor_ps(__m128 a) { -#if defined(__aarch64__) - return vreinterpretq_m128i_s8( - vzip1q_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); +#if (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ARM_FEATURE_DIRECTED_ROUNDING) + return vreinterpretq_m128_f32(vrndmq_f32(vreinterpretq_f32_m128(a))); #else - int8x8_t a1 = vreinterpret_s8_s16(vget_low_s16(vreinterpretq_s16_m128i(a))); - int8x8_t b1 = vreinterpret_s8_s16(vget_low_s16(vreinterpretq_s16_m128i(b))); - int8x8x2_t result = vzip_s8(a1, b1); - return vreinterpretq_m128i_s8(vcombine_s8(result.val[0], result.val[1])); + float *f = (float *) &a; + return _mm_set_ps(floorf(f[3]), floorf(f[2]), floorf(f[1]), floorf(f[0])); #endif } -// Interleaves the lower 4 signed or unsigned 16-bit integers in a with the -// lower 4 signed or unsigned 16-bit integers in b. -// -// r0 := a0 -// r1 := b0 -// r2 := a1 -// r3 := b1 -// r4 := a2 -// r5 := b2 -// r6 := a3 -// r7 := b3 -// -// https://msdn.microsoft.com/en-us/library/btxb17bw%28v=vs.90%29.aspx -FORCE_INLINE __m128i _mm_unpacklo_epi16(__m128i a, __m128i b) +// Round the lower double-precision (64-bit) floating-point element in b down to +// an integer value, store the result as a double-precision floating-point +// element in the lower element of dst, and copy the upper element from a to the +// upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_floor_sd +FORCE_INLINE __m128d _mm_floor_sd(__m128d a, __m128d b) { -#if defined(__aarch64__) - return vreinterpretq_m128i_s16( - vzip1q_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); -#else - int16x4_t a1 = vget_low_s16(vreinterpretq_s16_m128i(a)); - int16x4_t b1 = vget_low_s16(vreinterpretq_s16_m128i(b)); - int16x4x2_t result = vzip_s16(a1, b1); - return vreinterpretq_m128i_s16(vcombine_s16(result.val[0], result.val[1])); -#endif + return _mm_move_sd(a, _mm_floor_pd(b)); } -// Interleaves the lower 2 signed or unsigned 32 - bit integers in a with the -// lower 2 signed or unsigned 32 - bit integers in b. -// -// r0 := a0 -// r1 := b0 -// r2 := a1 -// r3 := b1 -// -// https://msdn.microsoft.com/en-us/library/x8atst9d(v=vs.100).aspx -FORCE_INLINE __m128i _mm_unpacklo_epi32(__m128i a, __m128i b) +// Round the lower single-precision (32-bit) floating-point element in b down to +// an integer value, store the result as a single-precision floating-point +// element in the lower element of dst, and copy the upper 3 packed elements +// from a to the upper elements of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_floor_ss +FORCE_INLINE __m128 _mm_floor_ss(__m128 a, __m128 b) +{ + return _mm_move_ss(a, _mm_floor_ps(b)); +} + +// Copy a to dst, and insert the 32-bit integer i into dst at the location +// specified by imm8. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_insert_epi32 +// FORCE_INLINE __m128i _mm_insert_epi32(__m128i a, int b, +// __constrange(0,4) int imm) +#define _mm_insert_epi32(a, b, imm) \ + vreinterpretq_m128i_s32( \ + vsetq_lane_s32((b), vreinterpretq_s32_m128i(a), (imm))) + +// Copy a to dst, and insert the 64-bit integer i into dst at the location +// specified by imm8. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_insert_epi64 +// FORCE_INLINE __m128i _mm_insert_epi64(__m128i a, __int64 b, +// __constrange(0,2) int imm) +#define _mm_insert_epi64(a, b, imm) \ + vreinterpretq_m128i_s64( \ + vsetq_lane_s64((b), vreinterpretq_s64_m128i(a), (imm))) + +// Copy a to dst, and insert the lower 8-bit integer from i into dst at the +// location specified by imm8. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_insert_epi8 +// FORCE_INLINE __m128i _mm_insert_epi8(__m128i a, int b, +// __constrange(0,16) int imm) +#define _mm_insert_epi8(a, b, imm) \ + vreinterpretq_m128i_s8(vsetq_lane_s8((b), vreinterpretq_s8_m128i(a), (imm))) + +// Copy a to tmp, then insert a single-precision (32-bit) floating-point +// element from b into tmp using the control in imm8. Store tmp to dst using +// the mask in imm8 (elements are zeroed out when the corresponding bit is set). +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=insert_ps +#define _mm_insert_ps(a, b, imm8) \ + _sse2neon_define2( \ + __m128, a, b, \ + float32x4_t tmp1 = \ + vsetq_lane_f32(vgetq_lane_f32(_b, (imm8 >> 6) & 0x3), \ + vreinterpretq_f32_m128(_a), 0); \ + float32x4_t tmp2 = \ + vsetq_lane_f32(vgetq_lane_f32(tmp1, 0), \ + vreinterpretq_f32_m128(_a), ((imm8 >> 4) & 0x3)); \ + const uint32_t data[4] = \ + _sse2neon_init(((imm8) & (1 << 0)) ? UINT32_MAX : 0, \ + ((imm8) & (1 << 1)) ? UINT32_MAX : 0, \ + ((imm8) & (1 << 2)) ? UINT32_MAX : 0, \ + ((imm8) & (1 << 3)) ? UINT32_MAX : 0); \ + uint32x4_t mask = vld1q_u32(data); \ + float32x4_t all_zeros = vdupq_n_f32(0); \ + \ + _sse2neon_return(vreinterpretq_m128_f32( \ + vbslq_f32(mask, all_zeros, vreinterpretq_f32_m128(tmp2))));) + +// Compare packed signed 32-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_epi32 +FORCE_INLINE __m128i _mm_max_epi32(__m128i a, __m128i b) { -#if defined(__aarch64__) return vreinterpretq_m128i_s32( - vzip1q_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); -#else - int32x2_t a1 = vget_low_s32(vreinterpretq_s32_m128i(a)); - int32x2_t b1 = vget_low_s32(vreinterpretq_s32_m128i(b)); - int32x2x2_t result = vzip_s32(a1, b1); - return vreinterpretq_m128i_s32(vcombine_s32(result.val[0], result.val[1])); -#endif + vmaxq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } -FORCE_INLINE __m128i _mm_unpacklo_epi64(__m128i a, __m128i b) +// Compare packed signed 8-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_epi8 +FORCE_INLINE __m128i _mm_max_epi8(__m128i a, __m128i b) { - int64x1_t a_l = vget_low_s64(vreinterpretq_s64_m128i(a)); - int64x1_t b_l = vget_low_s64(vreinterpretq_s64_m128i(b)); - return vreinterpretq_m128i_s64(vcombine_s64(a_l, b_l)); + return vreinterpretq_m128i_s8( + vmaxq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } -// Selects and interleaves the lower two single-precision, floating-point values -// from a and b. -// -// r0 := a0 -// r1 := b0 -// r2 := a1 -// r3 := b1 -// -// https://msdn.microsoft.com/en-us/library/25st103b%28v=vs.90%29.aspx -FORCE_INLINE __m128 _mm_unpacklo_ps(__m128 a, __m128 b) +// Compare packed unsigned 16-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_epu16 +FORCE_INLINE __m128i _mm_max_epu16(__m128i a, __m128i b) { -#if defined(__aarch64__) - return vreinterpretq_m128_f32( - vzip1q_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); -#else - float32x2_t a1 = vget_low_f32(vreinterpretq_f32_m128(a)); - float32x2_t b1 = vget_low_f32(vreinterpretq_f32_m128(b)); - float32x2x2_t result = vzip_f32(a1, b1); - return vreinterpretq_m128_f32(vcombine_f32(result.val[0], result.val[1])); -#endif + return vreinterpretq_m128i_u16( + vmaxq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); } -// Selects and interleaves the upper two single-precision, floating-point values -// from a and b. -// -// r0 := a2 -// r1 := b2 -// r2 := a3 -// r3 := b3 -// -// https://msdn.microsoft.com/en-us/library/skccxx7d%28v=vs.90%29.aspx -FORCE_INLINE __m128 _mm_unpackhi_ps(__m128 a, __m128 b) +// Compare packed unsigned 32-bit integers in a and b, and store packed maximum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_epu32 +FORCE_INLINE __m128i _mm_max_epu32(__m128i a, __m128i b) { -#if defined(__aarch64__) - return vreinterpretq_m128_f32( - vzip2q_f32(vreinterpretq_f32_m128(a), vreinterpretq_f32_m128(b))); -#else - float32x2_t a1 = vget_high_f32(vreinterpretq_f32_m128(a)); - float32x2_t b1 = vget_high_f32(vreinterpretq_f32_m128(b)); - float32x2x2_t result = vzip_f32(a1, b1); - return vreinterpretq_m128_f32(vcombine_f32(result.val[0], result.val[1])); -#endif + return vreinterpretq_m128i_u32( + vmaxq_u32(vreinterpretq_u32_m128i(a), vreinterpretq_u32_m128i(b))); } -// Interleaves the upper 8 signed or unsigned 8-bit integers in a with the upper -// 8 signed or unsigned 8-bit integers in b. -// -// r0 := a8 -// r1 := b8 -// r2 := a9 -// r3 := b9 -// ... -// r14 := a15 -// r15 := b15 -// -// https://msdn.microsoft.com/en-us/library/t5h7783k(v=vs.100).aspx -FORCE_INLINE __m128i _mm_unpackhi_epi8(__m128i a, __m128i b) +// Compare packed signed 32-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_epi32 +FORCE_INLINE __m128i _mm_min_epi32(__m128i a, __m128i b) { -#if defined(__aarch64__) - return vreinterpretq_m128i_s8( - vzip2q_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); -#else - int8x8_t a1 = - vreinterpret_s8_s16(vget_high_s16(vreinterpretq_s16_m128i(a))); - int8x8_t b1 = - vreinterpret_s8_s16(vget_high_s16(vreinterpretq_s16_m128i(b))); - int8x8x2_t result = vzip_s8(a1, b1); - return vreinterpretq_m128i_s8(vcombine_s8(result.val[0], result.val[1])); -#endif + return vreinterpretq_m128i_s32( + vminq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); } -// Interleaves the upper 4 signed or unsigned 16-bit integers in a with the -// upper 4 signed or unsigned 16-bit integers in b. -// -// r0 := a4 -// r1 := b4 -// r2 := a5 -// r3 := b5 -// r4 := a6 -// r5 := b6 -// r6 := a7 -// r7 := b7 -// -// https://msdn.microsoft.com/en-us/library/03196cz7(v=vs.100).aspx -FORCE_INLINE __m128i _mm_unpackhi_epi16(__m128i a, __m128i b) +// Compare packed signed 8-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_epi8 +FORCE_INLINE __m128i _mm_min_epi8(__m128i a, __m128i b) { -#if defined(__aarch64__) - return vreinterpretq_m128i_s16( - vzip2q_s16(vreinterpretq_s16_m128i(a), vreinterpretq_s16_m128i(b))); -#else - int16x4_t a1 = vget_high_s16(vreinterpretq_s16_m128i(a)); - int16x4_t b1 = vget_high_s16(vreinterpretq_s16_m128i(b)); - int16x4x2_t result = vzip_s16(a1, b1); - return vreinterpretq_m128i_s16(vcombine_s16(result.val[0], result.val[1])); -#endif + return vreinterpretq_m128i_s8( + vminq_s8(vreinterpretq_s8_m128i(a), vreinterpretq_s8_m128i(b))); } -// Interleaves the upper 2 signed or unsigned 32-bit integers in a with the -// upper 2 signed or unsigned 32-bit integers in b. -// https://msdn.microsoft.com/en-us/library/65sa7cbs(v=vs.100).aspx -FORCE_INLINE __m128i _mm_unpackhi_epi32(__m128i a, __m128i b) +// Compare packed unsigned 16-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_epu16 +FORCE_INLINE __m128i _mm_min_epu16(__m128i a, __m128i b) { -#if defined(__aarch64__) - return vreinterpretq_m128i_s32( - vzip2q_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); -#else - int32x2_t a1 = vget_high_s32(vreinterpretq_s32_m128i(a)); - int32x2_t b1 = vget_high_s32(vreinterpretq_s32_m128i(b)); - int32x2x2_t result = vzip_s32(a1, b1); - return vreinterpretq_m128i_s32(vcombine_s32(result.val[0], result.val[1])); -#endif + return vreinterpretq_m128i_u16( + vminq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b))); } -// Interleaves the upper signed or unsigned 64-bit integer in a with the -// upper signed or unsigned 64-bit integer in b. -// -// r0 := a1 -// r1 := b1 -FORCE_INLINE __m128i _mm_unpackhi_epi64(__m128i a, __m128i b) +// Compare packed unsigned 32-bit integers in a and b, and store packed minimum +// values in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_epu32 +FORCE_INLINE __m128i _mm_min_epu32(__m128i a, __m128i b) { - int64x1_t a_h = vget_high_s64(vreinterpretq_s64_m128i(a)); - int64x1_t b_h = vget_high_s64(vreinterpretq_s64_m128i(b)); - return vreinterpretq_m128i_s64(vcombine_s64(a_h, b_h)); + return vreinterpretq_m128i_u32( + vminq_u32(vreinterpretq_u32_m128i(a), vreinterpretq_u32_m128i(b))); } // Horizontally compute the minimum amongst the packed unsigned 16-bit integers // in a, store the minimum and index in dst, and zero the remaining bits in dst. -// -// index[2:0] := 0 -// min[15:0] := a[15:0] -// FOR j := 0 to 7 -// i := j*16 -// IF a[i+15:i] < min[15:0] -// index[2:0] := j -// min[15:0] := a[i+15:i] -// FI -// ENDFOR -// dst[15:0] := min[15:0] -// dst[18:16] := index[2:0] -// dst[127:19] := 0 -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_minpos_epu16 +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_minpos_epu16 FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) { __m128i dst; uint16_t min, idx = 0; +#if defined(__aarch64__) || defined(_M_ARM64) // Find the minimum value -#if defined(__aarch64__) min = vminvq_u16(vreinterpretq_u16_m128i(a)); + + // Get the index of the minimum value + static const uint16_t idxv[] = {0, 1, 2, 3, 4, 5, 6, 7}; + uint16x8_t minv = vdupq_n_u16(min); + uint16x8_t cmeq = vceqq_u16(minv, vreinterpretq_u16_m128i(a)); + idx = vminvq_u16(vornq_u16(vld1q_u16(idxv), cmeq)); #else + // Find the minimum value __m64 tmp; tmp = vreinterpret_m64_u16( vmin_u16(vget_low_u16(vreinterpretq_u16_m128i(a)), @@ -5583,7 +7258,6 @@ FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) tmp = vreinterpret_m64_u16( vpmin_u16(vreinterpret_u16_m64(tmp), vreinterpret_u16_m64(tmp))); min = vget_lane_u16(vreinterpret_u16_m64(tmp), 0); -#endif // Get the index of the minimum value int i; for (i = 0; i < 8; i++) { @@ -5593,6 +7267,7 @@ FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) } a = _mm_srli_si128(a, 2); } +#endif // Generate result dst = _mm_setzero_si128(); dst = vreinterpretq_m128i_u16( @@ -5602,32 +7277,356 @@ FORCE_INLINE __m128i _mm_minpos_epu16(__m128i a) return dst; } -// shift to right -// https://msdn.microsoft.com/en-us/library/bb514041(v=vs.120).aspx -// http://blog.csdn.net/hemmingway/article/details/44828303 -// Clang requires a macro here, as it is extremely picky about c being a -// literal. -#define _mm_alignr_epi8(a, b, c) \ - ((__m128i) vextq_s8((int8x16_t)(b), (int8x16_t)(a), (c))) +// Compute the sum of absolute differences (SADs) of quadruplets of unsigned +// 8-bit integers in a compared to those in b, and store the 16-bit results in +// dst. Eight SADs are performed using one quadruplet from b and eight +// quadruplets from a. One quadruplet is selected from b starting at on the +// offset specified in imm8. Eight quadruplets are formed from sequential 8-bit +// integers selected from a starting at the offset specified in imm8. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mpsadbw_epu8 +FORCE_INLINE __m128i _mm_mpsadbw_epu8(__m128i a, __m128i b, const int imm) +{ + uint8x16_t _a, _b; + + switch (imm & 0x4) { + case 0: + // do nothing + _a = vreinterpretq_u8_m128i(a); + break; + case 4: + _a = vreinterpretq_u8_u32(vextq_u32(vreinterpretq_u32_m128i(a), + vreinterpretq_u32_m128i(a), 1)); + break; + default: +#if defined(__GNUC__) || defined(__clang__) + __builtin_unreachable(); +#elif defined(_MSC_VER) + __assume(0); +#endif + break; + } + + switch (imm & 0x3) { + case 0: + _b = vreinterpretq_u8_u32( + vdupq_n_u32(vgetq_lane_u32(vreinterpretq_u32_m128i(b), 0))); + break; + case 1: + _b = vreinterpretq_u8_u32( + vdupq_n_u32(vgetq_lane_u32(vreinterpretq_u32_m128i(b), 1))); + break; + case 2: + _b = vreinterpretq_u8_u32( + vdupq_n_u32(vgetq_lane_u32(vreinterpretq_u32_m128i(b), 2))); + break; + case 3: + _b = vreinterpretq_u8_u32( + vdupq_n_u32(vgetq_lane_u32(vreinterpretq_u32_m128i(b), 3))); + break; + default: +#if defined(__GNUC__) || defined(__clang__) + __builtin_unreachable(); +#elif defined(_MSC_VER) + __assume(0); +#endif + break; + } + + int16x8_t c04, c15, c26, c37; + uint8x8_t low_b = vget_low_u8(_b); + c04 = vreinterpretq_s16_u16(vabdl_u8(vget_low_u8(_a), low_b)); + uint8x16_t _a_1 = vextq_u8(_a, _a, 1); + c15 = vreinterpretq_s16_u16(vabdl_u8(vget_low_u8(_a_1), low_b)); + uint8x16_t _a_2 = vextq_u8(_a, _a, 2); + c26 = vreinterpretq_s16_u16(vabdl_u8(vget_low_u8(_a_2), low_b)); + uint8x16_t _a_3 = vextq_u8(_a, _a, 3); + c37 = vreinterpretq_s16_u16(vabdl_u8(vget_low_u8(_a_3), low_b)); +#if defined(__aarch64__) || defined(_M_ARM64) + // |0|4|2|6| + c04 = vpaddq_s16(c04, c26); + // |1|5|3|7| + c15 = vpaddq_s16(c15, c37); + + int32x4_t trn1_c = + vtrn1q_s32(vreinterpretq_s32_s16(c04), vreinterpretq_s32_s16(c15)); + int32x4_t trn2_c = + vtrn2q_s32(vreinterpretq_s32_s16(c04), vreinterpretq_s32_s16(c15)); + return vreinterpretq_m128i_s16(vpaddq_s16(vreinterpretq_s16_s32(trn1_c), + vreinterpretq_s16_s32(trn2_c))); +#else + int16x4_t c01, c23, c45, c67; + c01 = vpadd_s16(vget_low_s16(c04), vget_low_s16(c15)); + c23 = vpadd_s16(vget_low_s16(c26), vget_low_s16(c37)); + c45 = vpadd_s16(vget_high_s16(c04), vget_high_s16(c15)); + c67 = vpadd_s16(vget_high_s16(c26), vget_high_s16(c37)); + + return vreinterpretq_m128i_s16( + vcombine_s16(vpadd_s16(c01, c23), vpadd_s16(c45, c67))); +#endif +} + +// Multiply the low signed 32-bit integers from each packed 64-bit element in +// a and b, and store the signed 64-bit results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mul_epi32 +FORCE_INLINE __m128i _mm_mul_epi32(__m128i a, __m128i b) +{ + // vmull_s32 upcasts instead of masking, so we downcast. + int32x2_t a_lo = vmovn_s64(vreinterpretq_s64_m128i(a)); + int32x2_t b_lo = vmovn_s64(vreinterpretq_s64_m128i(b)); + return vreinterpretq_m128i_s64(vmull_s32(a_lo, b_lo)); +} + +// Multiply the packed 32-bit integers in a and b, producing intermediate 64-bit +// integers, and store the low 32 bits of the intermediate integers in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mullo_epi32 +FORCE_INLINE __m128i _mm_mullo_epi32(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_s32( + vmulq_s32(vreinterpretq_s32_m128i(a), vreinterpretq_s32_m128i(b))); +} + +// Convert packed signed 32-bit integers from a and b to packed 16-bit integers +// using unsigned saturation, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_packus_epi32 +FORCE_INLINE __m128i _mm_packus_epi32(__m128i a, __m128i b) +{ + return vreinterpretq_m128i_u16( + vcombine_u16(vqmovun_s32(vreinterpretq_s32_m128i(a)), + vqmovun_s32(vreinterpretq_s32_m128i(b)))); +} + +// Round the packed double-precision (64-bit) floating-point elements in a using +// the rounding parameter, and store the results as packed double-precision +// floating-point elements in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_round_pd +FORCE_INLINE __m128d _mm_round_pd(__m128d a, int rounding) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + switch (rounding) { + case (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC): + return vreinterpretq_m128d_f64(vrndnq_f64(vreinterpretq_f64_m128d(a))); + case (_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC): + return _mm_floor_pd(a); + case (_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC): + return _mm_ceil_pd(a); + case (_MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC): + return vreinterpretq_m128d_f64(vrndq_f64(vreinterpretq_f64_m128d(a))); + default: //_MM_FROUND_CUR_DIRECTION + return vreinterpretq_m128d_f64(vrndiq_f64(vreinterpretq_f64_m128d(a))); + } +#else + double *v_double = (double *) &a; + + if (rounding == (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC) || + (rounding == _MM_FROUND_CUR_DIRECTION && + _MM_GET_ROUNDING_MODE() == _MM_ROUND_NEAREST)) { + double res[2], tmp; + for (int i = 0; i < 2; i++) { + tmp = (v_double[i] < 0) ? -v_double[i] : v_double[i]; + double roundDown = floor(tmp); // Round down value + double roundUp = ceil(tmp); // Round up value + double diffDown = tmp - roundDown; + double diffUp = roundUp - tmp; + if (diffDown < diffUp) { + /* If it's closer to the round down value, then use it */ + res[i] = roundDown; + } else if (diffDown > diffUp) { + /* If it's closer to the round up value, then use it */ + res[i] = roundUp; + } else { + /* If it's equidistant between round up and round down value, + * pick the one which is an even number */ + double half = roundDown / 2; + if (half != floor(half)) { + /* If the round down value is odd, return the round up value + */ + res[i] = roundUp; + } else { + /* If the round up value is odd, return the round down value + */ + res[i] = roundDown; + } + } + res[i] = (v_double[i] < 0) ? -res[i] : res[i]; + } + return _mm_set_pd(res[1], res[0]); + } else if (rounding == (_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC) || + (rounding == _MM_FROUND_CUR_DIRECTION && + _MM_GET_ROUNDING_MODE() == _MM_ROUND_DOWN)) { + return _mm_floor_pd(a); + } else if (rounding == (_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC) || + (rounding == _MM_FROUND_CUR_DIRECTION && + _MM_GET_ROUNDING_MODE() == _MM_ROUND_UP)) { + return _mm_ceil_pd(a); + } + return _mm_set_pd(v_double[1] > 0 ? floor(v_double[1]) : ceil(v_double[1]), + v_double[0] > 0 ? floor(v_double[0]) : ceil(v_double[0])); +#endif +} + +// Round the packed single-precision (32-bit) floating-point elements in a using +// the rounding parameter, and store the results as packed single-precision +// floating-point elements in dst. +// software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_round_ps +FORCE_INLINE __m128 _mm_round_ps(__m128 a, int rounding) +{ +#if (defined(__aarch64__) || defined(_M_ARM64)) || \ + defined(__ARM_FEATURE_DIRECTED_ROUNDING) + switch (rounding) { + case (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC): + return vreinterpretq_m128_f32(vrndnq_f32(vreinterpretq_f32_m128(a))); + case (_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC): + return _mm_floor_ps(a); + case (_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC): + return _mm_ceil_ps(a); + case (_MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC): + return vreinterpretq_m128_f32(vrndq_f32(vreinterpretq_f32_m128(a))); + default: //_MM_FROUND_CUR_DIRECTION + return vreinterpretq_m128_f32(vrndiq_f32(vreinterpretq_f32_m128(a))); + } +#else + float *v_float = (float *) &a; + + if (rounding == (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC) || + (rounding == _MM_FROUND_CUR_DIRECTION && + _MM_GET_ROUNDING_MODE() == _MM_ROUND_NEAREST)) { + uint32x4_t signmask = vdupq_n_u32(0x80000000); + float32x4_t half = vbslq_f32(signmask, vreinterpretq_f32_m128(a), + vdupq_n_f32(0.5f)); /* +/- 0.5 */ + int32x4_t r_normal = vcvtq_s32_f32(vaddq_f32( + vreinterpretq_f32_m128(a), half)); /* round to integer: [a + 0.5]*/ + int32x4_t r_trunc = vcvtq_s32_f32( + vreinterpretq_f32_m128(a)); /* truncate to integer: [a] */ + int32x4_t plusone = vreinterpretq_s32_u32(vshrq_n_u32( + vreinterpretq_u32_s32(vnegq_s32(r_trunc)), 31)); /* 1 or 0 */ + int32x4_t r_even = vbicq_s32(vaddq_s32(r_trunc, plusone), + vdupq_n_s32(1)); /* ([a] + {0,1}) & ~1 */ + float32x4_t delta = vsubq_f32( + vreinterpretq_f32_m128(a), + vcvtq_f32_s32(r_trunc)); /* compute delta: delta = (a - [a]) */ + uint32x4_t is_delta_half = + vceqq_f32(delta, half); /* delta == +/- 0.5 */ + return vreinterpretq_m128_f32( + vcvtq_f32_s32(vbslq_s32(is_delta_half, r_even, r_normal))); + } else if (rounding == (_MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC) || + (rounding == _MM_FROUND_CUR_DIRECTION && + _MM_GET_ROUNDING_MODE() == _MM_ROUND_DOWN)) { + return _mm_floor_ps(a); + } else if (rounding == (_MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC) || + (rounding == _MM_FROUND_CUR_DIRECTION && + _MM_GET_ROUNDING_MODE() == _MM_ROUND_UP)) { + return _mm_ceil_ps(a); + } + return _mm_set_ps(v_float[3] > 0 ? floorf(v_float[3]) : ceilf(v_float[3]), + v_float[2] > 0 ? floorf(v_float[2]) : ceilf(v_float[2]), + v_float[1] > 0 ? floorf(v_float[1]) : ceilf(v_float[1]), + v_float[0] > 0 ? floorf(v_float[0]) : ceilf(v_float[0])); +#endif +} + +// Round the lower double-precision (64-bit) floating-point element in b using +// the rounding parameter, store the result as a double-precision floating-point +// element in the lower element of dst, and copy the upper element from a to the +// upper element of dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_round_sd +FORCE_INLINE __m128d _mm_round_sd(__m128d a, __m128d b, int rounding) +{ + return _mm_move_sd(a, _mm_round_pd(b, rounding)); +} + +// Round the lower single-precision (32-bit) floating-point element in b using +// the rounding parameter, store the result as a single-precision floating-point +// element in the lower element of dst, and copy the upper 3 packed elements +// from a to the upper elements of dst. Rounding is done according to the +// rounding[3:0] parameter, which can be one of: +// (_MM_FROUND_TO_NEAREST_INT |_MM_FROUND_NO_EXC) // round to nearest, and +// suppress exceptions +// (_MM_FROUND_TO_NEG_INF |_MM_FROUND_NO_EXC) // round down, and +// suppress exceptions +// (_MM_FROUND_TO_POS_INF |_MM_FROUND_NO_EXC) // round up, and suppress +// exceptions +// (_MM_FROUND_TO_ZERO |_MM_FROUND_NO_EXC) // truncate, and suppress +// exceptions _MM_FROUND_CUR_DIRECTION // use MXCSR.RC; see +// _MM_SET_ROUNDING_MODE +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_round_ss +FORCE_INLINE __m128 _mm_round_ss(__m128 a, __m128 b, int rounding) +{ + return _mm_move_ss(a, _mm_round_ps(b, rounding)); +} + +// Load 128-bits of integer data from memory into dst using a non-temporal +// memory hint. mem_addr must be aligned on a 16-byte boundary or a +// general-protection exception may be generated. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_stream_load_si128 +FORCE_INLINE __m128i _mm_stream_load_si128(__m128i *p) +{ +#if __has_builtin(__builtin_nontemporal_store) + return __builtin_nontemporal_load(p); +#else + return vreinterpretq_m128i_s64(vld1q_s64((int64_t *) p)); +#endif +} + +// Compute the bitwise NOT of a and then AND with a 128-bit vector containing +// all 1's, and return 1 if the result is zero, otherwise return 0. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_test_all_ones +FORCE_INLINE int _mm_test_all_ones(__m128i a) +{ + return (uint64_t) (vgetq_lane_s64(a, 0) & vgetq_lane_s64(a, 1)) == + ~(uint64_t) 0; +} + +// Compute the bitwise AND of 128 bits (representing integer data) in a and +// mask, and return 1 if the result is zero, otherwise return 0. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_test_all_zeros +FORCE_INLINE int _mm_test_all_zeros(__m128i a, __m128i mask) +{ + int64x2_t a_and_mask = + vandq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(mask)); + return !(vgetq_lane_s64(a_and_mask, 0) | vgetq_lane_s64(a_and_mask, 1)); +} + +// Compute the bitwise AND of 128 bits (representing integer data) in a and +// mask, and set ZF to 1 if the result is zero, otherwise set ZF to 0. Compute +// the bitwise NOT of a and then AND with mask, and set CF to 1 if the result is +// zero, otherwise set CF to 0. Return 1 if both the ZF and CF values are zero, +// otherwise return 0. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=mm_test_mix_ones_zero +FORCE_INLINE int _mm_test_mix_ones_zeros(__m128i a, __m128i mask) +{ + uint64x2_t zf = + vandq_u64(vreinterpretq_u64_m128i(mask), vreinterpretq_u64_m128i(a)); + uint64x2_t cf = + vbicq_u64(vreinterpretq_u64_m128i(mask), vreinterpretq_u64_m128i(a)); + uint64x2_t result = vandq_u64(zf, cf); + return !(vgetq_lane_u64(result, 0) | vgetq_lane_u64(result, 1)); +} // Compute the bitwise AND of 128 bits (representing integer data) in a and b, // and set ZF to 1 if the result is zero, otherwise set ZF to 0. Compute the // bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, // otherwise set CF to 0. Return the CF value. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_testc_si128 +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_testc_si128 FORCE_INLINE int _mm_testc_si128(__m128i a, __m128i b) { int64x2_t s64 = - vandq_s64(vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_m128i(a))), - vreinterpretq_s64_m128i(b)); + vbicq_s64(vreinterpretq_s64_m128i(b), vreinterpretq_s64_m128i(a)); return !(vgetq_lane_s64(s64, 0) | vgetq_lane_s64(s64, 1)); } +// Compute the bitwise AND of 128 bits (representing integer data) in a and b, +// and set ZF to 1 if the result is zero, otherwise set ZF to 0. Compute the +// bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, +// otherwise set CF to 0. Return 1 if both the ZF and CF values are zero, +// otherwise return 0. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_testnzc_si128 +#define _mm_testnzc_si128(a, b) _mm_test_mix_ones_zeros(a, b) + // Compute the bitwise AND of 128 bits (representing integer data) in a and b, // and set ZF to 1 if the result is zero, otherwise set ZF to 0. Compute the // bitwise NOT of a and then AND with b, and set CF to 1 if the result is zero, // otherwise set CF to 0. Return the ZF value. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_testz_si128 +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_testz_si128 FORCE_INLINE int _mm_testz_si128(__m128i a, __m128i b) { int64x2_t s64 = @@ -5635,299 +7634,864 @@ FORCE_INLINE int _mm_testz_si128(__m128i a, __m128i b) return !(vgetq_lane_s64(s64, 0) | vgetq_lane_s64(s64, 1)); } -// Extracts the selected signed or unsigned 8-bit integer from a and zero -// extends. -// FORCE_INLINE int _mm_extract_epi8(__m128i a, __constrange(0,16) int imm) -#define _mm_extract_epi8(a, imm) vgetq_lane_u8(vreinterpretq_u8_m128i(a), (imm)) +/* SSE4.2 */ -// Inserts the least significant 8 bits of b into the selected 8-bit integer -// of a. -// FORCE_INLINE __m128i _mm_insert_epi8(__m128i a, int b, -// __constrange(0,16) int imm) -#define _mm_insert_epi8(a, b, imm) \ - __extension__({ \ - vreinterpretq_m128i_s8( \ - vsetq_lane_s8((b), vreinterpretq_s8_m128i(a), (imm))); \ - }) +static const uint16_t ALIGN_STRUCT(16) _sse2neon_cmpestr_mask16b[8] = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, +}; +static const uint8_t ALIGN_STRUCT(16) _sse2neon_cmpestr_mask8b[16] = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, +}; -// Extracts the selected signed or unsigned 16-bit integer from a and zero -// extends. -// https://msdn.microsoft.com/en-us/library/6dceta0c(v=vs.100).aspx -// FORCE_INLINE int _mm_extract_epi16(__m128i a, __constrange(0,8) int imm) -#define _mm_extract_epi16(a, imm) \ - vgetq_lane_u16(vreinterpretq_u16_m128i(a), (imm)) +/* specify the source data format */ +#define _SIDD_UBYTE_OPS 0x00 /* unsigned 8-bit characters */ +#define _SIDD_UWORD_OPS 0x01 /* unsigned 16-bit characters */ +#define _SIDD_SBYTE_OPS 0x02 /* signed 8-bit characters */ +#define _SIDD_SWORD_OPS 0x03 /* signed 16-bit characters */ + +/* specify the comparison operation */ +#define _SIDD_CMP_EQUAL_ANY 0x00 /* compare equal any: strchr */ +#define _SIDD_CMP_RANGES 0x04 /* compare ranges */ +#define _SIDD_CMP_EQUAL_EACH 0x08 /* compare equal each: strcmp */ +#define _SIDD_CMP_EQUAL_ORDERED 0x0C /* compare equal ordered */ + +/* specify the polarity */ +#define _SIDD_POSITIVE_POLARITY 0x00 +#define _SIDD_MASKED_POSITIVE_POLARITY 0x20 +#define _SIDD_NEGATIVE_POLARITY 0x10 /* negate results */ +#define _SIDD_MASKED_NEGATIVE_POLARITY \ + 0x30 /* negate results only before end of string */ + +/* specify the output selection in _mm_cmpXstri */ +#define _SIDD_LEAST_SIGNIFICANT 0x00 +#define _SIDD_MOST_SIGNIFICANT 0x40 + +/* specify the output selection in _mm_cmpXstrm */ +#define _SIDD_BIT_MASK 0x00 +#define _SIDD_UNIT_MASK 0x40 + +/* Pattern Matching for C macros. + * https://github.com/pfultz2/Cloak/wiki/C-Preprocessor-tricks,-tips,-and-idioms + */ -// Inserts the least significant 16 bits of b into the selected 16-bit integer -// of a. -// https://msdn.microsoft.com/en-us/library/kaze8hz1%28v=vs.100%29.aspx -// FORCE_INLINE __m128i _mm_insert_epi16(__m128i a, int b, -// __constrange(0,8) int imm) -#define _mm_insert_epi16(a, b, imm) \ - __extension__({ \ - vreinterpretq_m128i_s16( \ - vsetq_lane_s16((b), vreinterpretq_s16_m128i(a), (imm))); \ - }) +/* catenate */ +#define SSE2NEON_PRIMITIVE_CAT(a, ...) a##__VA_ARGS__ +#define SSE2NEON_CAT(a, b) SSE2NEON_PRIMITIVE_CAT(a, b) + +#define SSE2NEON_IIF(c) SSE2NEON_PRIMITIVE_CAT(SSE2NEON_IIF_, c) +/* run the 2nd parameter */ +#define SSE2NEON_IIF_0(t, ...) __VA_ARGS__ +/* run the 1st parameter */ +#define SSE2NEON_IIF_1(t, ...) t + +#define SSE2NEON_COMPL(b) SSE2NEON_PRIMITIVE_CAT(SSE2NEON_COMPL_, b) +#define SSE2NEON_COMPL_0 1 +#define SSE2NEON_COMPL_1 0 + +#define SSE2NEON_DEC(x) SSE2NEON_PRIMITIVE_CAT(SSE2NEON_DEC_, x) +#define SSE2NEON_DEC_1 0 +#define SSE2NEON_DEC_2 1 +#define SSE2NEON_DEC_3 2 +#define SSE2NEON_DEC_4 3 +#define SSE2NEON_DEC_5 4 +#define SSE2NEON_DEC_6 5 +#define SSE2NEON_DEC_7 6 +#define SSE2NEON_DEC_8 7 +#define SSE2NEON_DEC_9 8 +#define SSE2NEON_DEC_10 9 +#define SSE2NEON_DEC_11 10 +#define SSE2NEON_DEC_12 11 +#define SSE2NEON_DEC_13 12 +#define SSE2NEON_DEC_14 13 +#define SSE2NEON_DEC_15 14 +#define SSE2NEON_DEC_16 15 + +/* detection */ +#define SSE2NEON_CHECK_N(x, n, ...) n +#define SSE2NEON_CHECK(...) SSE2NEON_CHECK_N(__VA_ARGS__, 0, ) +#define SSE2NEON_PROBE(x) x, 1, + +#define SSE2NEON_NOT(x) SSE2NEON_CHECK(SSE2NEON_PRIMITIVE_CAT(SSE2NEON_NOT_, x)) +#define SSE2NEON_NOT_0 SSE2NEON_PROBE(~) + +#define SSE2NEON_BOOL(x) SSE2NEON_COMPL(SSE2NEON_NOT(x)) +#define SSE2NEON_IF(c) SSE2NEON_IIF(SSE2NEON_BOOL(c)) + +#define SSE2NEON_EAT(...) +#define SSE2NEON_EXPAND(...) __VA_ARGS__ +#define SSE2NEON_WHEN(c) SSE2NEON_IF(c)(SSE2NEON_EXPAND, SSE2NEON_EAT) + +/* recursion */ +/* deferred expression */ +#define SSE2NEON_EMPTY() +#define SSE2NEON_DEFER(id) id SSE2NEON_EMPTY() +#define SSE2NEON_OBSTRUCT(...) __VA_ARGS__ SSE2NEON_DEFER(SSE2NEON_EMPTY)() +#define SSE2NEON_EXPAND(...) __VA_ARGS__ + +#define SSE2NEON_EVAL(...) \ + SSE2NEON_EVAL1(SSE2NEON_EVAL1(SSE2NEON_EVAL1(__VA_ARGS__))) +#define SSE2NEON_EVAL1(...) \ + SSE2NEON_EVAL2(SSE2NEON_EVAL2(SSE2NEON_EVAL2(__VA_ARGS__))) +#define SSE2NEON_EVAL2(...) \ + SSE2NEON_EVAL3(SSE2NEON_EVAL3(SSE2NEON_EVAL3(__VA_ARGS__))) +#define SSE2NEON_EVAL3(...) __VA_ARGS__ + +#define SSE2NEON_REPEAT(count, macro, ...) \ + SSE2NEON_WHEN(count) \ + (SSE2NEON_OBSTRUCT(SSE2NEON_REPEAT_INDIRECT)()( \ + SSE2NEON_DEC(count), macro, \ + __VA_ARGS__) SSE2NEON_OBSTRUCT(macro)(SSE2NEON_DEC(count), \ + __VA_ARGS__)) +#define SSE2NEON_REPEAT_INDIRECT() SSE2NEON_REPEAT + +#define SSE2NEON_SIZE_OF_byte 8 +#define SSE2NEON_NUMBER_OF_LANES_byte 16 +#define SSE2NEON_SIZE_OF_word 16 +#define SSE2NEON_NUMBER_OF_LANES_word 8 + +#define SSE2NEON_COMPARE_EQUAL_THEN_FILL_LANE(i, type) \ + mtx[i] = vreinterpretq_m128i_##type(vceqq_##type( \ + vdupq_n_##type(vgetq_lane_##type(vreinterpretq_##type##_m128i(b), i)), \ + vreinterpretq_##type##_m128i(a))); + +#define SSE2NEON_FILL_LANE(i, type) \ + vec_b[i] = \ + vdupq_n_##type(vgetq_lane_##type(vreinterpretq_##type##_m128i(b), i)); + +#define PCMPSTR_RANGES(a, b, mtx, data_type_prefix, type_prefix, size, \ + number_of_lanes, byte_or_word) \ + do { \ + SSE2NEON_CAT( \ + data_type_prefix, \ + SSE2NEON_CAT(size, \ + SSE2NEON_CAT(x, SSE2NEON_CAT(number_of_lanes, _t)))) \ + vec_b[number_of_lanes]; \ + __m128i mask = SSE2NEON_IIF(byte_or_word)( \ + vreinterpretq_m128i_u16(vdupq_n_u16(0xff)), \ + vreinterpretq_m128i_u32(vdupq_n_u32(0xffff))); \ + SSE2NEON_EVAL(SSE2NEON_REPEAT(number_of_lanes, SSE2NEON_FILL_LANE, \ + SSE2NEON_CAT(type_prefix, size))) \ + for (int i = 0; i < number_of_lanes; i++) { \ + mtx[i] = SSE2NEON_CAT(vreinterpretq_m128i_u, \ + size)(SSE2NEON_CAT(vbslq_u, size)( \ + SSE2NEON_CAT(vreinterpretq_u, \ + SSE2NEON_CAT(size, _m128i))(mask), \ + SSE2NEON_CAT(vcgeq_, SSE2NEON_CAT(type_prefix, size))( \ + vec_b[i], \ + SSE2NEON_CAT( \ + vreinterpretq_, \ + SSE2NEON_CAT(type_prefix, \ + SSE2NEON_CAT(size, _m128i(a))))), \ + SSE2NEON_CAT(vcleq_, SSE2NEON_CAT(type_prefix, size))( \ + vec_b[i], \ + SSE2NEON_CAT( \ + vreinterpretq_, \ + SSE2NEON_CAT(type_prefix, \ + SSE2NEON_CAT(size, _m128i(a))))))); \ + } \ + } while (0) -// Copy a to dst, and insert the 16-bit integer i into dst at the location -// specified by imm8. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_insert_pi16 -#define _mm_insert_pi16(a, b, imm) \ - __extension__({ \ - vreinterpret_m64_s16( \ - vset_lane_s16((b), vreinterpret_s16_m64(a), (imm))); \ - }) +#define PCMPSTR_EQ(a, b, mtx, size, number_of_lanes) \ + do { \ + SSE2NEON_EVAL(SSE2NEON_REPEAT(number_of_lanes, \ + SSE2NEON_COMPARE_EQUAL_THEN_FILL_LANE, \ + SSE2NEON_CAT(u, size))) \ + } while (0) -// Extracts the selected signed or unsigned 32-bit integer from a and zero -// extends. -// FORCE_INLINE int _mm_extract_epi32(__m128i a, __constrange(0,4) int imm) -#define _mm_extract_epi32(a, imm) \ - vgetq_lane_s32(vreinterpretq_s32_m128i(a), (imm)) +#define SSE2NEON_CMP_EQUAL_ANY_IMPL(type) \ + static int _sse2neon_cmp_##type##_equal_any(__m128i a, int la, __m128i b, \ + int lb) \ + { \ + __m128i mtx[16]; \ + PCMPSTR_EQ(a, b, mtx, SSE2NEON_CAT(SSE2NEON_SIZE_OF_, type), \ + SSE2NEON_CAT(SSE2NEON_NUMBER_OF_LANES_, type)); \ + return SSE2NEON_CAT( \ + _sse2neon_aggregate_equal_any_, \ + SSE2NEON_CAT( \ + SSE2NEON_CAT(SSE2NEON_SIZE_OF_, type), \ + SSE2NEON_CAT(x, SSE2NEON_CAT(SSE2NEON_NUMBER_OF_LANES_, \ + type))))(la, lb, mtx); \ + } -// Extracts the selected single-precision (32-bit) floating-point from a. -// FORCE_INLINE int _mm_extract_ps(__m128 a, __constrange(0,4) int imm) -#define _mm_extract_ps(a, imm) vgetq_lane_s32(vreinterpretq_s32_m128(a), (imm)) +#define SSE2NEON_CMP_RANGES_IMPL(type, data_type, us, byte_or_word) \ + static int _sse2neon_cmp_##us##type##_ranges(__m128i a, int la, __m128i b, \ + int lb) \ + { \ + __m128i mtx[16]; \ + PCMPSTR_RANGES( \ + a, b, mtx, data_type, us, SSE2NEON_CAT(SSE2NEON_SIZE_OF_, type), \ + SSE2NEON_CAT(SSE2NEON_NUMBER_OF_LANES_, type), byte_or_word); \ + return SSE2NEON_CAT( \ + _sse2neon_aggregate_ranges_, \ + SSE2NEON_CAT( \ + SSE2NEON_CAT(SSE2NEON_SIZE_OF_, type), \ + SSE2NEON_CAT(x, SSE2NEON_CAT(SSE2NEON_NUMBER_OF_LANES_, \ + type))))(la, lb, mtx); \ + } -// Inserts the least significant 32 bits of b into the selected 32-bit integer -// of a. -// FORCE_INLINE __m128i _mm_insert_epi32(__m128i a, int b, -// __constrange(0,4) int imm) -#define _mm_insert_epi32(a, b, imm) \ - __extension__({ \ - vreinterpretq_m128i_s32( \ - vsetq_lane_s32((b), vreinterpretq_s32_m128i(a), (imm))); \ - }) +#define SSE2NEON_CMP_EQUAL_ORDERED_IMPL(type) \ + static int _sse2neon_cmp_##type##_equal_ordered(__m128i a, int la, \ + __m128i b, int lb) \ + { \ + __m128i mtx[16]; \ + PCMPSTR_EQ(a, b, mtx, SSE2NEON_CAT(SSE2NEON_SIZE_OF_, type), \ + SSE2NEON_CAT(SSE2NEON_NUMBER_OF_LANES_, type)); \ + return SSE2NEON_CAT( \ + _sse2neon_aggregate_equal_ordered_, \ + SSE2NEON_CAT( \ + SSE2NEON_CAT(SSE2NEON_SIZE_OF_, type), \ + SSE2NEON_CAT(x, \ + SSE2NEON_CAT(SSE2NEON_NUMBER_OF_LANES_, type))))( \ + SSE2NEON_CAT(SSE2NEON_NUMBER_OF_LANES_, type), la, lb, mtx); \ + } -// Extracts the selected signed or unsigned 64-bit integer from a and zero -// extends. -// FORCE_INLINE __int64 _mm_extract_epi64(__m128i a, __constrange(0,2) int imm) -#define _mm_extract_epi64(a, imm) \ - vgetq_lane_s64(vreinterpretq_s64_m128i(a), (imm)) +static int _sse2neon_aggregate_equal_any_8x16(int la, int lb, __m128i mtx[16]) +{ + int res = 0; + int m = (1 << la) - 1; + uint8x8_t vec_mask = vld1_u8(_sse2neon_cmpestr_mask8b); + uint8x8_t t_lo = vtst_u8(vdup_n_u8(m & 0xff), vec_mask); + uint8x8_t t_hi = vtst_u8(vdup_n_u8(m >> 8), vec_mask); + uint8x16_t vec = vcombine_u8(t_lo, t_hi); + for (int j = 0; j < lb; j++) { + mtx[j] = vreinterpretq_m128i_u8( + vandq_u8(vec, vreinterpretq_u8_m128i(mtx[j]))); + mtx[j] = vreinterpretq_m128i_u8( + vshrq_n_u8(vreinterpretq_u8_m128i(mtx[j]), 7)); + int tmp = _sse2neon_vaddvq_u8(vreinterpretq_u8_m128i(mtx[j])) ? 1 : 0; + res |= (tmp << j); + } + return res; +} + +static int _sse2neon_aggregate_equal_any_16x8(int la, int lb, __m128i mtx[16]) +{ + int res = 0; + int m = (1 << la) - 1; + uint16x8_t vec = + vtstq_u16(vdupq_n_u16(m), vld1q_u16(_sse2neon_cmpestr_mask16b)); + for (int j = 0; j < lb; j++) { + mtx[j] = vreinterpretq_m128i_u16( + vandq_u16(vec, vreinterpretq_u16_m128i(mtx[j]))); + mtx[j] = vreinterpretq_m128i_u16( + vshrq_n_u16(vreinterpretq_u16_m128i(mtx[j]), 15)); + int tmp = _sse2neon_vaddvq_u16(vreinterpretq_u16_m128i(mtx[j])) ? 1 : 0; + res |= (tmp << j); + } + return res; +} + +/* clang-format off */ +#define SSE2NEON_GENERATE_CMP_EQUAL_ANY(prefix) \ + prefix##IMPL(byte) \ + prefix##IMPL(word) +/* clang-format on */ + +SSE2NEON_GENERATE_CMP_EQUAL_ANY(SSE2NEON_CMP_EQUAL_ANY_) + +static int _sse2neon_aggregate_ranges_16x8(int la, int lb, __m128i mtx[16]) +{ + int res = 0; + int m = (1 << la) - 1; + uint16x8_t vec = + vtstq_u16(vdupq_n_u16(m), vld1q_u16(_sse2neon_cmpestr_mask16b)); + for (int j = 0; j < lb; j++) { + mtx[j] = vreinterpretq_m128i_u16( + vandq_u16(vec, vreinterpretq_u16_m128i(mtx[j]))); + mtx[j] = vreinterpretq_m128i_u16( + vshrq_n_u16(vreinterpretq_u16_m128i(mtx[j]), 15)); + __m128i tmp = vreinterpretq_m128i_u32( + vshrq_n_u32(vreinterpretq_u32_m128i(mtx[j]), 16)); + uint32x4_t vec_res = vandq_u32(vreinterpretq_u32_m128i(mtx[j]), + vreinterpretq_u32_m128i(tmp)); +#if defined(__aarch64__) || defined(_M_ARM64) + int t = vaddvq_u32(vec_res) ? 1 : 0; +#else + uint64x2_t sumh = vpaddlq_u32(vec_res); + int t = vgetq_lane_u64(sumh, 0) + vgetq_lane_u64(sumh, 1); +#endif + res |= (t << j); + } + return res; +} + +static int _sse2neon_aggregate_ranges_8x16(int la, int lb, __m128i mtx[16]) +{ + int res = 0; + int m = (1 << la) - 1; + uint8x8_t vec_mask = vld1_u8(_sse2neon_cmpestr_mask8b); + uint8x8_t t_lo = vtst_u8(vdup_n_u8(m & 0xff), vec_mask); + uint8x8_t t_hi = vtst_u8(vdup_n_u8(m >> 8), vec_mask); + uint8x16_t vec = vcombine_u8(t_lo, t_hi); + for (int j = 0; j < lb; j++) { + mtx[j] = vreinterpretq_m128i_u8( + vandq_u8(vec, vreinterpretq_u8_m128i(mtx[j]))); + mtx[j] = vreinterpretq_m128i_u8( + vshrq_n_u8(vreinterpretq_u8_m128i(mtx[j]), 7)); + __m128i tmp = vreinterpretq_m128i_u16( + vshrq_n_u16(vreinterpretq_u16_m128i(mtx[j]), 8)); + uint16x8_t vec_res = vandq_u16(vreinterpretq_u16_m128i(mtx[j]), + vreinterpretq_u16_m128i(tmp)); + int t = _sse2neon_vaddvq_u16(vec_res) ? 1 : 0; + res |= (t << j); + } + return res; +} + +#define SSE2NEON_CMP_RANGES_IS_BYTE 1 +#define SSE2NEON_CMP_RANGES_IS_WORD 0 + +/* clang-format off */ +#define SSE2NEON_GENERATE_CMP_RANGES(prefix) \ + prefix##IMPL(byte, uint, u, prefix##IS_BYTE) \ + prefix##IMPL(byte, int, s, prefix##IS_BYTE) \ + prefix##IMPL(word, uint, u, prefix##IS_WORD) \ + prefix##IMPL(word, int, s, prefix##IS_WORD) +/* clang-format on */ + +SSE2NEON_GENERATE_CMP_RANGES(SSE2NEON_CMP_RANGES_) + +#undef SSE2NEON_CMP_RANGES_IS_BYTE +#undef SSE2NEON_CMP_RANGES_IS_WORD + +static int _sse2neon_cmp_byte_equal_each(__m128i a, int la, __m128i b, int lb) +{ + uint8x16_t mtx = + vceqq_u8(vreinterpretq_u8_m128i(a), vreinterpretq_u8_m128i(b)); + int m0 = (la < lb) ? 0 : ((1 << la) - (1 << lb)); + int m1 = 0x10000 - (1 << la); + int tb = 0x10000 - (1 << lb); + uint8x8_t vec_mask, vec0_lo, vec0_hi, vec1_lo, vec1_hi; + uint8x8_t tmp_lo, tmp_hi, res_lo, res_hi; + vec_mask = vld1_u8(_sse2neon_cmpestr_mask8b); + vec0_lo = vtst_u8(vdup_n_u8(m0), vec_mask); + vec0_hi = vtst_u8(vdup_n_u8(m0 >> 8), vec_mask); + vec1_lo = vtst_u8(vdup_n_u8(m1), vec_mask); + vec1_hi = vtst_u8(vdup_n_u8(m1 >> 8), vec_mask); + tmp_lo = vtst_u8(vdup_n_u8(tb), vec_mask); + tmp_hi = vtst_u8(vdup_n_u8(tb >> 8), vec_mask); + + res_lo = vbsl_u8(vec0_lo, vdup_n_u8(0), vget_low_u8(mtx)); + res_hi = vbsl_u8(vec0_hi, vdup_n_u8(0), vget_high_u8(mtx)); + res_lo = vbsl_u8(vec1_lo, tmp_lo, res_lo); + res_hi = vbsl_u8(vec1_hi, tmp_hi, res_hi); + res_lo = vand_u8(res_lo, vec_mask); + res_hi = vand_u8(res_hi, vec_mask); + + int res = _sse2neon_vaddv_u8(res_lo) + (_sse2neon_vaddv_u8(res_hi) << 8); + return res; +} + +static int _sse2neon_cmp_word_equal_each(__m128i a, int la, __m128i b, int lb) +{ + uint16x8_t mtx = + vceqq_u16(vreinterpretq_u16_m128i(a), vreinterpretq_u16_m128i(b)); + int m0 = (la < lb) ? 0 : ((1 << la) - (1 << lb)); + int m1 = 0x100 - (1 << la); + int tb = 0x100 - (1 << lb); + uint16x8_t vec_mask = vld1q_u16(_sse2neon_cmpestr_mask16b); + uint16x8_t vec0 = vtstq_u16(vdupq_n_u16(m0), vec_mask); + uint16x8_t vec1 = vtstq_u16(vdupq_n_u16(m1), vec_mask); + uint16x8_t tmp = vtstq_u16(vdupq_n_u16(tb), vec_mask); + mtx = vbslq_u16(vec0, vdupq_n_u16(0), mtx); + mtx = vbslq_u16(vec1, tmp, mtx); + mtx = vandq_u16(mtx, vec_mask); + return _sse2neon_vaddvq_u16(mtx); +} + +#define SSE2NEON_AGGREGATE_EQUAL_ORDER_IS_UBYTE 1 +#define SSE2NEON_AGGREGATE_EQUAL_ORDER_IS_UWORD 0 + +#define SSE2NEON_AGGREGATE_EQUAL_ORDER_IMPL(size, number_of_lanes, data_type) \ + static int _sse2neon_aggregate_equal_ordered_##size##x##number_of_lanes( \ + int bound, int la, int lb, __m128i mtx[16]) \ + { \ + int res = 0; \ + int m1 = SSE2NEON_IIF(data_type)(0x10000, 0x100) - (1 << la); \ + uint##size##x8_t vec_mask = SSE2NEON_IIF(data_type)( \ + vld1_u##size(_sse2neon_cmpestr_mask##size##b), \ + vld1q_u##size(_sse2neon_cmpestr_mask##size##b)); \ + uint##size##x##number_of_lanes##_t vec1 = SSE2NEON_IIF(data_type)( \ + vcombine_u##size(vtst_u##size(vdup_n_u##size(m1), vec_mask), \ + vtst_u##size(vdup_n_u##size(m1 >> 8), vec_mask)), \ + vtstq_u##size(vdupq_n_u##size(m1), vec_mask)); \ + uint##size##x##number_of_lanes##_t vec_minusone = vdupq_n_u##size(-1); \ + uint##size##x##number_of_lanes##_t vec_zero = vdupq_n_u##size(0); \ + for (int j = 0; j < lb; j++) { \ + mtx[j] = vreinterpretq_m128i_u##size(vbslq_u##size( \ + vec1, vec_minusone, vreinterpretq_u##size##_m128i(mtx[j]))); \ + } \ + for (int j = lb; j < bound; j++) { \ + mtx[j] = vreinterpretq_m128i_u##size( \ + vbslq_u##size(vec1, vec_minusone, vec_zero)); \ + } \ + unsigned SSE2NEON_IIF(data_type)(char, short) *ptr = \ + (unsigned SSE2NEON_IIF(data_type)(char, short) *) mtx; \ + for (int i = 0; i < bound; i++) { \ + int val = 1; \ + for (int j = 0, k = i; j < bound - i && k < bound; j++, k++) \ + val &= ptr[k * bound + j]; \ + res += val << i; \ + } \ + return res; \ + } + +/* clang-format off */ +#define SSE2NEON_GENERATE_AGGREGATE_EQUAL_ORDER(prefix) \ + prefix##IMPL(8, 16, prefix##IS_UBYTE) \ + prefix##IMPL(16, 8, prefix##IS_UWORD) +/* clang-format on */ + +SSE2NEON_GENERATE_AGGREGATE_EQUAL_ORDER(SSE2NEON_AGGREGATE_EQUAL_ORDER_) + +#undef SSE2NEON_AGGREGATE_EQUAL_ORDER_IS_UBYTE +#undef SSE2NEON_AGGREGATE_EQUAL_ORDER_IS_UWORD -// Inserts the least significant 64 bits of b into the selected 64-bit integer -// of a. -// FORCE_INLINE __m128i _mm_insert_epi64(__m128i a, __int64 b, -// __constrange(0,2) int imm) -#define _mm_insert_epi64(a, b, imm) \ - __extension__({ \ - vreinterpretq_m128i_s64( \ - vsetq_lane_s64((b), vreinterpretq_s64_m128i(a), (imm))); \ - }) +/* clang-format off */ +#define SSE2NEON_GENERATE_CMP_EQUAL_ORDERED(prefix) \ + prefix##IMPL(byte) \ + prefix##IMPL(word) +/* clang-format on */ -// Count the number of bits set to 1 in unsigned 32-bit integer a, and -// return that count in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_popcnt_u32 -FORCE_INLINE int _mm_popcnt_u32(unsigned int a) +SSE2NEON_GENERATE_CMP_EQUAL_ORDERED(SSE2NEON_CMP_EQUAL_ORDERED_) + +#define SSE2NEON_CMPESTR_LIST \ + _(CMP_UBYTE_EQUAL_ANY, cmp_byte_equal_any) \ + _(CMP_UWORD_EQUAL_ANY, cmp_word_equal_any) \ + _(CMP_SBYTE_EQUAL_ANY, cmp_byte_equal_any) \ + _(CMP_SWORD_EQUAL_ANY, cmp_word_equal_any) \ + _(CMP_UBYTE_RANGES, cmp_ubyte_ranges) \ + _(CMP_UWORD_RANGES, cmp_uword_ranges) \ + _(CMP_SBYTE_RANGES, cmp_sbyte_ranges) \ + _(CMP_SWORD_RANGES, cmp_sword_ranges) \ + _(CMP_UBYTE_EQUAL_EACH, cmp_byte_equal_each) \ + _(CMP_UWORD_EQUAL_EACH, cmp_word_equal_each) \ + _(CMP_SBYTE_EQUAL_EACH, cmp_byte_equal_each) \ + _(CMP_SWORD_EQUAL_EACH, cmp_word_equal_each) \ + _(CMP_UBYTE_EQUAL_ORDERED, cmp_byte_equal_ordered) \ + _(CMP_UWORD_EQUAL_ORDERED, cmp_word_equal_ordered) \ + _(CMP_SBYTE_EQUAL_ORDERED, cmp_byte_equal_ordered) \ + _(CMP_SWORD_EQUAL_ORDERED, cmp_word_equal_ordered) + +enum { +#define _(name, func_suffix) name, + SSE2NEON_CMPESTR_LIST +#undef _ +}; +typedef int (*cmpestr_func_t)(__m128i a, int la, __m128i b, int lb); +static cmpestr_func_t _sse2neon_cmpfunc_table[] = { +#define _(name, func_suffix) _sse2neon_##func_suffix, + SSE2NEON_CMPESTR_LIST +#undef _ +}; + +FORCE_INLINE int _sse2neon_sido_negative(int res, int lb, int imm8, int bound) { -#if defined(__aarch64__) -#if __has_builtin(__builtin_popcount) - return __builtin_popcount(a); -#else - return (int) vaddlv_u8(vcnt_u8(vcreate_u8((uint64_t) a))); -#endif -#else - uint32_t count = 0; - uint8x8_t input_val, count8x8_val; - uint16x4_t count16x4_val; - uint32x2_t count32x2_val; + switch (imm8 & 0x30) { + case _SIDD_NEGATIVE_POLARITY: + res ^= 0xffffffff; + break; + case _SIDD_MASKED_NEGATIVE_POLARITY: + res ^= (1 << lb) - 1; + break; + default: + break; + } - input_val = vld1_u8((uint8_t *) &a); - count8x8_val = vcnt_u8(input_val); - count16x4_val = vpaddl_u8(count8x8_val); - count32x2_val = vpaddl_u16(count16x4_val); + return res & ((bound == 8) ? 0xFF : 0xFFFF); +} - vst1_u32(&count, count32x2_val); - return count; +FORCE_INLINE int _sse2neon_clz(unsigned int x) +{ +#ifdef _MSC_VER + unsigned long cnt = 0; + if (_BitScanReverse(&cnt, x)) + return 31 - cnt; + return 32; +#else + return x != 0 ? __builtin_clz(x) : 32; #endif } -// Count the number of bits set to 1 in unsigned 64-bit integer a, and -// return that count in dst. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_popcnt_u64 -FORCE_INLINE int64_t _mm_popcnt_u64(uint64_t a) +FORCE_INLINE int _sse2neon_ctz(unsigned int x) { -#if defined(__aarch64__) -#if __has_builtin(__builtin_popcountll) - return __builtin_popcountll(a); +#ifdef _MSC_VER + unsigned long cnt = 0; + if (_BitScanForward(&cnt, x)) + return cnt; + return 32; #else - return (int64_t) vaddlv_u8(vcnt_u8(vcreate_u8(a))); + return x != 0 ? __builtin_ctz(x) : 32; #endif -#else - uint64_t count = 0; - uint8x8_t input_val, count8x8_val; - uint16x4_t count16x4_val; - uint32x2_t count32x2_val; - uint64x1_t count64x1_val; +} - input_val = vld1_u8((uint8_t *) &a); - count8x8_val = vcnt_u8(input_val); - count16x4_val = vpaddl_u8(count8x8_val); - count32x2_val = vpaddl_u16(count16x4_val); - count64x1_val = vpaddl_u32(count32x2_val); - vst1_u64(&count, count64x1_val); - return count; +FORCE_INLINE int _sse2neon_ctzll(unsigned long long x) +{ +#ifdef _MSC_VER + unsigned long cnt; +#if defined(SSE2NEON_HAS_BITSCAN64) + if (_BitScanForward64(&cnt, x)) + return (int) (cnt); +#else + if (_BitScanForward(&cnt, (unsigned long) (x))) + return (int) cnt; + if (_BitScanForward(&cnt, (unsigned long) (x >> 32))) + return (int) (cnt + 32); +#endif /* SSE2NEON_HAS_BITSCAN64 */ + return 64; +#else /* assume GNU compatible compilers */ + return x != 0 ? __builtin_ctzll(x) : 64; #endif } -// Macro: Transpose the 4x4 matrix formed by the 4 rows of single-precision -// (32-bit) floating-point elements in row0, row1, row2, and row3, and store the -// transposed matrix in these vectors (row0 now contains column 0, etc.). -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=MM_TRANSPOSE4_PS -#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ - do { \ - float32x4x2_t ROW01 = vtrnq_f32(row0, row1); \ - float32x4x2_t ROW23 = vtrnq_f32(row2, row3); \ - row0 = vcombine_f32(vget_low_f32(ROW01.val[0]), \ - vget_low_f32(ROW23.val[0])); \ - row1 = vcombine_f32(vget_low_f32(ROW01.val[1]), \ - vget_low_f32(ROW23.val[1])); \ - row2 = vcombine_f32(vget_high_f32(ROW01.val[0]), \ - vget_high_f32(ROW23.val[0])); \ - row3 = vcombine_f32(vget_high_f32(ROW01.val[1]), \ - vget_high_f32(ROW23.val[1])); \ +#define SSE2NEON_MIN(x, y) (x) < (y) ? (x) : (y) + +#define SSE2NEON_CMPSTR_SET_UPPER(var, imm) \ + const int var = (imm & 0x01) ? 8 : 16 + +#define SSE2NEON_CMPESTRX_LEN_PAIR(a, b, la, lb) \ + int tmp1 = la ^ (la >> 31); \ + la = tmp1 - (la >> 31); \ + int tmp2 = lb ^ (lb >> 31); \ + lb = tmp2 - (lb >> 31); \ + la = SSE2NEON_MIN(la, bound); \ + lb = SSE2NEON_MIN(lb, bound) + +// Compare all pairs of character in string a and b, +// then aggregate the result. +// As the only difference of PCMPESTR* and PCMPISTR* is the way to calculate the +// length of string, we use SSE2NEON_CMP{I,E}STRX_GET_LEN to get the length of +// string a and b. +#define SSE2NEON_COMP_AGG(a, b, la, lb, imm8, IE) \ + SSE2NEON_CMPSTR_SET_UPPER(bound, imm8); \ + SSE2NEON_##IE##_LEN_PAIR(a, b, la, lb); \ + int r2 = (_sse2neon_cmpfunc_table[imm8 & 0x0f])(a, la, b, lb); \ + r2 = _sse2neon_sido_negative(r2, lb, imm8, bound) + +#define SSE2NEON_CMPSTR_GENERATE_INDEX(r2, bound, imm8) \ + return (r2 == 0) ? bound \ + : ((imm8 & 0x40) ? (31 - _sse2neon_clz(r2)) \ + : _sse2neon_ctz(r2)) + +#define SSE2NEON_CMPSTR_GENERATE_MASK(dst) \ + __m128i dst = vreinterpretq_m128i_u8(vdupq_n_u8(0)); \ + if (imm8 & 0x40) { \ + if (bound == 8) { \ + uint16x8_t tmp = vtstq_u16(vdupq_n_u16(r2), \ + vld1q_u16(_sse2neon_cmpestr_mask16b)); \ + dst = vreinterpretq_m128i_u16(vbslq_u16( \ + tmp, vdupq_n_u16(-1), vreinterpretq_u16_m128i(dst))); \ + } else { \ + uint8x16_t vec_r2 = \ + vcombine_u8(vdup_n_u8(r2), vdup_n_u8(r2 >> 8)); \ + uint8x16_t tmp = \ + vtstq_u8(vec_r2, vld1q_u8(_sse2neon_cmpestr_mask8b)); \ + dst = vreinterpretq_m128i_u8( \ + vbslq_u8(tmp, vdupq_n_u8(-1), vreinterpretq_u8_m128i(dst))); \ + } \ + } else { \ + if (bound == 16) { \ + dst = vreinterpretq_m128i_u16( \ + vsetq_lane_u16(r2 & 0xffff, vreinterpretq_u16_m128i(dst), 0)); \ + } else { \ + dst = vreinterpretq_m128i_u8( \ + vsetq_lane_u8(r2 & 0xff, vreinterpretq_u8_m128i(dst), 0)); \ + } \ + } \ + return dst + +// Compare packed strings in a and b with lengths la and lb using the control +// in imm8, and returns 1 if b did not contain a null character and the +// resulting mask was zero, and 0 otherwise. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestra +FORCE_INLINE int _mm_cmpestra(__m128i a, + int la, + __m128i b, + int lb, + const int imm8) +{ + int lb_cpy = lb; + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPESTRX); + return !r2 & (lb_cpy > bound); +} + +// Compare packed strings in a and b with lengths la and lb using the control in +// imm8, and returns 1 if the resulting mask was non-zero, and 0 otherwise. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestrc +FORCE_INLINE int _mm_cmpestrc(__m128i a, + int la, + __m128i b, + int lb, + const int imm8) +{ + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPESTRX); + return r2 != 0; +} + +// Compare packed strings in a and b with lengths la and lb using the control +// in imm8, and store the generated index in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestri +FORCE_INLINE int _mm_cmpestri(__m128i a, + int la, + __m128i b, + int lb, + const int imm8) +{ + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPESTRX); + SSE2NEON_CMPSTR_GENERATE_INDEX(r2, bound, imm8); +} + +// Compare packed strings in a and b with lengths la and lb using the control +// in imm8, and store the generated mask in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestrm +FORCE_INLINE __m128i +_mm_cmpestrm(__m128i a, int la, __m128i b, int lb, const int imm8) +{ + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPESTRX); + SSE2NEON_CMPSTR_GENERATE_MASK(dst); +} + +// Compare packed strings in a and b with lengths la and lb using the control in +// imm8, and returns bit 0 of the resulting bit mask. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestro +FORCE_INLINE int _mm_cmpestro(__m128i a, + int la, + __m128i b, + int lb, + const int imm8) +{ + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPESTRX); + return r2 & 1; +} + +// Compare packed strings in a and b with lengths la and lb using the control in +// imm8, and returns 1 if any character in a was null, and 0 otherwise. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestrs +FORCE_INLINE int _mm_cmpestrs(__m128i a, + int la, + __m128i b, + int lb, + const int imm8) +{ + (void) a; + (void) b; + (void) lb; + SSE2NEON_CMPSTR_SET_UPPER(bound, imm8); + return la <= (bound - 1); +} + +// Compare packed strings in a and b with lengths la and lb using the control in +// imm8, and returns 1 if any character in b was null, and 0 otherwise. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestrz +FORCE_INLINE int _mm_cmpestrz(__m128i a, + int la, + __m128i b, + int lb, + const int imm8) +{ + (void) a; + (void) b; + (void) la; + SSE2NEON_CMPSTR_SET_UPPER(bound, imm8); + return lb <= (bound - 1); +} + +#define SSE2NEON_CMPISTRX_LENGTH(str, len, imm8) \ + do { \ + if (imm8 & 0x01) { \ + uint16x8_t equal_mask_##str = \ + vceqq_u16(vreinterpretq_u16_m128i(str), vdupq_n_u16(0)); \ + uint8x8_t res_##str = vshrn_n_u16(equal_mask_##str, 4); \ + uint64_t matches_##str = \ + vget_lane_u64(vreinterpret_u64_u8(res_##str), 0); \ + len = _sse2neon_ctzll(matches_##str) >> 3; \ + } else { \ + uint16x8_t equal_mask_##str = vreinterpretq_u16_u8( \ + vceqq_u8(vreinterpretq_u8_m128i(str), vdupq_n_u8(0))); \ + uint8x8_t res_##str = vshrn_n_u16(equal_mask_##str, 4); \ + uint64_t matches_##str = \ + vget_lane_u64(vreinterpret_u64_u8(res_##str), 0); \ + len = _sse2neon_ctzll(matches_##str) >> 2; \ + } \ } while (0) -/* Crypto Extensions */ +#define SSE2NEON_CMPISTRX_LEN_PAIR(a, b, la, lb) \ + int la, lb; \ + do { \ + SSE2NEON_CMPISTRX_LENGTH(a, la, imm8); \ + SSE2NEON_CMPISTRX_LENGTH(b, lb, imm8); \ + } while (0) -#if defined(__ARM_FEATURE_CRYPTO) -// Wraps vmull_p64 -FORCE_INLINE uint64x2_t _sse2neon_vmull_p64(uint64x1_t _a, uint64x1_t _b) +// Compare packed strings with implicit lengths in a and b using the control in +// imm8, and returns 1 if b did not contain a null character and the resulting +// mask was zero, and 0 otherwise. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpistra +FORCE_INLINE int _mm_cmpistra(__m128i a, __m128i b, const int imm8) { - poly64_t a = vget_lane_p64(vreinterpret_p64_u64(_a), 0); - poly64_t b = vget_lane_p64(vreinterpret_p64_u64(_b), 0); - return vreinterpretq_u64_p128(vmull_p64(a, b)); + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPISTRX); + return !r2 & (lb >= bound); } -#else // ARMv7 polyfill -// ARMv7/some A64 lacks vmull_p64, but it has vmull_p8. -// -// vmull_p8 calculates 8 8-bit->16-bit polynomial multiplies, but we need a -// 64-bit->128-bit polynomial multiply. -// -// It needs some work and is somewhat slow, but it is still faster than all -// known scalar methods. -// -// Algorithm adapted to C from -// https://www.workofard.com/2017/07/ghash-for-low-end-cores/, which is adapted -// from "Fast Software Polynomial Multiplication on ARM Processors Using the -// NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and Ricardo Dahab -// (https://hal.inria.fr/hal-01506572) -static uint64x2_t _sse2neon_vmull_p64(uint64x1_t _a, uint64x1_t _b) + +// Compare packed strings with implicit lengths in a and b using the control in +// imm8, and returns 1 if the resulting mask was non-zero, and 0 otherwise. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpistrc +FORCE_INLINE int _mm_cmpistrc(__m128i a, __m128i b, const int imm8) { - poly8x8_t a = vreinterpret_p8_u64(_a); - poly8x8_t b = vreinterpret_p8_u64(_b); + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPISTRX); + return r2 != 0; +} - // Masks - uint8x16_t k48_32 = vcombine_u8(vcreate_u8(0x0000ffffffffffff), - vcreate_u8(0x00000000ffffffff)); - uint8x16_t k16_00 = vcombine_u8(vcreate_u8(0x000000000000ffff), - vcreate_u8(0x0000000000000000)); +// Compare packed strings with implicit lengths in a and b using the control in +// imm8, and store the generated index in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpistri +FORCE_INLINE int _mm_cmpistri(__m128i a, __m128i b, const int imm8) +{ + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPISTRX); + SSE2NEON_CMPSTR_GENERATE_INDEX(r2, bound, imm8); +} - // Do the multiplies, rotating with vext to get all combinations - uint8x16_t d = vreinterpretq_u8_p16(vmull_p8(a, b)); // D = A0 * B0 - uint8x16_t e = - vreinterpretq_u8_p16(vmull_p8(a, vext_p8(b, b, 1))); // E = A0 * B1 - uint8x16_t f = - vreinterpretq_u8_p16(vmull_p8(vext_p8(a, a, 1), b)); // F = A1 * B0 - uint8x16_t g = - vreinterpretq_u8_p16(vmull_p8(a, vext_p8(b, b, 2))); // G = A0 * B2 - uint8x16_t h = - vreinterpretq_u8_p16(vmull_p8(vext_p8(a, a, 2), b)); // H = A2 * B0 - uint8x16_t i = - vreinterpretq_u8_p16(vmull_p8(a, vext_p8(b, b, 3))); // I = A0 * B3 - uint8x16_t j = - vreinterpretq_u8_p16(vmull_p8(vext_p8(a, a, 3), b)); // J = A3 * B0 - uint8x16_t k = - vreinterpretq_u8_p16(vmull_p8(a, vext_p8(b, b, 4))); // L = A0 * B4 +// Compare packed strings with implicit lengths in a and b using the control in +// imm8, and store the generated mask in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpistrm +FORCE_INLINE __m128i _mm_cmpistrm(__m128i a, __m128i b, const int imm8) +{ + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPISTRX); + SSE2NEON_CMPSTR_GENERATE_MASK(dst); +} - // Add cross products - uint8x16_t l = veorq_u8(e, f); // L = E + F - uint8x16_t m = veorq_u8(g, h); // M = G + H - uint8x16_t n = veorq_u8(i, j); // N = I + J +// Compare packed strings with implicit lengths in a and b using the control in +// imm8, and returns bit 0 of the resulting bit mask. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpistro +FORCE_INLINE int _mm_cmpistro(__m128i a, __m128i b, const int imm8) +{ + SSE2NEON_COMP_AGG(a, b, la, lb, imm8, CMPISTRX); + return r2 & 1; +} - // Interleave. Using vzip1 and vzip2 prevents Clang from emitting TBL - // instructions. -#if defined(__aarch64__) - uint8x16_t lm_p0 = vreinterpretq_u8_u64( - vzip1q_u64(vreinterpretq_u64_u8(l), vreinterpretq_u64_u8(m))); - uint8x16_t lm_p1 = vreinterpretq_u8_u64( - vzip2q_u64(vreinterpretq_u64_u8(l), vreinterpretq_u64_u8(m))); - uint8x16_t nk_p0 = vreinterpretq_u8_u64( - vzip1q_u64(vreinterpretq_u64_u8(n), vreinterpretq_u64_u8(k))); - uint8x16_t nk_p1 = vreinterpretq_u8_u64( - vzip2q_u64(vreinterpretq_u64_u8(n), vreinterpretq_u64_u8(k))); +// Compare packed strings with implicit lengths in a and b using the control in +// imm8, and returns 1 if any character in a was null, and 0 otherwise. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpistrs +FORCE_INLINE int _mm_cmpistrs(__m128i a, __m128i b, const int imm8) +{ + (void) b; + SSE2NEON_CMPSTR_SET_UPPER(bound, imm8); + int la; + SSE2NEON_CMPISTRX_LENGTH(a, la, imm8); + return la <= (bound - 1); +} + +// Compare packed strings with implicit lengths in a and b using the control in +// imm8, and returns 1 if any character in b was null, and 0 otherwise. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpistrz +FORCE_INLINE int _mm_cmpistrz(__m128i a, __m128i b, const int imm8) +{ + (void) a; + SSE2NEON_CMPSTR_SET_UPPER(bound, imm8); + int lb; + SSE2NEON_CMPISTRX_LENGTH(b, lb, imm8); + return lb <= (bound - 1); +} + +// Compares the 2 signed 64-bit integers in a and the 2 signed 64-bit integers +// in b for greater than. +FORCE_INLINE __m128i _mm_cmpgt_epi64(__m128i a, __m128i b) +{ +#if defined(__aarch64__) || defined(_M_ARM64) + return vreinterpretq_m128i_u64( + vcgtq_s64(vreinterpretq_s64_m128i(a), vreinterpretq_s64_m128i(b))); #else - uint8x16_t lm_p0 = vcombine_u8(vget_low_u8(l), vget_low_u8(m)); - uint8x16_t lm_p1 = vcombine_u8(vget_high_u8(l), vget_high_u8(m)); - uint8x16_t nk_p0 = vcombine_u8(vget_low_u8(n), vget_low_u8(k)); - uint8x16_t nk_p1 = vcombine_u8(vget_high_u8(n), vget_high_u8(k)); + return vreinterpretq_m128i_s64(vshrq_n_s64( + vqsubq_s64(vreinterpretq_s64_m128i(b), vreinterpretq_s64_m128i(a)), + 63)); #endif - // t0 = (L) (P0 + P1) << 8 - // t1 = (M) (P2 + P3) << 16 - uint8x16_t t0t1_tmp = veorq_u8(lm_p0, lm_p1); - uint8x16_t t0t1_h = vandq_u8(lm_p1, k48_32); - uint8x16_t t0t1_l = veorq_u8(t0t1_tmp, t0t1_h); +} - // t2 = (N) (P4 + P5) << 24 - // t3 = (K) (P6 + P7) << 32 - uint8x16_t t2t3_tmp = veorq_u8(nk_p0, nk_p1); - uint8x16_t t2t3_h = vandq_u8(nk_p1, k16_00); - uint8x16_t t2t3_l = veorq_u8(t2t3_tmp, t2t3_h); +// Starting with the initial value in crc, accumulates a CRC32 value for +// unsigned 16-bit integer v, and stores the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_crc32_u16 +FORCE_INLINE uint32_t _mm_crc32_u16(uint32_t crc, uint16_t v) +{ +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) + __asm__ __volatile__("crc32ch %w[c], %w[c], %w[v]\n\t" + : [c] "+r"(crc) + : [v] "r"(v)); +#elif ((__ARM_ARCH == 8) && defined(__ARM_FEATURE_CRC32)) || \ + (defined(_M_ARM64) && !defined(__clang__)) + crc = __crc32ch(crc, v); +#else + crc = _mm_crc32_u8(crc, v & 0xff); + crc = _mm_crc32_u8(crc, (v >> 8) & 0xff); +#endif + return crc; +} - // De-interleave -#if defined(__aarch64__) - uint8x16_t t0 = vreinterpretq_u8_u64( - vuzp1q_u64(vreinterpretq_u64_u8(t0t1_l), vreinterpretq_u64_u8(t0t1_h))); - uint8x16_t t1 = vreinterpretq_u8_u64( - vuzp2q_u64(vreinterpretq_u64_u8(t0t1_l), vreinterpretq_u64_u8(t0t1_h))); - uint8x16_t t2 = vreinterpretq_u8_u64( - vuzp1q_u64(vreinterpretq_u64_u8(t2t3_l), vreinterpretq_u64_u8(t2t3_h))); - uint8x16_t t3 = vreinterpretq_u8_u64( - vuzp2q_u64(vreinterpretq_u64_u8(t2t3_l), vreinterpretq_u64_u8(t2t3_h))); +// Starting with the initial value in crc, accumulates a CRC32 value for +// unsigned 32-bit integer v, and stores the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_crc32_u32 +FORCE_INLINE uint32_t _mm_crc32_u32(uint32_t crc, uint32_t v) +{ +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) + __asm__ __volatile__("crc32cw %w[c], %w[c], %w[v]\n\t" + : [c] "+r"(crc) + : [v] "r"(v)); +#elif ((__ARM_ARCH == 8) && defined(__ARM_FEATURE_CRC32)) || \ + (defined(_M_ARM64) && !defined(__clang__)) + crc = __crc32cw(crc, v); #else - uint8x16_t t1 = vcombine_u8(vget_high_u8(t0t1_l), vget_high_u8(t0t1_h)); - uint8x16_t t0 = vcombine_u8(vget_low_u8(t0t1_l), vget_low_u8(t0t1_h)); - uint8x16_t t3 = vcombine_u8(vget_high_u8(t2t3_l), vget_high_u8(t2t3_h)); - uint8x16_t t2 = vcombine_u8(vget_low_u8(t2t3_l), vget_low_u8(t2t3_h)); + crc = _mm_crc32_u16(crc, v & 0xffff); + crc = _mm_crc32_u16(crc, (v >> 16) & 0xffff); #endif - // Shift the cross products - uint8x16_t t0_shift = vextq_u8(t0, t0, 15); // t0 << 8 - uint8x16_t t1_shift = vextq_u8(t1, t1, 14); // t1 << 16 - uint8x16_t t2_shift = vextq_u8(t2, t2, 13); // t2 << 24 - uint8x16_t t3_shift = vextq_u8(t3, t3, 12); // t3 << 32 + return crc; +} - // Accumulate the products - uint8x16_t cross1 = veorq_u8(t0_shift, t1_shift); - uint8x16_t cross2 = veorq_u8(t2_shift, t3_shift); - uint8x16_t mix = veorq_u8(d, cross1); - uint8x16_t r = veorq_u8(mix, cross2); - return vreinterpretq_u64_u8(r); +// Starting with the initial value in crc, accumulates a CRC32 value for +// unsigned 64-bit integer v, and stores the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_crc32_u64 +FORCE_INLINE uint64_t _mm_crc32_u64(uint64_t crc, uint64_t v) +{ +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) + __asm__ __volatile__("crc32cx %w[c], %w[c], %x[v]\n\t" + : [c] "+r"(crc) + : [v] "r"(v)); +#elif (defined(_M_ARM64) && !defined(__clang__)) + crc = __crc32cd((uint32_t) crc, v); +#else + crc = _mm_crc32_u32((uint32_t) (crc), v & 0xffffffff); + crc = _mm_crc32_u32((uint32_t) (crc), (v >> 32) & 0xffffffff); +#endif + return crc; } -#endif // ARMv7 polyfill -FORCE_INLINE __m128i _mm_clmulepi64_si128(__m128i _a, __m128i _b, const int imm) +// Starting with the initial value in crc, accumulates a CRC32 value for +// unsigned 8-bit integer v, and stores the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_crc32_u8 +FORCE_INLINE uint32_t _mm_crc32_u8(uint32_t crc, uint8_t v) { - uint64x2_t a = vreinterpretq_u64_m128i(_a); - uint64x2_t b = vreinterpretq_u64_m128i(_b); - switch (imm & 0x11) { - case 0x00: - return vreinterpretq_m128i_u64( - _sse2neon_vmull_p64(vget_low_u64(a), vget_low_u64(b))); - case 0x01: - return vreinterpretq_m128i_u64( - _sse2neon_vmull_p64(vget_high_u64(a), vget_low_u64(b))); - case 0x10: - return vreinterpretq_m128i_u64( - _sse2neon_vmull_p64(vget_low_u64(a), vget_high_u64(b))); - case 0x11: - return vreinterpretq_m128i_u64( - _sse2neon_vmull_p64(vget_high_u64(a), vget_high_u64(b))); - default: - abort(); +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) + __asm__ __volatile__("crc32cb %w[c], %w[c], %w[v]\n\t" + : [c] "+r"(crc) + : [v] "r"(v)); +#elif ((__ARM_ARCH == 8) && defined(__ARM_FEATURE_CRC32)) || \ + (defined(_M_ARM64) && !defined(__clang__)) + crc = __crc32cb(crc, v); +#else + crc ^= v; + for (int bit = 0; bit < 8; bit++) { + if (crc & 1) + crc = (crc >> 1) ^ UINT32_C(0x82f63b78); + else + crc = (crc >> 1); } +#endif + return crc; } -#if !defined(__ARM_FEATURE_CRYPTO) +/* AES */ + +#if !defined(__ARM_FEATURE_CRYPTO) && (!defined(_M_ARM64) || defined(__clang__)) /* clang-format off */ -#define SSE2NEON_AES_DATA(w) \ +#define SSE2NEON_AES_SBOX(w) \ { \ w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), \ w(0xc5), w(0x30), w(0x01), w(0x67), w(0x2b), w(0xfe), w(0xd7), \ @@ -5967,53 +8531,114 @@ FORCE_INLINE __m128i _mm_clmulepi64_si128(__m128i _a, __m128i _b, const int imm) w(0xe6), w(0x42), w(0x68), w(0x41), w(0x99), w(0x2d), w(0x0f), \ w(0xb0), w(0x54), w(0xbb), w(0x16) \ } +#define SSE2NEON_AES_RSBOX(w) \ + { \ + w(0x52), w(0x09), w(0x6a), w(0xd5), w(0x30), w(0x36), w(0xa5), \ + w(0x38), w(0xbf), w(0x40), w(0xa3), w(0x9e), w(0x81), w(0xf3), \ + w(0xd7), w(0xfb), w(0x7c), w(0xe3), w(0x39), w(0x82), w(0x9b), \ + w(0x2f), w(0xff), w(0x87), w(0x34), w(0x8e), w(0x43), w(0x44), \ + w(0xc4), w(0xde), w(0xe9), w(0xcb), w(0x54), w(0x7b), w(0x94), \ + w(0x32), w(0xa6), w(0xc2), w(0x23), w(0x3d), w(0xee), w(0x4c), \ + w(0x95), w(0x0b), w(0x42), w(0xfa), w(0xc3), w(0x4e), w(0x08), \ + w(0x2e), w(0xa1), w(0x66), w(0x28), w(0xd9), w(0x24), w(0xb2), \ + w(0x76), w(0x5b), w(0xa2), w(0x49), w(0x6d), w(0x8b), w(0xd1), \ + w(0x25), w(0x72), w(0xf8), w(0xf6), w(0x64), w(0x86), w(0x68), \ + w(0x98), w(0x16), w(0xd4), w(0xa4), w(0x5c), w(0xcc), w(0x5d), \ + w(0x65), w(0xb6), w(0x92), w(0x6c), w(0x70), w(0x48), w(0x50), \ + w(0xfd), w(0xed), w(0xb9), w(0xda), w(0x5e), w(0x15), w(0x46), \ + w(0x57), w(0xa7), w(0x8d), w(0x9d), w(0x84), w(0x90), w(0xd8), \ + w(0xab), w(0x00), w(0x8c), w(0xbc), w(0xd3), w(0x0a), w(0xf7), \ + w(0xe4), w(0x58), w(0x05), w(0xb8), w(0xb3), w(0x45), w(0x06), \ + w(0xd0), w(0x2c), w(0x1e), w(0x8f), w(0xca), w(0x3f), w(0x0f), \ + w(0x02), w(0xc1), w(0xaf), w(0xbd), w(0x03), w(0x01), w(0x13), \ + w(0x8a), w(0x6b), w(0x3a), w(0x91), w(0x11), w(0x41), w(0x4f), \ + w(0x67), w(0xdc), w(0xea), w(0x97), w(0xf2), w(0xcf), w(0xce), \ + w(0xf0), w(0xb4), w(0xe6), w(0x73), w(0x96), w(0xac), w(0x74), \ + w(0x22), w(0xe7), w(0xad), w(0x35), w(0x85), w(0xe2), w(0xf9), \ + w(0x37), w(0xe8), w(0x1c), w(0x75), w(0xdf), w(0x6e), w(0x47), \ + w(0xf1), w(0x1a), w(0x71), w(0x1d), w(0x29), w(0xc5), w(0x89), \ + w(0x6f), w(0xb7), w(0x62), w(0x0e), w(0xaa), w(0x18), w(0xbe), \ + w(0x1b), w(0xfc), w(0x56), w(0x3e), w(0x4b), w(0xc6), w(0xd2), \ + w(0x79), w(0x20), w(0x9a), w(0xdb), w(0xc0), w(0xfe), w(0x78), \ + w(0xcd), w(0x5a), w(0xf4), w(0x1f), w(0xdd), w(0xa8), w(0x33), \ + w(0x88), w(0x07), w(0xc7), w(0x31), w(0xb1), w(0x12), w(0x10), \ + w(0x59), w(0x27), w(0x80), w(0xec), w(0x5f), w(0x60), w(0x51), \ + w(0x7f), w(0xa9), w(0x19), w(0xb5), w(0x4a), w(0x0d), w(0x2d), \ + w(0xe5), w(0x7a), w(0x9f), w(0x93), w(0xc9), w(0x9c), w(0xef), \ + w(0xa0), w(0xe0), w(0x3b), w(0x4d), w(0xae), w(0x2a), w(0xf5), \ + w(0xb0), w(0xc8), w(0xeb), w(0xbb), w(0x3c), w(0x83), w(0x53), \ + w(0x99), w(0x61), w(0x17), w(0x2b), w(0x04), w(0x7e), w(0xba), \ + w(0x77), w(0xd6), w(0x26), w(0xe1), w(0x69), w(0x14), w(0x63), \ + w(0x55), w(0x21), w(0x0c), w(0x7d) \ + } /* clang-format on */ /* X Macro trick. See https://en.wikipedia.org/wiki/X_Macro */ #define SSE2NEON_AES_H0(x) (x) -static const uint8_t SSE2NEON_sbox[256] = SSE2NEON_AES_DATA(SSE2NEON_AES_H0); +static const uint8_t _sse2neon_sbox[256] = SSE2NEON_AES_SBOX(SSE2NEON_AES_H0); +static const uint8_t _sse2neon_rsbox[256] = SSE2NEON_AES_RSBOX(SSE2NEON_AES_H0); #undef SSE2NEON_AES_H0 -// In the absence of crypto extensions, implement aesenc using regular neon +/* x_time function and matrix multiply function */ +#if !defined(__aarch64__) && !defined(_M_ARM64) +#define SSE2NEON_XT(x) (((x) << 1) ^ ((((x) >> 7) & 1) * 0x1b)) +#define SSE2NEON_MULTIPLY(x, y) \ + (((y & 1) * x) ^ ((y >> 1 & 1) * SSE2NEON_XT(x)) ^ \ + ((y >> 2 & 1) * SSE2NEON_XT(SSE2NEON_XT(x))) ^ \ + ((y >> 3 & 1) * SSE2NEON_XT(SSE2NEON_XT(SSE2NEON_XT(x)))) ^ \ + ((y >> 4 & 1) * SSE2NEON_XT(SSE2NEON_XT(SSE2NEON_XT(SSE2NEON_XT(x)))))) +#endif + +// In the absence of crypto extensions, implement aesenc using regular NEON // intrinsics instead. See: // https://www.workofard.com/2017/01/accelerated-aes-for-the-arm64-linux-kernel/ // https://www.workofard.com/2017/07/ghash-for-low-end-cores/ and -// https://github.com/ColinIanKing/linux-next-mirror/blob/b5f466091e130caaf0735976648f72bd5e09aa84/crypto/aegis128-neon-inner.c#L52 -// for more information Reproduced with permission of the author. -FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey) +// for more information. +FORCE_INLINE __m128i _mm_aesenc_si128(__m128i a, __m128i RoundKey) { -#if defined(__aarch64__) - static const uint8_t shift_rows[] = {0x0, 0x5, 0xa, 0xf, 0x4, 0x9, - 0xe, 0x3, 0x8, 0xd, 0x2, 0x7, - 0xc, 0x1, 0x6, 0xb}; - static const uint8_t ror32by8[] = {0x1, 0x2, 0x3, 0x0, 0x5, 0x6, 0x7, 0x4, - 0x9, 0xa, 0xb, 0x8, 0xd, 0xe, 0xf, 0xc}; +#if defined(__aarch64__) || defined(_M_ARM64) + static const uint8_t shift_rows[] = { + 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3, + 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb, + }; + static const uint8_t ror32by8[] = { + 0x1, 0x2, 0x3, 0x0, 0x5, 0x6, 0x7, 0x4, + 0x9, 0xa, 0xb, 0x8, 0xd, 0xe, 0xf, 0xc, + }; uint8x16_t v; - uint8x16_t w = vreinterpretq_u8_m128i(EncBlock); + uint8x16_t w = vreinterpretq_u8_m128i(a); - // shift rows + /* shift rows */ w = vqtbl1q_u8(w, vld1q_u8(shift_rows)); - // sub bytes - v = vqtbl4q_u8(vld1q_u8_x4(SSE2NEON_sbox), w); - v = vqtbx4q_u8(v, vld1q_u8_x4(SSE2NEON_sbox + 0x40), w - 0x40); - v = vqtbx4q_u8(v, vld1q_u8_x4(SSE2NEON_sbox + 0x80), w - 0x80); - v = vqtbx4q_u8(v, vld1q_u8_x4(SSE2NEON_sbox + 0xc0), w - 0xc0); - - // mix columns - w = (v << 1) ^ (uint8x16_t)(((int8x16_t) v >> 7) & 0x1b); + /* sub bytes */ + // Here, we separate the whole 256-bytes table into 4 64-bytes tables, and + // look up each of the table. After each lookup, we load the next table + // which locates at the next 64-bytes. In the meantime, the index in the + // table would be smaller than it was, so the index parameters of + // `vqtbx4q_u8()` need to be added the same constant as the loaded tables. + v = vqtbl4q_u8(_sse2neon_vld1q_u8_x4(_sse2neon_sbox), w); + // 'w-0x40' equals to 'vsubq_u8(w, vdupq_n_u8(0x40))' + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0x40), w - 0x40); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0x80), w - 0x80); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0xc0), w - 0xc0); + + /* mix columns */ + w = (v << 1) ^ (uint8x16_t) (((int8x16_t) v >> 7) & 0x1b); w ^= (uint8x16_t) vrev32q_u16((uint16x8_t) v); w ^= vqtbl1q_u8(v ^ w, vld1q_u8(ror32by8)); - // add round key + /* add round key */ return vreinterpretq_m128i_u8(w) ^ RoundKey; -#else /* ARMv7-A NEON implementation */ -#define SSE2NEON_AES_B2W(b0, b1, b2, b3) \ - (((uint32_t)(b3) << 24) | ((uint32_t)(b2) << 16) | ((uint32_t)(b1) << 8) | \ - (b0)) +#else /* ARMv7-A implementation for a table-based AES */ +#define SSE2NEON_AES_B2W(b0, b1, b2, b3) \ + (((uint32_t) (b3) << 24) | ((uint32_t) (b2) << 16) | \ + ((uint32_t) (b1) << 8) | (uint32_t) (b0)) +// muliplying 'x' by 2 in GF(2^8) #define SSE2NEON_AES_F2(x) ((x << 1) ^ (((x >> 7) & 1) * 0x011b /* WPOLY */)) +// muliplying 'x' by 3 in GF(2^8) #define SSE2NEON_AES_F3(x) (SSE2NEON_AES_F2(x) ^ x) #define SSE2NEON_AES_U0(p) \ SSE2NEON_AES_B2W(SSE2NEON_AES_F2(p), p, p, SSE2NEON_AES_F3(p)) @@ -6023,11 +8648,14 @@ FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey) SSE2NEON_AES_B2W(p, SSE2NEON_AES_F3(p), SSE2NEON_AES_F2(p), p) #define SSE2NEON_AES_U3(p) \ SSE2NEON_AES_B2W(p, p, SSE2NEON_AES_F3(p), SSE2NEON_AES_F2(p)) + + // this generates a table containing every possible permutation of + // shift_rows() and sub_bytes() with mix_columns(). static const uint32_t ALIGN_STRUCT(16) aes_table[4][256] = { - SSE2NEON_AES_DATA(SSE2NEON_AES_U0), - SSE2NEON_AES_DATA(SSE2NEON_AES_U1), - SSE2NEON_AES_DATA(SSE2NEON_AES_U2), - SSE2NEON_AES_DATA(SSE2NEON_AES_U3), + SSE2NEON_AES_SBOX(SSE2NEON_AES_U0), + SSE2NEON_AES_SBOX(SSE2NEON_AES_U1), + SSE2NEON_AES_SBOX(SSE2NEON_AES_U2), + SSE2NEON_AES_SBOX(SSE2NEON_AES_U3), }; #undef SSE2NEON_AES_B2W #undef SSE2NEON_AES_F2 @@ -6037,11 +8665,15 @@ FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey) #undef SSE2NEON_AES_U2 #undef SSE2NEON_AES_U3 - uint32_t x0 = _mm_cvtsi128_si32(EncBlock); - uint32_t x1 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0x55)); - uint32_t x2 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0xAA)); - uint32_t x3 = _mm_cvtsi128_si32(_mm_shuffle_epi32(EncBlock, 0xFF)); + uint32_t x0 = _mm_cvtsi128_si32(a); // get a[31:0] + uint32_t x1 = + _mm_cvtsi128_si32(_mm_shuffle_epi32(a, 0x55)); // get a[63:32] + uint32_t x2 = + _mm_cvtsi128_si32(_mm_shuffle_epi32(a, 0xAA)); // get a[95:64] + uint32_t x3 = + _mm_cvtsi128_si32(_mm_shuffle_epi32(a, 0xFF)); // get a[127:96] + // finish the modulo addition step in mix_columns() __m128i out = _mm_set_epi32( (aes_table[0][x3 & 0xff] ^ aes_table[1][(x0 >> 8) & 0xff] ^ aes_table[2][(x1 >> 16) & 0xff] ^ aes_table[3][x2 >> 24]), @@ -6056,51 +8688,254 @@ FORCE_INLINE __m128i _mm_aesenc_si128(__m128i EncBlock, __m128i RoundKey) #endif } +// Perform one round of an AES decryption flow on data (state) in a using the +// round key in RoundKey, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec_si128 +FORCE_INLINE __m128i _mm_aesdec_si128(__m128i a, __m128i RoundKey) +{ +#if defined(__aarch64__) + static const uint8_t inv_shift_rows[] = { + 0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb, + 0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3, + }; + static const uint8_t ror32by8[] = { + 0x1, 0x2, 0x3, 0x0, 0x5, 0x6, 0x7, 0x4, + 0x9, 0xa, 0xb, 0x8, 0xd, 0xe, 0xf, 0xc, + }; + + uint8x16_t v; + uint8x16_t w = vreinterpretq_u8_m128i(a); + + // inverse shift rows + w = vqtbl1q_u8(w, vld1q_u8(inv_shift_rows)); + + // inverse sub bytes + v = vqtbl4q_u8(_sse2neon_vld1q_u8_x4(_sse2neon_rsbox), w); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_rsbox + 0x40), w - 0x40); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_rsbox + 0x80), w - 0x80); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_rsbox + 0xc0), w - 0xc0); + + // inverse mix columns + // multiplying 'v' by 4 in GF(2^8) + w = (v << 1) ^ (uint8x16_t) (((int8x16_t) v >> 7) & 0x1b); + w = (w << 1) ^ (uint8x16_t) (((int8x16_t) w >> 7) & 0x1b); + v ^= w; + v ^= (uint8x16_t) vrev32q_u16((uint16x8_t) w); + + w = (v << 1) ^ (uint8x16_t) (((int8x16_t) v >> 7) & + 0x1b); // muliplying 'v' by 2 in GF(2^8) + w ^= (uint8x16_t) vrev32q_u16((uint16x8_t) v); + w ^= vqtbl1q_u8(v ^ w, vld1q_u8(ror32by8)); + + // add round key + return vreinterpretq_m128i_u8(w) ^ RoundKey; + +#else /* ARMv7-A NEON implementation */ + /* FIXME: optimized for NEON */ + uint8_t i, e, f, g, h, v[4][4]; + uint8_t *_a = (uint8_t *) &a; + for (i = 0; i < 16; ++i) { + v[((i / 4) + (i % 4)) % 4][i % 4] = _sse2neon_rsbox[_a[i]]; + } + + // inverse mix columns + for (i = 0; i < 4; ++i) { + e = v[i][0]; + f = v[i][1]; + g = v[i][2]; + h = v[i][3]; + + v[i][0] = SSE2NEON_MULTIPLY(e, 0x0e) ^ SSE2NEON_MULTIPLY(f, 0x0b) ^ + SSE2NEON_MULTIPLY(g, 0x0d) ^ SSE2NEON_MULTIPLY(h, 0x09); + v[i][1] = SSE2NEON_MULTIPLY(e, 0x09) ^ SSE2NEON_MULTIPLY(f, 0x0e) ^ + SSE2NEON_MULTIPLY(g, 0x0b) ^ SSE2NEON_MULTIPLY(h, 0x0d); + v[i][2] = SSE2NEON_MULTIPLY(e, 0x0d) ^ SSE2NEON_MULTIPLY(f, 0x09) ^ + SSE2NEON_MULTIPLY(g, 0x0e) ^ SSE2NEON_MULTIPLY(h, 0x0b); + v[i][3] = SSE2NEON_MULTIPLY(e, 0x0b) ^ SSE2NEON_MULTIPLY(f, 0x0d) ^ + SSE2NEON_MULTIPLY(g, 0x09) ^ SSE2NEON_MULTIPLY(h, 0x0e); + } + + return vreinterpretq_m128i_u8(vld1q_u8((uint8_t *) v)) ^ RoundKey; +#endif +} + +// Perform the last round of an AES encryption flow on data (state) in a using +// the round key in RoundKey, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenclast_si128 FORCE_INLINE __m128i _mm_aesenclast_si128(__m128i a, __m128i RoundKey) { +#if defined(__aarch64__) + static const uint8_t shift_rows[] = { + 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3, + 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb, + }; + + uint8x16_t v; + uint8x16_t w = vreinterpretq_u8_m128i(a); + + // shift rows + w = vqtbl1q_u8(w, vld1q_u8(shift_rows)); + + // sub bytes + v = vqtbl4q_u8(_sse2neon_vld1q_u8_x4(_sse2neon_sbox), w); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0x40), w - 0x40); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0x80), w - 0x80); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0xc0), w - 0xc0); + + // add round key + return vreinterpretq_m128i_u8(v) ^ RoundKey; + +#else /* ARMv7-A implementation */ + uint8_t v[16] = { + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 0)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 5)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 10)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 15)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 4)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 9)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 14)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 3)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 8)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 13)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 2)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 7)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 12)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 1)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 6)], + _sse2neon_sbox[vgetq_lane_u8(vreinterpretq_u8_m128i(a), 11)], + }; + + return vreinterpretq_m128i_u8(vld1q_u8(v)) ^ RoundKey; +#endif +} + +// Perform the last round of an AES decryption flow on data (state) in a using +// the round key in RoundKey, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdeclast_si128 +FORCE_INLINE __m128i _mm_aesdeclast_si128(__m128i a, __m128i RoundKey) +{ +#if defined(__aarch64__) + static const uint8_t inv_shift_rows[] = { + 0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb, + 0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3, + }; + + uint8x16_t v; + uint8x16_t w = vreinterpretq_u8_m128i(a); + + // inverse shift rows + w = vqtbl1q_u8(w, vld1q_u8(inv_shift_rows)); + + // inverse sub bytes + v = vqtbl4q_u8(_sse2neon_vld1q_u8_x4(_sse2neon_rsbox), w); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_rsbox + 0x40), w - 0x40); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_rsbox + 0x80), w - 0x80); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_rsbox + 0xc0), w - 0xc0); + + // add round key + return vreinterpretq_m128i_u8(v) ^ RoundKey; + +#else /* ARMv7-A NEON implementation */ /* FIXME: optimized for NEON */ - uint8_t v[4][4] = { - [0] = {SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 0)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 5)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 10)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 15)]}, - [1] = {SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 4)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 9)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 14)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 3)]}, - [2] = {SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 8)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 13)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 2)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 7)]}, - [3] = {SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 12)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 1)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 6)], - SSE2NEON_sbox[vreinterpretq_nth_u8_m128i(a, 11)]}, + uint8_t v[4][4]; + uint8_t *_a = (uint8_t *) &a; + for (int i = 0; i < 16; ++i) { + v[((i / 4) + (i % 4)) % 4][i % 4] = _sse2neon_rsbox[_a[i]]; + } + + return vreinterpretq_m128i_u8(vld1q_u8((uint8_t *) v)) ^ RoundKey; +#endif +} + +// Perform the InvMixColumns transformation on a and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesimc_si128 +FORCE_INLINE __m128i _mm_aesimc_si128(__m128i a) +{ +#if defined(__aarch64__) + static const uint8_t ror32by8[] = { + 0x1, 0x2, 0x3, 0x0, 0x5, 0x6, 0x7, 0x4, + 0x9, 0xa, 0xb, 0x8, 0xd, 0xe, 0xf, 0xc, }; - for (int i = 0; i < 16; i++) - vreinterpretq_nth_u8_m128i(a, i) = - v[i / 4][i % 4] ^ vreinterpretq_nth_u8_m128i(RoundKey, i); - return a; + uint8x16_t v = vreinterpretq_u8_m128i(a); + uint8x16_t w; + + // multiplying 'v' by 4 in GF(2^8) + w = (v << 1) ^ (uint8x16_t) (((int8x16_t) v >> 7) & 0x1b); + w = (w << 1) ^ (uint8x16_t) (((int8x16_t) w >> 7) & 0x1b); + v ^= w; + v ^= (uint8x16_t) vrev32q_u16((uint16x8_t) w); + + // multiplying 'v' by 2 in GF(2^8) + w = (v << 1) ^ (uint8x16_t) (((int8x16_t) v >> 7) & 0x1b); + w ^= (uint8x16_t) vrev32q_u16((uint16x8_t) v); + w ^= vqtbl1q_u8(v ^ w, vld1q_u8(ror32by8)); + return vreinterpretq_m128i_u8(w); + +#else /* ARMv7-A NEON implementation */ + uint8_t i, e, f, g, h, v[4][4]; + vst1q_u8((uint8_t *) v, vreinterpretq_u8_m128i(a)); + for (i = 0; i < 4; ++i) { + e = v[i][0]; + f = v[i][1]; + g = v[i][2]; + h = v[i][3]; + + v[i][0] = SSE2NEON_MULTIPLY(e, 0x0e) ^ SSE2NEON_MULTIPLY(f, 0x0b) ^ + SSE2NEON_MULTIPLY(g, 0x0d) ^ SSE2NEON_MULTIPLY(h, 0x09); + v[i][1] = SSE2NEON_MULTIPLY(e, 0x09) ^ SSE2NEON_MULTIPLY(f, 0x0e) ^ + SSE2NEON_MULTIPLY(g, 0x0b) ^ SSE2NEON_MULTIPLY(h, 0x0d); + v[i][2] = SSE2NEON_MULTIPLY(e, 0x0d) ^ SSE2NEON_MULTIPLY(f, 0x09) ^ + SSE2NEON_MULTIPLY(g, 0x0e) ^ SSE2NEON_MULTIPLY(h, 0x0b); + v[i][3] = SSE2NEON_MULTIPLY(e, 0x0b) ^ SSE2NEON_MULTIPLY(f, 0x0d) ^ + SSE2NEON_MULTIPLY(g, 0x09) ^ SSE2NEON_MULTIPLY(h, 0x0e); + } + + return vreinterpretq_m128i_u8(vld1q_u8((uint8_t *) v)); +#endif } +// Assist in expanding the AES cipher key by computing steps towards generating +// a round key for encryption cipher using data from a and an 8-bit round +// constant specified in imm8, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aeskeygenassist_si128 +// // Emits the Advanced Encryption Standard (AES) instruction aeskeygenassist. // This instruction generates a round key for AES encryption. See // https://kazakov.life/2017/11/01/cryptocurrency-mining-on-ios-devices/ // for details. -// -// https://msdn.microsoft.com/en-us/library/cc714138(v=vs.120).aspx -FORCE_INLINE __m128i _mm_aeskeygenassist_si128(__m128i key, const int rcon) +FORCE_INLINE __m128i _mm_aeskeygenassist_si128(__m128i a, const int rcon) { - uint32_t X1 = _mm_cvtsi128_si32(_mm_shuffle_epi32(key, 0x55)); - uint32_t X3 = _mm_cvtsi128_si32(_mm_shuffle_epi32(key, 0xFF)); +#if defined(__aarch64__) + uint8x16_t _a = vreinterpretq_u8_m128i(a); + uint8x16_t v = vqtbl4q_u8(_sse2neon_vld1q_u8_x4(_sse2neon_sbox), _a); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0x40), _a - 0x40); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0x80), _a - 0x80); + v = vqtbx4q_u8(v, _sse2neon_vld1q_u8_x4(_sse2neon_sbox + 0xc0), _a - 0xc0); + + uint32x4_t v_u32 = vreinterpretq_u32_u8(v); + uint32x4_t ror_v = vorrq_u32(vshrq_n_u32(v_u32, 8), vshlq_n_u32(v_u32, 24)); + uint32x4_t ror_xor_v = veorq_u32(ror_v, vdupq_n_u32(rcon)); + + return vreinterpretq_m128i_u32(vtrn2q_u32(v_u32, ror_xor_v)); + +#else /* ARMv7-A NEON implementation */ + uint32_t X1 = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, 0x55)); + uint32_t X3 = _mm_cvtsi128_si32(_mm_shuffle_epi32(a, 0xFF)); for (int i = 0; i < 4; ++i) { - ((uint8_t *) &X1)[i] = SSE2NEON_sbox[((uint8_t *) &X1)[i]]; - ((uint8_t *) &X3)[i] = SSE2NEON_sbox[((uint8_t *) &X3)[i]]; + ((uint8_t *) &X1)[i] = _sse2neon_sbox[((uint8_t *) &X1)[i]]; + ((uint8_t *) &X3)[i] = _sse2neon_sbox[((uint8_t *) &X3)[i]]; } return _mm_set_epi32(((X3 >> 8) | (X3 << 24)) ^ rcon, X3, ((X1 >> 8) | (X1 << 24)) ^ rcon, X1); +#endif } -#undef SSE2NEON_AES_DATA +#undef SSE2NEON_AES_SBOX +#undef SSE2NEON_AES_RSBOX + +#if defined(__aarch64__) +#undef SSE2NEON_XT +#undef SSE2NEON_MULTIPLY +#endif #else /* __ARM_FEATURE_CRYPTO */ // Implements equivalent of 'aesenc' by combining AESE (with an empty key) and @@ -6111,12 +8946,24 @@ FORCE_INLINE __m128i _mm_aeskeygenassist_si128(__m128i key, const int rcon) // for more details. FORCE_INLINE __m128i _mm_aesenc_si128(__m128i a, __m128i b) { - return vreinterpretq_m128i_u8( - vaesmcq_u8(vaeseq_u8(vreinterpretq_u8_m128i(a), vdupq_n_u8(0))) ^ - vreinterpretq_u8_m128i(b)); + return vreinterpretq_m128i_u8(veorq_u8( + vaesmcq_u8(vaeseq_u8(vreinterpretq_u8_m128i(a), vdupq_n_u8(0))), + vreinterpretq_u8_m128i(b))); +} + +// Perform one round of an AES decryption flow on data (state) in a using the +// round key in RoundKey, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec_si128 +FORCE_INLINE __m128i _mm_aesdec_si128(__m128i a, __m128i RoundKey) +{ + return vreinterpretq_m128i_u8(veorq_u8( + vaesimcq_u8(vaesdq_u8(vreinterpretq_u8_m128i(a), vdupq_n_u8(0))), + vreinterpretq_u8_m128i(RoundKey))); } -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_aesenclast_si128 +// Perform the last round of an AES encryption flow on data (state) in a using +// the round key in RoundKey, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenclast_si128 FORCE_INLINE __m128i _mm_aesenclast_si128(__m128i a, __m128i RoundKey) { return _mm_xor_si128(vreinterpretq_m128i_u8(vaeseq_u8( @@ -6124,11 +8971,33 @@ FORCE_INLINE __m128i _mm_aesenclast_si128(__m128i a, __m128i RoundKey) RoundKey); } +// Perform the last round of an AES decryption flow on data (state) in a using +// the round key in RoundKey, and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdeclast_si128 +FORCE_INLINE __m128i _mm_aesdeclast_si128(__m128i a, __m128i RoundKey) +{ + return vreinterpretq_m128i_u8( + veorq_u8(vaesdq_u8(vreinterpretq_u8_m128i(a), vdupq_n_u8(0)), + vreinterpretq_u8_m128i(RoundKey))); +} + +// Perform the InvMixColumns transformation on a and store the result in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesimc_si128 +FORCE_INLINE __m128i _mm_aesimc_si128(__m128i a) +{ + return vreinterpretq_m128i_u8(vaesimcq_u8(vreinterpretq_u8_m128i(a))); +} + +// Assist in expanding the AES cipher key by computing steps towards generating +// a round key for encryption cipher using data from a and an 8-bit round +// constant specified in imm8, and store the result in dst." +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aeskeygenassist_si128 FORCE_INLINE __m128i _mm_aeskeygenassist_si128(__m128i a, const int rcon) { // AESE does ShiftRows and SubBytes on A uint8x16_t u8 = vaeseq_u8(vreinterpretq_u8_m128i(a), vdupq_n_u8(0)); +#ifndef _MSC_VER uint8x16_t dest = { // Undo ShiftRows step from AESE and extract X1 and X3 u8[0x4], u8[0x1], u8[0xE], u8[0xB], // SubBytes(X1) @@ -6138,156 +9007,209 @@ FORCE_INLINE __m128i _mm_aeskeygenassist_si128(__m128i a, const int rcon) }; uint32x4_t r = {0, (unsigned) rcon, 0, (unsigned) rcon}; return vreinterpretq_m128i_u8(dest) ^ vreinterpretq_m128i_u32(r); +#else + // We have to do this hack because MSVC is strictly adhering to the CPP + // standard, in particular C++03 8.5.1 sub-section 15, which states that + // unions must be initialized by their first member type. + + // As per the Windows ARM64 ABI, it is always little endian, so this works + __n128 dest{ + ((uint64_t) u8.n128_u8[0x4] << 0) | ((uint64_t) u8.n128_u8[0x1] << 8) | + ((uint64_t) u8.n128_u8[0xE] << 16) | + ((uint64_t) u8.n128_u8[0xB] << 24) | + ((uint64_t) u8.n128_u8[0x1] << 32) | + ((uint64_t) u8.n128_u8[0xE] << 40) | + ((uint64_t) u8.n128_u8[0xB] << 48) | + ((uint64_t) u8.n128_u8[0x4] << 56), + ((uint64_t) u8.n128_u8[0xC] << 0) | ((uint64_t) u8.n128_u8[0x9] << 8) | + ((uint64_t) u8.n128_u8[0x6] << 16) | + ((uint64_t) u8.n128_u8[0x3] << 24) | + ((uint64_t) u8.n128_u8[0x9] << 32) | + ((uint64_t) u8.n128_u8[0x6] << 40) | + ((uint64_t) u8.n128_u8[0x3] << 48) | + ((uint64_t) u8.n128_u8[0xC] << 56)}; + + dest.n128_u32[1] = dest.n128_u32[1] ^ rcon; + dest.n128_u32[3] = dest.n128_u32[3] ^ rcon; + + return dest; +#endif } #endif -/* Streaming Extensions */ +/* Others */ -// Guarantees that every preceding store is globally visible before any -// subsequent store. -// https://msdn.microsoft.com/en-us/library/5h2w73d1%28v=vs.90%29.aspx -FORCE_INLINE void _mm_sfence(void) +// Perform a carry-less multiplication of two 64-bit integers, selected from a +// and b according to imm8, and store the results in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_clmulepi64_si128 +FORCE_INLINE __m128i _mm_clmulepi64_si128(__m128i _a, __m128i _b, const int imm) { - __sync_synchronize(); + uint64x2_t a = vreinterpretq_u64_m128i(_a); + uint64x2_t b = vreinterpretq_u64_m128i(_b); + switch (imm & 0x11) { + case 0x00: + return vreinterpretq_m128i_u64( + _sse2neon_vmull_p64(vget_low_u64(a), vget_low_u64(b))); + case 0x01: + return vreinterpretq_m128i_u64( + _sse2neon_vmull_p64(vget_high_u64(a), vget_low_u64(b))); + case 0x10: + return vreinterpretq_m128i_u64( + _sse2neon_vmull_p64(vget_low_u64(a), vget_high_u64(b))); + case 0x11: + return vreinterpretq_m128i_u64( + _sse2neon_vmull_p64(vget_high_u64(a), vget_high_u64(b))); + default: + abort(); + } } -// Store 128-bits (composed of 4 packed single-precision (32-bit) floating- -// point elements) from a into memory using a non-temporal memory hint. -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_stream_ps -FORCE_INLINE void _mm_stream_ps(float *p, __m128 a) +FORCE_INLINE unsigned int _sse2neon_mm_get_denormals_zero_mode(void) { -#if __has_builtin(__builtin_nontemporal_store) - __builtin_nontemporal_store(a, (float32x4_t *) p); + union { + fpcr_bitfield field; +#if defined(__aarch64__) || defined(_M_ARM64) + uint64_t value; #else - vst1q_f32(p, vreinterpretq_f32_m128(a)); + uint32_t value; #endif -} + } r; -// Stores the data in a to the address p without polluting the caches. If the -// cache line containing address p is already in the cache, the cache will be -// updated. -// https://msdn.microsoft.com/en-us/library/ba08y07y%28v=vs.90%29.aspx -FORCE_INLINE void _mm_stream_si128(__m128i *p, __m128i a) -{ -#if __has_builtin(__builtin_nontemporal_store) - __builtin_nontemporal_store(a, p); +#if defined(__aarch64__) || defined(_M_ARM64) + r.value = _sse2neon_get_fpcr(); #else - vst1q_s64((int64_t *) p, vreinterpretq_s64_m128i(a)); + __asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ #endif + + return r.field.bit24 ? _MM_DENORMALS_ZERO_ON : _MM_DENORMALS_ZERO_OFF; } -// Load 128-bits of integer data from memory into dst using a non-temporal -// memory hint. mem_addr must be aligned on a 16-byte boundary or a -// general-protection exception may be generated. -// -// dst[127:0] := MEM[mem_addr+127:mem_addr] -// -// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_stream_load_si128 -FORCE_INLINE __m128i _mm_stream_load_si128(__m128i *p) +// Count the number of bits set to 1 in unsigned 32-bit integer a, and +// return that count in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_popcnt_u32 +FORCE_INLINE int _mm_popcnt_u32(unsigned int a) { -#if __has_builtin(__builtin_nontemporal_store) - return __builtin_nontemporal_load(p); +#if defined(__aarch64__) || defined(_M_ARM64) +#if __has_builtin(__builtin_popcount) + return __builtin_popcount(a); +#elif defined(_MSC_VER) + return _CountOneBits(a); #else - return vreinterpretq_m128i_s64(vld1q_s64((int64_t *) p)); + return (int) vaddlv_u8(vcnt_u8(vcreate_u8((uint64_t) a))); #endif -} +#else + uint32_t count = 0; + uint8x8_t input_val, count8x8_val; + uint16x4_t count16x4_val; + uint32x2_t count32x2_val; -// Cache line containing p is flushed and invalidated from all caches in the -// coherency domain. : -// https://msdn.microsoft.com/en-us/library/ba08y07y(v=vs.100).aspx -FORCE_INLINE void _mm_clflush(void const *p) -{ - (void) p; - // no corollary for Neon? -} + input_val = vld1_u8((uint8_t *) &a); + count8x8_val = vcnt_u8(input_val); + count16x4_val = vpaddl_u8(count8x8_val); + count32x2_val = vpaddl_u16(count16x4_val); -// Allocate aligned blocks of memory. -// https://software.intel.com/en-us/ -// cpp-compiler-developer-guide-and-reference-allocating-and-freeing-aligned-memory-blocks -FORCE_INLINE void *_mm_malloc(size_t size, size_t align) -{ - void *ptr; - if (align == 1) - return malloc(size); - if (align == 2 || (sizeof(void *) == 8 && align == 4)) - align = sizeof(void *); - if (!posix_memalign(&ptr, align, size)) - return ptr; - return NULL; + vst1_u32(&count, count32x2_val); + return count; +#endif } -FORCE_INLINE void _mm_free(void *addr) +// Count the number of bits set to 1 in unsigned 64-bit integer a, and +// return that count in dst. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_popcnt_u64 +FORCE_INLINE int64_t _mm_popcnt_u64(uint64_t a) { - free(addr); +#if defined(__aarch64__) || defined(_M_ARM64) +#if __has_builtin(__builtin_popcountll) + return __builtin_popcountll(a); +#elif defined(_MSC_VER) + return _CountOneBits64(a); +#else + return (int64_t) vaddlv_u8(vcnt_u8(vcreate_u8(a))); +#endif +#else + uint64_t count = 0; + uint8x8_t input_val, count8x8_val; + uint16x4_t count16x4_val; + uint32x2_t count32x2_val; + uint64x1_t count64x1_val; + + input_val = vld1_u8((uint8_t *) &a); + count8x8_val = vcnt_u8(input_val); + count16x4_val = vpaddl_u8(count8x8_val); + count32x2_val = vpaddl_u16(count16x4_val); + count64x1_val = vpaddl_u32(count32x2_val); + vst1_u64(&count, count64x1_val); + return count; +#endif } -// Starting with the initial value in crc, accumulates a CRC32 value for -// unsigned 8-bit integer v. -// https://msdn.microsoft.com/en-us/library/bb514036(v=vs.100) -FORCE_INLINE uint32_t _mm_crc32_u8(uint32_t crc, uint8_t v) +FORCE_INLINE void _sse2neon_mm_set_denormals_zero_mode(unsigned int flag) { -#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) - __asm__ __volatile__("crc32cb %w[c], %w[c], %w[v]\n\t" - : [c] "+r"(crc) - : [v] "r"(v)); + // AArch32 Advanced SIMD arithmetic always uses the Flush-to-zero setting, + // regardless of the value of the FZ bit. + union { + fpcr_bitfield field; +#if defined(__aarch64__) || defined(_M_ARM64) + uint64_t value; #else - crc ^= v; - for (int bit = 0; bit < 8; bit++) { - if (crc & 1) - crc = (crc >> 1) ^ UINT32_C(0x82f63b78); - else - crc = (crc >> 1); - } + uint32_t value; #endif - return crc; -} + } r; -// Starting with the initial value in crc, accumulates a CRC32 value for -// unsigned 16-bit integer v. -// https://msdn.microsoft.com/en-us/library/bb531411(v=vs.100) -FORCE_INLINE uint32_t _mm_crc32_u16(uint32_t crc, uint16_t v) -{ -#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) - __asm__ __volatile__("crc32ch %w[c], %w[c], %w[v]\n\t" - : [c] "+r"(crc) - : [v] "r"(v)); +#if defined(__aarch64__) || defined(_M_ARM64) + r.value = _sse2neon_get_fpcr(); #else - crc = _mm_crc32_u8(crc, v & 0xff); - crc = _mm_crc32_u8(crc, (v >> 8) & 0xff); + __asm__ __volatile__("vmrs %0, FPSCR" : "=r"(r.value)); /* read */ #endif - return crc; -} -// Starting with the initial value in crc, accumulates a CRC32 value for -// unsigned 32-bit integer v. -// https://msdn.microsoft.com/en-us/library/bb531394(v=vs.100) -FORCE_INLINE uint32_t _mm_crc32_u32(uint32_t crc, uint32_t v) -{ -#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) - __asm__ __volatile__("crc32cw %w[c], %w[c], %w[v]\n\t" - : [c] "+r"(crc) - : [v] "r"(v)); + r.field.bit24 = (flag & _MM_DENORMALS_ZERO_MASK) == _MM_DENORMALS_ZERO_ON; + +#if defined(__aarch64__) || defined(_M_ARM64) + _sse2neon_set_fpcr(r.value); #else - crc = _mm_crc32_u16(crc, v & 0xffff); - crc = _mm_crc32_u16(crc, (v >> 16) & 0xffff); + __asm__ __volatile__("vmsr FPSCR, %0" ::"r"(r)); /* write */ #endif - return crc; } -// Starting with the initial value in crc, accumulates a CRC32 value for -// unsigned 64-bit integer v. -// https://msdn.microsoft.com/en-us/library/bb514033(v=vs.100) -FORCE_INLINE uint64_t _mm_crc32_u64(uint64_t crc, uint64_t v) +// Return the current 64-bit value of the processor's time-stamp counter. +// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=rdtsc +FORCE_INLINE uint64_t _rdtsc(void) { -#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) - __asm__ __volatile__("crc32cx %w[c], %w[c], %x[v]\n\t" - : [c] "+r"(crc) - : [v] "r"(v)); +#if defined(__aarch64__) || defined(_M_ARM64) + uint64_t val; + + /* According to ARM DDI 0487F.c, from Armv8.0 to Armv8.5 inclusive, the + * system counter is at least 56 bits wide; from Armv8.6, the counter + * must be 64 bits wide. So the system counter could be less than 64 + * bits wide and it is attributed with the flag 'cap_user_time_short' + * is true. + */ +#if defined(_MSC_VER) + val = _ReadStatusReg(ARM64_SYSREG(3, 3, 14, 0, 2)); #else - crc = _mm_crc32_u32((uint32_t)(crc), v & 0xffffffff); - crc = _mm_crc32_u32((uint32_t)(crc), (v >> 32) & 0xffffffff); + __asm__ __volatile__("mrs %0, cntvct_el0" : "=r"(val)); +#endif + + return val; +#else + uint32_t pmccntr, pmuseren, pmcntenset; + // Read the user mode Performance Monitoring Unit (PMU) + // User Enable Register (PMUSERENR) access permissions. + __asm__ __volatile__("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren)); + if (pmuseren & 1) { // Allows reading PMUSERENR for user mode code. + __asm__ __volatile__("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset)); + if (pmcntenset & 0x80000000UL) { // Is it counting? + __asm__ __volatile__("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr)); + // The counter is set up to count every 64th cycle + return (uint64_t) (pmccntr) << 6; + } + } + + // Fallback to syscall as we can't enable PMUSERENR in user mode. + struct timeval tv; + gettimeofday(&tv, NULL); + return (uint64_t) (tv.tv_sec) * 1000000 + tv.tv_usec; #endif - return crc; } #if defined(__GNUC__) || defined(__clang__) @@ -6295,7 +9217,7 @@ FORCE_INLINE uint64_t _mm_crc32_u64(uint64_t crc, uint64_t v) #pragma pop_macro("FORCE_INLINE") #endif -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(__clang__) #pragma GCC pop_options #endif From 4b77564031401c6b9fc58f12c9848143627c07bc Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Wed, 11 Oct 2023 11:02:23 +0300 Subject: [PATCH 025/421] Replace __ARM64_ARCH_8__ with __aarch64__ It is supported by both Mac M1/M2 and Linux ARM64 Signed-off-by: Martin Tzvetanov Grigorov --- libhmsbeagle/CPU/BeagleCPUPlugin.cpp | 2 +- libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUPlugin.cpp b/libhmsbeagle/CPU/BeagleCPUPlugin.cpp index 03738e79..7b69eecf 100644 --- a/libhmsbeagle/CPU/BeagleCPUPlugin.cpp +++ b/libhmsbeagle/CPU/BeagleCPUPlugin.cpp @@ -17,7 +17,7 @@ BeagleCPUPlugin::BeagleCPUPlugin() : Plugin("CPU", "CPU") { BeagleResource resource; -#ifdef __ARM64_ARCH_8__ +#ifdef __aarch64__ resource.name = (char*) "CPU (arm64)"; #else resource.name = (char*) "CPU (x86_64)"; diff --git a/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp b/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp index ad3e0008..c8d5aaab 100644 --- a/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp +++ b/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp @@ -24,7 +24,7 @@ BeagleCPUSSEPlugin::BeagleCPUSSEPlugin() : Plugin("CPU-SSE", "CPU-SSE") { BeagleResource resource; -#ifdef __ARM64_ARCH_8__ +#ifdef __aarch64__ resource.name = (char*) "CPU (arm64)"; #else resource.name = (char*) "CPU (x86_64)"; @@ -160,8 +160,8 @@ bool check_sse2() return result[3] & 0x04000000; #endif // HAVE_CPUID.H } -#else // For Mac OS X GNU C -#if defined(__ARM64_ARCH_8__) +#else +#if defined(__aarch64__) bool check_sse2() { return 1; } #else bool check_sse2(){ From bf35ad4a28e5f03edaa10dfd77c1b54eee0d9595 Mon Sep 17 00:00:00 2001 From: "Marc A. Suchard" Date: Fri, 13 Oct 2023 05:51:53 -0700 Subject: [PATCH 026/421] bump patch number re: linux-arm64 fix --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21110afe..b8e37830 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ project(BEAGLE) set(BEAGLE_PACKAGE_NAME "libhmsbeagle") set(BEAGLE_VERSION_MAJOR "4") set(BEAGLE_VERSION_MINOR "0") -set(BEAGLE_VERSION_PATCH "0") +set(BEAGLE_VERSION_PATCH "1") set(GENERIC_API_VERSION "1") From e1113699a43f85feaa8f613c0f3742f28821ad2c Mon Sep 17 00:00:00 2001 From: Marc Suchard Date: Fri, 13 Oct 2023 15:46:35 -0700 Subject: [PATCH 027/421] remove macos deprecation via sprintf warning --- libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp b/libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp index 2b68493a..c60de3f1 100644 --- a/libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp +++ b/libhmsbeagle/GPU/GPUInterfaceOpenCL.cpp @@ -1039,7 +1039,7 @@ void GPUInterface::GetDeviceName(int deviceNumber, sizeof(cl_uint), &mpCount, NULL)); char mpCountStr[12]; - sprintf(mpCountStr, "%d", mpCount); + snprintf(mpCountStr, 12, "%d", mpCount); strcat(deviceName, " ("); strcat(deviceName, mpCountStr); (mpCount==1?strcat(deviceName, " compute unit)"):strcat(deviceName, " compute units)")); @@ -1086,7 +1086,7 @@ void GPUInterface::GetDeviceDescription(int deviceNumber, SAFE_CL(clGetDeviceInfo(tmpOpenClDevice, CL_DEVICE_MAX_COMPUTE_UNITS, sizeof(unsigned int), &mpCount, NULL)); - sprintf(deviceDescription, + snprintf(deviceDescription, 128, "Global memory (MB): %d | Clock speed (Ghz): %1.2f | Number of compute units: %d", int(totalGlobalMemory / 1024.0 / 1024.0), clockSpeed / 1000.0, mpCount); From 25ffe1d081b67212d6054b0ede6dd8d78f83663b Mon Sep 17 00:00:00 2001 From: Marc Suchard Date: Mon, 11 Dec 2023 10:01:26 -0800 Subject: [PATCH 028/421] function signatures --- libhmsbeagle/BeagleImpl.h | 10 ++++++++ libhmsbeagle/CPU/BeagleCPUImpl.h | 8 ++++++ libhmsbeagle/CPU/BeagleCPUImpl.hpp | 22 +++++++++++++++++ libhmsbeagle/GPU/BeagleGPUImpl.h | 10 +++++++- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 39 ++++++++++++++++++++++++++++++ libhmsbeagle/beagle.cpp | 35 +++++++++++++++++++++++++++ libhmsbeagle/beagle.h | 27 ++++++++++++++++++++- 7 files changed, 149 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/BeagleImpl.h b/libhmsbeagle/BeagleImpl.h index ab7a1071..76c54f94 100644 --- a/libhmsbeagle/BeagleImpl.h +++ b/libhmsbeagle/BeagleImpl.h @@ -263,6 +263,16 @@ class BeagleImpl virtual int getSiteDerivatives(double* outFirstDerivatives, double* outSecondDerivatives) = 0; + + + virtual int updateBastaPartials(const int* operations, + int operationCount) = 0; + + virtual int accumulateBastaPartials(const int* operations, + int operationCount, + const int* segments, + int segmentCount) = 0; + //protected: int resourceNumber; }; diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index 8efb86b1..95b61878 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -444,6 +444,14 @@ class BeagleCPUImpl : public BeagleImpl { int getSiteDerivatives(double* outFirstDerivatives, double* outSecondDerivatives); + + int updateBastaPartials(const int* operations, + int operationCount); + + int accumulateBastaPartials(const int* operations, + int operationCount, + const int* segments, + int segmentCount); int block(void); diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 8ff9d0a5..54b6ffa9 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -1328,6 +1328,28 @@ int BeagleCPUImpl::updatePrePartials(const int *operations, return returnCode; } +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::updateBastaPartials(const int* operations, + int count) { + int returnCode = BEAGLE_ERROR_GENERAL; + + fprintf(stderr, "BeagleCPUImpl::updateBastaPartials\n"); + + return returnCode; +} + +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::accumulateBastaPartials(const int* operations, + int operationCount, + const int* segments, + int segmentCount) { + int returnCode = BEAGLE_ERROR_GENERAL; + + fprintf(stderr, "BeagleCPUImpl::accumulateBastaPartials\n"); + + return returnCode; +} + //BEAGLE_CPU_TEMPLATE //int BeagleCPUImpl::calculateEdgeDerivative(const int *postBufferIndices, // const int *preBufferIndices, diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 16e32f08..c74fed9f 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -458,7 +458,15 @@ class BeagleGPUImpl : public BeagleImpl { int getSiteDerivatives(double* outFirstDerivatives, double* outSecondDerivatives); - + + int updateBastaPartials(const int* operations, + int operationCount); + + int accumulateBastaPartials(const int* operations, + int operationCount, + const int* segments, + int segmentCount); + private: char* getInstanceName(); diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 1cc10d80..90f54a5e 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2199,6 +2199,45 @@ int BeagleGPUImpl::updatePrePartials(const int *operations, return returnCode; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::updateBastaPartials(const int* operations, + int count) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); +#endif + + int returnCode = BEAGLE_ERROR_GENERAL; + + fprintf(stderr, "BeagleGPUImpl::updateBastaPartials\n"); + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUImpl::updateBastaPartials\n"); +#endif + + return returnCode; +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::accumulateBastaPartials(const int* operations, + int operationCount, + const int* segments, + int segmentCount) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); +#endif + + int returnCode = BEAGLE_ERROR_GENERAL; + + fprintf(stderr, "BeagleGPUImpl::accumulateBastaPartials\n"); + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); +#endif + + return returnCode; +} + + BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::calculateEdgeDerivative(const int *postBufferIndices, const int *preBufferIndices, diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 4e91d709..f1160054 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -1863,3 +1863,38 @@ int beagleCalculateEdgeDerivative(int instance, const int *postBufferIndices, co return BEAGLE_ERROR_NO_IMPLEMENTATION; } +int beagleUpdateBastaPartials(const int instance, + const BastaOperation* operations, + int operationCount) { + DEBUG_START_TIME(); + + beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) { + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + } + + int returnValue = beagleInstance->updateBastaPartials((const int*) operations, operationCount); + + DEBUG_END_TIME(); + return returnValue; +} + +int beagleAccumulateBastaPartials(const int instance, + const BastaOperation* operations, + int operationCount, + const int* segments, + int segmentCount) { + DEBUG_START_TIME(); + + beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) { + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + } + + int returnValue = beagleInstance->accumulateBastaPartials((const int*) operations, operationCount, + segments, segmentCount); + + DEBUG_END_TIME(); + return returnValue; + +} diff --git a/libhmsbeagle/beagle.h b/libhmsbeagle/beagle.h index d055d256..d49e79e9 100644 --- a/libhmsbeagle/beagle.h +++ b/libhmsbeagle/beagle.h @@ -1,7 +1,7 @@ /** * @file beagle.h * - * Copyright 2009-2013 Phylogenetic Likelihood Working Group + * Copyright 2009-2024 Phylogenetic Likelihood Working Group * * This file is part of BEAGLE. * @@ -1397,6 +1397,31 @@ BEAGLE_DLLEXPORT int beagleGetSiteLogLikelihoods(int instance, BEAGLE_DLLEXPORT int beagleGetSiteDerivatives(int instance, double* outFirstDerivatives, double* outSecondDerivatives); + +// START OF BASTA + +typedef struct { + int destinationPartials; /**< index of destination, or parent, partials buffer */ + int child1Partials; /**< index of first child partials buffer */ + int child1TransitionMatrix; /**< index of transition matrix of first partials child buffer */ + int child2Partials; /**< index of second child partials buffer */ + int child2TransitionMatrix; /**< index of transition matrix of second partials child buffer */ + int accumulation1Partials; + int accumulation2Partials2; + int intervalNumber; +} BastaOperation; + +BEAGLE_DLLEXPORT int beagleUpdateBastaPartials(const int instance, + const BastaOperation* operations, + int operationCount); + +BEAGLE_DLLEXPORT int beagleAccumulateBastaPartials(const int instance, + const BastaOperation* operations, + int operationCount, + const int* segments, + int segmentCount); + +// END OF BASTA /* using C calling conventions so that C programs can successfully link the beagle library * (closing brace) From 3f84717aa4121e8d3585a9008cceb1cc00aaef46 Mon Sep 17 00:00:00 2001 From: Marc Suchard Date: Mon, 11 Dec 2023 14:21:05 -0800 Subject: [PATCH 029/421] initial infrastructure for BEAGLE-based BASTA --- CMakeLists.txt | 1 + build.xml | 2 +- java/beagle/BeagleFactory.java | 2 +- java/beagle/BeagleJNIImpl.java | 2 +- libhmsbeagle/BeagleImpl.h | 3 +- libhmsbeagle/CPU/BeagleCPUImpl.h | 3 +- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 3 +- libhmsbeagle/GPU/BeagleGPUImpl.h | 3 +- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 3 +- libhmsbeagle/JNI/CMakeLists.txt | 22 ++++++++++ .../JNI/beagle_basta_BastaJNIWrapper.cpp | 41 +++++++++++++++++++ .../JNI/beagle_basta_BastaJNIWrapper.h | 29 +++++++++++++ libhmsbeagle/beagle.cpp | 5 ++- libhmsbeagle/beagle.h | 3 +- 14 files changed, 111 insertions(+), 11 deletions(-) create mode 100644 libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp create mode 100644 libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b8e37830..0dea8fd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ option(BUILD_OPENCL "Build beagle with OpenCL library" ON) option(BUILD_CUDA "Build beagle with CUDA library" ON) option(BUILD_JNI "Build beagle with JNI library" ON) option(BUILD_SSE "Build beagle with SSE library" ON) +option(BUILD_BASTA "Build beagle with mbasta library" OFF) # Old config.h settings diff --git a/build.xml b/build.xml index dd1540c5..c8c6dff1 100755 --- a/build.xml +++ b/build.xml @@ -34,7 +34,7 @@ - + diff --git a/java/beagle/BeagleFactory.java b/java/beagle/BeagleFactory.java index b44e4bc2..1cc733f6 100644 --- a/java/beagle/BeagleFactory.java +++ b/java/beagle/BeagleFactory.java @@ -162,7 +162,7 @@ public static Beagle loadBeagleInstance( ); } - private static BeagleJNIWrapper getBeagleJNIWrapper() { + protected static BeagleJNIWrapper getBeagleJNIWrapper() { if (BeagleJNIWrapper.INSTANCE == null) { try { BeagleJNIWrapper.loadBeagleLibrary(); diff --git a/java/beagle/BeagleJNIImpl.java b/java/beagle/BeagleJNIImpl.java index 7575fbac..c3637475 100644 --- a/java/beagle/BeagleJNIImpl.java +++ b/java/beagle/BeagleJNIImpl.java @@ -15,7 +15,7 @@ public class BeagleJNIImpl implements Beagle { - private int instance = -1; + protected int instance = -1; private InstanceDetails details = new InstanceDetails(); public BeagleJNIImpl(int tipCount, diff --git a/libhmsbeagle/BeagleImpl.h b/libhmsbeagle/BeagleImpl.h index 76c54f94..ab4b956e 100644 --- a/libhmsbeagle/BeagleImpl.h +++ b/libhmsbeagle/BeagleImpl.h @@ -266,7 +266,8 @@ class BeagleImpl virtual int updateBastaPartials(const int* operations, - int operationCount) = 0; + int operationCount, + int populationSizesIndex) = 0; virtual int accumulateBastaPartials(const int* operations, int operationCount, diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index 95b61878..e83c06b0 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -446,7 +446,8 @@ class BeagleCPUImpl : public BeagleImpl { double* outSecondDerivatives); int updateBastaPartials(const int* operations, - int operationCount); + int operationCount, + int populationSizesIndex); int accumulateBastaPartials(const int* operations, int operationCount, diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 54b6ffa9..5a450489 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -1330,7 +1330,8 @@ int BeagleCPUImpl::updatePrePartials(const int *operations, BEAGLE_CPU_TEMPLATE int BeagleCPUImpl::updateBastaPartials(const int* operations, - int count) { + int count, + int populationSizesIndex) { int returnCode = BEAGLE_ERROR_GENERAL; fprintf(stderr, "BeagleCPUImpl::updateBastaPartials\n"); diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index c74fed9f..58105301 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -460,7 +460,8 @@ class BeagleGPUImpl : public BeagleImpl { double* outSecondDerivatives); int updateBastaPartials(const int* operations, - int operationCount); + int operationCount, + int populationSizesIndex); int accumulateBastaPartials(const int* operations, int operationCount, diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 90f54a5e..6ffd9a96 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2201,7 +2201,8 @@ int BeagleGPUImpl::updatePrePartials(const int *operations, BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::updateBastaPartials(const int* operations, - int count) { + int count, + int populationSizesIndex) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif diff --git a/libhmsbeagle/JNI/CMakeLists.txt b/libhmsbeagle/JNI/CMakeLists.txt index 72578278..0b3bfdc9 100644 --- a/libhmsbeagle/JNI/CMakeLists.txt +++ b/libhmsbeagle/JNI/CMakeLists.txt @@ -19,3 +19,25 @@ endif(APPLE) install(TARGETS hmsbeagle-jni DESTINATION ${BEAGLE_INSTALL_DIR} ) + +if(BUILD_BASTA) + add_library(hmsbeagle-jni-basta SHARED + beagle_basta_BastaJNIWrapper.cpp + beagle_basta_BastaJNIWrapper.h + ) + + target_link_libraries(hmsbeagle-jni-basta hmsbeagle) + + if(APPLE) + set_target_properties(hmsbeagle-jni-basta PROPERTIES PREFIX "lib" SUFFIX ".jnilib") + elseif(WIN32) + set_target_properties(hmsbeagle-jni-basta PROPERTIES PREFIX "" SUFFIX "64.dll") + elseif(UNIX) + set_target_properties(hmsbeagle-jni-basta PROPERTIES PREFIX "lib" SUFFIX ".so") + endif(APPLE) + + install(TARGETS hmsbeagle-jni-basta + DESTINATION ${BEAGLE_INSTALL_DIR} + ) + +endif(BUILD_BASTA) diff --git a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp new file mode 100644 index 00000000..9bf449f0 --- /dev/null +++ b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp @@ -0,0 +1,41 @@ +#include +#include +#include +#include +#include + +#include "libhmsbeagle/beagle.h" +#include "libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h" + + +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: updateBastaPartials + * Signature: (I[III)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartials + (JNIEnv *env, jobject obj, jint instance, jintArray inOperations, + jint operationCount, jint populationSizesIndex) { + + jint *operations = env->GetIntArrayElements(inOperations, NULL); + + jint errCode = (jint)beagleUpdateBastaPartials(instance, + (BastaOperation*) operations, operationCount, populationSizesIndex); + + env->ReleaseIntArrayElements(inOperations, operations, JNI_ABORT); + + return errCode; + + } + +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: accumulateBastaPartials + * Signature: (I[II[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_accumulateBastaPartials + (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint) { + + return 0; + + } diff --git a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h new file mode 100644 index 00000000..a8e47b73 --- /dev/null +++ b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class beagle_basta_BastaJNIWrapper */ + +#ifndef _Included_beagle_basta_BastaJNIWrapper +#define _Included_beagle_basta_BastaJNIWrapper +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: updateBastaPartials + * Signature: (I[III)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartials + (JNIEnv *, jobject, jint, jintArray, jint, jint); + +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: accumulateBastaPartials + * Signature: (I[II[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_accumulateBastaPartials + (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index f1160054..844d1e7a 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -1865,7 +1865,8 @@ int beagleCalculateEdgeDerivative(int instance, const int *postBufferIndices, co int beagleUpdateBastaPartials(const int instance, const BastaOperation* operations, - int operationCount) { + int operationCount, + int populationSizesIndex) { DEBUG_START_TIME(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); @@ -1873,7 +1874,7 @@ int beagleUpdateBastaPartials(const int instance, return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; } - int returnValue = beagleInstance->updateBastaPartials((const int*) operations, operationCount); + int returnValue = beagleInstance->updateBastaPartials((const int*) operations, operationCount, populationSizesIndex); DEBUG_END_TIME(); return returnValue; diff --git a/libhmsbeagle/beagle.h b/libhmsbeagle/beagle.h index d49e79e9..c805e9f0 100644 --- a/libhmsbeagle/beagle.h +++ b/libhmsbeagle/beagle.h @@ -1413,7 +1413,8 @@ typedef struct { BEAGLE_DLLEXPORT int beagleUpdateBastaPartials(const int instance, const BastaOperation* operations, - int operationCount); + int operationCount, + int populationSizeIndex); BEAGLE_DLLEXPORT int beagleAccumulateBastaPartials(const int instance, const BastaOperation* operations, From 6bd8aa9b96e0fd7653072675cceb6f0c49703b9a Mon Sep 17 00:00:00 2001 From: "Marc A. Suchard" Date: Sun, 31 Dec 2023 19:16:12 -0800 Subject: [PATCH 030/421] first CPU version of BASTA operations --- java/beagle/BeagleFlag.java | 3 +- libhmsbeagle/BeagleImpl.h | 20 +- libhmsbeagle/CPU/BeagleCPUImpl.h | 34 +- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 297 +++++++++++++++++- libhmsbeagle/GPU/BeagleGPUImpl.h | 17 +- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 49 ++- .../JNI/beagle_basta_BastaJNIWrapper.cpp | 72 ++++- .../JNI/beagle_basta_BastaJNIWrapper.h | 24 +- libhmsbeagle/beagle.cpp | 52 ++- libhmsbeagle/beagle.h | 26 +- 10 files changed, 548 insertions(+), 46 deletions(-) diff --git a/java/beagle/BeagleFlag.java b/java/beagle/BeagleFlag.java index bc3cd188..c9efe485 100644 --- a/java/beagle/BeagleFlag.java +++ b/java/beagle/BeagleFlag.java @@ -43,7 +43,8 @@ public enum BeagleFlag { PARALLELOPS_GRID(1 << 29, "Operations in updatePartials may be folded into single kernel launch (necessary for partitions; typically performs better for problems with fewer pattern sites)"), PREORDER_TRANSPOSE_MANUAL(1 << 30, "Pre-order transition matrices passed to BEAGLE have been transposed"), - PREORDER_TRANSPOSE_AUTO(1 << 31, "Automatically transpose pre-order transition matrices"); + PREORDER_TRANSPOSE_AUTO(1L << 31, "Automatically transpose pre-order transition matrices"), + PREORDER_TRANSPOSE_LOW_MEMORY(1L << 32, "Use low-memory automatic transposition of pre-order transition matrices"); BeagleFlag(long mask, String meaning) { this.mask = mask; diff --git a/libhmsbeagle/BeagleImpl.h b/libhmsbeagle/BeagleImpl.h index ab4b956e..d199710c 100644 --- a/libhmsbeagle/BeagleImpl.h +++ b/libhmsbeagle/BeagleImpl.h @@ -263,16 +263,28 @@ class BeagleImpl virtual int getSiteDerivatives(double* outFirstDerivatives, double* outSecondDerivatives) = 0; - - + virtual int updateBastaPartials(const int* operations, int operationCount, - int populationSizesIndex) = 0; + const int* intervals, + int intervalCount, + int populationSizesIndex, + int coalescentIndex) = 0; virtual int accumulateBastaPartials(const int* operations, int operationCount, const int* segments, - int segmentCount) = 0; + int segmentCount, + const double* intervalLengths, + const int populationSizesIndex, + int coalescentIndex, + double* out) = 0; + + virtual int allocateBastaBuffers(int bufferCount, + int bufferLength) = 0; + + virtual int getBastaBuffer(int bufferIndex, + double* out) = 0; //protected: int resourceNumber; diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index e83c06b0..7fc24400 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -133,6 +133,11 @@ class BeagleCPUImpl : public BeagleImpl { REALTYPE* ones; REALTYPE* zeros; + std::vector gBastaBuffers; + std::vector gCoalescentBuffers; + int kCoalescentBufferLength; + int kCoalescentBufferCount; + struct threadData { std::thread t; // The thread object @@ -447,12 +452,37 @@ class BeagleCPUImpl : public BeagleImpl { int updateBastaPartials(const int* operations, int operationCount, - int populationSizesIndex); + const int* intervals, + int intervalCount, + int populationSizesIndex, + int coalescentIndex); int accumulateBastaPartials(const int* operations, int operationCount, const int* segments, - int segmentCount); + int segmentCount, + const double* intervalLengths, + const int populationSizesIndex, + int coalescentIndex, + double* out); + + int allocateBastaBuffers(int bufferCount, + int bufferLength); + + int getBastaBuffer(int bufferIndex, + double* out); + + void reduceWithinInterval(REALTYPE* e, REALTYPE* f, // TODO move to protected + REALTYPE* g, REALTYPE* h, + int startBuffer1, int startBuffer2, + int endBuffer1, int endBuffer2, + int interval); + + REALTYPE reduceAcrossIntervals(REALTYPE* e, REALTYPE* f, // TODO move to protected + REALTYPE* g, REALTYPE* h, + int interval, REALTYPE length, + const REALTYPE* sizes, + const REALTYPE* coalescent); int block(void); diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 5a450489..8d24a593 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -1328,26 +1328,297 @@ int BeagleCPUImpl::updatePrePartials(const int *operations, return returnCode; } +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::allocateBastaBuffers(int bufferCount, + int bufferLength) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleCPUImpl::allocateBastaBuffers\n"); +#endif + + kCoalescentBufferLength = bufferLength; + kCoalescentBufferCount = bufferCount; + + gCoalescentBuffers.resize(kCoalescentBufferCount * kCoalescentBufferLength); + gBastaBuffers.resize(kPartialsPaddedStateCount * kCoalescentBufferLength * 4); + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleCPUImpl::allocateBastaBuffers\n"); +#endif + + return BEAGLE_SUCCESS; +} + +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::getBastaBuffer(int bufferIndex, + double* out) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleCPUImpl::getBastaBuffer\n"); +#endif + + if (bufferIndex >= 0 && bufferIndex < kCoalescentBufferCount) { + beagleMemCpy(out, gCoalescentBuffers.data() + bufferIndex * kCoalescentBufferLength, + kCoalescentBufferLength); + } else if (bufferIndex >= kCoalescentBufferCount && bufferIndex < kCoalescentBufferCount + 4) { + bufferIndex -= kCoalescentBufferCount; + beagleMemCpy(out, gBastaBuffers.data() + bufferIndex * kPartialsPaddedStateCount, kStateCount); + } else if (bufferIndex == -1) { + for (int i = 0; i < 4; ++i) { + beagleMemCpy(out + i * kStateCount, + gBastaBuffers.data() + i * kPartialsPaddedStateCount, + kStateCount); + } + } + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleCPUImpl::getBastaBuffer\n"); +#endif + + return BEAGLE_SUCCESS; +} + BEAGLE_CPU_TEMPLATE int BeagleCPUImpl::updateBastaPartials(const int* operations, - int count, - int populationSizesIndex) { - int returnCode = BEAGLE_ERROR_GENERAL; - - fprintf(stderr, "BeagleCPUImpl::updateBastaPartials\n"); - + const int count, + const int* intervals, + const int intervalCount, + const int populationSizesIndex, + const int coalescentIndex) { + int returnCode = BEAGLE_SUCCESS; + + REALTYPE* coalescent = gCoalescentBuffers.data() + kCoalescentBufferLength * coalescentIndex; + std::fill(coalescent, coalescent + kCoalescentBufferLength, REALTYPE(0)); + + for (int op = 0; op < count; ++op) { + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int child1PartialIndex = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + + const int matrixIncr = kStateCount + T_PAD; + const int stateCountModFour = (kStateCount / 4) * 4; + + REALTYPE* destPartial = gPartials[destinationPartialIndex]; + + // child 1 + const REALTYPE* partial1 = gPartials[child1PartialIndex]; + const REALTYPE* matrix1 = gTransitionMatrices[child1TransMatIndex]; + + for (int i = 0; i < kStateCount; i++) { + const REALTYPE* matrices1Ptr = matrix1 + i * matrixIncr; + REALTYPE sum1A = 0.0; + REALTYPE sum1B = 0.0; + + int j = 0; + for (; j < stateCountModFour; j += 4) { + sum1A += matrices1Ptr[j + 0] * partial1[j + 0]; + sum1B += matrices1Ptr[j + 1] * partial1[j + 1]; + sum1A += matrices1Ptr[j + 2] * partial1[j + 2]; + sum1B += matrices1Ptr[j + 3] * partial1[j + 3]; + } + + for (; j < kStateCount; j++) { + sum1A += matrices1Ptr[j] * partial1[j]; + } + + destPartial[i] = sum1A + sum1B; + } + + if (child2PartialIndex >= 0) { + + // child 2 + const REALTYPE* partial2 = gPartials[child2PartialIndex]; + const REALTYPE* matrix2 = gTransitionMatrices[child2TransMatIndex]; + const REALTYPE* sizes = gStateFrequencies[populationSizesIndex]; + + REALTYPE* accumulation1 = gPartials[accumulation1PartialIndex]; + REALTYPE* accumulation2 = gPartials[accumulation2PartialIndex]; + + REALTYPE prob = REALTYPE(0); + + for (int i = 0; i < kStateCount; i++) { + const REALTYPE* matrices2Ptr = matrix2 + i * matrixIncr; + REALTYPE sum2A = 0.0; + REALTYPE sum2B = 0.0; + + int j = 0; + for (; j < stateCountModFour; j += 4) { + sum2A += matrices2Ptr[j + 0] * partial2[j + 0]; + sum2B += matrices2Ptr[j + 1] * partial2[j + 1]; + sum2A += matrices2Ptr[j + 2] * partial2[j + 2]; + sum2B += matrices2Ptr[j + 3] * partial2[j + 3]; + } + + for (; j < kStateCount; j++) { + sum2A += matrices2Ptr[j] * partial2[j]; + } + + REALTYPE child1 = destPartial[i]; + REALTYPE child2 = sum2A + sum2B; + REALTYPE parent = child1 * child2 / sizes[i]; + + accumulation1[i] = child1; + accumulation2[i] = child2; + destPartial[i] = parent; + + prob += parent; + } + + for (int i = 0; i < kStateCount; i++) { + destPartial[i] /= prob; + } + + coalescent[intervalNumber] = prob; + } + } + return returnCode; } +BEAGLE_CPU_TEMPLATE +REALTYPE BeagleCPUImpl::reduceAcrossIntervals( + REALTYPE* e, REALTYPE* f, + REALTYPE* g, REALTYPE* h, + int interval, REALTYPE length, + const REALTYPE* sizes, + const REALTYPE* coalescent) { + + e += interval * kPartialsPaddedStateCount; + f += interval * kPartialsPaddedStateCount; + g += interval * kPartialsPaddedStateCount; + h += interval * kPartialsPaddedStateCount; + + REALTYPE sum = REALTYPE(0); + for (int k = 0; k < kPartialsPaddedStateCount; ++k) { + sum += (e[k] * e[k] - f[k] + + g[k] * g[k] - h[k]) / sizes[k]; + } + + REALTYPE logL = -length * sum / 4; + + REALTYPE prob = coalescent[interval]; + if (prob != REALTYPE(0)) { + logL += log(prob); + } + + return logL; +} + +BEAGLE_CPU_TEMPLATE +void BeagleCPUImpl::reduceWithinInterval(REALTYPE* e, REALTYPE* f, + REALTYPE* g, REALTYPE* h, + int startBuffer1, int startBuffer2, + int endBuffer1, int endBuffer2, + int interval) { + + e += interval * kPartialsPaddedStateCount; + f += interval * kPartialsPaddedStateCount; + g += interval * kPartialsPaddedStateCount; + h += interval * kPartialsPaddedStateCount; + + const REALTYPE* startPartials = gPartials[startBuffer1]; + const REALTYPE* endPartials = gPartials[endBuffer1]; + + for (int i = 0; i < kPartialsPaddedStateCount; ++i) { + REALTYPE startP = startPartials[i]; + e[i] += startP; + f[i] += startP * startP; + + REALTYPE endP = endPartials[i]; + g[i] += endP; + h[i] += endP * endP; + } + + if (startBuffer2 >= 0) { + + const REALTYPE* startPartials = gPartials[startBuffer2]; + const REALTYPE* endPartials = gPartials[endBuffer2]; + + for (int i = 0; i < kPartialsPaddedStateCount; ++i) { + REALTYPE startP = startPartials[i]; + e[i] += startP; + f[i] += startP * startP; + + REALTYPE endP = endPartials[i]; + g[i] += endP; + h[i] += endP * endP; + } + } +} + BEAGLE_CPU_TEMPLATE int BeagleCPUImpl::accumulateBastaPartials(const int* operations, - int operationCount, - const int* segments, - int segmentCount) { - int returnCode = BEAGLE_ERROR_GENERAL; - - fprintf(stderr, "BeagleCPUImpl::accumulateBastaPartials\n"); - + const int operationCount, + const int* intervalStarts, + const int intervalStartsCount, + const double* intervalLengths, + const int populationSizesIndex, + const int coalescentIndex, + double* out) { + int returnCode = BEAGLE_SUCCESS; + + std::fill(gBastaBuffers.begin(), gBastaBuffers.end(), REALTYPE(0)); + + REALTYPE* e = gBastaBuffers.data() + 0 * kPartialsPaddedStateCount * kCoalescentBufferLength; + REALTYPE* f = gBastaBuffers.data() + 1 * kPartialsPaddedStateCount * kCoalescentBufferLength; + REALTYPE* g = gBastaBuffers.data() + 2 * kPartialsPaddedStateCount * kCoalescentBufferLength; + REALTYPE* h = gBastaBuffers.data() + 3 * kPartialsPaddedStateCount * kCoalescentBufferLength; + + const REALTYPE* coalescent = gCoalescentBuffers.data() + kCoalescentBufferLength * coalescentIndex; + const REALTYPE* sizes = gStateFrequencies[populationSizesIndex]; + + for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { // TODO execute in parallel (no race conditions) + const int start = intervalStarts[interval]; + const int end = intervalStarts[interval + 1]; + + for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int child1PartialIndex = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + + reduceWithinInterval(e, f, g, h, + child1PartialIndex, child2PartialIndex, + accumulation1PartialIndex, accumulation2PartialIndex, + intervalNumber); + } + } + + REALTYPE logL = 0.0; + for (int i = 0; i < intervalStartsCount - 1; ++i) { // TODO execute in parallel + const int op = intervalStarts[i]; + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int child1PartialIndex = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + + logL += reduceAcrossIntervals(e, f, g, h, + intervalNumber, intervalLengths[i], + sizes, coalescent); + } + + out[0] = (double) logL; + return returnCode; } diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index eb18432b..99943aa2 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -462,12 +462,25 @@ class BeagleGPUImpl : public BeagleImpl { int updateBastaPartials(const int* operations, int operationCount, - int populationSizesIndex); + const int* intervals, + int intervalCount, + int populationSizesIndex, + int coalescentIndex); int accumulateBastaPartials(const int* operations, int operationCount, const int* segments, - int segmentCount); + int segmentCount, + const double* intervalLengths, + const int populationSizesIndex, + int coalescentIndex, + double* out); + + int allocateBastaBuffers(int bufferCount, + int bufferLength); + + int getBastaBuffer(int bufferIndex, + double* out); private: diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index e2aab586..a4effbdc 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2199,10 +2199,47 @@ int BeagleGPUImpl::updatePrePartials(const int *operations, return returnCode; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, + int bufferLength) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUImpl::allocateBastaBuffers\n"); +#endif + + int returnCode = BEAGLE_ERROR_GENERAL; + + fprintf(stderr, "BeagleGPUImpl::allocateBastaBuffers\n"); + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUImpl::allocateBastaBuffers\n"); +#endif + + return returnCode; +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::getBastaBuffer(int bufferIndex, + double* out) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUImpl::getBastaBuffer\n"); +#endif + + fprintf(stderr, "BeagleGPUImpl::getBastaBuffer\n"); + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUImpl::getBastaBuffer\n"); +#endif + + return BEAGLE_SUCCESS; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::updateBastaPartials(const int* operations, - int count, - int populationSizesIndex) { + const int count, + const int* intervals, + const int intervalCount, + const int populationSizesIndex, + const int coalescentIndex) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif @@ -2221,8 +2258,12 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::accumulateBastaPartials(const int* operations, int operationCount, - const int* segments, - int segmentCount) { + const int* intervalStarts, + int intervalStartsCount, + const double* intervalLengths, + const int populationSizesIndex, + int coalescentIndex, + double* out) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); #endif diff --git a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp index 9bf449f0..87948285 100644 --- a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp +++ b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp @@ -8,6 +8,19 @@ #include "libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h" +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: allocateCoalescentBuffers + * Signature: (III)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_allocateCoalescentBuffers + (JNIEnv *env, jobject obj, jint instance, jint bufferCount, jint bufferLength) { + + jint errCode = (jint)beagleAllocateBastaBuffers(instance, bufferCount, bufferLength); + + return errCode; +} + /* * Class: beagle_basta_BastaJNIWrapper * Method: updateBastaPartials @@ -15,18 +28,39 @@ */ JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartials (JNIEnv *env, jobject obj, jint instance, jintArray inOperations, - jint operationCount, jint populationSizesIndex) { + jint operationCount, jintArray inIntervals, jint intervalCount, + jint populationSizesIndex, jint coalescentIndex) { - jint *operations = env->GetIntArrayElements(inOperations, NULL); + jint *operations = env->GetIntArrayElements(inOperations, NULL); + jint *intervals = env->GetIntArrayElements(inIntervals, NULL); jint errCode = (jint)beagleUpdateBastaPartials(instance, - (BastaOperation*) operations, operationCount, populationSizesIndex); + (BastaOperation*) operations, operationCount, + intervals, intervalCount, + populationSizesIndex, coalescentIndex); env->ReleaseIntArrayElements(inOperations, operations, JNI_ABORT); + env->ReleaseIntArrayElements(inIntervals, intervals, JNI_ABORT); return errCode; } + +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: getBastaBuffer + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_getBastaBuffer + (JNIEnv *env, jobject object, jint instance, jint bufferIndex, jdoubleArray out) { + + jdouble *array = env->GetDoubleArrayElements(out, NULL); + jint errCode = beagleGetBastaBuffer(instance, bufferIndex, (double *)array); + + // not using JNI_ABORT flag here because we want the values to be copied back... + env->ReleaseDoubleArrayElements(out, array, 0); + return errCode; +} /* * Class: beagle_basta_BastaJNIWrapper @@ -34,8 +68,34 @@ JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartials * Signature: (I[II[II)I */ JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_accumulateBastaPartials - (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint) { - - return 0; + (JNIEnv *env, jobject object, jint instance, + jintArray inOperations, jint operationCount, + jintArray inIntervals, jint intervalCount, + jdoubleArray inIntervalLengths, + jint populationSizesIndex, + jint coalescentIndex, + jdoubleArray out) { + + jint *operations = env->GetIntArrayElements(inOperations, NULL); + jint *intervals = env->GetIntArrayElements(inIntervals, NULL); + + jdouble *array = env->GetDoubleArrayElements(out, NULL); + jdouble *intervalLengths = env->GetDoubleArrayElements(inIntervalLengths, NULL); + + jint errCode = beagleAccumulateBastaPartials(instance, + (const BastaOperation*) operations, operationCount, + intervals, intervalCount, + (double *)intervalLengths, + populationSizesIndex, + coalescentIndex, (double *)array); + + env->ReleaseIntArrayElements(inOperations, operations, JNI_ABORT); + env->ReleaseIntArrayElements(inIntervals, intervals, JNI_ABORT); + env->ReleaseDoubleArrayElements(inIntervalLengths, intervalLengths, JNI_ABORT); + + // not using JNI_ABORT flag here because we want the values to be copied back... + env->ReleaseDoubleArrayElements(out, array, 0); + + return errCode; } diff --git a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h index a8e47b73..3e765f98 100644 --- a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h +++ b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h @@ -7,21 +7,37 @@ #ifdef __cplusplus extern "C" { #endif +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: allocateCoalescentBuffers + * Signature: (III)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_allocateCoalescentBuffers + (JNIEnv *, jobject, jint, jint, jint); + +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: getBastaBuffer + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_getBastaBuffer + (JNIEnv *, jobject, jint, jint, jdoubleArray); + /* * Class: beagle_basta_BastaJNIWrapper * Method: updateBastaPartials - * Signature: (I[III)I + * Signature: (I[II[IIII)I */ JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartials - (JNIEnv *, jobject, jint, jintArray, jint, jint); + (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint, jint, jint); /* * Class: beagle_basta_BastaJNIWrapper * Method: accumulateBastaPartials - * Signature: (I[II[II)I + * Signature: (I[II[III[D)I */ JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_accumulateBastaPartials - (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint, jdoubleArray, jint, jint, jdoubleArray); #ifdef __cplusplus } diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 4b5c8734..a98c11e0 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -1942,10 +1942,45 @@ int beagleCalculateEdgeDerivative(int instance, const int *postBufferIndices, co return BEAGLE_ERROR_NO_IMPLEMENTATION; } +int beagleAllocateBastaBuffers(const int instance, + const int bufferCount, + const int bufferLength) { + DEBUG_START_TIME(); + + beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) { + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + } + + int returnValue = beagleInstance->allocateBastaBuffers(bufferCount, bufferLength); + + DEBUG_END_TIME(); + return returnValue; +} + + +int beagleGetBastaBuffer(const int instance, + const int bufferIndex, + double* out) { + DEBUG_START_TIME(); + + beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) { + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + } + + int returnValue = beagleInstance->getBastaBuffer(bufferIndex, out); + DEBUG_END_TIME(); + return returnValue; +} + int beagleUpdateBastaPartials(const int instance, const BastaOperation* operations, int operationCount, - int populationSizesIndex) { + const int* intervals, + int intervalCount, + int populationSizesIndex, + int coalescentIndex) { DEBUG_START_TIME(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); @@ -1953,7 +1988,8 @@ int beagleUpdateBastaPartials(const int instance, return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; } - int returnValue = beagleInstance->updateBastaPartials((const int*) operations, operationCount, populationSizesIndex); + int returnValue = beagleInstance->updateBastaPartials((const int*) operations, + operationCount, intervals, intervalCount, populationSizesIndex, coalescentIndex); DEBUG_END_TIME(); return returnValue; @@ -1962,8 +1998,12 @@ int beagleUpdateBastaPartials(const int instance, int beagleAccumulateBastaPartials(const int instance, const BastaOperation* operations, int operationCount, - const int* segments, - int segmentCount) { + const int* intervalStarts, + int intervalCount, + const double* intervalLengths, + const int populationSizesIndex, + int coalescentIndex, + double* out) { DEBUG_START_TIME(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); @@ -1972,7 +2012,9 @@ int beagleAccumulateBastaPartials(const int instance, } int returnValue = beagleInstance->accumulateBastaPartials((const int*) operations, operationCount, - segments, segmentCount); + intervalStarts, intervalCount, intervalLengths, + populationSizesIndex, coalescentIndex, + out); DEBUG_END_TIME(); return returnValue; diff --git a/libhmsbeagle/beagle.h b/libhmsbeagle/beagle.h index c805e9f0..3e46e03c 100644 --- a/libhmsbeagle/beagle.h +++ b/libhmsbeagle/beagle.h @@ -201,6 +201,7 @@ enum BeagleBenchmarkFlags { enum BeagleOpCodes { BEAGLE_OP_COUNT = 7, /**< Total number of integers per beagleUpdatePartials operation */ BEAGLE_PARTITION_OP_COUNT = 9, /**< Total number of integers per beagleUpdatePartialsByPartition operation */ + BEAGLE_BASTA_OP_COUNT = 8, BEAGLE_OP_NONE = -1 /**< Specify no use for indexed buffer */ }; @@ -1414,13 +1415,28 @@ typedef struct { BEAGLE_DLLEXPORT int beagleUpdateBastaPartials(const int instance, const BastaOperation* operations, int operationCount, - int populationSizeIndex); + const int* intervalStarts, + int intervalCount, + int populationSizeIndex, + int coalescentIndex); BEAGLE_DLLEXPORT int beagleAccumulateBastaPartials(const int instance, - const BastaOperation* operations, - int operationCount, - const int* segments, - int segmentCount); + const BastaOperation* operations, + int operationCount, + const int* intervalStarts, + int intervalCount, + const double* intervalLengths, + const int populationSizesIndex, + int coalescentIndex, + double* out); + +BEAGLE_DLLEXPORT int beagleAllocateBastaBuffers(const int instance, + const int bufferCount, + const int bufferLength); + +BEAGLE_DLLEXPORT int beagleGetBastaBuffer(const int instance, + const int bufferIndex, + double* out); // END OF BASTA From 3763c0136a9d933860d02b632828919b5eb41efd Mon Sep 17 00:00:00 2001 From: "Marc A. Suchard" Date: Mon, 1 Jan 2024 06:33:20 -0800 Subject: [PATCH 031/421] macros for energy benchmarking --- libhmsbeagle/beagle.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index a98c11e0..590ab6ca 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -1938,7 +1938,7 @@ int beagleCalculateEdgeDerivative(int instance, const int *postBufferIndices, co const int categoryWeightsIndex, const int categoryRatesIndex, const int stateFrequenciesIndex, const int *cumulativeScaleIndices, int count, double *outFirstDerivative, double *outDiagonalSecondDerivative) { - fprintf(stderr, "Depricated"); + fprintf(stderr, "Function beagleCalculateEdgeDerivative is deprecated.\n"); return BEAGLE_ERROR_NO_IMPLEMENTATION; } @@ -1946,6 +1946,7 @@ int beagleAllocateBastaBuffers(const int instance, const int bufferCount, const int bufferLength) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -1955,6 +1956,8 @@ int beagleAllocateBastaBuffers(const int instance, int returnValue = beagleInstance->allocateBastaBuffers(bufferCount, bufferLength); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); + return returnValue; } @@ -1963,6 +1966,7 @@ int beagleGetBastaBuffer(const int instance, const int bufferIndex, double* out) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -1971,6 +1975,8 @@ int beagleGetBastaBuffer(const int instance, int returnValue = beagleInstance->getBastaBuffer(bufferIndex, out); DEBUG_END_TIME(); + DEBUG_END_ENERGY(); + return returnValue; } @@ -1982,6 +1988,7 @@ int beagleUpdateBastaPartials(const int instance, int populationSizesIndex, int coalescentIndex) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -1991,7 +1998,9 @@ int beagleUpdateBastaPartials(const int instance, int returnValue = beagleInstance->updateBastaPartials((const int*) operations, operationCount, intervals, intervalCount, populationSizesIndex, coalescentIndex); - DEBUG_END_TIME(); + DEBUG_END_TIME(); + DEBUG_END_ENERGY(); + return returnValue; } @@ -2005,6 +2014,7 @@ int beagleAccumulateBastaPartials(const int instance, int coalescentIndex, double* out) { DEBUG_START_TIME(); + DEBUG_START_ENERGY(); beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); if (beagleInstance == NULL) { @@ -2016,7 +2026,9 @@ int beagleAccumulateBastaPartials(const int instance, populationSizesIndex, coalescentIndex, out); - DEBUG_END_TIME(); + DEBUG_END_TIME(); + DEBUG_END_ENERGY(); + return returnValue; } From 9a8376f194c24e9c2b212c91b03f6d43fc7a2fac Mon Sep 17 00:00:00 2001 From: "Marc A. Suchard" Date: Fri, 5 Jan 2024 10:41:28 -0800 Subject: [PATCH 032/421] poor parallelization --- libhmsbeagle/CPU/BeagleCPUImpl.h | 19 +- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 353 +++++++++++++++++++++-------- libhmsbeagle/beagle.cpp | 1 + 3 files changed, 282 insertions(+), 91 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index 7fc24400..2e337562 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -34,6 +34,8 @@ #include #include +//#define BEAGLE_CACHE_FRIENDLY + #define BEAGLE_CPU_GENERIC REALTYPE, T_PAD, P_PAD #define BEAGLE_CPU_TEMPLATE template @@ -106,6 +108,10 @@ class BeagleCPUImpl : public BeagleImpl { int** gTipStates; REALTYPE** gScaleBuffers; +#ifdef BEAGLE_CACHE_FRIENDLY + REALTYPE* gPartialsCache; +#endif + signed short** gAutoScaleBuffers; int* gActiveScalingFactors; @@ -199,7 +205,7 @@ class BeagleCPUImpl : public BeagleImpl { // set the pre-order partials for root node // - // bufferIndices the indices of root nodes (may countain multiple) + // bufferIndices the indices of root nodes (may contain multiple) // stateFrequenciesIndices the indices of state frequencies at root // count number of root nodes int setRootPrePartials(const int* bufferIndices, @@ -805,6 +811,17 @@ class BeagleCPUImpl : public BeagleImpl { void threadWaiting(threadData* tData); + void updateInnerBastaPartials(const int* operations, // TODO make protected + const int begin, + const int end, + const REALTYPE* sizes, + REALTYPE* coalescent); + + void updateInnerBastaPartials2(const int* operations, // TODO make protected + const int op, + const REALTYPE* sizes, + REALTYPE* coalescent); + private: template diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 8d24a593..024f27a3 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -134,11 +134,16 @@ BeagleCPUImpl::~BeagleCPUImpl() { free(gTransitionMatrices); for(unsigned int i=0; i::createInstance(int tipCount, gTipStates[i] = NULL; } +#ifdef BEAGLE_CACHE_FRIENDLY + gPartialsCache = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPartialsSize * (kBufferCount - kTipCount)); +#endif + for (int i = kTipCount; i < kBufferCount; i++) { +#ifdef BEAGLE_CACHE_FRIENDLY + gPartials[i] = gPartialsCache + (i - kTipCount) * kPartialsSize; +#else gPartials[i] = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPartialsSize); +#endif if (gPartials[i] == NULL) throw std::bad_alloc(); } @@ -1376,6 +1389,228 @@ int BeagleCPUImpl::getBastaBuffer(int bufferIndex, return BEAGLE_SUCCESS; } +BEAGLE_CPU_TEMPLATE +void BeagleCPUImpl::updateInnerBastaPartials(const int* operations, + const int begin, + const int end, + const REALTYPE* sizes, + REALTYPE* coalescent) { + for (int op = begin; op < end; ++op) { + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int child1PartialIndex = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + + const int matrixIncr = kStateCount + T_PAD; + const int stateCountModFour = (kStateCount / 4) * 4; + + REALTYPE *destPartial = gPartials[destinationPartialIndex]; + + // child 1 + const REALTYPE *partial1 = gPartials[child1PartialIndex]; + const REALTYPE *matrix1 = gTransitionMatrices[child1TransMatIndex]; + + for (int i = 0; i < kStateCount; i++) { + const REALTYPE *matrices1Ptr = matrix1 + i * matrixIncr; + REALTYPE sum1A = 0.0; + REALTYPE sum1B = 0.0; + + int j = 0; + for (; j < stateCountModFour; j += 4) { + sum1A += matrices1Ptr[j + 0] * partial1[j + 0]; + sum1B += matrices1Ptr[j + 1] * partial1[j + 1]; + sum1A += matrices1Ptr[j + 2] * partial1[j + 2]; + sum1B += matrices1Ptr[j + 3] * partial1[j + 3]; + } + + for (; j < kStateCount; j++) { + sum1A += matrices1Ptr[j] * partial1[j]; + } + + destPartial[i] = sum1A + sum1B; + } + + if (child2PartialIndex >= 0) { + + // child 2 + const REALTYPE *partial2 = gPartials[child2PartialIndex]; + const REALTYPE *matrix2 = gTransitionMatrices[child2TransMatIndex]; + + REALTYPE *accumulation1 = gPartials[accumulation1PartialIndex]; + REALTYPE *accumulation2 = gPartials[accumulation2PartialIndex]; + + REALTYPE prob = REALTYPE(0); + + for (int i = 0; i < kStateCount; i++) { + const REALTYPE *matrices2Ptr = matrix2 + i * matrixIncr; + REALTYPE sum2A = 0.0; + REALTYPE sum2B = 0.0; + + int j = 0; + for (; j < stateCountModFour; j += 4) { + sum2A += matrices2Ptr[j + 0] * partial2[j + 0]; + sum2B += matrices2Ptr[j + 1] * partial2[j + 1]; + sum2A += matrices2Ptr[j + 2] * partial2[j + 2]; + sum2B += matrices2Ptr[j + 3] * partial2[j + 3]; + } + + for (; j < kStateCount; j++) { + sum2A += matrices2Ptr[j] * partial2[j]; + } + + REALTYPE child1 = destPartial[i]; + REALTYPE child2 = sum2A + sum2B; + REALTYPE parent = child1 * child2 / sizes[i]; + + accumulation1[i] = child1; + accumulation2[i] = child2; + destPartial[i] = parent; + + prob += parent; + } + + for (int i = 0; i < kStateCount; i++) { + destPartial[i] /= prob; + } + + coalescent[intervalNumber] = prob; + } + } +} + +BEAGLE_CPU_TEMPLATE +void BeagleCPUImpl::updateInnerBastaPartials2(const int* operations, + const int op, + const REALTYPE* sizes, + REALTYPE* coalescent) { + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int child1PartialIndex = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + + const int matrixIncr = kStateCount + T_PAD; + const int stateCountModFour = (kStateCount / 4) * 4; + + REALTYPE *destPartial = gPartials[destinationPartialIndex]; + + // child 1 + const REALTYPE *partial1 = gPartials[child1PartialIndex]; + const REALTYPE *matrix1 = gTransitionMatrices[child1TransMatIndex]; + + for (int i = 0; i < kStateCount; i++) { + const REALTYPE *matrices1Ptr = matrix1 + i * matrixIncr; + REALTYPE sum1A = 0.0; + REALTYPE sum1B = 0.0; + + int j = 0; + for (; j < stateCountModFour; j += 4) { + sum1A += matrices1Ptr[j + 0] * partial1[j + 0]; + sum1B += matrices1Ptr[j + 1] * partial1[j + 1]; + sum1A += matrices1Ptr[j + 2] * partial1[j + 2]; + sum1B += matrices1Ptr[j + 3] * partial1[j + 3]; + } + + for (; j < kStateCount; j++) { + sum1A += matrices1Ptr[j] * partial1[j]; + } + + destPartial[i] = sum1A + sum1B; + } + + if (child2PartialIndex >= 0) { + + // child 2 + const REALTYPE *partial2 = gPartials[child2PartialIndex]; + const REALTYPE *matrix2 = gTransitionMatrices[child2TransMatIndex]; + + REALTYPE *accumulation1 = gPartials[accumulation1PartialIndex]; + REALTYPE *accumulation2 = gPartials[accumulation2PartialIndex]; + + REALTYPE prob = REALTYPE(0); + + for (int i = 0; i < kStateCount; i++) { + const REALTYPE *matrices2Ptr = matrix2 + i * matrixIncr; + REALTYPE sum2A = 0.0; + REALTYPE sum2B = 0.0; + + int j = 0; + for (; j < stateCountModFour; j += 4) { + sum2A += matrices2Ptr[j + 0] * partial2[j + 0]; + sum2B += matrices2Ptr[j + 1] * partial2[j + 1]; + sum2A += matrices2Ptr[j + 2] * partial2[j + 2]; + sum2B += matrices2Ptr[j + 3] * partial2[j + 3]; + } + + for (; j < kStateCount; j++) { + sum2A += matrices2Ptr[j] * partial2[j]; + } + + REALTYPE child1 = destPartial[i]; + REALTYPE child2 = sum2A + sum2B; + REALTYPE parent = child1 * child2 / sizes[i]; + + accumulation1[i] = child1; + accumulation2[i] = child2; + destPartial[i] = parent; + + prob += parent; + } + + for (int i = 0; i < kStateCount; i++) { + destPartial[i] /= prob; + } + + coalescent[intervalNumber] = prob; + } +} + +template +inline void for_each(Integer begin, Integer end, Function function, int global_num_threads) { + std::vector threads; + threads.reserve(global_num_threads); + + // Calculate the chunk size for each thread + size_t chunk_size = (end - begin) / global_num_threads; + size_t remainder = (end - begin) % global_num_threads; + + // Perform the operation in parallel using threads + for (size_t i = 0; i < global_num_threads; ++i) { + Integer thread_begin = begin + i * chunk_size; + Integer thread_end = thread_begin + chunk_size; + if (i == global_num_threads - 1) { + // Include the remainder in the last thread + thread_end += remainder; + } + + threads.emplace_back([thread_begin, thread_end, &function] { +// threads.emplace_back([&] { + function(thread_begin, thread_end); +// for (int j = thread_begin; j < thread_end; ++j) { +// function(j); +// } + }); + } + + // Wait for all threads to finish + for (auto &thread : threads) { + thread.join(); + } +} + + BEAGLE_CPU_TEMPLATE int BeagleCPUImpl::updateBastaPartials(const int* operations, const int count, @@ -1387,97 +1622,35 @@ int BeagleCPUImpl::updateBastaPartials(const int* operations REALTYPE* coalescent = gCoalescentBuffers.data() + kCoalescentBufferLength * coalescentIndex; std::fill(coalescent, coalescent + kCoalescentBufferLength, REALTYPE(0)); - - for (int op = 0; op < count; ++op) { - - const int numOps = BEAGLE_BASTA_OP_COUNT; - - const int destinationPartialIndex = operations[op * numOps]; - const int child1PartialIndex = operations[op * numOps + 1]; - const int child1TransMatIndex = operations[op * numOps + 2]; - const int child2PartialIndex = operations[op * numOps + 3]; - const int child2TransMatIndex = operations[op * numOps + 4]; - const int accumulation1PartialIndex = operations[op * numOps + 5]; - const int accumulation2PartialIndex = operations[op * numOps + 6]; - const int intervalNumber = operations[op * numOps + 7]; - - const int matrixIncr = kStateCount + T_PAD; - const int stateCountModFour = (kStateCount / 4) * 4; - - REALTYPE* destPartial = gPartials[destinationPartialIndex]; - - // child 1 - const REALTYPE* partial1 = gPartials[child1PartialIndex]; - const REALTYPE* matrix1 = gTransitionMatrices[child1TransMatIndex]; - - for (int i = 0; i < kStateCount; i++) { - const REALTYPE* matrices1Ptr = matrix1 + i * matrixIncr; - REALTYPE sum1A = 0.0; - REALTYPE sum1B = 0.0; - - int j = 0; - for (; j < stateCountModFour; j += 4) { - sum1A += matrices1Ptr[j + 0] * partial1[j + 0]; - sum1B += matrices1Ptr[j + 1] * partial1[j + 1]; - sum1A += matrices1Ptr[j + 2] * partial1[j + 2]; - sum1B += matrices1Ptr[j + 3] * partial1[j + 3]; - } - - for (; j < kStateCount; j++) { - sum1A += matrices1Ptr[j] * partial1[j]; - } - - destPartial[i] = sum1A + sum1B; - } - - if (child2PartialIndex >= 0) { - - // child 2 - const REALTYPE* partial2 = gPartials[child2PartialIndex]; - const REALTYPE* matrix2 = gTransitionMatrices[child2TransMatIndex]; - const REALTYPE* sizes = gStateFrequencies[populationSizesIndex]; - - REALTYPE* accumulation1 = gPartials[accumulation1PartialIndex]; - REALTYPE* accumulation2 = gPartials[accumulation2PartialIndex]; - - REALTYPE prob = REALTYPE(0); - - for (int i = 0; i < kStateCount; i++) { - const REALTYPE* matrices2Ptr = matrix2 + i * matrixIncr; - REALTYPE sum2A = 0.0; - REALTYPE sum2B = 0.0; - - int j = 0; - for (; j < stateCountModFour; j += 4) { - sum2A += matrices2Ptr[j + 0] * partial2[j + 0]; - sum2B += matrices2Ptr[j + 1] * partial2[j + 1]; - sum2A += matrices2Ptr[j + 2] * partial2[j + 2]; - sum2B += matrices2Ptr[j + 3] * partial2[j + 3]; - } - - for (; j < kStateCount; j++) { - sum2A += matrices2Ptr[j] * partial2[j]; - } - - REALTYPE child1 = destPartial[i]; - REALTYPE child2 = sum2A + sum2B; - REALTYPE parent = child1 * child2 / sizes[i]; - - accumulation1[i] = child1; - accumulation2[i] = child2; - destPartial[i] = parent; - - prob += parent; - } - - for (int i = 0; i < kStateCount; i++) { - destPartial[i] /= prob; - } - coalescent[intervalNumber] = prob; - } - } - + const REALTYPE* sizes = gStateFrequencies[populationSizesIndex]; + + bool THREADING = true; + int CUT_POINT = 1280000; + int nThreads = 2; + + if (THREADING) { + + for (int i = 0; i < intervalCount - 1; ++i) { + + const int begin = intervals[i]; + const int end = intervals[i + 1]; + + if (end - begin > CUT_POINT) { + for_each(begin, end, [&](const int threadBegin, const int threadEnd) { + updateInnerBastaPartials(operations, threadBegin, threadEnd, sizes, coalescent); + }, nThreads); +// for_each(begin, end, [&](const int j) { +// updateInnerBastaPartials2(operations, j, sizes, coalescent); +// }, nThreads); + } else { + updateInnerBastaPartials(operations, begin, end, sizes, coalescent); + } + } + } else { + updateInnerBastaPartials(operations, 0, count, sizes, coalescent); + } + return returnCode; } diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 590ab6ca..e1adcc76 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -1974,6 +1974,7 @@ int beagleGetBastaBuffer(const int instance, } int returnValue = beagleInstance->getBastaBuffer(bufferIndex, out); + DEBUG_END_TIME(); DEBUG_END_ENERGY(); From 79f7f1a8c7b5a3b9740ebf01286828efd63b8fa3 Mon Sep 17 00:00:00 2001 From: gkarthik Date: Sat, 20 Apr 2024 15:58:37 -0500 Subject: [PATCH 033/421] Added skeleton for innerBastaPartials kernels --- CMakeLists.txt | 4 + libhmsbeagle/GPU/kernels/kernelsX.cu | 94 ++++++++++++++ tests/basta/CMakeLists.txt | 26 ++++ .../test_x_state_updateInnerBastaPartials.cu | 122 ++++++++++++++++++ 4 files changed, 246 insertions(+) create mode 100644 tests/basta/CMakeLists.txt create mode 100644 tests/basta/test_x_state_updateInnerBastaPartials.cu diff --git a/CMakeLists.txt b/CMakeLists.txt index d682d1a0..bba46f7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -179,6 +179,10 @@ enable_testing() add_subdirectory(examples) +IF(BEAGLE_DEBUG_KERNELS) + add_subdirectory(tests/basta) +ENDIF(BEAGLE_DEBUG_KERNELS) + set(CPACK_PACKAGE_VERSION "${BEAGLE_VERSION}") set(CPACK_PACKAGE_MAJOR "${BEAGLE_VERSION_MAJOR}") set(CPACK_PACKAGE_MINOR "${BEAGLE_VERSION_MINOR}") diff --git a/libhmsbeagle/GPU/kernels/kernelsX.cu b/libhmsbeagle/GPU/kernels/kernelsX.cu index f94d757f..28965291 100644 --- a/libhmsbeagle/GPU/kernels/kernelsX.cu +++ b/libhmsbeagle/GPU/kernels/kernelsX.cu @@ -1384,6 +1384,100 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsAutoScaling(KW_GLOBAL_VAR REAL* dResult[pattern] = (log(sum[state]) + (M_LN2 * maxScaleFactor)); } +/* + * BASTA kernels + */ + +// TODO: More appropriate kernel name +KW_GLOBAL_KERNEL void kernelInnerBastaPartials(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, + KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, + KW_GLOBAL_VAR REAL* KW_RESTRICT matrices1, + int totalPatterns) { + DETERMINE_INDICES_X_GPU(); + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ + /* Load values into shared memory */ + KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + int y = deltaPartialsByState + deltaPartialsByMatrix; + /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ + /* These are all coherent global memory reads; checked in Profiler */ + if (pattern < totalPatterns) { + sPartials1[patIdx][state] = partials1[y + state]; + } else { + sPartials1[patIdx][state] = 0; + } + REAL sum1 = 0; + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { + /* load one row of matrices */ + if (patIdx < BLOCK_PEELING_SIZE) { + /* These are all coherent global memory reads. */ + sMatrix1[patIdx][state] = matrix1[patIdx * PADDED_STATE_COUNT + state]; + /* sMatrix now filled with starting in state and ending in i */ + matrix1 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; + } + KW_LOCAL_FENCE; + for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { + FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); + } + KW_LOCAL_FENCE; + } + + if (pattern < totalPatterns) + partials3[u] = sum1; +} + +// TODO: More appropriate kernel name +KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, + KW_GLOBAL_VAR REAL* KW_RESTRICT partials2, + KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, + KW_GLOBAL_VAR REAL* KW_RESTRICT matrices1, + KW_GLOBAL_VAR REAL* KW_RESTRICT matrices2, + int totalPatterns) { +DETERMINE_INDICES_X_GPU(); + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix; /* Points to *this* matrix */ + /* Load values into shared memory */ + KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sMatrix2[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials2[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + + int y = deltaPartialsByState + deltaPartialsByMatrix; + /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ + /* These are all coherent global memory reads; checked in Profiler */ + if (pattern < totalPatterns) { + sPartials1[patIdx][state] = partials1[y + state]; + sPartials2[patIdx][state] = partials2[y + state]; + } else { + sPartials1[patIdx][state] = 0; + sPartials2[patIdx][state] = 0; + } + REAL sum1 = 0, sum2 = 0; + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { + /* load one row of matrices */ + if (patIdx < BLOCK_PEELING_SIZE) { + /* These are all coherent global memory reads. */ + sMatrix1[patIdx][state] = matrix1[patIdx * PADDED_STATE_COUNT + state]; + sMatrix2[patIdx][state] = matrix2[patIdx * PADDED_STATE_COUNT + state]; + /* sMatrix now filled with starting in state and ending in i */ + matrix1 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; + matrix2 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; + } + KW_LOCAL_FENCE; + for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { + FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); + FMA(sMatrix2[j][state], sPartials2[patIdx][i + j], sum2); + } + KW_LOCAL_FENCE; + } + + if (pattern < totalPatterns) + partials3[u] = sum1 * sum2; +} + + + + #ifdef CUDA #include "kernelsXDerivatives.cu" #endif // CUDA diff --git a/tests/basta/CMakeLists.txt b/tests/basta/CMakeLists.txt new file mode 100644 index 00000000..1f15b48e --- /dev/null +++ b/tests/basta/CMakeLists.txt @@ -0,0 +1,26 @@ +include_directories( + ${CUDA_INCLUDE_DIRS} +) + +link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib + ${CUDA_TOOLKIT_ROOT_DIR}/lib/x64 + ${CUDA_TOOLKIT_ROOT_DIR}/lib64 +) + +enable_language("CUDA") + +set(STATE_SIZE 16) + +set(BASTA_TEST_NAME "test_updateInnerBastaPartials_${STATE_SIZE}") +add_executable(${BASTA_TEST_NAME} test_x_state_updateInnerBastaPartials.cu) +target_compile_definitions( ${BASTA_TEST_NAME} PRIVATE STATE_SIZE=${STATE_SIZE} ) +target_compile_options(${BASTA_TEST_NAME} PRIVATE + $<$: + -I/usr/local/cuda/include; + -lineinfo + > +) +target_link_libraries(${BASTA_TEST_NAME} cuda) +set_source_files_properties(test_x_state_updateInnerBastaPartials.cu PROPERTIES LANGUAGE CUDA) + +add_dependencies(${BASTA_TEST_NAME} CudaKernels) \ No newline at end of file diff --git a/tests/basta/test_x_state_updateInnerBastaPartials.cu b/tests/basta/test_x_state_updateInnerBastaPartials.cu new file mode 100644 index 00000000..1da7f3b7 --- /dev/null +++ b/tests/basta/test_x_state_updateInnerBastaPartials.cu @@ -0,0 +1,122 @@ +// +// Created by gkarthik on 4/20/24. +// + +#include +#include +#include +#include +#include "../../libhmsbeagle/GPU/kernels/BeagleTensorCore_kernels.h" + +#define NPATTERNS 1 + +#define CONCAT(a, b) CONCAT_INNER(a, b) +#define CONCAT_INNER(a, b) a ## _ ## b + +#define GET_VAR_NAME(VAR) CONCAT(VAR, STATE_SIZE) + +#define KERNEL_STRING CONCAT(KERNELS_STRING_DP, STATE_SIZE) + +double partials1Host_16[] = {0.137572756735608, 0.477254124591127, 0.428745979443192, 0.218426858074963, 0.529428300447762, 0.110051316907629, 0.000466231256723404, 0.912552231224254, 0.472219149814919, 0.928942704107612, 0.560180699685588, 0.217537123244256, 0.697977353585884, 0.623952043708414, 0.675538737559691, 0.843222103314474}; +double partials2Host_16[] = {0.828149347100407, 0.864858906250447, 0.350309345172718, 0.155412578489631, 0.571178458165377, 0.0642696765717119, 0.767122565768659, 0.989166378509253, 0.807592692552134, 0.629633222240955, 0.759149922290817, 0.947871233569458, 0.514384098816663, 0.985400719335303, 0.437582379207015, 0.961291362997144}; +double matrices_16[] = {0.318755098618567, 0.740407602395862, 0.874102377332747, 0.690571578918025, 0.701961038634181, 0.795170336496085, 0.348821813939139, 0.719763568369672, 0.895593842957169, 0.0806103076320142, 0.0329403523355722, 0.594381110742688, 0.359542798716575, 0.0881143372971565, 0.649336193455383, 0.699423802085221, 0.194982040207833, 0.0281000500544906, 0.771264882525429, 0.364646208938211, 0.0918229892849922, 0.936870703473687, 0.37309667118825, 0.0921725274529308, 0.159810511628166, 0.626376076601446, 0.692677799146622, 0.207317036110908, 0.970800938550383, 0.101070829667151, 0.944739747559652, 0.842726089060307, 0.207157449331135, 0.9937439779751, 0.821067644748837, 0.698300351854414, 0.782907243119553, 0.973109148675576, 0.77274006837979, 0.618896092521027, 0.300635955762118, 0.499129123752937, 0.656451256712899, 0.900395354721695, 0.56438325252384, 0.860600786982104, 0.49170877225697, 0.889088666066527, 0.0156871858052909, 0.81213018624112, 0.977085781516507, 0.295148188248277, 0.726798894582316, 0.916058843955398, 0.608332335250452, 0.93036122340709, 0.740293311886489, 0.477501145098358, 0.528508115326986, 0.359984828624874, 0.115733818151057, 0.162383440183476, 0.599637272302061, 0.727882955456153, 0.674989679362625, 0.862724364968017, 0.736116931773722, 0.934994351351634, 0.225678282091394, 0.0690643971320242, 0.642801651265472, 0.0847296256106347, 0.11830615834333, 0.203358805505559, 0.326086097629741, 0.730495651019737, 0.29963503475301, 0.561327135190368, 0.0538779601920396, 0.904581440845504, 0.807041098363698, 0.0369221286382526, 0.676678945310414, 0.814023691928014, 0.651314357761294, 0.901343589415774, 0.429449937539175, 0.628918903879821, 0.0323536104988307, 0.718082484090701, 0.4828297609929, 0.284520726883784, 0.997801034012809, 0.577156581915915, 0.48068075068295, 0.720765815814957, 0.403965384466574, 0.170624090125784, 0.876439346699044, 0.593427885789424, 0.769376888172701, 0.647851618006825, 0.337206641677767, 0.408452160190791, 0.0607802064623684, 0.676360174780712, 0.462037036195397, 0.347813759464771, 0.946434140671045, 0.604583449661732, 0.742167289135978, 0.0429284807760268, 0.626676751766354, 0.64561746106483, 0.673989933682606, 0.662993856705725, 0.856125113321468, 0.870688743656501, 0.308341189753264, 0.538020107662305, 0.538775310153142, 0.258829150116071, 0.407576418248937, 0.0590927223674953, 0.520942375296727, 0.434115749085322, 0.340059822658077, 0.17571780225262, 0.53029696107842, 0.975732607766986, 0.831993991043419, 0.983791405102238, 0.55990690854378, 0.710094312205911, 0.0787766603752971, 0.589558490784839, 0.673550385283306, 0.673261155607179, 0.974599071079865, 0.340651801321656, 0.0923955119214952, 0.194945273222402, 0.141258440678939, 0.639831262873486, 0.339301661355421, 0.884930776199326, 0.289516024757177, 0.126225155545399, 0.446420934982598, 0.0850053902249783, 0.395572019508108, 0.157721624244004, 0.929064428433776, 0.875803815200925, 0.36426582490094, 0.161793374456465, 0.387928539188579, 0.59461420099251, 0.174750925740227, 0.115827345522121, 0.152487984858453, 0.035088024334982, 0.983126569306478, 0.575697488617152, 0.209149486385286, 0.201207815203816, 0.110956045333296, 0.0174690247513354, 0.775130218127742, 0.497412796830758, 0.443602121900767, 0.0095906644128263, 0.198237520409748, 0.304748482536525, 0.518023134209216, 0.547018625540659, 0.526118801673874, 0.493327406002209, 0.913669281871989, 0.077805265551433, 0.929841819219291, 0.29471311555244, 0.118532110471278, 0.14501806977205, 0.436429607449099, 0.205097107682377, 0.708817657083273, 0.452689809259027, 0.0368115562014282, 0.239382016705349, 0.471294831018895, 0.290052818134427, 0.141418197192252, 0.406050314893946, 0.123278650688007, 0.793885040329769, 0.803798166336492, 0.653874792391434, 0.570221147732809, 0.238239917438477, 0.29717841069214, 0.753295891918242, 0.86618359782733, 0.0173538792878389, 0.262981676263735, 0.870678049977869, 0.235948005225509, 0.518082186812535, 0.80484891217202, 0.0546407853253186, 0.849502737633884, 0.749514716677368, 0.744097977643833, 0.543421939015388, 0.416765965521336, 0.725398944690824, 0.33459063549526, 0.478168275207281, 0.308823636500165, 0.518112531863153, 0.0382656576111913, 0.569260101066902, 0.542835577391088, 0.786958181066439, 0.191143112257123, 0.756994087249041, 0.930127918720245, 0.0538773911539465, 0.108583583729342, 0.0136260520666838, 0.0493806877639145, 0.0609419206157327, 0.157695269677788, 0.351834069937468, 0.396392825525254, 0.179786366643384, 0.822698512114584, 0.391177944373339, 0.0950508427340537, 0.0245800712145865, 0.28290240559727, 0.772152968682349, 0.14831333537586, 0.672689846949652, 0.826513405656442, 0.395902422023937, 0.837914811447263, 0.927084936527535, 0.0562630859203637, 0.810627286788076, 0.67597354692407, 0.622439709492028, 0.0950460624881089, 0.0380097457673401, 0.166366128250957, 0.867308233864605}; +double expectedPartials_16[] = {12.9769783870594, 25.8647148645348, 37.1028089862958, 26.553395775949, 28.2498807522581, 21.9503211762829, 12.9202534609119, 13.9146869398328, 14.9317655768219, 22.3371483393775, 22.9624904769671, 9.58813665982792, 11.4421566478761, 13.2079833456249, 11.7415865195237, 23.4516262593406}; + +int main() { + char functionName[] = "kernelInnerBastaPartials"; + CUmodule cudaModule; + CUresult res; + cudaError_t err; + + cuInit(0); + + int numDevices; + cuDeviceGetCount(&numDevices); + std::cout << "Number of devices " << numDevices << std::endl; + + CUdevice cuDevice; + res = cuDeviceGet(&cuDevice, 0); + + CUcontext context; + res = cuCtxCreate(&context, 0, cuDevice); + + res = cuModuleLoadData(&cudaModule, KERNEL_STRING); + + CUfunction cudaFunction; + + res = cuModuleGetFunction(&cudaFunction, cudaModule, functionName); + + if(res != CUDA_SUCCESS) { + std::cerr << "Error loading function " << functionName < tolerance) { + if(pass == 0) + std::cerr << "Unequal values at positions " << std::endl; + pass = -1; + std::cerr << i << " Expected: " << GET_VAR_NAME(expectedPartials)[i] << " Observed: " << resHost[i] << std::endl; + } + } + + if(pass != 0) + std::cerr << std::endl << std::endl; + + if(pass == 0) { + std::cerr << "PASS: Test for post-order kernel for state size " << STATE_SIZE << std::endl; + } else { + std::cerr << "FAIL: Test for post-order kernel for state size " << STATE_SIZE << std::endl; + } + cuModuleUnload(cudaModule); + + cudaFree(matrices1); + cudaFree(matrices2); + cudaFree(partials1); + cudaFree(partials2); + cudaFree(partials3); + free(resHost); + + return pass; +} \ No newline at end of file From b656a357ff3a6265e4a3fdfcc1b5f9a153b9470e Mon Sep 17 00:00:00 2001 From: gkarthik Date: Sat, 20 Apr 2024 16:05:18 -0500 Subject: [PATCH 034/421] Set blockDim.y to PATTERN_BLOCK_SIZE of 8. --- tests/basta/test_x_state_updateInnerBastaPartials.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basta/test_x_state_updateInnerBastaPartials.cu b/tests/basta/test_x_state_updateInnerBastaPartials.cu index 1da7f3b7..833e4eac 100644 --- a/tests/basta/test_x_state_updateInnerBastaPartials.cu +++ b/tests/basta/test_x_state_updateInnerBastaPartials.cu @@ -77,7 +77,7 @@ int main() { &partials1, &partials3, &matrices1, &totalPatterns }; - res = cuLaunchKernel(cudaFunction, 1, 1, 1, STATE_SIZE, 4, 1, 0, NULL, params, NULL); + res = cuLaunchKernel(cudaFunction, 1, 1, 1, STATE_SIZE, 8, 1, 0, NULL, params, NULL); cudaDeviceSynchronize(); From 380cf71100d770b9a2930da8219cb3ffe0f75525 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 6 May 2024 17:59:32 -0500 Subject: [PATCH 035/421] a very crude plugin start --- libhmsbeagle/GPU/ActionCUDAPlugin.cpp | 86 ++++++++++++++++++++ libhmsbeagle/GPU/ActionCUDAPlugin.h | 59 ++++++++++++++ libhmsbeagle/GPU/CMakeLists.txt | 5 ++ libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 64 +++++++++++++++ 4 files changed, 214 insertions(+) create mode 100644 libhmsbeagle/GPU/ActionCUDAPlugin.cpp create mode 100644 libhmsbeagle/GPU/ActionCUDAPlugin.h create mode 100644 libhmsbeagle/GPU/CMake_Action/CMakeLists.txt diff --git a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp new file mode 100644 index 00000000..965daaae --- /dev/null +++ b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp @@ -0,0 +1,86 @@ +/** + * libhmsbeagle plugin system + * @author Xiang Ji + * @author Benjamin Redelings + * @author Marc Suchard + */ + +#include "libhmsbeagle/GPU/BeagleGPUImpl.h" +#include "ActionCUDAPlugin.h" + + +namespace beagle { + namespace gpu { + + ActionCUDAPlugin::ActionCUDAPlugin() : + Plugin("GPU-Action-CUDA", "GPU-Action-CUDA") + { + GPUInterface gpu; + bool anyGPUSupportsCUDA = false; + bool anyGPUSupportsDP = false; + if (gpu.Initialize()) { + int gpuDeviceCount = gpu.GetDeviceCount(); + anyGPUSupportsCUDA = (gpuDeviceCount > 0); + for (int i = 0; i < gpuDeviceCount; i++) { + int nameDescSize = 256; + char* dName = (char*) malloc(sizeof(char) * nameDescSize); + char* dDesc = (char*) malloc(sizeof(char) * nameDescSize); + gpu.GetDeviceName(i, dName, nameDescSize); + gpu.GetDeviceDescription(i, dDesc); + + BeagleResource resource; + resource.name = dName; + resource.description = dDesc; + resource.supportFlags = BEAGLE_FLAG_COMPUTATION_SYNCH | BEAGLE_FLAG_COMPUTATION_ASYNCH | + BEAGLE_FLAG_PRECISION_SINGLE | + BEAGLE_FLAG_SCALING_MANUAL | BEAGLE_FLAG_SCALING_ALWAYS | + BEAGLE_FLAG_SCALING_AUTO | BEAGLE_FLAG_SCALING_DYNAMIC | + BEAGLE_FLAG_THREADING_NONE | + BEAGLE_FLAG_VECTOR_NONE | + BEAGLE_FLAG_PROCESSOR_GPU | + BEAGLE_FLAG_SCALERS_LOG | BEAGLE_FLAG_SCALERS_RAW | + BEAGLE_FLAG_EIGEN_COMPLEX | BEAGLE_FLAG_EIGEN_REAL | + BEAGLE_FLAG_INVEVEC_STANDARD | BEAGLE_FLAG_INVEVEC_TRANSPOSED | + BEAGLE_FLAG_PREORDER_TRANSPOSE_MANUAL | BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO | + BEAGLE_FLAG_PARALLELOPS_GRID | BEAGLE_FLAG_PARALLELOPS_STREAMS | + BEAGLE_FLAG_COMPUTATION_ACTION | + BEAGLE_FLAG_FRAMEWORK_CUDA; + // Determine DP capability + if (gpu.GetSupportsDoublePrecision(i)) { + resource.supportFlags |= BEAGLE_FLAG_PRECISION_DOUBLE; + anyGPUSupportsDP = true; + } + + resource.requiredFlags = BEAGLE_FLAG_FRAMEWORK_CUDA; + + beagleResources.push_back(resource); + } + } + + // Optional for plugins: check if the hardware is compatible and only populate + // list with compatible factories +// if(beagleResources.size() > 0) { + if (anyGPUSupportsCUDA) { + using namespace cuda; + if (anyGPUSupportsDP) { + // TODO Uncomment when working + beagleFactories.push_back(new BeagleGPUImplFactory()); + } + beagleFactories.push_back(new BeagleGPUImplFactory()); + } + } + + ActionCUDAPlugin::~ActionCUDAPlugin() + { + } + + } // namespace gpu +} // namespace beagle + + +extern "C" { +void* plugin_init(void){ + return new beagle::gpu::ActionCUDAPlugin(); +} +} + diff --git a/libhmsbeagle/GPU/ActionCUDAPlugin.h b/libhmsbeagle/GPU/ActionCUDAPlugin.h new file mode 100644 index 00000000..cf658925 --- /dev/null +++ b/libhmsbeagle/GPU/ActionCUDAPlugin.h @@ -0,0 +1,59 @@ +/** + * libhmsbeagle plugin system + * @author Xiang Ji + * @author Benjamin Redelings + * @author Marc Suchard + */ + +#ifndef __BEAGLE_ACTIONCUDAPLUGIN_H__ +#define __BEAGLE_ACTIONCUDAPLUGIN_H__ + +#ifdef HAVE_CONFIG_H +#include "libhmsbeagle/config.h" +#endif + +#include "libhmsbeagle/platform.h" +#include "libhmsbeagle/plugin/Plugin.h" + + + +namespace beagle { + namespace gpu { + + class BEAGLE_DLLEXPORT ActionCUDAPlugin : public beagle::plugin::Plugin + { + public: + ActionCUDAPlugin(); + ~ActionCUDAPlugin(); + private: + ActionCUDAPlugin( const ActionCUDAPlugin& cp ); // disallow copy by defining this private + }; + + } // namespace gpu +} // namespace beagle + +extern "C" { + BEAGLE_DLLEXPORT void* plugin_init(void); +} + + + + + + + + + + + + + + + + + + + + + +#endif //BEAGLE_ACTIONCUDAPLUGIN_H diff --git a/libhmsbeagle/GPU/CMakeLists.txt b/libhmsbeagle/GPU/CMakeLists.txt index 22b3c1ef..5944fcee 100644 --- a/libhmsbeagle/GPU/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMakeLists.txt @@ -11,3 +11,8 @@ if(CUDA_FOUND AND BUILD_CUDA) message(STATUS "CUDA Includes: ${CUDA_INCLUDE_DIRS}") add_subdirectory("CMake_CUDA") endif(CUDA_FOUND AND BUILD_CUDA) + +if(CUDA_FOUND AND BUILD_ACTION) + message(STATUS "CUDA Includes: ${CUDA_INCLUDE_DIRS}") + add_subdirectory("CMake_Action") +endif(CUDA_FOUND AND BUILD_ACTION) diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt new file mode 100644 index 00000000..76ed04b0 --- /dev/null +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -0,0 +1,64 @@ + +include_directories( + ${CUDA_INCLUDE_DIRS} +) + +add_definitions(-DCUDA) + +link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib + ${CUDA_TOOLKIT_ROOT_DIR}/lib/x64 + ${CUDA_TOOLKIT_ROOT_DIR}/lib64 +) + +if(WIN32) + set(NVCC "nvcc.exe") + set(NVCCFLAGS "-O3 -Wno-deprecated-gpu-targets") + set(NVCC_INC "-I../../..") + set(CMD_NAME createCUDAKernels.bat) +else(WIN32) + set(NVCC "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc") + set(NVCCFLAGS "-O3 -Wno-deprecated-gpu-targets -ccbin ${CMAKE_CXX_COMPILER} -m64 -D_POSIX_C_SOURCE -std=c++11") + set(NVCC_INC "-I${CMAKE_SOURCE_DIR}") + set(CMD_NAME make_cuda_kernels.sh) +endif(WIN32) + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/../kernels/BeagleCUDA_kernels.h + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/../kernels/${CMD_NAME} ${NVCC} "${NVCCFLAGS}" "${NVCC_INC}" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../kernels + DEPENDS "../kernels/kernels4.cu" "../kernels/kernels4Derivatives.cu" "../kernels/kernelsXDerivatives.cu" "../kernels/kernelsAll.cu" + COMMENT "Building CUDA kernels" + VERBATIM) + +#add_custom_target(CudaKernels DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../kernels/BeagleCUDA_kernels.h) + +add_library(hmsbeagle-action-cuda SHARED + ../BeagleGPUImpl.h + ../BeagleGPUImpl.hpp + ../GPUImplDefs.h + ../GPUImplHelper.cpp + ../GPUImplHelper.h + ../GPUInterfaceCUDA.cpp + ../KernelLauncher.cpp + ../KernelLauncher.h + ../KernelResource.cpp + ../KernelResource.h + ../ActionCUDAPlugin.cpp + ../ActionCUDAPlugin.h + ../Precision.h +) + +add_dependencies(hmsbeagle-action-cuda CudaKernels) + +target_link_libraries(hmsbeagle-action-cuda cuda ${CUDA_LIBRARIES}) + +install(TARGETS hmsbeagle-action-cuda + DESTINATION ${BEAGLE_INSTALL_DIR} + COMPONENT action-cuda +) + +SET_TARGET_PROPERTIES(hmsbeagle-action-cuda + PROPERTIES + SOVERSION "${BEAGLE_PLUGIN_VERSION_EXTENDED}" + SUFFIX "${BEAGLE_PLUGIN_SUFFIX}" +) From c600bf9f0f028dc05f29e97e2b992549664a87c0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 8 May 2024 14:40:59 -0500 Subject: [PATCH 036/421] Add BeagleGPUActionImpl.h --- libhmsbeagle/GPU/ActionCUDAPlugin.cpp | 2 +- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 59 ++++++++++++++++++++ libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 4 +- 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 libhmsbeagle/GPU/BeagleGPUActionImpl.h diff --git a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp index 965daaae..a66794b1 100644 --- a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp +++ b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp @@ -5,7 +5,7 @@ * @author Marc Suchard */ -#include "libhmsbeagle/GPU/BeagleGPUImpl.h" +#include "libhmsbeagle/GPU/BeagleGPUActionImpl.h" #include "ActionCUDAPlugin.h" diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h new file mode 100644 index 00000000..a1c35926 --- /dev/null +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -0,0 +1,59 @@ +/* + * BeagleGPUActionImpl.h + * BEAGLE + * + * Copyright 2024 Phylogenetic Likelihood Working Group + * + * This file is part of BEAGLE. + * + * BEAGLE is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * BEAGLE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with BEAGLE. If not, see + * . + * + * @author Xiang Ji + * @author Benjamin Redelings + * @author Marc Suchard + */ + + +#ifndef BEAGLE_BEAGLEGPUACTIONIMPL_H +#define BEAGLE_BEAGLEGPUACTIONIMPL_H + +#ifdef HAVE_CONFIG_H +#include "libhmsbeagle/config.h" +#endif + +#include "libhmsbeagle/GPU/BeagleGPUImpl.h" + +namespace beagle { +namespace gpu { + +#ifdef CUDA + namespace cuda { +#else + namespace opencl { +#endif + +BEAGLE_GPU_TEMPLATE +class BeagleGPUActionImpl : public BeagleGPUImpl { +}; + +BEAGLE_GPU_TEMPLATE +class BeagleGPUActionImplFactory : public BeagleGPUImplFactory { +}; + +} // namspace device +} // namespace gpu +} // namespace beagle + +#endif diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt index 76ed04b0..f2e2d114 100644 --- a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -33,8 +33,8 @@ add_custom_command( #add_custom_target(CudaKernels DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../kernels/BeagleCUDA_kernels.h) add_library(hmsbeagle-action-cuda SHARED - ../BeagleGPUImpl.h - ../BeagleGPUImpl.hpp + ../BeagleGPUActionImpl.h +# ../BeagleGPUActionImpl.hpp ../GPUImplDefs.h ../GPUImplHelper.cpp ../GPUImplHelper.h From 73b46a9f5aa4d64ff366e470a61ed74c0a77beb2 Mon Sep 17 00:00:00 2001 From: yucais Date: Mon, 13 May 2024 01:37:09 -0700 Subject: [PATCH 037/421] peel partials for basta --- libhmsbeagle/GPU/BeagleGPUImpl.h | 35 +++- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 118 ++++++++++++- libhmsbeagle/GPU/KernelLauncher.cpp | 35 ++++ libhmsbeagle/GPU/KernelLauncher.h | 5 + libhmsbeagle/GPU/kernels/kernelsAll.cu | 158 ++++++++++++++++++ libhmsbeagle/GPU/kernels/kernelsX.cu | 91 ---------- .../test_x_state_updateInnerBastaPartials.cu | 120 ++++++++++++- 7 files changed, 452 insertions(+), 110 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 99943aa2..92b7514f 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -212,6 +212,11 @@ class BeagleGPUImpl : public BeagleImpl { int* hStreamIndices; + GPUPtr gBastaBuffers; + GPUPtr gCoalescentBuffers; + int kCoalescentBufferLength; + int kCoalescentBufferCount; + public: BeagleGPUImpl(); @@ -308,6 +313,10 @@ class BeagleGPUImpl : public BeagleImpl { const int* secondDerivativeIndices, const double* edgeLengths, int count); + + // int updateTransitionMatricesGrad(const int* probabilityIndices, + // const double* edgeLengths, + // int count); int updateTransitionMatricesWithModelCategories(int* eigenIndices, const int* probabilityIndices, @@ -459,14 +468,23 @@ class BeagleGPUImpl : public BeagleImpl { int getSiteDerivatives(double* outFirstDerivatives, double* outSecondDerivatives); - + + int updateInnerBastaPartials(const int * operations, int i, int count, GPUPtr gpu, GPUPtr coalescent); + int updateBastaPartials(const int* operations, - int operationCount, - const int* intervals, - int intervalCount, + int operationCount, + const int* intervals, + int intervalCount, int populationSizesIndex, int coalescentIndex); + // int updateBastaPartialsGrad(const int* operations, + // int operationCount, + // const int* intervals, + // int intervalCount, + // int populationSizesIndex, + // int coalescentIndex); + int accumulateBastaPartials(const int* operations, int operationCount, const int* segments, @@ -475,6 +493,15 @@ class BeagleGPUImpl : public BeagleImpl { const int populationSizesIndex, int coalescentIndex, double* out); + + // int accumulateBastaPartialsGrad(const int *operations, + // const int operationCount, + // const int *intervalStarts, + // const int intervalStartsCount, + // const double *intervalLengths, + // const int populationSizesIndex, + // const int coalescentIndex, + // double *out); int allocateBastaBuffers(int bufferCount, int bufferLength); diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index a4effbdc..c7eb193e 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -77,6 +77,9 @@ BeagleGPUImpl::BeagleGPUImpl() { dMaxScalingFactors = (GPUPtr)NULL; dIndexMaxScalingFactors = (GPUPtr)NULL; + gBastaBuffers = (GPUPtr)NULL; + gCoalescentBuffers = (GPUPtr)NULL; + dEigenValues = NULL; dEvec = NULL; dIevc = NULL; @@ -1887,6 +1890,14 @@ int BeagleGPUImpl::transposeTransitionMatrices( return BEAGLE_SUCCESS; } +// BEAGLE_GPU_TEMPLATE +// int BeagleGPUImpl::updateTransitionMatricesGrad(const int* probabilityIndices, +// const double* edgeLengths, +// int count) { +// return BEAGLE_ERROR_NO_IMPLEMENTATION; +// } +// + BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, @@ -2206,15 +2217,19 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, fprintf(stderr, "\tEntering BeagleGPUImpl::allocateBastaBuffers\n"); #endif - int returnCode = BEAGLE_ERROR_GENERAL; + kCoalescentBufferLength = bufferLength; + kCoalescentBufferCount = bufferCount; + + gCoalescentBuffers = gpu->AllocateMemory(kCoalescentBufferCount * kCoalescentBufferLength * sizeof(Real)); + gBastaBuffers = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * 4 * sizeof(Real)); - fprintf(stderr, "BeagleGPUImpl::allocateBastaBuffers\n"); + // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUImpl::allocateBastaBuffers\n"); #endif - return returnCode; + return BEAGLE_SUCCESS; } BEAGLE_GPU_TEMPLATE @@ -2244,9 +2259,11 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - int returnCode = BEAGLE_ERROR_GENERAL; - - fprintf(stderr, "BeagleGPUImpl::updateBastaPartials\n"); + GPUPtr coalescent = gCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); + + const GPUPtr sizes = dFrequencies[populationSizesIndex]; + int returnCode = updateInnerBastaPartials(operations, 0, count, sizes, coalescent); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUImpl::updateBastaPartials\n"); @@ -2255,6 +2272,70 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations return returnCode; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::updateInnerBastaPartials(const int* operations, + const int begin, + const int end, + const GPUPtr sizes, + GPUPtr coalescent) { + + + std::vector newOperations; + for (int op = begin; op < end; op++) { + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int parIndex = operations[op * numOps]; + const int child1Index = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2Index = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + + GPUPtr matrices1 = dMatrices[child1TransMatIndex]; + GPUPtr matrices2 = dMatrices[child2TransMatIndex]; + + GPUPtr partials1 = dPartials[child1Index]; + GPUPtr partials2 = dPartials[child2Index]; + GPUPtr partials3 = dPartials[parIndex]; + + + GPUPtr accumulation1 = dPartials[accumulation1PartialIndex]; + GPUPtr accumulation2 = dPartials[accumulation2PartialIndex]; + kernels->InnerBastaPartialsCoalescent(partials1, partials2, partials3, + matrices1, matrices2, sizes, accumulation1, accumulation2, coalescent, intervalNumber, 1, child2Index); + } + +#ifdef BEAGLE_DEBUG_SYNCH + gpu->SynchronizeHost(); +#endif + return BEAGLE_SUCCESS; +} + +// BEAGLE_GPU_TEMPLATE +// int BeagleGPUImpl::updateBastaPartialsGrad(const int* operations, +// const int count, +// const int* intervals, +// const int intervalCount, +// const int populationSizesIndex, +// const int coalescentIndex) { +// #ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); +// #endif +// +// int returnCode = BEAGLE_ERROR_GENERAL; +// +// fprintf(stderr, "BeagleGPUImpl::updateBastaPartials\n"); +// +// #ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tLeaving BeagleGPUImpl::updateBastaPartials\n"); +// #endif +// +// return returnCode; +// } + BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::accumulateBastaPartials(const int* operations, int operationCount, @@ -2280,6 +2361,31 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat } +// BEAGLE_GPU_TEMPLATE +// int BeagleGPUImpl::accumulateBastaPartialsGrad(const int *operations, +// const int operationCount, +// const int *intervalStarts, +// const int intervalStartsCount, +// const double *intervalLengths, +// const int populationSizesIndex, +// const int coalescentIndex, +// double *out) { +// #ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); +// #endif +// +// int returnCode = BEAGLE_ERROR_GENERAL; +// +// fprintf(stderr, "BeagleGPUImpl::accumulateBastaPartials\n"); +// +// #ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); +// #endif +// +// return returnCode; +// } + + BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::calculateEdgeDerivative(const int *postBufferIndices, const int *preBufferIndices, diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 43347701..2242163e 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -401,6 +401,9 @@ void KernelLauncher::LoadKernels() { fSumSites2 = gpu->GetFunction("kernelSumSites2"); fSumSites3 = gpu->GetFunction("kernelSumSites3"); + fInnerBastaPartialsCoalescent = gpu->GetFunction("kernelInnerBastaPartialsCoalescent"); + + fReorderPatterns = gpu->GetFunction("kernelReorderPatterns"); // partitioning and multi-op kernels @@ -2396,6 +2399,38 @@ void KernelLauncher::SumSites3(GPUPtr dArray1, } + +void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials1, + GPUPtr partials2, + GPUPtr partials3, + GPUPtr matrices1, + GPUPtr matrices2, + GPUPtr accumulation1, + GPUPtr accumulation2, + const GPUPtr sizes, + GPUPtr coalescent, + unsigned int intervalNUmber, + unsigned int patternCount, + unsigned int child2Index) { +#ifdef BEAGLE_DEBUG_FLOW +fprintf(stderr, "\t\tEntering KernelLauncher::SumSites3\n"); +#endif + + int parameterCountV = 9; + int totalParameterCount = 12; + gpu->LaunchKernel(fInnerBastaPartialsCoalescent, + bgPeelingBlock, bgPeelingGrid, + parameterCountV, totalParameterCount, + partials1, partials2, partials3, matrices1, matrices2, accumulation1, accumulation2, sizes, coalescent, + intervalNUmber, patternCount, child2Index); + +#ifdef BEAGLE_DEBUG_FLOW +fprintf(stderr, "\t\tLeaving KernelLauncher::SumSites3\n"); +#endif + +} + + }; // namespace diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index d9a6e501..f7108a47 100644 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -106,6 +106,8 @@ class KernelLauncher { GPUFunction fSumSites2; GPUFunction fSumSites3; + GPUFunction fInnerBastaPartialsCoalescent; + GPUFunction fReorderPatterns; Dim3Int bgTransitionProbabilitiesBlock; @@ -605,6 +607,9 @@ class KernelLauncher { GPUPtr dPatternWeights, unsigned int patternCount); + void InnerBastaPartialsCoalescent(GPUPtr partials1, GPUPtr partials2, GPUPtr partials3, GPUPtr matrices1, + GPUPtr matrices2, GPUPtr accumulation1, GPUPtr accumulation2, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, unsigned int patternCount, unsigned int child2Index); + void SetupKernelBlocksAndGrids(); protected: diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 107467fb..9e604906 100644 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -1973,6 +1973,164 @@ KW_GLOBAL_KERNEL void kernelAccumulateFactorsAutoScaling(KW_GLOBAL_VAR signed ch rootScaling[index] = total; } + +/* + * BASTA kernels + */ + +// TODO: More appropriate kernel name +KW_GLOBAL_KERNEL void kernelInnerBastaPartials(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, + KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, + KW_GLOBAL_VAR REAL* KW_RESTRICT matrices1, + int totalPatterns) { + int state = KW_LOCAL_ID_0; + int patIdx = KW_LOCAL_ID_1; + int pattern = __umul24(KW_GROUP_ID_0,PATTERN_BLOCK_SIZE) + patIdx; + int matrix = KW_GROUP_ID_1; + int patternCount = totalPatterns; + int deltaPartialsByState = pattern * PADDED_STATE_COUNT; + int deltaPartialsByMatrix = matrix * PADDED_STATE_COUNT * patternCount; + int deltaMatrix = matrix * PADDED_STATE_COUNT * PADDED_STATE_COUNT; + int u = state + deltaPartialsByState + deltaPartialsByMatrix; + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ + /* Load values into shared memory */ + KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + int y = deltaPartialsByState + deltaPartialsByMatrix; + /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ + /* These are all coherent global memory reads; checked in Profiler */ + if (pattern < totalPatterns) { + sPartials1[patIdx][state] = partials1[y + state]; + } else { + sPartials1[patIdx][state] = 0; + } + REAL sum1 = 0; + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { + /* load one row of matrices */ + if (patIdx < BLOCK_PEELING_SIZE) { + /* These are all coherent global memory reads. */ + sMatrix1[patIdx][state] = matrix1[patIdx * PADDED_STATE_COUNT + state]; + /* sMatrix now filled with starting in state and ending in i */ + matrix1 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; + } + KW_LOCAL_FENCE; + for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { + FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); + } + KW_LOCAL_FENCE; + } + + if (pattern < totalPatterns) + partials3[u] = sum1; +} + +// TODO: More appropriate kernel name +KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, + KW_GLOBAL_VAR REAL* KW_RESTRICT partials2, + KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, + KW_GLOBAL_VAR REAL* KW_RESTRICT matrices1, + KW_GLOBAL_VAR REAL* KW_RESTRICT matrices2, + KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation1, + KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation2, + KW_GLOBAL_VAR REAL* KW_RESTRICT sizes, + KW_GLOBAL_VAR REAL* KW_RESTRICT coalescent, + int intervalNumber, + int totalPatterns, + int child2Index) { + int state = KW_LOCAL_ID_0; + int patIdx = KW_LOCAL_ID_1; + int pattern = __umul24(KW_GROUP_ID_0,PATTERN_BLOCK_SIZE) + patIdx; + int matrix = KW_GROUP_ID_1; + int patternCount = totalPatterns; + int deltaPartialsByState = pattern * PADDED_STATE_COUNT; + int deltaPartialsByMatrix = matrix * PADDED_STATE_COUNT * patternCount; + int deltaMatrix = matrix * PADDED_STATE_COUNT * PADDED_STATE_COUNT; + int u = state + deltaPartialsByState + deltaPartialsByMatrix; + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ + /* Load values into shared memory */ + KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; + + KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + + + int y = deltaPartialsByState + deltaPartialsByMatrix; + if (pattern < totalPatterns) { + sPartials1[patIdx][state] = partials1[y + state]; + } else { + sPartials1[patIdx][state] = 0; + } + REAL sum1 = 0; + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { + /* load one row of matrices */ + if (patIdx < BLOCK_PEELING_SIZE) { + /* These are all coherent global memory reads. */ + sMatrix1[patIdx][state] = matrix1[patIdx * PADDED_STATE_COUNT + state]; + /* sMatrix now filled with starting in state and ending in i */ + matrix1 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; + } + KW_LOCAL_FENCE; + for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { + FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); + } + KW_LOCAL_FENCE; + } + + if (pattern < totalPatterns) + partials3[u] = sum1; + /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ + /* These are all coherent global memory reads; checked in Profiler */ + if (child2Index >= 0) { + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix; + KW_LOCAL_MEM REAL sMatrix2[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials2[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials3[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + + if (pattern < totalPatterns) { + sPartials2[patIdx][state] = partials2[y + state]; + } else { + sPartials2[patIdx][state] = 0; + } + REAL sum2 = 0; + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { + /* load one row of matrices */ + if (patIdx < BLOCK_PEELING_SIZE) { + /* These are all coherent global memory reads. */ + sMatrix2[patIdx][state] = matrix2[patIdx * PADDED_STATE_COUNT + state]; + /* sMatrix now filled with starting in state and ending in i */ + matrix2 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; + } + + KW_LOCAL_FENCE; + for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { + FMA(sMatrix2[j][state], sPartials2[patIdx][i + j], sum2); + } + KW_LOCAL_FENCE; + } + + if (pattern < totalPatterns) { + accumulation1[u] = sum1; + accumulation2[u] = sum2; + partials3[u] = sum1 * sum2 / sizes[state]; + sPartials3[patIdx][state] = partials3[y + state]; + +#ifdef IS_POWER_OF_TWO + // parallelized reduction *** only works for powers-of-2 **** + for (int i = PADDED_STATE_COUNT / 2; i > 0; i >>= 1) { + if (state < i) { +#else + for (int i = SMALLEST_POWER_OF_TWO / 2; i > 0; i >>= 1) { + if (state < i && state + i < PADDED_STATE_COUNT ) { +#endif // IS_POWER_OF_TWO + sPartials3[patIdx][state] += sPartials3[patIdx][state + i]; + } + KW_LOCAL_FENCE; + } + coalescent[intervalNumber] = sPartials3[patIdx][0]; + partials3[u] = partials3[u] / coalescent[intervalNumber]; + } + } +} + #ifdef CUDA } // extern "C" #endif //CUDA diff --git a/libhmsbeagle/GPU/kernels/kernelsX.cu b/libhmsbeagle/GPU/kernels/kernelsX.cu index 28965291..90f159d8 100644 --- a/libhmsbeagle/GPU/kernels/kernelsX.cu +++ b/libhmsbeagle/GPU/kernels/kernelsX.cu @@ -1384,97 +1384,6 @@ KW_GLOBAL_KERNEL void kernelIntegrateLikelihoodsAutoScaling(KW_GLOBAL_VAR REAL* dResult[pattern] = (log(sum[state]) + (M_LN2 * maxScaleFactor)); } -/* - * BASTA kernels - */ - -// TODO: More appropriate kernel name -KW_GLOBAL_KERNEL void kernelInnerBastaPartials(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, - KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, - KW_GLOBAL_VAR REAL* KW_RESTRICT matrices1, - int totalPatterns) { - DETERMINE_INDICES_X_GPU(); - KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ - /* Load values into shared memory */ - KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; - int y = deltaPartialsByState + deltaPartialsByMatrix; - /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ - /* These are all coherent global memory reads; checked in Profiler */ - if (pattern < totalPatterns) { - sPartials1[patIdx][state] = partials1[y + state]; - } else { - sPartials1[patIdx][state] = 0; - } - REAL sum1 = 0; - for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { - /* load one row of matrices */ - if (patIdx < BLOCK_PEELING_SIZE) { - /* These are all coherent global memory reads. */ - sMatrix1[patIdx][state] = matrix1[patIdx * PADDED_STATE_COUNT + state]; - /* sMatrix now filled with starting in state and ending in i */ - matrix1 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; - } - KW_LOCAL_FENCE; - for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { - FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); - } - KW_LOCAL_FENCE; - } - - if (pattern < totalPatterns) - partials3[u] = sum1; -} - -// TODO: More appropriate kernel name -KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, - KW_GLOBAL_VAR REAL* KW_RESTRICT partials2, - KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, - KW_GLOBAL_VAR REAL* KW_RESTRICT matrices1, - KW_GLOBAL_VAR REAL* KW_RESTRICT matrices2, - int totalPatterns) { -DETERMINE_INDICES_X_GPU(); - KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ - KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix; /* Points to *this* matrix */ - /* Load values into shared memory */ - KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sMatrix2[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartials2[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; - - int y = deltaPartialsByState + deltaPartialsByMatrix; - /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ - /* These are all coherent global memory reads; checked in Profiler */ - if (pattern < totalPatterns) { - sPartials1[patIdx][state] = partials1[y + state]; - sPartials2[patIdx][state] = partials2[y + state]; - } else { - sPartials1[patIdx][state] = 0; - sPartials2[patIdx][state] = 0; - } - REAL sum1 = 0, sum2 = 0; - for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { - /* load one row of matrices */ - if (patIdx < BLOCK_PEELING_SIZE) { - /* These are all coherent global memory reads. */ - sMatrix1[patIdx][state] = matrix1[patIdx * PADDED_STATE_COUNT + state]; - sMatrix2[patIdx][state] = matrix2[patIdx * PADDED_STATE_COUNT + state]; - /* sMatrix now filled with starting in state and ending in i */ - matrix1 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; - matrix2 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; - } - KW_LOCAL_FENCE; - for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { - FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); - FMA(sMatrix2[j][state], sPartials2[patIdx][i + j], sum2); - } - KW_LOCAL_FENCE; - } - - if (pattern < totalPatterns) - partials3[u] = sum1 * sum2; -} - diff --git a/tests/basta/test_x_state_updateInnerBastaPartials.cu b/tests/basta/test_x_state_updateInnerBastaPartials.cu index 833e4eac..120eca2d 100644 --- a/tests/basta/test_x_state_updateInnerBastaPartials.cu +++ b/tests/basta/test_x_state_updateInnerBastaPartials.cu @@ -6,7 +6,7 @@ #include #include #include -#include "../../libhmsbeagle/GPU/kernels/BeagleTensorCore_kernels.h" +#include #define NPATTERNS 1 @@ -17,13 +17,30 @@ #define KERNEL_STRING CONCAT(KERNELS_STRING_DP, STATE_SIZE) + double partials1Host_16[] = {0.137572756735608, 0.477254124591127, 0.428745979443192, 0.218426858074963, 0.529428300447762, 0.110051316907629, 0.000466231256723404, 0.912552231224254, 0.472219149814919, 0.928942704107612, 0.560180699685588, 0.217537123244256, 0.697977353585884, 0.623952043708414, 0.675538737559691, 0.843222103314474}; double partials2Host_16[] = {0.828149347100407, 0.864858906250447, 0.350309345172718, 0.155412578489631, 0.571178458165377, 0.0642696765717119, 0.767122565768659, 0.989166378509253, 0.807592692552134, 0.629633222240955, 0.759149922290817, 0.947871233569458, 0.514384098816663, 0.985400719335303, 0.437582379207015, 0.961291362997144}; double matrices_16[] = {0.318755098618567, 0.740407602395862, 0.874102377332747, 0.690571578918025, 0.701961038634181, 0.795170336496085, 0.348821813939139, 0.719763568369672, 0.895593842957169, 0.0806103076320142, 0.0329403523355722, 0.594381110742688, 0.359542798716575, 0.0881143372971565, 0.649336193455383, 0.699423802085221, 0.194982040207833, 0.0281000500544906, 0.771264882525429, 0.364646208938211, 0.0918229892849922, 0.936870703473687, 0.37309667118825, 0.0921725274529308, 0.159810511628166, 0.626376076601446, 0.692677799146622, 0.207317036110908, 0.970800938550383, 0.101070829667151, 0.944739747559652, 0.842726089060307, 0.207157449331135, 0.9937439779751, 0.821067644748837, 0.698300351854414, 0.782907243119553, 0.973109148675576, 0.77274006837979, 0.618896092521027, 0.300635955762118, 0.499129123752937, 0.656451256712899, 0.900395354721695, 0.56438325252384, 0.860600786982104, 0.49170877225697, 0.889088666066527, 0.0156871858052909, 0.81213018624112, 0.977085781516507, 0.295148188248277, 0.726798894582316, 0.916058843955398, 0.608332335250452, 0.93036122340709, 0.740293311886489, 0.477501145098358, 0.528508115326986, 0.359984828624874, 0.115733818151057, 0.162383440183476, 0.599637272302061, 0.727882955456153, 0.674989679362625, 0.862724364968017, 0.736116931773722, 0.934994351351634, 0.225678282091394, 0.0690643971320242, 0.642801651265472, 0.0847296256106347, 0.11830615834333, 0.203358805505559, 0.326086097629741, 0.730495651019737, 0.29963503475301, 0.561327135190368, 0.0538779601920396, 0.904581440845504, 0.807041098363698, 0.0369221286382526, 0.676678945310414, 0.814023691928014, 0.651314357761294, 0.901343589415774, 0.429449937539175, 0.628918903879821, 0.0323536104988307, 0.718082484090701, 0.4828297609929, 0.284520726883784, 0.997801034012809, 0.577156581915915, 0.48068075068295, 0.720765815814957, 0.403965384466574, 0.170624090125784, 0.876439346699044, 0.593427885789424, 0.769376888172701, 0.647851618006825, 0.337206641677767, 0.408452160190791, 0.0607802064623684, 0.676360174780712, 0.462037036195397, 0.347813759464771, 0.946434140671045, 0.604583449661732, 0.742167289135978, 0.0429284807760268, 0.626676751766354, 0.64561746106483, 0.673989933682606, 0.662993856705725, 0.856125113321468, 0.870688743656501, 0.308341189753264, 0.538020107662305, 0.538775310153142, 0.258829150116071, 0.407576418248937, 0.0590927223674953, 0.520942375296727, 0.434115749085322, 0.340059822658077, 0.17571780225262, 0.53029696107842, 0.975732607766986, 0.831993991043419, 0.983791405102238, 0.55990690854378, 0.710094312205911, 0.0787766603752971, 0.589558490784839, 0.673550385283306, 0.673261155607179, 0.974599071079865, 0.340651801321656, 0.0923955119214952, 0.194945273222402, 0.141258440678939, 0.639831262873486, 0.339301661355421, 0.884930776199326, 0.289516024757177, 0.126225155545399, 0.446420934982598, 0.0850053902249783, 0.395572019508108, 0.157721624244004, 0.929064428433776, 0.875803815200925, 0.36426582490094, 0.161793374456465, 0.387928539188579, 0.59461420099251, 0.174750925740227, 0.115827345522121, 0.152487984858453, 0.035088024334982, 0.983126569306478, 0.575697488617152, 0.209149486385286, 0.201207815203816, 0.110956045333296, 0.0174690247513354, 0.775130218127742, 0.497412796830758, 0.443602121900767, 0.0095906644128263, 0.198237520409748, 0.304748482536525, 0.518023134209216, 0.547018625540659, 0.526118801673874, 0.493327406002209, 0.913669281871989, 0.077805265551433, 0.929841819219291, 0.29471311555244, 0.118532110471278, 0.14501806977205, 0.436429607449099, 0.205097107682377, 0.708817657083273, 0.452689809259027, 0.0368115562014282, 0.239382016705349, 0.471294831018895, 0.290052818134427, 0.141418197192252, 0.406050314893946, 0.123278650688007, 0.793885040329769, 0.803798166336492, 0.653874792391434, 0.570221147732809, 0.238239917438477, 0.29717841069214, 0.753295891918242, 0.86618359782733, 0.0173538792878389, 0.262981676263735, 0.870678049977869, 0.235948005225509, 0.518082186812535, 0.80484891217202, 0.0546407853253186, 0.849502737633884, 0.749514716677368, 0.744097977643833, 0.543421939015388, 0.416765965521336, 0.725398944690824, 0.33459063549526, 0.478168275207281, 0.308823636500165, 0.518112531863153, 0.0382656576111913, 0.569260101066902, 0.542835577391088, 0.786958181066439, 0.191143112257123, 0.756994087249041, 0.930127918720245, 0.0538773911539465, 0.108583583729342, 0.0136260520666838, 0.0493806877639145, 0.0609419206157327, 0.157695269677788, 0.351834069937468, 0.396392825525254, 0.179786366643384, 0.822698512114584, 0.391177944373339, 0.0950508427340537, 0.0245800712145865, 0.28290240559727, 0.772152968682349, 0.14831333537586, 0.672689846949652, 0.826513405656442, 0.395902422023937, 0.837914811447263, 0.927084936527535, 0.0562630859203637, 0.810627286788076, 0.67597354692407, 0.622439709492028, 0.0950460624881089, 0.0380097457673401, 0.166366128250957, 0.867308233864605}; -double expectedPartials_16[] = {12.9769783870594, 25.8647148645348, 37.1028089862958, 26.553395775949, 28.2498807522581, 21.9503211762829, 12.9202534609119, 13.9146869398328, 14.9317655768219, 22.3371483393775, 22.9624904769671, 9.58813665982792, 11.4421566478761, 13.2079833456249, 11.7415865195237, 23.4516262593406}; +// double expectedPartials_16[] = {12.9769783870594, 25.8647148645348, 37.1028089862958, 26.553395775949, 28.2498807522581, 21.9503211762829, 12.9202534609119, 13.9146869398328, 14.9317655768219, 22.3371483393775, 22.9624904769671, 9.58813665982792, 11.4421566478761, 13.2079833456249, 11.7415865195237, 23.4516262593406}; +// double sizes_16[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; +double expectedPartials_16[] = {0.0305735894520103, 0.0609369261453154, 0.0437068636330068, 0.0417062995096076, 0.133112690863239, 0.103429332771742, 0.0608798925524519, 0.0655656368012336, 0.0703580844358418, 0.105252052132227, 0.10819864774354, 0.0451790463240147, 0.0539151394667653, 0.0311178340792188, 0.018442015033903, 0.0276258606529718}; +double expectedPartials0_16[] = {2.98143795102797, 4.65854349159416, 4.91424944421801, 4.36942401534397, 4.40172642450781, 3.8340440457869, 3.24990363502535, 3.1248633106177, 3.32838756850764, 4.25163890981238, 4.16627182696517, 2.51423509758302, 2.89070331372981, 3.34836030429601, 2.60373247077971, 4.13703860672182}; +double expectedPartials1_16[] = {4.35259046145336, 5.55210333684872, 7.55004592409327, 6.07709292636792, 6.4179092537349, 5.72510928777757, 3.97558048234593, 4.45289459303814, 4.48618595926221, 5.25377361840994, 5.51151999453036, 3.8135402170804, 3.95826046676252, 3.94461233119948, 4.50952110145465, 5.66869891454158}; +double sizes_16[] = {2, 2, 4, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4}; +double coalescent_16[] = {2}; + + +//double partials1Host_4[] = {1, 0, 0, 0}; +//double partials2Host_4[] = {0, 0, 1, 0}; +//double matrices_4[] = {0.9974538211778671, 5.991008993253103E-4, 0.0010484265738193166, 8.986513489883663E-4, 4.493256744940543E-4, 0.9976035964026984, 0.0010484265738197607, 8.986513489877002E-4, 4.49325674494082E-4, 5.991008993254314E-4, 0.9980529220771925, 8.986513489880332E-4, 4.49325674494082E-4, 5.991008993254456E-4, 0.0010484265738197607, 0.9979031468523611}; +//double sizes_4[] = {2, 2, 4, 3}; +//double expectedPartials_4[] = {0.8229333698706953, 3.7070898284225927E-4, 0.1764487818245675, 2.4713932189485474E-4}; +//double expectedPartials0_4[] = {0.9974538211778671, 4.493256744940543E-4, 4.49325674494082E-4, 4.49325674494082E-4}; +//double expectedPartials1_4[] = {0.0010484265738193166, 0.0010484265738197607, 0.9980529220771925, 0.0010484265738197607}; +double expectedProb_16[] = {212.2253}; int main() { - char functionName[] = "kernelInnerBastaPartials"; + char functionName[] = "kernelInnerBastaPartialsCoalescent"; CUmodule cudaModule; CUresult res; cudaError_t err; @@ -54,15 +71,24 @@ int main() { - double *partials1, *partials2, *partials3, *matrices1, *matrices2, *tmpAcc; + double *partials1, *partials2, *partials3, *matrices1, *matrices2, *accumulation1, *accumulation2, *sizes, *coalescent, *tmpAcc; int totalPatterns = NPATTERNS; + int intervalNumber = 1; double *resHost = (double*) malloc(sizeof(double) * STATE_SIZE * NPATTERNS); + double *accumHost1 = (double*) malloc(sizeof(double) * STATE_SIZE * NPATTERNS); + double *accumHost2 = (double*) malloc(sizeof(double) * STATE_SIZE * NPATTERNS); + float *coal = static_cast(malloc(sizeof(double) * intervalNumber)); cuMemAlloc((CUdeviceptr*)&partials1, sizeof(double) * STATE_SIZE * NPATTERNS); cuMemAlloc((CUdeviceptr*)&partials2, sizeof(double) * STATE_SIZE * NPATTERNS); cuMemAlloc((CUdeviceptr*)&partials3, sizeof(double) * STATE_SIZE * NPATTERNS); - cuMemAlloc((CUdeviceptr*)&tmpAcc, sizeof(double) * STATE_SIZE * STATE_SIZE); + cuMemAlloc((CUdeviceptr*)&accumulation1, sizeof(double) * STATE_SIZE * NPATTERNS); + cuMemAlloc((CUdeviceptr*)&accumulation2, sizeof(double) * STATE_SIZE * NPATTERNS); + cuMemAlloc((CUdeviceptr*)&sizes, sizeof(double) * STATE_SIZE * NPATTERNS); + cuMemAlloc((CUdeviceptr*)&coalescent, sizeof(float) * intervalNumber); + cuMemAlloc((CUdeviceptr*)&partials3, sizeof(double) * STATE_SIZE * NPATTERNS); + cuMemAlloc((CUdeviceptr*)&tmpAcc, sizeof(double) * 1); cuMemAlloc((CUdeviceptr*)&matrices1, sizeof(double) * STATE_SIZE * STATE_SIZE); cuMemAlloc((CUdeviceptr*)&matrices2, sizeof(double) * STATE_SIZE * STATE_SIZE); @@ -70,11 +96,11 @@ int main() { cudaMemcpy(matrices1, GET_VAR_NAME(matrices), sizeof(double) * STATE_SIZE * STATE_SIZE, cudaMemcpyHostToDevice); cudaMemcpy(matrices2, GET_VAR_NAME(matrices), sizeof(double) * STATE_SIZE * STATE_SIZE, cudaMemcpyHostToDevice); cudaMemcpy(partials1, GET_VAR_NAME(partials1Host), sizeof(double) * STATE_SIZE * NPATTERNS, cudaMemcpyHostToDevice); - err = cudaMemcpy(partials2, GET_VAR_NAME(partials2Host), sizeof(double) * STATE_SIZE * NPATTERNS, cudaMemcpyHostToDevice); - std::cerr << "Memcpy partials2 " << err << std::endl; - + cudaMemcpy(partials2, GET_VAR_NAME(partials2Host), sizeof(double) * STATE_SIZE * NPATTERNS, cudaMemcpyHostToDevice); + cudaMemcpy(sizes, GET_VAR_NAME(sizes), sizeof(double) * STATE_SIZE * NPATTERNS, cudaMemcpyHostToDevice); + cudaMemcpy(coalescent, GET_VAR_NAME(coalescent), sizeof(float) * intervalNumber, cudaMemcpyHostToDevice); void *params[] = { - &partials1, &partials3, &matrices1, &totalPatterns + &partials1, &partials2, &partials3, &matrices1, &matrices2, &accumulation1, &accumulation2, &sizes, &coalescent, &intervalNumber, &totalPatterns }; res = cuLaunchKernel(cudaFunction, 1, 1, 1, STATE_SIZE, 8, 1, 0, NULL, params, NULL); @@ -90,6 +116,15 @@ int main() { err = cudaMemcpy(resHost, partials3, NPATTERNS * STATE_SIZE * sizeof(double), cudaMemcpyDeviceToHost); std::cerr << "Memcpy partials3 " << err << std::endl; + err = cudaMemcpy(accumHost1, accumulation1, NPATTERNS * STATE_SIZE * sizeof(double), cudaMemcpyDeviceToHost); + std::cerr << "Memcpy accum1 " << err << std::endl; + + err = cudaMemcpy(accumHost2, accumulation2, NPATTERNS * STATE_SIZE * sizeof(double), cudaMemcpyDeviceToHost); + std::cerr << "Memcpy accum2 " << err << std::endl; + + err = cudaMemcpy(coal, coalescent, intervalNumber * sizeof(float), cudaMemcpyDeviceToHost); + std::cerr << "Memcpy coal " << err << std::endl; + double tolerance=1E-5; int pass = 0; for(int i = 0; i < STATE_SIZE; i++){ @@ -109,6 +144,66 @@ int main() { } else { std::cerr << "FAIL: Test for post-order kernel for state size " << STATE_SIZE << std::endl; } + + int pass0 = 0; + for(int i = 0; i < STATE_SIZE; i++){ + if(abs(GET_VAR_NAME(expectedPartials0)[i] - accumHost1[i]) > tolerance) { + if(pass0 == 0) + std::cerr << "Unequal values at positions " << std::endl; + pass0 = -1; + std::cerr << i << " Expected: " << GET_VAR_NAME(expectedPartials0)[i] << " Observed: " << accumHost1[i] << std::endl; + } + } + + if(pass0 != 0) + std::cerr << std::endl << std::endl; + + if(pass0 == 0) { + std::cerr << "PASS0: Test for post-order kernel for state size " << STATE_SIZE << std::endl; + } else { + std::cerr << "FAIL0: Test for post-order kernel for state size " << STATE_SIZE << std::endl; + } + + + int pass1 = 0; + for(int i = 0; i < STATE_SIZE; i++){ + if(abs(GET_VAR_NAME(expectedPartials1)[i] - accumHost2[i]) > tolerance) { + if(pass1 == 0) + std::cerr << "Unequal values at positions " << std::endl; + pass1 = -1; + std::cerr << i << " Expected: " << GET_VAR_NAME(expectedPartials1)[i] << " Observed: " << accumHost2[i] << std::endl; + } + } + + if(pass1 != 0) + std::cerr << std::endl << std::endl; + + if(pass1 == 0) { + std::cerr << "PASS1: Test for post-order kernel for state size " << STATE_SIZE << std::endl; + } else { + std::cerr << "FAIL1: Test for post-order kernel for state size " << STATE_SIZE << std::endl; + } + + + int pass2 = 0; + for(int i = 0; i < intervalNumber; i++){ + if(abs(GET_VAR_NAME(expectedProb)[i] - coal[i]) > tolerance) { + if(pass2 == 0) + std::cerr << "Unequal values at positions " << std::endl; + pass2 = -1; + std::cerr << i << " Expected: " << GET_VAR_NAME(expectedProb)[i] << " Observed: " << coal[i] << std::endl; + } + } + + if(pass2 != 0) + std::cerr << std::endl << std::endl; + + if(pass2 == 0) { + std::cerr << "PASS2: Test for post-order kernel for state size " << STATE_SIZE << std::endl; + } else { + std::cerr << "FAIL2: Test for post-order kernel for state size " << STATE_SIZE << std::endl; + } + cuModuleUnload(cudaModule); cudaFree(matrices1); @@ -116,7 +211,14 @@ int main() { cudaFree(partials1); cudaFree(partials2); cudaFree(partials3); + cudaFree(accumulation1); + cudaFree(accumulation2); + cudaFree(sizes); + cudaFree(coalescent); free(resHost); + free(accumHost1); + free(accumHost2); + free(coal); return pass; } \ No newline at end of file From 93a3348bf67e461cfcdaa8139d124792ab92da45 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 13 May 2024 16:28:14 -0400 Subject: [PATCH 038/421] Add BeagleGPUActionImpl.hpp --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 20 ++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 114 +++++++++++++++++++ libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 2 +- 3 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 libhmsbeagle/GPU/BeagleGPUActionImpl.hpp diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index a1c35926..2f2354bb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -50,10 +50,30 @@ class BeagleGPUActionImpl : public BeagleGPUImpl { BEAGLE_GPU_TEMPLATE class BeagleGPUActionImplFactory : public BeagleGPUImplFactory { + virtual BeagleImpl* createImpl(int tipCount, + int partialsBufferCount, + int compactBufferCount, + int stateCount, + int patternCount, + int eigenBufferCount, + int matrixBufferCount, + int categoryCount, + int scaleBufferCount, + int resourceNumber, + int pluginResourceNumber, + long long preferenceFlags, + long long requirementFlags, + int* errorCode); + + virtual const char* getName(); + virtual long long getFlags(); }; } // namspace device } // namespace gpu } // namespace beagle +#include "libhmsbeagle/GPU/BeagleGPUActionImpl.hpp" + #endif + diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp new file mode 100644 index 00000000..afb3f8cc --- /dev/null +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -0,0 +1,114 @@ + +/* + * BeagleGPUImpl.cpp + * BEAGLE + * + * Copyright 2024 Phylogenetic Likelihood Working Group + * + * This file is part of BEAGLE. + * + * BEAGLE is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * BEAGLE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with BEAGLE. If not, see + * . + * + * @author Xiang Ji + * @author Benjamin Redelings + * @author Marc Suchard + */ +#ifdef HAVE_CONFIG_H +#include "libhmsbeagle/config.h" +#endif + +#ifndef BEAGLE_BEAGLEGPUACTIONIMPL_HPP +#define BEAGLE_BEAGLEGPUACTIONIMPL_HPP + +namespace beagle { +namespace gpu { + +#ifdef CUDA + namespace cuda { +#else + namespace opencl { +#endif + +BEAGLE_GPU_TEMPLATE +BeagleImpl* BeagleGPUActionImplFactory::createImpl(int tipCount, + int partialsBufferCount, + int compactBufferCount, + int stateCount, + int patternCount, + int eigenBufferCount, + int matrixBufferCount, + int categoryCount, + int scaleBufferCount, + int resourceNumber, + int pluginResourceNumber, + long long preferenceFlags, + long long requirementFlags, + int* errorCode) { + BeagleImpl * impl = new BeagleGPUActionImpl(); + try { + *errorCode = + impl->createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, + patternCount, eigenBufferCount, matrixBufferCount, + categoryCount,scaleBufferCount, resourceNumber, pluginResourceNumber, preferenceFlags, requirementFlags); + if (*errorCode == BEAGLE_SUCCESS) { + return impl; + } + delete impl; + return NULL; + } + catch(...) + { + delete impl; + *errorCode = BEAGLE_ERROR_GENERAL; + throw; + } + delete impl; + *errorCode = BEAGLE_ERROR_GENERAL; + return NULL; +} + +#ifdef CUDA +template<> +const char* BeagleGPUActionImplFactory::getName() { + return "GPU-DP-CUDA-Action"; +} + +template<> +const char* BeagleGPUActionImplFactory::getName() { + return "GPU-SP-CUDA-Action"; +} +#elif defined(FW_OPENCL) +template<> +const char* BeagleGPUActionImplFactory::getName() { + return "DP-OpenCL-Action"; + +} +template<> +const char* BeagleGPUActionImplFactory::getName() { + return "SP-OpenCL-Action"; +} +#endif + +BEAGLE_GPU_TEMPLATE +long long BeagleGPUActionImplFactory::getFlags() { + return BeagleGPUImplFactory::getFlags() | BEAGLE_FLAG_COMPUTATION_ACTION; +} + +} // end of device namespace +} // end of gpu namespace +} // end of beagle namespace + + +#endif diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt index f2e2d114..a1ae7a5c 100644 --- a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -34,7 +34,7 @@ add_custom_command( add_library(hmsbeagle-action-cuda SHARED ../BeagleGPUActionImpl.h -# ../BeagleGPUActionImpl.hpp + ../BeagleGPUActionImpl.hpp ../GPUImplDefs.h ../GPUImplHelper.cpp ../GPUImplHelper.h From f27d6dbe6bcc966efeac478a04e844537816022f Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 13 May 2024 16:35:50 -0400 Subject: [PATCH 039/421] Make derived version of createInstance( ). --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 17 ++++++++++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 32 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 2f2354bb..8ba1a804 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -45,7 +45,22 @@ namespace gpu { #endif BEAGLE_GPU_TEMPLATE -class BeagleGPUActionImpl : public BeagleGPUImpl { +class BeagleGPUActionImpl : public BeagleGPUImpl +{ + int createInstance(int tipCount, + int partialsBufferCount, + int compactBufferCount, + int stateCount, + int patternCount, + int eigenDecompositionCount, + int matrixCount, + int categoryCount, + int scaleBufferCount, + int globalResourceNumber, + int pluginResourceNumber, + long long preferenceFlags, + long long requirementFlags); + }; BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index afb3f8cc..dacd2c41 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -41,6 +41,38 @@ namespace gpu { namespace opencl { #endif +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::createInstance(int tipCount, + int partialsBufferCount, + int compactBufferCount, + int stateCount, + int patternCount, + int eigenDecompositionCount, + int matrixCount, + int categoryCount, + int scaleBufferCount, + int globalResourceNumber, + int pluginResourceNumber, + long long preferenceFlags, + long long requirementFlags) +{ + + return BeagleGPUImpl::create_instance(tipCount, + partialsBufferCount, + compactBufferCount, + stateCount, + patternCount, + eigenDecompositionCount, + matrixCount, + categoryCount, + scaleBufferCount, + globalResourceNumber, + pluginResourceNumber, + preferenceFlags, + requirementFlags); +} + + BEAGLE_GPU_TEMPLATE BeagleImpl* BeagleGPUActionImplFactory::createImpl(int tipCount, int partialsBufferCount, From 050347edac68689774ee073a0ed5fd1ad742ab56 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 13 May 2024 16:42:28 -0400 Subject: [PATCH 040/421] Add getName and getFlags to BeagleGPUActionImpl --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 4 ++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 8ba1a804..ba4340e2 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -47,6 +47,10 @@ namespace gpu { BEAGLE_GPU_TEMPLATE class BeagleGPUActionImpl : public BeagleGPUImpl { + const char* getName(); + + long long getFlags(); + int createInstance(int tipCount, int partialsBufferCount, int compactBufferCount, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index dacd2c41..b0e684b4 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -41,6 +41,23 @@ namespace gpu { namespace opencl { #endif +BEAGLE_GPU_TEMPLATE +const char* BeagleGPUActionImpl::getName() +{ + return BeagleGPUActionImplFactory::getName(); +} + + +BEAGLE_GPU_TEMPLATE +long long BeagleGPUActionImpl::getFlags() +{ + auto flags = BeagleGPUImpl::getFlags(); + + flags |= BEAGLE_FLAG_COMPUTATION_ACTION; + + return flags; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::createInstance(int tipCount, int partialsBufferCount, From 274b52e6338610db46e18138a2977b67f0cf294b Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 13 May 2024 16:46:37 -0400 Subject: [PATCH 041/421] Disallow setTipStates( ). --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 ++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index ba4340e2..8a8fc7bb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -47,6 +47,7 @@ namespace gpu { BEAGLE_GPU_TEMPLATE class BeagleGPUActionImpl : public BeagleGPUImpl { +public: const char* getName(); long long getFlags(); @@ -65,6 +66,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl long long preferenceFlags, long long requirementFlags); + int setTipStates(int tipIndex, const int* inStates); }; BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index b0e684b4..3f2eb8f2 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -90,6 +90,20 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setTipStates(int tipIndex, const int* inStates) +{ + std::cerr<<"\nBEAGLE: When using action-based likelihood computations, setTipStates( ) is not allowed.\n"; + std::cerr<<" Use setTipPartials( ) instead.\n\n"; + + // There does not appear to be a simple method of throwing C++ exceptions into Java through the JNI. + // However, throwing this exception makes Java print a stack trace that shows where the setTipStates( ) + // call is coming from. + throw std::runtime_error("This message will not be seen"); + + std::abort(); +} + BEAGLE_GPU_TEMPLATE BeagleImpl* BeagleGPUActionImplFactory::createImpl(int tipCount, int partialsBufferCount, From 91f89f0e959ed5b49f1a3f500978bb841c94b614 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 13 May 2024 16:57:08 -0400 Subject: [PATCH 042/421] Add stubs for other relevant functions. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 25 +++++++++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 45 ++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 8a8fc7bb..f912c9c4 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -67,6 +67,31 @@ class BeagleGPUActionImpl : public BeagleGPUImpl long long requirementFlags); int setTipStates(int tipIndex, const int* inStates); + +protected: + + int upPartials(bool byPartition, + const int *operations, + int operationCount, + int cumulativeScalingIndex); + + int upPrePartials(bool byPartition, + const int *operations, + int operationCount, + int cumulativeScalingIndex); + +private: + int setEigenDecomposition(int eigenIndex, + const double *inEigenVectors, + const double *inInverseEigenVectors, + const double *inEigenValues); + + int updateTransitionMatrices(int eigenIndex, + const int* probabilityIndices, + const int* firstDerivativeIndices, + const int* secondDerivativeIndices, + const double* edgeLengths, + int count); }; BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 3f2eb8f2..e1a8a064 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -104,6 +104,51 @@ int BeagleGPUActionImpl::setTipStates(int tipIndex, const in std::abort(); } + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::upPartials(bool byPartition, + const int *operations, + int operationCount, + int cumulativeScalingIndex) +{ + return BeagleGPUImpl::upPartials(byPartition, operations, operationCount, cumulativeScalingIndex); +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::upPrePartials(bool byPartition, + const int *operations, + int operationCount, + int cumulativeScalingIndex) +{ + return BeagleGPUImpl::upPrePartials(byPartition, operations, operationCount, cumulativeScalingIndex); +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setEigenDecomposition(int eigenIndex, + const double *inEigenVectors, + const double *inInverseEigenVectors, + const double *inEigenValues) +{ + return BeagleGPUImpl::setEigenDecomposition(eigenIndex, inEigenVectors, inInverseEigenVectors, inEigenValues); +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::updateTransitionMatrices(int eigenIndex, + const int* probabilityIndices, + const int* firstDerivativeIndices, + const int* secondDerivativeIndices, + const double* edgeLengths, + int count) +{ + return BeagleGPUImpl::updateTransitionProbabilityMatrices(eigenIndex, + probabilityIndices, + firstDerivativeIndices, + secondDerivativeIndices, + edgeLengths, + count); +} + +///-------------------------------- Factory -------------------------------------/// BEAGLE_GPU_TEMPLATE BeagleImpl* BeagleGPUActionImplFactory::createImpl(int tipCount, int partialsBufferCount, From f7212734b8e5de81386255553eea21a9361a0a4d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 15 May 2024 11:25:11 -0400 Subject: [PATCH 043/421] scoreFlags needs to take 64bit flags. --- libhmsbeagle/beagle.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 15b5bc6f..2548512f 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -334,7 +334,7 @@ BeagleResourceList* beagleGetResourceList() { return rsrcList; } -int scoreFlags(long flags1, long flags2) { +int scoreFlags(long long flags1, long long flags2) { int score = 0; unsigned long trait = 1; for(int bits=0; bits<64; bits++) { From e3b6d24dff4a7d49e87ef9b469e29b013a241b31 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 15 May 2024 11:41:45 -0400 Subject: [PATCH 044/421] Change more flags from long to long long. --- libhmsbeagle/beagle.cpp | 12 ++++++------ libhmsbeagle/beagle.h | 8 ++++---- libhmsbeagle/benchmark/BeagleBenchmark.cpp | 2 +- libhmsbeagle/benchmark/BeagleBenchmark.h | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 2548512f..6d2bc2dd 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -369,7 +369,7 @@ int filterResources(int* resourceList, for(PairedList::iterator it = possibleResources->begin(); it != possibleResources->end(); ++it) { int resource = (*it).second; - long resourceFlag = rsrcList->list[resource].supportFlags; + long long resourceFlag = rsrcList->list[resource].supportFlags; if ( (resourceFlag & requirementFlags) < requirementFlags) { if(it==possibleResources->begin()){ possibleResources->remove(*(it)); @@ -401,8 +401,8 @@ int rankResourceImplementationPairs(long long preferenceFlags, for(PairedList::iterator it = possibleResources->begin(); it != possibleResources->end(); ++it) { int resource = (*it).second; - long resourceRequiredFlags = rsrcList->list[resource].requiredFlags; - long resourceSupportedFlags = rsrcList->list[resource].supportFlags; + long long resourceRequiredFlags = rsrcList->list[resource].requiredFlags; + long long resourceSupportedFlags = rsrcList->list[resource].supportFlags; int resourceScore = (*it).first; #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr,"Possible resource: %s (%d)\n",rsrcList->list[resource].name,resourceScore); @@ -410,7 +410,7 @@ int rankResourceImplementationPairs(long long preferenceFlags, for (std::list::iterator factory = implFactory->begin(); factory != implFactory->end(); factory++) { - long factoryFlags = (*factory)->getFlags(); + long long factoryFlags = (*factory)->getFlags(); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr,"\tExamining implementation: %s\n",(*factory)->getName()); #endif @@ -467,7 +467,7 @@ BeagleBenchmarkedResourceList* beagleGetBenchmarkedResourceList(int tipCount, int eigenModelCount, int partitionCount, int calculateDerivatives, - long benchmarkFlags) { + long long benchmarkFlags) { #ifdef BEAGLE_DEBUG_FP_REDUCED_PRECISION debugPatternCount = patternCount; @@ -518,7 +518,7 @@ BeagleBenchmarkedResourceList* beagleGetBenchmarkedResourceList(int tipCount, int resourceNumber; char* implName; - long benchedFlags; + long long benchedFlags; double benchmarkResultCPU; bool instOnly = false; diff --git a/libhmsbeagle/beagle.h b/libhmsbeagle/beagle.h index 146039b0..35804106 100644 --- a/libhmsbeagle/beagle.h +++ b/libhmsbeagle/beagle.h @@ -237,7 +237,7 @@ typedef struct { char* name; /**< Name of resource as a NULL-terminated character string */ char* description; /**< Description of resource as a NULL-terminated character string */ long long supportFlags; /**< Bit-flags of supported capabilities on resource */ - long requiredFlags;/**< Bit-flags that identify resource type */ + long long requiredFlags;/**< Bit-flags that identify resource type */ } BeagleResource; /** @@ -256,10 +256,10 @@ typedef struct { char* name; /**< Name of resource as a NULL-terminated character string */ char* description; /**< Description of resource as a NULL-terminated character string */ long long supportFlags; /**< Bit-flags of supported capabilities on resource */ - long requiredFlags;/**< Bit-flags that identify resource type */ + long long requiredFlags;/**< Bit-flags that identify resource type */ int returnCode; /**< Return code of for benchmark attempt (see BeagleReturnCodes) */ char* implName; /**< Name of implementation used to benchmark resource */ - long benchedFlags; /**< Bit-flags that characterize the activate + long long benchedFlags; /**< Bit-flags that characterize the activate * capabilities of the resource and implementation for this benchmark */ double benchmarkResult; /**< Benchmark result in milliseconds */ double performanceRatio; /**< Performance ratio relative to default CPU resource */ @@ -354,7 +354,7 @@ BEAGLE_DLLEXPORT BeagleBenchmarkedResourceList* beagleGetBenchmarkedResourceList int eigenModelCount, int partitionCount, int calculateDerivatives, - long benchmarkFlags); + long long benchmarkFlags); /** * @brief Create a single instance diff --git a/libhmsbeagle/benchmark/BeagleBenchmark.cpp b/libhmsbeagle/benchmark/BeagleBenchmark.cpp index 972b026a..619e51dd 100644 --- a/libhmsbeagle/benchmark/BeagleBenchmark.cpp +++ b/libhmsbeagle/benchmark/BeagleBenchmark.cpp @@ -135,7 +135,7 @@ int benchmarkResource(int resource, long long requirementFlags, int* resourceNumber, char** implName, - long* benchedFlags, + long long* benchedFlags, double* benchmarkResult, bool instOnly) { diff --git a/libhmsbeagle/benchmark/BeagleBenchmark.h b/libhmsbeagle/benchmark/BeagleBenchmark.h index 036be002..5def04a0 100644 --- a/libhmsbeagle/benchmark/BeagleBenchmark.h +++ b/libhmsbeagle/benchmark/BeagleBenchmark.h @@ -73,7 +73,7 @@ int benchmarkResource(int resource, long long requirementFlags, int* resourceNumber, char** implName, - long* benchedFlags, + long long* benchedFlags, double* benchmarkResult, bool instOnly); From 4e106c3e6e5fdca898df1c04af7de95cbc43d7f3 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 15 May 2024 12:57:01 -0400 Subject: [PATCH 045/421] Allow constructing a string representation of the flags for debugging. --- examples/actiontest/actiontest.cpp | 91 ++++++++++++++++++++---------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/examples/actiontest/actiontest.cpp b/examples/actiontest/actiontest.cpp index 9fcc9b60..84c11f6c 100644 --- a/examples/actiontest/actiontest.cpp +++ b/examples/actiontest/actiontest.cpp @@ -1,12 +1,13 @@ -#include -#include -#include +#include +#include +#include #include #include #include #include #include #include +#include typedef Eigen::Matrix MatrixXd; typedef Eigen::Map MapType; @@ -104,33 +105,61 @@ double* getPartials(char *sequence, int repeats) { return partials; } -void printFlags(long inFlags) { - if (inFlags & BEAGLE_FLAG_PROCESSOR_CPU) fprintf(stdout, " PROCESSOR_CPU"); - if (inFlags & BEAGLE_FLAG_PROCESSOR_GPU) fprintf(stdout, " PROCESSOR_GPU"); - if (inFlags & BEAGLE_FLAG_PROCESSOR_FPGA) fprintf(stdout, " PROCESSOR_FPGA"); - if (inFlags & BEAGLE_FLAG_PROCESSOR_CELL) fprintf(stdout, " PROCESSOR_CELL"); - if (inFlags & BEAGLE_FLAG_PRECISION_DOUBLE) fprintf(stdout, " PRECISION_DOUBLE"); - if (inFlags & BEAGLE_FLAG_PRECISION_SINGLE) fprintf(stdout, " PRECISION_SINGLE"); - if (inFlags & BEAGLE_FLAG_COMPUTATION_ASYNCH) fprintf(stdout, " COMPUTATION_ASYNCH"); - if (inFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) fprintf(stdout, " COMPUTATION_SYNCH"); - if (inFlags & BEAGLE_FLAG_EIGEN_REAL) fprintf(stdout, " EIGEN_REAL"); - if (inFlags & BEAGLE_FLAG_EIGEN_COMPLEX) fprintf(stdout, " EIGEN_COMPLEX"); - if (inFlags & BEAGLE_FLAG_SCALING_MANUAL) fprintf(stdout, " SCALING_MANUAL"); - if (inFlags & BEAGLE_FLAG_SCALING_AUTO) fprintf(stdout, " SCALING_AUTO"); - if (inFlags & BEAGLE_FLAG_SCALING_ALWAYS) fprintf(stdout, " SCALING_ALWAYS"); - if (inFlags & BEAGLE_FLAG_SCALING_DYNAMIC) fprintf(stdout, " SCALING_DYNAMIC"); - if (inFlags & BEAGLE_FLAG_SCALERS_RAW) fprintf(stdout, " SCALERS_RAW"); - if (inFlags & BEAGLE_FLAG_SCALERS_LOG) fprintf(stdout, " SCALERS_LOG"); - if (inFlags & BEAGLE_FLAG_VECTOR_NONE) fprintf(stdout, " VECTOR_NONE"); - if (inFlags & BEAGLE_FLAG_VECTOR_SSE) fprintf(stdout, " VECTOR_SSE"); - if (inFlags & BEAGLE_FLAG_VECTOR_AVX) fprintf(stdout, " VECTOR_AVX"); - if (inFlags & BEAGLE_FLAG_THREADING_NONE) fprintf(stdout, " THREADING_NONE"); - if (inFlags & BEAGLE_FLAG_THREADING_CPP) fprintf(stdout, " THREADING_CPP"); - if (inFlags & BEAGLE_FLAG_THREADING_OPENMP) fprintf(stdout, " THREADING_OPENMP"); - if (inFlags & BEAGLE_FLAG_FRAMEWORK_CPU) fprintf(stdout, " FRAMEWORK_CPU"); - if (inFlags & BEAGLE_FLAG_FRAMEWORK_CUDA) fprintf(stdout, " FRAMEWORK_CUDA"); - if (inFlags & BEAGLE_FLAG_FRAMEWORK_OPENCL) fprintf(stdout, " FRAMEWORK_OPENCL"); - if (inFlags & BEAGLE_FLAG_COMPUTATION_ACTION) fprintf(stdout, " COMPUTATION_ACTION"); + +std::ostream& operator<<(std::ostream& o, BeagleFlags inFlags) +{ + if (inFlags & BEAGLE_FLAG_PRECISION_SINGLE) o<<" PRECISION_SINGLE"; + if (inFlags & BEAGLE_FLAG_PRECISION_DOUBLE) o<<" PRECISION_DOUBLE"; + if (inFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) o<<" COMPUTATION_SYNCH"; + if (inFlags & BEAGLE_FLAG_COMPUTATION_ASYNCH) o<<" COMPUTATION_ASYNCH"; + if (inFlags & BEAGLE_FLAG_EIGEN_REAL) o<<" EIGEN_REAL"; + if (inFlags & BEAGLE_FLAG_EIGEN_COMPLEX) o<<" EIGEN_COMPLEX"; + + if (inFlags & BEAGLE_FLAG_SCALING_MANUAL) o<<" SCALING_MANUAL"; + if (inFlags & BEAGLE_FLAG_SCALING_AUTO) o<<" SCALING_AUTO"; + if (inFlags & BEAGLE_FLAG_SCALING_ALWAYS) o<<" SCALING_ALWAYS"; + if (inFlags & BEAGLE_FLAG_SCALING_DYNAMIC) o<<" SCALING_DYNAMIC"; + if (inFlags & BEAGLE_FLAG_SCALERS_RAW) o<<" SCALERS_RAW"; + if (inFlags & BEAGLE_FLAG_SCALERS_LOG) o<<" SCALERS_LOG"; + + if (inFlags & BEAGLE_FLAG_INVEVEC_STANDARD) o<<" INVENV_STANDARD"; + if (inFlags & BEAGLE_FLAG_INVEVEC_TRANSPOSED) o<<" INVENV_STANDARD"; + + if (inFlags & BEAGLE_FLAG_VECTOR_SSE) o<<" VECTOR_SSE"; + if (inFlags & BEAGLE_FLAG_VECTOR_AVX) o<<" VECTOR_AVX"; + if (inFlags & BEAGLE_FLAG_VECTOR_NONE) o<<" VECTOR_NONE"; + + if (inFlags & BEAGLE_FLAG_THREADING_CPP) o<<" THREADING_CPP"; + if (inFlags & BEAGLE_FLAG_THREADING_OPENMP) o<<" THREADING_OPENMP"; + if (inFlags & BEAGLE_FLAG_THREADING_NONE) o<<" THREADING_NONE"; + + if (inFlags & BEAGLE_FLAG_PROCESSOR_CPU) o<<" PROCESSOR_CPU"; + if (inFlags & BEAGLE_FLAG_PROCESSOR_GPU) o<<" PROCESSOR_GPU"; + if (inFlags & BEAGLE_FLAG_PROCESSOR_FPGA) o<<" PROCESSOR_FPGA"; + if (inFlags & BEAGLE_FLAG_PROCESSOR_CELL) o<<" PROCESSOR_CELL"; + if (inFlags & BEAGLE_FLAG_PROCESSOR_PHI) o<<" PROCESSOR_PHI"; + if (inFlags & BEAGLE_FLAG_PROCESSOR_OTHER) o<<" PROCESSOR_OTHER"; + + if (inFlags & BEAGLE_FLAG_FRAMEWORK_CUDA) o<<" FRAMEWORK_CUDA"; + if (inFlags & BEAGLE_FLAG_FRAMEWORK_OPENCL) o<<" FRAMEWORK_OPENCL"; + if (inFlags & BEAGLE_FLAG_FRAMEWORK_CPU) o<<" FRAMEWORK_CPU"; + + if (inFlags & BEAGLE_FLAG_PARALLELOPS_STREAMS) o<<" PARALLEL_OPS_STREAMS"; + if (inFlags & BEAGLE_FLAG_PARALLELOPS_GRID) o<<" PARALLEL_OPS_GRID"; + + if (inFlags & BEAGLE_FLAG_PREORDER_TRANSPOSE_MANUAL) o<<" PREORDER_TRANSPOSE_MANUAL"; + if (inFlags & BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO) o<<" PREORDER_TRANSPOSE_MANUAL"; + + if (inFlags & BEAGLE_FLAG_COMPUTATION_ACTION) o<<" COMPUTATION_ACTION"; + + return o; +} + +std::string showFlags(BeagleFlags flags) +{ + std::ostringstream o; + o<list[i].name); fprintf(stdout, "\t\tDesc : %s\n", rList->list[i].description); fprintf(stdout, "\t\tFlags:"); - printFlags(rList->list[i].supportFlags); + fprintf(stdout, showFlags( BeagleFlags(rList->list[i].supportFlags ) ).c_str() ); fprintf(stdout, "\n"); } fprintf(stdout, "\n"); From 23292edfaf3cf965e636c4521718bf050967c5c0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 15 May 2024 13:03:10 -0400 Subject: [PATCH 046/421] Convert the rest of fprintf to iostreams in actiontest. --- examples/actiontest/actiontest.cpp | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/examples/actiontest/actiontest.cpp b/examples/actiontest/actiontest.cpp index 84c11f6c..fa281ccf 100644 --- a/examples/actiontest/actiontest.cpp +++ b/examples/actiontest/actiontest.cpp @@ -167,15 +167,13 @@ int main( int argc, const char* argv[] ) // print resource list BeagleResourceList* rList; rList = beagleGetResourceList(); - fprintf(stdout, "Available resources:\n"); + std::cout<<"Available resources:\n"; for (int i = 0; i < rList->length; i++) { - fprintf(stdout, "\tResource %i:\n\t\tName : %s\n", i, rList->list[i].name); - fprintf(stdout, "\t\tDesc : %s\n", rList->list[i].description); - fprintf(stdout, "\t\tFlags:"); - fprintf(stdout, showFlags( BeagleFlags(rList->list[i].supportFlags ) ).c_str() ); - fprintf(stdout, "\n"); + std::cout<<"\tResource "<list[i].name<<"\n"; + std::cout<<"\t\tDesc : "<list[i].description<<"\n"; + std::cout<<"\t\tFlags:"<list[i].supportFlags )<<"\n"; } - fprintf(stdout, "\n"); + std::cout<<"\n"; bool scaling = true; @@ -269,17 +267,17 @@ int main( int argc, const char* argv[] ) requirementFlags, /**< Bit-flags indicating required implementation characteristics, see BeagleFlags (input) */ &instDetails); if (instance < 0) { - fprintf(stderr, "Failed to obtain BEAGLE instance\n\n"); + std::cerr<<"Failed to obtain BEAGLE instance\n\n"; exit(1); } int rNumber = instDetails.resourceNumber; - fprintf(stdout, "Using resource %i:\n", rNumber); - fprintf(stdout, "\tRsrc Name : %s\n",instDetails.resourceName); - fprintf(stdout, "\tImpl Name : %s\n", instDetails.implName); - fprintf(stdout, "\tImpl Desc : %s\n", instDetails.implDescription); - fprintf(stdout, "\n"); + std::cout<<"Using resource "< Date: Wed, 15 May 2024 13:20:53 -0400 Subject: [PATCH 047/421] Make --gpu work in action test. --- examples/actiontest/actiontest.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/examples/actiontest/actiontest.cpp b/examples/actiontest/actiontest.cpp index fa281ccf..74b2fc97 100644 --- a/examples/actiontest/actiontest.cpp +++ b/examples/actiontest/actiontest.cpp @@ -214,7 +214,10 @@ int main( int argc, const char* argv[] ) bool useTipStates = false; int whichDevice = -1; - if (useGpu) { + if (useGpu) + { + useSSE = false; + if (argc > 2) { whichDevice = atol(argv[2]); if (whichDevice < 0) { @@ -225,12 +228,19 @@ int main( int argc, const char* argv[] ) BeagleInstanceDetails instDetails; +// long requirementFlags = BEAGLE_FLAG_EIGEN_REAL; + long long requirementFlags = 0; long long preferenceFlags = BEAGLE_FLAG_COMPUTATION_ACTION; if (useGpu) { - preferenceFlags |= BEAGLE_FLAG_PROCESSOR_GPU; + requirementFlags |= BEAGLE_FLAG_PROCESSOR_GPU; } else { - preferenceFlags |= BEAGLE_FLAG_PROCESSOR_CPU; + requirementFlags |= BEAGLE_FLAG_PROCESSOR_CPU; + if (useSSE) { + requirementFlags |= BEAGLE_FLAG_VECTOR_SSE; + } else { + requirementFlags |= BEAGLE_FLAG_VECTOR_NONE; + } } if (singlePrecision) { @@ -239,14 +249,6 @@ int main( int argc, const char* argv[] ) preferenceFlags |= BEAGLE_FLAG_PRECISION_DOUBLE; } -// long requirementFlags = BEAGLE_FLAG_EIGEN_REAL; - long long requirementFlags = BEAGLE_FLAG_FRAMEWORK_CPU; - if (useSSE) { - requirementFlags |= BEAGLE_FLAG_VECTOR_SSE; - } else { - requirementFlags |= BEAGLE_FLAG_VECTOR_NONE; - } - if (useThreading) preferenceFlags |= BEAGLE_FLAG_THREADING_CPP; From 80879e42b801f20ef182af93987a35fa5020c4c9 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 16 May 2024 11:10:33 -0500 Subject: [PATCH 048/421] let action plugin call correct implFactory which generates new errors --- libhmsbeagle/GPU/ActionCUDAPlugin.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp index a66794b1..9488990e 100644 --- a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp +++ b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp @@ -63,10 +63,9 @@ namespace beagle { if (anyGPUSupportsCUDA) { using namespace cuda; if (anyGPUSupportsDP) { - // TODO Uncomment when working - beagleFactories.push_back(new BeagleGPUImplFactory()); + beagleFactories.push_back(new BeagleGPUActionImplFactory()); } - beagleFactories.push_back(new BeagleGPUImplFactory()); + beagleFactories.push_back(new BeagleGPUActionImplFactory()); } } From 577b292c37723671ede7bc3df4798a61dc021674 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 16 May 2024 11:32:42 -0500 Subject: [PATCH 049/421] add more search path for debug --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed856e07..770931ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,7 +176,7 @@ elseif(UNIX) SET(CMAKE_INSTALL_RPATH "\$ORIGIN") # $ORIGIN/{CPU,GPU,JNI} is so that libhmsbeagle.so can find the plugins in their build location. # $ORIGIN/../libhmsbeagle is so that examples/* can find libhmsbeagle.so - SET(CMAKE_BUILD_RPATH "\$ORIGIN:\$ORIGIN/CPU:\$ORIGIN/GPU:\$ORIGIN/JNI:\$ORIGIN/../libhmsbeagle") + SET(CMAKE_BUILD_RPATH "\$ORIGIN:\$ORIGIN/CPU:\$ORIGIN/GPU:\$ORIGIN/GPU/CMake_Action:\$ORIGIN/GPU/CMake_CUDA:\$ORIGIN/GPU/CMake_OpenCL:\$ORIGIN/JNI:\$ORIGIN/../libhmsbeagle") endif(APPLE) if (BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH) From 00d6882455e697b6ff0063fecb6dcf55c801c26a Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 16 May 2024 11:50:30 -0500 Subject: [PATCH 050/421] fix plugin load issue --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 33 ++++++++++-------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index e1a8a064..bf9d8a1f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -74,19 +74,19 @@ int BeagleGPUActionImpl::createInstance(int tipCount, long long requirementFlags) { - return BeagleGPUImpl::create_instance(tipCount, - partialsBufferCount, - compactBufferCount, - stateCount, - patternCount, - eigenDecompositionCount, - matrixCount, - categoryCount, - scaleBufferCount, - globalResourceNumber, - pluginResourceNumber, - preferenceFlags, - requirementFlags); + return BeagleGPUImpl::createInstance(tipCount, + partialsBufferCount, + compactBufferCount, + stateCount, + patternCount, + eigenDecompositionCount, + matrixCount, + categoryCount, + scaleBufferCount, + globalResourceNumber, + pluginResourceNumber, + preferenceFlags, + requirementFlags); } @@ -140,12 +140,7 @@ int BeagleGPUActionImpl::updateTransitionMatrices(int eigenI const double* edgeLengths, int count) { - return BeagleGPUImpl::updateTransitionProbabilityMatrices(eigenIndex, - probabilityIndices, - firstDerivativeIndices, - secondDerivativeIndices, - edgeLengths, - count); + return BEAGLE_ERROR_NO_IMPLEMENTATION; } ///-------------------------------- Factory -------------------------------------/// From 97bb092688fd4d92cff0940ef16fba42f4ad6d74 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 16 May 2024 11:51:36 -0500 Subject: [PATCH 051/421] let beagle try to load action-cuda --- libhmsbeagle/beagle.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 6d2bc2dd..5f624656 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -30,6 +30,9 @@ #include "libhmsbeagle/config.h" #endif +#define BEAGLE_DEBUG_FLOW +#define BEAGLE_DEBUG_LOAD + #ifdef _WIN32 #include #endif @@ -179,6 +182,19 @@ void beagleLoadPlugins(void) { plugins->push_back(actionplug); }catch(beagle::plugin::SharedLibraryException sle){} + try{ +#ifdef BEAGLE_DEBUG_LOAD + std::cerr << "Loading hmsbeagle-action-cuda" << std::endl; +#endif + beagle::plugin::Plugin* actiongpuplug = pm.findPlugin("hmsbeagle-action-cuda"); + plugins->push_back(actiongpuplug); + }catch(beagle::plugin::SharedLibraryException sle){ +#ifdef BEAGLE_DEBUG_LOAD + std::cerr << "Unable to load hmsbeagle-action-cuda: " << sle.getError() << std::endl; +#endif + } + + try{ beagle::plugin::Plugin* avxplug = pm.findPlugin("hmsbeagle-cpu-avx"); plugins->push_back(avxplug); From fdc432b67db2499f9528592e351e387f580fc1a3 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 16 May 2024 11:52:33 -0500 Subject: [PATCH 052/421] remove lazy macro define --- libhmsbeagle/beagle.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 5f624656..43a0697d 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -30,9 +30,6 @@ #include "libhmsbeagle/config.h" #endif -#define BEAGLE_DEBUG_FLOW -#define BEAGLE_DEBUG_LOAD - #ifdef _WIN32 #include #endif From 1cfdabf0f20387e140790836bc4f38b40f6f2b3f Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 16 May 2024 12:56:30 -0500 Subject: [PATCH 053/421] correct instance name --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 ++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index f912c9c4..397698b5 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -81,6 +81,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int cumulativeScalingIndex); private: + char* getInstanceName(); + int setEigenDecomposition(int eigenIndex, const double *inEigenVectors, const double *inInverseEigenVectors, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index bf9d8a1f..08a33da3 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -89,6 +89,27 @@ int BeagleGPUActionImpl::createInstance(int tipCount, requirementFlags); } +#ifdef CUDA +template<> +char* BeagleGPUActionImpl::getInstanceName() { + return (char*) "Action-CUDA-Double"; +} + +template<> +char* BeagleGPUActionImpl::getInstanceName() { + return (char*) "Action-CUDA-Single"; +} +#elif defined(FW_OPENCL) + template<> +char* BeagleGPUActionImpl::getInstanceName() { + return (char*) "Action-OpenCL-Double"; +} + +template<> +char* BeagleGPUActionImpl::getInstanceName() { + return (char*) "Action-OpenCL-Single"; +} +#endif BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setTipStates(int tipIndex, const int* inStates) From f13242a725083ea75bea4509152a9dbc589cf340 Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 17 May 2024 19:13:31 -0500 Subject: [PATCH 054/421] not sure why inheritence did not work as expected for showing correct implName --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 ++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 397698b5..4f192c2d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -52,6 +52,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl long long getFlags(); + int getInstanceDetails(BeagleInstanceDetails* retunInfo); + int createInstance(int tipCount, int partialsBufferCount, int compactBufferCount, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 08a33da3..7654d9f7 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -89,6 +89,13 @@ int BeagleGPUActionImpl::createInstance(int tipCount, requirementFlags); } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::getInstanceDetails(BeagleInstanceDetails* returnInfo) { + BeagleGPUImpl::getInstanceDetails(returnInfo); + returnInfo->implName = getInstanceName(); + return BEAGLE_SUCCESS; +} + #ifdef CUDA template<> char* BeagleGPUActionImpl::getInstanceName() { From 5b214a5d31d6c400df8210a1a8e7280444a3ed4a Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 17 May 2024 19:23:00 -0500 Subject: [PATCH 055/421] let DEBUG_FLOW not complain --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 4b65886c..fda8f9d4 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -648,7 +648,7 @@ namespace beagle { << " edge length multiplier: " << edgeLengths[i] << " edgeMultiplier: "<< gEdgeMultipliers[nodeIndex * kCategoryCount + category] << " nodeIndex: "<< nodeIndex - << std::endl << gScaledQs[nodeIndex][category]< Date: Fri, 24 May 2024 16:51:14 -0500 Subject: [PATCH 056/421] change all private members of BeagleGPUImpl to protected and use them in BeagleGPUActionImpl --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 117 +++++++++++++++++++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 3 +- libhmsbeagle/GPU/BeagleGPUImpl.h | 2 +- 3 files changed, 120 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 4f192c2d..9abef4d5 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -72,6 +72,123 @@ class BeagleGPUActionImpl : public BeagleGPUImpl protected: + using BeagleGPUImpl::kInitialized; + using BeagleGPUImpl::kTipCount; + using BeagleGPUImpl::kPartialsBufferCount; + using BeagleGPUImpl::kCompactBufferCount; + using BeagleGPUImpl::kStateCount; + using BeagleGPUImpl::kPatternCount; + using BeagleGPUImpl::kEigenDecompCount; + using BeagleGPUImpl::kMatrixCount; + using BeagleGPUImpl::kCategoryCount; + using BeagleGPUImpl::kScaleBufferCount; + using BeagleGPUImpl::kExtraMatrixCount; + using BeagleGPUImpl::kPartitionCount; + using BeagleGPUImpl::kMaxPartitionCount; + using BeagleGPUImpl::kPartitionsInitialised; + using BeagleGPUImpl::kPatternsReordered; + + using BeagleGPUImpl::resourceNumber; + + using BeagleGPUImpl::kTipPartialsBufferCount; + using BeagleGPUImpl::kBufferCount; + + using BeagleGPUImpl::kInternalPartialsBufferCount; + using BeagleGPUImpl::kPaddedStateCount; + + using BeagleGPUImpl::gpu; + using BeagleGPUImpl::kDeviceType; + using BeagleGPUImpl::kDeviceCode; + + using BeagleGPUImpl::kPaddedPatternCount; + using BeagleGPUImpl::kResultPaddedPatterns; + + using BeagleGPUImpl::kFlags; + using BeagleGPUImpl::kScaleBufferSize; + + using BeagleGPUImpl::kSumSitesBlockSize; + using BeagleGPUImpl::kSumSitesBlockCount; + + using BeagleGPUImpl::kPartialsSize; + using BeagleGPUImpl::kMatrixSize; + + using BeagleGPUImpl::kEigenValuesSize; + using BeagleGPUImpl::kLastCompactBufferIndex; + using BeagleGPUImpl::kLastTipPartialsBufferIndex; + using BeagleGPUImpl::kernels; + + using BeagleGPUImpl::hWeightsCache; + using BeagleGPUImpl::hFrequenciesCache; + using BeagleGPUImpl::hPartialsCache; + using BeagleGPUImpl::hStatesCache; + using BeagleGPUImpl::hLogLikelihoodsCache; + using BeagleGPUImpl::hMatrixCache; + using BeagleGPUImpl::dEvec; + using BeagleGPUImpl::dIevc; + using BeagleGPUImpl::dEigenValues; + using BeagleGPUImpl::dWeights; + using BeagleGPUImpl::dFrequencies; + + using BeagleGPUImpl::dMatrices; + using BeagleGPUImpl::kIndexOffsetMat; + using BeagleGPUImpl::dMatricesOrigin; + using BeagleGPUImpl::dScalingFactors; + using BeagleGPUImpl::dScalingFactorsMaster; + using BeagleGPUImpl::hRescalingTrigger; + using BeagleGPUImpl::dRescalingTrigger; + using BeagleGPUImpl::kEvecOffset; + using BeagleGPUImpl::kEvalOffset; + using BeagleGPUImpl::kWeightsOffset; + using BeagleGPUImpl::kFrequenciesOffset; + using BeagleGPUImpl::dIntegrationTmp; + using BeagleGPUImpl::dPatternWeights; + using BeagleGPUImpl::dSumLogLikelihood; + using BeagleGPUImpl::dPartialsTmp; + + using BeagleGPUImpl::kDerivBuffersInitialised; + using BeagleGPUImpl::kMultipleDerivativesLength; + + using BeagleGPUImpl::dPartials; + using BeagleGPUImpl::dPartialsOrigin; + using BeagleGPUImpl::hPartialsOffsets; + using BeagleGPUImpl::kIndexOffsetPat; + + using BeagleGPUImpl::dStatesOrigin; + using BeagleGPUImpl::dStates; + using BeagleGPUImpl::hStatesOffsets; + using BeagleGPUImpl::kIndexOffsetStates; + using BeagleGPUImpl::dCompactBuffers; + using BeagleGPUImpl::dTipPartialsBuffers; + using BeagleGPUImpl::hStreamIndices; + + using BeagleGPUImpl::dBranchLengths; + using BeagleGPUImpl::dDistanceQueue; + using BeagleGPUImpl::hDistanceQueue; + using BeagleGPUImpl::dPtrQueue; + using BeagleGPUImpl::hPtrQueue; + + using BeagleGPUImpl::dDerivativeQueue; + using BeagleGPUImpl::hDerivativeQueue; + + using BeagleGPUImpl::kSitesPerIntegrateBlock; + using BeagleGPUImpl::kSitesPerBlock; + using BeagleGPUImpl::kNumPatternBlocks; + using BeagleGPUImpl::kPaddedPartitionBlocks; + using BeagleGPUImpl::kMaxPaddedPartitionBlocks; + using BeagleGPUImpl::kPaddedPartitionIntegrateBlocks; + using BeagleGPUImpl::kMaxPaddedPartitionIntegrateBlocks; + using BeagleGPUImpl::kUsingMultiGrid; + using BeagleGPUImpl::hPartitionOffsets; + using BeagleGPUImpl::hCategoryRates; + using BeagleGPUImpl::hPatternWeightsCache; + using BeagleGPUImpl::dMaxScalingFactors; + using BeagleGPUImpl::dIndexMaxScalingFactors; + using BeagleGPUImpl::dAccumulatedScalingFactors; + using BeagleGPUImpl::kUsingAutoTranspose; + + void allocateMultiGridBuffers(); + + int upPartials(bool byPartition, const int *operations, int operationCount, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 7654d9f7..8604c3e6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -74,7 +74,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, long long requirementFlags) { - return BeagleGPUImpl::createInstance(tipCount, + int parentCode = BeagleGPUImpl::createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, @@ -87,6 +87,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, pluginResourceNumber, preferenceFlags, requirementFlags); + return BEAGLE_SUCCESS; } BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index d2761d88..2ca94b46 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -63,7 +63,7 @@ BEAGLE_GPU_TEMPLATE class BeagleGPUImpl : public BeagleImpl { -private: +protected: GPUInterface* gpu; KernelLauncher* kernels; From b01ec58859690e24d6ae03f53d1d087c1ddb540e Mon Sep 17 00:00:00 2001 From: yucais Date: Wed, 29 May 2024 09:53:24 -0700 Subject: [PATCH 057/421] first working reduction kernels --- libhmsbeagle/GPU/BeagleGPUImpl.h | 10 +- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 129 ++++++++++-- libhmsbeagle/GPU/KernelLauncher.cpp | 71 ++++++- libhmsbeagle/GPU/KernelLauncher.h | 14 ++ libhmsbeagle/GPU/kernels/kernelsAll.cu | 195 ++++++++++++------ .../test_x_state_updateInnerBastaPartials.cu | 16 +- 6 files changed, 344 insertions(+), 91 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 92b7514f..38713f2a 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -212,8 +212,14 @@ class BeagleGPUImpl : public BeagleImpl { int* hStreamIndices; - GPUPtr gBastaBuffers; - GPUPtr gCoalescentBuffers; + GPUPtr dBastaBuffers; + GPUPtr dBastaE; + GPUPtr dBastaF; + GPUPtr dBastaG; + GPUPtr dBastaH; + GPUPtr dBastaLogL; + GPUPtr dBastaDistance; + GPUPtr dCoalescentBuffers; int kCoalescentBufferLength; int kCoalescentBufferCount; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index c7eb193e..561309a0 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -43,9 +43,8 @@ namespace gpu { #else namespace opencl { #endif - -BEAGLE_GPU_TEMPLATE -BeagleGPUImpl::BeagleGPUImpl() { + BEAGLE_GPU_TEMPLATE + BeagleGPUImpl::BeagleGPUImpl() { gpu = NULL; kernels = NULL; @@ -77,8 +76,14 @@ BeagleGPUImpl::BeagleGPUImpl() { dMaxScalingFactors = (GPUPtr)NULL; dIndexMaxScalingFactors = (GPUPtr)NULL; - gBastaBuffers = (GPUPtr)NULL; - gCoalescentBuffers = (GPUPtr)NULL; + dBastaE = (GPUPtr)NULL; + dBastaF = (GPUPtr)NULL; + dBastaG = (GPUPtr)NULL; + dBastaH = (GPUPtr)NULL; + dBastaLogL = (GPUPtr)NULL; + dBastaDistance = (GPUPtr)NULL; + dBastaBuffers = (GPUPtr)NULL; + dCoalescentBuffers = (GPUPtr)NULL; dEigenValues = NULL; dEvec = NULL; @@ -247,6 +252,15 @@ BeagleGPUImpl::~BeagleGPUImpl() { free(hPartialsOffsets); free(hStatesOffsets); + + gpu->FreeMemory(dBastaBuffers); + gpu->FreeMemory(dBastaE); + gpu->FreeMemory(dBastaF); + gpu->FreeMemory(dBastaG); + gpu->FreeMemory(dBastaH); + gpu->FreeMemory(dBastaLogL); + gpu->FreeMemory(dBastaDistance); + gpu->FreeHostMemory(hPtrQueue); gpu->FreeHostMemory(hDerivativeQueue); @@ -2220,8 +2234,13 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, kCoalescentBufferLength = bufferLength; kCoalescentBufferCount = bufferCount; - gCoalescentBuffers = gpu->AllocateMemory(kCoalescentBufferCount * kCoalescentBufferLength * sizeof(Real)); - gBastaBuffers = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * 4 * sizeof(Real)); + dCoalescentBuffers = gpu->AllocateMemory(kCoalescentBufferCount * kCoalescentBufferLength * sizeof(Real)); + dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaLogL = gpu->AllocateMemory(1 * sizeof(Real)); + dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); @@ -2259,7 +2278,7 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - GPUPtr coalescent = gCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2305,7 +2324,13 @@ int BeagleGPUImpl::updateInnerBastaPartials(const int* opera GPUPtr accumulation1 = dPartials[accumulation1PartialIndex]; GPUPtr accumulation2 = dPartials[accumulation2PartialIndex]; kernels->InnerBastaPartialsCoalescent(partials1, partials2, partials3, - matrices1, matrices2, sizes, accumulation1, accumulation2, coalescent, intervalNumber, 1, child2Index); + matrices1, matrices2, accumulation1, accumulation2, sizes, coalescent, intervalNumber, 1, child2Index); + // Real r = 0; + // fprintf(stderr, "The op is: %d\n", op); + // fprintf(stderr, "The index is: %d\n", child1TransMatIndex); + // fprintf(stderr,"root partials = \n"); + // gpu->PrintfDeviceVector(partials3, 4, r); + // gpu->PrintfDeviceVector(sizes, 4, r); } #ifdef BEAGLE_DEBUG_SYNCH @@ -2349,15 +2374,93 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); #endif - int returnCode = BEAGLE_ERROR_GENERAL; - - fprintf(stderr, "BeagleGPUImpl::accumulateBastaPartials\n"); + + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + const GPUPtr sizes = dFrequencies[populationSizesIndex]; + Real* zeroes = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaE, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaF, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaG, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaH, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { + // TODO execute in parallel (no race conditions) + const int start = intervalStarts[interval]; + const int end = intervalStarts[interval + 1]; + for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int child1PartialIndex = operations[op * numOps + 1]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + GPUPtr startPartials1 = dPartials[child1PartialIndex]; + GPUPtr startPartials2 = dPartials[child2PartialIndex]; + GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; + GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; + Real r = 0; + kernels->reduceWithinInterval(dBastaE, dBastaF, dBastaG, dBastaH, + startPartials1, startPartials2, + endPartials1, endPartials2, + intervalNumber, child2PartialIndex); + + // fprintf(stderr, "startPartials1:\n"); + // gpu->PrintfDeviceVector(startPartials1, kPaddedStateCount, r); + // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + } + } + Real* hBastaDistance = (Real*) gpu->CallocHost(sizeof(Real), kCoalescentBufferLength); + for (int i = 0; i < kCoalescentBufferLength; ++i) { + hBastaDistance[i] = (Real) intervalLengths[i]; + } + Real r = 0; + Real* hBastaLogL = (Real*) gpu->CallocHost(sizeof(Real), 1); + gpu->MemcpyHostToDevice(dBastaLogL, hBastaLogL, sizeof(Real) * 1); + gpu->MemcpyHostToDevice(dBastaDistance, hBastaDistance, sizeof(Real) * kCoalescentBufferLength); + + + for (int i = 0; i < intervalStartsCount - 1; ++i) { + // TODO execute in parallel + Real r = 0; + const int op = intervalStarts[i]; + const int numOps = BEAGLE_BASTA_OP_COUNT; + const int intervalNumber = operations[op * numOps + 7]; + Real intLength = (Real) intervalLengths[i]; + + kernels->reduceAcrossIntervals(dBastaE, dBastaF, dBastaG, dBastaH, dBastaDistance, dBastaLogL, sizes, coalescent, + intervalNumber); + // fprintf(stderr, "The interval is: %d\n", i); + // fprintf(stderr, "The interval length is: %f\n", hBastaDistance[i]); + // fprintf(stderr, "logL:\n"); + // gpu->PrintfDeviceVector(dBastaLogL, 1, r); + // fprintf(stderr, "e:\n"); + // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "f:\n"); + // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "g:\n"); + // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "sizes:\n"); + // gpu->PrintfDeviceVector(sizes, 4, r); + // fprintf(stderr, "coalescent:\n"); + // gpu->PrintfDeviceVector(coalescent, kCoalescentBufferLength, r); + } + + gpu->MemcpyDeviceToHost(hBastaLogL, dBastaLogL, sizeof(Real) * 1); + out[0] += hBastaLogL[0]; + gpu->FreeHostMemory(zeroes); + gpu->FreeHostMemory(hBastaLogL); + gpu->FreeHostMemory(hBastaDistance); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); #endif - return returnCode; + return BEAGLE_SUCCESS; } diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 2242163e..7e25ce30 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -236,8 +236,20 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { } bgReorderPatternsGrid = Dim3Int((kUnpaddedPatternCount + REORDER_BLOCK_SIZE - 1) / REORDER_BLOCK_SIZE, kCategoryCount); + //Set up block for basta partials + + bgBastaPeelingBlock = Dim3Int(kPaddedStateCount, 1); + bgBastaPeelingGrid = Dim3Int(1,1); + + bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 4); + bgBastaReductionGrid = Dim3Int(1,1); + + bgBastaSumBlock = Dim3Int(kPaddedStateCount, 64); + bgBastaSumGrid = Dim3Int(1,1); } + + void KernelLauncher::LoadKernels() { #ifdef FW_OPENCL_TESTING @@ -402,6 +414,8 @@ void KernelLauncher::LoadKernels() { fSumSites3 = gpu->GetFunction("kernelSumSites3"); fInnerBastaPartialsCoalescent = gpu->GetFunction("kernelInnerBastaPartialsCoalescent"); + fReduceWithinInterval = gpu->GetFunction("kernelBastaReduceWithinInterval"); + fReduceAcrossInterval = gpu->GetFunction("kernelBastaReduceAcrossInterval"); fReorderPatterns = gpu->GetFunction("kernelReorderPatterns"); @@ -2413,24 +2427,75 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials1, unsigned int patternCount, unsigned int child2Index) { #ifdef BEAGLE_DEBUG_FLOW -fprintf(stderr, "\t\tEntering KernelLauncher::SumSites3\n"); +fprintf(stderr, "\t\tEntering KernelLauncher::InnerBastaPartialsCoalescent\n"); #endif int parameterCountV = 9; int totalParameterCount = 12; gpu->LaunchKernel(fInnerBastaPartialsCoalescent, - bgPeelingBlock, bgPeelingGrid, + bgBastaPeelingBlock, bgBastaPeelingGrid, parameterCountV, totalParameterCount, partials1, partials2, partials3, matrices1, matrices2, accumulation1, accumulation2, sizes, coalescent, intervalNUmber, patternCount, child2Index); #ifdef BEAGLE_DEBUG_FLOW -fprintf(stderr, "\t\tLeaving KernelLauncher::SumSites3\n"); +fprintf(stderr, "\t\tLeaving KernelLauncher::InnerBastaPartialsCoalescent\n"); +#endif + +} + +void KernelLauncher::reduceWithinInterval(GPUPtr e, + GPUPtr f, + GPUPtr g, + GPUPtr h, + GPUPtr startPartials1, + GPUPtr startPartials2, + GPUPtr endPartials1, + GPUPtr endPartials2, + unsigned int intervalNUmber, + unsigned int child2PartialIndex) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); +#endif + + int parameterCountV = 8; + int totalParameterCount = 10; + gpu->LaunchKernel(fReduceWithinInterval, + bgBastaReductionBlock, bgBastaReductionGrid, + parameterCountV, totalParameterCount, + e, f, g, h, startPartials1, startPartials2, endPartials1, endPartials2, intervalNUmber, child2PartialIndex); + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); #endif } +void KernelLauncher::reduceAcrossIntervals(GPUPtr e, + GPUPtr f, + GPUPtr g, + GPUPtr h, + GPUPtr distance, + GPUPtr dLogL, + const GPUPtr sizes, + GPUPtr coalescent, + unsigned int intervalNumber) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); +#endif + int parameterCountV = 8; + int totalParameterCount = 9; + gpu->LaunchKernel(fReduceAcrossInterval, + bgBastaSumBlock, bgBastaSumGrid, + parameterCountV, totalParameterCount, + e, f, g, h, distance, dLogL, sizes, coalescent, intervalNumber); +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); +#endif + +} + }; // namespace diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index f7108a47..c25a1ae6 100644 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -107,6 +107,8 @@ class KernelLauncher { GPUFunction fSumSites3; GPUFunction fInnerBastaPartialsCoalescent; + GPUFunction fReduceWithinInterval; + GPUFunction fReduceAcrossInterval; GPUFunction fReorderPatterns; @@ -130,6 +132,13 @@ class KernelLauncher { Dim3Int bgMultiNodeSumGrid; Dim3Int bgCrossProductBlock; Dim3Int bgCrossProductGrid; + Dim3Int bgBastaPeelingBlock; + Dim3Int bgBastaPeelingGrid; + Dim3Int bgBastaReductionBlock; + Dim3Int bgBastaReductionGrid; + Dim3Int bgBastaSumBlock; + Dim3Int bgBastaSumGrid; + unsigned int kPaddedStateCount; unsigned int kCategoryCount; @@ -610,6 +619,11 @@ class KernelLauncher { void InnerBastaPartialsCoalescent(GPUPtr partials1, GPUPtr partials2, GPUPtr partials3, GPUPtr matrices1, GPUPtr matrices2, GPUPtr accumulation1, GPUPtr accumulation2, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, unsigned int patternCount, unsigned int child2Index); + void reduceWithinInterval(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr startPartials1, GPUPtr startPartials2, + GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex); + + void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); + void SetupKernelBlocksAndGrids(); protected: diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 9e604906..266327be 100644 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -17,6 +17,7 @@ #include #include #include + #include extern "C" { #elif defined(FW_OPENCL) #ifdef DOUBLE_PRECISION @@ -1978,53 +1979,6 @@ KW_GLOBAL_KERNEL void kernelAccumulateFactorsAutoScaling(KW_GLOBAL_VAR signed ch * BASTA kernels */ -// TODO: More appropriate kernel name -KW_GLOBAL_KERNEL void kernelInnerBastaPartials(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, - KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, - KW_GLOBAL_VAR REAL* KW_RESTRICT matrices1, - int totalPatterns) { - int state = KW_LOCAL_ID_0; - int patIdx = KW_LOCAL_ID_1; - int pattern = __umul24(KW_GROUP_ID_0,PATTERN_BLOCK_SIZE) + patIdx; - int matrix = KW_GROUP_ID_1; - int patternCount = totalPatterns; - int deltaPartialsByState = pattern * PADDED_STATE_COUNT; - int deltaPartialsByMatrix = matrix * PADDED_STATE_COUNT * patternCount; - int deltaMatrix = matrix * PADDED_STATE_COUNT * PADDED_STATE_COUNT; - int u = state + deltaPartialsByState + deltaPartialsByMatrix; - KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ - /* Load values into shared memory */ - KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; - int y = deltaPartialsByState + deltaPartialsByMatrix; - /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ - /* These are all coherent global memory reads; checked in Profiler */ - if (pattern < totalPatterns) { - sPartials1[patIdx][state] = partials1[y + state]; - } else { - sPartials1[patIdx][state] = 0; - } - REAL sum1 = 0; - for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { - /* load one row of matrices */ - if (patIdx < BLOCK_PEELING_SIZE) { - /* These are all coherent global memory reads. */ - sMatrix1[patIdx][state] = matrix1[patIdx * PADDED_STATE_COUNT + state]; - /* sMatrix now filled with starting in state and ending in i */ - matrix1 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; - } - KW_LOCAL_FENCE; - for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { - FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); - } - KW_LOCAL_FENCE; - } - - if (pattern < totalPatterns) - partials3[u] = sum1; -} - -// TODO: More appropriate kernel name KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, KW_GLOBAL_VAR REAL* KW_RESTRICT partials2, KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, @@ -2037,6 +1991,10 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ int intervalNumber, int totalPatterns, int child2Index) { + + #define PATTERN_BLOCK_SIZE_B 1 + #define BLOCK_PEELING_SIZE_B 1 + int state = KW_LOCAL_ID_0; int patIdx = KW_LOCAL_ID_1; int pattern = __umul24(KW_GROUP_ID_0,PATTERN_BLOCK_SIZE) + patIdx; @@ -2048,9 +2006,9 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ int u = state + deltaPartialsByState + deltaPartialsByMatrix; KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ /* Load values into shared memory */ - KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; int y = deltaPartialsByState + deltaPartialsByMatrix; @@ -2060,17 +2018,17 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ sPartials1[patIdx][state] = 0; } REAL sum1 = 0; - for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE_B) { /* load one row of matrices */ - if (patIdx < BLOCK_PEELING_SIZE) { + if (patIdx < BLOCK_PEELING_SIZE_B) { /* These are all coherent global memory reads. */ sMatrix1[patIdx][state] = matrix1[patIdx * PADDED_STATE_COUNT + state]; /* sMatrix now filled with starting in state and ending in i */ - matrix1 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; + matrix1 += BLOCK_PEELING_SIZE_B * PADDED_STATE_COUNT; } KW_LOCAL_FENCE; - for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { - FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); + for(int j = 0; j < BLOCK_PEELING_SIZE_B; j++) { + FMA(sMatrix1[j][state], sPartials1[patIdx][i + j], sum1); } KW_LOCAL_FENCE; } @@ -2081,27 +2039,29 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ /* These are all coherent global memory reads; checked in Profiler */ if (child2Index >= 0) { KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix; - KW_LOCAL_MEM REAL sMatrix2[BLOCK_PEELING_SIZE][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartials2[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartials3[PATTERN_BLOCK_SIZE][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sMatrix2[BLOCK_PEELING_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials2[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials3[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL popSizes[PADDED_STATE_COUNT]; if (pattern < totalPatterns) { sPartials2[patIdx][state] = partials2[y + state]; + popSizes[state] = sizes[state]; } else { sPartials2[patIdx][state] = 0; } REAL sum2 = 0; - for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE) { + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE_B) { /* load one row of matrices */ - if (patIdx < BLOCK_PEELING_SIZE) { + if (patIdx < BLOCK_PEELING_SIZE_B) { /* These are all coherent global memory reads. */ sMatrix2[patIdx][state] = matrix2[patIdx * PADDED_STATE_COUNT + state]; /* sMatrix now filled with starting in state and ending in i */ - matrix2 += BLOCK_PEELING_SIZE * PADDED_STATE_COUNT; + matrix2 += BLOCK_PEELING_SIZE_B * PADDED_STATE_COUNT; } KW_LOCAL_FENCE; - for(int j = 0; j < BLOCK_PEELING_SIZE; j++) { + for(int j = 0; j < BLOCK_PEELING_SIZE_B; j++) { FMA(sMatrix2[j][state], sPartials2[patIdx][i + j], sum2); } KW_LOCAL_FENCE; @@ -2110,7 +2070,7 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ if (pattern < totalPatterns) { accumulation1[u] = sum1; accumulation2[u] = sum2; - partials3[u] = sum1 * sum2 / sizes[state]; + partials3[u] = sum1 * sum2 / popSizes[state]; sPartials3[patIdx][state] = partials3[y + state]; #ifdef IS_POWER_OF_TWO @@ -2119,18 +2079,123 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ if (state < i) { #else for (int i = SMALLEST_POWER_OF_TWO / 2; i > 0; i >>= 1) { - if (state < i && state + i < PADDED_STATE_COUNT ) { + if (state < i && state + i < PADDED_STATE_COUNT) { #endif // IS_POWER_OF_TWO sPartials3[patIdx][state] += sPartials3[patIdx][state + i]; } KW_LOCAL_FENCE; } - coalescent[intervalNumber] = sPartials3[patIdx][0]; - partials3[u] = partials3[u] / coalescent[intervalNumber]; + REAL denominator = sPartials3[patIdx][0]; + partials3[u] = partials3[u] / denominator; + coalescent[intervalNumber] = denominator; } + } +} + +KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* e, + KW_GLOBAL_VAR REAL* f, + KW_GLOBAL_VAR REAL* g, + KW_GLOBAL_VAR REAL* h, + KW_GLOBAL_VAR REAL* KW_RESTRICT startPartials1, + KW_GLOBAL_VAR REAL* KW_RESTRICT startPartials2, + KW_GLOBAL_VAR REAL* KW_RESTRICT endPartials1, + KW_GLOBAL_VAR REAL* KW_RESTRICT endPartials2, + int intervalNumber, + int child2PartialIndex) { + + #define SUM_SITES_BLOCK_SIZE_B 64 + #define SUM_PARTIAL_BLOCK_SIZE_B 4 + int state = KW_LOCAL_ID_0; + int u = KW_LOCAL_ID_0 + KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE_B; + int y = intervalNumber * PADDED_STATE_COUNT; + KW_LOCAL_MEM REAL sPartials1[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + + if (state < PADDED_STATE_COUNT) { + sPartials1[0][state] = startPartials1[u]; + sPartials1[1][state] = startPartials1[u] * startPartials1[u]; + sPartials1[2][state] = endPartials1[u]; + sPartials1[3][state] = endPartials1[u] * endPartials1[u]; + } else { + sPartials1[0][state] = 0; + sPartials1[1][state] = 0; + sPartials1[2][state] = 0; + sPartials1[3][state] = 0; + } + e[y + state] += sPartials1[0][state]; + f[y + state] += sPartials1[1][state]; + g[y + state] += sPartials1[2][state]; + h[y + state] += sPartials1[3][state]; + + KW_LOCAL_MEM REAL sPartials2[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + if (child2PartialIndex >= 0) { + if (state < PADDED_STATE_COUNT) { + sPartials2[0][state] = startPartials2[u]; + sPartials2[1][state] = startPartials2[u] * startPartials2[u]; + sPartials2[2][state] = endPartials2[u]; + sPartials2[3][state] = endPartials2[u] * endPartials2[u]; + } else { + sPartials2[0][state] = 0; + sPartials2[1][state] = 0; + sPartials2[2][state] = 0; + sPartials2[3][state] = 0; + } + e[y + state] += sPartials2[0][state]; + f[y + state] += sPartials2[1][state]; + g[y + state] += sPartials2[2][state]; + h[y + state] += sPartials2[3][state]; + } } +KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, + KW_GLOBAL_VAR REAL* KW_RESTRICT f, + KW_GLOBAL_VAR REAL* KW_RESTRICT g, + KW_GLOBAL_VAR REAL* KW_RESTRICT h, + KW_GLOBAL_VAR REAL* KW_RESTRICT distance, + KW_GLOBAL_VAR REAL* KW_RESTRICT dLogL, + KW_GLOBAL_VAR REAL* KW_RESTRICT sizes, + KW_GLOBAL_VAR REAL* KW_RESTRICT coalescent, + int intervalNumber) { + +#define SUM_SITES_BLOCK_SIZE_B2 1 + int state = KW_LOCAL_ID_0; + int u = KW_LOCAL_ID_0 + KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE_B2; + int y = intervalNumber * PADDED_STATE_COUNT; + + KW_LOCAL_MEM REAL sPartials1[PADDED_STATE_COUNT]; + + if (state < PADDED_STATE_COUNT) { + sPartials1[state] = (e[y + u] * e[y + u] - f[y + u] + + g[y + u] * g[y + u] - h[y + u]) / sizes[u]; + } + KW_LOCAL_FENCE; + +#ifdef IS_POWER_OF_TWO + // parallelized reduction *** only works for powers-of-2 **** + for (int i = PADDED_STATE_COUNT / 2; i > 0; i >>= 1) { + if (state < i) { +#else + for (int i = SMALLEST_POWER_OF_TWO / 2; i > 0; i >>= 1) { + if (state < i && state + i < PADDED_STATE_COUNT ) { +#endif // IS_POWER_OF_TWO + sPartials1[state] += sPartials1[state + i]; + } + KW_LOCAL_FENCE; + } + + + if (state == 0) { + REAL temp = -distance[intervalNumber] * sPartials1[0] / 4; + REAL prob = coalescent[intervalNumber]; + if (prob != 0) { + temp += logf(prob); + } + dLogL[0] += temp; + } + +} + + #ifdef CUDA } // extern "C" #endif //CUDA diff --git a/tests/basta/test_x_state_updateInnerBastaPartials.cu b/tests/basta/test_x_state_updateInnerBastaPartials.cu index 120eca2d..d6c3f679 100644 --- a/tests/basta/test_x_state_updateInnerBastaPartials.cu +++ b/tests/basta/test_x_state_updateInnerBastaPartials.cu @@ -71,14 +71,15 @@ int main() { - double *partials1, *partials2, *partials3, *matrices1, *matrices2, *accumulation1, *accumulation2, *sizes, *coalescent, *tmpAcc; + double *partials1, *partials2, *partials3, *matrices1, *matrices2, *accumulation1, *accumulation2, *sizes, *coalescent; int totalPatterns = NPATTERNS; int intervalNumber = 1; + int child2Index = 1; double *resHost = (double*) malloc(sizeof(double) * STATE_SIZE * NPATTERNS); double *accumHost1 = (double*) malloc(sizeof(double) * STATE_SIZE * NPATTERNS); double *accumHost2 = (double*) malloc(sizeof(double) * STATE_SIZE * NPATTERNS); - float *coal = static_cast(malloc(sizeof(double) * intervalNumber)); + double *coal = (double*) malloc(sizeof(double) * intervalNumber); cuMemAlloc((CUdeviceptr*)&partials1, sizeof(double) * STATE_SIZE * NPATTERNS); cuMemAlloc((CUdeviceptr*)&partials2, sizeof(double) * STATE_SIZE * NPATTERNS); @@ -86,9 +87,8 @@ int main() { cuMemAlloc((CUdeviceptr*)&accumulation1, sizeof(double) * STATE_SIZE * NPATTERNS); cuMemAlloc((CUdeviceptr*)&accumulation2, sizeof(double) * STATE_SIZE * NPATTERNS); cuMemAlloc((CUdeviceptr*)&sizes, sizeof(double) * STATE_SIZE * NPATTERNS); - cuMemAlloc((CUdeviceptr*)&coalescent, sizeof(float) * intervalNumber); + cuMemAlloc((CUdeviceptr*)&coalescent, sizeof(double) * intervalNumber); cuMemAlloc((CUdeviceptr*)&partials3, sizeof(double) * STATE_SIZE * NPATTERNS); - cuMemAlloc((CUdeviceptr*)&tmpAcc, sizeof(double) * 1); cuMemAlloc((CUdeviceptr*)&matrices1, sizeof(double) * STATE_SIZE * STATE_SIZE); cuMemAlloc((CUdeviceptr*)&matrices2, sizeof(double) * STATE_SIZE * STATE_SIZE); @@ -98,12 +98,12 @@ int main() { cudaMemcpy(partials1, GET_VAR_NAME(partials1Host), sizeof(double) * STATE_SIZE * NPATTERNS, cudaMemcpyHostToDevice); cudaMemcpy(partials2, GET_VAR_NAME(partials2Host), sizeof(double) * STATE_SIZE * NPATTERNS, cudaMemcpyHostToDevice); cudaMemcpy(sizes, GET_VAR_NAME(sizes), sizeof(double) * STATE_SIZE * NPATTERNS, cudaMemcpyHostToDevice); - cudaMemcpy(coalescent, GET_VAR_NAME(coalescent), sizeof(float) * intervalNumber, cudaMemcpyHostToDevice); + cudaMemcpy(coalescent, GET_VAR_NAME(coalescent), sizeof(double) * intervalNumber, cudaMemcpyHostToDevice); void *params[] = { - &partials1, &partials2, &partials3, &matrices1, &matrices2, &accumulation1, &accumulation2, &sizes, &coalescent, &intervalNumber, &totalPatterns + &partials1, &partials2, &partials3, &matrices1, &matrices2, &accumulation1, &accumulation2, &sizes, &coalescent, &intervalNumber, &totalPatterns, &child2Index }; - res = cuLaunchKernel(cudaFunction, 1, 1, 1, STATE_SIZE, 8, 1, 0, NULL, params, NULL); + res = cuLaunchKernel(cudaFunction, 1, 1, 1, STATE_SIZE, 1, 1, 0, NULL, params, NULL); cudaDeviceSynchronize(); @@ -122,7 +122,7 @@ int main() { err = cudaMemcpy(accumHost2, accumulation2, NPATTERNS * STATE_SIZE * sizeof(double), cudaMemcpyDeviceToHost); std::cerr << "Memcpy accum2 " << err << std::endl; - err = cudaMemcpy(coal, coalescent, intervalNumber * sizeof(float), cudaMemcpyDeviceToHost); + err = cudaMemcpy(coal, coalescent, intervalNumber * sizeof(double), cudaMemcpyDeviceToHost); std::cerr << "Memcpy coal " << err << std::endl; double tolerance=1E-5; From f61e974b454499e36b8c38821ae25a6123f33f83 Mon Sep 17 00:00:00 2001 From: yucais Date: Wed, 29 May 2024 11:41:16 -0700 Subject: [PATCH 058/421] temporary patch for opencl --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 561309a0..585ebb03 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2278,7 +2278,7 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2375,7 +2375,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; Real* zeroes = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); gpu->MemcpyHostToDevice(dBastaE, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); From f3ff34283eac608c9a93468259c1ba070d411498 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 29 May 2024 19:26:50 -0500 Subject: [PATCH 059/421] start messing up caching structure --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 9 + libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 165 +++++++++++++++++-- libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 2 +- 3 files changed, 162 insertions(+), 14 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 9abef4d5..77aeacd6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -34,6 +34,13 @@ #endif #include "libhmsbeagle/GPU/BeagleGPUImpl.h" +#include +#include +#include + +typedef Eigen::Matrix MatrixXd; +typedef Eigen::Map MapType; +typedef Eigen::SparseMatrix SpMatrix; namespace beagle { namespace gpu { @@ -72,6 +79,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl protected: + std::vector hInstantaneousMatrices; + using BeagleGPUImpl::kInitialized; using BeagleGPUImpl::kTipCount; using BeagleGPUImpl::kPartialsBufferCount; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 8604c3e6..ce887f2f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -74,19 +74,158 @@ int BeagleGPUActionImpl::createInstance(int tipCount, long long requirementFlags) { - int parentCode = BeagleGPUImpl::createInstance(tipCount, - partialsBufferCount, - compactBufferCount, - stateCount, - patternCount, - eigenDecompositionCount, - matrixCount, - categoryCount, - scaleBufferCount, - globalResourceNumber, - pluginResourceNumber, - preferenceFlags, - requirementFlags); +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::createInstance\n"); +#endif + + kInitialized = 0; + + kTipCount = tipCount; + kPartialsBufferCount = partialsBufferCount; + kCompactBufferCount = 0; // XJ: ignore compact buffer count because of using all partials for tip states + kStateCount = stateCount; + kEigenDecompCount = eigenDecompositionCount; + kMatrixCount = matrixCount; + kCategoryCount = categoryCount; + kScaleBufferCount = scaleBufferCount; + + kPartitionCount = 1; // XJ: seems related to stream setup + kMaxPartitionCount = kPartitionCount; + kPartitionsInitialised = false; + kPatternsReordered = false; + + resourceNumber = globalResourceNumber; + + kTipPartialsBufferCount = kTipCount; // use Partials for all tip states + kBufferCount = kPartialsBufferCount; + + kInternalPartialsBufferCount = kBufferCount - kTipCount; + + //TODO: check if pad state is necessary + kPaddedStateCount = kStateCount; + + gpu = new GPUInterface(); + + gpu->Initialize(); + + int numDevices = 0; + numDevices = gpu->GetDeviceCount(); + if (numDevices == 0) { + fprintf(stderr, "Error: No GPU devices\n"); + return BEAGLE_ERROR_NO_RESOURCE; + } + if (pluginResourceNumber > numDevices) { + fprintf(stderr,"Error: Trying to initialize device # %d (which does not exist)\n",resourceNumber); + return BEAGLE_ERROR_NO_RESOURCE; + } + + //TODO: check if pad patterns is necessary, ignored for now + bool CPUImpl = false; + + kDeviceType = gpu->GetDeviceTypeFlag(pluginResourceNumber); + kDeviceCode = gpu->GetDeviceImplementationCode(pluginResourceNumber); + + kPaddedPatternCount = kPatternCount; + kScaleBufferSize = kPaddedPatternCount; + + kFlags = 0; + + if (preferenceFlags & BEAGLE_FLAG_SCALING_AUTO || requirementFlags & BEAGLE_FLAG_SCALING_AUTO) { + kFlags |= BEAGLE_FLAG_SCALING_AUTO; + kFlags |= BEAGLE_FLAG_SCALERS_LOG; + kScaleBufferCount = kInternalPartialsBufferCount; + kScaleBufferSize *= kCategoryCount; + } else if (preferenceFlags & BEAGLE_FLAG_SCALING_ALWAYS || requirementFlags & BEAGLE_FLAG_SCALING_ALWAYS) { + kFlags |= BEAGLE_FLAG_SCALING_ALWAYS; + kFlags |= BEAGLE_FLAG_SCALERS_LOG; + kScaleBufferCount = kInternalPartialsBufferCount + 1; // +1 for temp buffer used by edgelikelihood + } else if (preferenceFlags & BEAGLE_FLAG_SCALING_DYNAMIC || requirementFlags & BEAGLE_FLAG_SCALING_DYNAMIC) { + kFlags |= BEAGLE_FLAG_SCALING_DYNAMIC; + kFlags |= BEAGLE_FLAG_SCALERS_RAW; + } else if (preferenceFlags & BEAGLE_FLAG_SCALERS_LOG || requirementFlags & BEAGLE_FLAG_SCALERS_LOG) { + kFlags |= BEAGLE_FLAG_SCALING_MANUAL; + kFlags |= BEAGLE_FLAG_SCALERS_LOG; + } else { + kFlags |= BEAGLE_FLAG_SCALING_MANUAL; + kFlags |= BEAGLE_FLAG_SCALERS_RAW; + } + //TODO: remember to implement scaling + +// if (preferenceFlags & BEAGLE_FLAG_EIGEN_COMPLEX || requirementFlags & BEAGLE_FLAG_EIGEN_COMPLEX) { +// kFlags |= BEAGLE_FLAG_EIGEN_COMPLEX; +// } else { +// kFlags |= BEAGLE_FLAG_EIGEN_REAL; +// } + + if (requirementFlags & BEAGLE_FLAG_INVEVEC_TRANSPOSED || preferenceFlags & BEAGLE_FLAG_INVEVEC_TRANSPOSED) + kFlags |= BEAGLE_FLAG_INVEVEC_TRANSPOSED; + else + kFlags |= BEAGLE_FLAG_INVEVEC_STANDARD; + + + // TODO: this chunk of control is not checked + if (kDeviceCode == BEAGLE_OPENCL_DEVICE_APPLE_CPU) + kFlags |= BEAGLE_FLAG_PARALLELOPS_STREAMS; + else if (requirementFlags & BEAGLE_FLAG_PARALLELOPS_STREAMS || preferenceFlags & BEAGLE_FLAG_PARALLELOPS_STREAMS) + kFlags |= BEAGLE_FLAG_PARALLELOPS_STREAMS; + else if (requirementFlags & BEAGLE_FLAG_PARALLELOPS_GRID || preferenceFlags & BEAGLE_FLAG_PARALLELOPS_GRID) + kFlags |= BEAGLE_FLAG_PARALLELOPS_GRID; + + if (preferenceFlags & BEAGLE_FLAG_COMPUTATION_ASYNCH || requirementFlags & BEAGLE_FLAG_COMPUTATION_ASYNCH) { + kFlags |= BEAGLE_FLAG_COMPUTATION_ASYNCH; + } else { + kFlags |= BEAGLE_FLAG_COMPUTATION_SYNCH; + } + + if (preferenceFlags & BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO || requirementFlags & BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO) { + kFlags |= BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO; + } else { + kFlags |= BEAGLE_FLAG_PREORDER_TRANSPOSE_MANUAL; + } + + Real r = 0; + modifyFlagsForPrecision(&kFlags, r); + + //TODO: sum block size ignored + kPartialsSize = kPaddedPatternCount * kPaddedStateCount * kCategoryCount; + kMatrixSize = kPaddedStateCount * kPaddedStateCount; + + kLastCompactBufferIndex = -1; + kLastTipPartialsBufferIndex = -1; + + //TODO: we probably don't use the kernel resource anyway, separate it? + gpu->SetDevice(pluginResourceNumber, kPaddedStateCount, kCategoryCount, + kPaddedPatternCount, kPatternCount, kTipCount, kFlags); + + int ptrQueueLength = kMatrixCount * kCategoryCount * 3 * 3; // first '3' for derivatives, last '3' is for 3 ops for uTMWMM + if (kInternalPartialsBufferCount > ptrQueueLength) + ptrQueueLength = kInternalPartialsBufferCount; + + // TODO: not sure if need kernels +// kernels = new KernelLauncher(gpu); + + hWeightsCache = (Real*) gpu->CallocHost(kCategoryCount, sizeof(Real)); + hFrequenciesCache = (Real*) gpu->CallocHost(kPaddedStateCount, sizeof(Real)); + hPartialsCache = (Real*) gpu->CallocHost(kPartialsSize, sizeof(Real)); + + int hMatrixCacheSize = kMatrixSize * kCategoryCount * BEAGLE_CACHED_MATRICES_COUNT; //TODO: use Eigen csr representation? + hLogLikelihoodsCache = (Real*) gpu->MallocHost(kPatternCount * sizeof(Real)); + hMatrixCache = (Real*) gpu->CallocHost(hMatrixCacheSize, sizeof(Real)); + hInstantaneousMatrices.resize(kEigenDecompCount); + for (int i = 0; i < kEigenDecompCount; i++) { + hInstantaneousMatrices[i] = SpMatrix(kStateCount, kStateCount); + } + + dWeights = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); + dFrequencies = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); + + dMatrices = (GPUPtr*) malloc(sizeof(GPUPtr) * kMatrixCount); + + + + + + return BEAGLE_SUCCESS; } diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt index a1ae7a5c..17cd96cd 100644 --- a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -50,7 +50,7 @@ add_library(hmsbeagle-action-cuda SHARED add_dependencies(hmsbeagle-action-cuda CudaKernels) -target_link_libraries(hmsbeagle-action-cuda cuda ${CUDA_LIBRARIES}) +target_link_libraries(hmsbeagle-action-cuda cuda ${CUDA_LIBRARIES} Eigen3::Eigen) install(TARGETS hmsbeagle-action-cuda DESTINATION ${BEAGLE_INSTALL_DIR} From 703e0070c8244779e636dc5292466f82315c5fb8 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 29 May 2024 19:41:41 -0500 Subject: [PATCH 060/421] add new API function for setting sparse matrix --- libhmsbeagle/BeagleImpl.h | 6 ++++ libhmsbeagle/CPU/BeagleCPUActionImpl.h | 6 ++++ libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 45 ++++++++++++++++++++++++ libhmsbeagle/CPU/BeagleCPUImpl.h | 6 ++++ libhmsbeagle/GPU/BeagleGPUActionImpl.h | 6 ++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 10 ++++++ libhmsbeagle/GPU/BeagleGPUImpl.h | 6 ++++ libhmsbeagle/GPU/BeagleGPUImpl.hpp | 9 +++++ libhmsbeagle/beagle.cpp | 15 ++++++++ libhmsbeagle/beagle.h | 21 +++++++++++ 10 files changed, 130 insertions(+) diff --git a/libhmsbeagle/BeagleImpl.h b/libhmsbeagle/BeagleImpl.h index 0b29bc0f..33249dcb 100644 --- a/libhmsbeagle/BeagleImpl.h +++ b/libhmsbeagle/BeagleImpl.h @@ -82,6 +82,12 @@ class BeagleImpl const double* inEigenVectors, const double* inInverseEigenVectors, const double* inEigenValues) = 0; + + virtual int setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros) = 0; virtual int setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies) = 0; diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index eca3e00c..3e7eea91 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -194,6 +194,12 @@ namespace beagle { const double *inInverseEigenVectors, const double *inEigenValues); + virtual int setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros); + virtual int updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, const int* firstDerivativeIndices, diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index fda8f9d4..01af7e06 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -610,6 +610,51 @@ namespace beagle { // gSimpleActions[eigenIndex]->setInstantaneousMatrix(tripletList); // gSimpleActions[eigenIndex]->fireMatrixChanged(); +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"In vlaues: \n"; + for (int i = 0; i < numNonZeros; i++) { + std::cerr<< "("<::setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros) { + + + std::vector tripletList; + for (int i = 0; i < numNonZeros; i++) { + tripletList.push_back(Triplet(rowIndices[i], colIndices[i], values[i])); + } + gInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); + + double mu_B = 0.0; + for (int i = 0; i < kStateCount; i++) { + mu_B += gInstantaneousMatrices[matrixIndex].coeff(i, i); + } + mu_B /= (double) kStateCount; + gMuBs[matrixIndex] = mu_B; + gBs[matrixIndex] = gInstantaneousMatrices[matrixIndex] - mu_B * identity; + gB1Norms[matrixIndex] = normP1(gBs[matrixIndex]); + + ds[matrixIndex].clear(); + + int pMax = getPMax(); + for(int p=0;p <= pMax+1; p++) + { + int t = 5; + double approx_norm = normest1( gBs[matrixIndex], p, t); + + // equation 3.7 in Al-Mohy and Higham + ds[matrixIndex].push_back( pow( approx_norm, 1.0/double(p) ) ); + } + #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"In vlaues: \n"; for (int i = 0; i < numNonZeros; i++) { diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index 12324497..38ddd447 100644 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -229,6 +229,12 @@ class BeagleCPUImpl : public BeagleImpl { const double* inInverseEigenVectors, const double* inEigenValues); + int setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros); + int setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies); diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 77aeacd6..d82b1914 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -216,6 +216,12 @@ class BeagleGPUActionImpl : public BeagleGPUImpl const double *inInverseEigenVectors, const double *inEigenValues); + int setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros); + int updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, const int* firstDerivativeIndices, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index ce887f2f..bf32bbd6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -300,6 +300,16 @@ int BeagleGPUActionImpl::setEigenDecomposition(int eigenInde return BeagleGPUImpl::setEigenDecomposition(eigenIndex, inEigenVectors, inInverseEigenVectors, inEigenValues); } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros) +{ + return BEAGLE_ERROR_NO_IMPLEMENTATION; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 2ca94b46..ae67d1f1 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -267,6 +267,12 @@ class BeagleGPUImpl : public BeagleImpl { const double* inEigenVectors, const double* inInverseEigenVectors, const double* inEigenValues); + + int setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros); int setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies); diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 12ee74bf..07064f32 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -1151,6 +1151,15 @@ int BeagleGPUImpl::setEigenDecomposition(int eigenIndex, return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros) { + return BEAGLE_ERROR_NO_IMPLEMENTATION; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies) { diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index 43a0697d..130401db 100644 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -924,6 +924,21 @@ int beagleSetEigenDecomposition(int instance, } } +int beagleSetSparseMatrix(int instance, + int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros) { + DEBUG_START_TIME(); + beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + int returnValue = beagleInstance->setSparseMatrix(matrixIndex, rowIndices, colIndices, values, numNonZeros); + DEBUG_END_TIME(); + return returnValue; +} + int beagleSetStateFrequencies(int instance, int stateFrequenciesIndex, const double* inStateFrequencies) { diff --git a/libhmsbeagle/beagle.h b/libhmsbeagle/beagle.h index 35804106..42109c74 100644 --- a/libhmsbeagle/beagle.h +++ b/libhmsbeagle/beagle.h @@ -549,6 +549,27 @@ BEAGLE_DLLEXPORT int beagleSetEigenDecomposition(int instance, const double* inInverseEigenVectors, const double* inEigenValues); +/** + * @brief Set a sparse matrix using non-zero entries + * + * This function sets a sparse matrix in an instance buffer. + * + * @param instance Instance number (input) + * @param matrixIndex Index of eigen-decomposition buffer (input) + * @param rowIndices List of row indices of non-zero entries (input) + * @param colIndices List of column indices of non-zero entries (input) + * @param values List of non-zero entry values + * @param numNonZeros Number of non-zero entries + * + * @return error code + */ +BEAGLE_DLLEXPORT int beagleSetSparseMatrix(int instance, + int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros); + /** * @brief Set a state frequency buffer * From 64685b746b0cd5250d67a149ec232af2b21a080a Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 29 May 2024 19:45:48 -0500 Subject: [PATCH 061/421] remove set eigen for action gpu code --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 5 ----- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 9 --------- 2 files changed, 14 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index d82b1914..b6292235 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -211,11 +211,6 @@ class BeagleGPUActionImpl : public BeagleGPUImpl private: char* getInstanceName(); - int setEigenDecomposition(int eigenIndex, - const double *inEigenVectors, - const double *inInverseEigenVectors, - const double *inEigenValues); - int setSparseMatrix(int matrixIndex, const int* rowIndices, const int* colIndices, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index bf32bbd6..78a0243e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -291,15 +291,6 @@ int BeagleGPUActionImpl::upPrePartials(bool byPartition, return BeagleGPUImpl::upPrePartials(byPartition, operations, operationCount, cumulativeScalingIndex); } -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::setEigenDecomposition(int eigenIndex, - const double *inEigenVectors, - const double *inInverseEigenVectors, - const double *inEigenValues) -{ - return BeagleGPUImpl::setEigenDecomposition(eigenIndex, inEigenVectors, inInverseEigenVectors, inEigenValues); -} - BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, const int* rowIndices, From e2fedd1d86d17e88b2b2488bb5dce395ae114ea4 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 30 May 2024 10:09:27 -0500 Subject: [PATCH 062/421] make CPU build happy with API change --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 7 ------- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 10 ++++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 01af7e06..3fe3d89c 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -655,13 +655,6 @@ namespace beagle { ds[matrixIndex].push_back( pow( approx_norm, 1.0/double(p) ) ); } -#ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"In vlaues: \n"; - for (int i = 0; i < numNonZeros; i++) { - std::cerr<< "("<::setEigenDecomposition(int eigenIndex, return BEAGLE_SUCCESS; } +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros) { + + return BEAGLE_ERROR_NO_IMPLEMENTATION; +} + BEAGLE_CPU_TEMPLATE int BeagleCPUImpl::setCategoryRates(const double* inCategoryRates) { int categoryRatesIndex=0; From d99a79778eab22e30330ab591b14f960b40f18ba Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 30 May 2024 15:15:11 -0500 Subject: [PATCH 063/421] use cusparseDnMat for device partials --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 34 ++++++++++++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 53 +++++++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index b6292235..871fe22f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -37,10 +37,34 @@ #include #include #include +#include // cudaMalloc, cudaMemcpy, etc. +#include // cusparseSpMV + +#define CHECK_CUDA(func) \ +{ \ + cudaError_t status = (func); \ + if (status != cudaSuccess) { \ + printf("CUDA API failed at line %d with error: %s (%d)\n", \ + __LINE__, cudaGetErrorString(status), status); \ + return EXIT_FAILURE; \ + } \ +} + +#define CHECK_CUSPARSE(func) \ +{ \ + cusparseStatus_t status = (func); \ + if (status != CUSPARSE_STATUS_SUCCESS) { \ + printf("CUSPARSE API failed at line %d with error: %s (%d)\n", \ + __LINE__, cusparseGetErrorString(status), status); \ + return EXIT_FAILURE; \ + } \ +} + typedef Eigen::Matrix MatrixXd; typedef Eigen::Map MapType; typedef Eigen::SparseMatrix SpMatrix; +typedef Eigen::Triplet Triplet; namespace beagle { namespace gpu { @@ -77,9 +101,15 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setTipStates(int tipIndex, const int* inStates); + int setTipPartials(int tipIndex, const double* inPartials); + protected: std::vector hInstantaneousMatrices; + cusparseSpMatDescr_t* dInstantaneousMatrices; + cusparseDnMatDescr_t* dPartials; + + using BeagleGPUImpl::kInitialized; using BeagleGPUImpl::kTipCount; @@ -157,8 +187,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::kDerivBuffersInitialised; using BeagleGPUImpl::kMultipleDerivativesLength; - using BeagleGPUImpl::dPartials; - using BeagleGPUImpl::dPartialsOrigin; +// using BeagleGPUImpl::dPartials; +// using BeagleGPUImpl::dPartialsOrigin; using BeagleGPUImpl::hPartialsOffsets; using BeagleGPUImpl::kIndexOffsetPat; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 78a0243e..9d001dfb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -213,7 +213,12 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hMatrixCache = (Real*) gpu->CallocHost(hMatrixCacheSize, sizeof(Real)); hInstantaneousMatrices.resize(kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { - hInstantaneousMatrices[i] = SpMatrix(kStateCount, kStateCount); + hInstantaneousMatrices[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); + } + dInstantaneousMatrices = (cusparseSpMatDescr_t *) gpu->calloc(sizeof(cusparseSpMatDescr_t), kEigenDecompCount); + dPartials = (cusparseDnMatDescr_t *) gpu->calloc(sizeof(cusparseDnMatDescr_t), kPartialsBufferCount * kCategoryCount); + for (int i = 0; i < kPartialsBufferCount * kCategoryCount; i++) { + dPartials[i] = NULL; } dWeights = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); @@ -272,6 +277,47 @@ int BeagleGPUActionImpl::setTipStates(int tipIndex, const in std::abort(); } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setTipPartials(int tipIndex, const double* inPartials) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::setTipPartials\n"); +#endif + + if (tipIndex < 0 || tipIndex >= kTipCount) + return BEAGLE_ERROR_OUT_OF_RANGE; + + const double* inPartialsOffset = inPartials; + Real* tmpRealPartialsOffset = hPartialsCache; + for (int i = 0; i < kPatternCount; i++) { +//#ifdef DOUBLE_PRECISION +// memcpy(tmpRealPartialsOffset, inPartialsOffset, sizeof(Real) * kStateCount); +//#else +// MEMCNV(tmpRealPartialsOffset, inPartialsOffset, kStateCount, Real); +//#endif + beagleMemCpy(tmpRealPartialsOffset, inPartialsOffset, kStateCount); + tmpRealPartialsOffset += kPaddedStateCount; + inPartialsOffset += kStateCount; + } + +// int partialsLength = kPaddedPatternCount * kPaddedStateCount; +// for (int i = 1; i < kCategoryCount; i++) { +// memcpy(hPartialsCache + i * partialsLength, hPartialsCache, partialsLength * sizeof(Real)); +// } + + if (tipIndex < kTipCount) { + if (dPartials[tipIndex] == NULL) { + for (int i = 0; i < kCategoryCount; i++) { + CHECK_CUSPARSE(cusparseCreateDnMat(dPartials[kPartialsBufferCount * i + tipIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, hPartialsCache, + CUDA_R_64F, CUSPARSE_ORDER_COL)) + } + } + } +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setTipPartials\n"); +#endif + + return BEAGLE_SUCCESS; +} BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPartials(bool byPartition, @@ -298,6 +344,11 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, const double* values, int numNonZeros) { + std::vector tripletList; + for (int i = 0; i < numNonZeros; i++) { + tripletList.push_back(Triplet(rowIndices[i], colIndices[i], values[i])); + } + hInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); return BEAGLE_ERROR_NO_IMPLEMENTATION; } From 91af0f8c4bf4254fc1838255ab190e3c835af9d8 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 30 May 2024 15:51:09 -0500 Subject: [PATCH 064/421] action gpu set sparse matrix --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 3 +++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 27 +++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 871fe22f..24155c62 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -108,6 +108,9 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector hInstantaneousMatrices; cusparseSpMatDescr_t* dInstantaneousMatrices; cusparseDnMatDescr_t* dPartials; + int *dMatrixCsrOffsetsCache, *dMatrixCsrColumnsCache; + Real *dMatrixCsrValuesCache; + int currentCacheNNZ; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 9d001dfb..35825216 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -216,6 +216,13 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hInstantaneousMatrices[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); } dInstantaneousMatrices = (cusparseSpMatDescr_t *) gpu->calloc(sizeof(cusparseSpMatDescr_t), kEigenDecompCount); + for (int i = 0; i < kEigenDecompCount; i++) { + dInstantaneousMatrices[i] = NULL; + } + CHECK_CUDA(cudaMalloc((void**) &dMatrixCsrOffsetsCache, (kPaddedStateCount + 1) * sizeof(int))) + dMatrixCsrColumnsCache = NULL; + currentCacheNNZ = 0; + dPartials = (cusparseDnMatDescr_t *) gpu->calloc(sizeof(cusparseDnMatDescr_t), kPartialsBufferCount * kCategoryCount); for (int i = 0; i < kPartialsBufferCount * kCategoryCount; i++) { dPartials[i] = NULL; @@ -224,7 +231,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dWeights = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); dFrequencies = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); - dMatrices = (GPUPtr*) malloc(sizeof(GPUPtr) * kMatrixCount); +// dMatrices = (GPUPtr*) malloc(sizeof(GPUPtr) * kMatrixCount); @@ -349,6 +356,24 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, tripletList.push_back(Triplet(rowIndices[i], colIndices[i], values[i])); } hInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); + + const int currentNNZ = hInstantaneousMatrices[matrixIndex].nonZeros(); + const int paddedNNZ = currentNNZ + 16 - currentNNZ%16; + if (currentCacheNNZ < paddedNNZ) { + currentCacheNNZ = paddedNNZ; + CHECK_CUDA(cudaMalloc((void**) &dMatrixCsrColumnsCache, currentCacheNNZ * sizeof(int))) + CHECK_CUDA(cudaMalloc((void**) &dMatrixCsrValuesCache, currentCacheNNZ * sizeof(Real))) + } + + CHECK_CUDA(cudaMemcpy(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), sizeof(int) * (kPaddedStateCount + 1), cudaMemcpyHostToDevice)) + CHECK_CUDA(cudaMemcpy(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), sizeof(int) * currentNNZ, cudaMemcpyHostToDevice)) + CHECK_CUDA(cudaMemcpy(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), sizeof(Real) * currentNNZ, cudaMemcpyHostToDevice)) + + CHECK_CUSPARSE(cusparseCreateCsr(dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, + dMatrixCsrOffsetsCache, dMatrixCsrColumnsCache, dMatrixCsrValuesCache, + CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, + CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F) ) + return BEAGLE_ERROR_NO_IMPLEMENTATION; } From 180d219681de1a75e57a58ebb46727bacbdf16a0 Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 31 May 2024 10:23:51 -0500 Subject: [PATCH 065/421] that one line for #include cusparse --- libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt index 17cd96cd..0c7d09be 100644 --- a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -50,7 +50,7 @@ add_library(hmsbeagle-action-cuda SHARED add_dependencies(hmsbeagle-action-cuda CudaKernels) -target_link_libraries(hmsbeagle-action-cuda cuda ${CUDA_LIBRARIES} Eigen3::Eigen) +target_link_libraries(hmsbeagle-action-cuda cuda ${CUDA_LIBRARIES} ${CUDA_cusparse_LIBRARY} Eigen3::Eigen) install(TARGETS hmsbeagle-action-cuda DESTINATION ${BEAGLE_INSTALL_DIR} From f6ae39e16f4888d1da01db4515c0668f3bc714b1 Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 31 May 2024 17:15:57 -0500 Subject: [PATCH 066/421] set pattern weights --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 18 +++++---- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 49 +++++++++++++++++++++--- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 24155c62..c0d00a49 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -103,6 +103,14 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setTipPartials(int tipIndex, const double* inPartials); + int setSparseMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros); + + int setPatternWeights(const double* inPatternWeights); + protected: std::vector hInstantaneousMatrices; @@ -111,6 +119,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int *dMatrixCsrOffsetsCache, *dMatrixCsrColumnsCache; Real *dMatrixCsrValuesCache; int currentCacheNNZ; + Real *dPatternWeightsCache; + cusparseDnVecDescr_t* dPatternWeights; @@ -183,7 +193,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::kWeightsOffset; using BeagleGPUImpl::kFrequenciesOffset; using BeagleGPUImpl::dIntegrationTmp; - using BeagleGPUImpl::dPatternWeights; +// using BeagleGPUImpl::dPatternWeights; using BeagleGPUImpl::dSumLogLikelihood; using BeagleGPUImpl::dPartialsTmp; @@ -244,12 +254,6 @@ class BeagleGPUActionImpl : public BeagleGPUImpl private: char* getInstanceName(); - int setSparseMatrix(int matrixIndex, - const int* rowIndices, - const int* colIndices, - const double* values, - int numNonZeros); - int updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, const int* firstDerivativeIndices, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 35825216..f8dfb65f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -125,6 +125,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, kDeviceType = gpu->GetDeviceTypeFlag(pluginResourceNumber); kDeviceCode = gpu->GetDeviceImplementationCode(pluginResourceNumber); + kPatternCount = patternCount; kPaddedPatternCount = kPatternCount; kScaleBufferSize = kPaddedPatternCount; @@ -205,6 +206,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, // kernels = new KernelLauncher(gpu); hWeightsCache = (Real*) gpu->CallocHost(kCategoryCount, sizeof(Real)); + hPatternWeightsCache = (Real*) gpu->CallocHost(kPatternCount, sizeof(Real)); hFrequenciesCache = (Real*) gpu->CallocHost(kPaddedStateCount, sizeof(Real)); hPartialsCache = (Real*) gpu->CallocHost(kPartialsSize, sizeof(Real)); @@ -215,7 +217,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, for (int i = 0; i < kEigenDecompCount; i++) { hInstantaneousMatrices[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); } - dInstantaneousMatrices = (cusparseSpMatDescr_t *) gpu->calloc(sizeof(cusparseSpMatDescr_t), kEigenDecompCount); + dInstantaneousMatrices = (cusparseSpMatDescr_t *) calloc(sizeof(cusparseSpMatDescr_t), kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { dInstantaneousMatrices[i] = NULL; } @@ -223,7 +225,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dMatrixCsrColumnsCache = NULL; currentCacheNNZ = 0; - dPartials = (cusparseDnMatDescr_t *) gpu->calloc(sizeof(cusparseDnMatDescr_t), kPartialsBufferCount * kCategoryCount); + dPartials = (cusparseDnMatDescr_t *) calloc(sizeof(cusparseDnMatDescr_t), kPartialsBufferCount * kCategoryCount); for (int i = 0; i < kPartialsBufferCount * kCategoryCount; i++) { dPartials[i] = NULL; } @@ -231,7 +233,12 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dWeights = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); dFrequencies = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); + hCategoryRates = (double**) calloc(sizeof(double*),kEigenDecompCount); // Keep in double-precision + hCategoryRates[0] = (double*) gpu->MallocHost(sizeof(double) * kCategoryCount); + checkHostMemory(hCategoryRates[0]); + // dMatrices = (GPUPtr*) malloc(sizeof(GPUPtr) * kMatrixCount); + CHECK_CUDA(cudaMalloc((void**) &dPatternWeightsCache, kPatternCount * sizeof(Real))) @@ -270,6 +277,31 @@ char* BeagleGPUActionImpl::getInstanceName() { } #endif +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setPatternWeights(const double* inPatternWeights) { + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::setPatternWeights\n"); +#endif + +//#ifdef DOUBLE_PRECISION +// const double* tmpWeights = inPatternWeights; +//#else +// Real* tmpWeights = hPatternWeightsCache; +// MEMCNV(hPatternWeightsCache, inPatternWeights, kPatternCount, Real); +//#endif +// const Real* tmpWeights = beagleCastIfNecessary(inPatternWeights, hPatternWeightsCache, kPatternCount); + CHECK_CUDA(cudaMemcpy(dPatternWeightsCache, inPatternWeights, kPatternCount * sizeof(Real), cudaMemcpyHostToDevice)) + + CHECK_CUSPARSE(cusparseCreateDnVec(dPatternWeights, kPatternCount, dPatternWeightsCache, CUDA_R_64F)) + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setPatternWeights\n"); +#endif + + return BEAGLE_SUCCESS; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setTipStates(int tipIndex, const int* inStates) { @@ -314,7 +346,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const if (tipIndex < kTipCount) { if (dPartials[tipIndex] == NULL) { for (int i = 0; i < kCategoryCount; i++) { - CHECK_CUSPARSE(cusparseCreateDnMat(dPartials[kPartialsBufferCount * i + tipIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, hPartialsCache, + CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[kPartialsBufferCount * i + tipIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, hPartialsCache, CUDA_R_64F, CUSPARSE_ORDER_COL)) } } @@ -369,12 +401,17 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, CHECK_CUDA(cudaMemcpy(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), sizeof(int) * currentNNZ, cudaMemcpyHostToDevice)) CHECK_CUDA(cudaMemcpy(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), sizeof(Real) * currentNNZ, cudaMemcpyHostToDevice)) - CHECK_CUSPARSE(cusparseCreateCsr(dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, + CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, dMatrixCsrOffsetsCache, dMatrixCsrColumnsCache, dMatrixCsrValuesCache, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, - CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F) ) + CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F)) - return BEAGLE_ERROR_NO_IMPLEMENTATION; + +//#ifdef BEAGLE_DEBUG_FLOW +// std::cerr<<"Setting host matrix: "< Date: Fri, 31 May 2024 16:27:55 -0700 Subject: [PATCH 067/421] draft basta gradient cpu code --- examples/hmctest/hmcBastaTest.cpp | 499 +++++++++++++++ libhmsbeagle/BeagleImpl.h | 19 + libhmsbeagle/CPU/BeagleCPUImpl.h | 53 +- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 593 +++++++++++++++++- libhmsbeagle/GPU/BeagleGPUImpl.h | 34 +- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 102 +-- .../JNI/beagle_basta_BastaJNIWrapper.cpp | 71 +++ .../JNI/beagle_basta_BastaJNIWrapper.h | 9 + libhmsbeagle/beagle.cpp | 68 ++ libhmsbeagle/beagle.h | 24 + 10 files changed, 1396 insertions(+), 76 deletions(-) create mode 100755 examples/hmctest/hmcBastaTest.cpp mode change 100644 => 100755 libhmsbeagle/BeagleImpl.h mode change 100644 => 100755 libhmsbeagle/CPU/BeagleCPUImpl.h mode change 100644 => 100755 libhmsbeagle/CPU/BeagleCPUImpl.hpp mode change 100644 => 100755 libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp mode change 100644 => 100755 libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h mode change 100644 => 100755 libhmsbeagle/beagle.cpp mode change 100644 => 100755 libhmsbeagle/beagle.h diff --git a/examples/hmctest/hmcBastaTest.cpp b/examples/hmctest/hmcBastaTest.cpp new file mode 100755 index 00000000..60415ef2 --- /dev/null +++ b/examples/hmctest/hmcBastaTest.cpp @@ -0,0 +1,499 @@ +/* + * tinyTest.c + * BEAGLE + * + * Created by Andrew Rambaut on 20/03/2009. + * Copyright 2009 __MyCompanyName__. All rights reserved. + * + */ +#include +#include +#include +#include +#include +#include +#include + +//#define JC + +#ifdef _WIN32 +#include +#endif + +#include "libhmsbeagle/beagle.h" + +char *human = (char*)"A"; +char *chimp = (char*)"G"; +char *gorilla = (char*)"G"; + +//char *human = (char*)"G"; +//char *chimp = (char*)"G"; +//char *gorilla = (char*)"A"; + + +//char *human = (char*)"GAGAAATATGTCTGATAAAAGAGTTACTTTGATAGAGTAAATAATAGGAGCTTAAACCCCCTTATTTCTACTAGGACTATGAGAATCGAACCCATCCCTGAGAATCCAAAATTCTCCGTGCCACCTATCACACCCCATCCTAAGTAAGGTCAGCTAAATAAGCTATCGGGCCCATACCCCGAAAATGTTGGTTATACCCTTCCCGTACTAAGAAATTTAGGTTAAATACAGACCAAGAGCCTTCAAAGCCCTCAGTAAGTTG-CAATACTTAATTTCTGTAAGGACTGCAAAACCCCACTCTGCATCAACTGAACGCAAATCAGCCACTTTAATTAAGCTAAGCCCTTCTAGACCAATGGGACTTAAACCCACAAACACTTAGTTAACAGCTAAGCACCCTAATCAAC-TGGCTTCAATCTAAAGCCCCGGCAGG-TTTGAAGCTGCTTCTTCGAATTTGCAATTCAATATGAAAA-TCACCTCGGAGCTTGGTAAAAAGAGGCCTAACCCCTGTCTTTAGATTTACAGTCCAATGCTTCA-CTCAGCCATTTTACCACAAAAAAGGAAGGAATCGAACCCCCCAAAGCTGGTTTCAAGCCAACCCCATGGCCTCCATGACTTTTTCAAAAGGTATTAGAAAAACCATTTCATAACTTTGTCAAAGTTAAATTATAGGCT-AAATCCTATATATCTTA-CACTGTAAAGCTAACTTAGCATTAACCTTTTAAGTTAAAGATTAAGAGAACCAACACCTCTTTACAGTGA"; +//char *chimp = (char*)"GGGAAATATGTCTGATAAAAGAATTACTTTGATAGAGTAAATAATAGGAGTTCAAATCCCCTTATTTCTACTAGGACTATAAGAATCGAACTCATCCCTGAGAATCCAAAATTCTCCGTGCCACCTATCACACCCCATCCTAAGTAAGGTCAGCTAAATAAGCTATCGGGCCCATACCCCGAAAATGTTGGTTACACCCTTCCCGTACTAAGAAATTTAGGTTAAGCACAGACCAAGAGCCTTCAAAGCCCTCAGCAAGTTA-CAATACTTAATTTCTGTAAGGACTGCAAAACCCCACTCTGCATCAACTGAACGCAAATCAGCCACTTTAATTAAGCTAAGCCCTTCTAGATTAATGGGACTTAAACCCACAAACATTTAGTTAACAGCTAAACACCCTAATCAAC-TGGCTTCAATCTAAAGCCCCGGCAGG-TTTGAAGCTGCTTCTTCGAATTTGCAATTCAATATGAAAA-TCACCTCAGAGCTTGGTAAAAAGAGGCTTAACCCCTGTCTTTAGATTTACAGTCCAATGCTTCA-CTCAGCCATTTTACCACAAAAAAGGAAGGAATCGAACCCCCTAAAGCTGGTTTCAAGCCAACCCCATGACCTCCATGACTTTTTCAAAAGATATTAGAAAAACTATTTCATAACTTTGTCAAAGTTAAATTACAGGTT-AACCCCCGTATATCTTA-CACTGTAAAGCTAACCTAGCATTAACCTTTTAAGTTAAAGATTAAGAGGACCGACACCTCTTTACAGTGA"; +//char *gorilla = (char*)"AGAAAATATGTCTGATAAAAGAGTTACTTTGATAGAGTAAATAATAGAGGTTTAAACCCCCTTATTTCTACTAGGACTATGAGAATTGAACCCATCCCTGAGAATCCAAAATTCTCCGTGCCACCTGTCACACCCCATCCTAAGTAAGGTCAGCTAAATAAGCTATCGGGCCCATACCCCGAAAATGTTGGTCACATCCTTCCCGTACTAAGAAATTTAGGTTAAACATAGACCAAGAGCCTTCAAAGCCCTTAGTAAGTTA-CAACACTTAATTTCTGTAAGGACTGCAAAACCCTACTCTGCATCAACTGAACGCAAATCAGCCACTTTAATTAAGCTAAGCCCTTCTAGATCAATGGGACTCAAACCCACAAACATTTAGTTAACAGCTAAACACCCTAGTCAAC-TGGCTTCAATCTAAAGCCCCGGCAGG-TTTGAAGCTGCTTCTTCGAATTTGCAATTCAATATGAAAT-TCACCTCGGAGCTTGGTAAAAAGAGGCCCAGCCTCTGTCTTTAGATTTACAGTCCAATGCCTTA-CTCAGCCATTTTACCACAAAAAAGGAAGGAATCGAACCCCCCAAAGCTGGTTTCAAGCCAACCCCATGACCTTCATGACTTTTTCAAAAGATATTAGAAAAACTATTTCATAACTTTGTCAAGGTTAAATTACGGGTT-AAACCCCGTATATCTTA-CACTGTAAAGCTAACCTAGCGTTAACCTTTTAAGTTAAAGATTAAGAGTATCGGCACCTCTTTGCAGTGA"; + +int* getStates(char *sequence, int repeats) { + int n = strlen(sequence); + int *states = (int*) malloc(sizeof(int) * n * repeats); + + int k = 0; + for (int r = 0; r < repeats; ++r) { + for (int i = 0; i < n; i++) { + switch (sequence[i]) { + case 'A': + states[k++] = 0; + break; + case 'C': + states[k++] = 1; + break; + case 'G': + states[k++] = 2; + break; + case 'T': + states[k++] = 3; + break; + default: + states[k++] = 4; + break; + } + } + } + return states; +} + +double* getPartials(char *sequence, int repeats) { + int n = strlen(sequence); + double *partials = (double*)malloc(sizeof(double) * n * 4); + + int k = 0; + for (int i = 0; i < n; i++) { + switch (sequence[i]) { + case 'A': + partials[k++] = 1; + partials[k++] = 0; + partials[k++] = 0; + partials[k++] = 0; + break; + case 'C': + partials[k++] = 0; + partials[k++] = 1; + partials[k++] = 0; + partials[k++] = 0; + break; + case 'G': + partials[k++] = 0; + partials[k++] = 0; + partials[k++] = 1; + partials[k++] = 0; + break; + case 'T': + partials[k++] = 0; + partials[k++] = 0; + partials[k++] = 0; + partials[k++] = 1; + break; + default: + partials[k++] = 1; + partials[k++] = 1; + partials[k++] = 1; + partials[k++] = 1; + break; + } + } + return partials; +} + +void printFlags(long inFlags) { + if (inFlags & BEAGLE_FLAG_PROCESSOR_CPU) fprintf(stdout, " PROCESSOR_CPU"); + if (inFlags & BEAGLE_FLAG_PROCESSOR_GPU) fprintf(stdout, " PROCESSOR_GPU"); + if (inFlags & BEAGLE_FLAG_PROCESSOR_FPGA) fprintf(stdout, " PROCESSOR_FPGA"); + if (inFlags & BEAGLE_FLAG_PROCESSOR_CELL) fprintf(stdout, " PROCESSOR_CELL"); + if (inFlags & BEAGLE_FLAG_PRECISION_DOUBLE) fprintf(stdout, " PRECISION_DOUBLE"); + if (inFlags & BEAGLE_FLAG_PRECISION_SINGLE) fprintf(stdout, " PRECISION_SINGLE"); + if (inFlags & BEAGLE_FLAG_COMPUTATION_ASYNCH) fprintf(stdout, " COMPUTATION_ASYNCH"); + if (inFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) fprintf(stdout, " COMPUTATION_SYNCH"); + if (inFlags & BEAGLE_FLAG_EIGEN_REAL) fprintf(stdout, " EIGEN_REAL"); + if (inFlags & BEAGLE_FLAG_EIGEN_COMPLEX) fprintf(stdout, " EIGEN_COMPLEX"); + if (inFlags & BEAGLE_FLAG_SCALING_MANUAL) fprintf(stdout, " SCALING_MANUAL"); + if (inFlags & BEAGLE_FLAG_SCALING_AUTO) fprintf(stdout, " SCALING_AUTO"); + if (inFlags & BEAGLE_FLAG_SCALING_ALWAYS) fprintf(stdout, " SCALING_ALWAYS"); + if (inFlags & BEAGLE_FLAG_SCALING_DYNAMIC) fprintf(stdout, " SCALING_DYNAMIC"); + if (inFlags & BEAGLE_FLAG_SCALERS_RAW) fprintf(stdout, " SCALERS_RAW"); + if (inFlags & BEAGLE_FLAG_SCALERS_LOG) fprintf(stdout, " SCALERS_LOG"); + if (inFlags & BEAGLE_FLAG_VECTOR_NONE) fprintf(stdout, " VECTOR_NONE"); + if (inFlags & BEAGLE_FLAG_VECTOR_SSE) fprintf(stdout, " VECTOR_SSE"); + if (inFlags & BEAGLE_FLAG_VECTOR_AVX) fprintf(stdout, " VECTOR_AVX"); + if (inFlags & BEAGLE_FLAG_THREADING_NONE) fprintf(stdout, " THREADING_NONE"); + if (inFlags & BEAGLE_FLAG_THREADING_OPENMP) fprintf(stdout, " THREADING_OPENMP"); + if (inFlags & BEAGLE_FLAG_FRAMEWORK_CPU) fprintf(stdout, " FRAMEWORK_CPU"); + if (inFlags & BEAGLE_FLAG_FRAMEWORK_CUDA) fprintf(stdout, " FRAMEWORK_CUDA"); + if (inFlags & BEAGLE_FLAG_FRAMEWORK_OPENCL) fprintf(stdout, " FRAMEWORK_OPENCL"); +} + +int main( int argc, const char* argv[] ) +{ + // print resource list + BeagleResourceList* rList; + rList = beagleGetResourceList(); + fprintf(stdout, "Available resources:\n"); + for (int i = 0; i < rList->length; i++) { + fprintf(stdout, "\tResource %i:\n\t\tName : %s\n", i, rList->list[i].name); + fprintf(stdout, "\t\tDesc : %s\n", rList->list[i].description); + fprintf(stdout, "\t\tFlags:"); + printFlags(rList->list[i].supportFlags); + fprintf(stdout, "\n"); + } + fprintf(stdout, "\n"); + +// bool scaling = true; + bool scaling = false; // disable scaling for now + + bool doJC = true; + + bool singlePrecision = false; + bool useSSE = true; + + // is nucleotides... + int stateCount = 4; + + int nRepeats = 1; + + // get the number of site patterns + int nPatterns = strlen(human) * nRepeats; + + // change # rate category to 2 +// int rateCategoryCount = 4; + int rateCategoryCount = 1; + + int scaleCount = (scaling ? 7 : 0); + + bool useGpu = argc > 1 && strcmp(argv[1] , "--gpu") == 0; + + bool useTipStates = false; + + int whichDevice = -1; + if (useGpu) { + if (argc > 2) { + whichDevice = atol(argv[2]); + if (whichDevice < 0) { + whichDevice = -1; + } + } + } + + BeagleInstanceDetails instDetails; + + long preferenceFlags = BEAGLE_FLAG_SCALERS_RAW; + + if (useGpu) { + preferenceFlags |= BEAGLE_FLAG_PROCESSOR_GPU; + } else { + preferenceFlags |= BEAGLE_FLAG_PROCESSOR_CPU; + } + + if (singlePrecision) { + preferenceFlags |= BEAGLE_FLAG_PRECISION_SINGLE; + } else { + preferenceFlags |= BEAGLE_FLAG_PRECISION_DOUBLE; + } + + long requirementFlags = BEAGLE_FLAG_EIGEN_REAL; + if (useSSE) { + requirementFlags |= BEAGLE_FLAG_VECTOR_SSE; + } else { + requirementFlags |= BEAGLE_FLAG_VECTOR_NONE; + } + + // create an instance of the BEAGLE library + int instance = beagleCreateInstance( + 0, /**< Number of tip data elements (input) */ + 20, /**< Number of partials buffers to create (input) */ + 0, /**< Number of compact state representation buffers to create (input) */ + stateCount, /**< Number of states in the continuous-time Markov chain (input) */ + 1, /**< Number of site patterns to be handled by the instance (input) */ + 2, /**< Number of rate matrix eigen-decomposition buffers to allocate (input) */ + 4, /**< Number of rate matrix buffers (input) */ + rateCategoryCount,/**< Number of rate categories (input) */ + 1, /**< Number of scaling buffers */ + NULL, /**< List of potential resource on which this instance is allowed (input, NULL implies no restriction */ + whichDevice >= 0 ? 1 : 0, /**< Length of resourceList list (input) */ + 0, + 32, /**< Bit-flags indicating required implementation characteristics, see BeagleFlags (input) */ + &instDetails); + + beagleAllocateBastaBuffers(instance,5,4); + + if (instance < 0) { + fprintf(stderr, "Failed to obtain BEAGLE instance\n\n"); + exit(1); + } + + int rNumber = instDetails.resourceNumber; + fprintf(stdout, "Using resource %i:\n", rNumber); + fprintf(stdout, "\tRsrc Name : %s\n",instDetails.resourceName); + fprintf(stdout, "\tImpl Name : %s\n", instDetails.implName); + fprintf(stdout, "\tImpl Desc : %s\n", instDetails.implDescription); + fprintf(stdout, "\n"); + + if (useTipStates) { + // set the sequences for each tip using state likelihood arrays + int *humanStates = getStates(human, nRepeats); + int *chimpStates = getStates(chimp, nRepeats); + int *gorillaStates = getStates(gorilla, nRepeats); + + beagleSetTipStates(instance, 0, humanStates); + beagleSetTipStates(instance, 1, chimpStates); + beagleSetTipStates(instance, 2, gorillaStates); + + free(humanStates); + free(chimpStates); + free(gorillaStates); + + } else { + // set the sequences for each tip using partial likelihood arrays + double *humanPartials = getPartials(human, nRepeats); + double *chimpPartials = getPartials(chimp, nRepeats); + double *gorillaPartials = getPartials(gorilla, nRepeats); + + beagleSetPartials(instance, 0, humanPartials); + beagleSetPartials(instance, 1, chimpPartials); + beagleSetPartials(instance, 2, gorillaPartials); + + free(humanPartials); + free(chimpPartials); + free(gorillaPartials); + } + +#ifdef _WIN32 + std::vector rates(rateCategoryCount); +#else + double rates[rateCategoryCount]; +#endif + + + + + rates[0] = 1.0; + + beagleSetCategoryRates(instance, &rates[0]); + + double* patternWeights = (double*) malloc(sizeof(double) * nPatterns); + + for (int i = 0; i < nPatterns; i++) { + patternWeights[i] = 1.0; + } + + beagleSetPatternWeights(instance, patternWeights); + + // create base frequency array +// double freqs[4] = { 0.1, 0.3, 0.2, 0.4 }; + double freqs[4] = { 0.25, 0.25, 0.25, 0.25 }; + + beagleSetStateFrequencies(instance, 0, freqs); + + // create an array containing site category weights +#ifdef _WIN32 + std::vector weights(rateCategoryCount); +#else + double weights[rateCategoryCount]; +#endif + for (int i = 0; i < rateCategoryCount; i++) { + weights[i] = 1.0/rateCategoryCount; +// weights[i] = 2.0 * double(i + 1)/ double(rateCategoryCount * (rateCategoryCount + 1)); + } + + beagleSetCategoryWeights(instance, 0, &weights[0]); + + + ///eigen decomposition of the HKY85 model + double evec[4 * 4] = {-0.6532830113402044, 0.4999979842955143, -0.27059808322435847, 0.5000000001499998, 0.2705993899275025, -0.4999994206514062, -0.6532813709797044, 0.4999999998499999, -0.27059644259896054, -0.5000005794441943, 0.65328170474785, 0.5000000000499999, 0.6532800643812152, 0.5000020156000878, 0.2705977493031389, 0.49999999994999994}; + + double ivec[4 * 4] = {-0.6532830113402049, 0.270599389927502, -0.27059644259896076, 0.6532800643812157, 0.49999798429551423, -0.49999942065140635, -0.5000005794441948, 0.500002015600088, -0.2705980832243587, -0.6532813709797037, 0.6532817047478499, 0.2705977493031391, 0.5000000001499999, 0.49999999985000004, 0.5000000000500006, 0.49999999994999994}; + + ///array of real parts + array of imaginary parts + double eval[8] = {-1.0000000001414213, -1.0000000000999998, -0.9999999998585786, 1.0000023031864202E-10, 0.0, 0.0, 0.0, 0.0}; + + ///Q^T matrix +// double QT[4 * 4] = { +// -1.2857138, 0.1428570, 0.1428570, 0.1428570, +// 0.4285712, -0.9999997, 0.4285714, 0.4285713, +// 0.2857142, 0.2857143, -1.1428568, 0.2857142, +// 0.5714284, 0.5714284, 0.5714284, -0.8571426 +// }; + + double Q[4 * 4 * 1] = { + -0.7499999999, 0.2499999999, 0.2500000001, 0.2499999999, + 0.2500000001, -0.7500000001, 0.2500000001, 0.2499999999, + 0.2500000001, 0.2499999999, -0.7499999999, 0.2499999999, + 0.2500000001, 0.2499999999, 0.2500000001, -0.7500000001 + }; + + + std::vector scaledQ(4 * 4 * 1); +// std::vector scaledQT(4 * 4 * 2); + + for (int rate = 0; rate < rateCategoryCount; ++rate) { + for (int entry = 0; entry < stateCount * stateCount; ++entry) { + scaledQ[entry + rate * stateCount * stateCount] = Q[entry + rate * stateCount * stateCount] * rates[rate]; + } + } + + // set the Eigen decomposition + beagleSetEigenDecomposition(instance, 0, evec, ivec, eval); + + // a list of indices and edge lengths + int transitionMatrixIndices[3] = { 0, 1, 2}; + double edgeLengths[3] = {0, 0.6, 0.7}; + //double edgeLengths[4] = { 1.0, 1.0, 1.0, 1.0}; + + // tell BEAGLE to populate the transition matrices for the above edge lengths + beagleUpdateTransitionMatrices(instance, // instance + 0, // eigenIndex + transitionMatrixIndices, // probabilityIndices + NULL, // firstDerivativeIndices + NULL, // secondDervativeIndices + edgeLengths, // edgeLengths + 3); // count + + beagleSetTransitionMatrix(instance, 3, scaledQ.data(), 0.0); + + + double* matrix1 = (double*) malloc(sizeof(double) * stateCount * stateCount * rateCategoryCount); + double* matrix2 = (double*) malloc(sizeof(double) * stateCount * stateCount * rateCategoryCount); + double* matrix3 = (double*) malloc(sizeof(double) * stateCount * stateCount * rateCategoryCount); + + beagleGetTransitionMatrix(instance, 0, matrix1); + beagleGetTransitionMatrix(instance, 1, matrix2); + beagleGetTransitionMatrix(instance, 2, matrix3); + + int nodeId = 0; + std::cout << "Matrix for node " << nodeId << std::endl; + double* mat = matrix1; + { + int offset = 0; + for (int r = 0; r < rateCategoryCount; r++) { + std::cout << " rate category" << r + 1 << ": \n"; + for (int i = 0; i < stateCount; i++) { + for (int j = 0; j < stateCount; j++) { + std::cout << mat[offset++] << ", "; + } + std::cout << std::endl; + } + std::cout << std::endl; + } + } + + std::cout << "Matrix for node " << nodeId << std::endl; + double* mat2 = matrix2; + { + int offset = 0; + for (int r = 0; r < rateCategoryCount; r++) { + std::cout << " rate category" << r + 1 << ": \n"; + for (int i = 0; i < stateCount; i++) { + for (int j = 0; j < stateCount; j++) { + std::cout << mat2[offset++] << ", "; + } + std::cout << std::endl; + } + std::cout << std::endl; + } + } + + std::cout << "Matrix for node " << nodeId << std::endl; + double* mat3 = matrix3; + { + int offset = 0; + for (int r = 0; r < rateCategoryCount; r++) { + std::cout << " rate category" << r + 1 << ": \n"; + for (int i = 0; i < stateCount; i++) { + for (int j = 0; j < stateCount; j++) { + std::cout << mat3[offset++] << ", "; + } + std::cout << std::endl; + } + std::cout << std::endl; + } + } + + // create a list of partial likelihood update operations + // the order is [dest, destScaling, source1, matrix1, source2, matrix2] + BastaOperation operations[4] = {3, 2, 0, -1, -1, 3, -1, 0, 4, 0, 1, 1, 1, 5, 6, 1, 7, 3, 1, -1, -1, 7, -1, 1, 8, 4, 2, 7, 2, 9, 10, 2}; + + int rootIndex = 4; + int intervals[4] = {0,1,3,4}; + int intervalSize = 4; + // update the partials + beagleUpdateBastaPartials(instance, + operations, + 4, + intervals, + intervalSize, + 0, + 0); + + + double * nodePartials = (double*) malloc(sizeof(double) * stateCount * nPatterns * rateCategoryCount); + + for(int i = 0; i < 5; i++){ + beagleGetPartials(instance, 3+i, BEAGLE_OP_NONE, nodePartials); + + std::cout<<"node Partial for node "<< 4-i << ": \n"; + + int l = 0; + for(int s = 0; s < rateCategoryCount; s++){ + std::cout<<" rate category"<< s+1<< ": \n"; + for(int j=0; j < stateCount; j++){ + std::cout< gBastaBuffers; + std::vector gBastaGradBuffers; std::vector gCoalescentBuffers; int kCoalescentBufferLength; int kCoalescentBufferCount; @@ -291,6 +297,10 @@ class BeagleCPUImpl : public BeagleImpl { const double* edgeLengths, int count); + int updateTransitionMatricesGrad(const int* probabilityIndices, + const double* edgeLengths, + int count); + int updateTransitionMatricesWithModelCategories(int* eigenIndices, const int* probabilityIndices, const int* firstDerivativeIndices, @@ -298,7 +308,6 @@ class BeagleCPUImpl : public BeagleImpl { const double* edgeLengths, int count); - int updateTransitionMatricesWithMultipleModels(const int* eigenIndices, const int* categoryRateIndices, const int* probabilityIndices, @@ -462,6 +471,13 @@ class BeagleCPUImpl : public BeagleImpl { int intervalCount, int populationSizesIndex, int coalescentIndex); + + int updateBastaPartialsGrad(const int* operations, + int operationCount, + const int* intervals, + int intervalCount, + int populationSizesIndex, + int coalescentIndex); int accumulateBastaPartials(const int* operations, int operationCount, @@ -472,6 +488,15 @@ class BeagleCPUImpl : public BeagleImpl { int coalescentIndex, double* out); + int accumulateBastaPartialsGrad(const int *operations, + const int operationCount, + const int *intervalStarts, + const int intervalStartsCount, + const double *intervalLengths, + const int populationSizesIndex, + const int coalescentIndex, + double *out); + int allocateBastaBuffers(int bufferCount, int bufferLength); @@ -484,11 +509,26 @@ class BeagleCPUImpl : public BeagleImpl { int endBuffer1, int endBuffer2, int interval); + void reduceWithinIntervalGrad(REALTYPE*** eGrad, REALTYPE*** fGrad, + REALTYPE*** gGrad, REALTYPE*** hGrad, + int startBuffer1, int startBuffer2, + int endBuffer1, int endBuffer2, + int interval); + REALTYPE reduceAcrossIntervals(REALTYPE* e, REALTYPE* f, // TODO move to protected REALTYPE* g, REALTYPE* h, int interval, REALTYPE length, const REALTYPE* sizes, const REALTYPE* coalescent); + + void reduceAcrossIntervalsGrad(REALTYPE *e, REALTYPE *f, + REALTYPE *g, REALTYPE *h, + REALTYPE ***eGrad, REALTYPE ***fGrad, + REALTYPE ***gGrad, REALTYPE ***hGrad, + REALTYPE **resultGrad, + int interval, REALTYPE length, + const REALTYPE *sizes, + const REALTYPE *coalescent); int block(void); @@ -821,6 +861,17 @@ class BeagleCPUImpl : public BeagleImpl { const int op, const REALTYPE* sizes, REALTYPE* coalescent); + + void updateInnerBastaPartialsGrad(const int* operations, + const int begin, + const int end, + const REALTYPE* sizes, + const REALTYPE* coalescent); + + void matrixMultiply(const REALTYPE* matrices1, + const REALTYPE* matrices2, + REALTYPE* output, + int n); private: diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp old mode 100644 new mode 100755 index 024f27a3..a3309e85 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -133,6 +133,50 @@ BeagleCPUImpl::~BeagleCPUImpl() { } free(gTransitionMatrices); + for (int i = 0; i < kStateCount; i++) { + for (int j = 0; j < kStateCount; j++) { + for (int k = 0; k < kMatrixCount; k++) { + if (gTransitionMatricesGrad[i][j][k] != NULL) + free(gTransitionMatricesGrad[i][j][k]); + } + if (gTransitionMatricesGrad[i][j] != NULL) + free(gTransitionMatricesGrad[i][j]); + } + if (gTransitionMatricesGrad[i] != NULL) + free(gTransitionMatricesGrad[i]); + } + free(gTransitionMatricesGrad); + + for (int i = 0; i < kStateCount; i++) { + for (int j = 0; j < kStateCount; j++) { + for (int k = 0; k < kBufferCount; k++) { + if (gPartialsGrad[i][j][k] != NULL) + free(gPartialsGrad[i][j][k]); + } + if (gPartialsGrad[i][j] != NULL) + free(gPartialsGrad[i][j]); + } + if (gPartialsGrad[i] != NULL) + free(gPartialsGrad[i]); + } + free(gPartialsGrad); + + for (int i = 0; i < kStateCount; i++) { + for (int j = 0; j < kStateCount; j++) { + if (coalescentGrad[i][j] != NULL) + free(coalescentGrad[i][j]); + } + if (coalescentGrad[i] != NULL) + free(coalescentGrad[i]); + } + free(coalescentGrad); + + + for (int i = 0; i < kStateCount; i++) { + if (tempGrad[i] != NULL) + free(tempGrad[i]); + } + for(unsigned int i=0; i::createInstance(int tipCount, if (gPartials == NULL) throw std::bad_alloc(); + // TODO: deconstrucor should free all of these + gPartialsGrad = (REALTYPE****) malloc(sizeof(REALTYPE***) * kStateCount); + for (int i = 0; i < kStateCount; i++) { + gPartialsGrad[i] = (REALTYPE***) malloc(sizeof(REALTYPE**) * kStateCount); + if (gPartialsGrad[i] == NULL) + throw std::bad_alloc(); + for (int j = 0; j < kStateCount; j++) { + gPartialsGrad[i][j] = (REALTYPE**) malloc(sizeof(REALTYPE*) * kBufferCount); + if (gPartialsGrad[i][j] == NULL) + throw std::bad_alloc(); + for (int k = 0; k < kBufferCount; k++) { + gPartialsGrad[i][j][k] = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPartialsSize); + if (gPartialsGrad[i][j][k] == NULL) + throw std::bad_alloc(); + } + } + } + // TODO: figure out the exact number of coalescent intervals + int MAX_NUM_COALESCENT_INTERVALS = 5000; + + coalescentGrad = (REALTYPE***) malloc(sizeof(REALTYPE***) * kStateCount); + for (int i = 0; i < kStateCount; i++) { + coalescentGrad[i] = (REALTYPE**) malloc(sizeof(REALTYPE**) * kStateCount); + if (coalescentGrad[i] == NULL) + throw std::bad_alloc(); + for (int j = 0; j < kStateCount; j++) { + coalescentGrad[i][j] = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * MAX_NUM_COALESCENT_INTERVALS); + if (coalescentGrad[i][j] == NULL) + throw std::bad_alloc(); + } + } + +// efghGrad = (REALTYPE****) malloc(sizeof(REALTYPE***) * 4); +// for (int i = 0; i < 4; i++) { +// efghGrad[i] = (REALTYPE***) malloc(sizeof(REALTYPE**) * kStateCount); +// if (efghGrad[i] == NULL) +// throw std::bad_alloc(); +// for (int j = 0; j < kStateCount; j++) { +// efghGrad[i][j] = (REALTYPE**) malloc(sizeof(REALTYPE*) * kStateCount); +// if (efghGrad[i][j] == NULL) +// throw std::bad_alloc(); +// for (int k = 0; k < kStateCount; k++) { +// efghGrad[i][j][k] = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPartialsPaddedStateCount * kCoalescentBufferLength); +// if (efghGrad[i][j][k] == NULL) +// throw std::bad_alloc(); +// } +// } +// } + + tempGrad = (REALTYPE**) malloc(sizeof(REALTYPE*) * kStateCount); + for (int i = 0; i < kStateCount; i++) { + tempGrad[i] = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kStateCount); + if (tempGrad[i] == NULL) + throw std::bad_alloc(); + } + gStateFrequencies = (REALTYPE**) calloc(sizeof(REALTYPE*), kEigenDecompCount); if (gStateFrequencies == NULL) throw std::bad_alloc(); @@ -431,6 +531,24 @@ int BeagleCPUImpl::createInstance(int tipCount, throw std::bad_alloc(); } + // TODO: deconstrucor should free all of these + gTransitionMatricesGrad = (REALTYPE****) malloc(sizeof(REALTYPE***) * kStateCount); + for (int i = 0; i < kStateCount; i++) { + gTransitionMatricesGrad[i] = (REALTYPE***) malloc(sizeof(REALTYPE**) * kStateCount); + if (gTransitionMatricesGrad[i] == NULL) + throw std::bad_alloc(); + for (int j = 0; j < kStateCount; j++) { + gTransitionMatricesGrad[i][j] = (REALTYPE**) malloc(sizeof(REALTYPE*) * kMatrixCount); + if (gTransitionMatricesGrad[i][j] == NULL) + throw std::bad_alloc(); + for (int k = 0; k < kMatrixCount; k++) { + gTransitionMatricesGrad[i][j][k] = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kMatrixSize * kCategoryCount); + if (gTransitionMatricesGrad[i][j][k] == NULL) + throw std::bad_alloc(); + } + } + } + integrationTmp = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPatternCount * kStateCount); firstDerivTmp = (REALTYPE*) malloc(sizeof(REALTYPE) * kPatternCount * kStateCount); secondDerivTmp = (REALTYPE*) malloc(sizeof(REALTYPE) * kPatternCount * kStateCount); @@ -1250,6 +1368,33 @@ int BeagleCPUImpl::updateTransitionMatrices(int eigenIndex, return BEAGLE_SUCCESS; } +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::updateTransitionMatricesGrad(const int* probabilityIndices, + const double* edgeLengths, + int count) { + const int matrixIncr = kStateCount + T_PAD; + + for (int u = 0; u < count; u++) { + REALTYPE* transitionMat = gTransitionMatrices[probabilityIndices[u]]; + const double edgeLength = edgeLengths[u]; + for (int a = 0; a < kStateCount; a++) { + for (int b = 0; b < kStateCount; b++) { + REALTYPE* transitionMatGrad = gTransitionMatricesGrad[a][b][probabilityIndices[u]]; + for (int c = 0; c < kStateCount; c++) { + for (int d = 0; d < kStateCount; d++) { + if (d == b) { + transitionMatGrad[c*matrixIncr + b] = edgeLength * transitionMat[c*matrixIncr + a]; + } else { + transitionMatGrad[c*matrixIncr + d] = 0.0; + } + } + } + } + } + } + return BEAGLE_SUCCESS; +} + BEAGLE_CPU_TEMPLATE int BeagleCPUImpl::updateTransitionMatricesWithModelCategories(int* eigenIndices, @@ -1353,6 +1498,7 @@ int BeagleCPUImpl::allocateBastaBuffers(int bufferCount, gCoalescentBuffers.resize(kCoalescentBufferCount * kCoalescentBufferLength); gBastaBuffers.resize(kPartialsPaddedStateCount * kCoalescentBufferLength * 4); + gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleCPUImpl::allocateBastaBuffers\n"); @@ -1396,7 +1542,6 @@ void BeagleCPUImpl::updateInnerBastaPartials(const int* oper const REALTYPE* sizes, REALTYPE* coalescent) { for (int op = begin; op < end; ++op) { - const int numOps = BEAGLE_BASTA_OP_COUNT; const int destinationPartialIndex = operations[op * numOps]; @@ -1412,7 +1557,6 @@ void BeagleCPUImpl::updateInnerBastaPartials(const int* oper const int stateCountModFour = (kStateCount / 4) * 4; REALTYPE *destPartial = gPartials[destinationPartialIndex]; - // child 1 const REALTYPE *partial1 = gPartials[child1PartialIndex]; const REALTYPE *matrix1 = gTransitionMatrices[child1TransMatIndex]; @@ -1428,6 +1572,7 @@ void BeagleCPUImpl::updateInnerBastaPartials(const int* oper sum1B += matrices1Ptr[j + 1] * partial1[j + 1]; sum1A += matrices1Ptr[j + 2] * partial1[j + 2]; sum1B += matrices1Ptr[j + 3] * partial1[j + 3]; + } for (; j < kStateCount; j++) { @@ -1577,6 +1722,168 @@ void BeagleCPUImpl::updateInnerBastaPartials2(const int* ope } } +BEAGLE_CPU_TEMPLATE +void BeagleCPUImpl::updateInnerBastaPartialsGrad(const int* operations, + const int begin, + const int end, + const REALTYPE* sizes, + const REALTYPE* coalescent) { + for (int op = begin; op < end; ++op) { + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int child1PartialIndex = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + + const int matrixIncr = kStateCount + T_PAD; + const int stateCountModFour = (kStateCount / 4) * 4; + + REALTYPE *destPartial = gPartials[destinationPartialIndex]; + + // child 1 + const REALTYPE *partial1 = gPartials[child1PartialIndex]; + const REALTYPE *matrix1 = gTransitionMatrices[child1TransMatIndex]; + for (int a = 0; a < kStateCount; a++) { + for (int b = 0; b < kStateCount; b++) { + const REALTYPE *partialGrad1 = gPartialsGrad[a][b][child1PartialIndex]; +// for (int i = 0; i < 4; i++) { +// std::cout << "partialGrad1:" << partialGrad1[i] << std::endl; +// } +// for (int i = 0; i < 4; i++) { +// std::cout << "partial1:" << partial1[i] << std::endl; +// } + const REALTYPE *matrixGrad1 = gTransitionMatricesGrad[a][b][child1TransMatIndex]; + +// for (int i = 0; i < kMatrixSize; ++i) { +// std::cout << "matrix grad:" << matrixGrad1[i] << std::endl; +// } + for (int i = 0; i < kStateCount; i++) { + REALTYPE sum = 0.0; + const REALTYPE *matrices1Ptr = matrix1 + i * matrixIncr; + const REALTYPE *matricesGrad1Ptr = matrixGrad1 + i * matrixIncr; + REALTYPE *destPartialGrad = gPartialsGrad[a][b][destinationPartialIndex]; + for (int j = 0; j < kStateCount; j++) { + sum += matrices1Ptr[j] * partialGrad1[j] + matricesGrad1Ptr[j] * partial1[j]; + } + destPartialGrad[i] = sum; + } + } + } + +// for (int a = 0; a < kStateCount; ++a) { +// for (int b = 0; b < kStateCount; ++b) { +// std::cout << "a:" << a << std::endl; +// std::cout << "b:" << b << std::endl; +// for (int i = 0; i < 4; i++) { +// std::cout << " result:" << "\n"; +// std::cout << gPartialsGrad[a][b][destinationPartialIndex][i] << ' '; +// } +// std::cout << " " << "\n"; +// } +// } + + if (child2PartialIndex >= 0) { + // child 2 + const REALTYPE *partial2 = gPartials[child2PartialIndex]; + const REALTYPE *matrix2 = gTransitionMatrices[child2TransMatIndex]; + + REALTYPE *accumulation1 = gPartials[accumulation1PartialIndex]; + REALTYPE *accumulation2 = gPartials[accumulation2PartialIndex]; + + for (int a = 0; a < kStateCount; a++) { + for (int b = 0; b < kStateCount; b++) { + REALTYPE J = coalescent[intervalNumber]; + + // first half + const REALTYPE *partialGrad2 = gPartialsGrad[a][b][child2PartialIndex]; + const REALTYPE *matrixGrad2 = gTransitionMatricesGrad[a][b][child2TransMatIndex]; + + REALTYPE *destPartialGrad = gPartialsGrad[a][b][destinationPartialIndex]; + REALTYPE *accumulationGrad1 = gPartialsGrad[a][b][accumulation1PartialIndex]; + REALTYPE *accumulationGrad2 = gPartialsGrad[a][b][accumulation2PartialIndex]; + + REALTYPE partial_J_ab = REALTYPE(0); + for (int i = 0; i < kStateCount; i++) { + // sum is child2Grad + REALTYPE sum = 0.0; + const REALTYPE *matrices2Ptr = matrix2 + i * matrixIncr; + const REALTYPE *matricesGrad2Ptr = matrixGrad2 + i * matrixIncr; + + for (int j = 0; j < kStateCount; j++) { + sum += matrices2Ptr[j] * partialGrad2[j] + matricesGrad2Ptr[j] * partial2[j]; + } + REALTYPE child1Grad = gPartialsGrad[a][b][destinationPartialIndex][i]; + REALTYPE child1 = accumulation1[i]; + REALTYPE child2 = accumulation2[i]; + + REALTYPE entry = (child1Grad * child2 + child1 * sum) / sizes[i]; + partial_J_ab += entry; + + destPartialGrad[i] = entry / J; + accumulationGrad1[i] = child1Grad; + accumulationGrad2[i] = sum; + } + + // second half + for (int i = 0; i < kStateCount; i++) { + REALTYPE entry = destPartial[i]; + destPartialGrad[i] -= partial_J_ab * entry / J; + } + coalescentGrad[a][b][intervalNumber] = partial_J_ab; + } + } + } +// for (int a = 0; a < kStateCount; ++a) { +// for (int b = 0; b < kStateCount; ++b) { +// std::cout << "a:" << a << std::endl; +// std::cout << "b:" << b << std::endl; +// if (child2PartialIndex >= 0) { +// for (int i = 0; i < 4; i++) { +// std::cout << " left:" << "\n"; +// std::cout << gPartialsGrad[a][b][accumulation1PartialIndex][i] << ' '; +// } +// +// for (int i = 0; i < 4; i++) { +// std::cout << " right:" << "\n"; +// std::cout << gPartialsGrad[a][b][accumulation2PartialIndex][i] << ' '; +// } +// } +// for (int i = 0; i < 4; i++) { +// std::cout << " result:" << "\n"; +// std::cout << gPartialsGrad[a][b][destinationPartialIndex][i] << ' '; +// } +// std::cout << " " << "\n"; +// } +// } + + } +} + + + +BEAGLE_CPU_TEMPLATE +void BeagleCPUImpl::matrixMultiply(const REALTYPE* matrix1, + const REALTYPE* matrix2, + REALTYPE* output, + int n) { + #pragma omp parallel for + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + REALTYPE sum = 0.0; + for (int k = 0; k < n; k++) { + sum += matrix1[i*n + k] * matrix2[k*n + j]; + } + output[i*n + j] = sum; + } + } +} + template inline void for_each(Integer begin, Integer end, Function function, int global_num_threads) { std::vector threads; @@ -1654,6 +1961,49 @@ int BeagleCPUImpl::updateBastaPartials(const int* operations return returnCode; } +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::updateBastaPartialsGrad(const int* operations, + const int count, + const int* intervals, + const int intervalCount, + const int populationSizesIndex, + const int coalescentIndex) { + int returnCode = BEAGLE_SUCCESS; + + // REALTYPE* coalescent = gCoalescentBuffers.data() + kCoalescentBufferLength * coalescentIndex; + // std::fill(coalescent, coalescent + kCoalescentBufferLength, REALTYPE(0)); + // just for test + const REALTYPE* coalescent = gCoalescentBuffers.data() + kCoalescentBufferLength * coalescentIndex; + const REALTYPE* sizes = gStateFrequencies[populationSizesIndex]; + + bool THREADING = true; + int CUT_POINT = 1280000; + int nThreads = 2; + + if (THREADING) { + + for (int i = 0; i < intervalCount - 1; ++i) { + const int begin = intervals[i]; + const int end = intervals[i + 1]; + + if (end - begin > CUT_POINT) { + for_each(begin, end, [&](const int threadBegin, const int threadEnd) { + updateInnerBastaPartialsGrad(operations, threadBegin, threadEnd, sizes, coalescent); + }, nThreads); +// for_each(begin, end, [&](const int j) { +// updateInnerBastaPartials2(operations, j, sizes, coalescent); +// }, nThreads); + } else { + updateInnerBastaPartialsGrad(operations, begin, end, sizes, coalescent); + } + } + } else { + updateInnerBastaPartialsGrad(operations, 0, count, sizes, coalescent); + } + + return returnCode; +} + BEAGLE_CPU_TEMPLATE REALTYPE BeagleCPUImpl::reduceAcrossIntervals( REALTYPE* e, REALTYPE* f, @@ -1666,9 +2016,8 @@ REALTYPE BeagleCPUImpl::reduceAcrossIntervals( f += interval * kPartialsPaddedStateCount; g += interval * kPartialsPaddedStateCount; h += interval * kPartialsPaddedStateCount; - REALTYPE sum = REALTYPE(0); - for (int k = 0; k < kPartialsPaddedStateCount; ++k) { + for (int k = 0; k < kStateCount; ++k) { sum += (e[k] * e[k] - f[k] + g[k] * g[k] - h[k]) / sizes[k]; } @@ -1683,6 +2032,44 @@ REALTYPE BeagleCPUImpl::reduceAcrossIntervals( return logL; } + +BEAGLE_CPU_TEMPLATE +void BeagleCPUImpl::reduceAcrossIntervalsGrad( + REALTYPE* e, REALTYPE* f, + REALTYPE* g, REALTYPE* h, + REALTYPE*** eGrad, REALTYPE*** fGrad, + REALTYPE*** gGrad, REALTYPE*** hGrad, + REALTYPE** resultGrad, + int interval, REALTYPE length, + const REALTYPE* sizes, + const REALTYPE* coalescent) { + + const int offset = interval * kPartialsPaddedStateCount; + + e += offset; + g += offset; + + for (int a = 0; a < kStateCount; ++a) { + for (int b = 0; b < kStateCount; ++b) { + REALTYPE sum = REALTYPE(0); + REALTYPE* eGrad_ab = (REALTYPE*) eGrad[a][b] + offset; + REALTYPE* fGrad_ab = (REALTYPE*) fGrad[a][b] + offset; + REALTYPE* gGrad_ab = (REALTYPE*) gGrad[a][b] + offset; + REALTYPE* hGrad_ab = (REALTYPE*) hGrad[a][b] + offset; + + for (int k = 0; k < kStateCount; ++k) { + sum += (2 * e[k] * eGrad_ab[k] - fGrad_ab[k] + + 2 * g[k] * gGrad_ab[k] - hGrad_ab[k]) / sizes[k]; + } + resultGrad[a][b] = -length * sum / 4; + REALTYPE prob = coalescent[interval]; + if (prob != REALTYPE(0)) { + resultGrad[a][b] += coalescentGrad[a][b][interval] / prob; + } + } + } +} + BEAGLE_CPU_TEMPLATE void BeagleCPUImpl::reduceWithinInterval(REALTYPE* e, REALTYPE* f, REALTYPE* g, REALTYPE* h, @@ -1698,7 +2085,7 @@ void BeagleCPUImpl::reduceWithinInterval(REALTYPE* e, REALTY const REALTYPE* startPartials = gPartials[startBuffer1]; const REALTYPE* endPartials = gPartials[endBuffer1]; - for (int i = 0; i < kPartialsPaddedStateCount; ++i) { + for (int i = 0; i < kStateCount; ++i) { REALTYPE startP = startPartials[i]; e[i] += startP; f[i] += startP * startP; @@ -1713,7 +2100,7 @@ void BeagleCPUImpl::reduceWithinInterval(REALTYPE* e, REALTY const REALTYPE* startPartials = gPartials[startBuffer2]; const REALTYPE* endPartials = gPartials[endBuffer2]; - for (int i = 0; i < kPartialsPaddedStateCount; ++i) { + for (int i = 0; i < kStateCount; ++i) { REALTYPE startP = startPartials[i]; e[i] += startP; f[i] += startP * startP; @@ -1725,6 +2112,97 @@ void BeagleCPUImpl::reduceWithinInterval(REALTYPE* e, REALTY } } + +BEAGLE_CPU_TEMPLATE +void BeagleCPUImpl::reduceWithinIntervalGrad(REALTYPE*** eGrad, REALTYPE*** fGrad, + REALTYPE*** gGrad, REALTYPE*** hGrad, + int startBuffer1, int startBuffer2, + int endBuffer1, int endBuffer2, + int interval) { + const int offset = interval * kPartialsPaddedStateCount; +// std::cout << "interval:" << interval << std::endl; + const REALTYPE* startPartials = gPartials[startBuffer1]; + const REALTYPE* endPartials = gPartials[endBuffer1]; + + for (int a = 0; a < kStateCount; ++a) { + for (int b = 0; b < kStateCount; ++b) { + REALTYPE* eGrad_ab = (REALTYPE*) eGrad[a][b] + offset; + REALTYPE* fGrad_ab = (REALTYPE*) fGrad[a][b] + offset; + REALTYPE* gGrad_ab = (REALTYPE*) gGrad[a][b] + offset; + REALTYPE* hGrad_ab = (REALTYPE*) hGrad[a][b] + offset; +// std::cout << "a:" << a << std::endl; +// std::cout << "b:" << b << std::endl; + for (int i = 0; i < kStateCount; ++i) { + REALTYPE startPGrad = gPartialsGrad[a][b][startBuffer1][i]; + REALTYPE startP = startPartials[i]; +// std::cout << "startPGrad:" << startPGrad << std::endl; + eGrad_ab[i] += startPGrad; + fGrad_ab[i] += 2 * startP * startPGrad; + + + //same for endPGrad and endP + REALTYPE endPGrad = gPartialsGrad[a][b][endBuffer1][i]; + REALTYPE endP = endPartials[i]; + gGrad_ab[i] += endPGrad; + hGrad_ab[i] += 2 * endP * endPGrad; +// std::cout << "endPGrad:" << endPGrad << std::endl; + } + +// for (int i = 0; i < 4; i++) { +// std::cout << "fGrad_ab:" << "\n"; +// std::cout << fGrad_ab[i] << ' '; +// } +// for (int i = 0; i < 4; i++) { +// std::cout << "hGrad_ab:" << "\n"; +// std::cout << hGrad_ab[i] << ' '; +// } + } + } + + + if (startBuffer2 >= 0) { + // same for startBuffer2 and endBuffer2 + const REALTYPE* startPartials = gPartials[startBuffer2]; + const REALTYPE* endPartials = gPartials[endBuffer2]; + + for (int a = 0; a < kStateCount; ++a) { + for (int b = 0; b < kStateCount; ++b) { + +// std::cout << "a:" << a << std::endl; +// std::cout << "b:" << b << std::endl; + + REALTYPE* eGrad_ab = (REALTYPE*) eGrad[a][b] + offset; + REALTYPE* fGrad_ab = (REALTYPE*) fGrad[a][b] + offset; + REALTYPE* gGrad_ab = (REALTYPE*) gGrad[a][b] + offset; + REALTYPE* hGrad_ab = (REALTYPE*) hGrad[a][b] + offset; + + for (int i = 0; i < kStateCount; ++i) { + REALTYPE startPGrad = gPartialsGrad[a][b][startBuffer2][i]; + REALTYPE startP = startPartials[i]; + eGrad_ab[i] += startPGrad; + fGrad_ab[i] += 2 * startP * startPGrad; + + //same for endPGrad and endP + REALTYPE endPGrad = gPartialsGrad[a][b][endBuffer2][i]; + REALTYPE endP = endPartials[i]; + gGrad_ab[i] += endPGrad; + hGrad_ab[i] += 2 * endP * endPGrad; + } + +// for (int i = 0; i < 4; i++) { +// std::cout << "fGrad_ab:" << "\n"; +// std::cout << fGrad_ab[i] << ' '; +// } +// for (int i = 0; i < 4; i++) { +// std::cout << "hGrad_ab:" << "\n"; +// std::cout << hGrad_ab[i] << ' '; +// } + + } + } + } +} + BEAGLE_CPU_TEMPLATE int BeagleCPUImpl::accumulateBastaPartials(const int* operations, const int operationCount, @@ -1745,7 +2223,6 @@ int BeagleCPUImpl::accumulateBastaPartials(const int* operat const REALTYPE* coalescent = gCoalescentBuffers.data() + kCoalescentBufferLength * coalescentIndex; const REALTYPE* sizes = gStateFrequencies[populationSizesIndex]; - for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { // TODO execute in parallel (no race conditions) const int start = intervalStarts[interval]; const int end = intervalStarts[interval + 1]; @@ -1795,6 +2272,108 @@ int BeagleCPUImpl::accumulateBastaPartials(const int* operat return returnCode; } + +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::accumulateBastaPartialsGrad(const int* operations, + const int operationCount, + const int* intervalStarts, + const int intervalStartsCount, + const double* intervalLengths, + const int populationSizesIndex, + const int coalescentIndex, + double* out) { + int returnCode = BEAGLE_SUCCESS; + + REALTYPE* e = gBastaBuffers.data() + 0 * kPartialsPaddedStateCount * kCoalescentBufferLength; + REALTYPE* f = gBastaBuffers.data() + 1 * kPartialsPaddedStateCount * kCoalescentBufferLength; + REALTYPE* g = gBastaBuffers.data() + 2 * kPartialsPaddedStateCount * kCoalescentBufferLength; + REALTYPE* h = gBastaBuffers.data() + 3 * kPartialsPaddedStateCount * kCoalescentBufferLength; + +// std::fill(&efghGrad[0][0][0][0], &efghGrad[0][0][0][0] + sizeof(efghGrad) / sizeof(efghGrad[0][0][0][0]), REALTYPE(0)); +// +// REALTYPE*** eGrad = efghGrad[0]; +// REALTYPE*** fGrad = efghGrad[1]; +// REALTYPE*** gGrad = efghGrad[2]; +// REALTYPE*** hGrad = efghGrad[3]; + std::fill(gBastaGradBuffers.begin(), gBastaGradBuffers.end(), REALTYPE(0)); + // Create the 4D array structure using pointers into the contiguous memory block + REALTYPE**** efghGrad = new REALTYPE***[4]; + for (int i = 0; i < 4; i++) { + efghGrad[i] = new REALTYPE**[kStateCount]; + for (int j = 0; j < kStateCount; j++) { + efghGrad[i][j] = new REALTYPE*[kStateCount]; + for (int k = 0; k < kStateCount; k++) { + int offset = ((i * kStateCount + j) * kStateCount + k) * kPartialsPaddedStateCount * kCoalescentBufferLength; + efghGrad[i][j][k] = gBastaGradBuffers.data() + offset; + } + } + } + + REALTYPE*** eGrad = efghGrad[0]; + REALTYPE*** fGrad = efghGrad[1]; + REALTYPE*** gGrad = efghGrad[2]; + REALTYPE*** hGrad = efghGrad[3]; + + const REALTYPE* coalescent = gCoalescentBuffers.data() + kCoalescentBufferLength * coalescentIndex; + const REALTYPE* sizes = gStateFrequencies[populationSizesIndex]; + + for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { + const int start = intervalStarts[interval]; + const int end = intervalStarts[interval + 1]; + + for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int child1PartialIndex = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + + reduceWithinIntervalGrad(eGrad, fGrad, gGrad, hGrad, + child1PartialIndex, child2PartialIndex, + accumulation1PartialIndex, accumulation2PartialIndex, + intervalNumber); + } + } + + for (int a = 0; a < kStateCount; ++a) { + for (int b = 0; b < kStateCount; ++b) { + out[a*kStateCount+b] = (double) 0; + } + } + + for (int i = 0; i < intervalStartsCount - 1; ++i) { + const int op = intervalStarts[i]; + + const int numOps = BEAGLE_BASTA_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int child1PartialIndex = operations[op * numOps + 1]; + const int child1TransMatIndex = operations[op * numOps + 2]; + const int child2PartialIndex = operations[op * numOps + 3]; + const int child2TransMatIndex = operations[op * numOps + 4]; + const int accumulation1PartialIndex = operations[op * numOps + 5]; + const int accumulation2PartialIndex = operations[op * numOps + 6]; + const int intervalNumber = operations[op * numOps + 7]; + reduceAcrossIntervalsGrad(e, f, g, h, + eGrad, fGrad, gGrad, hGrad, + tempGrad, // save output here! + intervalNumber, intervalLengths[i], + sizes, coalescent); + for (int a = 0; a < kStateCount; ++a) { + for (int b = 0; b < kStateCount; ++b) { + out[a*kStateCount+b] += tempGrad[a][b]; + } + } + } + return returnCode; +} + //BEAGLE_CPU_TEMPLATE //int BeagleCPUImpl::calculateEdgeDerivative(const int *postBufferIndices, // const int *preBufferIndices, diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 38713f2a..b7a1c177 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -320,9 +320,9 @@ class BeagleGPUImpl : public BeagleImpl { const double* edgeLengths, int count); - // int updateTransitionMatricesGrad(const int* probabilityIndices, - // const double* edgeLengths, - // int count); + int updateTransitionMatricesGrad(const int* probabilityIndices, + const double* edgeLengths, + int count); int updateTransitionMatricesWithModelCategories(int* eigenIndices, const int* probabilityIndices, @@ -484,12 +484,12 @@ class BeagleGPUImpl : public BeagleImpl { int populationSizesIndex, int coalescentIndex); - // int updateBastaPartialsGrad(const int* operations, - // int operationCount, - // const int* intervals, - // int intervalCount, - // int populationSizesIndex, - // int coalescentIndex); + int updateBastaPartialsGrad(const int* operations, + int operationCount, + const int* intervals, + int intervalCount, + int populationSizesIndex, + int coalescentIndex); int accumulateBastaPartials(const int* operations, int operationCount, @@ -500,14 +500,14 @@ class BeagleGPUImpl : public BeagleImpl { int coalescentIndex, double* out); - // int accumulateBastaPartialsGrad(const int *operations, - // const int operationCount, - // const int *intervalStarts, - // const int intervalStartsCount, - // const double *intervalLengths, - // const int populationSizesIndex, - // const int coalescentIndex, - // double *out); + int accumulateBastaPartialsGrad(const int *operations, + const int operationCount, + const int *intervalStarts, + const int intervalStartsCount, + const double *intervalLengths, + const int populationSizesIndex, + const int coalescentIndex, + double *out); int allocateBastaBuffers(int bufferCount, int bufferLength); diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 585ebb03..ab5b2b3d 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -1904,13 +1904,13 @@ int BeagleGPUImpl::transposeTransitionMatrices( return BEAGLE_SUCCESS; } -// BEAGLE_GPU_TEMPLATE -// int BeagleGPUImpl::updateTransitionMatricesGrad(const int* probabilityIndices, -// const double* edgeLengths, -// int count) { -// return BEAGLE_ERROR_NO_IMPLEMENTATION; -// } -// + BEAGLE_GPU_TEMPLATE + int BeagleGPUImpl::updateTransitionMatricesGrad(const int* probabilityIndices, + const double* edgeLengths, + int count) { + return BEAGLE_ERROR_NO_IMPLEMENTATION; + } + BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::updateTransitionMatrices(int eigenIndex, @@ -2339,27 +2339,27 @@ int BeagleGPUImpl::updateInnerBastaPartials(const int* opera return BEAGLE_SUCCESS; } -// BEAGLE_GPU_TEMPLATE -// int BeagleGPUImpl::updateBastaPartialsGrad(const int* operations, -// const int count, -// const int* intervals, -// const int intervalCount, -// const int populationSizesIndex, -// const int coalescentIndex) { -// #ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); -// #endif -// -// int returnCode = BEAGLE_ERROR_GENERAL; -// -// fprintf(stderr, "BeagleGPUImpl::updateBastaPartials\n"); -// -// #ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tLeaving BeagleGPUImpl::updateBastaPartials\n"); -// #endif -// -// return returnCode; -// } + BEAGLE_GPU_TEMPLATE + int BeagleGPUImpl::updateBastaPartialsGrad(const int* operations, + const int count, + const int* intervals, + const int intervalCount, + const int populationSizesIndex, + const int coalescentIndex) { + #ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); + #endif + + int returnCode = BEAGLE_ERROR_GENERAL; + + fprintf(stderr, "BeagleGPUImpl::updateBastaPartials\n"); + + #ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUImpl::updateBastaPartials\n"); + #endif + + return returnCode; + } BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::accumulateBastaPartials(const int* operations, @@ -2464,29 +2464,29 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat } -// BEAGLE_GPU_TEMPLATE -// int BeagleGPUImpl::accumulateBastaPartialsGrad(const int *operations, -// const int operationCount, -// const int *intervalStarts, -// const int intervalStartsCount, -// const double *intervalLengths, -// const int populationSizesIndex, -// const int coalescentIndex, -// double *out) { -// #ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); -// #endif -// -// int returnCode = BEAGLE_ERROR_GENERAL; -// -// fprintf(stderr, "BeagleGPUImpl::accumulateBastaPartials\n"); -// -// #ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); -// #endif -// -// return returnCode; -// } + BEAGLE_GPU_TEMPLATE + int BeagleGPUImpl::accumulateBastaPartialsGrad(const int *operations, + const int operationCount, + const int *intervalStarts, + const int intervalStartsCount, + const double *intervalLengths, + const int populationSizesIndex, + const int coalescentIndex, + double *out) { + #ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); + #endif + + int returnCode = BEAGLE_ERROR_GENERAL; + + fprintf(stderr, "BeagleGPUImpl::accumulateBastaPartials\n"); + + #ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); + #endif + + return returnCode; + } BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp old mode 100644 new mode 100755 index 87948285..a7e075e5 --- a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp +++ b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -46,6 +47,43 @@ JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartials } +/* + * Class: beagle_basta_BastaJNIWrapper + * Method: updateBastaPartialsGrad + * Signature: (I[III)I + */ +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartialsGrad + (JNIEnv *env, jobject obj, jint instance, jintArray inOperations, + jint operationCount, jintArray inIntervals, jint intervalCount, + jint populationSizesIndex, jint coalescentIndex) { + + jint *operations = env->GetIntArrayElements(inOperations, NULL); + jint *intervals = env->GetIntArrayElements(inIntervals, NULL); + + jint errCode = (jint)beagleUpdateBastaPartialsGrad(instance, + (BastaOperation*) operations, operationCount, + intervals, intervalCount, + populationSizesIndex, coalescentIndex); + + env->ReleaseIntArrayElements(inOperations, operations, JNI_ABORT); + env->ReleaseIntArrayElements(inIntervals, intervals, JNI_ABORT); + + return errCode; + + } + + +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateTransitionMatricesGrad + (JNIEnv *env, jobject obj, jint instance, jintArray inProbabilityIndices, jdoubleArray inEdgeLengths, jint count) +{ + jint *probabilityIndices = env->GetIntArrayElements(inProbabilityIndices, NULL); + jdouble *edgeLengths = env->GetDoubleArrayElements(inEdgeLengths, NULL); + jint errCode = (jint)beagleUpdateTransitionMatricesGrad(instance, (int *)probabilityIndices, (double *)edgeLengths, count); + + env->ReleaseDoubleArrayElements(inEdgeLengths, edgeLengths, JNI_ABORT); + env->ReleaseIntArrayElements(inProbabilityIndices, probabilityIndices, JNI_ABORT); + return errCode; +} /* * Class: beagle_basta_BastaJNIWrapper * Method: getBastaBuffer @@ -99,3 +137,36 @@ JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_accumulateBastaPartials return errCode; } + +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_accumulateBastaPartialsGrad + (JNIEnv *env, jobject object, jint instance, + jintArray inOperations, jint operationCount, + jintArray inIntervals, jint intervalCount, + jdoubleArray inIntervalLengths, + jint populationSizesIndex, + jint coalescentIndex, + jdoubleArray out) { + + jint *operations = env->GetIntArrayElements(inOperations, NULL); + jint *intervals = env->GetIntArrayElements(inIntervals, NULL); + + jdouble *array = env->GetDoubleArrayElements(out, NULL); + jdouble *intervalLengths = env->GetDoubleArrayElements(inIntervalLengths, NULL); + + jint errCode = beagleAccumulateBastaPartialsGrad(instance, + (const BastaOperation*) operations, operationCount, + intervals, intervalCount, + (double *)intervalLengths, + populationSizesIndex, + coalescentIndex, (double *)array); + + env->ReleaseIntArrayElements(inOperations, operations, JNI_ABORT); + env->ReleaseIntArrayElements(inIntervals, intervals, JNI_ABORT); + env->ReleaseDoubleArrayElements(inIntervalLengths, intervalLengths, JNI_ABORT); + + // not using JNI_ABORT flag here because we want the values to be copied back... + env->ReleaseDoubleArrayElements(out, array, 0); + + return errCode; + + } diff --git a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h old mode 100644 new mode 100755 index 3e765f98..79c03a92 --- a/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h +++ b/libhmsbeagle/JNI/beagle_basta_BastaJNIWrapper.h @@ -39,6 +39,15 @@ JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartials JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_accumulateBastaPartials (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint, jdoubleArray, jint, jint, jdoubleArray); +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_accumulateBastaPartialsGrad + (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint, jdoubleArray, jint, jint, jdoubleArray); + +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateBastaPartialsGrad + (JNIEnv *, jobject, jint, jintArray, jint, jintArray, jint, jint, jint); + +JNIEXPORT jint JNICALL Java_beagle_basta_BastaJNIWrapper_updateTransitionMatricesGrad + (JNIEnv *, jobject, jint, jintArray, jdoubleArray, jint); + #ifdef __cplusplus } #endif diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp old mode 100644 new mode 100755 index e1adcc76..2c8c0030 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -1182,6 +1182,21 @@ int beagleUpdateTransitionMatrices(int instance, // } } +int beagleUpdateTransitionMatricesGrad(int instance, + const int* probabilityIndices, + const double* edgeLengths, + int count) { + DEBUG_START_TIME(); + DEBUG_START_ENERGY(); + beagle::BeagleImpl* beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + int returnValue = beagleInstance->updateTransitionMatricesGrad(probabilityIndices, edgeLengths, count); + DEBUG_END_TIME(); + DEBUG_END_ENERGY(); + return returnValue; +} + int beagleUpdateTransitionMatricesWithModelCategories(int instance, int* eigenIndices, const int* probabilityIndices, @@ -2005,6 +2020,30 @@ int beagleUpdateBastaPartials(const int instance, return returnValue; } +int beagleUpdateBastaPartialsGrad(const int instance, + const BastaOperation* operations, + int operationCount, + const int* intervals, + int intervalCount, + int populationSizesIndex, + int coalescentIndex) { + DEBUG_START_TIME(); + DEBUG_START_ENERGY(); + + beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) { + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + } + + int returnValue = beagleInstance->updateBastaPartialsGrad((const int*) operations, + operationCount, intervals, intervalCount, populationSizesIndex, coalescentIndex); + + DEBUG_END_TIME(); + DEBUG_END_ENERGY(); + + return returnValue; +} + int beagleAccumulateBastaPartials(const int instance, const BastaOperation* operations, int operationCount, @@ -2033,3 +2072,32 @@ int beagleAccumulateBastaPartials(const int instance, return returnValue; } + +int beagleAccumulateBastaPartialsGrad(const int instance, + const BastaOperation* operations, + int operationCount, + const int* intervalStarts, + int intervalCount, + const double* intervalLengths, + const int populationSizesIndex, + int coalescentIndex, + double* out) { + DEBUG_START_TIME(); + DEBUG_START_ENERGY(); + + beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) { + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + } + + int returnValue = beagleInstance->accumulateBastaPartialsGrad((const int*) operations, operationCount, + intervalStarts, intervalCount, intervalLengths, + populationSizesIndex, coalescentIndex, + out); + + DEBUG_END_TIME(); + DEBUG_END_ENERGY(); + + return returnValue; + +} diff --git a/libhmsbeagle/beagle.h b/libhmsbeagle/beagle.h old mode 100644 new mode 100755 index 3e46e03c..24726ddb --- a/libhmsbeagle/beagle.h +++ b/libhmsbeagle/beagle.h @@ -721,6 +721,12 @@ BEAGLE_DLLEXPORT int beagleTransposeTransitionMatrices(int instance, * * @return error code */ + +BEAGLE_DLLEXPORT int beagleUpdateTransitionMatricesGrad(int instance, + const int* probabilityIndices, + const double* edgeLengths, + int count); + BEAGLE_DLLEXPORT int beagleUpdateTransitionMatrices(int instance, int eigenIndex, const int* probabilityIndices, @@ -1420,6 +1426,14 @@ BEAGLE_DLLEXPORT int beagleUpdateBastaPartials(const int instance, int populationSizeIndex, int coalescentIndex); +BEAGLE_DLLEXPORT int beagleUpdateBastaPartialsGrad(const int instance, + const BastaOperation* operations, + int operationCount, + const int* intervalStarts, + int intervalCount, + int populationSizeIndex, + int coalescentIndex); + BEAGLE_DLLEXPORT int beagleAccumulateBastaPartials(const int instance, const BastaOperation* operations, int operationCount, @@ -1430,6 +1444,16 @@ BEAGLE_DLLEXPORT int beagleAccumulateBastaPartials(const int instance, int coalescentIndex, double* out); +BEAGLE_DLLEXPORT int beagleAccumulateBastaPartialsGrad(const int instance, + const BastaOperation* operations, + int operationCount, + const int* intervalStarts, + int intervalCount, + const double* intervalLengths, + const int populationSizesIndex, + int coalescentIndex, + double* out); + BEAGLE_DLLEXPORT int beagleAllocateBastaBuffers(const int instance, const int bufferCount, const int bufferLength); From 7b221faff601880823b7b393c743e6bb7eae4325 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 09:36:16 -0500 Subject: [PATCH 068/421] frequencies --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 16 +++++--- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 47 +++++++++++++++++++++++- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index c0d00a49..382f250b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -111,11 +111,17 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setPatternWeights(const double* inPatternWeights); + int setStateFrequencies(int stateFrequenciesIndex, + const double* inStateFrequencies); + protected: std::vector hInstantaneousMatrices; cusparseSpMatDescr_t* dInstantaneousMatrices; cusparseDnMatDescr_t* dPartials; + Real **dFrequenciesCache, **dWeightsCache; + cusparseDnVecDescr_t *dFrequencies; + cusparseDnVecDescr_t *dWeights; int *dMatrixCsrOffsetsCache, *dMatrixCsrColumnsCache; Real *dMatrixCsrValuesCache; int currentCacheNNZ; @@ -175,11 +181,11 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::hStatesCache; using BeagleGPUImpl::hLogLikelihoodsCache; using BeagleGPUImpl::hMatrixCache; - using BeagleGPUImpl::dEvec; - using BeagleGPUImpl::dIevc; - using BeagleGPUImpl::dEigenValues; - using BeagleGPUImpl::dWeights; - using BeagleGPUImpl::dFrequencies; +// using BeagleGPUImpl::dEvec; +// using BeagleGPUImpl::dIevc; +// using BeagleGPUImpl::dEigenValues; +// using BeagleGPUImpl::dWeights; +// using BeagleGPUImpl::dFrequencies; using BeagleGPUImpl::dMatrices; using BeagleGPUImpl::kIndexOffsetMat; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index f8dfb65f..c2b5d66d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -230,8 +230,16 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dPartials[i] = NULL; } - dWeights = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); - dFrequencies = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); + dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); + dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; + dWeights = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kEigenDecompCount); + dFrequencies = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kEigenDecompCount); + for (int i = 0; i < kEigenDecompCount; i++) { + dFrequenciesCache[i] = NULL; + dWeightsCache[i] = NULL; + dWeights[i] = NULL; + dFrequencies[i] = NULL; + } hCategoryRates = (double**) calloc(sizeof(double*),kEigenDecompCount); // Keep in double-precision hCategoryRates[0] = (double*) gpu->MallocHost(sizeof(double) * kCategoryCount); @@ -302,6 +310,41 @@ int BeagleGPUActionImpl::setPatternWeights(const double* inP return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setStateFrequencies(int stateFrequenciesIndex, + const double* inStateFrequencies) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::setStateFrequencies\n"); +#endif + + if (stateFrequenciesIndex < 0 || stateFrequenciesIndex >= kEigenDecompCount) + return BEAGLE_ERROR_OUT_OF_RANGE; + +//#ifdef DOUBLE_PRECISION +// memcpy(hFrequenciesCache, inStateFrequencies, kStateCount * sizeof(Real)); +//#else +// MEMCNV(hFrequenciesCache, inStateFrequencies, kStateCount, Real); +//#endif + beagleMemCpy(hFrequenciesCache, inStateFrequencies, kStateCount); + + if (dFrequenciesCache[stateFrequenciesIndex] == NULL) { + CHECK_CUDA(cudaMalloc((void**) &dFrequenciesCache[stateFrequenciesIndex], sizeof(Real) * kPaddedStateCount)) + } + CHECK_CUDA(cudaMemcpy(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, kStateCount, cudaMemcpyHostToDevice)) + if (dFrequencies[stateFrequenciesIndex] == NULL) { + CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequencies[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], CUDA_R_64F)) + } else { + CHECK_CUSPARSE(cusparseDnVecSetValues(dFrequencies[stateFrequenciesIndex], dFrequenciesCache[stateFrequenciesIndex])) + } + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setStateFrequencies\n"); +#endif + + return BEAGLE_SUCCESS; +} + + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setTipStates(int tipIndex, const int* inStates) { From e9b6bd5ece3ec0c2f4b1caeabe7404bbe9e66848 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 09:44:38 -0500 Subject: [PATCH 069/421] set weights --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 ++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 43 +++++++++++++++++++----- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 382f250b..dac438cb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -114,6 +114,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies); + int setCategoryWeights(int categoryWeightsIndex, + const double* inCategoryWeights); protected: std::vector hInstantaneousMatrices; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c2b5d66d..a8667e1e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -231,14 +231,16 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); - dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; - dWeights = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kEigenDecompCount); + dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kCategoryCount);; + dWeights = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kCategoryCount); dFrequencies = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { dFrequenciesCache[i] = NULL; + dFrequencies[i] = NULL; + } + for (int i = 0; i < kCategoryCount; i++) { dWeightsCache[i] = NULL; dWeights[i] = NULL; - dFrequencies[i] = NULL; } hCategoryRates = (double**) calloc(sizeof(double*),kEigenDecompCount); // Keep in double-precision @@ -320,11 +322,6 @@ int BeagleGPUActionImpl::setStateFrequencies(int stateFreque if (stateFrequenciesIndex < 0 || stateFrequenciesIndex >= kEigenDecompCount) return BEAGLE_ERROR_OUT_OF_RANGE; -//#ifdef DOUBLE_PRECISION -// memcpy(hFrequenciesCache, inStateFrequencies, kStateCount * sizeof(Real)); -//#else -// MEMCNV(hFrequenciesCache, inStateFrequencies, kStateCount, Real); -//#endif beagleMemCpy(hFrequenciesCache, inStateFrequencies, kStateCount); if (dFrequenciesCache[stateFrequenciesIndex] == NULL) { @@ -344,6 +341,36 @@ int BeagleGPUActionImpl::setStateFrequencies(int stateFreque return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setCategoryWeights(int categoryWeightsIndex, + const double* inCategoryWeights) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::setCategoryWeights\n"); +#endif + + if (categoryWeightsIndex < 0 || categoryWeightsIndex >= kEigenDecompCount) + return BEAGLE_ERROR_OUT_OF_RANGE; + + beagleMemCpy(hWeightsCache, inCategoryWeights, kPaddedPatternCount); + + if (dWeightsCache[categoryWeightsIndex] == NULL) { + CHECK_CUDA(cudaMalloc((void**) &dWeightsCache[categoryWeightsIndex], sizeof(Real) * kPaddedPatternCount)) + } + CHECK_CUDA(cudaMemcpy(dWeightsCache[categoryWeightsIndex], hWeightsCache, kPaddedPatternCount, cudaMemcpyHostToDevice)) + + if (dWeights[categoryWeightsIndex] == NULL) { + CHECK_CUSPARSE(cusparseCreateDnVec(&dWeights[categoryWeightsIndex], kPaddedPatternCount, dWeightsCache[categoryWeightsIndex], CUDA_R_64F)) + } else { + CHECK_CUSPARSE(cusparseDnVecSetValues(dWeights[categoryWeightsIndex], dWeightsCache[categoryWeightsIndex])) + } + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setCategoryWeights\n"); +#endif + + return BEAGLE_SUCCESS; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setTipStates(int tipIndex, const int* inStates) From 6f651ae3d212d317b9bbc9118336e1a749ce31d1 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 09:48:25 -0500 Subject: [PATCH 070/421] correct category weight length --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index a8667e1e..8e5e3411 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -231,14 +231,12 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); - dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kCategoryCount);; - dWeights = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kCategoryCount); + dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; + dWeights = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kEigenDecompCount); dFrequencies = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { dFrequenciesCache[i] = NULL; dFrequencies[i] = NULL; - } - for (int i = 0; i < kCategoryCount; i++) { dWeightsCache[i] = NULL; dWeights[i] = NULL; } @@ -351,15 +349,15 @@ int BeagleGPUActionImpl::setCategoryWeights(int categoryWeig if (categoryWeightsIndex < 0 || categoryWeightsIndex >= kEigenDecompCount) return BEAGLE_ERROR_OUT_OF_RANGE; - beagleMemCpy(hWeightsCache, inCategoryWeights, kPaddedPatternCount); + beagleMemCpy(hWeightsCache, inCategoryWeights, kCategoryCount); if (dWeightsCache[categoryWeightsIndex] == NULL) { - CHECK_CUDA(cudaMalloc((void**) &dWeightsCache[categoryWeightsIndex], sizeof(Real) * kPaddedPatternCount)) + CHECK_CUDA(cudaMalloc((void**) &dWeightsCache[categoryWeightsIndex], sizeof(Real) * kCategoryCount)) } - CHECK_CUDA(cudaMemcpy(dWeightsCache[categoryWeightsIndex], hWeightsCache, kPaddedPatternCount, cudaMemcpyHostToDevice)) + CHECK_CUDA(cudaMemcpy(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount, cudaMemcpyHostToDevice)) if (dWeights[categoryWeightsIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dWeights[categoryWeightsIndex], kPaddedPatternCount, dWeightsCache[categoryWeightsIndex], CUDA_R_64F)) + CHECK_CUSPARSE(cusparseCreateDnVec(&dWeights[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], CUDA_R_64F)) } else { CHECK_CUSPARSE(cusparseDnVecSetValues(dWeights[categoryWeightsIndex], dWeightsCache[categoryWeightsIndex])) } From 75fb025da81a3d9299360a630074d6cb4a7cad56 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 09:56:58 -0500 Subject: [PATCH 071/421] action test uses set sparse matrix instead of set eigen --- examples/actiontest/actiontest.cpp | 40 +++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/examples/actiontest/actiontest.cpp b/examples/actiontest/actiontest.cpp index 74b2fc97..f05962de 100644 --- a/examples/actiontest/actiontest.cpp +++ b/examples/actiontest/actiontest.cpp @@ -427,6 +427,43 @@ int main( int argc, const char* argv[] ) 3, 3, }; + int rowIndices[16] = { + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 3, + }; + int colIndices[16] = { + 0, + 1, + 2, + 3, + 0, + 1, + 2, + 3, + 0, + 1, + 2, + 3, + 0, + 1, + 2, + 3, + }; + double ivec[1] = {16}; ///array of real parts + array of imaginary parts @@ -465,7 +502,8 @@ int main( int argc, const char* argv[] ) // set the Eigen decomposition - beagleSetEigenDecomposition(instance, 0, evec, ivec, eval); +// beagleSetEigenDecomposition(instance, 0, evec, ivec, eval); + beagleSetSparseMatrix(instance, 0, rowIndices, colIndices, eval, 16); // a list of indices and edge lengths int nodeIndices[4] = { 0, 1, 2, 3 }; From 0c8a55c098f770810d42d9e7460dd3bbc9c82afe Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 11:48:43 -0500 Subject: [PATCH 072/421] code duplication for ds and p statistics calculations --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 72 +++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 231 +++++++++++++++++++++++ 2 files changed, 303 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index dac438cb..b53f790c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -35,11 +35,34 @@ #include "libhmsbeagle/GPU/BeagleGPUImpl.h" #include +#include #include #include #include // cudaMalloc, cudaMemcpy, etc. #include // cusparseSpMV +using std::vector; +using std::tuple; +using Eigen::MatrixXi; + +template +double normP1(const T& matrix) { + return (Eigen::RowVectorXd::Ones(matrix.rows()) * matrix.cwiseAbs()).maxCoeff(); +} + +template +tuple ArgNormP1(const T& matrix) +{ + int x=-1; + double v = matrix.colwise().template lpNorm<1>().maxCoeff(&x); + return {v,x}; +} + +template +double normPInf(const T& matrix) { + return matrix.template lpNorm(); +} + #define CHECK_CUDA(func) \ { \ cudaError_t status = (func); \ @@ -119,6 +142,54 @@ class BeagleGPUActionImpl : public BeagleGPUImpl protected: std::vector hInstantaneousMatrices; + SpMatrix hIdentity; + std::vector hBs; + std::vector hMuBs; + std::vector hB1Norms; + const int mMax = 55; + std::vector> ds; + std::map thetaConstants = { + //The first 30 values are from table A.3 of Computing Matrix Functions. + // For double precision, tol = 2^(-53) + // TODO: maybe calculate this + {1, 2.29E-16}, + {2, 2.58E-8}, + {3, 1.39E-5}, + {4, 3.40E-4}, + {5, 2.40E-3}, + {6, 9.07E-3}, + {7, 2.38E-2}, + {8, 5.00E-2}, + {9, 8.96E-2}, + {10, 1.44E-1}, + {11, 2.14E-1}, + {12, 3.00E-1}, + {13, 4.00E-1}, + {14, 5.14E-1}, + {15, 6.41E-1}, + {16, 7.81E-1}, + {17, 9.31E-1}, + {18, 1.09}, + {19, 1.26}, + {20, 1.44}, + {21, 1.62}, + {22, 1.82}, + {23, 2.01}, + {24, 2.22}, + {25, 2.43}, + {26, 2.64}, + {27, 2.86}, + {28, 3.08}, + {29, 3.31}, + {30, 3.54}, + //The rest are from table 3.1 of Computing the Action of the Matrix Exponential. + {35, 4.7}, + {40, 6.0}, + {45, 7.2}, + {50, 8.5}, + {55, 9.9}, + }; + cusparseSpMatDescr_t* dInstantaneousMatrices; cusparseDnMatDescr_t* dPartials; Real **dFrequenciesCache, **dWeightsCache; @@ -268,6 +339,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl const int* secondDerivativeIndices, const double* edgeLengths, int count); + double getPMax() const; }; BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 8e5e3411..095c6eba 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -32,6 +32,205 @@ #ifndef BEAGLE_BEAGLEGPUACTIONIMPL_HPP #define BEAGLE_BEAGLEGPUACTIONIMPL_HPP + +// Duplicated from CPU code +std::independent_bits_engine engine; + +bool random_bool() +{ + return engine(); +} +double random_plus_minus_1_func(double x) +{ + if (random_bool()) + return 1; + else + return -1; +} +// Algorithm 2.4 from Higham and Tisseur (2000), A BLOCK ALGORITHM FOR MATRIX 1-NORM ESTIMATION, +// WITH AN APPLICATION TO 1-NORM PSEUDOSPECTRA. +// See OneNormEst in https://eprints.maths.manchester.ac.uk/2195/1/thesis-main.pdf +// This seems to have a bug where it checks if columns in S are parallel to EVERY column of S_old. +// See also https://github.com/gnu-octave/octave/blob/default/scripts/linear-algebra/normest1.m +// See dlacn1.f +double normest1(const SpMatrix& A, int p, int t=2, int itmax=5) +{ + assert(p >= 0); + assert(t != 0); // negative means t = n + assert(itmax >= 1); + + if (p == 0) return 1.0; + + // A is (n,n); + assert(A.rows() == A.cols()); + int n = A.cols(); + + // Handle t too large + t = std::min(n,t); + + // Interpret negative t as t == n + if (t < 0) t = n; + + // Defer to normP1 if p=1 and n is small or we want an exact answer. + if (p == 1 and (n <= 4 or t == n)) + return normP1(A); + + // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. + MatrixXd X(n,t); + // We choose the first column to be all 1s. + X.col(0).setOnes(); + // The other columns have randomly chosen {-1,+1} entries. + X = X.unaryExpr( &random_plus_minus_1_func ); + // Divide by n so that the norm of each column is 1. + X /= n; + + // 3. + std::vector ind_hist(n,0); + std::vector indices(n,0); + int ind_best = -1; + double est_old = 0; + MatrixXd S = MatrixXd::Ones(n,t); + MatrixXd S_old = MatrixXd::Ones(n,t); + MatrixXi prodS(t,t); + MatrixXd Y(n,t); + MatrixXd Z(n,t); + Eigen::VectorXd h(n); + + for(int k=1; k<=itmax; k++) + { + // std::cerr<<"iter "< est_old or k == 2) + { + // Note that j is in [0,t-1], but indices[j] is in [0,n-1]. + ind_best = indices[j]; + // w = Y.col(ind_best); + } + // std::cerr<<" est = "<= 2) + { + // std::cerr<<" The new estimate ("<= est_old); + + // S = sign(Y), 0.0 -> 1.0 + S = Y.unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); + + // prodS is (t,t) + prodS = (S_old.transpose() * S).matrix().cwiseAbs().cast() ; + + // (2) If each columns in S is parallel to SOME column of S_old + if (prodS.colwise().maxCoeff().sum() == n * t and k >= 2) + { + // std::cerr<<" All columns of S parallel to S_old\n"; + return est; + } + + if (t > 1) + { + // If S(j) is parallel to S_old(i), replace S(j) with random column + for(int j=0;j() ; + for(int i=0;i (t,t) + + h = Z.cwiseAbs().rowwise().maxCoeff(); + + // (4) of Algorithm 2.4 + if (k >= 2 and h.maxCoeff() == h[ind_best]) + { + // std::cerr<<" The best column ("< h[j];}); + + // (5) of Algorithm 2.4 + int n_found = 0; + for(int i=0;i= 2); + // std::cerr<<" All columns were found in the column history.\n"; + return est; + } + + // find the first t indices that are not in ind_hist + int l=0; + for(int i=0;i(t, indices.size()); + + X = MatrixXd::Zero(n, tmax); + for(int j=0; j < tmax; j++) + X(indices[j], j) = 1; // X(:,j) = e(indices[j]) + + for(int i: indices) + ind_hist[i] = true; + + S_old = S; + } + + return est_old; +} + + namespace beagle { namespace gpu { @@ -213,10 +412,15 @@ int BeagleGPUActionImpl::createInstance(int tipCount, int hMatrixCacheSize = kMatrixSize * kCategoryCount * BEAGLE_CACHED_MATRICES_COUNT; //TODO: use Eigen csr representation? hLogLikelihoodsCache = (Real*) gpu->MallocHost(kPatternCount * sizeof(Real)); hMatrixCache = (Real*) gpu->CallocHost(hMatrixCacheSize, sizeof(Real)); + hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); + hIdentity.setIdentity(); hInstantaneousMatrices.resize(kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { hInstantaneousMatrices[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); } + hBs.resize(kEigenDecompCount); + hMuBs.resize(kEigenDecompCount); + hB1Norms.resize(kEigenDecompCount); dInstantaneousMatrices = (cusparseSpMatDescr_t *) calloc(sizeof(cusparseSpMatDescr_t), kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { dInstantaneousMatrices[i] = NULL; @@ -473,7 +677,28 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, dMatrixCsrOffsetsCache, dMatrixCsrColumnsCache, dMatrixCsrValuesCache, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F)) + //TODO: use cusparse function for diagonal sum? + double mu_B = 0.0; + for (int i = 0; i < kStateCount; i++) { + mu_B += hInstantaneousMatrices[matrixIndex].coeff(i, i); + } + mu_B /= (double) kStateCount; + hMuBs[matrixIndex] = mu_B; + hBs[matrixIndex] = hInstantaneousMatrices[matrixIndex] - mu_B * hIdentity; + hB1Norms[matrixIndex] = normP1(hBs[matrixIndex]); + + ds[matrixIndex].clear(); + + int pMax = getPMax(); + for(int p=0;p <= pMax+1; p++) + { + int t = 5; + double approx_norm = normest1( hBs[matrixIndex], p, t); + + // equation 3.7 in Al-Mohy and Higham + ds[matrixIndex].push_back( pow( approx_norm, 1.0/double(p) ) ); + } //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"Setting host matrix: "<::setSparseMatrix(int matrixIndex, return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +double BeagleGPUActionImpl::getPMax() const +{ + return floor(0.5 + 0.5 * sqrt(5.0 + 4.0 * mMax)); +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, From a043e4e9240e5f0e0138f5bc849dea2b525bbc96 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 13:26:12 -0500 Subject: [PATCH 073/421] update transition matrices code duplication from CPU --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 5 ++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index b53f790c..67f8ed79 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -199,7 +199,10 @@ class BeagleGPUActionImpl : public BeagleGPUImpl Real *dMatrixCsrValuesCache; int currentCacheNNZ; Real *dPatternWeightsCache; - cusparseDnVecDescr_t* dPatternWeights; + cusparseDnVecDescr_t dPatternWeights; + + std::vector hEigenMaps; + std::vector hEdgeMultipliers; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 095c6eba..4b2cc572 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -421,6 +421,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hBs.resize(kEigenDecompCount); hMuBs.resize(kEigenDecompCount); hB1Norms.resize(kEigenDecompCount); + ds.resize(kEigenDecompCount); dInstantaneousMatrices = (cusparseSpMatDescr_t *) calloc(sizeof(cusparseSpMatDescr_t), kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { dInstantaneousMatrices[i] = NULL; @@ -451,6 +452,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, // dMatrices = (GPUPtr*) malloc(sizeof(GPUPtr) * kMatrixCount); CHECK_CUDA(cudaMalloc((void**) &dPatternWeightsCache, kPatternCount * sizeof(Real))) + dPatternWeights = NULL; @@ -505,7 +507,11 @@ int BeagleGPUActionImpl::setPatternWeights(const double* inP // const Real* tmpWeights = beagleCastIfNecessary(inPatternWeights, hPatternWeightsCache, kPatternCount); CHECK_CUDA(cudaMemcpy(dPatternWeightsCache, inPatternWeights, kPatternCount * sizeof(Real), cudaMemcpyHostToDevice)) - CHECK_CUSPARSE(cusparseCreateDnVec(dPatternWeights, kPatternCount, dPatternWeightsCache, CUDA_R_64F)) + if (dPatternWeights == NULL) { + CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, CUDA_R_64F)) + } else { + CHECK_CUSPARSE(cusparseDnVecSetValues(dPatternWeights, dPatternWeightsCache)) + } #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setPatternWeights\n"); @@ -721,6 +727,16 @@ int BeagleGPUActionImpl::updateTransitionMatrices(int eigenI const double* edgeLengths, int count) { + for (int i = 0; i < count; i++) { + const int nodeIndex = probabilityIndices[i]; + hEigenMaps[nodeIndex] = eigenIndex; + + for (int category = 0; category < kCategoryCount; category++) { + const double categoryRate = hCategoryRates[0][category]; // XJ: because rate categories are only set for first eigen index + hEdgeMultipliers[nodeIndex * kCategoryCount + category] = edgeLengths[i] * categoryRate; + } + } + // TODO: check if need to copy it from host to device afterwards return BEAGLE_ERROR_NO_IMPLEMENTATION; } From 8537a0a08195488de8631b50a9f7c8f2e70c960a Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 13:34:23 -0500 Subject: [PATCH 074/421] initialize vector and private function for partial index --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 ++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 67f8ed79..473263d3 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -343,6 +343,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl const double* edgeLengths, int count); double getPMax() const; + + int getPartialIndex(int nodeIndex, int categoryIndex); }; BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 4b2cc572..c57bcc51 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -446,6 +446,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dWeights[i] = NULL; } + hEigenMaps.resize(kPartialsBufferCount); + hEdgeMultipliers.resize(kPartialsBufferCount * kCategoryCount); + hCategoryRates = (double**) calloc(sizeof(double*),kEigenDecompCount); // Keep in double-precision hCategoryRates[0] = (double*) gpu->MallocHost(sizeof(double) * kCategoryCount); checkHostMemory(hCategoryRates[0]); @@ -624,7 +627,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const if (tipIndex < kTipCount) { if (dPartials[tipIndex] == NULL) { for (int i = 0; i < kCategoryCount; i++) { - CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[kPartialsBufferCount * i + tipIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, hPartialsCache, + CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[getPartialIndex(tipIndex, i)], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, hPartialsCache, CUDA_R_64F, CUSPARSE_ORDER_COL)) } } @@ -636,6 +639,11 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int categoryIndex) { + return kPartialsBufferCount * categoryIndex + nodeIndex; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPartials(bool byPartition, const int *operations, From 39388af5db58217e20ca1a026970375814603468 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 14:05:54 -0500 Subject: [PATCH 075/421] correctly initialize tip partial --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 1 + libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 473263d3..76e9893d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -192,6 +192,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl cusparseSpMatDescr_t* dInstantaneousMatrices; cusparseDnMatDescr_t* dPartials; + Real **dPartialCache; Real **dFrequenciesCache, **dWeightsCache; cusparseDnVecDescr_t *dFrequencies; cusparseDnVecDescr_t *dWeights; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c57bcc51..b4fbe186 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -431,8 +431,10 @@ int BeagleGPUActionImpl::createInstance(int tipCount, currentCacheNNZ = 0; dPartials = (cusparseDnMatDescr_t *) calloc(sizeof(cusparseDnMatDescr_t), kPartialsBufferCount * kCategoryCount); + dPartialCache = (Real **) calloc(sizeof(Real*), kPartialsBufferCount * kCategoryCount); for (int i = 0; i < kPartialsBufferCount * kCategoryCount; i++) { dPartials[i] = NULL; + dPartialCache[i] = NULL; } dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); @@ -538,7 +540,7 @@ int BeagleGPUActionImpl::setStateFrequencies(int stateFreque if (dFrequenciesCache[stateFrequenciesIndex] == NULL) { CHECK_CUDA(cudaMalloc((void**) &dFrequenciesCache[stateFrequenciesIndex], sizeof(Real) * kPaddedStateCount)) } - CHECK_CUDA(cudaMemcpy(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, kStateCount, cudaMemcpyHostToDevice)) + CHECK_CUDA(cudaMemcpy(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, sizeof(Real) * kStateCount, cudaMemcpyHostToDevice)) if (dFrequencies[stateFrequenciesIndex] == NULL) { CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequencies[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], CUDA_R_64F)) } else { @@ -609,15 +611,18 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const const double* inPartialsOffset = inPartials; Real* tmpRealPartialsOffset = hPartialsCache; for (int i = 0; i < kPatternCount; i++) { -//#ifdef DOUBLE_PRECISION -// memcpy(tmpRealPartialsOffset, inPartialsOffset, sizeof(Real) * kStateCount); -//#else -// MEMCNV(tmpRealPartialsOffset, inPartialsOffset, kStateCount, Real); -//#endif beagleMemCpy(tmpRealPartialsOffset, inPartialsOffset, kStateCount); tmpRealPartialsOffset += kPaddedStateCount; inPartialsOffset += kStateCount; } + for (int i = 0; i < kCategoryCount; i++) { + if (dPartialCache[getPartialIndex(tipIndex, i)] == NULL) { + CHECK_CUDA(cudaMalloc((void**) &dPartialCache[getPartialIndex(tipIndex, i)], kPaddedStateCount * kPaddedPatternCount)) + } + CHECK_CUDA(cudaMemcpy(dPartialCache[getPartialIndex(tipIndex, i)], hPartialsCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount)) + } + + // int partialsLength = kPaddedPatternCount * kPaddedStateCount; // for (int i = 1; i < kCategoryCount; i++) { @@ -627,7 +632,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const if (tipIndex < kTipCount) { if (dPartials[tipIndex] == NULL) { for (int i = 0; i < kCategoryCount; i++) { - CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[getPartialIndex(tipIndex, i)], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, hPartialsCache, + CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[getPartialIndex(tipIndex, i)], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[getPartialIndex(tipIndex, i)], CUDA_R_64F, CUSPARSE_ORDER_COL)) } } From c54ea8a0ad4d0b7518a5a603d0a56781a04b773c Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 14:28:51 -0500 Subject: [PATCH 076/421] get partials --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 4 +++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 44 +++++++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 76e9893d..a1793b54 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -139,6 +139,10 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setCategoryWeights(int categoryWeightsIndex, const double* inCategoryWeights); + + int getPartials(int bufferIndex, + int scaleIndex, + double* outPartials); protected: std::vector hInstantaneousMatrices; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index b4fbe186..1e7131b9 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -619,7 +619,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const if (dPartialCache[getPartialIndex(tipIndex, i)] == NULL) { CHECK_CUDA(cudaMalloc((void**) &dPartialCache[getPartialIndex(tipIndex, i)], kPaddedStateCount * kPaddedPatternCount)) } - CHECK_CUDA(cudaMemcpy(dPartialCache[getPartialIndex(tipIndex, i)], hPartialsCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount)) + CHECK_CUDA(cudaMemcpy(dPartialCache[getPartialIndex(tipIndex, i)], hPartialsCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyHostToDevice)) } @@ -630,10 +630,13 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const // } if (tipIndex < kTipCount) { - if (dPartials[tipIndex] == NULL) { - for (int i = 0; i < kCategoryCount; i++) { - CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[getPartialIndex(tipIndex, i)], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[getPartialIndex(tipIndex, i)], + for (int i = 0; i < kCategoryCount; i++) { + const int partialIndex = getPartialIndex(tipIndex, i); + if (dPartials[partialIndex] == NULL) { + CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[partialIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[partialIndex], CUDA_R_64F, CUSPARSE_ORDER_COL)) + } else { + CHECK_CUSPARSE(cusparseDnMatSetValues(dPartials[partialIndex], dPartialCache[partialIndex])) } } } @@ -644,6 +647,39 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const return BEAGLE_SUCCESS; } + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::getPartials(int bufferIndex, + int scaleIndex, + double* outPartials) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::getPartials\n"); +#endif + + for (int i = 0; i < kCategoryCount; i++) { + CHECK_CUDA(cudaMemcpy(hPartialsCache + i * kPaddedStateCount * kPaddedPatternCount, dPartialCache[getPartialIndex(bufferIndex, i)], + sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToHost)) + } + + double *outPartialsOffset = outPartials; + Real *tmpRealPartialsOffset = hPartialsCache; + + for (int c = 0; c < kCategoryCount; c++) { + for (int i = 0; i < kPatternCount; i++) { + beagleMemCpy(outPartialsOffset, tmpRealPartialsOffset, kStateCount); + tmpRealPartialsOffset += kPaddedStateCount; + outPartialsOffset += kStateCount; + } + tmpRealPartialsOffset += kPaddedStateCount * (kPaddedPatternCount - kPatternCount); + } + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::getPartials\n"); +#endif + + return BEAGLE_SUCCESS; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int categoryIndex) { return kPartialsBufferCount * categoryIndex + nodeIndex; From 1d2491ff07554d53a02eff53b8cc3f0d0bf0d2f0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 15:43:36 -0400 Subject: [PATCH 077/421] Convert some pointers to std::vector --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 10 +++++----- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 19 +++++-------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index a1793b54..5ffdaa0c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -194,12 +194,12 @@ class BeagleGPUActionImpl : public BeagleGPUImpl {55, 9.9}, }; - cusparseSpMatDescr_t* dInstantaneousMatrices; - cusparseDnMatDescr_t* dPartials; - Real **dPartialCache; + std::vector dInstantaneousMatrices; + std::vector dPartials; + std::vector dPartialCache; Real **dFrequenciesCache, **dWeightsCache; - cusparseDnVecDescr_t *dFrequencies; - cusparseDnVecDescr_t *dWeights; + std::vector dFrequencies; + std::vector dWeights; int *dMatrixCsrOffsetsCache, *dMatrixCsrColumnsCache; Real *dMatrixCsrValuesCache; int currentCacheNNZ; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 1e7131b9..c94e7627 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -422,30 +422,21 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hMuBs.resize(kEigenDecompCount); hB1Norms.resize(kEigenDecompCount); ds.resize(kEigenDecompCount); - dInstantaneousMatrices = (cusparseSpMatDescr_t *) calloc(sizeof(cusparseSpMatDescr_t), kEigenDecompCount); - for (int i = 0; i < kEigenDecompCount; i++) { - dInstantaneousMatrices[i] = NULL; - } + dInstantaneousMatrices = std::vector(kEigenDecompCount, nullptr); CHECK_CUDA(cudaMalloc((void**) &dMatrixCsrOffsetsCache, (kPaddedStateCount + 1) * sizeof(int))) dMatrixCsrColumnsCache = NULL; currentCacheNNZ = 0; - dPartials = (cusparseDnMatDescr_t *) calloc(sizeof(cusparseDnMatDescr_t), kPartialsBufferCount * kCategoryCount); - dPartialCache = (Real **) calloc(sizeof(Real*), kPartialsBufferCount * kCategoryCount); - for (int i = 0; i < kPartialsBufferCount * kCategoryCount; i++) { - dPartials[i] = NULL; - dPartialCache[i] = NULL; - } + dPartials = std::vector(kPartialsBufferCount * kCategoryCount, nullptr); + dPartialCache = std::vector(kPartialsBufferCount * kCategoryCount, nullptr); dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; - dWeights = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kEigenDecompCount); - dFrequencies = (cusparseDnVecDescr_t *) calloc(sizeof(cusparseDnVecDescr_t), kEigenDecompCount); + dWeights = std::vector(kEigenDecompCount, nullptr); + dFrequencies = std::vector(kEigenDecompCount, nullptr); for (int i = 0; i < kEigenDecompCount; i++) { dFrequenciesCache[i] = NULL; - dFrequencies[i] = NULL; dWeightsCache[i] = NULL; - dWeights[i] = NULL; } hEigenMaps.resize(kPartialsBufferCount); From 38e4a746baba6b72975abb6fb0a585c06342b837 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 15:49:42 -0400 Subject: [PATCH 078/421] Try to fix builds by disabling opencl. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9066ebda..5e862c31 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -93,11 +93,12 @@ jobs: sudo apt-get install -y g++-${{ matrix.version }} echo "CC=ccache gcc-${{ matrix.version }}" >> $GITHUB_ENV echo "CXX=ccache g++-${{ matrix.version }}" >> $GITHUB_ENV + echo "CMAKE_ARGS=-DBUILD_OPENCL=OFF" >> $GITHUB_ENV else sudo apt-get install -y clang-${{ matrix.version }} echo "CC=ccache clang-${{ matrix.version }}" >> $GITHUB_ENV echo "CXX=ccache clang++-${{ matrix.version }}" >> $GITHUB_ENV - echo "CMAKE_ARGS=-DBUILD_OPENMP=OFF" >> $GITHUB_ENV + echo "CMAKE_ARGS='-DBUILD_OPENCL=OFF -DBUILD_OPENMP=OFF'" >> $GITHUB_ENV fi - name: Install (Linux -> Windows [cross]) From ed778032145364501aa0ff9eae7ab6e80de12061 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 16:05:28 -0400 Subject: [PATCH 079/421] Fix missing "*sizeof(REAL)" --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 30 ++++++++++++++---------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c94e7627..332c959c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -257,6 +257,16 @@ long long BeagleGPUActionImpl::getFlags() return flags; } +template +T* cudaDeviceNew(int n) +{ + T* result; + auto status = cudaMalloc((void**)&result, n*sizeof(T)); + if (status != cudaSuccess) + throw std::runtime_error("cudaMalloc: failed!"); + return result; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::createInstance(int tipCount, int partialsBufferCount, @@ -423,7 +433,8 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hB1Norms.resize(kEigenDecompCount); ds.resize(kEigenDecompCount); dInstantaneousMatrices = std::vector(kEigenDecompCount, nullptr); - CHECK_CUDA(cudaMalloc((void**) &dMatrixCsrOffsetsCache, (kPaddedStateCount + 1) * sizeof(int))) + + dMatrixCsrOffsetsCache = cudaDeviceNew(kPaddedStateCount + 1); dMatrixCsrColumnsCache = NULL; currentCacheNNZ = 0; @@ -447,14 +458,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, checkHostMemory(hCategoryRates[0]); // dMatrices = (GPUPtr*) malloc(sizeof(GPUPtr) * kMatrixCount); - CHECK_CUDA(cudaMalloc((void**) &dPatternWeightsCache, kPatternCount * sizeof(Real))) + dPatternWeightsCache = cudaDeviceNew(kPatternCount); dPatternWeights = NULL; - - - - - return BEAGLE_SUCCESS; } @@ -529,7 +535,7 @@ int BeagleGPUActionImpl::setStateFrequencies(int stateFreque beagleMemCpy(hFrequenciesCache, inStateFrequencies, kStateCount); if (dFrequenciesCache[stateFrequenciesIndex] == NULL) { - CHECK_CUDA(cudaMalloc((void**) &dFrequenciesCache[stateFrequenciesIndex], sizeof(Real) * kPaddedStateCount)) + dFrequenciesCache[stateFrequenciesIndex] = cudaDeviceNew(kPaddedStateCount); } CHECK_CUDA(cudaMemcpy(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, sizeof(Real) * kStateCount, cudaMemcpyHostToDevice)) if (dFrequencies[stateFrequenciesIndex] == NULL) { @@ -558,7 +564,7 @@ int BeagleGPUActionImpl::setCategoryWeights(int categoryWeig beagleMemCpy(hWeightsCache, inCategoryWeights, kCategoryCount); if (dWeightsCache[categoryWeightsIndex] == NULL) { - CHECK_CUDA(cudaMalloc((void**) &dWeightsCache[categoryWeightsIndex], sizeof(Real) * kCategoryCount)) + dWeightsCache[categoryWeightsIndex] = cudaDeviceNew(kCategoryCount); } CHECK_CUDA(cudaMemcpy(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount, cudaMemcpyHostToDevice)) @@ -608,7 +614,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const } for (int i = 0; i < kCategoryCount; i++) { if (dPartialCache[getPartialIndex(tipIndex, i)] == NULL) { - CHECK_CUDA(cudaMalloc((void**) &dPartialCache[getPartialIndex(tipIndex, i)], kPaddedStateCount * kPaddedPatternCount)) + dPartialCache[getPartialIndex(tipIndex, i)] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); } CHECK_CUDA(cudaMemcpy(dPartialCache[getPartialIndex(tipIndex, i)], hPartialsCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyHostToDevice)) } @@ -711,8 +717,8 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, const int paddedNNZ = currentNNZ + 16 - currentNNZ%16; if (currentCacheNNZ < paddedNNZ) { currentCacheNNZ = paddedNNZ; - CHECK_CUDA(cudaMalloc((void**) &dMatrixCsrColumnsCache, currentCacheNNZ * sizeof(int))) - CHECK_CUDA(cudaMalloc((void**) &dMatrixCsrValuesCache, currentCacheNNZ * sizeof(Real))) + dMatrixCsrColumnsCache = cudaDeviceNew(currentCacheNNZ); + dMatrixCsrValuesCache = cudaDeviceNew(currentCacheNNZ); } CHECK_CUDA(cudaMemcpy(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), sizeof(int) * (kPaddedStateCount + 1), cudaMemcpyHostToDevice)) From b9bfb794d46946cb42cce2d2fac559775e4c8597 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 16:20:17 -0400 Subject: [PATCH 080/421] Check argument types for cudaMemcpy --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 332c959c..6d429d46 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -267,6 +267,14 @@ T* cudaDeviceNew(int n) return result; } +template +void MemcpyHostToDevice(T* dptr, const T* hptr, int n) +{ + auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyHostToDevice); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::createInstance(int tipCount, int partialsBufferCount, @@ -507,7 +515,7 @@ int BeagleGPUActionImpl::setPatternWeights(const double* inP // MEMCNV(hPatternWeightsCache, inPatternWeights, kPatternCount, Real); //#endif // const Real* tmpWeights = beagleCastIfNecessary(inPatternWeights, hPatternWeightsCache, kPatternCount); - CHECK_CUDA(cudaMemcpy(dPatternWeightsCache, inPatternWeights, kPatternCount * sizeof(Real), cudaMemcpyHostToDevice)) + MemcpyHostToDevice(dPatternWeightsCache, inPatternWeights, kPatternCount); if (dPatternWeights == NULL) { CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, CUDA_R_64F)) @@ -537,7 +545,7 @@ int BeagleGPUActionImpl::setStateFrequencies(int stateFreque if (dFrequenciesCache[stateFrequenciesIndex] == NULL) { dFrequenciesCache[stateFrequenciesIndex] = cudaDeviceNew(kPaddedStateCount); } - CHECK_CUDA(cudaMemcpy(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, sizeof(Real) * kStateCount, cudaMemcpyHostToDevice)) + MemcpyHostToDevice(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, kStateCount); if (dFrequencies[stateFrequenciesIndex] == NULL) { CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequencies[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], CUDA_R_64F)) } else { @@ -566,7 +574,7 @@ int BeagleGPUActionImpl::setCategoryWeights(int categoryWeig if (dWeightsCache[categoryWeightsIndex] == NULL) { dWeightsCache[categoryWeightsIndex] = cudaDeviceNew(kCategoryCount); } - CHECK_CUDA(cudaMemcpy(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount, cudaMemcpyHostToDevice)) + MemcpyHostToDevice(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount); if (dWeights[categoryWeightsIndex] == NULL) { CHECK_CUSPARSE(cusparseCreateDnVec(&dWeights[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], CUDA_R_64F)) @@ -616,7 +624,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const if (dPartialCache[getPartialIndex(tipIndex, i)] == NULL) { dPartialCache[getPartialIndex(tipIndex, i)] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); } - CHECK_CUDA(cudaMemcpy(dPartialCache[getPartialIndex(tipIndex, i)], hPartialsCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyHostToDevice)) + MemcpyHostToDevice(dPartialCache[getPartialIndex(tipIndex, i)], hPartialsCache, kPaddedStateCount * kPaddedPatternCount); } @@ -721,9 +729,9 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, dMatrixCsrValuesCache = cudaDeviceNew(currentCacheNNZ); } - CHECK_CUDA(cudaMemcpy(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), sizeof(int) * (kPaddedStateCount + 1), cudaMemcpyHostToDevice)) - CHECK_CUDA(cudaMemcpy(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), sizeof(int) * currentNNZ, cudaMemcpyHostToDevice)) - CHECK_CUDA(cudaMemcpy(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), sizeof(Real) * currentNNZ, cudaMemcpyHostToDevice)) + MemcpyHostToDevice(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); + MemcpyHostToDevice(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); + MemcpyHostToDevice(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, dMatrixCsrOffsetsCache, dMatrixCsrColumnsCache, dMatrixCsrValuesCache, From ca12391183ad1552317c5a36242332a3dea51419 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 16:33:40 -0400 Subject: [PATCH 081/421] Don't support Action with `Real = float`. --- libhmsbeagle/GPU/ActionCUDAPlugin.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp index 9488990e..394a6909 100644 --- a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp +++ b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp @@ -65,7 +65,8 @@ namespace beagle { if (anyGPUSupportsDP) { beagleFactories.push_back(new BeagleGPUActionImplFactory()); } - beagleFactories.push_back(new BeagleGPUActionImplFactory()); + // We don't support `Real = float`. + // beagleFactories.push_back(new BeagleGPUActionImplFactory()); } } From 69ff426d198e1b33354c40f9942e2fe9f9e7b5c0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 16:45:11 -0400 Subject: [PATCH 082/421] We do want to support Real = float. --- libhmsbeagle/GPU/ActionCUDAPlugin.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp index 394a6909..9488990e 100644 --- a/libhmsbeagle/GPU/ActionCUDAPlugin.cpp +++ b/libhmsbeagle/GPU/ActionCUDAPlugin.cpp @@ -65,8 +65,7 @@ namespace beagle { if (anyGPUSupportsDP) { beagleFactories.push_back(new BeagleGPUActionImplFactory()); } - // We don't support `Real = float`. - // beagleFactories.push_back(new BeagleGPUActionImplFactory()); + beagleFactories.push_back(new BeagleGPUActionImplFactory()); } } From c5bca5ca0e97840adb4c30757787ca8ee7d29353 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 3 Jun 2024 16:03:09 -0500 Subject: [PATCH 083/421] check tip partial on actiontest --- examples/actiontest/actiontest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/actiontest/actiontest.cpp b/examples/actiontest/actiontest.cpp index f05962de..f3d0ee64 100644 --- a/examples/actiontest/actiontest.cpp +++ b/examples/actiontest/actiontest.cpp @@ -531,7 +531,7 @@ int main( int argc, const char* argv[] ) double * seeprePartials = (double*) malloc(sizeof(double) * stateCount * nPatterns * rateCategoryCount); - beagleGetPartials(instance, 4, BEAGLE_OP_NONE, seeprePartials); + beagleGetPartials(instance, 0, BEAGLE_OP_NONE, seeprePartials); MapType * testMaps = (MapType *) malloc(sizeof(MapType) * rateCategoryCount); for (int i = 0; i < rateCategoryCount; i++) { new (& testMaps[i]) MapType(seeprePartials + i * nPatterns * stateCount, stateCount, nPatterns); @@ -543,7 +543,7 @@ int main( int argc, const char* argv[] ) int l = 0; for(int s = 0; s < rateCategoryCount; s++){ - std::cout<<"See partial"<< 3 <<" rate category"<< s+1<< ": \n"; + std::cout<<"See partial"<< 0 <<" rate category"<< s+1<< ": \n"; for(int k = 0; k Date: Mon, 3 Jun 2024 16:03:30 -0500 Subject: [PATCH 084/421] mess up Ben's wrapper before it works --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 6d429d46..0e736750 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -515,7 +515,7 @@ int BeagleGPUActionImpl::setPatternWeights(const double* inP // MEMCNV(hPatternWeightsCache, inPatternWeights, kPatternCount, Real); //#endif // const Real* tmpWeights = beagleCastIfNecessary(inPatternWeights, hPatternWeightsCache, kPatternCount); - MemcpyHostToDevice(dPatternWeightsCache, inPatternWeights, kPatternCount); + CHECK_CUDA(cudaMemcpy(dPatternWeightsCache, inPatternWeights, kPatternCount * sizeof(Real), cudaMemcpyHostToDevice)) if (dPatternWeights == NULL) { CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, CUDA_R_64F)) @@ -729,9 +729,9 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, dMatrixCsrValuesCache = cudaDeviceNew(currentCacheNNZ); } - MemcpyHostToDevice(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); - MemcpyHostToDevice(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); - MemcpyHostToDevice(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); + CHECK_CUDA(cudaMemcpy(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), sizeof(int) * (kPaddedStateCount + 1), cudaMemcpyHostToDevice)) + CHECK_CUDA(cudaMemcpy(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), sizeof(int) * currentNNZ, cudaMemcpyHostToDevice)) + CHECK_CUDA(cudaMemcpy(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), sizeof(Real) * currentNNZ, cudaMemcpyHostToDevice)) CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, dMatrixCsrOffsetsCache, dMatrixCsrColumnsCache, dMatrixCsrValuesCache, From acbafef34713fe6db9646f05612ff06f5b534c4f Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 17:15:58 -0400 Subject: [PATCH 085/421] Fix compilation when Real = float --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 46 ++++++++++++++---------- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 39 ++++++++++---------- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 5ffdaa0c..841528be 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -83,11 +83,21 @@ double normPInf(const T& matrix) { } \ } +template +using SpMatrix = Eigen::SparseMatrix; -typedef Eigen::Matrix MatrixXd; -typedef Eigen::Map MapType; -typedef Eigen::SparseMatrix SpMatrix; -typedef Eigen::Triplet Triplet; +template +using Triplet = Eigen::Triplet; + +//template +//using DnMatrix = Eigen::Matrix; + +//template +//using DnVector = Eigen::Matrix; +//using DnVector = Eigen::Matrix; + +//template +//using MapType = DnMatrix; namespace beagle { namespace gpu { @@ -124,34 +134,34 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setTipStates(int tipIndex, const int* inStates); - int setTipPartials(int tipIndex, const double* inPartials); + int setTipPartials(int tipIndex, const Real* inPartials); int setSparseMatrix(int matrixIndex, const int* rowIndices, const int* colIndices, - const double* values, + const Real* values, int numNonZeros); - int setPatternWeights(const double* inPatternWeights); + int setPatternWeights(const Real* inPatternWeights); int setStateFrequencies(int stateFrequenciesIndex, - const double* inStateFrequencies); + const Real* inStateFrequencies); int setCategoryWeights(int categoryWeightsIndex, - const double* inCategoryWeights); + const Real* inCategoryWeights); int getPartials(int bufferIndex, int scaleIndex, - double* outPartials); + Real* outPartials); protected: - std::vector hInstantaneousMatrices; - SpMatrix hIdentity; - std::vector hBs; - std::vector hMuBs; - std::vector hB1Norms; + std::vector> hInstantaneousMatrices; + SpMatrix hIdentity; + std::vector> hBs; + std::vector hMuBs; + std::vector hB1Norms; const int mMax = 55; - std::vector> ds; + std::vector> ds; std::map thetaConstants = { //The first 30 values are from table A.3 of Computing Matrix Functions. // For double precision, tol = 2^(-53) @@ -207,7 +217,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl cusparseDnVecDescr_t dPatternWeights; std::vector hEigenMaps; - std::vector hEdgeMultipliers; + std::vector hEdgeMultipliers; @@ -345,7 +355,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl const int* probabilityIndices, const int* firstDerivativeIndices, const int* secondDerivativeIndices, - const double* edgeLengths, + const Real* edgeLengths, int count); double getPMax() const; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 0e736750..ec2819ec 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -47,14 +47,17 @@ double random_plus_minus_1_func(double x) else return -1; } + // Algorithm 2.4 from Higham and Tisseur (2000), A BLOCK ALGORITHM FOR MATRIX 1-NORM ESTIMATION, // WITH AN APPLICATION TO 1-NORM PSEUDOSPECTRA. // See OneNormEst in https://eprints.maths.manchester.ac.uk/2195/1/thesis-main.pdf // This seems to have a bug where it checks if columns in S are parallel to EVERY column of S_old. // See also https://github.com/gnu-octave/octave/blob/default/scripts/linear-algebra/normest1.m // See dlacn1.f -double normest1(const SpMatrix& A, int p, int t=2, int itmax=5) +double normest1(const SpMatrix& A, int p, int t=2, int itmax=5) { + typedef Eigen::Matrix MatrixXd; + assert(p >= 0); assert(t != 0); // negative means t = n assert(itmax >= 1); @@ -430,11 +433,11 @@ int BeagleGPUActionImpl::createInstance(int tipCount, int hMatrixCacheSize = kMatrixSize * kCategoryCount * BEAGLE_CACHED_MATRICES_COUNT; //TODO: use Eigen csr representation? hLogLikelihoodsCache = (Real*) gpu->MallocHost(kPatternCount * sizeof(Real)); hMatrixCache = (Real*) gpu->CallocHost(hMatrixCacheSize, sizeof(Real)); - hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); + hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); hIdentity.setIdentity(); hInstantaneousMatrices.resize(kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { - hInstantaneousMatrices[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); + hInstantaneousMatrices[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); } hBs.resize(kEigenDecompCount); hMuBs.resize(kEigenDecompCount); @@ -502,7 +505,7 @@ char* BeagleGPUActionImpl::getInstanceName() { #endif BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::setPatternWeights(const double* inPatternWeights) { +int BeagleGPUActionImpl::setPatternWeights(const Real* inPatternWeights) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUActionImpl::setPatternWeights\n"); @@ -532,7 +535,7 @@ int BeagleGPUActionImpl::setPatternWeights(const double* inP BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setStateFrequencies(int stateFrequenciesIndex, - const double* inStateFrequencies) { + const Real* inStateFrequencies) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUActionImpl::setStateFrequencies\n"); #endif @@ -561,7 +564,7 @@ int BeagleGPUActionImpl::setStateFrequencies(int stateFreque BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setCategoryWeights(int categoryWeightsIndex, - const double* inCategoryWeights) { + const Real* inCategoryWeights) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUActionImpl::setCategoryWeights\n"); #endif @@ -605,7 +608,7 @@ int BeagleGPUActionImpl::setTipStates(int tipIndex, const in } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::setTipPartials(int tipIndex, const double* inPartials) { +int BeagleGPUActionImpl::setTipPartials(int tipIndex, const Real* inPartials) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUActionImpl::setTipPartials\n"); #endif @@ -613,7 +616,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const if (tipIndex < 0 || tipIndex >= kTipCount) return BEAGLE_ERROR_OUT_OF_RANGE; - const double* inPartialsOffset = inPartials; + const Real* inPartialsOffset = inPartials; Real* tmpRealPartialsOffset = hPartialsCache; for (int i = 0; i < kPatternCount; i++) { beagleMemCpy(tmpRealPartialsOffset, inPartialsOffset, kStateCount); @@ -656,7 +659,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartials(int bufferIndex, int scaleIndex, - double* outPartials) { + Real* outPartials) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUActionImpl::getPartials\n"); #endif @@ -666,7 +669,7 @@ int BeagleGPUActionImpl::getPartials(int bufferIndex, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToHost)) } - double *outPartialsOffset = outPartials; + Real *outPartialsOffset = outPartials; Real *tmpRealPartialsOffset = hPartialsCache; for (int c = 0; c < kCategoryCount; c++) { @@ -712,12 +715,12 @@ BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, const int* rowIndices, const int* colIndices, - const double* values, + const Real* values, int numNonZeros) { - std::vector tripletList; + std::vector> tripletList; for (int i = 0; i < numNonZeros; i++) { - tripletList.push_back(Triplet(rowIndices[i], colIndices[i], values[i])); + tripletList.push_back(Triplet(rowIndices[i], colIndices[i], values[i])); } hInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); @@ -738,11 +741,11 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F)) //TODO: use cusparse function for diagonal sum? - double mu_B = 0.0; + Real mu_B = 0.0; for (int i = 0; i < kStateCount; i++) { mu_B += hInstantaneousMatrices[matrixIndex].coeff(i, i); } - mu_B /= (double) kStateCount; + mu_B /= (Real) kStateCount; hMuBs[matrixIndex] = mu_B; hBs[matrixIndex] = hInstantaneousMatrices[matrixIndex] - mu_B * hIdentity; @@ -754,10 +757,10 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, for(int p=0;p <= pMax+1; p++) { int t = 5; - double approx_norm = normest1( hBs[matrixIndex], p, t); + Real approx_norm = normest1( hBs[matrixIndex], p, t); // equation 3.7 in Al-Mohy and Higham - ds[matrixIndex].push_back( pow( approx_norm, 1.0/double(p) ) ); + ds[matrixIndex].push_back( pow( approx_norm, 1.0/Real(p) ) ); } //#ifdef BEAGLE_DEBUG_FLOW @@ -778,7 +781,7 @@ int BeagleGPUActionImpl::updateTransitionMatrices(int eigenI const int* probabilityIndices, const int* firstDerivativeIndices, const int* secondDerivativeIndices, - const double* edgeLengths, + const Real* edgeLengths, int count) { for (int i = 0; i < count; i++) { From bff1bd519a17df91c336f5710ca0082f94caff94 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 17:17:54 -0400 Subject: [PATCH 086/421] Revert "mess up Ben's wrapper before it works" This reverts commit 4bb72adbd29cc7650910c78dea0ebf758c5f6970. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index ec2819ec..4e547098 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -518,7 +518,7 @@ int BeagleGPUActionImpl::setPatternWeights(const Real* inPat // MEMCNV(hPatternWeightsCache, inPatternWeights, kPatternCount, Real); //#endif // const Real* tmpWeights = beagleCastIfNecessary(inPatternWeights, hPatternWeightsCache, kPatternCount); - CHECK_CUDA(cudaMemcpy(dPatternWeightsCache, inPatternWeights, kPatternCount * sizeof(Real), cudaMemcpyHostToDevice)) + MemcpyHostToDevice(dPatternWeightsCache, inPatternWeights, kPatternCount); if (dPatternWeights == NULL) { CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, CUDA_R_64F)) @@ -732,9 +732,9 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, dMatrixCsrValuesCache = cudaDeviceNew(currentCacheNNZ); } - CHECK_CUDA(cudaMemcpy(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), sizeof(int) * (kPaddedStateCount + 1), cudaMemcpyHostToDevice)) - CHECK_CUDA(cudaMemcpy(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), sizeof(int) * currentNNZ, cudaMemcpyHostToDevice)) - CHECK_CUDA(cudaMemcpy(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), sizeof(Real) * currentNNZ, cudaMemcpyHostToDevice)) + MemcpyHostToDevice(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); + MemcpyHostToDevice(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); + MemcpyHostToDevice(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, dMatrixCsrOffsetsCache, dMatrixCsrColumnsCache, dMatrixCsrValuesCache, From 6fddd0245e6fc9855a7bd9957b9eca44a09e22c2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 17:42:11 -0400 Subject: [PATCH 087/421] Make normest1 work with both float and double. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 9 ++++---- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 27 ++++++++++++------------ 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 841528be..d3c2095f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -89,12 +89,11 @@ using SpMatrix = Eigen::SparseMatrix; template using Triplet = Eigen::Triplet; -//template -//using DnMatrix = Eigen::Matrix; +template +using DnMatrix = Eigen::Matrix; -//template -//using DnVector = Eigen::Matrix; -//using DnVector = Eigen::Matrix; +template +using DnVector = Eigen::Matrix; //template //using MapType = DnMatrix; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 4e547098..d7038e19 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -54,10 +54,9 @@ double random_plus_minus_1_func(double x) // This seems to have a bug where it checks if columns in S are parallel to EVERY column of S_old. // See also https://github.com/gnu-octave/octave/blob/default/scripts/linear-algebra/normest1.m // See dlacn1.f -double normest1(const SpMatrix& A, int p, int t=2, int itmax=5) +template +Real normest1(const SpMatrix& A, int p, int t=2, int itmax=5) { - typedef Eigen::Matrix MatrixXd; - assert(p >= 0); assert(t != 0); // negative means t = n assert(itmax >= 1); @@ -79,7 +78,7 @@ double normest1(const SpMatrix& A, int p, int t=2, int itmax=5) return normP1(A); // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. - MatrixXd X(n,t); + DnMatrix X(n,t); // We choose the first column to be all 1s. X.col(0).setOnes(); // The other columns have randomly chosen {-1,+1} entries. @@ -91,13 +90,13 @@ double normest1(const SpMatrix& A, int p, int t=2, int itmax=5) std::vector ind_hist(n,0); std::vector indices(n,0); int ind_best = -1; - double est_old = 0; - MatrixXd S = MatrixXd::Ones(n,t); - MatrixXd S_old = MatrixXd::Ones(n,t); + Real est_old = 0; + DnMatrix S = DnMatrix::Ones(n,t); + DnMatrix S_old = DnMatrix::Ones(n,t); MatrixXi prodS(t,t); - MatrixXd Y(n,t); - MatrixXd Z(n,t); - Eigen::VectorXd h(n); + DnMatrix Y(n,t); + DnMatrix Z(n,t); + DnVector h(n); for(int k=1; k<=itmax; k++) { @@ -129,10 +128,10 @@ double normest1(const SpMatrix& A, int p, int t=2, int itmax=5) assert(est >= est_old); // S = sign(Y), 0.0 -> 1.0 - S = Y.unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); + S = Y.unaryExpr([](const Real& x) {return (x>=0) ? 1.0 : -1.0 ;}); // prodS is (t,t) - prodS = (S_old.transpose() * S).matrix().cwiseAbs().cast() ; + prodS = (S_old.transpose() * S).matrix().cwiseAbs().template cast() ; // (2) If each columns in S is parallel to SOME column of S_old if (prodS.colwise().maxCoeff().sum() == n * t and k >= 2) @@ -156,7 +155,7 @@ double normest1(const SpMatrix& A, int p, int t=2, int itmax=5) } // If S(j) is parallel to S(i) for i() ; + prodS = (S.transpose() * S).matrix().template cast() ; for(int i=0;i& A, int p, int t=2, int itmax=5) int tmax = std::min(t, indices.size()); - X = MatrixXd::Zero(n, tmax); + X = DnMatrix::Zero(n, tmax); for(int j=0; j < tmax; j++) X(indices[j], j) = 1; // X(:,j) = e(indices[j]) From 9fd103e20c540d067fc25199f419c08470251556 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Jun 2024 17:43:33 -0400 Subject: [PATCH 088/421] Select CUDA_R_64F for double and CUDA_R_32F for float. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index d7038e19..17ad8640 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -259,6 +259,14 @@ long long BeagleGPUActionImpl::getFlags() return flags; } +template cusparseIndexType_t IndexType; +template <> cusparseIndexType_t IndexType = CUSPARSE_INDEX_32I; +template <> cusparseIndexType_t IndexType = CUSPARSE_INDEX_64I; + +template cudaDataType DataType; +template <> cudaDataType DataType = CUDA_R_32F; +template <> cudaDataType DataType = CUDA_R_64F; + template T* cudaDeviceNew(int n) { @@ -520,7 +528,7 @@ int BeagleGPUActionImpl::setPatternWeights(const Real* inPat MemcpyHostToDevice(dPatternWeightsCache, inPatternWeights, kPatternCount); if (dPatternWeights == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, CUDA_R_64F)) + CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, DataType)) } else { CHECK_CUSPARSE(cusparseDnVecSetValues(dPatternWeights, dPatternWeightsCache)) } @@ -549,7 +557,7 @@ int BeagleGPUActionImpl::setStateFrequencies(int stateFreque } MemcpyHostToDevice(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, kStateCount); if (dFrequencies[stateFrequenciesIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequencies[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], CUDA_R_64F)) + CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequencies[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], DataType)) } else { CHECK_CUSPARSE(cusparseDnVecSetValues(dFrequencies[stateFrequenciesIndex], dFrequenciesCache[stateFrequenciesIndex])) } @@ -579,7 +587,7 @@ int BeagleGPUActionImpl::setCategoryWeights(int categoryWeig MemcpyHostToDevice(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount); if (dWeights[categoryWeightsIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dWeights[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], CUDA_R_64F)) + CHECK_CUSPARSE(cusparseCreateDnVec(&dWeights[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], DataType)) } else { CHECK_CUSPARSE(cusparseDnVecSetValues(dWeights[categoryWeightsIndex], dWeightsCache[categoryWeightsIndex])) } @@ -641,7 +649,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const const int partialIndex = getPartialIndex(tipIndex, i); if (dPartials[partialIndex] == NULL) { CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[partialIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[partialIndex], - CUDA_R_64F, CUSPARSE_ORDER_COL)) + DataType, CUSPARSE_ORDER_COL)) } else { CHECK_CUSPARSE(cusparseDnMatSetValues(dPartials[partialIndex], dPartialCache[partialIndex])) } @@ -737,8 +745,8 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, dMatrixCsrOffsetsCache, dMatrixCsrColumnsCache, dMatrixCsrValuesCache, - CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, - CUSPARSE_INDEX_BASE_ZERO, CUDA_R_64F)) + IndexType, IndexType, + CUSPARSE_INDEX_BASE_ZERO, DataType)) //TODO: use cusparse function for diagonal sum? Real mu_B = 0.0; for (int i = 0; i < kStateCount; i++) { From cfc32b556867132e7678d037561636de2b814588 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 4 Jun 2024 14:46:52 -0500 Subject: [PATCH 089/421] reformat upPartials code --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 178 ++++++++++++----------- 1 file changed, 90 insertions(+), 88 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 3fe3d89c..f6926268 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -370,59 +370,59 @@ namespace beagle { inline const char* getBeagleCPUActionName(){ return "CPU-Action-Single"; }; - BEAGLE_CPU_ACTION_TEMPLATE + BEAGLE_CPU_ACTION_TEMPLATE int BeagleCPUActionImpl::upPartials(bool byPartition, - const int *operations, - int operationCount, - int cumulativeScaleIndex) { - double* cumulativeScaleBuffer = NULL; + const int *operations, + int operationCount, + int cumulativeScaleIndex) { + double *cumulativeScaleBuffer = NULL; if (cumulativeScaleIndex != BEAGLE_OP_NONE) cumulativeScaleBuffer = gScaleBuffers[cumulativeScaleIndex]; for (int op = 0; op < operationCount; op++) { int numOps = BEAGLE_OP_COUNT; - if (byPartition) - numOps = BEAGLE_PARTITION_OP_COUNT; + if (byPartition) + numOps = BEAGLE_PARTITION_OP_COUNT; - if (DEBUGGING_OUTPUT) { - fprintf(stderr, "op[%d] = ", op); - for (int j = 0; j < numOps; j++) { - std::cerr << operations[op*numOps+j] << " "; - } - fprintf(stderr, "\n"); - } + if (DEBUGGING_OUTPUT) { + fprintf(stderr, "op[%d] = ", op); + for (int j = 0; j < numOps; j++) { + std::cerr << operations[op * numOps + j] << " "; + } + fprintf(stderr, "\n"); + } - const int destinationPartialIndex = operations[op * numOps]; + const int destinationPartialIndex = operations[op * numOps]; const int writeScalingIndex = operations[op * numOps + 1]; const int readScalingIndex = operations[op * numOps + 2]; const int firstChildPartialIndex = operations[op * numOps + 3]; const int firstChildSubstitutionMatrixIndex = operations[op * numOps + 4]; const int secondChildPartialIndex = operations[op * numOps + 5]; const int secondChildSubstitutionMatrixIndex = operations[op * numOps + 6]; - int currentPartition = 0; - if (byPartition) { - currentPartition = operations[op * numOps + 7]; - cumulativeScaleIndex = operations[op * numOps + 8]; - if (cumulativeScaleIndex != BEAGLE_OP_NONE) - cumulativeScaleBuffer = gScaleBuffers[cumulativeScaleIndex]; - else - cumulativeScaleBuffer = NULL; - } + int currentPartition = 0; + if (byPartition) { + currentPartition = operations[op * numOps + 7]; + cumulativeScaleIndex = operations[op * numOps + 8]; + if (cumulativeScaleIndex != BEAGLE_OP_NONE) + cumulativeScaleBuffer = gScaleBuffers[cumulativeScaleIndex]; + else + cumulativeScaleBuffer = NULL; + } - int startPattern = 0; - int endPattern = kPatternCount; - if (byPartition) { - startPattern = this->gPatternPartitionsStartPatterns[currentPartition]; - endPattern = this->gPatternPartitionsStartPatterns[currentPartition + 1]; + int startPattern = 0; + int endPattern = kPatternCount; + if (byPartition) { + startPattern = this->gPatternPartitionsStartPatterns[currentPartition]; + endPattern = this->gPatternPartitionsStartPatterns[currentPartition + 1]; - assert(startPattern >= 0 and startPattern <= kPatternCount); - assert(endPattern >= 0 and endPattern <= kPatternCount); - assert(startPattern <= endPattern); - } + assert(startPattern >= 0 and startPattern <= kPatternCount); + assert(endPattern >= 0 and endPattern <= kPatternCount); + assert(startPattern <= endPattern); + } int rescale = BEAGLE_OP_NONE; - double* scalingFactors = NULL; + double *scalingFactors = NULL; if (writeScalingIndex >= 0) { rescale = 1; scalingFactors = gScaleBuffers[writeScalingIndex]; @@ -435,25 +435,26 @@ namespace beagle { #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"Updating partials for index: "<rescalePartialsByPartition(destPartials,scalingFactors,cumulativeScaleBuffer,0, currentPartition); - } else { - this->rescalePartials(destPartials,scalingFactors,cumulativeScaleBuffer,0); - } + double *destPartials = gPartials[destinationPartialIndex]; + if (byPartition) { + this->rescalePartialsByPartition(destPartials, scalingFactors, cumulativeScaleBuffer, 0, + currentPartition); + } else { + this->rescalePartials(destPartials, scalingFactors, cumulativeScaleBuffer, 0); + } } } @@ -462,10 +463,10 @@ namespace beagle { BEAGLE_CPU_ACTION_TEMPLATE int BeagleCPUActionImpl::upPrePartials(bool byPartition, - const int *operations, - int operationCount, - int cumulativeScaleIndex) { - double* cumulativeScaleBuffer = NULL; + const int *operations, + int operationCount, + int cumulativeScaleIndex) { + double *cumulativeScaleBuffer = NULL; if (cumulativeScaleIndex != BEAGLE_OP_NONE) cumulativeScaleBuffer = gScaleBuffers[cumulativeScaleIndex]; @@ -486,31 +487,31 @@ namespace beagle { const int substitutionMatrixIndex = operations[op * numOps + 4]; const int siblingIndex = operations[op * numOps + 5]; const int siblingSubstitutionMatrixIndex = operations[op * numOps + 6]; - int currentPartition = 0; - if (byPartition) { - currentPartition = operations[op * numOps + 7]; - cumulativeScaleIndex = operations[op * numOps + 8]; + int currentPartition = 0; + if (byPartition) { + currentPartition = operations[op * numOps + 7]; + cumulativeScaleIndex = operations[op * numOps + 8]; // if (cumulativeScaleIndex != BEAGLE_OP_NONE) // cumulativeScaleBuffer = gScaleBuffers[cumulativeScaleIndex]; // else // cumulativeScaleBuffer = NULL; - } + } - double *destPartials = gPartials[destinationPartialIndex]; + double *destPartials = gPartials[destinationPartialIndex]; - int startPattern = 0; - int endPattern = kPatternCount; - if (byPartition) { - startPattern = this->gPatternPartitionsStartPatterns[currentPartition]; - endPattern = this->gPatternPartitionsStartPatterns[currentPartition + 1]; + int startPattern = 0; + int endPattern = kPatternCount; + if (byPartition) { + startPattern = this->gPatternPartitionsStartPatterns[currentPartition]; + endPattern = this->gPatternPartitionsStartPatterns[currentPartition + 1]; - assert(startPattern >= 0 and startPattern <= kPatternCount); - assert(endPattern >= 0 and endPattern <= kPatternCount); - assert(startPattern <= endPattern); - } + assert(startPattern >= 0 and startPattern <= kPatternCount); + assert(endPattern >= 0 and endPattern <= kPatternCount); + assert(startPattern <= endPattern); + } int rescale = BEAGLE_OP_NONE; - double* scalingFactors = NULL; + double *scalingFactors = NULL; if (writeScalingIndex >= 0) { rescale = 1; scalingFactors = gScaleBuffers[writeScalingIndex]; @@ -523,36 +524,37 @@ namespace beagle { #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"Updating preorder partials for index: "<rescalePartialsByPartition(destPartials,scalingFactors,cumulativeScaleBuffer,0, currentPartition); - } else { - this->rescalePartials(destPartials,scalingFactors,cumulativeScaleBuffer,0); - } + double *destPartials = gPartials[destinationPartialIndex]; + if (byPartition) { + this->rescalePartialsByPartition(destPartials, scalingFactors, cumulativeScaleBuffer, 0, + currentPartition); + } else { + this->rescalePartials(destPartials, scalingFactors, cumulativeScaleBuffer, 0); + } } - if (DEBUGGING_OUTPUT) { - if (scalingFactors != NULL && rescale == 0) { - for (int i = 0; i < kPatternCount; i++) - fprintf(stderr, "old scaleFactor[%d] = %.5f\n", i, scalingFactors[i]); - } - fprintf(stderr, "Result partials:\n"); - for (int i = 0; i < this->kPartialsSize; i++) - fprintf(stderr, "destP[%d] = %.5f\n", i, destPartials[i]); - } + if (DEBUGGING_OUTPUT) { + if (scalingFactors != NULL && rescale == 0) { + for (int i = 0; i < kPatternCount; i++) + fprintf(stderr, "old scaleFactor[%d] = %.5f\n", i, scalingFactors[i]); + } + fprintf(stderr, "Result partials:\n"); + for (int i = 0; i < this->kPartialsSize; i++) + fprintf(stderr, "destP[%d] = %.5f\n", i, destPartials[i]); + } } return BEAGLE_SUCCESS; From e97a990e7089b9c138b8dc940f41ab3d9968391c Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 4 Jun 2024 16:23:38 -0500 Subject: [PATCH 090/421] intermediate gpu simple action 2 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 25 ++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 202 ++++++++++++++++++++--- 2 files changed, 204 insertions(+), 23 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index d3c2095f..313a3642 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -155,8 +155,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl protected: std::vector> hInstantaneousMatrices; - SpMatrix hIdentity; std::vector> hBs; + SpMatrix hIdentity; std::vector hMuBs; std::vector hB1Norms; const int mMax = 55; @@ -205,13 +205,16 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector dInstantaneousMatrices; std::vector dPartials; + std::vector dAs; + std::vector dACache; std::vector dPartialCache; Real **dFrequenciesCache, **dWeightsCache; std::vector dFrequencies; std::vector dWeights; - int *dMatrixCsrOffsetsCache, *dMatrixCsrColumnsCache; - Real *dMatrixCsrValuesCache; - int currentCacheNNZ; + std::vector dMatrixCsrOffsetsCache; + std::vector dMatrixCsrColumnsCache; + std::vector dMatrixCsrValuesCache; + std::vector currentCacheNNZs; Real *dPatternWeightsCache; cusparseDnVecDescr_t dPatternWeights; @@ -359,6 +362,20 @@ class BeagleGPUActionImpl : public BeagleGPUImpl double getPMax() const; int getPartialIndex(int nodeIndex, int categoryIndex); + + void calcPartialsPartials(int destPIndex, + int partials1Index, + int edgeIndex1, + int partials2Index, + int edgeIndex2); + + int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int edgeMultiplierIndex, bool transpose) const; + + std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, + int eigenIndex) const; + + double getDValue(int p, int eigenIndex) const; + }; BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 17ad8640..43ba1af8 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -443,22 +443,33 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); hIdentity.setIdentity(); hInstantaneousMatrices.resize(kEigenDecompCount); + hBs.resize(kEigenDecompCount); for (int i = 0; i < kEigenDecompCount; i++) { hInstantaneousMatrices[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); + hBs[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); } - hBs.resize(kEigenDecompCount); hMuBs.resize(kEigenDecompCount); hB1Norms.resize(kEigenDecompCount); ds.resize(kEigenDecompCount); dInstantaneousMatrices = std::vector(kEigenDecompCount, nullptr); - dMatrixCsrOffsetsCache = cudaDeviceNew(kPaddedStateCount + 1); - dMatrixCsrColumnsCache = NULL; - currentCacheNNZ = 0; + dMatrixCsrOffsetsCache.resize(kEigenDecompCount); + dMatrixCsrColumnsCache.resize(kEigenDecompCount); + dMatrixCsrValuesCache.resize(kEigenDecompCount); + currentCacheNNZs = std::vector(kEigenDecompCount, 0); + for (int i = 0; i < kEigenDecompCount; i++) { + dMatrixCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); + dMatrixCsrColumnsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); + dMatrixCsrValuesCache[i] = cudaDeviceNew(currentCacheNNZs[i] + 1); + } + dPartials = std::vector(kPartialsBufferCount * kCategoryCount, nullptr); dPartialCache = std::vector(kPartialsBufferCount * kCategoryCount, nullptr); + dAs = std::vector(kCategoryCount, nullptr); + dACache = std::vector(kCategoryCount, nullptr); + dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; dWeights = std::vector(kEigenDecompCount, nullptr); @@ -706,9 +717,168 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, int operationCount, int cumulativeScalingIndex) { - return BeagleGPUImpl::upPartials(byPartition, operations, operationCount, cumulativeScalingIndex); + //TODO: implement/re-use scaling code + + for (int op = 0; op < operationCount; op++) { + const int numOps = BEAGLE_OP_COUNT; + + const int destinationPartialIndex = operations[op * numOps]; + const int writeScalingIndex = operations[op * numOps + 1]; + const int readScalingIndex = operations[op * numOps + 2]; + const int firstChildPartialIndex = operations[op * numOps + 3]; + const int firstChildSubstitutionMatrixIndex = operations[op * numOps + 4]; + const int secondChildPartialIndex = operations[op * numOps + 5]; + const int secondChildSubstitutionMatrixIndex = operations[op * numOps + 6]; + + calcPartialsPartials(destinationPartialIndex, firstChildPartialIndex, firstChildSubstitutionMatrixIndex, + secondChildPartialIndex, secondChildSubstitutionMatrixIndex); + + } + return BEAGLE_SUCCESS; +} + +BEAGLE_GPU_TEMPLATE +void BeagleGPUActionImpl::calcPartialsPartials(int destPIndex, + int partials1Index, + int edgeIndex1, + int partials2Index, + int edgeIndex2) { + for (int category = 0; category < kCategoryCount; category++) + { + const int destPartialindex = getPartialIndex(destPIndex, category); + const int partial1Index = getPartialIndex(partial1Index, category); + const int partial2Index = getPartialIndex(partial2Index, category); + const int edgeMultiplier1 = edgeIndex1 * kCategoryCount + category; + const int edgeMultiplier2 = edgeIndex2 * kCategoryCount + category; + +// auto partials1 = partialsMap(partials1Index, category, startPattern, endPattern); +// auto partials1Cache = partialsCacheMap(partials1Index, category, startPattern, endPattern); +// simpleAction2(partials1Cache, partials1, edgeIndex1, category, false); +// +// auto partials2 = partialsMap(partials2Index, category, startPattern, endPattern); +// auto partials2Cache = partialsCacheMap(partials2Index, category, startPattern, endPattern); +// simpleAction2(partials2Cache, partials2, edgeIndex2, category, false); +// +// auto destP = partialsMap(destPIndex, category, startPattern, endPattern); +// destP = partials1Cache.cwiseProduct(partials2Cache); + } +} + +BEAGLE_GPU_TEMPLATE +double BeagleGPUActionImpl::getPMax() const +{ + return floor(0.5 + 0.5 * sqrt(5.0 + 4.0 * mMax)); } +BEAGLE_GPU_TEMPLATE +std::tuple BeagleGPUActionImpl::getStatistics2(double t, int nCol, + double edgeMultiplier, + int eigenIndex) const { + assert( t >= 0 ); + assert( nCol >= 0); + assert( edgeMultiplier >= 0 ); + assert( eigenIndex >= 0); + + if (t * hB1Norms[eigenIndex] == 0.0) + return {0, 1}; + + int bestM = INT_MAX; + double bestS = INT_MAX; // Not all the values of s can fit in a 32-bit int. + + const double theta = thetaConstants.at(mMax); + const double pMax = getPMax(); + // pMax is the largest positive integer such that p*(p-1) <= mMax + 1 + + const bool conditionFragment313 = hB1Norms[eigenIndex] * edgeMultiplier <= 2.0 * theta / ((double) nCol * mMax) * pMax * (pMax + 3); + // using l = 1 as in equation 3.13 + if (conditionFragment313) { + for (auto& [thisM, thetaM]: thetaConstants) { + const double thisS = ceil(hB1Norms[eigenIndex] * edgeMultiplier / thetaM); + if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { + bestS = thisS; + bestM = thisM; + } + } + } else { + for (int p = 2; p < pMax; p++) { + for (int thisM = p * (p - 1) - 1; thisM < mMax + 1; thisM++) { + auto it = thetaConstants.find(thisM); + if (it != thetaConstants.end()) { + // equation 3.7 in Al-Mohy and Higham + const double dValueP = getDValue(p, eigenIndex); + const double dValuePPlusOne = getDValue(p + 1, eigenIndex); + const double alpha = std::max(dValueP, dValuePPlusOne) * edgeMultiplier; + // part of equation 3.10 + const double thisS = ceil(alpha / thetaConstants.at(thisM)); + if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { + bestS = thisS; + bestM = thisM; + } + } + } + } + } + bestS = std::max(std::min(bestS, INT_MAX), 1.0); + assert( bestS >= 1 ); + assert( bestS <= INT_MAX ); + + int m = bestM; + int s = (int) bestS; + + assert(m >= 0); + assert(s >= 1); + + return {m,s}; +} + + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int edgeMultiplierIndex, bool transpose) const { + const double tol = pow(2.0, -53.0); + const double t = 1.0; + const int nCol = kPaddedStateCount * kPaddedPatternCount; + + const double edgeMultiplier = hEdgeMultipliers[edgeMultiplierIndex]; + + auto [m, s] = getStatistics2(t, nCol, edgeMultiplier, hEigenMaps[edgeIndex]); + + +#ifdef BEAGLE_DEBUG_FLOW + std::cerr << "simpleAction2: m = " << m << " s = " << s << std::endl; +#endif + +// destP = partials; + CHECK_CUSPARSE(cusparseDnMatSetValues(dPartials[destPIndex], dPartialCache[partialsIndex])) + SpMatrix A = hBs[hEigenMaps[edgeIndex]] * edgeMultiplier; + if (transpose) { + A = A.transpose(); + } + +// MatrixXd F(kStateCount, nCol); +// F = destP; +// +// const double eta = exp(t * gMuBs[gEigenMaps[edgeIndex]] * edgeMultiplier / (double) s); +// +// for (int i = 0; i < s; i++) { +// double c1 = normPInf(destP); +// for (int j = 1; j < m + 1; j++) { +// destP = A * destP; +// destP *= t / ((double) s * j); +// double c2 = normPInf(destP); +// F += destP; +// if (c1 + c2 <= tol * normPInf(F)) { +// break; +// } +// c1 = c2; +// } +// F *= eta; +// destP = F; +// } + + return BEAGLE_SUCCESS; +} + + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPrePartials(bool byPartition, const int *operations, @@ -733,18 +903,18 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, const int currentNNZ = hInstantaneousMatrices[matrixIndex].nonZeros(); const int paddedNNZ = currentNNZ + 16 - currentNNZ%16; - if (currentCacheNNZ < paddedNNZ) { - currentCacheNNZ = paddedNNZ; - dMatrixCsrColumnsCache = cudaDeviceNew(currentCacheNNZ); - dMatrixCsrValuesCache = cudaDeviceNew(currentCacheNNZ); + if (currentCacheNNZs[matrixIndex] < paddedNNZ) { + currentCacheNNZs[matrixIndex] = paddedNNZ; + dMatrixCsrColumnsCache[matrixIndex] = cudaDeviceNew(paddedNNZ); + dMatrixCsrValuesCache[matrixIndex] = cudaDeviceNew(paddedNNZ); } - MemcpyHostToDevice(dMatrixCsrOffsetsCache, hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); - MemcpyHostToDevice(dMatrixCsrColumnsCache, hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); - MemcpyHostToDevice(dMatrixCsrValuesCache, hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); + MemcpyHostToDevice(dMatrixCsrOffsetsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); + MemcpyHostToDevice(dMatrixCsrColumnsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); + MemcpyHostToDevice(dMatrixCsrValuesCache[matrixIndex], hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, - dMatrixCsrOffsetsCache, dMatrixCsrColumnsCache, dMatrixCsrValuesCache, + dMatrixCsrOffsetsCache[matrixIndex], dMatrixCsrColumnsCache[matrixIndex], dMatrixCsrValuesCache[matrixIndex], IndexType, IndexType, CUSPARSE_INDEX_BASE_ZERO, DataType)) //TODO: use cusparse function for diagonal sum? @@ -777,12 +947,6 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, return BEAGLE_SUCCESS; } -BEAGLE_GPU_TEMPLATE -double BeagleGPUActionImpl::getPMax() const -{ - return floor(0.5 + 0.5 * sqrt(5.0 + 4.0 * mMax)); -} - BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, From 294534e8ba27846791c1fca0051da72ba4c7b777 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 4 Jun 2024 20:01:26 -0500 Subject: [PATCH 091/421] intermediate simple action work, F cache size needs work --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 27 +++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 111 ++++++++++++++++++----- 2 files changed, 111 insertions(+), 27 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 313a3642..06bc82c5 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -40,6 +40,7 @@ #include #include // cudaMalloc, cudaMemcpy, etc. #include // cusparseSpMV +#include using std::vector; using std::tuple; @@ -83,6 +84,17 @@ double normPInf(const T& matrix) { } \ } +// cublas API error checking +#define CUBLAS_CHECK(err) \ + do { \ + cublasStatus_t err_ = (err); \ + if (err_ != CUBLAS_STATUS_SUCCESS) { \ + std::printf("cublas error %d at %s:%d\n", err_, __FILE__, __LINE__); \ + throw std::runtime_error("cublas error"); \ + } \ + } while (0) + + template using SpMatrix = Eigen::SparseMatrix; @@ -211,13 +223,16 @@ class BeagleGPUActionImpl : public BeagleGPUImpl Real **dFrequenciesCache, **dWeightsCache; std::vector dFrequencies; std::vector dWeights; - std::vector dMatrixCsrOffsetsCache; - std::vector dMatrixCsrColumnsCache; - std::vector dMatrixCsrValuesCache; + std::vector dInstantaneousMatrixCsrOffsetsCache; + std::vector dInstantaneousMatrixCsrColumnsCache; + std::vector dInstantaneousMatrixCsrValuesCache; + std::vector dACscValuesCache; std::vector currentCacheNNZs; Real *dPatternWeightsCache; cusparseDnVecDescr_t dPatternWeights; + cublasHandle_t cublasH; + std::vector hEigenMaps; std::vector hEdgeMultipliers; @@ -363,13 +378,17 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int getPartialIndex(int nodeIndex, int categoryIndex); + int getPartialCacheIndex(int nodeIndex, int categoryIndex); + void calcPartialsPartials(int destPIndex, int partials1Index, int edgeIndex1, int partials2Index, int edgeIndex2); - int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int edgeMultiplierIndex, bool transpose) const; + int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, bool transpose) const; + + int cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) const; std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, int eigenIndex) const; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 43ba1af8..26de106b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -453,21 +453,28 @@ int BeagleGPUActionImpl::createInstance(int tipCount, ds.resize(kEigenDecompCount); dInstantaneousMatrices = std::vector(kEigenDecompCount, nullptr); - dMatrixCsrOffsetsCache.resize(kEigenDecompCount); - dMatrixCsrColumnsCache.resize(kEigenDecompCount); - dMatrixCsrValuesCache.resize(kEigenDecompCount); + CUBLAS_CHECK(cublasCreate(&cublasH)); //TODO: destroyer: CUBLAS_CHECK(cublasDestroy(cublasH)); + + dInstantaneousMatrixCsrOffsetsCache.resize(kEigenDecompCount); + dInstantaneousMatrixCsrColumnsCache.resize(kEigenDecompCount); + dInstantaneousMatrixCsrValuesCache.resize(kEigenDecompCount); + dACscValuesCache.resize(kEigenDecompCount * kCategoryCount * 2); + dAs = std::vector(kEigenDecompCount * kCategoryCount * 2, nullptr); currentCacheNNZs = std::vector(kEigenDecompCount, 0); for (int i = 0; i < kEigenDecompCount; i++) { - dMatrixCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); - dMatrixCsrColumnsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); - dMatrixCsrValuesCache[i] = cudaDeviceNew(currentCacheNNZs[i] + 1); + dInstantaneousMatrixCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); + dInstantaneousMatrixCsrColumnsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); + dInstantaneousMatrixCsrValuesCache[i] = cudaDeviceNew(currentCacheNNZs[i] + 1); + for (int j = 0; j < kCategoryCount; j++) { + dACscValuesCache[i * kCategoryCount * 2 + 2 * j] = cudaDeviceNew(currentCacheNNZs[i] + 1); + dACscValuesCache[i * kCategoryCount * 2 + 2 * j + 1] = cudaDeviceNew(currentCacheNNZs[i] + 1); + } } - dPartials = std::vector(kPartialsBufferCount * kCategoryCount, nullptr); - dPartialCache = std::vector(kPartialsBufferCount * kCategoryCount, nullptr); + dPartials = std::vector(kPartialsBufferCount * kCategoryCount * 2, nullptr); + dPartialCache = std::vector(kPartialsBufferCount * kCategoryCount * 2, nullptr); - dAs = std::vector(kCategoryCount, nullptr); dACache = std::vector(kCategoryCount, nullptr); dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); @@ -711,6 +718,11 @@ int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int return kPartialsBufferCount * categoryIndex + nodeIndex; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::getPartialCacheIndex(int nodeIndex, int categoryIndex) { + return kPartialsBufferCount * kCategoryCount + kPartialsBufferCount * categoryIndex + nodeIndex; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPartials(bool byPartition, const int *operations, @@ -743,14 +755,24 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde int edgeIndex1, int partials2Index, int edgeIndex2) { + + cacheAMatrices(edgeIndex1, edgeIndex2, false); + cudaDeviceSynchronize(); + for (int category = 0; category < kCategoryCount; category++) { const int destPartialindex = getPartialIndex(destPIndex, category); const int partial1Index = getPartialIndex(partial1Index, category); const int partial2Index = getPartialIndex(partial2Index, category); - const int edgeMultiplier1 = edgeIndex1 * kCategoryCount + category; - const int edgeMultiplier2 = edgeIndex2 * kCategoryCount + category; + const int partial1CacheIndex = getPartialCacheIndex(partial1Index, category); + const int partial2CacheIndex = getPartialCacheIndex(partial2Index, category); + + simpleAction2(partial1CacheIndex, partial1Index, edgeIndex1, category, false); + simpleAction2(partial2CacheIndex, partial2Index, edgeIndex2, category, false); + + CUBLAS_CHECK(cublasSdgmm(cublasH, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, + dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); // auto partials1 = partialsMap(partials1Index, category, startPattern, endPattern); // auto partials1Cache = partialsCacheMap(partials1Index, category, startPattern, endPattern); // simpleAction2(partials1Cache, partials1, edgeIndex1, category, false); @@ -831,12 +853,49 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub return {m,s}; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) const { + for (int category = 0; category < kCategoryCount; category++) { + const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; + const int edgeMultiplierIndex1 = edgeIndex1 * kCategoryCount + category; + const double edgeMultiplier1 = hEdgeMultipliers[edgeMultiplierIndex1]; + + const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; + const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; + const double edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; + + CUBLAS_CHECK(cublasDscal(cublasH, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following + CUBLAS_CHECK(cublasDscal(cublasH, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following + + if (transpose) { + CHECK_CUSPARSE(cusparseCreateCsc(&dAs[matrixIndex1], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex1]], + dInstantaneousMatrixCsrOffsetsCache[hEigenMaps[edgeIndex1]], dInstantaneousMatrixCsrColumnsCache[hEigenMaps[edgeIndex1]], dACscValuesCache[matrixIndex1], + IndexType, IndexType, + CUSPARSE_INDEX_BASE_ZERO, DataType)) + CHECK_CUSPARSE(cusparseCreateCsc(&dAs[matrixIndex2], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex2]], + dInstantaneousMatrixCsrOffsetsCache[hEigenMaps[edgeIndex2]], dInstantaneousMatrixCsrColumnsCache[hEigenMaps[edgeIndex2]], dACscValuesCache[matrixIndex2], + IndexType, IndexType, + CUSPARSE_INDEX_BASE_ZERO, DataType)) + } else { + CHECK_CUSPARSE(cusparseCreateCsr(&dAs[matrixIndex1], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex1]], + dInstantaneousMatrixCsrOffsetsCache[hEigenMaps[edgeIndex1]], dInstantaneousMatrixCsrColumnsCache[hEigenMaps[edgeIndex1]], dACscValuesCache[matrixIndex1], + IndexType, IndexType, + CUSPARSE_INDEX_BASE_ZERO, DataType)) + CHECK_CUSPARSE(cusparseCreateCsr(&dAs[matrixIndex2], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex2]], + dInstantaneousMatrixCsrOffsetsCache[hEigenMaps[edgeIndex2]], dInstantaneousMatrixCsrColumnsCache[hEigenMaps[edgeIndex2]], dACscValuesCache[matrixIndex2], + IndexType, IndexType, + CUSPARSE_INDEX_BASE_ZERO, DataType)) + } + } + return BEAGLE_SUCCESS; +} BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int edgeMultiplierIndex, bool transpose) const { +int BeagleGPUActionImpl::simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, bool transpose) const { const double tol = pow(2.0, -53.0); const double t = 1.0; const int nCol = kPaddedStateCount * kPaddedPatternCount; + const int edgeMultiplierIndex = edgeIndex * kCategoryCount + category; const double edgeMultiplier = hEdgeMultipliers[edgeMultiplierIndex]; @@ -849,10 +908,10 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // destP = partials; CHECK_CUSPARSE(cusparseDnMatSetValues(dPartials[destPIndex], dPartialCache[partialsIndex])) - SpMatrix A = hBs[hEigenMaps[edgeIndex]] * edgeMultiplier; - if (transpose) { - A = A.transpose(); - } +// SpMatrix A = hBs[hEigenMaps[edgeIndex]] * edgeMultiplier; + const int matrixIndex = hEigenMaps[edgeIndex] * kCategoryCount * 2 + category; + cusparseSpMatDescr_t A = dAs[matrixIndex]; + // MatrixXd F(kStateCount, nCol); // F = destP; @@ -905,16 +964,19 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, const int paddedNNZ = currentNNZ + 16 - currentNNZ%16; if (currentCacheNNZs[matrixIndex] < paddedNNZ) { currentCacheNNZs[matrixIndex] = paddedNNZ; - dMatrixCsrColumnsCache[matrixIndex] = cudaDeviceNew(paddedNNZ); - dMatrixCsrValuesCache[matrixIndex] = cudaDeviceNew(paddedNNZ); + dInstantaneousMatrixCsrColumnsCache[matrixIndex] = cudaDeviceNew(paddedNNZ); + dInstantaneousMatrixCsrValuesCache[matrixIndex] = cudaDeviceNew(paddedNNZ); + for (int category = 0; category < kCategoryCount; category++) { + dACscValuesCache[matrixIndex * kCategoryCount + category] = cudaDeviceNew(paddedNNZ); + } } - MemcpyHostToDevice(dMatrixCsrOffsetsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); - MemcpyHostToDevice(dMatrixCsrColumnsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); - MemcpyHostToDevice(dMatrixCsrValuesCache[matrixIndex], hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); + MemcpyHostToDevice(dInstantaneousMatrixCsrOffsetsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); + MemcpyHostToDevice(dInstantaneousMatrixCsrColumnsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); + MemcpyHostToDevice(dInstantaneousMatrixCsrValuesCache[matrixIndex], hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, - dMatrixCsrOffsetsCache[matrixIndex], dMatrixCsrColumnsCache[matrixIndex], dMatrixCsrValuesCache[matrixIndex], + dInstantaneousMatrixCsrOffsetsCache[matrixIndex], dInstantaneousMatrixCsrColumnsCache[matrixIndex], dInstantaneousMatrixCsrValuesCache[matrixIndex], IndexType, IndexType, CUSPARSE_INDEX_BASE_ZERO, DataType)) //TODO: use cusparse function for diagonal sum? @@ -977,7 +1039,10 @@ BeagleImpl* BeagleGPUActionImplFactory::createImpl(int tipC int patternCount, int eigenBufferCount, int matrixBufferCount, - int categoryCount, + int categoryCount,CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, + dInstantaneousMatrixCsrOffsetsCache[matrixIndex], dInstantaneousMatrixCsrColumnsCache[matrixIndex], dInstantaneousMatrixCsrValuesCache[matrixIndex], + IndexType, IndexType, + CUSPARSE_INDEX_BASE_ZERO, DataType)) int scaleBufferCount, int resourceNumber, int pluginResourceNumber, From 3344a36997e78adc18b7d119b3724e9c180b97f6 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 4 Jun 2024 20:12:03 -0500 Subject: [PATCH 092/421] synchronize before partial partial element wise multiplication --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 +- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 42 +++++++++++++----------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 06bc82c5..8ce72319 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -386,7 +386,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int partials2Index, int edgeIndex2); - int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, bool transpose) const; + int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool transpose) const; int cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) const; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 26de106b..f9f2cd48 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -761,29 +761,32 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde for (int category = 0; category < kCategoryCount; category++) { - const int destPartialindex = getPartialIndex(destPIndex, category); - const int partial1Index = getPartialIndex(partial1Index, category); - const int partial2Index = getPartialIndex(partial2Index, category); + const int partial1Index = getPartialIndex(partials1Index, category); + const int partial2Index = getPartialIndex(partials2Index, category); + + const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); + const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); - const int partial1CacheIndex = getPartialCacheIndex(partial1Index, category); - const int partial2CacheIndex = getPartialCacheIndex(partial2Index, category); + const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; + const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; - simpleAction2(partial1CacheIndex, partial1Index, edgeIndex1, category, false); - simpleAction2(partial2CacheIndex, partial2Index, edgeIndex2, category, false); + simpleAction2(partial1CacheIndex, partial1Index, edgeIndex1, category, matrixIndex1, false); + simpleAction2(partial2CacheIndex, partial2Index, edgeIndex2, category, matrixIndex2, false); + } + cudaDeviceSynchronize(); + + for (int category = 0; category < kCategoryCount; category++) + { + const int destPartialindex = getPartialIndex(destPIndex, category); + + const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); + const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); CUBLAS_CHECK(cublasSdgmm(cublasH, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, - dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); -// auto partials1 = partialsMap(partials1Index, category, startPattern, endPattern); -// auto partials1Cache = partialsCacheMap(partials1Index, category, startPattern, endPattern); -// simpleAction2(partials1Cache, partials1, edgeIndex1, category, false); -// -// auto partials2 = partialsMap(partials2Index, category, startPattern, endPattern); -// auto partials2Cache = partialsCacheMap(partials2Index, category, startPattern, endPattern); -// simpleAction2(partials2Cache, partials2, edgeIndex2, category, false); -// -// auto destP = partialsMap(destPIndex, category, startPattern, endPattern); -// destP = partials1Cache.cwiseProduct(partials2Cache); + dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); } + cudaDeviceSynchronize(); + } BEAGLE_GPU_TEMPLATE @@ -891,7 +894,7 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, bool transpose) const { +int BeagleGPUActionImpl::simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool transpose) const { const double tol = pow(2.0, -53.0); const double t = 1.0; const int nCol = kPaddedStateCount * kPaddedPatternCount; @@ -909,7 +912,6 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // destP = partials; CHECK_CUSPARSE(cusparseDnMatSetValues(dPartials[destPIndex], dPartialCache[partialsIndex])) // SpMatrix A = hBs[hEigenMaps[edgeIndex]] * edgeMultiplier; - const int matrixIndex = hEigenMaps[edgeIndex] * kCategoryCount * 2 + category; cusparseSpMatDescr_t A = dAs[matrixIndex]; From 519c26cf9853c8efc4925f4eb04a06979876f116 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 15:15:42 -0500 Subject: [PATCH 093/421] include cuBlas library --- libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt index 0c7d09be..d3de92ac 100644 --- a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -50,7 +50,7 @@ add_library(hmsbeagle-action-cuda SHARED add_dependencies(hmsbeagle-action-cuda CudaKernels) -target_link_libraries(hmsbeagle-action-cuda cuda ${CUDA_LIBRARIES} ${CUDA_cusparse_LIBRARY} Eigen3::Eigen) +target_link_libraries(hmsbeagle-action-cuda cuda ${CUDA_LIBRARIES} ${CUDA_cusparse_LIBRARY} ${CUDA_cublas_LIBRARY} Eigen3::Eigen) install(TARGETS hmsbeagle-action-cuda DESTINATION ${BEAGLE_INSTALL_DIR} From 804cb94b89eaa3df52f4bbea048a26a15bc69b01 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 15:16:45 -0500 Subject: [PATCH 094/421] more broken action code --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 69 +++++++---- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 139 ++++++++++++++++++----- 2 files changed, 159 insertions(+), 49 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 8ce72319..92623bdb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -46,24 +46,6 @@ using std::vector; using std::tuple; using Eigen::MatrixXi; -template -double normP1(const T& matrix) { - return (Eigen::RowVectorXd::Ones(matrix.rows()) * matrix.cwiseAbs()).maxCoeff(); -} - -template -tuple ArgNormP1(const T& matrix) -{ - int x=-1; - double v = matrix.colwise().template lpNorm<1>().maxCoeff(&x); - return {v,x}; -} - -template -double normPInf(const T& matrix) { - return matrix.template lpNorm(); -} - #define CHECK_CUDA(func) \ { \ cudaError_t status = (func); \ @@ -95,6 +77,37 @@ double normPInf(const T& matrix) { } while (0) +template +double normP1(const T& matrix) { + return (Eigen::RowVectorXd::Ones(matrix.rows()) * matrix.cwiseAbs()).maxCoeff(); +} + +template +tuple ArgNormP1(const T& matrix) +{ + int x=-1; + double v = matrix.colwise().template lpNorm<1>().maxCoeff(&x); + return {v,x}; +} + +template +double normPInf(const T& matrix) { + return matrix.template lpNorm(); +} + + +double normPInf(double* matrix, int nRows, int nCols, cublasHandle_t cublasH) { + std::vector absoluteRowSums(nRows, 0); + for (int i = 0; i < nRows; i++) { + CUBLAS_CHECK(cublasDasum(cublasH, nCols, matrix + i * nCols, 1, &absoluteRowSums[i])); + } + cudaDeviceSynchronize(); + return *std::max_element(absoluteRowSums.begin(), absoluteRowSums.end()); +} + + + + template using SpMatrix = Eigen::SparseMatrix; @@ -217,9 +230,24 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector dInstantaneousMatrices; std::vector dPartials; + std::vector dFLeft; + std::vector dFRight; + std::vector dIntegrationTmpLeft; + std::vector dIntegrationTmpRight; std::vector dAs; std::vector dACache; std::vector dPartialCache; + std::vector dFLeftCache; + std::vector dFRightCache; + std::vector dIntegrationTmpLeftCache; + std::vector dIntegrationTmpRightCache; + std::vector integrationLeftBufferSize; + std::vector integrationLeftStoredBufferSize; + std::vector dIntegrationLeftBuffer; + std::vector integrationRightBufferSize; + std::vector integrationRightStoredBufferSize; + std::vector dIntegrationRightBuffer; + Real **dFrequenciesCache, **dWeightsCache; std::vector dFrequencies; std::vector dWeights; @@ -231,7 +259,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl Real *dPatternWeightsCache; cusparseDnVecDescr_t dPatternWeights; - cublasHandle_t cublasH; + cublasHandle_t cublasHandle; + cusparseHandle_t cusparseHandle; std::vector hEigenMaps; std::vector hEdgeMultipliers; @@ -386,7 +415,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int partials2Index, int edgeIndex2); - int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool transpose) const; + int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool left, bool transpose) const; int cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) const; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index f9f2cd48..9ae42248 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -453,7 +453,8 @@ int BeagleGPUActionImpl::createInstance(int tipCount, ds.resize(kEigenDecompCount); dInstantaneousMatrices = std::vector(kEigenDecompCount, nullptr); - CUBLAS_CHECK(cublasCreate(&cublasH)); //TODO: destroyer: CUBLAS_CHECK(cublasDestroy(cublasH)); + CUBLAS_CHECK(cublasCreate(&cublasHandle)); //TODO: destroyer: CUBLAS_CHECK(cublasDestroy(cublasHandle)); + CHECK_CUSPARSE(cusparseCreate(&cusparseHandle)); dInstantaneousMatrixCsrOffsetsCache.resize(kEigenDecompCount); dInstantaneousMatrixCsrColumnsCache.resize(kEigenDecompCount); @@ -475,6 +476,41 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dPartials = std::vector(kPartialsBufferCount * kCategoryCount * 2, nullptr); dPartialCache = std::vector(kPartialsBufferCount * kCategoryCount * 2, nullptr); + dFLeft = std::vector(kCategoryCount, nullptr); + dFLeftCache = std::vector(kCategoryCount, nullptr); + dFRight = std::vector(kCategoryCount, nullptr); + dFRightCache = std::vector(kCategoryCount, nullptr); + + dIntegrationTmpLeft = std::vector(kCategoryCount, nullptr); + dIntegrationTmpLeftCache = std::vector(kCategoryCount, nullptr); + dIntegrationTmpRight = std::vector(kCategoryCount, nullptr); + dIntegrationTmpRightCache = std::vector(kCategoryCount, nullptr); + integrationLeftBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); + integrationLeftStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); + dIntegrationLeftBuffer = std::vector(kCategoryCount, nullptr); + integrationRightBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); + integrationRightStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); + dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); + for (int category = 0; category < categoryCount; category++) { + dFLeftCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); + dFRightCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); + dIntegrationTmpLeftCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); + dIntegrationTmpRightCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); + + CHECK_CUSPARSE(cusparseCreateDnMat(&dFLeft[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dFLeftCache[category], + DataType, CUSPARSE_ORDER_COL)) + CHECK_CUSPARSE(cusparseCreateDnMat(&dFRight[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dFRightCache[category], + DataType, CUSPARSE_ORDER_COL)) + CHECK_CUSPARSE(cusparseCreateDnMat(&dIntegrationTmpLeft[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dIntegrationTmpLeftCache[category], + DataType, CUSPARSE_ORDER_COL)) + CHECK_CUSPARSE(cusparseCreateDnMat(&dIntegrationTmpRight[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dIntegrationTmpRightCache[category], + DataType, CUSPARSE_ORDER_COL)) + CHECK_CUDA(cudaMalloc(&dIntegrationLeftBuffer[category], integrationLeftBufferSize[category])) + CHECK_CUDA(cudaMalloc(&dIntegrationRightBuffer[category], integrationRightBufferSize[category])) + } + + + dACache = std::vector(kCategoryCount, nullptr); dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); @@ -770,8 +806,8 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; - simpleAction2(partial1CacheIndex, partial1Index, edgeIndex1, category, matrixIndex1, false); - simpleAction2(partial2CacheIndex, partial2Index, edgeIndex2, category, matrixIndex2, false); + simpleAction2(partial1CacheIndex, partial1Index, edgeIndex1, category, matrixIndex1, true, false); + simpleAction2(partial2CacheIndex, partial2Index, edgeIndex2, category, matrixIndex2, false, false); } cudaDeviceSynchronize(); @@ -782,7 +818,7 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); - CUBLAS_CHECK(cublasSdgmm(cublasH, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, + CUBLAS_CHECK(cublasSdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); } cudaDeviceSynchronize(); @@ -867,8 +903,8 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; const double edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; - CUBLAS_CHECK(cublasDscal(cublasH, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following - CUBLAS_CHECK(cublasDscal(cublasH, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following + CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following + CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following if (transpose) { CHECK_CUSPARSE(cusparseCreateCsc(&dAs[matrixIndex1], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex1]], @@ -894,7 +930,7 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool transpose) const { +int BeagleGPUActionImpl::simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool left, bool transpose) const { const double tol = pow(2.0, -53.0); const double t = 1.0; const int nCol = kPaddedStateCount * kPaddedPatternCount; @@ -909,32 +945,80 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p std::cerr << "simpleAction2: m = " << m << " s = " << s << std::endl; #endif -// destP = partials; - CHECK_CUSPARSE(cusparseDnMatSetValues(dPartials[destPIndex], dPartialCache[partialsIndex])) // SpMatrix A = hBs[hEigenMaps[edgeIndex]] * edgeMultiplier; cusparseSpMatDescr_t A = dAs[matrixIndex]; + vector F; + vector integrationTmp; + std::vector FCache; + std::vector integrationCache; + std::vector integrationBuffer; + std::vector integrationBufferSize; + std::vector integrationBufferStoredSize; + if (left) { + F = dFLeft; + integrationTmp = dIntegrationTmpLeft; + FCache = dFLeftCache; + integrationCache = dIntegrationTmpLeftCache; + integrationBuffer = dIntegrationLeftBuffer; + integrationBufferSize = integrationLeftBufferSize; + integrationBufferStoredSize = integrationLeftStoredBufferSize; + } else { + F = dFRight; + integrationTmp = dIntegrationTmpRight; + FCache = dFRightCache; + integrationCache = dIntegrationTmpRightCache; + integrationBuffer = dIntegrationRightBuffer; + integrationBufferSize = integrationRightBufferSize; + integrationBufferStoredSize = integrationRightStoredBufferSize; + } - +// destP = partials; + CHECK_CUSPARSE(cusparseDnMatSetValues(dPartials[destPIndex], dPartialCache[partialsIndex])) // MatrixXd F(kStateCount, nCol); // F = destP; -// -// const double eta = exp(t * gMuBs[gEigenMaps[edgeIndex]] * edgeMultiplier / (double) s); -// -// for (int i = 0; i < s; i++) { -// double c1 = normPInf(destP); -// for (int j = 1; j < m + 1; j++) { -// destP = A * destP; -// destP *= t / ((double) s * j); -// double c2 = normPInf(destP); + CHECK_CUSPARSE(cusparseDnMatSetValues(F[category], dPartialCache[partialsIndex])) // TODO: loop category within this function + + cudaDeviceSynchronize(); + + const Real eta = exp(t * hMuBs[hEigenMaps[edgeIndex]] * edgeMultiplier / (Real) s); + + + const Real beta = 0; + const Real one = 1; + for (int i = 0; i < s; i++) { + double c1 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + for (int j = 1; j < m + 1; j++) { + const Real alpha = t / ((Real) s * j); +// destP = alpha * A * destP; + CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A, dPartials[destPIndex], &beta, integrationTmp, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &integrationBufferSize[category])) + if(integrationBufferSize[category] > integrationBufferStoredSize[category]) { + CHECK_CUDA(cudaMalloc(&integrationBuffer[category], integrationBufferSize[category])) // TODO: is this necessary? Are there better ways to claim additional buffer? + integrationBufferStoredSize[category] = integrationBufferSize[category]; + } + + // integrationTmp = alpha * A * P + CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A, dPartials[destPIndex], &beta, integrationTmp, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, integrationBuffer[category])) //row-major layout provides higher performance (?) + + // P = IntegrationTmp + CHECK_CUDA(cudaMemcpy(dPartialCache[partialsIndex], integrationTmp[category], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) + + double c2 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); // F += destP; -// if (c1 + c2 <= tol * normPInf(F)) { -// break; -// } -// c1 = c2; -// } + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); + if (c1 + c2 <= tol * normPInf(FCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { + break; + } + c1 = c2; + } // F *= eta; + CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &eta, FCache[category], 1)); // destP = F; -// } + CHECK_CUDA(cudaMemcpy(dPartialCache[partialsIndex], FCache[category], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) + } return BEAGLE_SUCCESS; } @@ -1041,10 +1125,7 @@ BeagleImpl* BeagleGPUActionImplFactory::createImpl(int tipC int patternCount, int eigenBufferCount, int matrixBufferCount, - int categoryCount,CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, - dInstantaneousMatrixCsrOffsetsCache[matrixIndex], dInstantaneousMatrixCsrColumnsCache[matrixIndex], dInstantaneousMatrixCsrValuesCache[matrixIndex], - IndexType, IndexType, - CUSPARSE_INDEX_BASE_ZERO, DataType)) + int categoryCount, int scaleBufferCount, int resourceNumber, int pluginResourceNumber, From f76be34996c1e19a9857825c5ca528911ef32a4c Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 15:24:07 -0500 Subject: [PATCH 095/421] let there be updatePartials() so that compiler can complain --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 4 ++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 92623bdb..b80476e8 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -177,6 +177,10 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int getPartials(int bufferIndex, int scaleIndex, Real* outPartials); + + int updatePartials(const int* operations, + int operationCount, + int cumulativeScalingIndex); protected: std::vector> hInstantaneousMatrices; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 9ae42248..26582432 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -716,6 +716,27 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::updatePartials(const int* operations, + int operationCount, + int cumulativeScalingIndex) { + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::updatePartials\n"); +#endif + + bool byPartition = false; + int returnCode = upPartials(byPartition, + operations, + operationCount, + cumulativeScalingIndex); + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::updatePartials\n"); +#endif + + return returnCode; +} BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartials(int bufferIndex, From a179e7e2cc378f9f95a6379f5b229b3785d3ac1c Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 16:22:08 -0500 Subject: [PATCH 096/421] let it compile --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 22 +++++---- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 63 +++++++++++++++++------- 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index b80476e8..cf220b0f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -95,11 +95,15 @@ double normPInf(const T& matrix) { return matrix.template lpNorm(); } - -double normPInf(double* matrix, int nRows, int nCols, cublasHandle_t cublasH) { - std::vector absoluteRowSums(nRows, 0); +template +double normPInf(Real* matrix, int nRows, int nCols, cublasHandle_t cublasH) { + std::vector absoluteRowSums(nRows, 0); for (int i = 0; i < nRows; i++) { - CUBLAS_CHECK(cublasDasum(cublasH, nCols, matrix + i * nCols, 1, &absoluteRowSums[i])); + if constexpr (std::is_same::value) { + CUBLAS_CHECK(cublasSasum(cublasH, nCols, matrix + i * nCols, 1, &absoluteRowSums[i])); + } else { + CUBLAS_CHECK(cublasDasum(cublasH, nCols, matrix + i * nCols, 1, &absoluteRowSums[i])); + } } cudaDeviceSynchronize(); return *std::max_element(absoluteRowSums.begin(), absoluteRowSums.end()); @@ -245,11 +249,11 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector dFRightCache; std::vector dIntegrationTmpLeftCache; std::vector dIntegrationTmpRightCache; - std::vector integrationLeftBufferSize; - std::vector integrationLeftStoredBufferSize; + std::vector integrationLeftBufferSize; + std::vector integrationLeftStoredBufferSize; std::vector dIntegrationLeftBuffer; - std::vector integrationRightBufferSize; - std::vector integrationRightStoredBufferSize; + std::vector integrationRightBufferSize; + std::vector integrationRightStoredBufferSize; std::vector dIntegrationRightBuffer; Real **dFrequenciesCache, **dWeightsCache; @@ -421,7 +425,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool left, bool transpose) const; - int cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) const; + int cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose); std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, int eigenIndex) const; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 26582432..8ab1662c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -485,11 +485,11 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dIntegrationTmpLeftCache = std::vector(kCategoryCount, nullptr); dIntegrationTmpRight = std::vector(kCategoryCount, nullptr); dIntegrationTmpRightCache = std::vector(kCategoryCount, nullptr); - integrationLeftBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); - integrationLeftStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); + integrationLeftBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); + integrationLeftStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationLeftBuffer = std::vector(kCategoryCount, nullptr); - integrationRightBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); - integrationRightStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); + integrationRightBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); + integrationRightStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); for (int category = 0; category < categoryCount; category++) { dFLeftCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); @@ -839,8 +839,15 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); - CUBLAS_CHECK(cublasSdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, - dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); + if constexpr (std::is_same::value) { + CUBLAS_CHECK(cublasSdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, + dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); + + } else { + CUBLAS_CHECK(cublasDdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, + dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); + + } } cudaDeviceSynchronize(); @@ -852,6 +859,15 @@ double BeagleGPUActionImpl::getPMax() const return floor(0.5 + 0.5 * sqrt(5.0 + 4.0 * mMax)); } +BEAGLE_GPU_TEMPLATE +double BeagleGPUActionImpl::getDValue(int p, int eigenIndex) const +{ + assert(p >= 0 and p < ds[eigenIndex].size()); + + return ds[eigenIndex][p]; +} + + BEAGLE_GPU_TEMPLATE std::tuple BeagleGPUActionImpl::getStatistics2(double t, int nCol, double edgeMultiplier, @@ -914,18 +930,23 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) const { +int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) { for (int category = 0; category < kCategoryCount; category++) { const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; const int edgeMultiplierIndex1 = edgeIndex1 * kCategoryCount + category; - const double edgeMultiplier1 = hEdgeMultipliers[edgeMultiplierIndex1]; + const Real edgeMultiplier1 = hEdgeMultipliers[edgeMultiplierIndex1]; const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; - const double edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; + const Real edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; - CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following - CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following + if constexpr (std::is_same::value) { + CUBLAS_CHECK(cublasSscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following + CUBLAS_CHECK(cublasSscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following + } else { + CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following + CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following + } if (transpose) { CHECK_CUSPARSE(cusparseCreateCsc(&dAs[matrixIndex1], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex1]], @@ -973,8 +994,8 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p std::vector FCache; std::vector integrationCache; std::vector integrationBuffer; - std::vector integrationBufferSize; - std::vector integrationBufferStoredSize; + std::vector integrationBufferSize; + std::vector integrationBufferStoredSize; if (left) { F = dFLeft; integrationTmp = dIntegrationTmpLeft; @@ -1012,7 +1033,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p const Real alpha = t / ((Real) s * j); // destP = alpha * A * destP; CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A, dPartials[destPIndex], &beta, integrationTmp, DataType, + &alpha, A, dPartials[destPIndex], &beta, integrationTmp[category], DataType, CUSPARSE_SPMM_ALG_DEFAULT, &integrationBufferSize[category])) if(integrationBufferSize[category] > integrationBufferStoredSize[category]) { CHECK_CUDA(cudaMalloc(&integrationBuffer[category], integrationBufferSize[category])) // TODO: is this necessary? Are there better ways to claim additional buffer? @@ -1021,7 +1042,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // integrationTmp = alpha * A * P CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A, dPartials[destPIndex], &beta, integrationTmp, DataType, + &alpha, A, dPartials[destPIndex], &beta, integrationTmp[category], DataType, CUSPARSE_SPMM_ALG_DEFAULT, integrationBuffer[category])) //row-major layout provides higher performance (?) // P = IntegrationTmp @@ -1029,14 +1050,22 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p double c2 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); // F += destP; - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); + if constexpr (std::is_same::value) { + CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); + } else { + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); + } if (c1 + c2 <= tol * normPInf(FCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { break; } c1 = c2; } // F *= eta; - CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &eta, FCache[category], 1)); + if constexpr (std::is_same::value) { + CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &eta, FCache[category], 1)); + } else { + CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &eta, FCache[category], 1)); + } // destP = F; CHECK_CUDA(cudaMemcpy(dPartialCache[partialsIndex], FCache[category], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) } From bfaad38d00e1bd76943025cbed99432ce34a8226 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 16:27:41 -0500 Subject: [PATCH 097/421] NNZ number needs to be exact? --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 8ab1662c..85cb5eff 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1097,13 +1097,13 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, hInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); const int currentNNZ = hInstantaneousMatrices[matrixIndex].nonZeros(); - const int paddedNNZ = currentNNZ + 16 - currentNNZ%16; - if (currentCacheNNZs[matrixIndex] < paddedNNZ) { - currentCacheNNZs[matrixIndex] = paddedNNZ; - dInstantaneousMatrixCsrColumnsCache[matrixIndex] = cudaDeviceNew(paddedNNZ); - dInstantaneousMatrixCsrValuesCache[matrixIndex] = cudaDeviceNew(paddedNNZ); +// const int paddedNNZ = currentNNZ%16 == 0? currentNNZ : currentNNZ + 16 - currentNNZ%16; + if (currentCacheNNZs[matrixIndex] != currentNNZ) { + currentCacheNNZs[matrixIndex] = currentNNZ; + dInstantaneousMatrixCsrColumnsCache[matrixIndex] = cudaDeviceNew(currentNNZ); + dInstantaneousMatrixCsrValuesCache[matrixIndex] = cudaDeviceNew(currentNNZ); for (int category = 0; category < kCategoryCount; category++) { - dACscValuesCache[matrixIndex * kCategoryCount + category] = cudaDeviceNew(paddedNNZ); + dACscValuesCache[matrixIndex * kCategoryCount + category] = cudaDeviceNew(currentNNZ); } } From 394ace0008cdcef42078a37ace3a935a45fa5ddf Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 16:33:04 -0500 Subject: [PATCH 098/421] initialize all partials at first --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 85cb5eff..cee310d7 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -473,8 +473,14 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } - dPartials = std::vector(kPartialsBufferCount * kCategoryCount * 2, nullptr); - dPartialCache = std::vector(kPartialsBufferCount * kCategoryCount * 2, nullptr); + dPartials.resize(kPartialsBufferCount * kCategoryCount * 2); + dPartialCache.resize(kPartialsBufferCount * kCategoryCount * 2); + + for (int i = 0; i < kPartialsBufferCount * kCategoryCount * 2; i++) { + dPartialCache[i] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); + CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[i], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[i], + DataType, CUSPARSE_ORDER_COL)) + } dFLeft = std::vector(kCategoryCount, nullptr); dFLeftCache = std::vector(kCategoryCount, nullptr); From a02351f48264c0ffa16500e25e2ae8242ec4238c Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 16:36:03 -0500 Subject: [PATCH 099/421] copy over cache --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index cee310d7..eaea246e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1052,7 +1052,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p CUSPARSE_SPMM_ALG_DEFAULT, integrationBuffer[category])) //row-major layout provides higher performance (?) // P = IntegrationTmp - CHECK_CUDA(cudaMemcpy(dPartialCache[partialsIndex], integrationTmp[category], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpy(dPartialCache[partialsIndex], integrationCache[category], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) double c2 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); // F += destP; From 368f0d23c63b3d3c2342997ea71f56e12e8e6c9a Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 18:45:11 -0500 Subject: [PATCH 100/421] very bad code duplication of print device vector, checking numbers --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 13 +- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 174 ++++++++++++++++++++++- 2 files changed, 178 insertions(+), 9 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index cf220b0f..ca44f995 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -185,6 +185,13 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int updatePartials(const int* operations, int operationCount, int cumulativeScalingIndex); + + int calculateRootLogLikelihoods(const int* bufferIndices, + const int* categoryWeightsIndices, + const int* stateFrequenciesIndices, + const int* cumulativeScaleIndices, + int count, + double* outSumLogLikelihood); protected: std::vector> hInstantaneousMatrices; @@ -243,7 +250,6 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector dIntegrationTmpLeft; std::vector dIntegrationTmpRight; std::vector dAs; - std::vector dACache; std::vector dPartialCache; std::vector dFLeftCache; std::vector dFRightCache; @@ -423,7 +429,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int partials2Index, int edgeIndex2); - int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool left, bool transpose) const; + int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool left, bool transpose); int cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose); @@ -432,6 +438,9 @@ class BeagleGPUActionImpl : public BeagleGPUImpl double getDValue(int p, int eigenIndex) const; + int PrintfDeviceVector(Real* dPtr, int length, double checkValue, int *signal, Real r); + int PrintfDeviceVector(cusparseDnMatDescr_t dPtr, int length, double checkValue, int *signal, Real r); + }; BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index eaea246e..1748128d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -517,8 +517,6 @@ int BeagleGPUActionImpl::createInstance(int tipCount, - dACache = std::vector(kCategoryCount, nullptr); - dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; dWeights = std::vector(kEigenDecompCount, nullptr); @@ -744,6 +742,41 @@ int BeagleGPUActionImpl::updatePartials(const int* operation return returnCode; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::calculateRootLogLikelihoods(const int* bufferIndices, + const int* categoryWeightsIndices, + const int* stateFrequenciesIndices, + const int* cumulativeScaleIndices, + int count, + double* outSumLogLikelihood) { + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); +#endif + + int returnCode = BEAGLE_SUCCESS; + + if (count == 1) { + + } else { + return BEAGLE_ERROR_NO_IMPLEMENTATION; + } + +#ifdef BEAGLE_DEBUG_VALUES + Real r = 0; +fprintf(stderr, "parent = \n"); +gpu->PrintfDeviceVector(dIntegrationTmp, kPatternCount, r); +#endif + + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); +#endif + + return returnCode; +} + + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartials(int bufferIndex, int scaleIndex, @@ -812,6 +845,81 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::PrintfDeviceVector(Real* dPtr, + int length, + double checkValue, + int *signal, + Real r) { + Real* hPtr = (Real*) malloc(sizeof(Real) * length); + +// MemcpyDeviceToHost(hPtr, dPtr, sizeof(Real) * length); + CHECK_CUDA(cudaMemcpy(hPtr, dPtr, sizeof(Real) * length, cudaMemcpyDeviceToHost)) + printfVector(hPtr, length); + + if (checkValue != -1) { + double sum = 0; + for(int i=0; i checkValue) && (hPtr[i]-checkValue > 1.0E-4)) { + fprintf(stderr,"Check value exception! (%d) %2.5e > %2.5e (diff = %2.5e)\n", + i,hPtr[i],checkValue, (hPtr[i]-checkValue)); + if( signal != 0 ) + *signal = 1; + } + if (hPtr[i] != hPtr[i]) { + fprintf(stderr,"NaN found!\n"); + if( signal != 0 ) + *signal = 1; + } + } + if (sum == 0) { + fprintf(stderr,"Zero-sum vector!\n"); + if( signal != 0 ) + *signal = 1; + } + } + free(hPtr); +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::PrintfDeviceVector(cusparseDnMatDescr_t dPtr, + int length, + double checkValue, + int *signal, + Real r) { + Real* hPtr = (Real*) malloc(sizeof(Real) * length); + +// MemcpyDeviceToHost(hPtr, dPtr, sizeof(Real) * length); +// CHECK_CUDA(cudaMemcpy(hPtr, dPtr, sizeof(Real) * length, cudaMemcpyDeviceToHost)) + CHECK_CUSPARSE(cusparseConstDnMatGetValues(dPtr, reinterpret_cast(hPtr))) + printfVector(hPtr, length); + + if (checkValue != -1) { + double sum = 0; + for(int i=0; i checkValue) && (hPtr[i]-checkValue > 1.0E-4)) { + fprintf(stderr,"Check value exception! (%d) %2.5e > %2.5e (diff = %2.5e)\n", + i,hPtr[i],checkValue, (hPtr[i]-checkValue)); + if( signal != 0 ) + *signal = 1; + } + if (hPtr[i] != hPtr[i]) { + fprintf(stderr,"NaN found!\n"); + if( signal != 0 ) + *signal = 1; + } + } + if (sum == 0) { + fprintf(stderr,"Zero-sum vector!\n"); + if( signal != 0 ) + *signal = 1; + } + } + free(hPtr); +} + BEAGLE_GPU_TEMPLATE void BeagleGPUActionImpl::calcPartialsPartials(int destPIndex, int partials1Index, @@ -834,6 +942,12 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; simpleAction2(partial1CacheIndex, partial1Index, edgeIndex1, category, matrixIndex1, true, false); + + +#ifdef BEAGLE_DEBUG_FLOW + PrintfDeviceVector(dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); +#endif + simpleAction2(partial2CacheIndex, partial2Index, edgeIndex2, category, matrixIndex2, false, false); } cudaDeviceSynchronize(); @@ -954,6 +1068,16 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following } +#ifdef BEAGLE_DEBUG_FLOW + + std::cerr<<"matrixIndex1 = "< integrationBufferStoredSize[category]) { CHECK_CUDA(cudaMalloc(&integrationBuffer[category], integrationBufferSize[category])) // TODO: is this necessary? Are there better ways to claim additional buffer? integrationBufferStoredSize[category] = integrationBufferSize[category]; @@ -1116,6 +1273,9 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, MemcpyHostToDevice(dInstantaneousMatrixCsrOffsetsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); MemcpyHostToDevice(dInstantaneousMatrixCsrColumnsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); MemcpyHostToDevice(dInstantaneousMatrixCsrValuesCache[matrixIndex], hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); + for (int category = 0; category < kCategoryCount; category++) { + MemcpyHostToDevice(dACscValuesCache[matrixIndex * kCategoryCount + category], hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); + } CHECK_CUSPARSE(cusparseCreateCsr(&dInstantaneousMatrices[matrixIndex], kPaddedStateCount, kPaddedStateCount, currentNNZ, dInstantaneousMatrixCsrOffsetsCache[matrixIndex], dInstantaneousMatrixCsrColumnsCache[matrixIndex], dInstantaneousMatrixCsrValuesCache[matrixIndex], From d03ffb5564ca1296f0cca2b5c4c3f6db5fe3a7e7 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 5 Jun 2024 19:07:50 -0500 Subject: [PATCH 101/421] last bit of the day --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 1748128d..23612e8b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1060,6 +1060,9 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; const Real edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; + + CHECK_CUDA(cudaMemcpy(dACscValuesCache[matrixIndex1], dInstantaneousMatrixCsrValuesCache[hEigenMaps[edgeIndex1]], sizeof(Real) * currentCacheNNZs[hEigenMaps[edgeIndex1]], cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpy(dACscValuesCache[matrixIndex2], dInstantaneousMatrixCsrValuesCache[hEigenMaps[edgeIndex2]], sizeof(Real) * currentCacheNNZs[hEigenMaps[edgeIndex2]], cudaMemcpyDeviceToDevice)) if constexpr (std::is_same::value) { CUBLAS_CHECK(cublasSscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following CUBLAS_CHECK(cublasSscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following @@ -1195,7 +1198,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p CUSPARSE_SPMM_ALG_DEFAULT, &integrationBufferSize[category])) #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"AP = \n"< %2.5e (diff = %2.5e)\n", + i,hPtr[i],checkValue, (hPtr[i]-checkValue)); + if( signal != 0 ) + *signal = 1; + } + if (hPtr[i] != hPtr[i]) { + fprintf(stderr,"NaN found!\n"); + if( signal != 0 ) + *signal = 1; + } + } + if (sum == 0) { + fprintf(stderr,"Zero-sum vector!\n"); + if( signal != 0 ) + *signal = 1; + } + } + free(hPtr); +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::PrintfDeviceVector(Real* dPtr, int length, @@ -1262,6 +1299,24 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, } hInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); +#ifdef BEAGLE_DEBUG_FLOW + + std::cerr<<"CPU instantaneous matrix "<::setSparseMatrix(int matrixIndex, MemcpyHostToDevice(dInstantaneousMatrixCsrColumnsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); MemcpyHostToDevice(dInstantaneousMatrixCsrValuesCache[matrixIndex], hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); +#ifdef BEAGLE_DEBUG_FLOW + + std::cerr<<"\ndInstantaneousMatrixCsrOffsets for "<< matrixIndex <<"="< Date: Thu, 6 Jun 2024 20:25:53 -0500 Subject: [PATCH 105/421] still wrong number, but all positive now --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 15 ++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 76 ++++++++++++++++++------ 2 files changed, 71 insertions(+), 20 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 145e7917..279be976 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -96,13 +96,21 @@ double normPInf(const T& matrix) { } template -double normPInf(Real* matrix, int nRows, int nCols, cublasHandle_t cublasH) { +double normPInf(Real* matrix, Real* transposeBuffer, int nRows, int nCols, cublasHandle_t cublasH) { std::vector absoluteRowSums(nRows, 0); + const Real one = 1.0; + const Real zero = 0.0; + if constexpr (std::is_same::value) { + CUBLAS_CHECK(cublasSgeam(cublasH, CUBLAS_OP_T, CUBLAS_OP_N, nRows, nCols, &one, matrix, nRows, &zero, matrix, nRows, transposeBuffer, nRows)); + } else { + CUBLAS_CHECK(cublasDgeam(cublasH, CUBLAS_OP_T, CUBLAS_OP_N, nRows, nCols, &one, matrix, nRows, &zero, matrix, nRows, transposeBuffer, nRows)); + } + for (int i = 0; i < nRows; i++) { if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSasum(cublasH, nCols, matrix + i * nCols, 1, &absoluteRowSums[i])); + CUBLAS_CHECK(cublasSasum(cublasH, nCols, transposeBuffer + i * nCols, 1, &absoluteRowSums[i])); } else { - CUBLAS_CHECK(cublasDasum(cublasH, nCols, matrix + i * nCols, 1, &absoluteRowSums[i])); + CUBLAS_CHECK(cublasDasum(cublasH, nCols, transposeBuffer + i * nCols, 1, &absoluteRowSums[i])); } } cudaDeviceSynchronize(); @@ -261,6 +269,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector integrationRightBufferSize; std::vector integrationRightStoredBufferSize; std::vector dIntegrationRightBuffer; + Real* dTransposeBufferCache; Real **dFrequenciesCache, **dWeightsCache; std::vector dFrequencies; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5d65a3c5..39e75c2c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -497,6 +497,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, integrationRightBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); integrationRightStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); + CHECK_CUDA(cudaMalloc((void**) &dTransposeBufferCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount)) for (int category = 0; category < categoryCount; category++) { dFLeftCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); dFRightCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); @@ -965,7 +966,6 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde int edgeIndex2) { cacheAMatrices(edgeIndex1, edgeIndex2, false); - cudaDeviceSynchronize(); for (int category = 0; category < kCategoryCount; category++) { @@ -1005,6 +1005,15 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); } +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"Checking p_parent = p_1 * p_2, parent index = "<::simpleAction2(int destPIndex, int p CHECK_CUDA(cudaMemcpy(FCache[category], dPartialCache[partialsIndex], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) // CHECK_CUSPARSE(cusparseDnMatSetValues(F[category], dPartialCache[partialsIndex])) // TODO: loop category within this function #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"F = partials operation, FCache:\n"<::simpleAction2(int destPIndex, int p const Real beta = 0; const Real one = 1; for (int i = 0; i < s; i++) { - double c1 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + double c1 = normPInf(dPartialCache[partialsIndex], dTransposeBufferCache, kPaddedStateCount, kPaddedPatternCount, cublasHandle); + +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"Transposing:"<::value) { CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); } else { CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); } - if (c1 + c2 <= tol * normPInf(FCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { +#ifdef BEAGLE_DEBUG_FLOW + + std::cerr<<"F += destP, c1 = " < Date: Fri, 7 Jun 2024 11:58:07 -0500 Subject: [PATCH 106/421] fix CPU action PInf norm --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 87 ++++++++++++++---------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index f6926268..db3ee081 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -62,7 +62,11 @@ tuple ArgNormP1(const T& matrix) template double normPInf(const T& matrix) { - return matrix.template lpNorm(); +//#ifdef BEAGLE_DEBUG_FLOW +// std::cerr<<"matrix =\n" << matrix<().maxCoeff() << std::endl; +//#endif + return matrix.rowwise().template lpNorm<1>().maxCoeff(); } std::independent_bits_engine engine; @@ -742,55 +746,64 @@ namespace beagle { BEAGLE_CPU_ACTION_TEMPLATE void - BeagleCPUActionImpl::simpleAction2(MapType& destP, MapType& partials, int edgeIndex, + BeagleCPUActionImpl::simpleAction2(MapType &destP, MapType &partials, int edgeIndex, int category, bool transpose) const { #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"New impl 2\nRate category "< Date: Fri, 7 Jun 2024 12:06:40 -0500 Subject: [PATCH 107/421] seemingly correct numbers for updatePartials --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 16 +- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 195 +++++++++++------------ 2 files changed, 108 insertions(+), 103 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 279be976..7b37d5f7 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -113,6 +113,14 @@ double normPInf(Real* matrix, Real* transposeBuffer, int nRows, int nCols, cubla CUBLAS_CHECK(cublasDasum(cublasH, nCols, transposeBuffer + i * nCols, 1, &absoluteRowSums[i])); } } +// +//#ifdef BEAGLE_DEBUG_FLOW +// std::cerr<<"absoluteRowSums = "; +// for (int i = 0; i < nCols; i++) { +// std::cerr << absoluteRowSums[i]<<", "; +// } +//#endif + cudaDeviceSynchronize(); return *std::max_element(absoluteRowSums.begin(), absoluteRowSums.end()); } @@ -251,7 +259,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl {55, 9.9}, }; - std::vector dInstantaneousMatrices; +// std::vector dInstantaneousMatrices; std::vector dPartials; std::vector dFLeft; std::vector dFRight; @@ -274,9 +282,9 @@ class BeagleGPUActionImpl : public BeagleGPUImpl Real **dFrequenciesCache, **dWeightsCache; std::vector dFrequencies; std::vector dWeights; - std::vector dInstantaneousMatrixCsrOffsetsCache; - std::vector dInstantaneousMatrixCsrColumnsCache; - std::vector dInstantaneousMatrixCsrValuesCache; + std::vector dBsCsrOffsetsCache; + std::vector dBsCsrColumnsCache; + std::vector dBsCsrValuesCache; std::vector dACscValuesCache; std::vector currentCacheNNZs; Real *dPatternWeightsCache; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 39e75c2c..b3206443 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -451,21 +451,20 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hMuBs.resize(kEigenDecompCount); hB1Norms.resize(kEigenDecompCount); ds.resize(kEigenDecompCount); - dInstantaneousMatrices = std::vector(kEigenDecompCount, nullptr); CUBLAS_CHECK(cublasCreate(&cublasHandle)); //TODO: destroyer: CUBLAS_CHECK(cublasDestroy(cublasHandle)); CHECK_CUSPARSE(cusparseCreate(&cusparseHandle)); - dInstantaneousMatrixCsrOffsetsCache.resize(kEigenDecompCount); - dInstantaneousMatrixCsrColumnsCache.resize(kEigenDecompCount); - dInstantaneousMatrixCsrValuesCache.resize(kEigenDecompCount); + dBsCsrOffsetsCache.resize(kEigenDecompCount); + dBsCsrColumnsCache.resize(kEigenDecompCount); + dBsCsrValuesCache.resize(kEigenDecompCount); dACscValuesCache.resize(kEigenDecompCount * kCategoryCount * 2); dAs = std::vector(kEigenDecompCount * kCategoryCount * 2, nullptr); currentCacheNNZs = std::vector(kEigenDecompCount, kPaddedStateCount); for (int i = 0; i < kEigenDecompCount; i++) { - dInstantaneousMatrixCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); - dInstantaneousMatrixCsrColumnsCache[i] = cudaDeviceNew(currentCacheNNZs[i]); - dInstantaneousMatrixCsrValuesCache[i] = cudaDeviceNew(currentCacheNNZs[i]); + dBsCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); + dBsCsrColumnsCache[i] = cudaDeviceNew(currentCacheNNZs[i]); + dBsCsrValuesCache[i] = cudaDeviceNew(currentCacheNNZs[i]); for (int j = 0; j < kCategoryCount; j++) { dACscValuesCache[i * kCategoryCount * 2 + 2 * j] = cudaDeviceNew(currentCacheNNZs[i]); dACscValuesCache[i * kCategoryCount * 2 + 2 * j + 1] = cudaDeviceNew(currentCacheNNZs[i]); @@ -981,9 +980,9 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde simpleAction2(partial1CacheIndex, partial1Index, edgeIndex1, category, matrixIndex1, true, false); -#ifdef BEAGLE_DEBUG_FLOW - PrintfDeviceVector(dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); -#endif +//#ifdef BEAGLE_DEBUG_FLOW +// PrintfDeviceVector(dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); +//#endif simpleAction2(partial2CacheIndex, partial2Index, edgeIndex2, category, matrixIndex2, false, false); } @@ -1005,15 +1004,15 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); } -#ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"Checking p_parent = p_1 * p_2, parent index = "<::setSparseMatrix(int matrixIndex, #endif - const int currentNNZ = hInstantaneousMatrices[matrixIndex].nonZeros(); -// const int paddedNNZ = currentNNZ%16 == 0? currentNNZ : currentNNZ + 16 - currentNNZ%16; + //TODO: use cusparse function for diagonal sum? + Real mu_B = 0.0; + for (int i = 0; i < kStateCount; i++) { + mu_B += hInstantaneousMatrices[matrixIndex].coeff(i, i); + } + mu_B /= (Real) kStateCount; + + hMuBs[matrixIndex] = mu_B; + hBs[matrixIndex] = hInstantaneousMatrices[matrixIndex] - mu_B * hIdentity; + hB1Norms[matrixIndex] = normP1(hBs[matrixIndex]); + + const int currentNNZ = hBs[matrixIndex].nonZeros(); if (currentCacheNNZs[matrixIndex] != currentNNZ) { currentCacheNNZs[matrixIndex] = currentNNZ; - dInstantaneousMatrixCsrColumnsCache[matrixIndex] = cudaDeviceNew(currentNNZ); - dInstantaneousMatrixCsrValuesCache[matrixIndex] = cudaDeviceNew(currentNNZ); + dBsCsrColumnsCache[matrixIndex] = cudaDeviceNew(currentNNZ); + dBsCsrValuesCache[matrixIndex] = cudaDeviceNew(currentNNZ); for (int category = 0; category < kCategoryCount; category++) { dACscValuesCache[matrixIndex * kCategoryCount * 2 + category] = cudaDeviceNew(currentNNZ); dACscValuesCache[matrixIndex * kCategoryCount * 2 + kCategoryCount + category] = cudaDeviceNew(currentNNZ); } } - MemcpyHostToDevice(dInstantaneousMatrixCsrOffsetsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); - MemcpyHostToDevice(dInstantaneousMatrixCsrColumnsCache[matrixIndex], hInstantaneousMatrices[matrixIndex].innerIndexPtr(), currentNNZ); - MemcpyHostToDevice(dInstantaneousMatrixCsrValuesCache[matrixIndex], hInstantaneousMatrices[matrixIndex].valuePtr(), currentNNZ); + MemcpyHostToDevice(dBsCsrOffsetsCache[matrixIndex], hBs[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); + MemcpyHostToDevice(dBsCsrColumnsCache[matrixIndex], hBs[matrixIndex].innerIndexPtr(), currentNNZ); + MemcpyHostToDevice(dBsCsrValuesCache[matrixIndex], hBs[matrixIndex].valuePtr(), currentNNZ); #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"\ndInstantaneousMatrixCsrOffsets for "<< matrixIndex <<"="<().maxCoeff() << std::endl; //#endif - return matrix.rowwise().template lpNorm<1>().maxCoeff(); + return matrix.template lpNorm(); } std::independent_bits_engine engine; @@ -773,7 +773,7 @@ namespace beagle { #ifdef BEAGLE_DEBUG_FLOW std::cerr << "simpleAction2: m = " << m << " s = " << s << " eta = " << eta << " edgeMultiplier = " << edgeMultiplier << std::endl; - std::cerr << "edgeMultiplier = " << edgeMultiplier << "\nQ = " << gBs[gEigenMaps[edgeIndex]] < -double normPInf(Real* matrix, Real* transposeBuffer, int nRows, int nCols, cublasHandle_t cublasH) { - std::vector absoluteRowSums(nRows, 0); - const Real one = 1.0; - const Real zero = 0.0; +Real normPInf(Real* matrix, int nRows, int nCols, cublasHandle_t cublasH) { + int index; + Real result; if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSgeam(cublasH, CUBLAS_OP_T, CUBLAS_OP_N, nRows, nCols, &one, matrix, nRows, &zero, matrix, nRows, transposeBuffer, nRows)); + CUBLAS_CHECK(cublasIsamax(cublasH, nRows * nCols, matrix, 1, &index)); } else { - CUBLAS_CHECK(cublasDgeam(cublasH, CUBLAS_OP_T, CUBLAS_OP_N, nRows, nCols, &one, matrix, nRows, &zero, matrix, nRows, transposeBuffer, nRows)); + CUBLAS_CHECK(cublasIdamax(cublasH, nRows * nCols, matrix, 1, &index)); } - - for (int i = 0; i < nRows; i++) { - if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSasum(cublasH, nCols, transposeBuffer + i * nCols, 1, &absoluteRowSums[i])); - } else { - CUBLAS_CHECK(cublasDasum(cublasH, nCols, transposeBuffer + i * nCols, 1, &absoluteRowSums[i])); - } - } -// -//#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"absoluteRowSums = "; -// for (int i = 0; i < nCols; i++) { -// std::cerr << absoluteRowSums[i]<<", "; -// } -//#endif - cudaDeviceSynchronize(); - return *std::max_element(absoluteRowSums.begin(), absoluteRowSums.end()); + CHECK_CUDA(cudaMemcpy(&result, matrix + index, sizeof(Real), cudaMemcpyDeviceToHost)) + return std::abs(result); } @@ -277,7 +261,6 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector integrationRightBufferSize; std::vector integrationRightStoredBufferSize; std::vector dIntegrationRightBuffer; - Real* dTransposeBufferCache; Real **dFrequenciesCache, **dWeightsCache; std::vector dFrequencies; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index b3206443..374f3b7b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -496,7 +496,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, integrationRightBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); integrationRightStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); - CHECK_CUDA(cudaMalloc((void**) &dTransposeBufferCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount)) + for (int category = 0; category < categoryCount; category++) { dFLeftCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); dFRightCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); @@ -1231,7 +1231,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p const Real zero = 0; const Real one = 1; for (int i = 0; i < s; i++) { - double c1 = normPInf(dPartialCache[partialsIndex], dTransposeBufferCache, kPaddedStateCount, kPaddedPatternCount, cublasHandle); + double c1 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"Transposing:"<::simpleAction2(int destPIndex, int p // PrintfDeviceVector(dPartialCache[destPIndex], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); //#endif - double c2 = normPInf(integrationCache[category], dTransposeBufferCache, kPaddedStateCount, kPaddedPatternCount, cublasHandle); + double c2 = normPInf(integrationCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); // F += destP; if constexpr (std::is_same::value) { CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); @@ -1290,7 +1290,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // PrintfDeviceVector(FCache[category], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); //#endif - if (c1 + c2 <= tol * normPInf(FCache[category], dTransposeBufferCache, kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { + if (c1 + c2 <= tol * normPInf(FCache[category],kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { break; } c1 = c2; From 8237a3b4640fb8cff73771d4a47473ee2acc9ae6 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 10 Jun 2024 14:00:23 -0500 Subject: [PATCH 109/421] refactor dPartials to Wrapper in action --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 4 ++-- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 256d510d..34d6f0d9 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -244,7 +244,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl }; // std::vector dInstantaneousMatrices; - std::vector dPartials; + std::vector dPartialsWrapper; std::vector dFLeft; std::vector dFRight; std::vector dIntegrationTmpLeft; @@ -357,7 +357,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::kDerivBuffersInitialised; using BeagleGPUImpl::kMultipleDerivativesLength; -// using BeagleGPUImpl::dPartials; +// using BeagleGPUImpl::dPartialsWrapper; // using BeagleGPUImpl::dPartialsOrigin; using BeagleGPUImpl::hPartialsOffsets; using BeagleGPUImpl::kIndexOffsetPat; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 374f3b7b..1efd436e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -472,12 +472,12 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } - dPartials.resize(kPartialsBufferCount * kCategoryCount * 2); + dPartialsWrapper.resize(kPartialsBufferCount * kCategoryCount * 2); dPartialCache.resize(kPartialsBufferCount * kCategoryCount * 2); for (int i = 0; i < kPartialsBufferCount * kCategoryCount * 2; i++) { dPartialCache[i] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); - CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[i], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[i], + CHECK_CUSPARSE(cusparseCreateDnMat(&dPartialsWrapper[i], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[i], DataType, CUSPARSE_ORDER_COL)) } @@ -705,11 +705,11 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const if (tipIndex < kTipCount) { for (int i = 0; i < kCategoryCount; i++) { const int partialIndex = getPartialIndex(tipIndex, i); - if (dPartials[partialIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnMat(&dPartials[partialIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[partialIndex], + if (dPartialsWrapper[partialIndex] == NULL) { + CHECK_CUSPARSE(cusparseCreateDnMat(&dPartialsWrapper[partialIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[partialIndex], DataType, CUSPARSE_ORDER_COL)) } else { - CHECK_CUSPARSE(cusparseDnMatSetValues(dPartials[partialIndex], dPartialCache[partialIndex])) + CHECK_CUSPARSE(cusparseDnMatSetValues(dPartialsWrapper[partialIndex], dPartialCache[partialIndex])) } } } @@ -1198,12 +1198,12 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // std::cerr<<"Before destP = partials operation, destPCache:\n"<::simpleAction2(int destPIndex, int p //#endif // destP = alpha * A * destP; CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A, dPartials[destPIndex], &zero, integrationTmp[category], DataType, + &alpha, A, dPartialsWrapper[destPIndex], &zero, integrationTmp[category], DataType, CUSPARSE_SPMM_ALG_DEFAULT, &integrationBufferSize[category])) if(integrationBufferSize[category] > integrationBufferStoredSize[category]) { @@ -1257,7 +1257,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // integrationTmp = alpha * A * destP CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A, dPartials[destPIndex], &zero, integrationTmp[category], DataType, + &alpha, A, dPartialsWrapper[destPIndex], &zero, integrationTmp[category], DataType, CUSPARSE_SPMM_ALG_DEFAULT, integrationBuffer[category])) //row-major layout provides higher performance (?) //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"edge multiplier = "<< hEdgeMultipliers[edgeIndex * kCategoryCount + category]<<"\nB ="< Date: Mon, 10 Jun 2024 14:36:13 -0500 Subject: [PATCH 110/421] gradually remove unecessary code --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 6 +- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 143 ++--------------------- 2 files changed, 11 insertions(+), 138 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 34d6f0d9..9b4837bd 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -200,7 +200,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector hMuBs; std::vector hB1Norms; const int mMax = 55; - std::vector> ds; + std::vector> hds; std::map thetaConstants = { //The first 30 values are from table A.3 of Computing Matrix Functions. // For double precision, tol = 2^(-53) @@ -357,8 +357,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::kDerivBuffersInitialised; using BeagleGPUImpl::kMultipleDerivativesLength; -// using BeagleGPUImpl::dPartialsWrapper; -// using BeagleGPUImpl::dPartialsOrigin; + using BeagleGPUImpl::dPartials; + using BeagleGPUImpl::dPartialsOrigin; using BeagleGPUImpl::hPartialsOffsets; using BeagleGPUImpl::kIndexOffsetPat; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 1efd436e..5858f0e5 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -305,137 +305,10 @@ int BeagleGPUActionImpl::createInstance(int tipCount, fprintf(stderr, "\tEntering BeagleGPUActionImpl::createInstance\n"); #endif - kInitialized = 0; + BeagleGPUImpl::createInstance(tipCount, partialsBufferCount * 2, compactBufferCount, stateCount, patternCount, + eigenDecompositionCount, matrixCount, categoryCount, scaleBufferCount, globalResourceNumber, pluginResourceNumber, + preferenceFlags, requirementFlags); - kTipCount = tipCount; - kPartialsBufferCount = partialsBufferCount; - kCompactBufferCount = 0; // XJ: ignore compact buffer count because of using all partials for tip states - kStateCount = stateCount; - kEigenDecompCount = eigenDecompositionCount; - kMatrixCount = matrixCount; - kCategoryCount = categoryCount; - kScaleBufferCount = scaleBufferCount; - - kPartitionCount = 1; // XJ: seems related to stream setup - kMaxPartitionCount = kPartitionCount; - kPartitionsInitialised = false; - kPatternsReordered = false; - - resourceNumber = globalResourceNumber; - - kTipPartialsBufferCount = kTipCount; // use Partials for all tip states - kBufferCount = kPartialsBufferCount; - - kInternalPartialsBufferCount = kBufferCount - kTipCount; - - //TODO: check if pad state is necessary - kPaddedStateCount = kStateCount; - - gpu = new GPUInterface(); - - gpu->Initialize(); - - int numDevices = 0; - numDevices = gpu->GetDeviceCount(); - if (numDevices == 0) { - fprintf(stderr, "Error: No GPU devices\n"); - return BEAGLE_ERROR_NO_RESOURCE; - } - if (pluginResourceNumber > numDevices) { - fprintf(stderr,"Error: Trying to initialize device # %d (which does not exist)\n",resourceNumber); - return BEAGLE_ERROR_NO_RESOURCE; - } - - //TODO: check if pad patterns is necessary, ignored for now - bool CPUImpl = false; - - kDeviceType = gpu->GetDeviceTypeFlag(pluginResourceNumber); - kDeviceCode = gpu->GetDeviceImplementationCode(pluginResourceNumber); - - kPatternCount = patternCount; - kPaddedPatternCount = kPatternCount; - kScaleBufferSize = kPaddedPatternCount; - - kFlags = 0; - - if (preferenceFlags & BEAGLE_FLAG_SCALING_AUTO || requirementFlags & BEAGLE_FLAG_SCALING_AUTO) { - kFlags |= BEAGLE_FLAG_SCALING_AUTO; - kFlags |= BEAGLE_FLAG_SCALERS_LOG; - kScaleBufferCount = kInternalPartialsBufferCount; - kScaleBufferSize *= kCategoryCount; - } else if (preferenceFlags & BEAGLE_FLAG_SCALING_ALWAYS || requirementFlags & BEAGLE_FLAG_SCALING_ALWAYS) { - kFlags |= BEAGLE_FLAG_SCALING_ALWAYS; - kFlags |= BEAGLE_FLAG_SCALERS_LOG; - kScaleBufferCount = kInternalPartialsBufferCount + 1; // +1 for temp buffer used by edgelikelihood - } else if (preferenceFlags & BEAGLE_FLAG_SCALING_DYNAMIC || requirementFlags & BEAGLE_FLAG_SCALING_DYNAMIC) { - kFlags |= BEAGLE_FLAG_SCALING_DYNAMIC; - kFlags |= BEAGLE_FLAG_SCALERS_RAW; - } else if (preferenceFlags & BEAGLE_FLAG_SCALERS_LOG || requirementFlags & BEAGLE_FLAG_SCALERS_LOG) { - kFlags |= BEAGLE_FLAG_SCALING_MANUAL; - kFlags |= BEAGLE_FLAG_SCALERS_LOG; - } else { - kFlags |= BEAGLE_FLAG_SCALING_MANUAL; - kFlags |= BEAGLE_FLAG_SCALERS_RAW; - } - //TODO: remember to implement scaling - -// if (preferenceFlags & BEAGLE_FLAG_EIGEN_COMPLEX || requirementFlags & BEAGLE_FLAG_EIGEN_COMPLEX) { -// kFlags |= BEAGLE_FLAG_EIGEN_COMPLEX; -// } else { -// kFlags |= BEAGLE_FLAG_EIGEN_REAL; -// } - - if (requirementFlags & BEAGLE_FLAG_INVEVEC_TRANSPOSED || preferenceFlags & BEAGLE_FLAG_INVEVEC_TRANSPOSED) - kFlags |= BEAGLE_FLAG_INVEVEC_TRANSPOSED; - else - kFlags |= BEAGLE_FLAG_INVEVEC_STANDARD; - - - // TODO: this chunk of control is not checked - if (kDeviceCode == BEAGLE_OPENCL_DEVICE_APPLE_CPU) - kFlags |= BEAGLE_FLAG_PARALLELOPS_STREAMS; - else if (requirementFlags & BEAGLE_FLAG_PARALLELOPS_STREAMS || preferenceFlags & BEAGLE_FLAG_PARALLELOPS_STREAMS) - kFlags |= BEAGLE_FLAG_PARALLELOPS_STREAMS; - else if (requirementFlags & BEAGLE_FLAG_PARALLELOPS_GRID || preferenceFlags & BEAGLE_FLAG_PARALLELOPS_GRID) - kFlags |= BEAGLE_FLAG_PARALLELOPS_GRID; - - if (preferenceFlags & BEAGLE_FLAG_COMPUTATION_ASYNCH || requirementFlags & BEAGLE_FLAG_COMPUTATION_ASYNCH) { - kFlags |= BEAGLE_FLAG_COMPUTATION_ASYNCH; - } else { - kFlags |= BEAGLE_FLAG_COMPUTATION_SYNCH; - } - - if (preferenceFlags & BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO || requirementFlags & BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO) { - kFlags |= BEAGLE_FLAG_PREORDER_TRANSPOSE_AUTO; - } else { - kFlags |= BEAGLE_FLAG_PREORDER_TRANSPOSE_MANUAL; - } - - Real r = 0; - modifyFlagsForPrecision(&kFlags, r); - - //TODO: sum block size ignored - kPartialsSize = kPaddedPatternCount * kPaddedStateCount * kCategoryCount; - kMatrixSize = kPaddedStateCount * kPaddedStateCount; - - kLastCompactBufferIndex = -1; - kLastTipPartialsBufferIndex = -1; - - //TODO: we probably don't use the kernel resource anyway, separate it? - gpu->SetDevice(pluginResourceNumber, kPaddedStateCount, kCategoryCount, - kPaddedPatternCount, kPatternCount, kTipCount, kFlags); - - int ptrQueueLength = kMatrixCount * kCategoryCount * 3 * 3; // first '3' for derivatives, last '3' is for 3 ops for uTMWMM - if (kInternalPartialsBufferCount > ptrQueueLength) - ptrQueueLength = kInternalPartialsBufferCount; - - // TODO: not sure if need kernels -// kernels = new KernelLauncher(gpu); - - hWeightsCache = (Real*) gpu->CallocHost(kCategoryCount, sizeof(Real)); - hPatternWeightsCache = (Real*) gpu->CallocHost(kPatternCount, sizeof(Real)); - hFrequenciesCache = (Real*) gpu->CallocHost(kPaddedStateCount, sizeof(Real)); - hPartialsCache = (Real*) gpu->CallocHost(kPartialsSize, sizeof(Real)); int hMatrixCacheSize = kMatrixSize * kCategoryCount * BEAGLE_CACHED_MATRICES_COUNT; //TODO: use Eigen csr representation? hLogLikelihoodsCache = (Real*) gpu->MallocHost(kPatternCount * sizeof(Real)); @@ -450,7 +323,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } hMuBs.resize(kEigenDecompCount); hB1Norms.resize(kEigenDecompCount); - ds.resize(kEigenDecompCount); + hds.resize(kEigenDecompCount); CUBLAS_CHECK(cublasCreate(&cublasHandle)); //TODO: destroyer: CUBLAS_CHECK(cublasDestroy(cublasHandle)); CHECK_CUSPARSE(cusparseCreate(&cusparseHandle)); @@ -1027,9 +900,9 @@ double BeagleGPUActionImpl::getPMax() const BEAGLE_GPU_TEMPLATE double BeagleGPUActionImpl::getDValue(int p, int eigenIndex) const { - assert(p >= 0 and p < ds[eigenIndex].size()); + assert(p >= 0 and p < hds[eigenIndex].size()); - return ds[eigenIndex][p]; + return hds[eigenIndex][p]; } @@ -1400,7 +1273,7 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, #endif - ds[matrixIndex].clear(); + hds[matrixIndex].clear(); int pMax = getPMax(); for(int p=0;p <= pMax+1; p++) @@ -1409,7 +1282,7 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, Real approx_norm = normest1( hBs[matrixIndex], p, t); // equation 3.7 in Al-Mohy and Higham - ds[matrixIndex].push_back( pow( approx_norm, 1.0/Real(p) ) ); + hds[matrixIndex].push_back(pow(approx_norm, 1.0 / Real(p) ) ); } //#ifdef BEAGLE_DEBUG_FLOW From caac11441bd0f6b5e9d71a8fee66a216d530cab6 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 10 Jun 2024 17:05:25 -0500 Subject: [PATCH 111/421] re-use old memory partials now --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5858f0e5..c2fd1576 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -347,9 +347,14 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dPartialsWrapper.resize(kPartialsBufferCount * kCategoryCount * 2); dPartialCache.resize(kPartialsBufferCount * kCategoryCount * 2); + for (int i = 0; i < kPartialsBufferCount * 2; i++) { + for (int category = 0; category < kCategoryCount; category++) { + dPartialCache[i * kCategoryCount + category] = (Real *) gpu->CreateSubPointer(dPartialsOrigin, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * (kCategoryCount * i + category), sizeof(Real) * kPaddedStateCount * kPaddedPatternCount); + } + } for (int i = 0; i < kPartialsBufferCount * kCategoryCount * 2; i++) { - dPartialCache[i] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); +// dPartialCache[i] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); CHECK_CUSPARSE(cusparseCreateDnMat(&dPartialsWrapper[i], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[i], DataType, CUSPARSE_ORDER_COL)) } @@ -562,9 +567,9 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const inPartialsOffset += kStateCount; } for (int i = 0; i < kCategoryCount; i++) { - if (dPartialCache[getPartialIndex(tipIndex, i)] == NULL) { - dPartialCache[getPartialIndex(tipIndex, i)] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); - } +// if (dPartialCache[getPartialIndex(tipIndex, i)] == NULL) { +// dPartialCache[getPartialIndex(tipIndex, i)] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); +// } MemcpyHostToDevice(dPartialCache[getPartialIndex(tipIndex, i)], hPartialsCache, kPaddedStateCount * kPaddedPatternCount); } @@ -684,12 +689,12 @@ int BeagleGPUActionImpl::getPartials(int bufferIndex, BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int categoryIndex) { - return kPartialsBufferCount * categoryIndex + nodeIndex; + return nodeIndex * kCategoryCount + categoryIndex; } BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartialCacheIndex(int nodeIndex, int categoryIndex) { - return kPartialsBufferCount * kCategoryCount + kPartialsBufferCount * categoryIndex + nodeIndex; + return kPartialsBufferCount + getPartialIndex(nodeIndex, categoryIndex); } BEAGLE_GPU_TEMPLATE From 721e7538e7d5c7e6182485cf9db1e31695d9ba99 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 10 Jun 2024 17:38:36 -0500 Subject: [PATCH 112/421] fix cublas amax index --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 +- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 9b4837bd..8cd66f0b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -105,7 +105,7 @@ Real normPInf(Real* matrix, int nRows, int nCols, cublasHandle_t cublasH) { CUBLAS_CHECK(cublasIdamax(cublasH, nRows * nCols, matrix, 1, &index)); } cudaDeviceSynchronize(); - CHECK_CUDA(cudaMemcpy(&result, matrix + index, sizeof(Real), cudaMemcpyDeviceToHost)) + CHECK_CUDA(cudaMemcpy(&result, matrix + index - 1, sizeof(Real), cudaMemcpyDeviceToHost)) return std::abs(result); } diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c2fd1576..046f1afa 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1109,15 +1109,14 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p const Real zero = 0; const Real one = 1; for (int i = 0; i < s; i++) { - double c1 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); - //#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"Transposing:"< Date: Tue, 11 Jun 2024 10:13:55 -0500 Subject: [PATCH 113/421] frequencies --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 18 +-- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 138 +++++++++++------------ 2 files changed, 75 insertions(+), 81 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 8cd66f0b..98c90d25 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -170,7 +170,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl const Real* values, int numNonZeros); - int setPatternWeights(const Real* inPatternWeights); +// int setPatternWeights(const Real* inPatternWeights); int setStateFrequencies(int stateFrequenciesIndex, const Real* inStateFrequencies); @@ -186,12 +186,12 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int operationCount, int cumulativeScalingIndex); - int calculateRootLogLikelihoods(const int* bufferIndices, - const int* categoryWeightsIndices, - const int* stateFrequenciesIndices, - const int* cumulativeScaleIndices, - int count, - double* outSumLogLikelihood); +// int calculateRootLogLikelihoods(const int* bufferIndices, +// const int* categoryWeightsIndices, +// const int* stateFrequenciesIndices, +// const int* cumulativeScaleIndices, +// int count, +// double* outSumLogLikelihood); protected: std::vector> hInstantaneousMatrices; @@ -263,7 +263,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector dIntegrationRightBuffer; Real **dFrequenciesCache, **dWeightsCache; - std::vector dFrequencies; + std::vector dFrequenciesWrapper; std::vector dWeights; std::vector dBsCsrOffsetsCache; std::vector dBsCsrColumnsCache; @@ -336,7 +336,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // using BeagleGPUImpl::dIevc; // using BeagleGPUImpl::dEigenValues; // using BeagleGPUImpl::dWeights; -// using BeagleGPUImpl::dFrequencies; + using BeagleGPUImpl::dFrequencies; using BeagleGPUImpl::dMatrices; using BeagleGPUImpl::kIndexOffsetMat; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 046f1afa..87dba6a1 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -398,9 +398,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; dWeights = std::vector(kEigenDecompCount, nullptr); - dFrequencies = std::vector(kEigenDecompCount, nullptr); + dFrequenciesWrapper = std::vector(kEigenDecompCount, nullptr); for (int i = 0; i < kEigenDecompCount; i++) { - dFrequenciesCache[i] = NULL; + dFrequenciesCache[i] = dFrequencies[i]; dWeightsCache[i] = NULL; } @@ -446,35 +446,35 @@ char* BeagleGPUActionImpl::getInstanceName() { return (char*) "Action-OpenCL-Single"; } #endif - -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::setPatternWeights(const Real* inPatternWeights) { - -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tEntering BeagleGPUActionImpl::setPatternWeights\n"); -#endif - -//#ifdef DOUBLE_PRECISION -// const double* tmpWeights = inPatternWeights; -//#else -// Real* tmpWeights = hPatternWeightsCache; -// MEMCNV(hPatternWeightsCache, inPatternWeights, kPatternCount, Real); +// +//BEAGLE_GPU_TEMPLATE +//int BeagleGPUActionImpl::setPatternWeights(const Real* inPatternWeights) { +// +//#ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tEntering BeagleGPUActionImpl::setPatternWeights\n"); //#endif -// const Real* tmpWeights = beagleCastIfNecessary(inPatternWeights, hPatternWeightsCache, kPatternCount); - MemcpyHostToDevice(dPatternWeightsCache, inPatternWeights, kPatternCount); - - if (dPatternWeights == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, DataType)) - } else { - CHECK_CUSPARSE(cusparseDnVecSetValues(dPatternWeights, dPatternWeightsCache)) - } - -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setPatternWeights\n"); -#endif - - return BEAGLE_SUCCESS; -} +// +////#ifdef DOUBLE_PRECISION +//// const double* tmpWeights = inPatternWeights; +////#else +//// Real* tmpWeights = hPatternWeightsCache; +//// MEMCNV(hPatternWeightsCache, inPatternWeights, kPatternCount, Real); +////#endif +//// const Real* tmpWeights = beagleCastIfNecessary(inPatternWeights, hPatternWeightsCache, kPatternCount); +// MemcpyHostToDevice(dPatternWeightsCache, inPatternWeights, kPatternCount); +// +// if (dPatternWeights == NULL) { +// CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, DataType)) +// } else { +// CHECK_CUSPARSE(cusparseDnVecSetValues(dPatternWeights, dPatternWeightsCache)) +// } +// +//#ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setPatternWeights\n"); +//#endif +// +// return BEAGLE_SUCCESS; +//} BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setStateFrequencies(int stateFrequenciesIndex, @@ -483,19 +483,13 @@ int BeagleGPUActionImpl::setStateFrequencies(int stateFreque fprintf(stderr, "\tEntering BeagleGPUActionImpl::setStateFrequencies\n"); #endif - if (stateFrequenciesIndex < 0 || stateFrequenciesIndex >= kEigenDecompCount) - return BEAGLE_ERROR_OUT_OF_RANGE; - beagleMemCpy(hFrequenciesCache, inStateFrequencies, kStateCount); - if (dFrequenciesCache[stateFrequenciesIndex] == NULL) { - dFrequenciesCache[stateFrequenciesIndex] = cudaDeviceNew(kPaddedStateCount); - } MemcpyHostToDevice(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, kStateCount); - if (dFrequencies[stateFrequenciesIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequencies[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], DataType)) + if (dFrequenciesWrapper[stateFrequenciesIndex] == NULL) { + CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequenciesWrapper[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], DataType)) } else { - CHECK_CUSPARSE(cusparseDnVecSetValues(dFrequencies[stateFrequenciesIndex], dFrequenciesCache[stateFrequenciesIndex])) + CHECK_CUSPARSE(cusparseDnVecSetValues(dFrequenciesWrapper[stateFrequenciesIndex], dFrequenciesCache[stateFrequenciesIndex])) } #ifdef BEAGLE_DEBUG_FLOW @@ -620,39 +614,39 @@ int BeagleGPUActionImpl::updatePartials(const int* operation return returnCode; } -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::calculateRootLogLikelihoods(const int* bufferIndices, - const int* categoryWeightsIndices, - const int* stateFrequenciesIndices, - const int* cumulativeScaleIndices, - int count, - double* outSumLogLikelihood) { - -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tEntering BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); -#endif - - int returnCode = BEAGLE_SUCCESS; - - if (count == 1) { - - } else { - return BEAGLE_ERROR_NO_IMPLEMENTATION; - } - -#ifdef BEAGLE_DEBUG_VALUES - Real r = 0; -fprintf(stderr, "parent = \n"); -gpu->PrintfDeviceVector(dIntegrationTmp, kPatternCount, r); -#endif - - -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); -#endif - - return returnCode; -} +//BEAGLE_GPU_TEMPLATE +//int BeagleGPUActionImpl::calculateRootLogLikelihoods(const int* bufferIndices, +// const int* categoryWeightsIndices, +// const int* stateFrequenciesIndices, +// const int* cumulativeScaleIndices, +// int count, +// double* outSumLogLikelihood) { +// +//#ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tEntering BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); +//#endif +// +// int returnCode = BEAGLE_SUCCESS; +// +// if (count == 1) { +// +// } else { +// return BEAGLE_ERROR_NO_IMPLEMENTATION; +// } +// +//#ifdef BEAGLE_DEBUG_VALUES +// Real r = 0; +//fprintf(stderr, "parent = \n"); +//gpu->PrintfDeviceVector(dIntegrationTmp, kPatternCount, r); +//#endif +// +// +//#ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tLeaving BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); +//#endif +// +// return returnCode; +//} BEAGLE_GPU_TEMPLATE From 5aadd23546c46666198bdb89cf849b480dac2d85 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 11 Jun 2024 10:19:56 -0500 Subject: [PATCH 114/421] category weights --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 4 ++-- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 18 ++++++------------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 98c90d25..68f2bc41 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -264,7 +264,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl Real **dFrequenciesCache, **dWeightsCache; std::vector dFrequenciesWrapper; - std::vector dWeights; + std::vector dWeightsWrapper; std::vector dBsCsrOffsetsCache; std::vector dBsCsrColumnsCache; std::vector dBsCsrValuesCache; @@ -335,7 +335,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // using BeagleGPUImpl::dEvec; // using BeagleGPUImpl::dIevc; // using BeagleGPUImpl::dEigenValues; -// using BeagleGPUImpl::dWeights; + using BeagleGPUImpl::dWeights; using BeagleGPUImpl::dFrequencies; using BeagleGPUImpl::dMatrices; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 87dba6a1..86ceb6b9 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -397,11 +397,11 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; - dWeights = std::vector(kEigenDecompCount, nullptr); + dWeightsWrapper = std::vector(kEigenDecompCount, nullptr); dFrequenciesWrapper = std::vector(kEigenDecompCount, nullptr); for (int i = 0; i < kEigenDecompCount; i++) { - dFrequenciesCache[i] = dFrequencies[i]; - dWeightsCache[i] = NULL; + dFrequenciesCache[i] = (Real*) dFrequencies[i]; + dWeightsCache[i] = (Real*) dWeights[i]; } hEigenMaps.resize(kPartialsBufferCount); @@ -506,20 +506,14 @@ int BeagleGPUActionImpl::setCategoryWeights(int categoryWeig fprintf(stderr, "\tEntering BeagleGPUActionImpl::setCategoryWeights\n"); #endif - if (categoryWeightsIndex < 0 || categoryWeightsIndex >= kEigenDecompCount) - return BEAGLE_ERROR_OUT_OF_RANGE; - beagleMemCpy(hWeightsCache, inCategoryWeights, kCategoryCount); - if (dWeightsCache[categoryWeightsIndex] == NULL) { - dWeightsCache[categoryWeightsIndex] = cudaDeviceNew(kCategoryCount); - } MemcpyHostToDevice(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount); - if (dWeights[categoryWeightsIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dWeights[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], DataType)) + if (dWeightsWrapper[categoryWeightsIndex] == NULL) { + CHECK_CUSPARSE(cusparseCreateDnVec(&dWeightsWrapper[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], DataType)) } else { - CHECK_CUSPARSE(cusparseDnVecSetValues(dWeights[categoryWeightsIndex], dWeightsCache[categoryWeightsIndex])) + CHECK_CUSPARSE(cusparseDnVecSetValues(dWeightsWrapper[categoryWeightsIndex], dWeightsCache[categoryWeightsIndex])) } #ifdef BEAGLE_DEBUG_FLOW From 92006c7765a0f7fe6e3e1b513c3ab7989e2bb232 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 11 Jun 2024 10:25:39 -0500 Subject: [PATCH 115/421] reuse setTipPartials getPartials --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 8 -- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 107 ----------------------- 2 files changed, 115 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 68f2bc41..09c5bc7f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -162,26 +162,18 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setTipStates(int tipIndex, const int* inStates); - int setTipPartials(int tipIndex, const Real* inPartials); - int setSparseMatrix(int matrixIndex, const int* rowIndices, const int* colIndices, const Real* values, int numNonZeros); -// int setPatternWeights(const Real* inPatternWeights); - int setStateFrequencies(int stateFrequenciesIndex, const Real* inStateFrequencies); int setCategoryWeights(int categoryWeightsIndex, const Real* inCategoryWeights); - int getPartials(int bufferIndex, - int scaleIndex, - Real* outPartials); - int updatePartials(const int* operations, int operationCount, int cumulativeScalingIndex); diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 86ceb6b9..d0b67506 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -446,35 +446,6 @@ char* BeagleGPUActionImpl::getInstanceName() { return (char*) "Action-OpenCL-Single"; } #endif -// -//BEAGLE_GPU_TEMPLATE -//int BeagleGPUActionImpl::setPatternWeights(const Real* inPatternWeights) { -// -//#ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tEntering BeagleGPUActionImpl::setPatternWeights\n"); -//#endif -// -////#ifdef DOUBLE_PRECISION -//// const double* tmpWeights = inPatternWeights; -////#else -//// Real* tmpWeights = hPatternWeightsCache; -//// MEMCNV(hPatternWeightsCache, inPatternWeights, kPatternCount, Real); -////#endif -//// const Real* tmpWeights = beagleCastIfNecessary(inPatternWeights, hPatternWeightsCache, kPatternCount); -// MemcpyHostToDevice(dPatternWeightsCache, inPatternWeights, kPatternCount); -// -// if (dPatternWeights == NULL) { -// CHECK_CUSPARSE(cusparseCreateDnVec(&dPatternWeights, kPatternCount, dPatternWeightsCache, DataType)) -// } else { -// CHECK_CUSPARSE(cusparseDnVecSetValues(dPatternWeights, dPatternWeightsCache)) -// } -// -//#ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setPatternWeights\n"); -//#endif -// -// return BEAGLE_SUCCESS; -//} BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setStateFrequencies(int stateFrequenciesIndex, @@ -538,53 +509,6 @@ int BeagleGPUActionImpl::setTipStates(int tipIndex, const in std::abort(); } -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::setTipPartials(int tipIndex, const Real* inPartials) { -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tEntering BeagleGPUActionImpl::setTipPartials\n"); -#endif - - if (tipIndex < 0 || tipIndex >= kTipCount) - return BEAGLE_ERROR_OUT_OF_RANGE; - - const Real* inPartialsOffset = inPartials; - Real* tmpRealPartialsOffset = hPartialsCache; - for (int i = 0; i < kPatternCount; i++) { - beagleMemCpy(tmpRealPartialsOffset, inPartialsOffset, kStateCount); - tmpRealPartialsOffset += kPaddedStateCount; - inPartialsOffset += kStateCount; - } - for (int i = 0; i < kCategoryCount; i++) { -// if (dPartialCache[getPartialIndex(tipIndex, i)] == NULL) { -// dPartialCache[getPartialIndex(tipIndex, i)] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); -// } - MemcpyHostToDevice(dPartialCache[getPartialIndex(tipIndex, i)], hPartialsCache, kPaddedStateCount * kPaddedPatternCount); - } - - - -// int partialsLength = kPaddedPatternCount * kPaddedStateCount; -// for (int i = 1; i < kCategoryCount; i++) { -// memcpy(hPartialsCache + i * partialsLength, hPartialsCache, partialsLength * sizeof(Real)); -// } - - if (tipIndex < kTipCount) { - for (int i = 0; i < kCategoryCount; i++) { - const int partialIndex = getPartialIndex(tipIndex, i); - if (dPartialsWrapper[partialIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnMat(&dPartialsWrapper[partialIndex], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[partialIndex], - DataType, CUSPARSE_ORDER_COL)) - } else { - CHECK_CUSPARSE(cusparseDnMatSetValues(dPartialsWrapper[partialIndex], dPartialCache[partialIndex])) - } - } - } -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setTipPartials\n"); -#endif - - return BEAGLE_SUCCESS; -} BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::updatePartials(const int* operations, @@ -643,37 +567,6 @@ int BeagleGPUActionImpl::updatePartials(const int* operation //} -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::getPartials(int bufferIndex, - int scaleIndex, - Real* outPartials) { -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tEntering BeagleGPUActionImpl::getPartials\n"); -#endif - - for (int i = 0; i < kCategoryCount; i++) { - CHECK_CUDA(cudaMemcpy(hPartialsCache + i * kPaddedStateCount * kPaddedPatternCount, dPartialCache[getPartialIndex(bufferIndex, i)], - sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToHost)) - } - - Real *outPartialsOffset = outPartials; - Real *tmpRealPartialsOffset = hPartialsCache; - - for (int c = 0; c < kCategoryCount; c++) { - for (int i = 0; i < kPatternCount; i++) { - beagleMemCpy(outPartialsOffset, tmpRealPartialsOffset, kStateCount); - tmpRealPartialsOffset += kPaddedStateCount; - outPartialsOffset += kStateCount; - } - tmpRealPartialsOffset += kPaddedStateCount * (kPaddedPatternCount - kPatternCount); - } - -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUActionImpl::getPartials\n"); -#endif - - return BEAGLE_SUCCESS; -} BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int categoryIndex) { From 6d542fdc9be954b43e839023f7ae8981f59b71f4 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 11 Jun 2024 10:30:12 -0500 Subject: [PATCH 116/421] reuse more functions --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 14 +-- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 106 +++++++++++------------ 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 09c5bc7f..91917735 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -168,11 +168,11 @@ class BeagleGPUActionImpl : public BeagleGPUImpl const Real* values, int numNonZeros); - int setStateFrequencies(int stateFrequenciesIndex, - const Real* inStateFrequencies); +// int setStateFrequencies(int stateFrequenciesIndex, +// const Real* inStateFrequencies); - int setCategoryWeights(int categoryWeightsIndex, - const Real* inCategoryWeights); +// int setCategoryWeights(int categoryWeightsIndex, +// const Real* inCategoryWeights); int updatePartials(const int* operations, int operationCount, @@ -254,9 +254,9 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector integrationRightStoredBufferSize; std::vector dIntegrationRightBuffer; - Real **dFrequenciesCache, **dWeightsCache; - std::vector dFrequenciesWrapper; - std::vector dWeightsWrapper; +// Real **dFrequenciesCache, **dWeightsCache; +// std::vector dFrequenciesWrapper; +// std::vector dWeightsWrapper; std::vector dBsCsrOffsetsCache; std::vector dBsCsrColumnsCache; std::vector dBsCsrValuesCache; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index d0b67506..1bf6c7c0 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -395,14 +395,14 @@ int BeagleGPUActionImpl::createInstance(int tipCount, - dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); - dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; - dWeightsWrapper = std::vector(kEigenDecompCount, nullptr); - dFrequenciesWrapper = std::vector(kEigenDecompCount, nullptr); - for (int i = 0; i < kEigenDecompCount; i++) { - dFrequenciesCache[i] = (Real*) dFrequencies[i]; - dWeightsCache[i] = (Real*) dWeights[i]; - } +// dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); +// dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; +// dWeightsWrapper = std::vector(kEigenDecompCount, nullptr); +//// dFrequenciesWrapper = std::vector(kEigenDecompCount, nullptr); +// for (int i = 0; i < kEigenDecompCount; i++) { +//// dFrequenciesCache[i] = (Real*) dFrequencies[i]; +// dWeightsCache[i] = (Real*) dWeights[i]; +// } hEigenMaps.resize(kPartialsBufferCount); hEdgeMultipliers.resize(kPartialsBufferCount * kCategoryCount); @@ -447,52 +447,52 @@ char* BeagleGPUActionImpl::getInstanceName() { } #endif -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::setStateFrequencies(int stateFrequenciesIndex, - const Real* inStateFrequencies) { -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tEntering BeagleGPUActionImpl::setStateFrequencies\n"); -#endif - - beagleMemCpy(hFrequenciesCache, inStateFrequencies, kStateCount); - - MemcpyHostToDevice(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, kStateCount); - if (dFrequenciesWrapper[stateFrequenciesIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequenciesWrapper[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], DataType)) - } else { - CHECK_CUSPARSE(cusparseDnVecSetValues(dFrequenciesWrapper[stateFrequenciesIndex], dFrequenciesCache[stateFrequenciesIndex])) - } - -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setStateFrequencies\n"); -#endif - - return BEAGLE_SUCCESS; -} - -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::setCategoryWeights(int categoryWeightsIndex, - const Real* inCategoryWeights) { -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tEntering BeagleGPUActionImpl::setCategoryWeights\n"); -#endif - - beagleMemCpy(hWeightsCache, inCategoryWeights, kCategoryCount); - - MemcpyHostToDevice(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount); - - if (dWeightsWrapper[categoryWeightsIndex] == NULL) { - CHECK_CUSPARSE(cusparseCreateDnVec(&dWeightsWrapper[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], DataType)) - } else { - CHECK_CUSPARSE(cusparseDnVecSetValues(dWeightsWrapper[categoryWeightsIndex], dWeightsCache[categoryWeightsIndex])) - } - -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setCategoryWeights\n"); -#endif +//BEAGLE_GPU_TEMPLATE +//int BeagleGPUActionImpl::setStateFrequencies(int stateFrequenciesIndex, +// const Real* inStateFrequencies) { +//#ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tEntering BeagleGPUActionImpl::setStateFrequencies\n"); +//#endif +// +// beagleMemCpy(hFrequenciesCache, inStateFrequencies, kStateCount); +// +// MemcpyHostToDevice(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, kStateCount); +// if (dFrequenciesWrapper[stateFrequenciesIndex] == NULL) { +// CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequenciesWrapper[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], DataType)) +// } else { +// CHECK_CUSPARSE(cusparseDnVecSetValues(dFrequenciesWrapper[stateFrequenciesIndex], dFrequenciesCache[stateFrequenciesIndex])) +// } +// +//#ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setStateFrequencies\n"); +//#endif +// +// return BEAGLE_SUCCESS; +//} - return BEAGLE_SUCCESS; -} +//BEAGLE_GPU_TEMPLATE +//int BeagleGPUActionImpl::setCategoryWeights(int categoryWeightsIndex, +// const Real* inCategoryWeights) { +//#ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tEntering BeagleGPUActionImpl::setCategoryWeights\n"); +//#endif +// +// beagleMemCpy(hWeightsCache, inCategoryWeights, kCategoryCount); +// +// MemcpyHostToDevice(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount); +// +// if (dWeightsWrapper[categoryWeightsIndex] == NULL) { +// CHECK_CUSPARSE(cusparseCreateDnVec(&dWeightsWrapper[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], DataType)) +// } else { +// CHECK_CUSPARSE(cusparseDnVecSetValues(dWeightsWrapper[categoryWeightsIndex], dWeightsCache[categoryWeightsIndex])) +// } +// +//#ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setCategoryWeights\n"); +//#endif +// +// return BEAGLE_SUCCESS; +//} BEAGLE_GPU_TEMPLATE From 19627a5f617a3193650766381b7962eb4c376bcf Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 11 Jun 2024 10:33:32 -0500 Subject: [PATCH 117/421] more cleaning --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 23 +-------- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 65 ------------------------ 2 files changed, 1 insertion(+), 87 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 91917735..8f16e813 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -168,22 +168,10 @@ class BeagleGPUActionImpl : public BeagleGPUImpl const Real* values, int numNonZeros); -// int setStateFrequencies(int stateFrequenciesIndex, -// const Real* inStateFrequencies); - -// int setCategoryWeights(int categoryWeightsIndex, -// const Real* inCategoryWeights); - int updatePartials(const int* operations, int operationCount, int cumulativeScalingIndex); -// int calculateRootLogLikelihoods(const int* bufferIndices, -// const int* categoryWeightsIndices, -// const int* stateFrequenciesIndices, -// const int* cumulativeScaleIndices, -// int count, -// double* outSumLogLikelihood); protected: std::vector> hInstantaneousMatrices; @@ -254,16 +242,11 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector integrationRightStoredBufferSize; std::vector dIntegrationRightBuffer; -// Real **dFrequenciesCache, **dWeightsCache; -// std::vector dFrequenciesWrapper; -// std::vector dWeightsWrapper; std::vector dBsCsrOffsetsCache; std::vector dBsCsrColumnsCache; std::vector dBsCsrValuesCache; std::vector dACscValuesCache; std::vector currentCacheNNZs; - Real *dPatternWeightsCache; - cusparseDnVecDescr_t dPatternWeights; cublasHandle_t cublasHandle; cusparseHandle_t cusparseHandle; @@ -324,9 +307,6 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::hStatesCache; using BeagleGPUImpl::hLogLikelihoodsCache; using BeagleGPUImpl::hMatrixCache; -// using BeagleGPUImpl::dEvec; -// using BeagleGPUImpl::dIevc; -// using BeagleGPUImpl::dEigenValues; using BeagleGPUImpl::dWeights; using BeagleGPUImpl::dFrequencies; @@ -342,7 +322,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::kWeightsOffset; using BeagleGPUImpl::kFrequenciesOffset; using BeagleGPUImpl::dIntegrationTmp; -// using BeagleGPUImpl::dPatternWeights; + using BeagleGPUImpl::dPatternWeights; using BeagleGPUImpl::dSumLogLikelihood; using BeagleGPUImpl::dPartialsTmp; @@ -387,7 +367,6 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::dAccumulatedScalingFactors; using BeagleGPUImpl::kUsingAutoTranspose; - void allocateMultiGridBuffers(); int upPartials(bool byPartition, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 1bf6c7c0..aed3e89c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -393,28 +393,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, CHECK_CUDA(cudaMalloc(&dIntegrationRightBuffer[category], integrationRightBufferSize[category])) } - - -// dFrequenciesCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount); -// dWeightsCache = (Real**) gpu->MallocHost(sizeof(Real*) * kEigenDecompCount);; -// dWeightsWrapper = std::vector(kEigenDecompCount, nullptr); -//// dFrequenciesWrapper = std::vector(kEigenDecompCount, nullptr); -// for (int i = 0; i < kEigenDecompCount; i++) { -//// dFrequenciesCache[i] = (Real*) dFrequencies[i]; -// dWeightsCache[i] = (Real*) dWeights[i]; -// } - hEigenMaps.resize(kPartialsBufferCount); hEdgeMultipliers.resize(kPartialsBufferCount * kCategoryCount); - hCategoryRates = (double**) calloc(sizeof(double*),kEigenDecompCount); // Keep in double-precision - hCategoryRates[0] = (double*) gpu->MallocHost(sizeof(double) * kCategoryCount); - checkHostMemory(hCategoryRates[0]); - -// dMatrices = (GPUPtr*) malloc(sizeof(GPUPtr) * kMatrixCount); - dPatternWeightsCache = cudaDeviceNew(kPatternCount); - dPatternWeights = NULL; - return BEAGLE_SUCCESS; } @@ -447,52 +428,6 @@ char* BeagleGPUActionImpl::getInstanceName() { } #endif -//BEAGLE_GPU_TEMPLATE -//int BeagleGPUActionImpl::setStateFrequencies(int stateFrequenciesIndex, -// const Real* inStateFrequencies) { -//#ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tEntering BeagleGPUActionImpl::setStateFrequencies\n"); -//#endif -// -// beagleMemCpy(hFrequenciesCache, inStateFrequencies, kStateCount); -// -// MemcpyHostToDevice(dFrequenciesCache[stateFrequenciesIndex], hFrequenciesCache, kStateCount); -// if (dFrequenciesWrapper[stateFrequenciesIndex] == NULL) { -// CHECK_CUSPARSE(cusparseCreateDnVec(&dFrequenciesWrapper[stateFrequenciesIndex], kPaddedStateCount, dFrequenciesCache[stateFrequenciesIndex], DataType)) -// } else { -// CHECK_CUSPARSE(cusparseDnVecSetValues(dFrequenciesWrapper[stateFrequenciesIndex], dFrequenciesCache[stateFrequenciesIndex])) -// } -// -//#ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setStateFrequencies\n"); -//#endif -// -// return BEAGLE_SUCCESS; -//} - -//BEAGLE_GPU_TEMPLATE -//int BeagleGPUActionImpl::setCategoryWeights(int categoryWeightsIndex, -// const Real* inCategoryWeights) { -//#ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tEntering BeagleGPUActionImpl::setCategoryWeights\n"); -//#endif -// -// beagleMemCpy(hWeightsCache, inCategoryWeights, kCategoryCount); -// -// MemcpyHostToDevice(dWeightsCache[categoryWeightsIndex], hWeightsCache, kCategoryCount); -// -// if (dWeightsWrapper[categoryWeightsIndex] == NULL) { -// CHECK_CUSPARSE(cusparseCreateDnVec(&dWeightsWrapper[categoryWeightsIndex], kCategoryCount, dWeightsCache[categoryWeightsIndex], DataType)) -// } else { -// CHECK_CUSPARSE(cusparseDnVecSetValues(dWeightsWrapper[categoryWeightsIndex], dWeightsCache[categoryWeightsIndex])) -// } -// -//#ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setCategoryWeights\n"); -//#endif -// -// return BEAGLE_SUCCESS; -//} BEAGLE_GPU_TEMPLATE From 850a3b872e60a4f26039c7469bfc4f8793c1aa19 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 11 Jun 2024 12:02:52 -0500 Subject: [PATCH 118/421] setTipPartials mapping correct pointers --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 3 +++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 23 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 8f16e813..6c9e60a0 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -162,6 +162,9 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setTipStates(int tipIndex, const int* inStates); + int setTipPartials(int tipIndex, + const Real* inPartials); + int setSparseMatrix(int matrixIndex, const int* rowIndices, const int* colIndices, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index aed3e89c..abb76d64 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -444,6 +444,25 @@ int BeagleGPUActionImpl::setTipStates(int tipIndex, const in std::abort(); } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setTipPartials(int tipIndex, const Real* inPartials) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::setTipPartials\n"); +#endif + + BeagleGPUImpl::setTipPartials(tipIndex, inPartials); + for (int category = 0; category < kCategoryCount; category++) { + dPartialCache[getPartialIndex(tipIndex, category)] = (Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category; + } + + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setTipPartials\n"); +#endif + + return BEAGLE_SUCCESS; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::updatePartials(const int* operations, @@ -891,8 +910,6 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"Before destP = partials operation, destPCache:\n"<::simpleAction2(int destPIndex, int p // PrintfDeviceVector(dPartialCache[partialsIndex], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); //#endif - double c1 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + Real c1 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); for (int j = 1; j < m + 1; j++) { From 88ae186f1b34454b96a1186121a2afd8732164f1 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 11 Jun 2024 14:49:19 -0400 Subject: [PATCH 119/421] Add scaling code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 50 +++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index abb76d64..df990b5d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -538,7 +538,12 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, int operationCount, int cumulativeScalingIndex) { - //TODO: implement/re-use scaling code + GPUPtr cumulativeScalingBuffer = 0; + if (cumulativeScalingIndex != BEAGLE_OP_NONE) + cumulativeScalingBuffer = dScalingFactors[cumulativeScalingIndex]; + + int streamIndex = -1; + int waitIndex = -1; for (int op = 0; op < operationCount; op++) { const int numOps = BEAGLE_OP_COUNT; @@ -551,9 +556,52 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, const int secondChildPartialIndex = operations[op * numOps + 5]; const int secondChildSubstitutionMatrixIndex = operations[op * numOps + 6]; + int rescale = BEAGLE_OP_NONE; + GPUPtr scalingFactors = (GPUPtr)NULL; + + if (kFlags & BEAGLE_FLAG_SCALING_AUTO) { + int sIndex = destinationPartialIndex - kTipCount; + + rescale = 2; + scalingFactors = dScalingFactors[sIndex]; + } else if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { + rescale = 1; + scalingFactors = dScalingFactors[destinationPartialIndex - kTipCount]; + } else if ((kFlags & BEAGLE_FLAG_SCALING_MANUAL) && writeScalingIndex >= 0) { + rescale = 1; + scalingFactors = dScalingFactors[writeScalingIndex]; + } else if ((kFlags & BEAGLE_FLAG_SCALING_MANUAL) && readScalingIndex >= 0) { + rescale = 0; + scalingFactors = dScalingFactors[readScalingIndex]; + } + calcPartialsPartials(destinationPartialIndex, firstChildPartialIndex, firstChildSubstitutionMatrixIndex, secondChildPartialIndex, secondChildSubstitutionMatrixIndex); + + if (rescale == 1) + { + GPUPtr partials3 = dPartials[destinationPartialIndex]; + + kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, + kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); + } + + if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { + int parScalingIndex = destinationPartialIndex - kTipCount; + int child1ScalingIndex = firstChildPartialIndex - kTipCount; + int child2ScalingIndex = secondChildPartialIndex - kTipCount; + if (child1ScalingIndex >= 0 && child2ScalingIndex >= 0) { + int scalingIndices[2] = {child1ScalingIndex, child2ScalingIndex}; + BeagleGPUImpl::accumulateScaleFactors(scalingIndices, 2, parScalingIndex); + } else if (child1ScalingIndex >= 0) { + int scalingIndices[1] = {child1ScalingIndex}; + BeagleGPUImpl::accumulateScaleFactors(scalingIndices, 1, parScalingIndex); + } else if (child2ScalingIndex >= 0) { + int scalingIndices[1] = {child2ScalingIndex}; + BeagleGPUImpl::accumulateScaleFactors(scalingIndices, 1, parScalingIndex); + } + } } return BEAGLE_SUCCESS; } From f1afce24e359310d60d3afb3fa8aaaa8de0f933b Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 11 Jun 2024 14:49:58 -0400 Subject: [PATCH 120/421] Add some debug prints. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index df990b5d..b2ed1ee4 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -538,6 +538,11 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, int operationCount, int cumulativeScalingIndex) { + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::upPartials\n"); +#endif + GPUPtr cumulativeScalingBuffer = 0; if (cumulativeScalingIndex != BEAGLE_OP_NONE) cumulativeScalingBuffer = dScalingFactors[cumulativeScalingIndex]; @@ -603,6 +608,15 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, } } } + +#ifdef BEAGLE_DEBUG_SYNCH + gpu->SynchronizeHost(); +#endif + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::upPartials\n"); +#endif + return BEAGLE_SUCCESS; } From d90e2574bcbe89386a0eb40d87bfe8ce9bf8bcb5 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 5 Jun 2024 17:38:18 -0400 Subject: [PATCH 121/421] Just say OpenMP is off. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e862c31..f992a55d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -98,7 +98,7 @@ jobs: sudo apt-get install -y clang-${{ matrix.version }} echo "CC=ccache clang-${{ matrix.version }}" >> $GITHUB_ENV echo "CXX=ccache clang++-${{ matrix.version }}" >> $GITHUB_ENV - echo "CMAKE_ARGS='-DBUILD_OPENCL=OFF -DBUILD_OPENMP=OFF'" >> $GITHUB_ENV + echo "CMAKE_ARGS=-DBUILD_OPENMP=OFF" >> $GITHUB_ENV fi - name: Install (Linux -> Windows [cross]) From dc062198387d59ad73217fae2db51d71125050c6 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 5 Jun 2024 17:43:16 -0400 Subject: [PATCH 122/421] Just don't install opencl --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f992a55d..2a902c68 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,7 +83,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y cmake ninja-build ccache pkg-config - sudo apt-get install -y openjdk-17-jdk libeigen3-dev ocl-icd-opencl-dev + sudo apt-get install -y openjdk-17-jdk libeigen3-dev if [ "${{ matrix.compiler }}" = "gcc" ]; then # For newer GCCs, maybe. @@ -93,7 +93,6 @@ jobs: sudo apt-get install -y g++-${{ matrix.version }} echo "CC=ccache gcc-${{ matrix.version }}" >> $GITHUB_ENV echo "CXX=ccache g++-${{ matrix.version }}" >> $GITHUB_ENV - echo "CMAKE_ARGS=-DBUILD_OPENCL=OFF" >> $GITHUB_ENV else sudo apt-get install -y clang-${{ matrix.version }} echo "CC=ccache clang-${{ matrix.version }}" >> $GITHUB_ENV From f637a4eb6f09dee6d0552ac5f06558a6845e3230 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 11 Jun 2024 16:46:47 -0500 Subject: [PATCH 123/421] more cleaning on header --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 98 ++------------------------ 1 file changed, 4 insertions(+), 94 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 6c9e60a0..fd13c20c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -258,117 +258,23 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector hEdgeMultipliers; - - using BeagleGPUImpl::kInitialized; using BeagleGPUImpl::kTipCount; using BeagleGPUImpl::kPartialsBufferCount; - using BeagleGPUImpl::kCompactBufferCount; using BeagleGPUImpl::kStateCount; - using BeagleGPUImpl::kPatternCount; using BeagleGPUImpl::kEigenDecompCount; - using BeagleGPUImpl::kMatrixCount; using BeagleGPUImpl::kCategoryCount; - using BeagleGPUImpl::kScaleBufferCount; - using BeagleGPUImpl::kExtraMatrixCount; - using BeagleGPUImpl::kPartitionCount; - using BeagleGPUImpl::kMaxPartitionCount; - using BeagleGPUImpl::kPartitionsInitialised; - using BeagleGPUImpl::kPatternsReordered; - - using BeagleGPUImpl::resourceNumber; - using BeagleGPUImpl::kTipPartialsBufferCount; - using BeagleGPUImpl::kBufferCount; - using BeagleGPUImpl::kInternalPartialsBufferCount; using BeagleGPUImpl::kPaddedStateCount; - using BeagleGPUImpl::gpu; - using BeagleGPUImpl::kDeviceType; - using BeagleGPUImpl::kDeviceCode; - using BeagleGPUImpl::kPaddedPatternCount; - using BeagleGPUImpl::kResultPaddedPatterns; - using BeagleGPUImpl::kFlags; - using BeagleGPUImpl::kScaleBufferSize; - - using BeagleGPUImpl::kSumSitesBlockSize; - using BeagleGPUImpl::kSumSitesBlockCount; - - using BeagleGPUImpl::kPartialsSize; using BeagleGPUImpl::kMatrixSize; - - using BeagleGPUImpl::kEigenValuesSize; - using BeagleGPUImpl::kLastCompactBufferIndex; - using BeagleGPUImpl::kLastTipPartialsBufferIndex; using BeagleGPUImpl::kernels; - - using BeagleGPUImpl::hWeightsCache; - using BeagleGPUImpl::hFrequenciesCache; - using BeagleGPUImpl::hPartialsCache; - using BeagleGPUImpl::hStatesCache; - using BeagleGPUImpl::hLogLikelihoodsCache; - using BeagleGPUImpl::hMatrixCache; - using BeagleGPUImpl::dWeights; - using BeagleGPUImpl::dFrequencies; - - using BeagleGPUImpl::dMatrices; - using BeagleGPUImpl::kIndexOffsetMat; - using BeagleGPUImpl::dMatricesOrigin; using BeagleGPUImpl::dScalingFactors; - using BeagleGPUImpl::dScalingFactorsMaster; - using BeagleGPUImpl::hRescalingTrigger; - using BeagleGPUImpl::dRescalingTrigger; - using BeagleGPUImpl::kEvecOffset; - using BeagleGPUImpl::kEvalOffset; - using BeagleGPUImpl::kWeightsOffset; - using BeagleGPUImpl::kFrequenciesOffset; - using BeagleGPUImpl::dIntegrationTmp; - using BeagleGPUImpl::dPatternWeights; - using BeagleGPUImpl::dSumLogLikelihood; - using BeagleGPUImpl::dPartialsTmp; - - using BeagleGPUImpl::kDerivBuffersInitialised; - using BeagleGPUImpl::kMultipleDerivativesLength; - using BeagleGPUImpl::dPartials; using BeagleGPUImpl::dPartialsOrigin; - using BeagleGPUImpl::hPartialsOffsets; - using BeagleGPUImpl::kIndexOffsetPat; - - using BeagleGPUImpl::dStatesOrigin; - using BeagleGPUImpl::dStates; - using BeagleGPUImpl::hStatesOffsets; - using BeagleGPUImpl::kIndexOffsetStates; - using BeagleGPUImpl::dCompactBuffers; - using BeagleGPUImpl::dTipPartialsBuffers; - using BeagleGPUImpl::hStreamIndices; - - using BeagleGPUImpl::dBranchLengths; - using BeagleGPUImpl::dDistanceQueue; - using BeagleGPUImpl::hDistanceQueue; - using BeagleGPUImpl::dPtrQueue; - using BeagleGPUImpl::hPtrQueue; - - using BeagleGPUImpl::dDerivativeQueue; - using BeagleGPUImpl::hDerivativeQueue; - - using BeagleGPUImpl::kSitesPerIntegrateBlock; - using BeagleGPUImpl::kSitesPerBlock; - using BeagleGPUImpl::kNumPatternBlocks; - using BeagleGPUImpl::kPaddedPartitionBlocks; - using BeagleGPUImpl::kMaxPaddedPartitionBlocks; - using BeagleGPUImpl::kPaddedPartitionIntegrateBlocks; - using BeagleGPUImpl::kMaxPaddedPartitionIntegrateBlocks; - using BeagleGPUImpl::kUsingMultiGrid; - using BeagleGPUImpl::hPartitionOffsets; using BeagleGPUImpl::hCategoryRates; - using BeagleGPUImpl::hPatternWeightsCache; - using BeagleGPUImpl::dMaxScalingFactors; - using BeagleGPUImpl::dIndexMaxScalingFactors; - using BeagleGPUImpl::dAccumulatedScalingFactors; - using BeagleGPUImpl::kUsingAutoTranspose; @@ -405,6 +311,10 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool left, bool transpose); + int simpleAction3(int destPIndex1, int partialsIndex1, int edgeIndex1, int matrixIndex1, + int destPIndex2, int partialsIndex2, int edgeIndex2, int matrixIndex2, + bool transpose); + int cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose); std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, From aaf3a438742a37032464aee69240226a5e27100f Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 11 Jun 2024 20:13:39 -0500 Subject: [PATCH 124/421] intermediate simpleAction3 work, need to fix index assignment --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 7 + libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 204 ++++++++++++++++++++++- 2 files changed, 204 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index fd13c20c..20be9610 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -257,6 +257,13 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector hEigenMaps; std::vector hEdgeMultipliers; + std::vector> msCache; + std::vector etaCache; + std::vector c1Cache; + std::vector c2Cache; + std::vector alphaCache; + std::vector integrationMultipliers; + using BeagleGPUImpl::kTipCount; using BeagleGPUImpl::kPartialsBufferCount; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index b2ed1ee4..28f63e47 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -311,8 +311,6 @@ int BeagleGPUActionImpl::createInstance(int tipCount, int hMatrixCacheSize = kMatrixSize * kCategoryCount * BEAGLE_CACHED_MATRICES_COUNT; //TODO: use Eigen csr representation? - hLogLikelihoodsCache = (Real*) gpu->MallocHost(kPatternCount * sizeof(Real)); - hMatrixCache = (Real*) gpu->CallocHost(hMatrixCacheSize, sizeof(Real)); hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); hIdentity.setIdentity(); hInstantaneousMatrices.resize(kEigenDecompCount); @@ -361,13 +359,17 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dFLeft = std::vector(kCategoryCount, nullptr); dFLeftCache = std::vector(kCategoryCount, nullptr); + dFLeftCache[0] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); dFRight = std::vector(kCategoryCount, nullptr); dFRightCache = std::vector(kCategoryCount, nullptr); + dFRightCache[0] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); dIntegrationTmpLeft = std::vector(kCategoryCount, nullptr); dIntegrationTmpLeftCache = std::vector(kCategoryCount, nullptr); + dIntegrationTmpLeftCache[0] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); dIntegrationTmpRight = std::vector(kCategoryCount, nullptr); dIntegrationTmpRightCache = std::vector(kCategoryCount, nullptr); + dIntegrationTmpRightCache[0] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); integrationLeftBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); integrationLeftStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationLeftBuffer = std::vector(kCategoryCount, nullptr); @@ -375,11 +377,14 @@ int BeagleGPUActionImpl::createInstance(int tipCount, integrationRightStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); - for (int category = 0; category < categoryCount; category++) { - dFLeftCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); - dFRightCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); - dIntegrationTmpLeftCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); - dIntegrationTmpRightCache[category] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); + for (int category = 1; category < kCategoryCount; category++) { + dFLeftCache[category] = dFLeftCache[0] + kPaddedStateCount * kPaddedPatternCount * sizeof(Real); + dFRightCache[category] = dFRightCache[0] + kPaddedStateCount * kPaddedPatternCount * sizeof(Real); + dIntegrationTmpLeftCache[category] = dIntegrationTmpLeftCache[0] + kPaddedStateCount * kPaddedPatternCount * sizeof(Real); + dIntegrationTmpRightCache[category] = dIntegrationTmpRightCache[0] + kPaddedStateCount * kPaddedPatternCount * sizeof(Real); + } + + for (int category = 0; category < kCategoryCount; category++) { CHECK_CUSPARSE(cusparseCreateDnMat(&dFLeft[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dFLeftCache[category], DataType, CUSPARSE_ORDER_COL)) @@ -396,6 +401,13 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hEigenMaps.resize(kPartialsBufferCount); hEdgeMultipliers.resize(kPartialsBufferCount * kCategoryCount); + msCache = std::vector>(2 * kCategoryCount, tuple{0, 0}); + etaCache = std::vector(2 * kCategoryCount, 0); + c1Cache.resize(2 * kCategoryCount); + c2Cache.resize(2 * kCategoryCount); + alphaCache.resize(2 * kCategoryCount); + integrationMultipliers = std::vector(kCategoryCount * 2, 1); + return BEAGLE_SUCCESS; } @@ -1091,6 +1103,184 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::simpleAction3(int destPIndex1, int partialsIndex1, int edgeIndex1, int matrixIndex1, + int destPIndex2, int partialsIndex2, int edgeIndex2, int matrixIndex2, + bool transpose) { + const double tol = pow(2.0, -53.0); + const double t = 1.0; + const int nCol = kPaddedStateCount * kPaddedPatternCount; + + int m_max = 0, s_max = 0; + for (int category = 0; category < kCategoryCount; category++) { + const int edgeMultiplierIndex1 = edgeIndex1 * kCategoryCount + category; + const double edgeMultiplier1 = hEdgeMultipliers[edgeMultiplierIndex1]; + msCache[category] = getStatistics2(t, nCol, edgeMultiplier1, hEigenMaps[edgeIndex1]); + auto [m1, s1] = msCache[category]; + etaCache[category] = exp(t * hMuBs[hEigenMaps[edgeIndex1]] * edgeMultiplier1 / (Real) s1); + if (m_max < m1) m_max = m1; + if (s_max < s1) s_max = s1; + + const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; + const double edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; + msCache[kCategoryCount + category] = getStatistics2(t, nCol, edgeMultiplier2, hEigenMaps[edgeIndex2]); + auto [m2, s2] = msCache[kCategoryCount + category]; + etaCache[kCategoryCount + category] = exp(t * hMuBs[hEigenMaps[edgeIndex2]] * edgeMultiplier2 / (Real) s2); + if (m_max < m2) m_max = m2; + if (s_max < s2) s_max = s2; + } + + + + cusparseSpMatDescr_t ALeft = dAs[matrixIndex1]; + cusparseSpMatDescr_t ARight = dAs[matrixIndex2]; +// vector Fleft, FLeft; +// vector integrationTmp1, integrationTmp2; +// std::vector FCache; +// std::vector integrationCache; +// std::vector integrationBuffer; +// std::vector integrationBufferSize; +// std::vector integrationBufferStoredSize; +// if (left) { +// F = dFLeft; +// integrationTmp = dIntegrationTmpLeft; +// FCache = dFLeftCache; +// integrationCache = dIntegrationTmpLeftCache; +// integrationBuffer = dIntegrationLeftBuffer; +// integrationBufferSize = integrationLeftBufferSize; +// integrationBufferStoredSize = integrationLeftStoredBufferSize; +// } else { +// F = dFRight; +// integrationTmp = dIntegrationTmpRight; +// FCache = dFRightCache; +// integrationCache = dIntegrationTmpRightCache; +// integrationBuffer = dIntegrationRightBuffer; +// integrationBufferSize = integrationRightBufferSize; +// integrationBufferStoredSize = integrationRightStoredBufferSize; +// } + + // destP = partials; + CHECK_CUDA(cudaMemcpyAsync(dPartials[destPIndex1], dPartials[partialsIndex1], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartials[destPIndex2], dPartials[partialsIndex2], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + // F = destP; + CHECK_CUDA(cudaMemcpyAsync(dFLeftCache, dPartials[partialsIndex1], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dFRightCache, dPartials[partialsIndex2], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + + cudaDeviceSynchronize(); + + + + const Real zero = 0; + const Real one = 1; + for (int i = 0; i < s_max; i++) { + + for (int category = 0; category < kCategoryCount; category++) { + const Real c1Left = normPInf(dPartialCache[partialsIndex1 * kCategoryCount + category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + c1Cache[category] = c1Left; + const Real c1Right = normPInf(dPartialCache[partialsIndex2 * kCategoryCount + category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + c1Cache[kCategoryCount + category] = c1Right; + + etaCache[category] = i < std::get<1>(msCache[category]) ? etaCache[category] : 1; + etaCache[kCategoryCount + category] = i < std::get<1>(msCache[kCategoryCount + category]) ? etaCache[kCategoryCount + category] : 1; + } + std::fill(integrationMultipliers.begin(), integrationMultipliers.end(), 1.0); + + + for (int j = 1; j < m_max + 1; j++) { + + for (int category = 0; category < kCategoryCount; category++) { + alphaCache[category] = t / ((Real) std::get<0>(msCache[category]) * j) * integrationMultipliers[category] * (j < std::get<0>(msCache[category]) && i < std::get<1>(msCache[category])); + alphaCache[kCategoryCount + category] = t / ((Real) std::get<0>(msCache[category]) * j) * integrationMultipliers[kCategoryCount + category] * (j < std::get<0>(msCache[kCategoryCount + category]) && i < std::get<1>(msCache[kCategoryCount + category])); + } + + for (int category = 0; category < kCategoryCount; category++) { + CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &alphaCache[category], ALeft, dPartialsWrapper[destPIndex1], &zero, + dIntegrationTmpLeft[category], DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &integrationLeftBufferSize[category])) + + CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &alphaCache[kCategoryCount + category], ARight, dPartialsWrapper[destPIndex2], &zero, + dIntegrationTmpRight[category], DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &integrationRightBufferSize[kCategoryCount + category])) + if (integrationLeftBufferSize[category] > integrationLeftStoredBufferSize[category] && integrationMultipliers[category]) { + CHECK_CUDA(cudaMalloc(&dIntegrationLeftBuffer[category], + integrationLeftBufferSize[category])) // TODO: is this necessary? Are there better ways to claim additional buffer? + integrationLeftStoredBufferSize[category] = integrationLeftBufferSize[category]; + } + if (integrationRightBufferSize[category] > integrationRightStoredBufferSize[category] && integrationMultipliers[kCategoryCount + category]) { + CHECK_CUDA(cudaMalloc(&dIntegrationRightBuffer[category], + integrationRightBufferSize[category])) // TODO: is this necessary? Are there better ways to claim additional buffer? + integrationRightStoredBufferSize[category] = integrationRightBufferSize[category]; + + } + + // integrationTmp = alpha * A * destP + CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &alphaCache[category], ALeft, dPartialsWrapper[destPIndex1], &zero, + dIntegrationTmpLeft[category], DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &integrationLeftBufferSize[category])) + CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, + CUSPARSE_OPERATION_NON_TRANSPOSE, + &alphaCache[kCategoryCount + category], ARight, dPartialsWrapper[destPIndex2], &zero, + dIntegrationTmpRight[category], DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &integrationRightBufferSize[kCategoryCount + category])) + + } + cudaDeviceSynchronize(); + + // destP = IntegrationTmp + CHECK_CUDA(cudaMemcpyAsync(dPartials[destPIndex1], dIntegrationTmpLeftCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartials[destPIndex2], dIntegrationTmpRightCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + +// F += destP = IntegrationTmp; + if constexpr (std::is_same::value) { + CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache, 1, dFLeftCache, 1)); + CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache, 1, dFRightCache, 1)); + } else { + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache, 1, dFLeftCache, 1)); + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache, 1, dFRightCache, 1)); + } + + cudaDeviceSynchronize(); + + for (int category = 0; category < kCategoryCount; category++) { + c2Cache[category] = normPInf(dIntegrationTmpLeftCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + c2Cache[kCategoryCount + category] = normPInf(dIntegrationTmpRightCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + if (c1Cache[category] + c2Cache[category] < tol * normPInf(dFLeftCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { + integrationMultipliers[category] = 0; + } + if (c1Cache[kCategoryCount + category] + c2Cache[kCategoryCount + category] < tol * normPInf(dFRightCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { + integrationMultipliers[kCategoryCount + category] = 0; + } + c1Cache[category] = c2Cache[category]; + c1Cache[kCategoryCount + category] = c2Cache[kCategoryCount + category]; + } + + } +// F *= eta; + for (int category = 0; category < kCategoryCount; category++) { + if constexpr (std::is_same::value) { + CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[category], dFLeftCache[category], 1)); + CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[kCategoryCount + category], dFRightCache[category], 1)); + } else { + CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[category], dFLeftCache[category], 1)); + CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[kCategoryCount + category], dFRightCache[category], 1)); + } + } + cudaDeviceSynchronize(); +// destP = F; + CHECK_CUDA(cudaMemcpyAsync(dPartials[destPIndex1], dFLeftCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartials[destPIndex2], dFRightCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + cudaDeviceSynchronize(); + } + + return BEAGLE_SUCCESS; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPrePartials(bool byPartition, const int *operations, From 34cbee17a5446ac88b603f17563fed112889a737 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 12 Jun 2024 10:26:33 -0500 Subject: [PATCH 125/421] fix pointer issue --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 28f63e47..050a2818 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -310,7 +310,6 @@ int BeagleGPUActionImpl::createInstance(int tipCount, preferenceFlags, requirementFlags); - int hMatrixCacheSize = kMatrixSize * kCategoryCount * BEAGLE_CACHED_MATRICES_COUNT; //TODO: use Eigen csr representation? hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); hIdentity.setIdentity(); hInstantaneousMatrices.resize(kEigenDecompCount); @@ -359,17 +358,17 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dFLeft = std::vector(kCategoryCount, nullptr); dFLeftCache = std::vector(kCategoryCount, nullptr); - dFLeftCache[0] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + Real* dLeftCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); dFRight = std::vector(kCategoryCount, nullptr); dFRightCache = std::vector(kCategoryCount, nullptr); - dFRightCache[0] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + Real* dRightCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); dIntegrationTmpLeft = std::vector(kCategoryCount, nullptr); dIntegrationTmpLeftCache = std::vector(kCategoryCount, nullptr); - dIntegrationTmpLeftCache[0] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + Real* dIntegrationTmpLeftCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); dIntegrationTmpRight = std::vector(kCategoryCount, nullptr); dIntegrationTmpRightCache = std::vector(kCategoryCount, nullptr); - dIntegrationTmpRightCache[0] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + Real* dIntegrationTmpRightCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); integrationLeftBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); integrationLeftStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationLeftBuffer = std::vector(kCategoryCount, nullptr); @@ -377,11 +376,11 @@ int BeagleGPUActionImpl::createInstance(int tipCount, integrationRightStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); - for (int category = 1; category < kCategoryCount; category++) { - dFLeftCache[category] = dFLeftCache[0] + kPaddedStateCount * kPaddedPatternCount * sizeof(Real); - dFRightCache[category] = dFRightCache[0] + kPaddedStateCount * kPaddedPatternCount * sizeof(Real); - dIntegrationTmpLeftCache[category] = dIntegrationTmpLeftCache[0] + kPaddedStateCount * kPaddedPatternCount * sizeof(Real); - dIntegrationTmpRightCache[category] = dIntegrationTmpRightCache[0] + kPaddedStateCount * kPaddedPatternCount * sizeof(Real); + for (int category = 0; category < kCategoryCount; category++) { + dFLeftCache[category] = dLeftCachePtr + kPaddedStateCount * kPaddedPatternCount * category; + dFRightCache[category] = dRightCachePtr + kPaddedStateCount * kPaddedPatternCount * category; + dIntegrationTmpLeftCache[category] = dIntegrationTmpLeftCachePtr + kPaddedStateCount * kPaddedPatternCount * category; + dIntegrationTmpRightCache[category] = dIntegrationTmpRightCachePtr + kPaddedStateCount * kPaddedPatternCount * category; } for (int category = 0; category < kCategoryCount; category++) { From 1cfdc234edb9bc2c232f83eb9b3b8277f3f818d4 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 12 Jun 2024 12:35:22 -0400 Subject: [PATCH 126/421] Just request python 3.12, not 3.12.0 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2a902c68..de696a3a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,7 +76,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.12.0 + python-version: 3.12 - name: Install (Linux) if: runner.os == 'Linux' && matrix.name != 'windows' From 192ac7a4236df7ba6380f730e99cdf6656c127b2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 12 Jun 2024 13:59:20 -0400 Subject: [PATCH 127/421] Use MemcpyDeviceToDevice --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 050a2818..96d44014 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -285,6 +285,14 @@ void MemcpyHostToDevice(T* dptr, const T* hptr, int n) throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); } +template +void MemcpyDeviceToDevice(T* dptr, const T* hptr, int n) +{ + auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyDeviceToDevice); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::createInstance(int tipCount, int partialsBufferCount, @@ -892,8 +900,8 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int const Real edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; - CHECK_CUDA(cudaMemcpy(dACscValuesCache[matrixIndex1], dBsCsrValuesCache[hEigenMaps[edgeIndex1]], sizeof(Real) * currentCacheNNZs[hEigenMaps[edgeIndex1]], cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpy(dACscValuesCache[matrixIndex2], dBsCsrValuesCache[hEigenMaps[edgeIndex2]], sizeof(Real) * currentCacheNNZs[hEigenMaps[edgeIndex2]], cudaMemcpyDeviceToDevice)) + MemcpyDeviceToDevice(dACscValuesCache[matrixIndex1], dBsCsrValuesCache[hEigenMaps[edgeIndex1]], currentCacheNNZs[hEigenMaps[edgeIndex1]]); + MemcpyDeviceToDevice(dACscValuesCache[matrixIndex2], dBsCsrValuesCache[hEigenMaps[edgeIndex2]], currentCacheNNZs[hEigenMaps[edgeIndex2]]); if constexpr (std::is_same::value) { CUBLAS_CHECK(cublasSscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following CUBLAS_CHECK(cublasSscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following @@ -986,7 +994,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p //#endif // destP = partials; - CHECK_CUDA(cudaMemcpy(dPartialCache[destPIndex], dPartialCache[partialsIndex], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) + MemcpyDeviceToDevice(dPartialCache[destPIndex], dPartialCache[partialsIndex], kPaddedStateCount * kPaddedPatternCount); // CHECK_CUSPARSE(cusparseDnMatSetValues(dPartialsWrapper[destPIndex], dPartialCache[partialsIndex])) @@ -999,7 +1007,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // MatrixXd F(kStateCount, nCol); // F = destP; - CHECK_CUDA(cudaMemcpy(FCache[category], dPartialCache[partialsIndex], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) + MemcpyDeviceToDevice(FCache[category], dPartialCache[partialsIndex], kPaddedStateCount * kPaddedPatternCount); // CHECK_CUSPARSE(cusparseDnMatSetValues(F[category], dPartialCache[partialsIndex])) // TODO: loop category within this function //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"F = partials operation, FCache:"<::simpleAction2(int destPIndex, int p // integrationTmp = alpha * A * destP CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, A, dPartialsWrapper[destPIndex], &zero, integrationTmp[category], DataType, - CUSPARSE_SPMM_ALG_DEFAULT, integrationBuffer[category])) //row-major layout provides higher performance (?) + CUSPARSE_SPMM_ALG_DEFAULT, integrationBuffer[category])); //row-major layout provides higher performance (?) //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"edge multiplier = "<< hEdgeMultipliers[edgeIndex * kCategoryCount + category]<<"\nB ="<::simpleAction2(int destPIndex, int p //#endif // destP = IntegrationTmp - CHECK_CUDA(cudaMemcpy(dPartialCache[destPIndex], integrationCache[category], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) + MemcpyDeviceToDevice(dPartialCache[destPIndex], integrationCache[category], kPaddedStateCount * kPaddedPatternCount); //#ifdef BEAGLE_DEBUG_FLOW // @@ -1090,7 +1098,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // PrintfDeviceVector(FCache[category], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); //#endif // destP = F; - CHECK_CUDA(cudaMemcpy(dPartialCache[destPIndex], FCache[category], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount, cudaMemcpyDeviceToDevice)) + MemcpyDeviceToDevice(dPartialCache[destPIndex], FCache[category], kPaddedStateCount * kPaddedPatternCount); #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"destP = F:"< Date: Wed, 12 Jun 2024 14:28:18 -0400 Subject: [PATCH 128/421] Factor out some code into spMM --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 42 ++++++++++++++---------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 96d44014..7c62f5e3 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -944,6 +944,28 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int return BEAGLE_SUCCESS; } +template +int spMM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSpMatDescr_t A, cusparseDnMatDescr_t B, Real beta, void*& buffer, size_t& buffersize) +{ + size_t new_buffersize; + CHECK_CUSPARSE(cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A, B, &beta, C, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &new_buffersize)); + + if(new_buffersize > buffersize) + { + CHECK_CUDA(cudaFree(buffer)); + CHECK_CUDA(cudaMalloc(&buffer, buffersize)); + buffersize = new_buffersize; + } + + // integrationTmp = alpha * A * destP + CHECK_CUSPARSE(cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A, B, &beta, C, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, buffer)); //row-major layout provides higher performance (?) + return 0; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::simpleAction2(int destPIndex, int partialsIndex, int edgeIndex, int category, int matrixIndex, bool left, bool transpose) { const double tol = pow(2.0, -53.0); @@ -969,7 +991,6 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p std::vector integrationCache; std::vector integrationBuffer; std::vector integrationBufferSize; - std::vector integrationBufferStoredSize; if (left) { F = dFLeft; integrationTmp = dIntegrationTmpLeft; @@ -977,7 +998,6 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p integrationCache = dIntegrationTmpLeftCache; integrationBuffer = dIntegrationLeftBuffer; integrationBufferSize = integrationLeftBufferSize; - integrationBufferStoredSize = integrationLeftStoredBufferSize; } else { F = dFRight; integrationTmp = dIntegrationTmpRight; @@ -985,7 +1005,6 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p integrationCache = dIntegrationTmpRightCache; integrationBuffer = dIntegrationRightBuffer; integrationBufferSize = integrationRightBufferSize; - integrationBufferStoredSize = integrationRightStoredBufferSize; } //#ifdef BEAGLE_DEBUG_FLOW @@ -1030,26 +1049,13 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p Real c1 = normPInf(dPartialCache[partialsIndex], kPaddedStateCount, kPaddedPatternCount, cublasHandle); - for (int j = 1; j < m + 1; j++) { - const Real alpha = t / ((Real) s * j); //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"j/m = "<::createInstance(tipCount, partialsBufferCount * 2, compactBufferCount, stateCount, patternCount, + BeagleGPUImpl::createInstance(tipCount, partialsBufferCount * 2, 0, stateCount, patternCount, eigenDecompositionCount, matrixCount, categoryCount, scaleBufferCount, globalResourceNumber, pluginResourceNumber, preferenceFlags, requirementFlags); - - + kPartialsCacheOffset = partialsBufferCount; hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); hIdentity.setIdentity(); hInstantaneousMatrices.resize(kEigenDecompCount); @@ -350,15 +349,15 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } - dPartialsWrapper.resize(kPartialsBufferCount * kCategoryCount * 2); - dPartialCache.resize(kPartialsBufferCount * kCategoryCount * 2); - for (int i = 0; i < kPartialsBufferCount * 2; i++) { + dPartialsWrapper.resize(kPartialsBufferCount * kCategoryCount); + dPartialCache.resize(kPartialsBufferCount * kCategoryCount); + for (int i = 0; i < kPartialsBufferCount; i++) { for (int category = 0; category < kCategoryCount; category++) { dPartialCache[i * kCategoryCount + category] = (Real *) gpu->CreateSubPointer(dPartialsOrigin, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * (kCategoryCount * i + category), sizeof(Real) * kPaddedStateCount * kPaddedPatternCount); } } - for (int i = 0; i < kPartialsBufferCount * kCategoryCount * 2; i++) { + for (int i = 0; i < kPartialsBufferCount * kCategoryCount; i++) { // dPartialCache[i] = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount); CHECK_CUSPARSE(cusparseCreateDnMat(&dPartialsWrapper[i], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dPartialCache[i], DataType, CUSPARSE_ORDER_COL)) @@ -405,8 +404,8 @@ int BeagleGPUActionImpl::createInstance(int tipCount, CHECK_CUDA(cudaMalloc(&dIntegrationRightBuffer[category], integrationRightBufferSize[category])) } - hEigenMaps.resize(kPartialsBufferCount); - hEdgeMultipliers.resize(kPartialsBufferCount * kCategoryCount); + hEigenMaps.resize(kPartialsCacheOffset); + hEdgeMultipliers.resize(kPartialsCacheOffset * kCategoryCount); msCache = std::vector>(2 * kCategoryCount, tuple{0, 0}); etaCache = std::vector(2 * kCategoryCount, 0); @@ -548,7 +547,7 @@ int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartialCacheIndex(int nodeIndex, int categoryIndex) { - return kPartialsBufferCount + getPartialIndex(nodeIndex, categoryIndex); + return kPartialsCacheOffset + getPartialIndex(nodeIndex, categoryIndex); } BEAGLE_GPU_TEMPLATE @@ -760,52 +759,51 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde cacheAMatrices(edgeIndex1, edgeIndex2, false); - for (int category = 0; category < kCategoryCount; category++) - { - const int partial1Index = getPartialIndex(partials1Index, category); - const int partial2Index = getPartialIndex(partials2Index, category); - - const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); - const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); - - const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; - const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; - - simpleAction2(partial1CacheIndex, partial1Index, edgeIndex1, category, matrixIndex1, true, false); - - -//#ifdef BEAGLE_DEBUG_FLOW -// PrintfDeviceVector(dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); -//#endif +// for (int category = 0; category < kCategoryCount; category++) +// { +// const int partial1Index = getPartialIndex(partials1Index, category); +// const int partial2Index = getPartialIndex(partials2Index, category); +// +// const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); +// const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); +// +// const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; +// const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; +// +// simpleAction2(getPartialCacheIndex(partials1Index, category), getPartialIndex(partials1Index, category), +// edgeIndex1, category, hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category, true, false); +// +// simpleAction2(getPartialCacheIndex(partials2Index, category), getPartialIndex(partials2Index, category), +// edgeIndex2, category, hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category, false, false); +// } - simpleAction2(partial2CacheIndex, partial2Index, edgeIndex2, category, matrixIndex2, false, false); - } + simpleAction3(partials1Index, edgeIndex1, partials2Index, edgeIndex2); cudaDeviceSynchronize(); for (int category = 0; category < kCategoryCount; category++) { - const int destPartialindex = getPartialIndex(destPIndex, category); + const int destPartialIndex = getPartialIndex(destPIndex, category); const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); if constexpr (std::is_same::value) { CUBLAS_CHECK(cublasSdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, - dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); + dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialIndex], kPaddedStateCount * kPaddedPatternCount)); } else { CUBLAS_CHECK(cublasDdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialCache[partial1CacheIndex], kPaddedStateCount * kPaddedPatternCount, - dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialindex], kPaddedStateCount * kPaddedPatternCount)); + dPartialCache[partial2CacheIndex], 1, dPartialCache[destPartialIndex], kPaddedStateCount * kPaddedPatternCount)); } //#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"Checking p_parent = p_1 * p_2, parent index = "<(msCache[i])<<", "<(msCache[i])<<"), "; + std::cerr<<"("<(msCache[i + kCategoryCount])<<", "<(msCache[i + kCategoryCount])<<"), "; + } +#endif const Real zero = 0; const Real one = 1; for (int i = 0; i < s_max; i++) { for (int category = 0; category < kCategoryCount; category++) { - const Real c1Left = normPInf(dPartialCache[partialsIndex1 * kCategoryCount + category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + const Real c1Left = normPInf(dPartialCache[getPartialIndex(partialsIndex1, category)], kPaddedStateCount, kPaddedPatternCount, cublasHandle); c1Cache[category] = c1Left; - const Real c1Right = normPInf(dPartialCache[partialsIndex2 * kCategoryCount + category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + const Real c1Right = normPInf(dPartialCache[getPartialIndex(partialsIndex2, category)], kPaddedStateCount, kPaddedPatternCount, cublasHandle); c1Cache[kCategoryCount + category] = c1Right; - etaCache[category] = i < std::get<1>(msCache[category]) ? etaCache[category] : 1; + etaCache[category] = (i < std::get<1>(msCache[category])) * (etaCache[category] - 1) + 1; etaCache[kCategoryCount + category] = i < std::get<1>(msCache[kCategoryCount + category]) ? etaCache[kCategoryCount + category] : 1; } std::fill(integrationMultipliers.begin(), integrationMultipliers.end(), 1.0); +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"s/s_max = "<(msCache[category]) * j) * integrationMultipliers[category] * (j < std::get<0>(msCache[category]) && i < std::get<1>(msCache[category])); - alphaCache[kCategoryCount + category] = t / ((Real) std::get<0>(msCache[category]) * j) * integrationMultipliers[kCategoryCount + category] * (j < std::get<0>(msCache[kCategoryCount + category]) && i < std::get<1>(msCache[kCategoryCount + category])); + alphaCache[category] = t / ((Real) std::get<1>(msCache[category]) * j) * integrationMultipliers[category] * (j < std::get<0>(msCache[category]) && i < std::get<1>(msCache[category])); + alphaCache[kCategoryCount + category] = t / ((Real) std::get<1>(msCache[kCategoryCount + category]) * j) * integrationMultipliers[kCategoryCount + category] * (j < std::get<0>(msCache[kCategoryCount + category]) && i < std::get<1>(msCache[kCategoryCount + category])); + } + +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"\nj/m_max = "<(cusparseHandle, dIntegrationTmpLeft[category], alphaCache[category], ALeft, dPartialsWrapper[destPIndex1], 0, dIntegrationLeftBuffer[category], integrationLeftBufferSize[category]); // spMM(cusparseHandle, dIntegrationTmpRight[category], alphaCache[kCategoryCount + category], ARight, dPartialsWrapper[destPIndex2], 0, dIntegrationRightBuffer[category], integrationRightBufferSize[category]); CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alphaCache[category], ALeft, dPartialsWrapper[destPIndex1], &zero, + &alphaCache[category], dAs[hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex1, category)], &zero, dIntegrationTmpLeft[category], DataType, CUSPARSE_SPMM_ALG_DEFAULT, &integrationLeftBufferSize[category])) CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alphaCache[kCategoryCount + category], ARight, dPartialsWrapper[destPIndex2], &zero, + &alphaCache[kCategoryCount + category], dAs[hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex2, category)], &zero, dIntegrationTmpRight[category], DataType, CUSPARSE_SPMM_ALG_DEFAULT, &integrationRightBufferSize[kCategoryCount + category])) if (integrationLeftBufferSize[category] > integrationLeftStoredBufferSize[category] && integrationMultipliers[category]) { @@ -1236,29 +1268,35 @@ int BeagleGPUActionImpl::simpleAction3(int destPIndex1, int // integrationTmp = alpha * A * destP CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alphaCache[category], ALeft, dPartialsWrapper[destPIndex1], &zero, + &alphaCache[category], dAs[hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex1, category)], &zero, dIntegrationTmpLeft[category], DataType, - CUSPARSE_SPMM_ALG_DEFAULT, &integrationLeftBufferSize[category])) + CUSPARSE_SPMM_ALG_DEFAULT, dIntegrationLeftBuffer[category])) CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alphaCache[kCategoryCount + category], ARight, dPartialsWrapper[destPIndex2], &zero, + &alphaCache[kCategoryCount + category], dAs[hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex2, category)], &zero, dIntegrationTmpRight[category], DataType, - CUSPARSE_SPMM_ALG_DEFAULT, &integrationRightBufferSize[kCategoryCount + category])) + CUSPARSE_SPMM_ALG_DEFAULT, dIntegrationRightBuffer[category])) } cudaDeviceSynchronize(); +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"integrationTmp = alpha * A * destP = " <::value) { - CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache, 1, dFLeftCache, 1)); - CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache, 1, dFRightCache, 1)); + CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache[0], 1, dFLeftCache[0], 1)); + CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache[0], 1, dFRightCache[0], 1)); } else { - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache, 1, dFLeftCache, 1)); - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache, 1, dFRightCache, 1)); + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache[0], 1, dFLeftCache[0], 1)); + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache[0], 1, dFRightCache[0], 1)); } cudaDeviceSynchronize(); @@ -1272,10 +1310,27 @@ int BeagleGPUActionImpl::simpleAction3(int destPIndex1, int if (c1Cache[kCategoryCount + category] + c2Cache[kCategoryCount + category] < tol * normPInf(dFRightCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { integrationMultipliers[kCategoryCount + category] = 0; } + c1Cache[category] = c2Cache[category]; c1Cache[kCategoryCount + category] = c2Cache[kCategoryCount + category]; } +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"dFLeftCache = " <::simpleAction3(int destPIndex1, int } cudaDeviceSynchronize(); // destP = F; - CHECK_CUDA(cudaMemcpyAsync(dPartials[destPIndex1], dFLeftCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpyAsync(dPartials[destPIndex2], dFRightCache, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex1, 0)], dFLeftCache[0], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex2, 0)], dFRightCache[0], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) cudaDeviceSynchronize(); } From bcb79bcd0936fc0d97e91eeb4ff8c831cfc74a1c Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 12 Jun 2024 14:36:48 -0500 Subject: [PATCH 131/421] cleaning and better logic comparison calc --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 37 +----------------------- 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 4c6cd25b..ba4e7d71 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1140,41 +1140,6 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i if (s_max < s2) s_max = s2; } -// const int partial1Index = getPartialIndex(partials1Index, category); -// const int partial2Index = getPartialIndex(partials2Index, category); -// -// const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); -// const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); -// -// const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; -// const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; - -// cusparseSpMatDescr_t ALeft = dAs[matrixIndex1]; -// cusparseSpMatDescr_t ARight = dAs[matrixIndex2]; -// vector Fleft, FLeft; -// vector integrationTmp1, integrationTmp2; -// std::vector FCache; -// std::vector integrationCache; -// std::vector integrationBuffer; -// std::vector integrationBufferSize; -// std::vector integrationBufferStoredSize; -// if (left) { -// F = dFLeft; -// integrationTmp = dIntegrationTmpLeft; -// FCache = dFLeftCache; -// integrationCache = dIntegrationTmpLeftCache; -// integrationBuffer = dIntegrationLeftBuffer; -// integrationBufferSize = integrationLeftBufferSize; -// integrationBufferStoredSize = integrationLeftStoredBufferSize; -// } else { -// F = dFRight; -// integrationTmp = dIntegrationTmpRight; -// FCache = dFRightCache; -// integrationCache = dIntegrationTmpRightCache; -// integrationBuffer = dIntegrationRightBuffer; -// integrationBufferSize = integrationRightBufferSize; -// integrationBufferStoredSize = integrationRightStoredBufferSize; -// } // destP = partials; CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex1, 0)], dPartialCache[getPartialIndex(partialsIndex1, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) @@ -1210,7 +1175,7 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i c1Cache[kCategoryCount + category] = c1Right; etaCache[category] = (i < std::get<1>(msCache[category])) * (etaCache[category] - 1) + 1; - etaCache[kCategoryCount + category] = i < std::get<1>(msCache[kCategoryCount + category]) ? etaCache[kCategoryCount + category] : 1; + etaCache[kCategoryCount + category] = (i < std::get<1>(msCache[kCategoryCount + category])) * (etaCache[kCategoryCount + category] - 1) + 1; } std::fill(integrationMultipliers.begin(), integrationMultipliers.end(), 1.0); From 5c7fe136116b5b6dbf50c9fba9f6d983e62731b2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 12 Jun 2024 17:49:14 -0400 Subject: [PATCH 132/421] Ignore problem loading kernel that we don't use. If this breaks, blame Xiang! --- libhmsbeagle/GPU/GPUInterfaceCUDA.cpp | 2 ++ libhmsbeagle/GPU/KernelLauncher.cpp | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp index 570c2500..e09fe70f 100644 --- a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp +++ b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp @@ -251,6 +251,8 @@ void GPUInterface::InitializeKernelResource(int paddedStateCount, case 128: LOAD_KERNEL_INTO_RESOURCE(128, SP, 128); break; case 192: LOAD_KERNEL_INTO_RESOURCE(192, SP, 192); break; case 256: LOAD_KERNEL_INTO_RESOURCE(256, SP, 256); break; + default: printf(stderr,"ACTION CUDA: Stupidly loading 4-state kernel since we can't find the right one"); + LOAD_KERNEL_INTO_RESOURCE( 4, DP, 4); } } diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 3a81bfe3..17289cd6 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -205,7 +205,8 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { if (kPatternCount % 4 != 0) { bgScaleGrid.x += 1; // fprintf(stderr,"PATTERNS SHOULD BE PADDED! Inform Marc, please.\n"); - exit(-1); + fprintf(stderr,"Going ahead anyway..\n"); + // exit(-1); } } else { bgScaleBlock = Dim3Int(kPaddedStateCount, kMatrixBlockSize); From 6c2f3a0ac1205f3e93a1f2976c8670a0f8d369b0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 12 Jun 2024 17:52:32 -0400 Subject: [PATCH 133/421] fprintf --- libhmsbeagle/GPU/GPUInterfaceCUDA.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp index e09fe70f..771c2472 100644 --- a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp +++ b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp @@ -251,7 +251,7 @@ void GPUInterface::InitializeKernelResource(int paddedStateCount, case 128: LOAD_KERNEL_INTO_RESOURCE(128, SP, 128); break; case 192: LOAD_KERNEL_INTO_RESOURCE(192, SP, 192); break; case 256: LOAD_KERNEL_INTO_RESOURCE(256, SP, 256); break; - default: printf(stderr,"ACTION CUDA: Stupidly loading 4-state kernel since we can't find the right one"); + default: fprintf(stderr,"ACTION CUDA: Stupidly loading 4-state kernel since we can't find the right one"); LOAD_KERNEL_INTO_RESOURCE( 4, DP, 4); } } From 8286efd5f075e7faa260b687e2a30457b70ea745 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 12 Jun 2024 17:56:10 -0400 Subject: [PATCH 134/421] Destroy cublas handle. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 ++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 0e6e2027..f223b9db 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -296,6 +296,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int operationCount, int cumulativeScalingIndex); + ~BeagleGPUActionImpl(); + private: char* getInstanceName(); diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index ba4e7d71..cd0177b7 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -417,6 +417,12 @@ int BeagleGPUActionImpl::createInstance(int tipCount, return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +BeagleGPUActionImpl::~BeagleGPUActionImpl() +{ + cublasDestroy(cublasHandle); +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getInstanceDetails(BeagleInstanceDetails* returnInfo) { BeagleGPUImpl::getInstanceDetails(returnInfo); From 09fe3511350af830cd01439e4d2cd62d8e541624 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 12 Jun 2024 18:07:29 -0400 Subject: [PATCH 135/421] Add missing newline. --- libhmsbeagle/GPU/GPUInterfaceCUDA.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp index 771c2472..d5759631 100644 --- a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp +++ b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp @@ -251,7 +251,7 @@ void GPUInterface::InitializeKernelResource(int paddedStateCount, case 128: LOAD_KERNEL_INTO_RESOURCE(128, SP, 128); break; case 192: LOAD_KERNEL_INTO_RESOURCE(192, SP, 192); break; case 256: LOAD_KERNEL_INTO_RESOURCE(256, SP, 256); break; - default: fprintf(stderr,"ACTION CUDA: Stupidly loading 4-state kernel since we can't find the right one"); + default: fprintf(stderr,"ACTION CUDA: Stupidly loading 4-state kernel since we can't find the right one\n"); LOAD_KERNEL_INTO_RESOURCE( 4, DP, 4); } } From a10724784545838c9917d8dacccbe84d0dbf6287 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 12 Jun 2024 22:06:15 -0500 Subject: [PATCH 136/421] do not allocate eigen related memory for action in GPUImpl --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 44 ++++++++++++++++++------------ 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 07064f32..86a69aef 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -343,7 +343,7 @@ int BeagleGPUImpl::createInstance(int tipCount, } else if (kStateCount <= 256) { kPaddedStateCount = 256; } else { - kPaddedStateCount = kStateCount + kStateCount % 16; + kPaddedStateCount = kStateCount % 16 == 0 ? kStateCount : kStateCount + 16 - kStateCount % 16; } gpu = new GPUInterface(); @@ -572,9 +572,15 @@ int BeagleGPUImpl::createInstance(int tipCount, hLogLikelihoodsCache = (Real*) gpu->MallocHost(kPatternCount * sizeof(Real)); hMatrixCache = (Real*) gpu->CallocHost(hMatrixCacheSize, sizeof(Real)); - dEvec = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); - dIevc = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); - dEigenValues = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); + if (preferenceFlags & BEAGLE_FLAG_COMPUTATION_ACTION) { + dEvec = nullptr; + dIevc = nullptr; + dEigenValues = nullptr; + } else { + dEvec = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); + dIevc = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); + dEigenValues = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); + } dWeights = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); dFrequencies = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); @@ -620,22 +626,26 @@ int BeagleGPUImpl::createInstance(int tipCount, } } - ptrIncrement = gpu->AlignMemOffset(kMatrixSize * sizeof(Real)); - kEvecOffset = ptrIncrement/sizeof(Real); - GPUPtr dEvecOrigin = gpu->AllocateMemory(kEigenDecompCount * ptrIncrement); - GPUPtr dIevcOrigin = gpu->AllocateMemory(kEigenDecompCount * ptrIncrement); - for(int i=0; iCreateSubPointer(dEvecOrigin, ptrIncrement*i, ptrIncrement); - dIevc[i] = gpu->CreateSubPointer(dIevcOrigin, ptrIncrement*i, ptrIncrement); - } + if(!preferenceFlags & BEAGLE_FLAG_COMPUTATION_ACTION) { + ptrIncrement = gpu->AlignMemOffset(kMatrixSize * sizeof(Real)); + kEvecOffset = ptrIncrement/sizeof(Real); + GPUPtr dEvecOrigin = gpu->AllocateMemory(kEigenDecompCount * ptrIncrement); + GPUPtr dIevcOrigin = gpu->AllocateMemory(kEigenDecompCount * ptrIncrement); + for(int i=0; iCreateSubPointer(dEvecOrigin, ptrIncrement*i, ptrIncrement); + dIevc[i] = gpu->CreateSubPointer(dIevcOrigin, ptrIncrement*i, ptrIncrement); + } + + ptrIncrement = gpu->AlignMemOffset(kEigenValuesSize * sizeof(Real)); + kEvalOffset = ptrIncrement/sizeof(Real); + GPUPtr dEigenValuesOrigin = gpu->AllocateMemory(kEigenDecompCount * ptrIncrement); + for(int i=0; iCreateSubPointer(dEigenValuesOrigin, ptrIncrement*i, ptrIncrement); + } - ptrIncrement = gpu->AlignMemOffset(kEigenValuesSize * sizeof(Real)); - kEvalOffset = ptrIncrement/sizeof(Real); - GPUPtr dEigenValuesOrigin = gpu->AllocateMemory(kEigenDecompCount * ptrIncrement); - for(int i=0; iCreateSubPointer(dEigenValuesOrigin, ptrIncrement*i, ptrIncrement); } + ptrIncrement = gpu->AlignMemOffset(kCategoryCount * sizeof(Real)); kWeightsOffset = ptrIncrement/sizeof(Real); GPUPtr dWeightsOrigin = gpu->AllocateMemory(kEigenDecompCount * ptrIncrement); From 4c931786fbc9b345871827cbbbb1ebd18fd9f09d Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 12 Jun 2024 22:06:38 -0500 Subject: [PATCH 137/421] more destroyers, still have error --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index cd0177b7..cbfd10af 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -420,7 +420,36 @@ int BeagleGPUActionImpl::createInstance(int tipCount, BEAGLE_GPU_TEMPLATE BeagleGPUActionImpl::~BeagleGPUActionImpl() { +// BeagleGPUImpl::~BeagleGPUImpl<>(); + + for (int i = 0; i < dPartialsWrapper.size(); i++) { + cusparseDestroyDnMat(dPartialsWrapper[i]); + cudaFree(dPartialCache[i]); + } + for (int i = 0; i < kCategoryCount; i++) { + cusparseDestroyDnMat(dFLeft[i]); + cusparseDestroyDnMat(dFRight[i]); + cusparseDestroyDnMat(dIntegrationTmpLeft[i]); + cusparseDestroyDnMat(dIntegrationTmpRight[i]); + cudaFree(dFLeftCache[i]); + cudaFree(dFRightCache[i]); + cudaFree(dIntegrationTmpLeftCache[i]); + cudaFree(dIntegrationTmpRightCache[i]); + cudaFree(dIntegrationLeftBuffer[i]); + cudaFree(dIntegrationRightBuffer[i]); + } + + for (int i = 0; i < kEigenDecompCount; i++) { + cudaFree(dBsCsrOffsetsCache[i]); + cudaFree(dBsCsrColumnsCache[i]); + cudaFree(dBsCsrValuesCache[i]); + for (int j = 0; j < kCategoryCount * 2; j++) { + cudaFree(dACscValuesCache[i * kCategoryCount * 2 + j]); +// cusparseDestroySpMat(dAs[i * kCategoryCount * 2 + i]); + } + } cublasDestroy(cublasHandle); + cusparseDestroy(cusparseHandle); } BEAGLE_GPU_TEMPLATE From cac0d64cec4445409eae63e6a538affcdbf41004 Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 14 Jun 2024 16:07:14 -0500 Subject: [PATCH 138/421] JNI new function for setting sparse matrix --- java/beagle-lib.iml | 11 +++++++++++ java/beagle/Beagle.java | 18 ++++++++++++++++++ java/beagle/BeagleJNIImpl.java | 8 ++++++++ java/beagle/BeagleJNIWrapper.java | 7 +++++++ libhmsbeagle/JNI/beagle_BeagleJNIWrapper.cpp | 20 ++++++++++++++++++++ libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h | 12 ++++++++++-- 6 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 java/beagle-lib.iml diff --git a/java/beagle-lib.iml b/java/beagle-lib.iml new file mode 100644 index 00000000..b107a2dd --- /dev/null +++ b/java/beagle-lib.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/java/beagle/Beagle.java b/java/beagle/Beagle.java index 9bfb765c..29a3c8cc 100644 --- a/java/beagle/Beagle.java +++ b/java/beagle/Beagle.java @@ -287,6 +287,24 @@ void setEigenDecomposition( final double[] inInverseEigenVectors, final double[] inEigenValues); + /** + * Set a sparse matrix + * + * This function copies a sparse matrix into a instance buffer. + * + * @param maatrixIndex Index of sparse matrix buffer (input) + * @param inRowIndices Row indices of non-zero entries (input) + * @param inColIndices Column indices of non-zero entries (input) + * @param inValues Vector of non-zero values (input) + * @param numNonZeros Number of non-zero entries (input) + */ + void setSparseMatrix( + int maatrixIndex, + final int[] inRowIndices, + final int[] inColIndices, + final double[] inValues, + int numNonZeros); + /** * Set a set of state frequences. These will probably correspond to an * eigen-system. diff --git a/java/beagle/BeagleJNIImpl.java b/java/beagle/BeagleJNIImpl.java index 7575fbac..bad23d4e 100644 --- a/java/beagle/BeagleJNIImpl.java +++ b/java/beagle/BeagleJNIImpl.java @@ -141,6 +141,14 @@ public void setEigenDecomposition(int eigenIndex, } } + @Override + public void setSparseMatrix(int matrixIndex, int[] inRowIndices, int[] inColIndices, double[] inValues, int numNonZeros) { + int errCode = BeagleJNIWrapper.INSTANCE.setSparseMatrix(instance, matrixIndex, inRowIndices, inColIndices, inValues, numNonZeros); + if (errCode != 0) { + throw new BeagleException("setSparseMatrix", errCode); + } + } + public void setStateFrequencies(int stateFrequenciesIndex, final double[] stateFrequencies) { int errCode = BeagleJNIWrapper.INSTANCE.setStateFrequencies(instance, diff --git a/java/beagle/BeagleJNIWrapper.java b/java/beagle/BeagleJNIWrapper.java index b77c0920..60207f01 100644 --- a/java/beagle/BeagleJNIWrapper.java +++ b/java/beagle/BeagleJNIWrapper.java @@ -90,6 +90,13 @@ public native int setEigenDecomposition(int instance, final double[] inverseEigenValues, final double[] eigenValues); + public native int setSparseMatrix(int instance, + int matrixIndex, + final int[] rowIndices, + final int[] colIndices, + final double[] values, + int numNonZeros); + public native int setStateFrequencies(int instance, int stateFrequenciesIndex, final double[] stateFrequencies); diff --git a/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.cpp b/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.cpp index 1586b48d..266c71b9 100644 --- a/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.cpp +++ b/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.cpp @@ -634,6 +634,26 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setEigenDecomposition return errCode; } +/* + * Class: beagle_BeagleJNIWrapper + * Method: setSparseMatrix + * Signature: (II[I[I[DI)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setSparseMatrix + (JNIEnv *env, jobject obj, jint instance, jint matrixIndex, jintArray inRowIndices, jintArray inColIndices, jdoubleArray inValues, jint numNonZeros) { + jint *rowIndices = env->GetIntArrayElements(inRowIndices, NULL); + jint *colIndices = env->GetIntArrayElements(inColIndices, NULL); + jdouble *values = env->GetDoubleArrayElements(inValues, NULL); + + jint errCode = (jint)beagleSetSparseMatrix(instance, matrixIndex, (int *)rowIndices, (int *)colIndices, (double *)values, numNonZeros); + + env->ReleaseIntArrayElements(inRowIndices, rowIndices, JNI_ABORT); + env->ReleaseIntArrayElements(inColIndices, colIndices, JNI_ABORT); + env->ReleaseDoubleArrayElements(inValues, values, JNI_ABORT); + + return errCode; +} + /* * Class: beagle_BeagleJNIWrapper * Method: setStateFrequencies diff --git a/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h b/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h index 385f53a5..b362f377 100644 --- a/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h +++ b/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h @@ -143,6 +143,14 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getLogScaleFactors JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setEigenDecomposition (JNIEnv *, jobject, jint, jint, jdoubleArray, jdoubleArray, jdoubleArray); +/* + * Class: beagle_BeagleJNIWrapper + * Method: setSparseMatrix + * Signature: (II[I[I[DI)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setSparseMatrix + (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jdoubleArray, jint); + /* * Class: beagle_BeagleJNIWrapper * Method: setStateFrequencies @@ -213,7 +221,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_convolveTransitionMatrices * Signature: (I[I[I[II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_addTransitionMatrices - (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -370,7 +378,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateEdgeDifferentials /* * Class: beagle_BeagleJNIWrapper * Method: calculateCrossProductDifferentials - * Signature: (I[I[I[I[II[D[D[D)I + * Signature: (I[I[I[I[I[DI[D[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateCrossProductDifferentials (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jdoubleArray, jint, jdoubleArray, jdoubleArray); From fc7d00efe7ffe0b32b6c353e2ee3d2dbd6f704e8 Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 14 Jun 2024 21:29:31 -0500 Subject: [PATCH 139/421] updateTransitionMatrices return Success --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index cbfd10af..5df7e654 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1470,7 +1470,7 @@ int BeagleGPUActionImpl::updateTransitionMatrices(int eigenI } } // TODO: check if need to copy it from host to device afterwards - return BEAGLE_ERROR_NO_IMPLEMENTATION; + return BEAGLE_SUCCESS; } ///-------------------------------- Factory -------------------------------------/// From 24c8bd81a8ff243a43158b0bef1c4e9b0272aaf4 Mon Sep 17 00:00:00 2001 From: xji3 Date: Sat, 15 Jun 2024 11:44:01 -0500 Subject: [PATCH 140/421] free eigen device buffers only when used --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 86a69aef..3a7c05c2 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -137,9 +137,11 @@ BeagleGPUImpl::~BeagleGPUImpl() { // TODO: free subpointers gpu->FreeMemory(dMatrices[0]); // TODO Should try: gpu->FreeMemory(dMatricesOrigin); instead of above - gpu->FreeMemory(dEigenValues[0]); // TODO Here is where my Mac / Intel-GPU are throwing bad-exception - gpu->FreeMemory(dEvec[0]); // TODO Should be save and then release just d*Origin? - gpu->FreeMemory(dIevc[0]); + if(dEigenValues) { + gpu->FreeMemory(dEigenValues[0]); // TODO Here is where my Mac / Intel-GPU are throwing bad-exception + gpu->FreeMemory(dEvec[0]); // TODO Should be save and then release just d*Origin? + gpu->FreeMemory(dIevc[0]); + } gpu->FreeMemory(dWeights[0]); gpu->FreeMemory(dFrequencies[0]); From e35d8c5af671a473423cfa5d2e603cb1f213dadd Mon Sep 17 00:00:00 2001 From: xji3 Date: Sat, 15 Jun 2024 11:44:47 -0500 Subject: [PATCH 141/421] Ben's code magically solved memory issue --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 41 ++---------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5df7e654..466ed21b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -420,11 +420,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, BEAGLE_GPU_TEMPLATE BeagleGPUActionImpl::~BeagleGPUActionImpl() { -// BeagleGPUImpl::~BeagleGPUImpl<>(); for (int i = 0; i < dPartialsWrapper.size(); i++) { cusparseDestroyDnMat(dPartialsWrapper[i]); - cudaFree(dPartialCache[i]); } for (int i = 0; i < kCategoryCount; i++) { cusparseDestroyDnMat(dFLeft[i]); @@ -1239,43 +1237,8 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i for (int category = 0; category < kCategoryCount; category++) { -// spMM(cusparseHandle, dIntegrationTmpLeft[category], alphaCache[category], ALeft, dPartialsWrapper[destPIndex1], 0, dIntegrationLeftBuffer[category], integrationLeftBufferSize[category]); -// spMM(cusparseHandle, dIntegrationTmpRight[category], alphaCache[kCategoryCount + category], ARight, dPartialsWrapper[destPIndex2], 0, dIntegrationRightBuffer[category], integrationRightBufferSize[category]); - - CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, - CUSPARSE_OPERATION_NON_TRANSPOSE, - &alphaCache[category], dAs[hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex1, category)], &zero, - dIntegrationTmpLeft[category], DataType, - CUSPARSE_SPMM_ALG_DEFAULT, &integrationLeftBufferSize[category])) - - CHECK_CUSPARSE(cusparseSpMM_bufferSize(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, - CUSPARSE_OPERATION_NON_TRANSPOSE, - &alphaCache[kCategoryCount + category], dAs[hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex2, category)], &zero, - dIntegrationTmpRight[category], DataType, - CUSPARSE_SPMM_ALG_DEFAULT, &integrationRightBufferSize[kCategoryCount + category])) - if (integrationLeftBufferSize[category] > integrationLeftStoredBufferSize[category] && integrationMultipliers[category]) { - CHECK_CUDA(cudaMalloc(&dIntegrationLeftBuffer[category], - integrationLeftBufferSize[category])) // TODO: is this necessary? Are there better ways to claim additional buffer? - integrationLeftStoredBufferSize[category] = integrationLeftBufferSize[category]; - } - if (integrationRightBufferSize[category] > integrationRightStoredBufferSize[category] && integrationMultipliers[kCategoryCount + category]) { - CHECK_CUDA(cudaMalloc(&dIntegrationRightBuffer[category], - integrationRightBufferSize[category])) // TODO: is this necessary? Are there better ways to claim additional buffer? - integrationRightStoredBufferSize[category] = integrationRightBufferSize[category]; - - } - - // integrationTmp = alpha * A * destP - CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, - CUSPARSE_OPERATION_NON_TRANSPOSE, - &alphaCache[category], dAs[hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex1, category)], &zero, - dIntegrationTmpLeft[category], DataType, - CUSPARSE_SPMM_ALG_DEFAULT, dIntegrationLeftBuffer[category])) - CHECK_CUSPARSE(cusparseSpMM(cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, - CUSPARSE_OPERATION_NON_TRANSPOSE, - &alphaCache[kCategoryCount + category], dAs[hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex2, category)], &zero, - dIntegrationTmpRight[category], DataType, - CUSPARSE_SPMM_ALG_DEFAULT, dIntegrationRightBuffer[category])) + spMM(cusparseHandle, dIntegrationTmpLeft[category], alphaCache[category], dAs[hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex1, category)], 0, dIntegrationLeftBuffer[category], integrationLeftBufferSize[category]); + spMM(cusparseHandle, dIntegrationTmpRight[category], alphaCache[kCategoryCount + category], dAs[hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex2, category)], 0, dIntegrationRightBuffer[category], integrationRightBufferSize[category]); } cudaDeviceSynchronize(); From 05b39b6d77f5f73ee69507aa10a90f6aff664831 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 18 Jun 2024 08:59:27 -0400 Subject: [PATCH 142/421] Refactor dense matrix code somewhat. - Free dFLeft/dFRight memory in one block, not per-category. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 68 ++++++++++++++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 93 ++++++++++++------------ 2 files changed, 109 insertions(+), 52 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index f223b9db..13f95c8a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -124,6 +124,68 @@ using DnMatrix = Eigen::Matrix; template using DnVector = Eigen::Matrix; +template cusparseIndexType_t IndexType; +template <> cusparseIndexType_t IndexType = CUSPARSE_INDEX_32I; +template <> cusparseIndexType_t IndexType = CUSPARSE_INDEX_64I; + +template cudaDataType DataType; +template <> cudaDataType DataType = CUDA_R_32F; +template <> cudaDataType DataType = CUDA_R_64F; + +template +struct DnMatrixDevice +{ + cusparseDnMatDescr_t descr = nullptr; + Real* ptr = nullptr; + size_t size1 = 0; + size_t size2 = 0; + cusparseOrder_t order = CUSPARSE_ORDER_COL; + size_t size() const {return size1*size2;} + size_t byte_size() const {return size()*sizeof(Real);} + + // Disallow copying -- only one object can "own" the descriptor. + DnMatrixDevice& operator=(const DnMatrixDevice&) = delete; + // Allow moving. + DnMatrixDevice& operator=(DnMatrixDevice&& D) noexcept + { + std::swap(descr, D.descr); + std::swap(ptr, D.ptr); + std::swap(size1, D.size1); + std::swap(size2, D.size2); + std::swap(order, D.order); + return *this; + } + + // Disallow copying. + DnMatrixDevice(const DnMatrixDevice&) = delete; + // Allow moving. + DnMatrixDevice(DnMatrixDevice&& D) noexcept + { + operator=(std::move(D)); + } + + DnMatrixDevice(Real* p, size_t s1, size_t s2, cusparseOrder_t o = CUSPARSE_ORDER_COL) + :ptr(p), size1(s1), size2(s2), order(o) + { + auto status = (order == CUSPARSE_ORDER_COL) + ? cusparseCreateDnMat(&descr, size1, size2, size1, ptr, DataType, CUSPARSE_ORDER_COL) + : cusparseCreateDnMat(&descr, size1, size2, size2, ptr, DataType, CUSPARSE_ORDER_ROW); + + if (status != CUSPARSE_STATUS_SUCCESS) + { + std::cerr<<"cusparseCreateDnMat( ) failed!"; + std::exit(1); + } + } + + ~DnMatrixDevice() + { + // This class does not own the memory, so does not call cudaFree. + if (descr) cusparseDestroyDnMat(descr); + } +}; + + //template //using MapType = DnMatrix; @@ -229,14 +291,12 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // std::vector dInstantaneousMatrices; std::vector dPartialsWrapper; - std::vector dFLeft; - std::vector dFRight; + std::vector> dFLeft; + std::vector> dFRight; std::vector dIntegrationTmpLeft; std::vector dIntegrationTmpRight; std::vector dAs; std::vector dPartialCache; - std::vector dFLeftCache; - std::vector dFRightCache; std::vector dIntegrationTmpLeftCache; std::vector dIntegrationTmpRightCache; std::vector integrationLeftBufferSize; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 466ed21b..57d7a78e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -363,11 +363,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, DataType, CUSPARSE_ORDER_COL)) } - dFLeft = std::vector(kCategoryCount, nullptr); - dFLeftCache = std::vector(kCategoryCount, nullptr); Real* dLeftCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - dFRight = std::vector(kCategoryCount, nullptr); - dFRightCache = std::vector(kCategoryCount, nullptr); Real* dRightCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); dIntegrationTmpLeft = std::vector(kCategoryCount, nullptr); @@ -383,19 +379,24 @@ int BeagleGPUActionImpl::createInstance(int tipCount, integrationRightStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); - for (int category = 0; category < kCategoryCount; category++) { - dFLeftCache[category] = dLeftCachePtr + kPaddedStateCount * kPaddedPatternCount * category; - dFRightCache[category] = dRightCachePtr + kPaddedStateCount * kPaddedPatternCount * category; + dFLeft.clear(); + + dFRight.clear(); + + for (int category = 0; category < kCategoryCount; category++) + { + Real* leftPtr = dLeftCachePtr + kPaddedStateCount * kPaddedPatternCount * category; + dFLeft.emplace_back(leftPtr, kPaddedStateCount, kPaddedPatternCount); + + Real* rightPtr = dRightCachePtr + kPaddedStateCount * kPaddedPatternCount * category; + dFRight.emplace_back(rightPtr, kPaddedStateCount, kPaddedPatternCount); + dIntegrationTmpLeftCache[category] = dIntegrationTmpLeftCachePtr + kPaddedStateCount * kPaddedPatternCount * category; dIntegrationTmpRightCache[category] = dIntegrationTmpRightCachePtr + kPaddedStateCount * kPaddedPatternCount * category; } for (int category = 0; category < kCategoryCount; category++) { - CHECK_CUSPARSE(cusparseCreateDnMat(&dFLeft[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dFLeftCache[category], - DataType, CUSPARSE_ORDER_COL)) - CHECK_CUSPARSE(cusparseCreateDnMat(&dFRight[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dFRightCache[category], - DataType, CUSPARSE_ORDER_COL)) CHECK_CUSPARSE(cusparseCreateDnMat(&dIntegrationTmpLeft[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dIntegrationTmpLeftCache[category], DataType, CUSPARSE_ORDER_COL)) CHECK_CUSPARSE(cusparseCreateDnMat(&dIntegrationTmpRight[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dIntegrationTmpRightCache[category], @@ -425,17 +426,15 @@ BeagleGPUActionImpl::~BeagleGPUActionImpl() cusparseDestroyDnMat(dPartialsWrapper[i]); } for (int i = 0; i < kCategoryCount; i++) { - cusparseDestroyDnMat(dFLeft[i]); - cusparseDestroyDnMat(dFRight[i]); cusparseDestroyDnMat(dIntegrationTmpLeft[i]); cusparseDestroyDnMat(dIntegrationTmpRight[i]); - cudaFree(dFLeftCache[i]); - cudaFree(dFRightCache[i]); cudaFree(dIntegrationTmpLeftCache[i]); cudaFree(dIntegrationTmpRightCache[i]); cudaFree(dIntegrationLeftBuffer[i]); cudaFree(dIntegrationRightBuffer[i]); } + cudaFree(dFLeft[0].ptr); + cudaFree(dFRight[0].ptr); for (int i = 0; i < kEigenDecompCount; i++) { cudaFree(dBsCsrOffsetsCache[i]); @@ -1016,27 +1015,25 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // SpMatrix A = hBs[hEigenMaps[edgeIndex]] * edgeMultiplier; cusparseSpMatDescr_t A = dAs[matrixIndex]; - vector F; + vector>* FF; vector integrationTmp; - std::vector FCache; std::vector integrationCache; std::vector integrationBuffer; std::vector integrationBufferSize; if (left) { - F = dFLeft; + FF = &dFLeft; integrationTmp = dIntegrationTmpLeft; - FCache = dFLeftCache; integrationCache = dIntegrationTmpLeftCache; integrationBuffer = dIntegrationLeftBuffer; integrationBufferSize = integrationLeftBufferSize; } else { - F = dFRight; + FF = &dFRight; integrationTmp = dIntegrationTmpRight; - FCache = dFRightCache; integrationCache = dIntegrationTmpRightCache; integrationBuffer = dIntegrationRightBuffer; integrationBufferSize = integrationRightBufferSize; } + auto& F = *FF; //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"Before destP = partials operation, destPCache:\n"<::simpleAction2(int destPIndex, int p // MatrixXd F(kStateCount, nCol); // F = destP; - MemcpyDeviceToDevice(FCache[category], dPartialCache[partialsIndex], kPaddedStateCount * kPaddedPatternCount); + MemcpyDeviceToDevice(F[category].ptr, dPartialCache[partialsIndex], kPaddedStateCount * kPaddedPatternCount); // CHECK_CUSPARSE(cusparseDnMatSetValues(F[category], dPartialCache[partialsIndex])) // TODO: loop category within this function //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"F = partials operation, FCache:"<::simpleAction2(int destPIndex, int p double c2 = normPInf(integrationCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); // F += destP; if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); + CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, F[category].ptr, 1)); } else { - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, FCache[category], 1)); + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationCache[category], 1, F[category].ptr, 1)); } //#ifdef BEAGLE_DEBUG_FLOW // // std::cerr<<"F += destP, c1 = " <::simpleAction3(int partialsIndex1, i CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex1, 0)], dPartialCache[getPartialIndex(partialsIndex1, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex2, 0)], dPartialCache[getPartialIndex(partialsIndex2, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) // F = destP; - CHECK_CUDA(cudaMemcpyAsync(dFLeftCache[0], dPartialCache[getPartialIndex(partialsIndex1, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpyAsync(dFRightCache[0], dPartialCache[getPartialIndex(partialsIndex2, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dFLeft[0].ptr, dPartialCache[getPartialIndex(partialsIndex1, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dFRight[0].ptr, dPartialCache[getPartialIndex(partialsIndex2, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) cudaDeviceSynchronize(); @@ -1189,7 +1186,7 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i std::cerr<<"destP:" <(msCache[i])<<", "<(msCache[i])<<"), "; @@ -1255,11 +1252,11 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i // F += destP = IntegrationTmp; if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache[0], 1, dFLeftCache[0], 1)); - CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache[0], 1, dFRightCache[0], 1)); + CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache[0], 1, dFLeft[0].ptr, 1)); + CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache[0], 1, dFRight[0].ptr, 1)); } else { - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache[0], 1, dFLeftCache[0], 1)); - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache[0], 1, dFRightCache[0], 1)); + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeftCache[0], 1, dFLeft[0].ptr, 1)); + CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRightCache[0], 1, dFRight[0].ptr, 1)); } cudaDeviceSynchronize(); @@ -1267,10 +1264,10 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i for (int category = 0; category < kCategoryCount; category++) { c2Cache[category] = normPInf(dIntegrationTmpLeftCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); c2Cache[kCategoryCount + category] = normPInf(dIntegrationTmpRightCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle); - if (c1Cache[category] + c2Cache[category] < tol * normPInf(dFLeftCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { + if (c1Cache[category] + c2Cache[category] < tol * normPInf(dFLeft[category].ptr, kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { integrationMultipliers[category] = 0; } - if (c1Cache[kCategoryCount + category] + c2Cache[kCategoryCount + category] < tol * normPInf(dFRightCache[category], kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { + if (c1Cache[kCategoryCount + category] + c2Cache[kCategoryCount + category] < tol * normPInf(dFRight[category].ptr, kPaddedStateCount, kPaddedPatternCount, cublasHandle)) { integrationMultipliers[kCategoryCount + category] = 0; } @@ -1280,9 +1277,9 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"dFLeftCache = " <::simpleAction3(int partialsIndex1, i // F *= eta; for (int category = 0; category < kCategoryCount; category++) { if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[category], dFLeftCache[category], 1)); - CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[kCategoryCount + category], dFRightCache[category], 1)); + CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[category], dFLeft[category].ptr, 1)); + CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[kCategoryCount + category], dFRight[category].ptr, 1)); } else { - CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[category], dFLeftCache[category], 1)); - CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[kCategoryCount + category], dFRightCache[category], 1)); + CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[category], dFLeft[category].ptr, 1)); + CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[kCategoryCount + category], dFRight[category].ptr, 1)); } } cudaDeviceSynchronize(); // destP = F; - CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex1, 0)], dFLeftCache[0], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex2, 0)], dFRightCache[0], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex1, 0)], dFLeft[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex2, 0)], dFRight[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) cudaDeviceSynchronize(); } From f7c71bd0df91e95439df4acd0fbca1f170660288 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 18 Jun 2024 09:23:43 -0400 Subject: [PATCH 143/421] Convert dIntegrationTmpLeft and dIntegrationTmpRight --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 6 +- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 74 +++++++++++------------- 2 files changed, 35 insertions(+), 45 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 13f95c8a..8721da06 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -293,12 +293,10 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector dPartialsWrapper; std::vector> dFLeft; std::vector> dFRight; - std::vector dIntegrationTmpLeft; - std::vector dIntegrationTmpRight; + std::vector> dIntegrationTmpLeft; + std::vector> dIntegrationTmpRight; std::vector dAs; std::vector dPartialCache; - std::vector dIntegrationTmpLeftCache; - std::vector dIntegrationTmpRightCache; std::vector integrationLeftBufferSize; std::vector integrationLeftStoredBufferSize; std::vector dIntegrationLeftBuffer; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 57d7a78e..1427d590 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -366,12 +366,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, Real* dLeftCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); Real* dRightCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - dIntegrationTmpLeft = std::vector(kCategoryCount, nullptr); - dIntegrationTmpLeftCache = std::vector(kCategoryCount, nullptr); Real* dIntegrationTmpLeftCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - dIntegrationTmpRight = std::vector(kCategoryCount, nullptr); - dIntegrationTmpRightCache = std::vector(kCategoryCount, nullptr); Real* dIntegrationTmpRightCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + integrationLeftBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); integrationLeftStoredBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationLeftBuffer = std::vector(kCategoryCount, nullptr); @@ -383,24 +380,23 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dFRight.clear(); + dIntegrationTmpLeft.clear(); + + dIntegrationTmpRight.clear(); + for (int category = 0; category < kCategoryCount; category++) { - Real* leftPtr = dLeftCachePtr + kPaddedStateCount * kPaddedPatternCount * category; - dFLeft.emplace_back(leftPtr, kPaddedStateCount, kPaddedPatternCount); + size_t offset = kPaddedStateCount * kPaddedPatternCount * category; - Real* rightPtr = dRightCachePtr + kPaddedStateCount * kPaddedPatternCount * category; - dFRight.emplace_back(rightPtr, kPaddedStateCount, kPaddedPatternCount); + dFLeft.emplace_back(dLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); + dFRight.emplace_back(dRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); - dIntegrationTmpLeftCache[category] = dIntegrationTmpLeftCachePtr + kPaddedStateCount * kPaddedPatternCount * category; - dIntegrationTmpRightCache[category] = dIntegrationTmpRightCachePtr + kPaddedStateCount * kPaddedPatternCount * category; + dIntegrationTmpLeft.emplace_back(dIntegrationTmpLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); + dIntegrationTmpRight.emplace_back(dIntegrationTmpRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); } for (int category = 0; category < kCategoryCount; category++) { - CHECK_CUSPARSE(cusparseCreateDnMat(&dIntegrationTmpLeft[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dIntegrationTmpLeftCache[category], - DataType, CUSPARSE_ORDER_COL)) - CHECK_CUSPARSE(cusparseCreateDnMat(&dIntegrationTmpRight[category], kPaddedStateCount, kPaddedPatternCount, kPaddedStateCount, dIntegrationTmpRightCache[category], - DataType, CUSPARSE_ORDER_COL)) CHECK_CUDA(cudaMalloc(&dIntegrationLeftBuffer[category], integrationLeftBufferSize[category])) CHECK_CUDA(cudaMalloc(&dIntegrationRightBuffer[category], integrationRightBufferSize[category])) } @@ -426,15 +422,13 @@ BeagleGPUActionImpl::~BeagleGPUActionImpl() cusparseDestroyDnMat(dPartialsWrapper[i]); } for (int i = 0; i < kCategoryCount; i++) { - cusparseDestroyDnMat(dIntegrationTmpLeft[i]); - cusparseDestroyDnMat(dIntegrationTmpRight[i]); - cudaFree(dIntegrationTmpLeftCache[i]); - cudaFree(dIntegrationTmpRightCache[i]); cudaFree(dIntegrationLeftBuffer[i]); cudaFree(dIntegrationRightBuffer[i]); } cudaFree(dFLeft[0].ptr); cudaFree(dFRight[0].ptr); + cudaFree(dIntegrationTmpLeft[0].ptr); + cudaFree(dIntegrationTmpRight[0].ptr); for (int i = 0; i < kEigenDecompCount; i++) { cudaFree(dBsCsrOffsetsCache[i]); @@ -1016,24 +1010,22 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // SpMatrix A = hBs[hEigenMaps[edgeIndex]] * edgeMultiplier; cusparseSpMatDescr_t A = dAs[matrixIndex]; vector>* FF; - vector integrationTmp; - std::vector integrationCache; + vector>* integrationTmpPtr; std::vector integrationBuffer; std::vector integrationBufferSize; if (left) { FF = &dFLeft; - integrationTmp = dIntegrationTmpLeft; - integrationCache = dIntegrationTmpLeftCache; + integrationTmpPtr = &dIntegrationTmpLeft; integrationBuffer = dIntegrationLeftBuffer; integrationBufferSize = integrationLeftBufferSize; } else { FF = &dFRight; - integrationTmp = dIntegrationTmpRight; - integrationCache = dIntegrationTmpRightCache; + integrationTmpPtr = &dIntegrationTmpRight; integrationBuffer = dIntegrationRightBuffer; integrationBufferSize = integrationRightBufferSize; } auto& F = *FF; + auto& integrationTmp = *integrationTmpPtr; //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"Before destP = partials operation, destPCache:\n"<::simpleAction2(int destPIndex, int p // std::cerr<<"j/m = "<::simpleAction2(int destPIndex, int p for (int i = 0; i < s; i++) { //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"dPartials:"<::simpleAction2(int destPIndex, int p // PrintfDeviceVector(F[category].ptr, kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); //#endif // destP = F; - MemcpyDeviceToDevice(dPartialCache[destPIndex], F[category].ptr, kPaddedStateCount * kPaddedPatternCount); + MemcpyDeviceToDevice(dPartialsWrapper[destPIndex].ptr, F[category].ptr, kPaddedStateCount * kPaddedPatternCount); #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"destP = F:"<::simpleAction3(int partialsIndex1, i // destP = partials; - CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex1, 0)], dPartialCache[getPartialIndex(partialsIndex1, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex2, 0)], dPartialCache[getPartialIndex(partialsIndex2, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialsWrapper[getPartialCacheIndex(partialsIndex1, 0)].ptr, dPartialsWrapper[getPartialIndex(partialsIndex1, 0)].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialsWrapper[getPartialCacheIndex(partialsIndex2, 0)].ptr, dPartialsWrapper[getPartialIndex(partialsIndex2, 0)].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) // F = destP; - CHECK_CUDA(cudaMemcpyAsync(dFLeft[0].ptr, dPartialCache[getPartialIndex(partialsIndex1, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpyAsync(dFRight[0].ptr, dPartialCache[getPartialIndex(partialsIndex2, 0)], sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dFLeft[0].ptr, dPartialsWrapper[getPartialIndex(partialsIndex1, 0)].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dFRight[0].ptr, dPartialsWrapper[getPartialIndex(partialsIndex2, 0)].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) cudaDeviceSynchronize(); #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"P:" <::simpleAction3(int partialsIndex1, i for (int i = 0; i < s_max; i++) { for (int category = 0; category < kCategoryCount; category++) { - const Real c1Left = normPInf(dPartialCache[getPartialIndex(partialsIndex1, category)], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + const Real c1Left = normPInf(dPartialsWrapper[getPartialIndex(partialsIndex1, category)].ptr, kPaddedStateCount, kPaddedPatternCount, cublasHandle); c1Cache[category] = c1Left; - const Real c1Right = normPInf(dPartialCache[getPartialIndex(partialsIndex2, category)], kPaddedStateCount, kPaddedPatternCount, cublasHandle); + const Real c1Right = normPInf(dPartialsWrapper[getPartialIndex(partialsIndex2, category)].ptr, kPaddedStateCount, kPaddedPatternCount, cublasHandle); c1Cache[kCategoryCount + category] = c1Right; etaCache[category] = (i < std::get<1>(msCache[category])) * (etaCache[category] - 1) + 1; @@ -1226,8 +1227,8 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i for (int category = 0; category < kCategoryCount; category++) { - spMM(cusparseHandle, dIntegrationTmpLeft[category].descr, alphaCache[category], dAs[hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex1, category)], 0, dIntegrationLeftBuffer[category], integrationLeftBufferSize[category]); - spMM(cusparseHandle, dIntegrationTmpRight[category].descr, alphaCache[kCategoryCount + category], dAs[hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex2, category)], 0, dIntegrationRightBuffer[category], integrationRightBufferSize[category]); + spMM(cusparseHandle, dIntegrationTmpLeft[category], alphaCache[category], dAs[hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex1, category)], 0, dIntegrationLeftBuffer[category], integrationLeftBufferSize[category]); + spMM(cusparseHandle, dIntegrationTmpRight[category], alphaCache[kCategoryCount + category], dAs[hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex2, category)], 0, dIntegrationRightBuffer[category], integrationRightBufferSize[category]); } cudaDeviceSynchronize(); @@ -1239,8 +1240,8 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i #endif // destP = IntegrationTmp - CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex1, 0)], dIntegrationTmpLeft[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex2, 0)], dIntegrationTmpRight[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialsWrapper[getPartialCacheIndex(partialsIndex1, 0)].ptr, dIntegrationTmpLeft[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialsWrapper[getPartialCacheIndex(partialsIndex2, 0)].ptr, dIntegrationTmpRight[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) // F += destP = IntegrationTmp; if constexpr (std::is_same::value) { @@ -1296,8 +1297,8 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i } cudaDeviceSynchronize(); // destP = F; - CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex1, 0)], dFLeft[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpyAsync(dPartialCache[getPartialCacheIndex(partialsIndex2, 0)], dFRight[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialsWrapper[getPartialCacheIndex(partialsIndex1, 0)].ptr, dFLeft[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + CHECK_CUDA(cudaMemcpyAsync(dPartialsWrapper[getPartialCacheIndex(partialsIndex2, 0)].ptr, dFRight[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) cudaDeviceSynchronize(); } From feaa314377236b71eb78be4cca1d6ddd47c04b89 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 18 Jun 2024 10:12:12 -0400 Subject: [PATCH 146/421] Put the cublas handle into the struct. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 6 ++++-- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 10 +++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index cb138ebb..5625e542 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -135,6 +135,8 @@ template <> cudaDataType DataType = CUDA_R_64F; template struct DnMatrixDevice { + cublasHandle_t cublas = nullptr; + // Lets assume any cusparseHandle_t comes from a sparse matrix. cusparseDnMatDescr_t descr = nullptr; Real* ptr = nullptr; size_t size1 = 0; @@ -165,8 +167,8 @@ struct DnMatrixDevice } DnMatrixDevice() = default; - DnMatrixDevice(Real* p, size_t s1, size_t s2, cusparseOrder_t o = CUSPARSE_ORDER_COL) - :ptr(p), size1(s1), size2(s2), order(o) + DnMatrixDevice(cublasHandle_t cb, Real* p, size_t s1, size_t s2, cusparseOrder_t o = CUSPARSE_ORDER_COL) + :cublas(cb), ptr(p), size1(s1), size2(s2), order(o) { auto status = (order == CUSPARSE_ORDER_COL) ? cusparseCreateDnMat(&descr, size1, size2, size1, ptr, DataType, CUSPARSE_ORDER_COL) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 980c0c2c..161e6378 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -353,7 +353,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, for (int i = 0; i < kPartialsBufferCount; i++) { for (int category = 0; category < kCategoryCount; category++) { auto ptr = (Real *) gpu->CreateSubPointer(dPartialsOrigin, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * (kCategoryCount * i + category), sizeof(Real) * kPaddedStateCount * kPaddedPatternCount); - auto mat = DnMatrixDevice(ptr, kPaddedStateCount, kPaddedPatternCount); + auto mat = DnMatrixDevice(cublasHandle, ptr, kPaddedStateCount, kPaddedPatternCount); dPartialsWrapper[i * kCategoryCount + category] = std::move(mat); } } @@ -383,11 +383,11 @@ int BeagleGPUActionImpl::createInstance(int tipCount, { size_t offset = kPaddedStateCount * kPaddedPatternCount * category; - dFLeft.emplace_back(dLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); - dFRight.emplace_back(dRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); + dFLeft.emplace_back(cublasHandle, dLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); + dFRight.emplace_back(cublasHandle, dRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); - dIntegrationTmpLeft.emplace_back(dIntegrationTmpLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); - dIntegrationTmpRight.emplace_back(dIntegrationTmpRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); + dIntegrationTmpLeft.emplace_back(cublasHandle, dIntegrationTmpLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); + dIntegrationTmpRight.emplace_back(cublasHandle, dIntegrationTmpRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); } for (int category = 0; category < kCategoryCount; category++) { From 9809121a2b1c1d3f66c9c7d707e356bcdaabcd78 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 18 Jun 2024 10:45:36 -0400 Subject: [PATCH 147/421] Fix copying of cublasHandle --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 5625e542..cd19716c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -135,7 +135,7 @@ template <> cudaDataType DataType = CUDA_R_64F; template struct DnMatrixDevice { - cublasHandle_t cublas = nullptr; + cublasHandle_t cublasHandle = nullptr; // Lets assume any cusparseHandle_t comes from a sparse matrix. cusparseDnMatDescr_t descr = nullptr; Real* ptr = nullptr; @@ -150,6 +150,7 @@ struct DnMatrixDevice // Allow moving. DnMatrixDevice& operator=(DnMatrixDevice&& D) noexcept { + std::swap(cublasHandle, D.cublasHandle); std::swap(descr, D.descr); std::swap(ptr, D.ptr); std::swap(size1, D.size1); @@ -168,7 +169,7 @@ struct DnMatrixDevice DnMatrixDevice() = default; DnMatrixDevice(cublasHandle_t cb, Real* p, size_t s1, size_t s2, cusparseOrder_t o = CUSPARSE_ORDER_COL) - :cublas(cb), ptr(p), size1(s1), size2(s2), order(o) + :cublasHandle(cb), ptr(p), size1(s1), size2(s2), order(o) { auto status = (order == CUSPARSE_ORDER_COL) ? cusparseCreateDnMat(&descr, size1, size2, size1, ptr, DataType, CUSPARSE_ORDER_COL) From d0b66c2587c2d69d2aab4f3bb0239e333b3f27fa Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 18 Jun 2024 10:48:20 -0400 Subject: [PATCH 148/421] Factor out code for axpy/+= --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 15 +++++++++++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 12 +++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index cd19716c..b0d143e6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -159,6 +159,21 @@ struct DnMatrixDevice return *this; } + DnMatrixDevice& operator+=(const DnMatrixDevice& D) + { + assert(D.size1 == size1); + assert(D.size2 == size2); + assert(D.cublasHandle == cublasHandle); + Real one = 1; + if constexpr (std::is_same::value) { + cublasSaxpy(cublasHandle, size(), &one, D.ptr, 1, ptr, 1); + } else { + cublasDaxpy(cublasHandle, size(), &one, D.ptr, 1, ptr, 1); + } + + return *this; + } + // Disallow copying. DnMatrixDevice(const DnMatrixDevice&) = delete; // Allow moving. diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 161e6378..b40e116b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1097,11 +1097,13 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p double c2 = normPInf(integrationTmp[category].ptr, kPaddedStateCount, kPaddedPatternCount, cublasHandle); // F += destP; - if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationTmp[category].ptr, 1, F[category].ptr, 1)); - } else { - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationTmp[category].ptr, 1, F[category].ptr, 1)); - } + + F[category] += integrationTmp[category]; +// if constexpr (std::is_same::value) { +// CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationTmp[category].ptr, 1, F[category].ptr, 1)); +// } else { +// CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &one, integrationTmp[category].ptr, 1, F[category].ptr, 1)); +// } //#ifdef BEAGLE_DEBUG_FLOW // // std::cerr<<"F += destP, c1 = " <::setPartials(tipIndex, inPartials); + for (int category = 0; category < kCategoryCount; category++) { + dPartialCache[getPartialIndex(tipIndex, category)] = (Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category; + } + + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setPartials\n"); +#endif + + return BEAGLE_SUCCESS; +} + + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::updatePartials(const int* operations, From 952ea6b58a8a664e52096cf2079702300a2c9e1d Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 18 Jun 2024 11:10:13 -0500 Subject: [PATCH 153/421] sad to see pointers being killed --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 99e50465..0ad429ea 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -510,7 +510,7 @@ int BeagleGPUActionImpl::setPartials(int tipIndex, const Rea BeagleGPUImpl::setPartials(tipIndex, inPartials); for (int category = 0; category < kCategoryCount; category++) { - dPartialCache[getPartialIndex(tipIndex, category)] = (Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category; + dPartialsWrapper[getPartialIndex(tipIndex, category)].ptr = (Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category; } From b0e6e87ee57b259033ccfcfb15ead8fb5f85d92b Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 18 Jun 2024 17:25:38 -0400 Subject: [PATCH 154/421] Fix typo --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 0ad429ea..ba793c5d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1253,7 +1253,7 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"integrationTmp = alpha * A * destP = " < Date: Tue, 18 Jun 2024 17:25:57 -0400 Subject: [PATCH 155/421] Mark descr as being dead. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index f73ff9fd..fd6b1611 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -197,7 +197,11 @@ struct DnMatrixDevice ~DnMatrixDevice() { // This class does not own the memory, so does not call cudaFree. - if (descr) cusparseDestroyDnMat(descr); + if (descr) + { + cusparseDestroyDnMat(descr); + descr = nullptr; + } } }; From 493d88423f302055ce97d215c4de98486a768be3 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 18 Jun 2024 17:29:22 -0400 Subject: [PATCH 156/421] Make dAs into class SpMatrixDevice --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 76 +++++++++++++++++++++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 49 +++++++-------- 2 files changed, 97 insertions(+), 28 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index fd6b1611..d3ec8791 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -205,6 +205,80 @@ struct DnMatrixDevice } }; +enum class sparseFormat { none, csr, csc }; + +template +struct SpMatrixDevice +{ + cusparseHandle_t cusparseHandle = nullptr; + cusparseSpMatDescr_t descr = nullptr; + int size1 = 0; + int size2 = 0; + int num_non_zeros = 0; + Real* values = nullptr; + int* columns = nullptr; + int* offsets = nullptr; + sparseFormat format = sparseFormat::none; + + // Disallow copying -- only one object can "own" the descriptor. + SpMatrixDevice& operator=(const SpMatrixDevice&) = delete; + // Allow moving. + SpMatrixDevice& operator=(SpMatrixDevice&& D) noexcept + { + std::swap(cusparseHandle, D.cusparseHandle); + std::swap(descr, D.descr); + std::swap(size1, D.size1); + std::swap(size2, D.size2); + std::swap(num_non_zeros, D.num_non_zeros); + std::swap(values, D.values); + std::swap(columns, D.columns); + std::swap(offsets, D.offsets); + std::swap(format, D.format); + return *this; + } + + // Disallow copying. + SpMatrixDevice(const SpMatrixDevice&) = delete; + // Allow moving. + SpMatrixDevice(SpMatrixDevice&& D) noexcept + { + operator=(std::move(D)); + } + + SpMatrixDevice() = default; + SpMatrixDevice(cusparseHandle_t h, int s1, int s2, int n, Real* v, int* c, int* o, sparseFormat f) + :cusparseHandle(h), size1(s1), size2(s2), num_non_zeros(n), values(v), columns(c), offsets(o), format(f) + { + cusparseStatus_t status; + if (format == sparseFormat::csc) + { + status = cusparseCreateCsc(&descr, s1, s2, num_non_zeros, offsets, columns, values, + IndexType, IndexType, CUSPARSE_INDEX_BASE_ZERO, DataType); + } + else + { + status = cusparseCreateCsr(&descr, s1, s2, num_non_zeros, offsets, columns, values, + IndexType, IndexType, CUSPARSE_INDEX_BASE_ZERO, DataType); + } + + if (status != CUSPARSE_STATUS_SUCCESS) + { + std::cerr<<"cusparseCreateSpMat( ) failed!"; + std::exit(1); + } + } + + ~SpMatrixDevice() + { + if (descr) + { + cusparseDestroySpMat(descr); + descr = nullptr; + } + } +}; + + template auto normPInf(const DnMatrixDevice& M) { @@ -324,7 +398,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector> dFRight; std::vector> dIntegrationTmpLeft; std::vector> dIntegrationTmpRight; - std::vector dAs; + std::vector> dAs; std::vector integrationLeftBufferSize; std::vector integrationLeftStoredBufferSize; std::vector dIntegrationLeftBuffer; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index ba793c5d..a949bb04 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1,4 +1,3 @@ - /* * BeagleGPUImpl.cpp * BEAGLE @@ -336,7 +335,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dBsCsrColumnsCache.resize(kEigenDecompCount); dBsCsrValuesCache.resize(kEigenDecompCount); dACscValuesCache.resize(kEigenDecompCount * kCategoryCount * 2); - dAs = std::vector(kEigenDecompCount * kCategoryCount * 2, nullptr); + dAs = std::vector>(kEigenDecompCount * kCategoryCount * 2); currentCacheNNZs = std::vector(kEigenDecompCount, kPaddedStateCount); for (int i = 0; i < kEigenDecompCount; i++) { dBsCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); @@ -959,25 +958,21 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int PrintfDeviceVector(dACscValuesCache[matrixIndex1], currentCacheNNZs[hEigenMaps[edgeIndex1]], -1, 0, 0); #endif - if (transpose) { - CHECK_CUSPARSE(cusparseCreateCsc(&dAs[matrixIndex1], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex1]], - dBsCsrOffsetsCache[hEigenMaps[edgeIndex1]], dBsCsrColumnsCache[hEigenMaps[edgeIndex1]], dACscValuesCache[matrixIndex1], - IndexType, IndexType, - CUSPARSE_INDEX_BASE_ZERO, DataType)) - CHECK_CUSPARSE(cusparseCreateCsc(&dAs[matrixIndex2], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex2]], - dBsCsrOffsetsCache[hEigenMaps[edgeIndex2]], dBsCsrColumnsCache[hEigenMaps[edgeIndex2]], dACscValuesCache[matrixIndex2], - IndexType, IndexType, - CUSPARSE_INDEX_BASE_ZERO, DataType)) - } else { - CHECK_CUSPARSE(cusparseCreateCsr(&dAs[matrixIndex1], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex1]], - dBsCsrOffsetsCache[hEigenMaps[edgeIndex1]], dBsCsrColumnsCache[hEigenMaps[edgeIndex1]], dACscValuesCache[matrixIndex1], - IndexType, IndexType, - CUSPARSE_INDEX_BASE_ZERO, DataType)) - CHECK_CUSPARSE(cusparseCreateCsr(&dAs[matrixIndex2], kPaddedStateCount, kPaddedStateCount, currentCacheNNZs[hEigenMaps[edgeIndex2]], - dBsCsrOffsetsCache[hEigenMaps[edgeIndex2]], dBsCsrColumnsCache[hEigenMaps[edgeIndex2]], dACscValuesCache[matrixIndex2], - IndexType, IndexType, - CUSPARSE_INDEX_BASE_ZERO, DataType)) - } + auto format = transpose ? sparseFormat::csc : sparseFormat::csr; + + dAs[matrixIndex1] = SpMatrixDevice(cusparseHandle, kPaddedStateCount, kPaddedStateCount, + currentCacheNNZs[hEigenMaps[edgeIndex1]], + dACscValuesCache[matrixIndex1], + dBsCsrColumnsCache[hEigenMaps[edgeIndex1]], + dBsCsrOffsetsCache[hEigenMaps[edgeIndex1]], + format); + + dAs[matrixIndex2] = SpMatrixDevice(cusparseHandle, kPaddedStateCount, kPaddedStateCount, + currentCacheNNZs[hEigenMaps[edgeIndex2]], + dACscValuesCache[matrixIndex2], + dBsCsrColumnsCache[hEigenMaps[edgeIndex2]], + dBsCsrOffsetsCache[hEigenMaps[edgeIndex2]], + format); } cudaDeviceSynchronize(); return BEAGLE_SUCCESS; @@ -1006,9 +1001,9 @@ int spMM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSp } template -int spMM(cusparseHandle_t handle, DnMatrixDevice& C, Real alpha, cusparseSpMatDescr_t A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) +int spMM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) { - return spMM(handle, C.descr, alpha, A, B.descr, beta, buffer, buffersize); + return spMM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); } BEAGLE_GPU_TEMPLATE @@ -1029,7 +1024,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p #endif // SpMatrix A = hBs[hEigenMaps[edgeIndex]] * edgeMultiplier; - cusparseSpMatDescr_t A = dAs[matrixIndex]; + auto& A = dAs[matrixIndex]; vector>* FF; vector>* integrationTmpPtr; std::vector integrationBuffer; @@ -1095,7 +1090,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // std::cerr<<"j/m = "<Device): failed!"); } +template +void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) +{ + auto status = cudaMemcpy(hptr, dptr, n*sizeof(T), cudaMemcpyDeviceToHost); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); +} +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::createInstance(int tipCount, int partialsBufferCount, From 6089328d4eedbd28f3a1eebff174f9522fd41a5d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 19 Jun 2024 12:17:58 -0400 Subject: [PATCH 160/421] Add routine to print dense matrices. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index a4839233..88434eeb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -291,6 +291,98 @@ void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) if (status != cudaSuccess) throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); } + +// Create an io-manipulator so that we can write std::cerr< +class byRow +{ + const DnMatrixDevice& D; +public: + + friend std::ostream& operator<<(std::ostream& o, const byRow& pr) + { + auto& D = pr.D; + Real* hPtr = new Real[D.size()]; + + MemcpyDeviceToHost(hPtr, D.ptr, D.size()); + + o<<"Rows[ "; + for(int row=0;row& d):D(d) {} +}; + +// Create an io-manipulator so that we can write std::cerr< +class byCol +{ + const DnMatrixDevice& D; +public: + + friend std::ostream& operator<<(std::ostream& o, const byCol& pr) + { + auto& D = pr.D; + + Real* hPtr = new Real[D.size()]; + + MemcpyDeviceToHost(hPtr, D.ptr, D.size()); + + o<<"Cols[ "; + for(int col=0;col& d):D(d) {} +}; + +template +std::ostream& operator<<(std::ostream& o, const DnMatrixDevice& D) +{ + return o< Date: Wed, 19 Jun 2024 15:09:58 -0400 Subject: [PATCH 161/421] Add routine to output sparse device matrix. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 26 ++++-- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 114 ++++++++++++++++++++--- 2 files changed, 119 insertions(+), 21 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 504060b9..fc42aa90 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -212,14 +212,26 @@ struct SpMatrixDevice { cusparseHandle_t cusparseHandle = nullptr; cusparseSpMatDescr_t descr = nullptr; - int size1 = 0; - int size2 = 0; + int size1 = 0; // rows + int size2 = 0; // columns int num_non_zeros = 0; Real* values = nullptr; - int* columns = nullptr; + int* inner = nullptr; int* offsets = nullptr; sparseFormat format = sparseFormat::none; + int outer_dim_size() const + { + // With CSR, we return the number of row. + if (format == sparseFormat::csr) + return size1; + // With CSC, we return the number of columns; + else if (format == sparseFormat::csc) + return size2; + else + std::abort(); + } + // Disallow copying -- only one object can "own" the descriptor. SpMatrixDevice& operator=(const SpMatrixDevice&) = delete; // Allow moving. @@ -231,7 +243,7 @@ struct SpMatrixDevice std::swap(size2, D.size2); std::swap(num_non_zeros, D.num_non_zeros); std::swap(values, D.values); - std::swap(columns, D.columns); + std::swap(inner, D.inner); std::swap(offsets, D.offsets); std::swap(format, D.format); return *this; @@ -247,17 +259,17 @@ struct SpMatrixDevice SpMatrixDevice() = default; SpMatrixDevice(cusparseHandle_t h, int s1, int s2, int n, Real* v, int* c, int* o, sparseFormat f) - :cusparseHandle(h), size1(s1), size2(s2), num_non_zeros(n), values(v), columns(c), offsets(o), format(f) + :cusparseHandle(h), size1(s1), size2(s2), num_non_zeros(n), values(v), inner(c), offsets(o), format(f) { cusparseStatus_t status; if (format == sparseFormat::csc) { - status = cusparseCreateCsc(&descr, s1, s2, num_non_zeros, offsets, columns, values, + status = cusparseCreateCsc(&descr, s1, s2, num_non_zeros, offsets, inner, values, IndexType, IndexType, CUSPARSE_INDEX_BASE_ZERO, DataType); } else { - status = cusparseCreateCsr(&descr, s1, s2, num_non_zeros, offsets, columns, values, + status = cusparseCreateCsr(&descr, s1, s2, num_non_zeros, offsets, inner, values, IndexType, IndexType, CUSPARSE_INDEX_BASE_ZERO, DataType); } diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 88434eeb..beef899a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -292,19 +292,28 @@ void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); } +template +std::vector MemcpyDeviceToHostVector(const T* dptr, int n) +{ + std::vector host_vec(n); + auto status = cudaMemcpy(host_vec.data(), dptr, n*sizeof(T), cudaMemcpyDeviceToHost); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); + return host_vec; +} + // Create an io-manipulator so that we can write std::cerr< -class byRow +class byRowDense { const DnMatrixDevice& D; public: - friend std::ostream& operator<<(std::ostream& o, const byRow& pr) + friend std::ostream& operator<<(std::ostream& o, const byRowDense& pr) { auto& D = pr.D; - Real* hPtr = new Real[D.size()]; - MemcpyDeviceToHost(hPtr, D.ptr, D.size()); + auto hPtr = MemcpyDeviceToHostVector(D.ptr, D.size()); o<<"Rows[ "; for(int row=0;row& d):D(d) {} + byRowDense(const DnMatrixDevice& d):D(d) {} }; // Create an io-manipulator so that we can write std::cerr< -class byCol +class byColDense { const DnMatrixDevice& D; public: - friend std::ostream& operator<<(std::ostream& o, const byCol& pr) + friend std::ostream& operator<<(std::ostream& o, const byColDense& pr) { auto& D = pr.D; - Real* hPtr = new Real[D.size()]; - - MemcpyDeviceToHost(hPtr, D.ptr, D.size()); + auto hPtr = MemcpyDeviceToHostVector(D.ptr, D.size()); o<<"Cols[ "; for(int col=0;col& d):D(d) {} +}; + +std::ostream& operator<<(std::ostream& o, sparseFormat f) +{ + if (f == sparseFormat::csr) + o<<"csr"; + else if (f == sparseFormat::csc) + o<<"csc"; + else + o<<"sparseFormat=unknown"; + return o; +} + +// Create an io-manipulator so that we can write std::cerr< +class byRowSparse +{ + const SpMatrixDevice& S; +public: + + friend std::ostream& operator<<(std::ostream& o, const byRowSparse& pr) + { + auto& S = pr.S; + auto values = MemcpyDeviceToHostVector(S.values, S.num_non_zeros); + auto inner = MemcpyDeviceToHostVector(S.inner, S.num_non_zeros); + auto offsets = MemcpyDeviceToHostVector(S.offsets, S.outer_dim_size() + 1); + + std::vector< std::vector > D(S.size1, std::vector(S.size2, 0)); + for(int o=0; o< S.outer_dim_size();o++) + { + for(int index=offsets[o];index& d):D(d) {} + byRowSparse(const SpMatrixDevice& s):S(s) {} }; +template +byRowDense byRow(const DnMatrixDevice& D) +{ + return byRowDense(D); +} + +template +byRowSparse byRow(const SpMatrixDevice& D) +{ + return byRowSparse(D); +} + +template +byColDense byCol(const DnMatrixDevice& D) +{ + return byColDense(D); +} + template std::ostream& operator<<(std::ostream& o, const DnMatrixDevice& D) { return o< +std::ostream& operator<<(std::ostream& o, const SpMatrixDevice& D) +{ + return o<::createInstance(int tipCount, int partialsBufferCount, @@ -1181,6 +1266,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // std::cerr<<"j/m = "<(integrationTmp[category], t / ((Real) s * j), A, dPartialsWrapper[destPIndex], 0, integrationBuffer[category], integrationBufferSize[category]); //#ifdef BEAGLE_DEBUG_FLOW From 4704416c895bec02afce598b6bdce6e987bb152d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 19 Jun 2024 15:23:45 -0400 Subject: [PATCH 162/421] Remove warnings from PrintfDeviceVector --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index beef899a..452e0f3c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -871,7 +871,7 @@ int BeagleGPUActionImpl::PrintfDeviceVector(int* dPtr, for(int i=0; i checkValue) && (hPtr[i]-checkValue > 1.0E-4)) { - fprintf(stderr,"Check value exception! (%d) %2.5e > %2.5e (diff = %2.5e)\n", + fprintf(stderr,"Check value exception! (%d) %2.5d > %2.5e (diff = %2.5e)\n", i,hPtr[i],checkValue, (hPtr[i]-checkValue)); if( signal != 0 ) *signal = 1; @@ -889,6 +889,7 @@ int BeagleGPUActionImpl::PrintfDeviceVector(int* dPtr, } } free(hPtr); + return 0; } BEAGLE_GPU_TEMPLATE @@ -926,6 +927,7 @@ int BeagleGPUActionImpl::PrintfDeviceVector(Real* dPtr, } } free(hPtr); + return 0; } BEAGLE_GPU_TEMPLATE From 3302207a2d3fb5e73b120cdf55c9546a9a3fd5ad Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 19 Jun 2024 15:33:12 -0400 Subject: [PATCH 163/421] Comment out debugging print. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 452e0f3c..960d111a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1268,7 +1268,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // std::cerr<<"j/m = "<(integrationTmp[category], t / ((Real) s * j), A, dPartialsWrapper[destPIndex], 0, integrationBuffer[category], integrationBufferSize[category]); //#ifdef BEAGLE_DEBUG_FLOW From 113420b35db646187fd88be81188af2daa659430 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 19 Jun 2024 15:35:40 -0400 Subject: [PATCH 164/421] Default to printing dense matrices on multiple rows. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 31 ++++++++++++++---------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 960d111a..4db42d63 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -307,28 +307,33 @@ template class byRowDense { const DnMatrixDevice& D; + bool multiline = false; public: friend std::ostream& operator<<(std::ostream& o, const byRowDense& pr) { auto& D = pr.D; + bool multiline = pr.multiline; auto hPtr = MemcpyDeviceToHostVector(D.ptr, D.size()); - o<<"Rows[ "; + if (not multiline) o<<"Rows[ "; for(int row=0;row& d):D(d) {} + byRowDense(const DnMatrixDevice& d, bool m=false):D(d),multiline(m) {} }; // Create an io-manipulator so that we can write std::cerr< -byRowDense byRow(const DnMatrixDevice& D) +byRowDense byRow(const DnMatrixDevice& D, bool m) { - return byRowDense(D); + return byRowDense(D, m); } template @@ -461,7 +466,7 @@ byColDense byCol(const DnMatrixDevice& D) template std::ostream& operator<<(std::ostream& o, const DnMatrixDevice& D) { - return o< From 47de00e16e55a7c930bed658e6d40d7178213a43 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 19 Jun 2024 15:42:16 -0400 Subject: [PATCH 165/421] Move cuda memory wrappers to declaration header. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 44 ++++++++++++++++++++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 44 ------------------------ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index fc42aa90..1dd09d48 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -109,6 +109,50 @@ Real normPInf(Real* matrix, int nRows, int nCols, cublasHandle_t cublasH) { return std::abs(result); } +template +T* cudaDeviceNew(int n) +{ + T* result; + auto status = cudaMalloc((void**)&result, n*sizeof(T)); + if (status != cudaSuccess) + throw std::runtime_error("cudaMalloc: failed!"); + return result; +} + +template +void MemcpyHostToDevice(T* dptr, const T* hptr, int n) +{ + auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyHostToDevice); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); +} + +template +void MemcpyDeviceToDevice(T* dptr, const T* hptr, int n) +{ + auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyDeviceToDevice); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); +} + +template +void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) +{ + auto status = cudaMemcpy(hptr, dptr, n*sizeof(T), cudaMemcpyDeviceToHost); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); +} + +template +std::vector MemcpyDeviceToHostVector(const T* dptr, int n) +{ + std::vector host_vec(n); + auto status = cudaMemcpy(host_vec.data(), dptr, n*sizeof(T), cudaMemcpyDeviceToHost); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); + return host_vec; +} + template using SpMatrix = Eigen::SparseMatrix; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 4db42d63..42471dfe 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -258,50 +258,6 @@ long long BeagleGPUActionImpl::getFlags() return flags; } -template -T* cudaDeviceNew(int n) -{ - T* result; - auto status = cudaMalloc((void**)&result, n*sizeof(T)); - if (status != cudaSuccess) - throw std::runtime_error("cudaMalloc: failed!"); - return result; -} - -template -void MemcpyHostToDevice(T* dptr, const T* hptr, int n) -{ - auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyHostToDevice); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); -} - -template -void MemcpyDeviceToDevice(T* dptr, const T* hptr, int n) -{ - auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyDeviceToDevice); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); -} - -template -void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) -{ - auto status = cudaMemcpy(hptr, dptr, n*sizeof(T), cudaMemcpyDeviceToHost); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); -} - -template -std::vector MemcpyDeviceToHostVector(const T* dptr, int n) -{ - std::vector host_vec(n); - auto status = cudaMemcpy(host_vec.data(), dptr, n*sizeof(T), cudaMemcpyDeviceToHost); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); - return host_vec; -} - // Create an io-manipulator so that we can write std::cerr< class byRowDense From fd4445e93b07ce2d17fb3257989b230961d8f14f Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 19 Jun 2024 15:43:14 -0400 Subject: [PATCH 166/421] Use new copyFrom member to copy dense device matrices. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 8 ++++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 1dd09d48..891cdcf5 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -186,6 +186,14 @@ struct DnMatrixDevice size_t size() const {return size1*size2;} size_t byte_size() const {return size()*sizeof(Real);} + void copyFrom(const DnMatrixDevice& D) + { + assert(order == D.order); + assert(size1 == D.size1); + assert(size2 == D.size2); + MemcpyDeviceToDevice(ptr, D.ptr, size()); + } + // Disallow copying -- only one object can "own" the descriptor. DnMatrixDevice& operator=(const DnMatrixDevice&) = delete; // Allow moving. diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 42471dfe..df03854f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1188,8 +1188,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p //#endif // destP = partials; - MemcpyDeviceToDevice(dPartialsWrapper[destPIndex].ptr, dPartialsWrapper[partialsIndex].ptr, kPaddedStateCount * kPaddedPatternCount); -// CHECK_CUSPARSE(cusparseDnMatSetValues(dPartialsWrapper[destPIndex], dPartialsWrapper[partialsIndex].ptr)) + dPartialsWrapper[destPIndex].copyFrom( dPartialsWrapper[partialsIndex] ); //#ifdef BEAGLE_DEBUG_FLOW @@ -1201,7 +1200,8 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // MatrixXd F(kStateCount, nCol); // F = destP; - MemcpyDeviceToDevice(F[category].ptr, dPartialsWrapper[partialsIndex].ptr, kPaddedStateCount * kPaddedPatternCount); + F[category].copyFrom( dPartialsWrapper[partialsIndex] ); + // CHECK_CUSPARSE(cusparseDnMatSetValues(F[category], dPartialsWrapper[partialsIndex].ptr)) // TODO: loop category within this function //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"F = partials operation, FCache:"< Date: Wed, 19 Jun 2024 15:46:32 -0400 Subject: [PATCH 167/421] Print column/row-major tag on last line. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index df03854f..5f30d01d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -284,10 +284,10 @@ class byRowDense else o< Date: Wed, 19 Jun 2024 16:12:24 -0400 Subject: [PATCH 168/421] Use some of the new printing operations. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5f30d01d..cbefe08c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1185,25 +1185,22 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"Before destP = partials operation, destPCache:\n"<::simpleAction2(int destPIndex, int p // destP = F; MemcpyDeviceToDevice(dPartialsWrapper[destPIndex].ptr, F[category].ptr, kPaddedStateCount * kPaddedPatternCount); #ifdef BEAGLE_DEBUG_FLOW - - std::cerr<<"destP = F:"< Date: Wed, 19 Jun 2024 16:19:23 -0400 Subject: [PATCH 169/421] Add operator*= for dense device matrices. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 891cdcf5..30e51039 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -208,6 +208,17 @@ struct DnMatrixDevice return *this; } + DnMatrixDevice& operator*=(Real d) + { + if constexpr (std::is_same::value) { + cublasSscal(cublasHandle, size(), &d, ptr, 1); + } else { + cublasDscal(cublasHandle, size(), &d, ptr, 1); + } + + return *this; + } + DnMatrixDevice& operator+=(const DnMatrixDevice& D) { assert(D.size1 == size1); From 334d505a86d8823ac3002fa7cfb5dd0885aea3d3 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 19 Jun 2024 16:22:09 -0400 Subject: [PATCH 170/421] Use copyFrom --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index cbefe08c..e5e7ad35 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1240,7 +1240,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p //#endif // destP = IntegrationTmp - MemcpyDeviceToDevice(dPartialsWrapper[destPIndex].ptr, integrationTmp[category].ptr, kPaddedStateCount * kPaddedPatternCount); + dPartialsWrapper[destPIndex].copyFrom( integrationTmp[category] ); //#ifdef BEAGLE_DEBUG_FLOW // @@ -1275,7 +1275,7 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p // PrintfDeviceVector(F[category].ptr, kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); //#endif // destP = F; - MemcpyDeviceToDevice(dPartialsWrapper[destPIndex].ptr, F[category].ptr, kPaddedStateCount * kPaddedPatternCount); + dPartialsWrapper[destPIndex].copyFrom ( F[category] ); #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"destP = F:\n"; std::cerr<<" "< Date: Wed, 19 Jun 2024 16:22:22 -0400 Subject: [PATCH 171/421] Use *= --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index e5e7ad35..9e942e10 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1263,12 +1263,8 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p } c1 = c2; } -// F *= eta; - if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &eta, F[category].ptr, 1)); - } else { - CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &eta, F[category].ptr, 1)); - } + + F[category] *= eta; //#ifdef BEAGLE_DEBUG_FLOW // // std::cerr<<"F *= eta (eta ="< Date: Wed, 19 Jun 2024 16:22:48 -0400 Subject: [PATCH 172/421] Use << --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 9e942e10..7f9396e7 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1243,17 +1243,15 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p dPartialsWrapper[destPIndex].copyFrom( integrationTmp[category] ); //#ifdef BEAGLE_DEBUG_FLOW -// // std::cerr<<"P = IntegrationTmp ="< Date: Wed, 19 Jun 2024 16:38:48 -0400 Subject: [PATCH 173/421] Print sparse matrices in multiline format by default. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 25 +++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 7f9396e7..2cc4f7e6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -360,11 +360,14 @@ template class byRowSparse { const SpMatrixDevice& S; + bool multiline = false; public: friend std::ostream& operator<<(std::ostream& o, const byRowSparse& pr) { auto& S = pr.S; + bool multiline = pr.multiline; + auto values = MemcpyDeviceToHostVector(S.values, S.num_non_zeros); auto inner = MemcpyDeviceToHostVector(S.inner, S.num_non_zeros); auto offsets = MemcpyDeviceToHostVector(S.offsets, S.outer_dim_size() + 1); @@ -381,37 +384,41 @@ class byRowSparse } } - o<<"Row[ "; + if (not multiline) o<<"Row[ "; for(int row=0;row& s):S(s) {} + byRowSparse(const SpMatrixDevice& s, bool m = false):S(s),multiline(m) {} }; template -byRowDense byRow(const DnMatrixDevice& D, bool m) +byRowDense byRow(const DnMatrixDevice& D, bool m = false) { return byRowDense(D, m); } template -byRowSparse byRow(const SpMatrixDevice& D) +byRowSparse byRow(const SpMatrixDevice& D, bool m = false) { - return byRowSparse(D); + return byRowSparse(D, m); } template @@ -429,7 +436,7 @@ std::ostream& operator<<(std::ostream& o, const DnMatrixDevice& D) template std::ostream& operator<<(std::ostream& o, const SpMatrixDevice& D) { - return o< Date: Wed, 19 Jun 2024 16:39:08 -0400 Subject: [PATCH 174/421] Simplify debug output slightly. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 2cc4f7e6..63ad25f8 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1233,17 +1233,15 @@ int BeagleGPUActionImpl::simpleAction2(int destPIndex, int p //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"j/m = "<(integrationTmp[category], t / ((Real) s * j), A, dPartialsWrapper[destPIndex], 0, integrationBuffer[category], integrationBufferSize[category]); //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"edge multiplier = "<< hEdgeMultipliers[edgeIndex * kCategoryCount + category]<<"\nB ="<::simpleAction2(int destPIndex, int p // PrintfDeviceVector(dPartialsWrapper[destPIndex].ptr, kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); //#endif - Real c2 = normPInf(integrationTmp[category]); // F += destP; F[category] += integrationTmp[category]; + + Real c2 = normPInf(integrationTmp[category]); //#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"F += destP, c1 = " <::simpleAction2(int destPIndex, int p for (int i = 0; i < s; i++) { //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"dPartials:"<::simpleAction2(int destPIndex, int p //#endif // destP = t / (s * j) * A * destP; - spMM(integrationTmp[category], t / ((Real) s * j), A, dPartialsWrapper[destPIndex], 0, integrationBuffer[category], integrationBufferSize[category]); + spMM(integrationTmp, t / ((Real) s * j), A, destP, 0, integrationBuffer[category], integrationBufferSize[category]); //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"edge multiplier = "<< hEdgeMultipliers[edgeIndex * kCategoryCount + category]<<"\nB ="<::simpleAction2(int destPIndex, int p //#endif // destP = IntegrationTmp - dPartialsWrapper[destPIndex].copyFrom( integrationTmp[category] ); + destP.copyFrom( integrationTmp ); //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"P = IntegrationTmp ="< Date: Wed, 19 Jun 2024 17:01:03 -0400 Subject: [PATCH 176/421] Simplify calls to simpleAction2( ). --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 4 ++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 52f7232e..ff008408 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -546,6 +546,10 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int getPartialCacheIndex(int nodeIndex, int categoryIndex); + DnMatrixDevice& getPartialsWrapper(int nodeIndex, int categoryIndex); + + DnMatrixDevice& getPartialsCacheWrapper(int nodeIndex, int categoryIndex); + void calcPartialsPartials(int destPIndex, int partials1Index, int edgeIndex1, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index ffb33c58..23908fc0 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -735,6 +735,16 @@ int BeagleGPUActionImpl::getPartialCacheIndex(int nodeIndex, return kPartialsCacheOffset + getPartialIndex(nodeIndex, categoryIndex); } +BEAGLE_GPU_TEMPLATE +DnMatrixDevice& BeagleGPUActionImpl::getPartialsWrapper(int nodeIndex, int categoryIndex) { + return dPartialsWrapper[getPartialIndex(nodeIndex, categoryIndex)]; +} + +BEAGLE_GPU_TEMPLATE +DnMatrixDevice& BeagleGPUActionImpl::getPartialsCacheWrapper(int nodeIndex, int categoryIndex) { + return dPartialsWrapper[getPartialCacheIndex(nodeIndex, categoryIndex)]; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPartials(bool byPartition, const int *operations, @@ -957,11 +967,11 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; - simpleAction2(dPartialsWrapper[getPartialCacheIndex(partials1Index, category)], dPartialsWrapper[getPartialIndex(partials1Index, category)], - edgeIndex1, category, hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category, true, false); + simpleAction2(getPartialsCacheWrapper(partials1Index, category), getPartialsWrapper(partials1Index, category), + edgeIndex1, category, matrixIndex1, true, false); - simpleAction2(dPartialsWrapper[getPartialCacheIndex(partials2Index, category)], dPartialsWrapper[getPartialIndex(partials2Index, category)], - edgeIndex2, category, hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category, false, false); + simpleAction2(getPartialsCacheWrapper(partials2Index, category), getPartialsWrapper(partials2Index, category), + edgeIndex2, category, matrixIndex2, false, false); } // simpleAction3(partials1Index, edgeIndex1, partials2Index, edgeIndex2); @@ -974,6 +984,7 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); + // element-wise multiply if constexpr (std::is_same::value) { CUBLAS_CHECK(cublasSdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialsWrapper[partial1CacheIndex].ptr, kPaddedStateCount * kPaddedPatternCount, dPartialsWrapper[partial2CacheIndex].ptr, 1, dPartialsWrapper[destPartialIndex].ptr, kPaddedStateCount * kPaddedPatternCount)); From ae37a9e895940ae68ad50ed686b30bacd2c0e8a1 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 19 Jun 2024 17:13:23 -0400 Subject: [PATCH 177/421] Tweak comments. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 23908fc0..66e7e712 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1245,14 +1245,14 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& // std::cerr<<"j/m = "< Date: Thu, 20 Jun 2024 20:15:28 -0500 Subject: [PATCH 178/421] Bug FIX: cache partial index wrong, need to multiply kCategoryCount! --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 66e7e712..1497880e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -732,7 +732,7 @@ int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartialCacheIndex(int nodeIndex, int categoryIndex) { - return kPartialsCacheOffset + getPartialIndex(nodeIndex, categoryIndex); + return kPartialsCacheOffset * kCategoryCount + getPartialIndex(nodeIndex, categoryIndex); } BEAGLE_GPU_TEMPLATE From b992d623115333d30b0efd2001b47def9e3b459c Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 24 Jun 2024 14:29:21 -0400 Subject: [PATCH 179/421] Implement SpMatrix::operator*= --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 63 ++++++++++++++++++------ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 41 +++++++-------- 2 files changed, 66 insertions(+), 38 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index ff008408..df20dc53 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -210,28 +210,40 @@ struct DnMatrixDevice DnMatrixDevice& operator*=(Real d) { + cublasStatus_t status; if constexpr (std::is_same::value) { - cublasSscal(cublasHandle, size(), &d, ptr, 1); + status = cublasSscal(cublasHandle, size(), &d, ptr, 1); } else { - cublasDscal(cublasHandle, size(), &d, ptr, 1); + status = cublasDscal(cublasHandle, size(), &d, ptr, 1); } - return *this; + if (status != CUBLAS_STATUS_SUCCESS) + { + std::cerr<<"cublas error "<::operator*=\n"; + exit(1); + } + return *this; } DnMatrixDevice& operator+=(const DnMatrixDevice& D) { - assert(D.size1 == size1); - assert(D.size2 == size2); - assert(D.cublasHandle == cublasHandle); - Real one = 1; - if constexpr (std::is_same::value) { - cublasSaxpy(cublasHandle, size(), &one, D.ptr, 1, ptr, 1); - } else { - cublasDaxpy(cublasHandle, size(), &one, D.ptr, 1, ptr, 1); - } + cublasStatus_t status; + assert(D.size1 == size1); + assert(D.size2 == size2); + assert(D.cublasHandle == cublasHandle); + Real one = 1; + if constexpr (std::is_same::value) { + status = cublasSaxpy(cublasHandle, size(), &one, D.ptr, 1, ptr, 1); + } else { + status = cublasDaxpy(cublasHandle, size(), &one, D.ptr, 1, ptr, 1); + } - return *this; + if (status != CUBLAS_STATUS_SUCCESS) + { + std::cerr<<"cublas error "<::operator+=\n"; + exit(1); + } + return *this; } // Disallow copying. @@ -273,6 +285,7 @@ enum class sparseFormat { none, csr, csc }; template struct SpMatrixDevice { + cublasHandle_t cublasHandle = nullptr; cusparseHandle_t cusparseHandle = nullptr; cusparseSpMatDescr_t descr = nullptr; int size1 = 0; // rows @@ -300,6 +313,7 @@ struct SpMatrixDevice // Allow moving. SpMatrixDevice& operator=(SpMatrixDevice&& D) noexcept { + std::swap(cublasHandle, D.cublasHandle); std::swap(cusparseHandle, D.cusparseHandle); std::swap(descr, D.descr); std::swap(size1, D.size1); @@ -312,7 +326,24 @@ struct SpMatrixDevice return *this; } - // Disallow copying. + SpMatrixDevice& operator*=(Real d) + { + cublasStatus_t status; + if constexpr (std::is_same::value) { + status = cublasSscal(cublasHandle, num_non_zeros, &d, values, 1); + } else { + status = cublasDscal(cublasHandle, num_non_zeros, &d, values, 1); + } + + if (status != CUBLAS_STATUS_SUCCESS) + { + std::cerr<<"cublas error "<::operator*=\n"; + exit(1); + } + return *this; + } + + // Disallow copying. SpMatrixDevice(const SpMatrixDevice&) = delete; // Allow moving. SpMatrixDevice(SpMatrixDevice&& D) noexcept @@ -321,8 +352,8 @@ struct SpMatrixDevice } SpMatrixDevice() = default; - SpMatrixDevice(cusparseHandle_t h, int s1, int s2, int n, Real* v, int* c, int* o, sparseFormat f) - :cusparseHandle(h), size1(s1), size2(s2), num_non_zeros(n), values(v), inner(c), offsets(o), format(f) + SpMatrixDevice(cublasHandle_t h1, cusparseHandle_t h2, int s1, int s2, int n, Real* v, int* c, int* o, sparseFormat f) + :cublasHandle(h1), cusparseHandle(h2), size1(s1), size2(s2), num_non_zeros(n), values(v), inner(c), offsets(o), format(f) { cusparseStatus_t status; if (format == sparseFormat::csc) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 1497880e..c1ee2fed 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1085,7 +1085,8 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) { +int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) +{ for (int category = 0; category < kCategoryCount; category++) { const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; const int edgeMultiplierIndex1 = edgeIndex1 * kCategoryCount + category; @@ -1095,42 +1096,38 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; const Real edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; - MemcpyDeviceToDevice(dACscValuesCache[matrixIndex1], dBsCsrValuesCache[hEigenMaps[edgeIndex1]], currentCacheNNZs[hEigenMaps[edgeIndex1]]); MemcpyDeviceToDevice(dACscValuesCache[matrixIndex2], dBsCsrValuesCache[hEigenMaps[edgeIndex2]], currentCacheNNZs[hEigenMaps[edgeIndex2]]); - if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following - CUBLAS_CHECK(cublasSscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following - } else { - CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex1]], &edgeMultiplier1, dACscValuesCache[matrixIndex1], 1)); //Check if this is asynchronous with the following - CUBLAS_CHECK(cublasDscal(cublasHandle, currentCacheNNZs[hEigenMaps[edgeIndex2]], &edgeMultiplier2, dACscValuesCache[matrixIndex2], 1)); //Check if this is asynchronous with the following - } - -#ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"category = "< Date: Mon, 24 Jun 2024 14:36:07 -0400 Subject: [PATCH 180/421] Use destP instead of integrationTmp. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c1ee2fed..4bbf6225 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1252,19 +1252,15 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& // std::cerr<<"AP * alpha = "<::simpleAction2(DnMatrixDevice& const Real one = 1; for (int i = 0; i < s; i++) { //#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"dPartials:"< Date: Tue, 25 Jun 2024 09:15:13 -0700 Subject: [PATCH 182/421] optimize peeling --- libhmsbeagle/GPU/BeagleGPUImpl.h | 4 +- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 384 ++++++++++++++--------- libhmsbeagle/GPU/KernelLauncher.cpp | 106 ++++--- libhmsbeagle/GPU/KernelLauncher.h | 11 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 405 ++++++++++++++++--------- 5 files changed, 594 insertions(+), 316 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index b7a1c177..57b2839b 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -219,9 +219,11 @@ class BeagleGPUImpl : public BeagleImpl { GPUPtr dBastaH; GPUPtr dBastaLogL; GPUPtr dBastaDistance; + GPUPtr dBastaOperationQueue; GPUPtr dCoalescentBuffers; int kCoalescentBufferLength; int kCoalescentBufferCount; + int kBastaIntervalBlockCount; public: BeagleGPUImpl(); @@ -475,7 +477,7 @@ class BeagleGPUImpl : public BeagleImpl { int getSiteDerivatives(double* outFirstDerivatives, double* outSecondDerivatives); - int updateInnerBastaPartials(const int * operations, int i, int count, GPUPtr gpu, GPUPtr coalescent); + int updateInnerBastaPartials(const int * operations, const int * intervals, int i, int begin, int end, GPUPtr sizes, GPUPtr coalescent); int updateBastaPartials(const int* operations, int operationCount, diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index ab5b2b3d..c990560c 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -82,6 +82,7 @@ namespace gpu { dBastaH = (GPUPtr)NULL; dBastaLogL = (GPUPtr)NULL; dBastaDistance = (GPUPtr)NULL; + dBastaOperationQueue = (GPUPtr)NULL; dBastaBuffers = (GPUPtr)NULL; dCoalescentBuffers = (GPUPtr)NULL; @@ -260,6 +261,7 @@ BeagleGPUImpl::~BeagleGPUImpl() { gpu->FreeMemory(dBastaH); gpu->FreeMemory(dBastaLogL); gpu->FreeMemory(dBastaDistance); + gpu->FreeMemory(dBastaOperationQueue); gpu->FreeHostMemory(hPtrQueue); @@ -1904,12 +1906,12 @@ int BeagleGPUImpl::transposeTransitionMatrices( return BEAGLE_SUCCESS; } - BEAGLE_GPU_TEMPLATE - int BeagleGPUImpl::updateTransitionMatricesGrad(const int* probabilityIndices, - const double* edgeLengths, - int count) { - return BEAGLE_ERROR_NO_IMPLEMENTATION; - } +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::updateTransitionMatricesGrad(const int* probabilityIndices, + const double* edgeLengths, + int count) { + return BEAGLE_ERROR_NO_IMPLEMENTATION; +} BEAGLE_GPU_TEMPLATE @@ -2233,13 +2235,14 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, kCoalescentBufferLength = bufferLength; kCoalescentBufferCount = bufferCount; + kBastaIntervalBlockCount = bufferLength / 32 + 1; dCoalescentBuffers = gpu->AllocateMemory(kCoalescentBufferCount * kCoalescentBufferLength * sizeof(Real)); dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaLogL = gpu->AllocateMemory(1 * sizeof(Real)); + dBastaLogL = gpu->AllocateMemory(kBastaIntervalBlockCount * sizeof(Real)); dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); @@ -2278,11 +2281,11 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - GPUPtr coalescent = (GPUPtr) NULL; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; - int returnCode = updateInnerBastaPartials(operations, 0, count, sizes, coalescent); + int returnCode = updateInnerBastaPartials(operations, intervals, intervalCount, 0, count, sizes, coalescent); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUImpl::updateBastaPartials\n"); @@ -2293,45 +2296,108 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::updateInnerBastaPartials(const int* operations, + const int* intervals, + const int intervalCount, const int begin, const int end, const GPUPtr sizes, GPUPtr coalescent) { + std::vector newOperations; + const int numOps = BEAGLE_BASTA_OP_COUNT; + const int operationCount = end - begin; + Real* hBastaOperationQueue = (Real*) gpu->CallocHost(sizeof(Real), operationCount * numOps); + + for (int i = 0; i < operationCount; ++i) { + hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; + hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; + hBastaOperationQueue[i * numOps + 2] = operations[i * numOps + 2] * kPaddedStateCount * kPaddedStateCount; + if (operations[i * numOps + 3] >= 0) { + hBastaOperationQueue[i * numOps + 3] = hPartialsOffsets[operations[i * numOps + 3]]; + hBastaOperationQueue[i * numOps + 4] = operations[i * numOps + 4] * kPaddedStateCount * kPaddedStateCount; + } else { + hBastaOperationQueue[i * numOps + 3] = operations[i * numOps + 3]; + hBastaOperationQueue[i * numOps + 4] = operations[i * numOps + 4]; + } + hBastaOperationQueue[i * numOps + 5] = hPartialsOffsets[operations[i * numOps + 5]]; + hBastaOperationQueue[i * numOps + 6] = hPartialsOffsets[operations[i * numOps + 6]]; + hBastaOperationQueue[i * numOps + 7] = operations[i * numOps + 7]; + } - std::vector newOperations; - for (int op = begin; op < end; op++) { - - const int numOps = BEAGLE_BASTA_OP_COUNT; + dBastaOperationQueue = gpu->AllocateMemory(operationCount * numOps * sizeof(Real)); + gpu->MemcpyHostToDevice(dBastaOperationQueue, hBastaOperationQueue, sizeof(Real) * operationCount * numOps); + gpu->FreeHostMemory(hBastaOperationQueue); + + // Real r = 0; + // fprintf(stderr, "e:\n"); + // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "f:\n"); + // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "g:\n"); + // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - const int parIndex = operations[op * numOps]; - const int child1Index = operations[op * numOps + 1]; - const int child1TransMatIndex = operations[op * numOps + 2]; - const int child2Index = operations[op * numOps + 3]; - const int child2TransMatIndex = operations[op * numOps + 4]; - const int accumulation1PartialIndex = operations[op * numOps + 5]; - const int accumulation2PartialIndex = operations[op * numOps + 6]; - const int intervalNumber = operations[op * numOps + 7]; + for (int interval = 0; interval < intervalCount - 1; ++interval) { + // TODO execute in parallel (no race conditions) + const int start = intervals[interval]; + const int end = intervals[interval + 1]; + const int opCounts = end - start; + kernels->InnerBastaPartialsCoalescent(dPartialsOrigin, dMatricesOrigin, dBastaOperationQueue, + sizes, coalescent, interval, start, numOps, opCounts); - GPUPtr matrices1 = dMatrices[child1TransMatIndex]; - GPUPtr matrices2 = dMatrices[child2TransMatIndex]; + } - GPUPtr partials1 = dPartials[child1Index]; - GPUPtr partials2 = dPartials[child2Index]; - GPUPtr partials3 = dPartials[parIndex]; - - GPUPtr accumulation1 = dPartials[accumulation1PartialIndex]; - GPUPtr accumulation2 = dPartials[accumulation2PartialIndex]; - kernels->InnerBastaPartialsCoalescent(partials1, partials2, partials3, - matrices1, matrices2, accumulation1, accumulation2, sizes, coalescent, intervalNumber, 1, child2Index); - // Real r = 0; - // fprintf(stderr, "The op is: %d\n", op); - // fprintf(stderr, "The index is: %d\n", child1TransMatIndex); - // fprintf(stderr,"root partials = \n"); - // gpu->PrintfDeviceVector(partials3, 4, r); - // gpu->PrintfDeviceVector(sizes, 4, r); - } + // for (int op = begin; op < end; op++) { + // + // const int numOps = BEAGLE_BASTA_OP_COUNT; + // + // const int parIndex = operations[op * numOps]; + // const int child1Index = operations[op * numOps + 1]; + // const int child1TransMatIndex = operations[op * numOps + 2]; + // const int child2Index = operations[op * numOps + 3]; + // const int child2TransMatIndex = operations[op * numOps + 4]; + // const int accumulation1PartialIndex = operations[op * numOps + 5]; + // const int accumulation2PartialIndex = operations[op * numOps + 6]; + // const int intervalNumber = operations[op * numOps + 7]; + // + // GPUPtr matrices1 = dMatrices[child1TransMatIndex]; + // GPUPtr matrices2 = dMatrices[child2TransMatIndex]; + // + // GPUPtr partials1 = dPartials[child1Index]; + // GPUPtr partials2 = dPartials[child2Index]; + // GPUPtr partials3 = dPartials[parIndex]; + // GPUPtr accumulation1 = dPartials[accumulation1PartialIndex]; + // GPUPtr accumulation2 = dPartials[accumulation2PartialIndex]; + + // Real r = 1; + // fprintf(stderr, "The partial index is: %d\n", child1Index); + // fprintf(stderr, "The matrix index is: %d\n", child1TransMatIndex); + // fprintf(stderr, "partial1:\n"); + // gpu->PrintfDeviceVector(partials1, kPaddedStateCount, r); + // fprintf(stderr, "matice1:\n"); + // gpu->PrintfDeviceVector(matrices1, kMatrixSize, r); + // kernels->InnerBastaPartialsCoalescent(partials1, partials2, partials3, + // matrices1, matrices2, accumulation1, accumulation2, sizes, coalescent, intervalNumber, 1, child2Index); + // Real r = 0; + // fprintf(stderr, "The child1index is: %d\n", child1Index); + // fprintf(stderr, "partials1:\n"); + // gpu->PrintfDeviceVector(partials1, kPaddedStateCount, r); + // fprintf(stderr, "The parindex is: %d\n", parIndex); + // fprintf(stderr, "partials3:\n"); + // gpu->PrintfDeviceVector(partials3, kPaddedStateCount, r); + + // Real r = 0; + // fprintf(stderr, "The op is: %d\n", op); + // fprintf(stderr, "The index is: %d\n", child1TransMatIndex); + //fprintf(stderr,"matrices = \n"); + // fprintf(stderr, "partials:\n"); + // gpu->PrintfDeviceVector(dPartialsOrigin, 524 * kPaddedStateCount, r); + // // gpu->PrintfDeviceVector(dMatricesOrigin, kPaddedStateCount * kPaddedStateCount * 3, r); + // // gpu->PrintfDeviceVector(sizes, 4, r); + // gpu->PrintfDeviceVector(coalescent, 4 * kCoalescentBufferLength, r); + //} #ifdef BEAGLE_DEBUG_SYNCH gpu->SynchronizeHost(); @@ -2339,27 +2405,27 @@ int BeagleGPUImpl::updateInnerBastaPartials(const int* opera return BEAGLE_SUCCESS; } - BEAGLE_GPU_TEMPLATE - int BeagleGPUImpl::updateBastaPartialsGrad(const int* operations, - const int count, - const int* intervals, - const int intervalCount, - const int populationSizesIndex, - const int coalescentIndex) { - #ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); - #endif +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::updateBastaPartialsGrad(const int* operations, + const int count, + const int* intervals, + const int intervalCount, + const int populationSizesIndex, + const int coalescentIndex) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); +#endif - int returnCode = BEAGLE_ERROR_GENERAL; + int returnCode = BEAGLE_ERROR_GENERAL; - fprintf(stderr, "BeagleGPUImpl::updateBastaPartials\n"); + fprintf(stderr, "BeagleGPUImpl::updateBastaPartials\n"); - #ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUImpl::updateBastaPartials\n"); - #endif +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUImpl::updateBastaPartials\n"); +#endif - return returnCode; - } + return returnCode; +} BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::accumulateBastaPartials(const int* operations, @@ -2375,67 +2441,63 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #endif - GPUPtr coalescent = (GPUPtr) NULL; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; const GPUPtr sizes = dFrequencies[populationSizesIndex]; - Real* zeroes = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaE, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaF, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaG, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaH, zeroes, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + const int numOps = BEAGLE_BASTA_OP_COUNT; + + // Real* hBastaOperationQueue = (Real*) gpu->CallocHost(sizeof(Real), operationCount * numOps); + + + // + // for (int i = 0; i < operationCount; ++i) { + // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; + // hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; + // hBastaOperationQueue[i * numOps + 2] = operations[i * numOps + 2] * kPaddedStateCount * kPaddedStateCount; + // if (operations[i * numOps + 3] >= 0) { + // hBastaOperationQueue[i * numOps + 3] = hPartialsOffsets[operations[i * numOps + 3]]; + // hBastaOperationQueue[i * numOps + 4] = operations[i * numOps + 4] * kPaddedStateCount * kPaddedStateCount; + // } else { + // hBastaOperationQueue[i * numOps + 3] = operations[i * numOps + 3]; + // hBastaOperationQueue[i * numOps + 4] = operations[i * numOps + 4]; + // } + // hBastaOperationQueue[i * numOps + 5] = hPartialsOffsets[operations[i * numOps + 5]]; + // hBastaOperationQueue[i * numOps + 6] = hPartialsOffsets[operations[i * numOps + 6]]; + // hBastaOperationQueue[i * numOps + 7] = operations[i * numOps + 7]; + // } + // + // dBastaOperationQueue = gpu->AllocateMemory(operationCount * numOps * sizeof(Real)); + // gpu->MemcpyHostToDevice(dBastaOperationQueue, hBastaOperationQueue, sizeof(Real) * operationCount * numOps); + // gpu->FreeHostMemory(hBastaOperationQueue); + + // Real r = 0; + // fprintf(stderr, "e:\n"); + // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "f:\n"); + // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "g:\n"); + // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { // TODO execute in parallel (no race conditions) const int start = intervalStarts[interval]; const int end = intervalStarts[interval + 1]; - for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h - - const int numOps = BEAGLE_BASTA_OP_COUNT; - - const int child1PartialIndex = operations[op * numOps + 1]; - const int child2PartialIndex = operations[op * numOps + 3]; - const int accumulation1PartialIndex = operations[op * numOps + 5]; - const int accumulation2PartialIndex = operations[op * numOps + 6]; - const int intervalNumber = operations[op * numOps + 7]; - GPUPtr startPartials1 = dPartials[child1PartialIndex]; - GPUPtr startPartials2 = dPartials[child2PartialIndex]; - GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; - GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; - Real r = 0; - kernels->reduceWithinInterval(dBastaE, dBastaF, dBastaG, dBastaH, - startPartials1, startPartials2, - endPartials1, endPartials2, - intervalNumber, child2PartialIndex); - - // fprintf(stderr, "startPartials1:\n"); - // gpu->PrintfDeviceVector(startPartials1, kPaddedStateCount, r); - // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); - // fprintf(stderr, "h:\n"); - // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - } - } - Real* hBastaDistance = (Real*) gpu->CallocHost(sizeof(Real), kCoalescentBufferLength); - for (int i = 0; i < kCoalescentBufferLength; ++i) { - hBastaDistance[i] = (Real) intervalLengths[i]; - } - Real r = 0; - Real* hBastaLogL = (Real*) gpu->CallocHost(sizeof(Real), 1); - gpu->MemcpyHostToDevice(dBastaLogL, hBastaLogL, sizeof(Real) * 1); - gpu->MemcpyHostToDevice(dBastaDistance, hBastaDistance, sizeof(Real) * kCoalescentBufferLength); - for (int i = 0; i < intervalStartsCount - 1; ++i) { - // TODO execute in parallel - Real r = 0; - const int op = intervalStarts[i]; - const int numOps = BEAGLE_BASTA_OP_COUNT; - const int intervalNumber = operations[op * numOps + 7]; - Real intLength = (Real) intervalLengths[i]; - - kernels->reduceAcrossIntervals(dBastaE, dBastaF, dBastaG, dBastaH, dBastaDistance, dBastaLogL, sizes, coalescent, - intervalNumber); - // fprintf(stderr, "The interval is: %d\n", i); - // fprintf(stderr, "The interval length is: %f\n", hBastaDistance[i]); - // fprintf(stderr, "logL:\n"); - // gpu->PrintfDeviceVector(dBastaLogL, 1, r); + // const int child1PartialIndex = operations[op * numOps + 1]; + // const int child2PartialIndex = operations[op * numOps + 3]; + // const int accumulation1PartialIndex = operations[op * numOps + 5]; + // const int accumulation2PartialIndex = operations[op * numOps + 6]; + // const int intervalNumber = operations[op * numOps + 7]; + // GPUPtr startPartials1 = dPartials[child1PartialIndex]; + // GPUPtr startPartials2 = dPartials[child2PartialIndex]; + // GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; + // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; + // Real r = 0; + kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaE, dBastaF, dBastaG, dBastaH, numOps, + interval, start, end); + // fprintf(stderr, "The interval is: %d\n", interval); // fprintf(stderr, "e:\n"); // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); // fprintf(stderr, "f:\n"); @@ -2444,15 +2506,67 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); // fprintf(stderr, "h:\n"); // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "sizes:\n"); - // gpu->PrintfDeviceVector(sizes, 4, r); - // fprintf(stderr, "coalescent:\n"); - // gpu->PrintfDeviceVector(coalescent, kCoalescentBufferLength, r); + // Real r = 0; + // // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); + // fprintf(stderr, "partials:\n"); + // gpu->PrintfDeviceVector(dPartialsOrigin, 24 * kPaddedStateCount, r); + // fprintf(stderr, "operations:\n"); + // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); } - gpu->MemcpyDeviceToHost(hBastaLogL, dBastaLogL, sizeof(Real) * 1); - out[0] += hBastaLogL[0]; - gpu->FreeHostMemory(zeroes); + // for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { + // // TODO execute in parallel (no race conditions) + // const int start = intervalStarts[interval]; + // const int end = intervalStarts[interval + 1]; + // int renew = 0; + // for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h + // + // const int numOps = BEAGLE_BASTA_OP_COUNT; + // + // const int child1PartialIndex = operations[op * numOps + 1]; + // const int child2PartialIndex = operations[op * numOps + 3]; + // const int accumulation1PartialIndex = operations[op * numOps + 5]; + // const int accumulation2PartialIndex = operations[op * numOps + 6]; + // const int intervalNumber = operations[op * numOps + 7]; + // GPUPtr startPartials1 = dPartials[child1PartialIndex]; + // GPUPtr startPartials2 = dPartials[child2PartialIndex]; + // GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; + // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; + // Real r = 0; + // kernels->reduceWithinInterval(dBastaE, dBastaF, dBastaG, dBastaH, + // startPartials1, startPartials2, + // endPartials1, endPartials2, + // intervalNumber, child2PartialIndex, renew); + // renew = 1; + // fprintf(stderr, "startPartials1:\n"); + // gpu->PrintfDeviceVector(startPartials1, kPaddedStateCount, r); + // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + // } + + + Real* hBastaDistance = (Real*) gpu->CallocHost(sizeof(Real), kCoalescentBufferLength); + for (int i = 0; i < kCoalescentBufferLength; ++i) { + hBastaDistance[i] = (Real) intervalLengths[i]; + } + Real* hBastaLogL = (Real*) gpu->CallocHost(sizeof(Real), kBastaIntervalBlockCount); + gpu->MemcpyHostToDevice(dBastaDistance, hBastaDistance, sizeof(Real) * kCoalescentBufferLength); + + + // kernels->reduceAcrossIntervals(dBastaE, dBastaF, dBastaG, dBastaH, dBastaDistance, dBastaLogL, sizes, coalescent, + // intervalStartsCount); + // fprintf(stderr, "The block count is: %d\n", kBastaIntervalBlockCount); + // fprintf(stderr, "logL:\n"); + // gpu->PrintfDeviceVector(dBastaLogL, kBastaIntervalBlockCount, r); + // fprintf(stderr, "sizes:\n"); + // gpu->PrintfDeviceVector(sizes, 6, r); + // fprintf(stderr, "coalescent:\n"); + // gpu->PrintfDeviceVector(coalescent, kCoalescentBufferLength, r); + gpu->MemcpyDeviceToHost(hBastaLogL, dBastaLogL, sizeof(Real) * kBastaIntervalBlockCount); + for (int i = 0; i < kBastaIntervalBlockCount; i++) { + out[0] += hBastaLogL[i]; + } gpu->FreeHostMemory(hBastaLogL); gpu->FreeHostMemory(hBastaDistance); @@ -2464,29 +2578,29 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat } - BEAGLE_GPU_TEMPLATE - int BeagleGPUImpl::accumulateBastaPartialsGrad(const int *operations, - const int operationCount, - const int *intervalStarts, - const int intervalStartsCount, - const double *intervalLengths, - const int populationSizesIndex, - const int coalescentIndex, - double *out) { - #ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); - #endif +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::accumulateBastaPartialsGrad(const int *operations, + const int operationCount, + const int *intervalStarts, + const int intervalStartsCount, + const double *intervalLengths, + const int populationSizesIndex, + const int coalescentIndex, + double *out) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); +#endif - int returnCode = BEAGLE_ERROR_GENERAL; + int returnCode = BEAGLE_ERROR_GENERAL; - fprintf(stderr, "BeagleGPUImpl::accumulateBastaPartials\n"); + fprintf(stderr, "BeagleGPUImpl::accumulateBastaPartials\n"); - #ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); - #endif +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); +#endif - return returnCode; - } + return returnCode; +} BEAGLE_GPU_TEMPLATE diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 7e25ce30..b7c781c2 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -238,14 +238,14 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { //Set up block for basta partials - bgBastaPeelingBlock = Dim3Int(kPaddedStateCount, 1); + bgBastaPeelingBlock = Dim3Int(kPaddedStateCount, 32); bgBastaPeelingGrid = Dim3Int(1,1); - bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 4); - bgBastaReductionGrid = Dim3Int(1,1); + bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 32); + bgBastaReductionGrid = Dim3Int(1); - bgBastaSumBlock = Dim3Int(kPaddedStateCount, 64); - bgBastaSumGrid = Dim3Int(1,1); + bgBastaSumBlock = Dim3Int(128); + bgBastaSumGrid = Dim3Int(32); } @@ -2414,56 +2414,83 @@ void KernelLauncher::SumSites3(GPUPtr dArray1, } -void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials1, - GPUPtr partials2, - GPUPtr partials3, - GPUPtr matrices1, - GPUPtr matrices2, - GPUPtr accumulation1, - GPUPtr accumulation2, - const GPUPtr sizes, - GPUPtr coalescent, - unsigned int intervalNUmber, - unsigned int patternCount, - unsigned int child2Index) { +void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, + GPUPtr matrices, + GPUPtr operations, + const GPUPtr sizes, + GPUPtr coalescent, + unsigned int intervalNUmber, + unsigned int start, + unsigned int numOps, + unsigned int patternCount) { #ifdef BEAGLE_DEBUG_FLOW -fprintf(stderr, "\t\tEntering KernelLauncher::InnerBastaPartialsCoalescent\n"); + fprintf(stderr, "\t\tEntering KernelLauncher::InnerBastaPartialsCoalescent\n"); #endif - int parameterCountV = 9; - int totalParameterCount = 12; - gpu->LaunchKernel(fInnerBastaPartialsCoalescent, - bgBastaPeelingBlock, bgBastaPeelingGrid, - parameterCountV, totalParameterCount, - partials1, partials2, partials3, matrices1, matrices2, accumulation1, accumulation2, sizes, coalescent, - intervalNUmber, patternCount, child2Index); + int parameterCountV = 5; + int totalParameterCount = 9; + gpu->LaunchKernel(fInnerBastaPartialsCoalescent, + bgBastaPeelingBlock, bgBastaPeelingGrid, + parameterCountV, totalParameterCount, + partials, matrices, operations, sizes, coalescent, + intervalNUmber, start, numOps, patternCount); #ifdef BEAGLE_DEBUG_FLOW -fprintf(stderr, "\t\tLeaving KernelLauncher::InnerBastaPartialsCoalescent\n"); + fprintf(stderr, "\t\tLeaving KernelLauncher::InnerBastaPartialsCoalescent\n"); #endif -} + } -void KernelLauncher::reduceWithinInterval(GPUPtr e, - GPUPtr f, - GPUPtr g, - GPUPtr h, - GPUPtr startPartials1, - GPUPtr startPartials2, - GPUPtr endPartials1, - GPUPtr endPartials2, - unsigned int intervalNUmber, - unsigned int child2PartialIndex) { +// void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials1, +// GPUPtr partials2, +// GPUPtr partials3, +// GPUPtr matrices1, +// GPUPtr matrices2, +// GPUPtr accumulation1, +// GPUPtr accumulation2, +// const GPUPtr sizes, +// GPUPtr coalescent, +// unsigned int intervalNUmber, +// unsigned int patternCount, +// unsigned int child2Index) { +// #ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\t\tEntering KernelLauncher::InnerBastaPartialsCoalescent\n"); +// #endif +// +// int parameterCountV = 9; +// int totalParameterCount = 12; +// gpu->LaunchKernel(fInnerBastaPartialsCoalescent, +// bgBastaPeelingBlock, bgBastaPeelingGrid, +// parameterCountV, totalParameterCount, +// partials1, partials2, partials3, matrices1, matrices2, accumulation1, accumulation2, sizes, coalescent, +// intervalNUmber, patternCount, child2Index); +// +// #ifdef BEAGLE_DEBUG_FLOW +// fprintf(stderr, "\t\tLeaving KernelLauncher::InnerBastaPartialsCoalescent\n"); +// #endif +// +// } + +void KernelLauncher::reduceWithinInterval(GPUPtr operations, + GPUPtr partials, + GPUPtr e, + GPUPtr f, + GPUPtr g, + GPUPtr h, + unsigned int numOps, + unsigned int intervalNumber, + unsigned int start, + unsigned int end) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); #endif - int parameterCountV = 8; + int parameterCountV = 6; int totalParameterCount = 10; gpu->LaunchKernel(fReduceWithinInterval, bgBastaReductionBlock, bgBastaReductionGrid, parameterCountV, totalParameterCount, - e, f, g, h, startPartials1, startPartials2, endPartials1, endPartials2, intervalNUmber, child2PartialIndex); + operations, partials, e, f, g, h, numOps, intervalNumber, start, end); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); @@ -2472,6 +2499,7 @@ void KernelLauncher::reduceWithinInterval(GPUPtr e, } + void KernelLauncher::reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index c25a1ae6..ba84a9fd 100644 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -616,12 +616,15 @@ class KernelLauncher { GPUPtr dPatternWeights, unsigned int patternCount); - void InnerBastaPartialsCoalescent(GPUPtr partials1, GPUPtr partials2, GPUPtr partials3, GPUPtr matrices1, - GPUPtr matrices2, GPUPtr accumulation1, GPUPtr accumulation2, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, unsigned int patternCount, unsigned int child2Index); + // void InnerBastaPartialsCoalescent(GPUPtr partials1, GPUPtr partials2, GPUPtr partials3, GPUPtr matrices1, + // GPUPtr matrices2, GPUPtr accumulation1, GPUPtr accumulation2, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, unsigned int patternCount, unsigned int child2Index); - void reduceWithinInterval(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr startPartials1, GPUPtr startPartials2, - GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex); + void InnerBastaPartialsCoalescent(GPUPtr partials, GPUPtr matrices, +GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, unsigned int start, unsigned int numOps, unsigned int patternCount); + // void reduceWithinInterval(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr startPartials1, GPUPtr startPartials2, + // GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex, unsigned int renew); + void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, unsigned int numOps, unsigned int intervalNumber,unsigned int start, unsigned int end); void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); void SetupKernelBlocksAndGrids(); diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 266327be..a23f6838 100644 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -1979,45 +1979,48 @@ KW_GLOBAL_KERNEL void kernelAccumulateFactorsAutoScaling(KW_GLOBAL_VAR signed ch * BASTA kernels */ -KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_RESTRICT partials1, - KW_GLOBAL_VAR REAL* KW_RESTRICT partials2, - KW_GLOBAL_VAR REAL* KW_RESTRICT partials3, - KW_GLOBAL_VAR REAL* KW_RESTRICT matrices1, - KW_GLOBAL_VAR REAL* KW_RESTRICT matrices2, - KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation1, - KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation2, +KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_RESTRICT partials, + KW_GLOBAL_VAR REAL* KW_RESTRICT matrices, + KW_GLOBAL_VAR REAL* KW_RESTRICT operations, KW_GLOBAL_VAR REAL* KW_RESTRICT sizes, KW_GLOBAL_VAR REAL* KW_RESTRICT coalescent, int intervalNumber, - int totalPatterns, - int child2Index) { + int start, + int numOps, + int totalPatterns) { - #define PATTERN_BLOCK_SIZE_B 1 - #define BLOCK_PEELING_SIZE_B 1 + #define PATTERN_BLOCK_SIZE_B 32 + #define BLOCK_PEELING_SIZE_B PADDED_STATE_COUNT int state = KW_LOCAL_ID_0; int patIdx = KW_LOCAL_ID_1; - int pattern = __umul24(KW_GROUP_ID_0,PATTERN_BLOCK_SIZE) + patIdx; - int matrix = KW_GROUP_ID_1; - int patternCount = totalPatterns; - int deltaPartialsByState = pattern * PADDED_STATE_COUNT; - int deltaPartialsByMatrix = matrix * PADDED_STATE_COUNT * patternCount; - int deltaMatrix = matrix * PADDED_STATE_COUNT * PADDED_STATE_COUNT; - int u = state + deltaPartialsByState + deltaPartialsByMatrix; - KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices1 + deltaMatrix; /* Points to *this* matrix */ - /* Load values into shared memory */ + int pattern = __umul24(KW_GROUP_ID_0,PATTERN_BLOCK_SIZE_B) + patIdx; + KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + int op = pattern + start; + int maxOp = start + totalPatterns - 1; + int desIndex = operations[op * numOps]; + int child1PartialIndex = operations[op * numOps + 1]; + int child1TransIndex = operations[maxOp * numOps + 2]; + int child2PartialIndex = operations[op * numOps + 3]; + int child2TransIndex = operations[op * numOps + 4]; + int accumulation1PartialIndex = operations[op * numOps + 5]; + int accumulation2PartialIndex = operations[op * numOps + 6]; + + KW_LOCAL_MEM REAL sPartials3[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL popSizes[PADDED_STATE_COUNT]; - - int y = deltaPartialsByState + deltaPartialsByMatrix; if (pattern < totalPatterns) { - sPartials1[patIdx][state] = partials1[y + state]; + KW_GLOBAL_VAR REAL* KW_RESTRICT partials1 = partials + child1PartialIndex; + sPartials1[patIdx][state] = partials1[state]; } else { sPartials1[patIdx][state] = 0; } REAL sum1 = 0; + + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix1 = matrices + child1TransIndex; for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE_B) { /* load one row of matrices */ if (patIdx < BLOCK_PEELING_SIZE_B) { @@ -2033,45 +2036,53 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ KW_LOCAL_FENCE; } - if (pattern < totalPatterns) - partials3[u] = sum1; + + if (pattern < totalPatterns) { + KW_GLOBAL_VAR REAL* KW_RESTRICT partials3 = partials + desIndex; + partials3[state] = sum1; + } + + /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ /* These are all coherent global memory reads; checked in Profiler */ - if (child2Index >= 0) { - KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices2 + deltaMatrix; - KW_LOCAL_MEM REAL sMatrix2[BLOCK_PEELING_SIZE_B][PADDED_STATE_COUNT]; + if (child2PartialIndex >= 0) { KW_LOCAL_MEM REAL sPartials2[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartials3[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL popSizes[PADDED_STATE_COUNT]; if (pattern < totalPatterns) { - sPartials2[patIdx][state] = partials2[y + state]; + KW_GLOBAL_VAR REAL* KW_RESTRICT partials2 = partials + child2PartialIndex; + sPartials2[patIdx][state] = partials2[state]; popSizes[state] = sizes[state]; } else { sPartials2[patIdx][state] = 0; + popSizes[state] = 0; } + + if (patIdx == 0) { + popSizes[state] = sizes[state]; + } + REAL sum2 = 0; - for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE_B) { - /* load one row of matrices */ - if (patIdx < BLOCK_PEELING_SIZE_B) { - /* These are all coherent global memory reads. */ - sMatrix2[patIdx][state] = matrix2[patIdx * PADDED_STATE_COUNT + state]; - /* sMatrix now filled with starting in state and ending in i */ - matrix2 += BLOCK_PEELING_SIZE_B * PADDED_STATE_COUNT; - } - KW_LOCAL_FENCE; + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE_B) { for(int j = 0; j < BLOCK_PEELING_SIZE_B; j++) { - FMA(sMatrix2[j][state], sPartials2[patIdx][i + j], sum2); + FMA(sMatrix1[j][state], sPartials2[patIdx][i + j], sum2); } KW_LOCAL_FENCE; } + if (pattern < totalPatterns) { - accumulation1[u] = sum1; - accumulation2[u] = sum2; - partials3[u] = sum1 * sum2 / popSizes[state]; - sPartials3[patIdx][state] = partials3[y + state]; + KW_GLOBAL_VAR REAL* KW_RESTRICT partials3 = partials + desIndex; + KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation1 = partials + accumulation1PartialIndex; + KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation2 = partials + accumulation2PartialIndex; + accumulation1[state] = sum1; + accumulation2[state] = sum2; + if (popSizes[state] > 0) { + partials3[state] = sum1 * sum2 / popSizes[state]; + } else { + partials3[state] = 0; + } + sPartials3[patIdx][state] = partials3[state]; #ifdef IS_POWER_OF_TWO // parallelized reduction *** only works for powers-of-2 **** @@ -2086,114 +2097,234 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ KW_LOCAL_FENCE; } REAL denominator = sPartials3[patIdx][0]; - partials3[u] = partials3[u] / denominator; + partials3[state] = partials3[state] / denominator; coalescent[intervalNumber] = denominator; } } } -KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* e, - KW_GLOBAL_VAR REAL* f, - KW_GLOBAL_VAR REAL* g, - KW_GLOBAL_VAR REAL* h, - KW_GLOBAL_VAR REAL* KW_RESTRICT startPartials1, - KW_GLOBAL_VAR REAL* KW_RESTRICT startPartials2, - KW_GLOBAL_VAR REAL* KW_RESTRICT endPartials1, - KW_GLOBAL_VAR REAL* KW_RESTRICT endPartials2, - int intervalNumber, - int child2PartialIndex) { - - #define SUM_SITES_BLOCK_SIZE_B 64 - #define SUM_PARTIAL_BLOCK_SIZE_B 4 - int state = KW_LOCAL_ID_0; - int u = KW_LOCAL_ID_0 + KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE_B; - int y = intervalNumber * PADDED_STATE_COUNT; - KW_LOCAL_MEM REAL sPartials1[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - - if (state < PADDED_STATE_COUNT) { - sPartials1[0][state] = startPartials1[u]; - sPartials1[1][state] = startPartials1[u] * startPartials1[u]; - sPartials1[2][state] = endPartials1[u]; - sPartials1[3][state] = endPartials1[u] * endPartials1[u]; - } else { - sPartials1[0][state] = 0; - sPartials1[1][state] = 0; - sPartials1[2][state] = 0; - sPartials1[3][state] = 0; - } - e[y + state] += sPartials1[0][state]; - f[y + state] += sPartials1[1][state]; - g[y + state] += sPartials1[2][state]; - h[y + state] += sPartials1[3][state]; - KW_LOCAL_MEM REAL sPartials2[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - if (child2PartialIndex >= 0) { - if (state < PADDED_STATE_COUNT) { - sPartials2[0][state] = startPartials2[u]; - sPartials2[1][state] = startPartials2[u] * startPartials2[u]; - sPartials2[2][state] = endPartials2[u]; - sPartials2[3][state] = endPartials2[u] * endPartials2[u]; - } else { - sPartials2[0][state] = 0; - sPartials2[1][state] = 0; - sPartials2[2][state] = 0; - sPartials2[3][state] = 0; - } - e[y + state] += sPartials2[0][state]; - f[y + state] += sPartials2[1][state]; - g[y + state] += sPartials2[2][state]; - h[y + state] += sPartials2[3][state]; - } -} +//KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* e, + //KW_GLOBAL_VAR REAL* f, + //KW_GLOBAL_VAR REAL* g, + //KW_GLOBAL_VAR REAL* h, + //KW_GLOBAL_VAR REAL* KW_RESTRICT startPartials1, + //KW_GLOBAL_VAR REAL* KW_RESTRICT startPartials2, + //KW_GLOBAL_VAR REAL* KW_RESTRICT endPartials1, + //KW_GLOBAL_VAR REAL* KW_RESTRICT endPartials2, + //int intervalNumber, + //int child2PartialIndex, + //int renew) { + + //#define SUM_SITES_BLOCK_SIZE_B 64 + //#define SUM_PARTIAL_BLOCK_SIZE_B 4 + //int state = KW_LOCAL_ID_0; + //int u = KW_LOCAL_ID_0 + KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE_B; + //int y = intervalNumber * PADDED_STATE_COUNT; + //KW_LOCAL_MEM REAL sPartials1[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + + //if (state < PADDED_STATE_COUNT) { + //sPartials1[0][state] = startPartials1[u]; + //sPartials1[1][state] = startPartials1[u] * startPartials1[u]; + //sPartials1[2][state] = endPartials1[u]; + //sPartials1[3][state] = endPartials1[u] * endPartials1[u]; + //} else { + //sPartials1[0][state] = 0; + //sPartials1[1][state] = 0; + //sPartials1[2][state] = 0; + //sPartials1[3][state] = 0; + //} + + //if (renew == 0) { + //e[y + state] = sPartials1[0][state]; + //f[y + state] = sPartials1[1][state]; + //g[y + state] = sPartials1[2][state]; + //h[y + state] = sPartials1[3][state]; + //} else { + //e[y + state] += sPartials1[0][state]; + //f[y + state] += sPartials1[1][state]; + //g[y + state] += sPartials1[2][state]; + //h[y + state] += sPartials1[3][state]; + //} + + //KW_LOCAL_MEM REAL sPartials2[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + //if (child2PartialIndex >= 0) { + //if (state < PADDED_STATE_COUNT) { + //sPartials2[0][state] = startPartials2[u]; + //sPartials2[1][state] = startPartials2[u] * startPartials2[u]; + //sPartials2[2][state] = endPartials2[u]; + //sPartials2[3][state] = endPartials2[u] * endPartials2[u]; + //} else { + //sPartials2[0][state] = 0; + //sPartials2[1][state] = 0; + //sPartials2[2][state] = 0; + //sPartials2[3][state] = 0; + ///} + //e[y + state] += sPartials2[0][state]; + //f[y + state] += sPartials2[1][state]; + //g[y + state] += sPartials2[2][state]; + //h[y + state] += sPartials2[3][state]; + //} +//} + +KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, + KW_GLOBAL_VAR REAL* KW_RESTRICT partials, + KW_GLOBAL_VAR REAL* e, + KW_GLOBAL_VAR REAL* f, + KW_GLOBAL_VAR REAL* g, + KW_GLOBAL_VAR REAL* h, + int numOps, + int intervalNumber, + int start, + int end) { + +#define SUM_PARTIAL_BLOCK_SIZE_B 32 +#define SUM_PARTIAL_GRID_SIZE_B 1 + + int state = KW_LOCAL_ID_0; + int opIdx = KW_LOCAL_ID_1; + int opBlock = KW_GROUP_ID_0; + int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; + int y = intervalNumber * PADDED_STATE_COUNT; + int opCount = end - start; + KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + + if (opNumber < opCount && state < PADDED_STATE_COUNT) { + int op = opNumber + start; + int child1PartialIndex = operations[op * numOps + 1]; + int accumulation1PartialIndex = operations[op * numOps + 5]; + KW_GLOBAL_VAR REAL* startPartials1 = partials + child1PartialIndex; + KW_GLOBAL_VAR REAL* endPartials1 = partials + accumulation1PartialIndex; + + //if (state == 0 && opIdx == 0) { + // printf("start %d\n", start); + //printf("end %d\n", end); + // printf("ops %d\n", child1PartialIndex / PADDED_STATE_COUNT); + // printf("opIdx %d\n", opIdx); + // printf("added0 %1.2e\n", startPartials1[0]); + // printf("added1 %1.2e\n", startPartials1[1]); + // printf("added2 %1.2e\n", startPartials1[2]); + // printf("added3 %1.2e\n", startPartials1[3]); + //} + + sPartialsE[opIdx][state] = startPartials1[state]; + sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; + sPartialsG[opIdx][state] = endPartials1[state]; + sPartialsH[opIdx][state] = endPartials1[state] * endPartials1[state]; + } else { + sPartialsE[opIdx][state] = 0; + sPartialsF[opIdx][state] = 0; + sPartialsG[opIdx][state] = 0; + sPartialsH[opIdx][state] = 0; + } -KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, - KW_GLOBAL_VAR REAL* KW_RESTRICT f, - KW_GLOBAL_VAR REAL* KW_RESTRICT g, - KW_GLOBAL_VAR REAL* KW_RESTRICT h, - KW_GLOBAL_VAR REAL* KW_RESTRICT distance, - KW_GLOBAL_VAR REAL* KW_RESTRICT dLogL, - KW_GLOBAL_VAR REAL* KW_RESTRICT sizes, - KW_GLOBAL_VAR REAL* KW_RESTRICT coalescent, - int intervalNumber) { - -#define SUM_SITES_BLOCK_SIZE_B2 1 - int state = KW_LOCAL_ID_0; - int u = KW_LOCAL_ID_0 + KW_GROUP_ID_0 * SUM_SITES_BLOCK_SIZE_B2; - int y = intervalNumber * PADDED_STATE_COUNT; - - KW_LOCAL_MEM REAL sPartials1[PADDED_STATE_COUNT]; - - if (state < PADDED_STATE_COUNT) { - sPartials1[state] = (e[y + u] * e[y + u] - f[y + u] + - g[y + u] * g[y + u] - h[y + u]) / sizes[u]; + KW_LOCAL_FENCE; + for (int s = SUM_PARTIAL_BLOCK_SIZE_B / 2; s > 0; s >>= 1) { + if (opIdx < s && opIdx + s < opCount) { + sPartialsE[opIdx][state] += sPartialsE[opIdx + s][state]; + sPartialsF[opIdx][state] += sPartialsF[opIdx + s][state]; + sPartialsG[opIdx][state] += sPartialsG[opIdx + s][state]; + sPartialsH[opIdx][state] += sPartialsH[opIdx + s][state]; } KW_LOCAL_FENCE; + } -#ifdef IS_POWER_OF_TWO - // parallelized reduction *** only works for powers-of-2 **** - for (int i = PADDED_STATE_COUNT / 2; i > 0; i >>= 1) { - if (state < i) { -#else - for (int i = SMALLEST_POWER_OF_TWO / 2; i > 0; i >>= 1) { - if (state < i && state + i < PADDED_STATE_COUNT ) { -#endif // IS_POWER_OF_TWO - sPartials1[state] += sPartials1[state + i]; - } - KW_LOCAL_FENCE; + KW_LOCAL_FENCE; + if (opIdx == 0 && (state < PADDED_STATE_COUNT)) { + e[y + state] = sPartialsE[0][state]; + f[y + state] = sPartialsF[0][state]; + g[y + state] = sPartialsG[0][state]; + h[y + state] = sPartialsH[0][state]; + } + + KW_LOCAL_FENCE; + + if (opNumber < opCount && state < PADDED_STATE_COUNT) { + int op = opNumber + start; + int child2PartialIndex = operations[op * numOps + 3]; + int accumulation2PartialIndex = operations[op * numOps + 6]; + KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; + KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; + + if (child2PartialIndex >= 0) { + e[y + state] += startPartials2[state]; + f[y + state] += startPartials2[state] * startPartials2[state]; + g[y + state] += endPartials2[state]; + h[y + state] += endPartials2[state] * endPartials2[state]; } + } +} - if (state == 0) { - REAL temp = -distance[intervalNumber] * sPartials1[0] / 4; - REAL prob = coalescent[intervalNumber]; - if (prob != 0) { - temp += logf(prob); - } - dLogL[0] += temp; + + + + + + + +KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, + KW_GLOBAL_VAR REAL* KW_RESTRICT f, + KW_GLOBAL_VAR REAL* KW_RESTRICT g, + KW_GLOBAL_VAR REAL* KW_RESTRICT h, + KW_GLOBAL_VAR REAL* KW_RESTRICT distance, + KW_GLOBAL_VAR REAL* KW_RESTRICT dLogL, + KW_GLOBAL_VAR REAL* KW_RESTRICT sizes, + KW_GLOBAL_VAR REAL* KW_RESTRICT coalescent, + int intervalStartsCount) { + +#define SUM_INTERVAL_BLOCK_SIZE 32 + + int intervalCount = intervalStartsCount - 1; + int tid = KW_LOCAL_ID_0; + int tidTotal = __umul24(KW_GROUP_ID_0, SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT) + tid; + int state = tid % PADDED_STATE_COUNT; + int intervalIdx = tid / PADDED_STATE_COUNT; + int intervalNumber = __umul24(KW_GROUP_ID_0, SUM_INTERVAL_BLOCK_SIZE) + intervalIdx; + int u = state + intervalNumber * PADDED_STATE_COUNT; + + KW_LOCAL_MEM REAL sPartials1[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials2[SUM_INTERVAL_BLOCK_SIZE]; + + if (intervalNumber < intervalCount && (sizes[state] > 0)) { + sPartials1[tid] = (e[u] * e[u] - f[u] + + g[u] * g[u] - h[u]) * distance[intervalNumber] / sizes[state]; + } else { + sPartials1[tid] = 0; + } + KW_LOCAL_FENCE; + + if (tidTotal < intervalCount && (coalescent[u] != 0)) { + sPartials2[tid] = logf(coalescent[u]); + } else { + sPartials2[tid] = 0; + } + KW_LOCAL_FENCE; + + + for (int i = SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT / 2; i > 0; i >>= 1) { + if (tid < i) { + sPartials1[tid] += sPartials1[tid + i]; } + KW_LOCAL_FENCE; + } -} + for (int i = SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT / 2; i > 0; i >>= 1) { + if (tid < i) { + sPartials2[tid] += sPartials2[tid + i]; + } + KW_LOCAL_FENCE; + } + + if (tid == 0) { + REAL temp = - sPartials1[0] / 4 + sPartials2[0]; + dLogL[KW_GROUP_ID_0] = temp; + } + } #ifdef CUDA From 5820a41de995a5e7e34043f9eca06793956296d1 Mon Sep 17 00:00:00 2001 From: yucais Date: Tue, 25 Jun 2024 09:33:01 -0700 Subject: [PATCH 183/421] some optimization to reduction --- libhmsbeagle/GPU/BeagleGPUImpl.h | 1 + libhmsbeagle/GPU/BeagleGPUImpl.hpp | 53 +++++++++++++++---------- libhmsbeagle/GPU/KernelLauncher.cpp | 26 ++++++------ libhmsbeagle/GPU/KernelLauncher.h | 2 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 55 +++++++++++++++----------- 5 files changed, 79 insertions(+), 58 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 57b2839b..860952d3 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -221,6 +221,7 @@ class BeagleGPUImpl : public BeagleImpl { GPUPtr dBastaDistance; GPUPtr dBastaOperationQueue; GPUPtr dCoalescentBuffers; + GPUPtr dBastaInterval; int kCoalescentBufferLength; int kCoalescentBufferCount; int kBastaIntervalBlockCount; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index c990560c..088ca9ee 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -85,7 +85,7 @@ namespace gpu { dBastaOperationQueue = (GPUPtr)NULL; dBastaBuffers = (GPUPtr)NULL; dCoalescentBuffers = (GPUPtr)NULL; - + dBastaInterval = (GPUPtr)NULL;; dEigenValues = NULL; dEvec = NULL; dIevc = NULL; @@ -262,6 +262,7 @@ BeagleGPUImpl::~BeagleGPUImpl() { gpu->FreeMemory(dBastaLogL); gpu->FreeMemory(dBastaDistance); gpu->FreeMemory(dBastaOperationQueue); + gpu->FreeMemory(dBastaInterval); gpu->FreeHostMemory(hPtrQueue); @@ -595,7 +596,10 @@ int BeagleGPUImpl::createInstance(int tipCount, for (int i = 0; i < kMatrixCount; i++) { dMatrices[i] = gpu->CreateSubPointer(dMatricesOrigin, ptrIncrement*i, ptrIncrement); } - + fprintf(stderr, "The matrice increment is: %d\n", ptrIncrement); + fprintf(stderr, "The matrix size is: %d\n", kMatrixSize); + fprintf(stderr, "The state size is: %d\n", kPaddedStateCount); + fprintf(stderr, "The matrix count is: %d\n", kMatrixCount); if (kScaleBufferCount > 0) { if (kFlags & BEAGLE_FLAG_SCALING_AUTO) { dScalingFactors = (GPUPtr*) malloc(sizeof(GPUPtr) * kScaleBufferCount); @@ -690,7 +694,9 @@ int BeagleGPUImpl::createInstance(int tipCount, ptrIncrement = gpu->AlignMemOffset(kPartialsSize * sizeof(Real)); GPUPtr dPartialsTmpOrigin = gpu->AllocateMemory(partialsBufferCountTotal * ptrIncrement); dPartialsOrigin = gpu->CreateSubPointer(dPartialsTmpOrigin, 0, ptrIncrement); + fprintf(stderr, "The increment is: %d\n", ptrIncrement); hPartialsOffsets = (unsigned int*) calloc(sizeof(unsigned int), bufferCountTotal); + fprintf(stderr, "The partial buffer count is: %d\n", partialsBufferCountTotal); kIndexOffsetPat = gpu->AlignMemOffset(kPartialsSize * sizeof(Real)) / sizeof(Real); size_t ptrIncrementStates = gpu->AlignMemOffset(kPaddedPatternCount * sizeof(int)); @@ -2238,12 +2244,10 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, kBastaIntervalBlockCount = bufferLength / 32 + 1; dCoalescentBuffers = gpu->AllocateMemory(kCoalescentBufferCount * kCoalescentBufferLength * sizeof(Real)); - dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaLogL = gpu->AllocateMemory(kBastaIntervalBlockCount * sizeof(Real)); dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); + dBastaInterval = gpu->AllocateMemory(kBufferCount * sizeof(Real)); // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); @@ -2281,7 +2285,8 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + // GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2441,14 +2446,22 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + // GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; - + const int chunkSize = 64; // Real* hBastaOperationQueue = (Real*) gpu->CallocHost(sizeof(Real), operationCount * numOps); - - - // + Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + for (int i = 0; i < operationCount; ++i) { + hBastaInterval[i] = (Real) operations[i * numOps + 7]; + } + gpu->MemcpyHostToDevice(dBastaInterval, hBastaInterval, sizeof(Real) * operationCount); + gpu->FreeHostMemory(hBastaInterval); + dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); // for (int i = 0; i < operationCount; ++i) { // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; // hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; @@ -2479,10 +2492,10 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // fprintf(stderr, "h:\n"); // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { + for (int i = 0; i < operationCount; i += chunkSize) { // TODO execute in parallel (no race conditions) - const int start = intervalStarts[interval]; - const int end = intervalStarts[interval + 1]; + const int start = i; + const int end = std::min(operationCount, i + chunkSize); // const int child1PartialIndex = operations[op * numOps + 1]; @@ -2495,9 +2508,9 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; // Real r = 0; - kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaE, dBastaF, dBastaG, dBastaH, numOps, - interval, start, end); - // fprintf(stderr, "The interval is: %d\n", interval); + + kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaE, dBastaF, dBastaG, dBastaH, dBastaInterval, numOps, + start, end); // fprintf(stderr, "e:\n"); // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); // fprintf(stderr, "f:\n"); @@ -2554,8 +2567,8 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat gpu->MemcpyHostToDevice(dBastaDistance, hBastaDistance, sizeof(Real) * kCoalescentBufferLength); - // kernels->reduceAcrossIntervals(dBastaE, dBastaF, dBastaG, dBastaH, dBastaDistance, dBastaLogL, sizes, coalescent, - // intervalStartsCount); + kernels->reduceAcrossIntervals(dBastaE, dBastaF, dBastaG, dBastaH, dBastaDistance, dBastaLogL, sizes, coalescent, + intervalStartsCount); // fprintf(stderr, "The block count is: %d\n", kBastaIntervalBlockCount); // fprintf(stderr, "logL:\n"); // gpu->PrintfDeviceVector(dBastaLogL, kBastaIntervalBlockCount, r); diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index b7c781c2..029e6a24 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -241,7 +241,7 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgBastaPeelingBlock = Dim3Int(kPaddedStateCount, 32); bgBastaPeelingGrid = Dim3Int(1,1); - bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 32); + bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 64); bgBastaReductionGrid = Dim3Int(1); bgBastaSumBlock = Dim3Int(128); @@ -2471,26 +2471,26 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, // // } -void KernelLauncher::reduceWithinInterval(GPUPtr operations, - GPUPtr partials, - GPUPtr e, - GPUPtr f, - GPUPtr g, - GPUPtr h, - unsigned int numOps, - unsigned int intervalNumber, - unsigned int start, - unsigned int end) { + void KernelLauncher::reduceWithinInterval(GPUPtr operations, + GPUPtr partials, + GPUPtr e, + GPUPtr f, + GPUPtr g, + GPUPtr h, + GPUPtr intervals, + unsigned int numOps, + unsigned int start, + unsigned int end) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); #endif - int parameterCountV = 6; + int parameterCountV = 7; int totalParameterCount = 10; gpu->LaunchKernel(fReduceWithinInterval, bgBastaReductionBlock, bgBastaReductionGrid, parameterCountV, totalParameterCount, - operations, partials, e, f, g, h, numOps, intervalNumber, start, end); + operations, partials, e, f, g, h, intervals, numOps, start, end); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index ba84a9fd..b037bbca 100644 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -624,7 +624,7 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, // void reduceWithinInterval(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr startPartials1, GPUPtr startPartials2, // GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex, unsigned int renew); - void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, unsigned int numOps, unsigned int intervalNumber,unsigned int start, unsigned int end); + void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end); void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); void SetupKernelBlocksAndGrids(); diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index a23f6838..deb5bc9c 100644 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2174,31 +2174,33 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES KW_GLOBAL_VAR REAL* f, KW_GLOBAL_VAR REAL* g, KW_GLOBAL_VAR REAL* h, + KW_GLOBAL_VAR REAL* intervals, int numOps, - int intervalNumber, int start, int end) { -#define SUM_PARTIAL_BLOCK_SIZE_B 32 +#define SUM_PARTIAL_BLOCK_SIZE_B 64 #define SUM_PARTIAL_GRID_SIZE_B 1 int state = KW_LOCAL_ID_0; int opIdx = KW_LOCAL_ID_1; int opBlock = KW_GROUP_ID_0; int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; - int y = intervalNumber * PADDED_STATE_COUNT; int opCount = end - start; KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsIntervals1[SUM_PARTIAL_BLOCK_SIZE_B]; + KW_LOCAL_MEM REAL sPartialsIntervals2[SUM_PARTIAL_BLOCK_SIZE_B]; - if (opNumber < opCount && state < PADDED_STATE_COUNT) { + if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { int op = opNumber + start; int child1PartialIndex = operations[op * numOps + 1]; int accumulation1PartialIndex = operations[op * numOps + 5]; KW_GLOBAL_VAR REAL* startPartials1 = partials + child1PartialIndex; KW_GLOBAL_VAR REAL* endPartials1 = partials + accumulation1PartialIndex; + KW_GLOBAL_VAR REAL* partialIntervals = intervals + start; //if (state == 0 && opIdx == 0) { // printf("start %d\n", start); @@ -2215,30 +2217,42 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; sPartialsG[opIdx][state] = endPartials1[state]; sPartialsH[opIdx][state] = endPartials1[state] * endPartials1[state]; + sPartialsIntervals1[opIdx] = partialIntervals[opIdx]; + sPartialsIntervals2[opIdx] = partialIntervals[opIdx]; } else { sPartialsE[opIdx][state] = 0; sPartialsF[opIdx][state] = 0; sPartialsG[opIdx][state] = 0; sPartialsH[opIdx][state] = 0; + sPartialsIntervals1[opIdx] = -1; + sPartialsIntervals2[opIdx] = -1; } KW_LOCAL_FENCE; + + int intervalNumber = sPartialsIntervals1[opNumber]; + int y = intervalNumber * PADDED_STATE_COUNT; for (int s = SUM_PARTIAL_BLOCK_SIZE_B / 2; s > 0; s >>= 1) { - if (opIdx < s && opIdx + s < opCount) { - sPartialsE[opIdx][state] += sPartialsE[opIdx + s][state]; - sPartialsF[opIdx][state] += sPartialsF[opIdx + s][state]; - sPartialsG[opIdx][state] += sPartialsG[opIdx + s][state]; - sPartialsH[opIdx][state] += sPartialsH[opIdx + s][state]; + int neighbor = opIdx + s; + if (neighbor < opCount && (sPartialsIntervals1[opIdx] == sPartialsIntervals1[neighbor])) { + sPartialsIntervals1[neighbor] = -1; + sPartialsE[opIdx][state] += sPartialsE[neighbor][state]; + sPartialsF[opIdx][state] += sPartialsF[neighbor][state]; + sPartialsG[opIdx][state] += sPartialsG[neighbor][state]; + sPartialsH[opIdx][state] += sPartialsH[neighbor][state]; } KW_LOCAL_FENCE; } KW_LOCAL_FENCE; - if (opIdx == 0 && (state < PADDED_STATE_COUNT)) { - e[y + state] = sPartialsE[0][state]; - f[y + state] = sPartialsF[0][state]; - g[y + state] = sPartialsG[0][state]; - h[y + state] = sPartialsH[0][state]; + + if ((opIdx == 0 || sPartialsIntervals2[opIdx] != sPartialsIntervals2[opIdx - 1]) && (opNumber < opCount) && (state < PADDED_STATE_COUNT)) { + if (y >= 0) { + e[y + state] += sPartialsE[opIdx][state]; + f[y + state] += sPartialsF[opIdx][state]; + g[y + state] += sPartialsG[opIdx][state]; + h[y + state] += sPartialsH[opIdx][state]; + } } KW_LOCAL_FENCE; @@ -2247,10 +2261,11 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES int op = opNumber + start; int child2PartialIndex = operations[op * numOps + 3]; int accumulation2PartialIndex = operations[op * numOps + 6]; - KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; - KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; if (child2PartialIndex >= 0) { + KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; + KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; + e[y + state] += startPartials2[state]; f[y + state] += startPartials2[state] * startPartials2[state]; g[y + state] += endPartials2[state]; @@ -2259,14 +2274,6 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES } } - - - - - - - - KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, KW_GLOBAL_VAR REAL* KW_RESTRICT f, KW_GLOBAL_VAR REAL* KW_RESTRICT g, From 36dbb002289861b1d343f26be2cfda81ba4c89a1 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 25 Jun 2024 15:03:00 -0400 Subject: [PATCH 184/421] Specialize calculateRootLogLikelihoods( ) --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 23 +++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 146 +++++++++++++++++++++++ 2 files changed, 168 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index df20dc53..3e3c9950 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -548,7 +548,21 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::dPartialsOrigin; using BeagleGPUImpl::hCategoryRates; - + // For calculateRootLogLikelihoods + using BeagleGPUImpl::dAccumulatedScalingFactors; + using BeagleGPUImpl::dIntegrationTmp; + using BeagleGPUImpl::dWeights; + using BeagleGPUImpl::dFrequencies; + using BeagleGPUImpl::dSumLogLikelihood; + using BeagleGPUImpl::dPatternWeights; + using BeagleGPUImpl::hLogLikelihoodsCache; + using BeagleGPUImpl::kSumSitesBlockCount; + using BeagleGPUImpl::kPatternCount; + using BeagleGPUImpl::kScaleBufferSize; + using BeagleGPUImpl::dPtrQueue; + using BeagleGPUImpl::hPtrQueue; + using BeagleGPUImpl::dMaxScalingFactors; + using BeagleGPUImpl::dIndexMaxScalingFactors; int upPartials(bool byPartition, const int *operations, @@ -560,6 +574,13 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int operationCount, int cumulativeScalingIndex); + int calculateRootLogLikelihoods(const int* bufferIndices, + const int* categoryWeightsIndices, + const int* stateFrequenciesIndices, + const int* cumulativeScaleIndices, + int count, + double* outSumLogLikelihood); + ~BeagleGPUActionImpl(); private: diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5d2e6987..5361595e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1022,6 +1022,152 @@ double BeagleGPUActionImpl::getDValue(int p, int eigenIndex) return hds[eigenIndex][p]; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::calculateRootLogLikelihoods(const int* bufferIndices, + const int* categoryWeightsIndices, + const int* stateFrequenciesIndices, + const int* cumulativeScaleIndices, + int count, + double* outSumLogLikelihood) { + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tEntering BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); +#endif + + int returnCode = BEAGLE_SUCCESS; + + if (count == 1) { + const int rootNodeIndex = bufferIndices[0]; + const int categoryWeightsIndex = categoryWeightsIndices[0]; + const int stateFrequenciesIndex = stateFrequenciesIndices[0]; + + + GPUPtr dCumulativeScalingFactor; + bool scale = 1; + if (kFlags & BEAGLE_FLAG_SCALING_AUTO) + dCumulativeScalingFactor = dAccumulatedScalingFactors; + else if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) + dCumulativeScalingFactor = dScalingFactors[bufferIndices[0] - kTipCount]; + else if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE) + dCumulativeScalingFactor = dScalingFactors[cumulativeScaleIndices[0]]; + else + scale = 0; + +#ifdef BEAGLE_DEBUG_VALUES + Real r = 0; + fprintf(stderr,"root partials = \n"); + gpu->PrintfDeviceVector(dPartials[rootNodeIndex], kPaddedPatternCount, r); +#endif + + if (scale) { + kernels->IntegrateLikelihoodsDynamicScaling(dIntegrationTmp, dPartials[rootNodeIndex], + dWeights[categoryWeightsIndex], + dFrequencies[stateFrequenciesIndex], + dCumulativeScalingFactor, + kPaddedPatternCount, + kCategoryCount); + } else { + kernels->IntegrateLikelihoods(dIntegrationTmp, dPartials[rootNodeIndex], + dWeights[categoryWeightsIndex], + dFrequencies[stateFrequenciesIndex], + kPaddedPatternCount, kCategoryCount); + } + +#ifdef BEAGLE_DEBUG_VALUES + fprintf(stderr,"before SumSites1 = \n"); + gpu->PrintfDeviceVector(dIntegrationTmp, kPaddedPatternCount, r); +#endif + + kernels->SumSites1(dIntegrationTmp, dSumLogLikelihood, dPatternWeights, + kPatternCount); + + if (kFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) { + gpu->MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); + + *outSumLogLikelihood = 0.0; + for (int i = 0; i < kSumSitesBlockCount; i++) { + if (hLogLikelihoodsCache[i] != hLogLikelihoodsCache[i]) + returnCode = BEAGLE_ERROR_FLOATING_POINT; + + *outSumLogLikelihood += hLogLikelihoodsCache[i]; + } + } + + } else { + // TODO: evaluate performance, maybe break up kernels below for each subsetIndex case + + if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { + for(int n = 0; n < count; n++) { + int cumulativeScalingFactor = bufferIndices[n] - kTipCount; + hPtrQueue[n] = cumulativeScalingFactor * kScaleBufferSize; + } + gpu->MemcpyHostToDevice(dPtrQueue, hPtrQueue, sizeof(unsigned int) * count); + } else if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE) { + for(int n = 0; n < count; n++) + hPtrQueue[n] = cumulativeScaleIndices[n] * kScaleBufferSize; + gpu->MemcpyHostToDevice(dPtrQueue, hPtrQueue, sizeof(unsigned int) * count); + } + + for (int subsetIndex = 0 ; subsetIndex < count; ++subsetIndex ) { + + const GPUPtr tmpDWeights = dWeights[categoryWeightsIndices[subsetIndex]]; + const GPUPtr tmpDFrequencies = dFrequencies[stateFrequenciesIndices[subsetIndex]]; + const int rootNodeIndex = bufferIndices[subsetIndex]; + + if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE || (kFlags & BEAGLE_FLAG_SCALING_ALWAYS)) { + kernels->IntegrateLikelihoodsFixedScaleMulti(dIntegrationTmp, dPartials[rootNodeIndex], tmpDWeights, + tmpDFrequencies, dScalingFactors[0], dPtrQueue, dMaxScalingFactors, + dIndexMaxScalingFactors, + kPaddedPatternCount, + kCategoryCount, count, subsetIndex); + } else { + if (subsetIndex == 0) { + kernels->IntegrateLikelihoodsMulti(dIntegrationTmp, dPartials[rootNodeIndex], tmpDWeights, + tmpDFrequencies, + kPaddedPatternCount, kCategoryCount, 0); + } else if (subsetIndex == count - 1) { + kernels->IntegrateLikelihoodsMulti(dIntegrationTmp, dPartials[rootNodeIndex], tmpDWeights, + tmpDFrequencies, + kPaddedPatternCount, kCategoryCount, 1); + } else { + kernels->IntegrateLikelihoodsMulti(dIntegrationTmp, dPartials[rootNodeIndex], tmpDWeights, + tmpDFrequencies, + kPaddedPatternCount, kCategoryCount, 2); + } + } + + + kernels->SumSites1(dIntegrationTmp, dSumLogLikelihood, dPatternWeights, + kPatternCount); + + if (kFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) { + gpu->MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); + + *outSumLogLikelihood = 0.0; + for (int i = 0; i < kSumSitesBlockCount; i++) { + if (hLogLikelihoodsCache[i] != hLogLikelihoodsCache[i]) + returnCode = BEAGLE_ERROR_FLOATING_POINT; + + *outSumLogLikelihood += hLogLikelihoodsCache[i]; + } + } + } + } + +#ifdef BEAGLE_DEBUG_VALUES + Real r = 0; + fprintf(stderr, "parent = \n"); + gpu->PrintfDeviceVector(dIntegrationTmp, kPatternCount, r); +#endif + + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); +#endif + + return returnCode; +} + BEAGLE_GPU_TEMPLATE std::tuple BeagleGPUActionImpl::getStatistics2(double t, int nCol, From af76e5f105a4b1eb28634922215b654ae093e1e2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 25 Jun 2024 15:18:18 -0400 Subject: [PATCH 185/421] Use cublas for the dot product. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 12 ++++++++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 3e3c9950..218ba75b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -384,6 +384,18 @@ struct SpMatrixDevice } }; +template +void dotProduct(Real* out, cublasHandle_t handle, int n, Real* v1, Real* v2) +{ + if constexpr (std::is_same::value) + { + cublasSdot(handle, n, v1, 1, v2, 1, out); + } + else + { + cublasDdot(handle, n, v1, 1, v2, 1, out); + } +}; template auto normPInf(const DnMatrixDevice& M) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5361595e..2cc69d30 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1078,8 +1078,8 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i gpu->PrintfDeviceVector(dIntegrationTmp, kPaddedPatternCount, r); #endif - kernels->SumSites1(dIntegrationTmp, dSumLogLikelihood, dPatternWeights, - kPatternCount); + // Take the dot product of the pattern log-likelihoods and the pattern weights. + dotProduct((Real*)dSumLogLikelihood, cublasHandle, kPatternCount, (Real*)dIntegrationTmp, (Real*)dPatternWeights); if (kFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) { gpu->MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); From 977e584d12dcfc3e1ebe021b304c05aa5d8e92d3 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 25 Jun 2024 15:48:39 -0400 Subject: [PATCH 186/421] Chop out the multigrid version to simplify things. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 152 +++++++---------------- 1 file changed, 48 insertions(+), 104 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 2cc69d30..c653304e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1036,121 +1036,65 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i int returnCode = BEAGLE_SUCCESS; - if (count == 1) { - const int rootNodeIndex = bufferIndices[0]; - const int categoryWeightsIndex = categoryWeightsIndices[0]; - const int stateFrequenciesIndex = stateFrequenciesIndices[0]; - - - GPUPtr dCumulativeScalingFactor; - bool scale = 1; - if (kFlags & BEAGLE_FLAG_SCALING_AUTO) - dCumulativeScalingFactor = dAccumulatedScalingFactors; - else if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) - dCumulativeScalingFactor = dScalingFactors[bufferIndices[0] - kTipCount]; - else if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE) - dCumulativeScalingFactor = dScalingFactors[cumulativeScaleIndices[0]]; - else - scale = 0; + const int rootNodeIndex = bufferIndices[0]; + const int categoryWeightsIndex = categoryWeightsIndices[0]; + const int stateFrequenciesIndex = stateFrequenciesIndices[0]; + + GPUPtr dCumulativeScalingFactor; + bool scale = 1; + if (kFlags & BEAGLE_FLAG_SCALING_AUTO) + dCumulativeScalingFactor = dAccumulatedScalingFactors; + else if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) + dCumulativeScalingFactor = dScalingFactors[bufferIndices[0] - kTipCount]; + else if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE) + dCumulativeScalingFactor = dScalingFactors[cumulativeScaleIndices[0]]; + else + scale = 0; #ifdef BEAGLE_DEBUG_VALUES - Real r = 0; - fprintf(stderr,"root partials = \n"); - gpu->PrintfDeviceVector(dPartials[rootNodeIndex], kPaddedPatternCount, r); + Real r = 0; + fprintf(stderr,"root partials = \n"); + gpu->PrintfDeviceVector(dPartials[rootNodeIndex], kPaddedPatternCount, r); #endif - if (scale) { - kernels->IntegrateLikelihoodsDynamicScaling(dIntegrationTmp, dPartials[rootNodeIndex], - dWeights[categoryWeightsIndex], - dFrequencies[stateFrequenciesIndex], - dCumulativeScalingFactor, - kPaddedPatternCount, - kCategoryCount); - } else { - kernels->IntegrateLikelihoods(dIntegrationTmp, dPartials[rootNodeIndex], - dWeights[categoryWeightsIndex], - dFrequencies[stateFrequenciesIndex], - kPaddedPatternCount, kCategoryCount); - } + // We want to sum dRootPartials(category, pattern, state) * dWeights[category] * dFrequencies[state] + // over (category,state). + // dRootPartials(c,p,s) = dRootPartials[c*kPatternCount*kPaddedStateCount + p*kPaddedStateCount + s] + + if (scale) { + // See kernelIntegrateLikelihoodsAutoScaling in KernelsX.cu + kernels->IntegrateLikelihoodsDynamicScaling(dIntegrationTmp, dPartials[rootNodeIndex], + dWeights[categoryWeightsIndex], + dFrequencies[stateFrequenciesIndex], + dCumulativeScalingFactor, + kPaddedPatternCount, + kCategoryCount); + } else { + // See kernelIntegrateLikelihoods in KernelsX.cu + kernels->IntegrateLikelihoods(dIntegrationTmp, dPartials[rootNodeIndex], + dWeights[categoryWeightsIndex], + dFrequencies[stateFrequenciesIndex], + kPaddedPatternCount, kCategoryCount); + } #ifdef BEAGLE_DEBUG_VALUES - fprintf(stderr,"before SumSites1 = \n"); - gpu->PrintfDeviceVector(dIntegrationTmp, kPaddedPatternCount, r); + fprintf(stderr,"before SumSites1 = \n"); + gpu->PrintfDeviceVector(dIntegrationTmp, kPaddedPatternCount, r); #endif - // Take the dot product of the pattern log-likelihoods and the pattern weights. - dotProduct((Real*)dSumLogLikelihood, cublasHandle, kPatternCount, (Real*)dIntegrationTmp, (Real*)dPatternWeights); - - if (kFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) { - gpu->MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); + // Take the dot product of the pattern log-likelihoods and the pattern weights. + // The output (dSumLogLikelihood) needs to be a device pointer. + dotProduct((Real*)dSumLogLikelihood, cublasHandle, kPatternCount, (Real*)dIntegrationTmp, (Real*)dPatternWeights); - *outSumLogLikelihood = 0.0; - for (int i = 0; i < kSumSitesBlockCount; i++) { - if (hLogLikelihoodsCache[i] != hLogLikelihoodsCache[i]) - returnCode = BEAGLE_ERROR_FLOATING_POINT; + if (kFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) { + gpu->MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); - *outSumLogLikelihood += hLogLikelihoodsCache[i]; - } - } - - } else { - // TODO: evaluate performance, maybe break up kernels below for each subsetIndex case + *outSumLogLikelihood = 0.0; + for (int i = 0; i < kSumSitesBlockCount; i++) { + if (hLogLikelihoodsCache[i] != hLogLikelihoodsCache[i]) + returnCode = BEAGLE_ERROR_FLOATING_POINT; - if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { - for(int n = 0; n < count; n++) { - int cumulativeScalingFactor = bufferIndices[n] - kTipCount; - hPtrQueue[n] = cumulativeScalingFactor * kScaleBufferSize; - } - gpu->MemcpyHostToDevice(dPtrQueue, hPtrQueue, sizeof(unsigned int) * count); - } else if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE) { - for(int n = 0; n < count; n++) - hPtrQueue[n] = cumulativeScaleIndices[n] * kScaleBufferSize; - gpu->MemcpyHostToDevice(dPtrQueue, hPtrQueue, sizeof(unsigned int) * count); - } - - for (int subsetIndex = 0 ; subsetIndex < count; ++subsetIndex ) { - - const GPUPtr tmpDWeights = dWeights[categoryWeightsIndices[subsetIndex]]; - const GPUPtr tmpDFrequencies = dFrequencies[stateFrequenciesIndices[subsetIndex]]; - const int rootNodeIndex = bufferIndices[subsetIndex]; - - if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE || (kFlags & BEAGLE_FLAG_SCALING_ALWAYS)) { - kernels->IntegrateLikelihoodsFixedScaleMulti(dIntegrationTmp, dPartials[rootNodeIndex], tmpDWeights, - tmpDFrequencies, dScalingFactors[0], dPtrQueue, dMaxScalingFactors, - dIndexMaxScalingFactors, - kPaddedPatternCount, - kCategoryCount, count, subsetIndex); - } else { - if (subsetIndex == 0) { - kernels->IntegrateLikelihoodsMulti(dIntegrationTmp, dPartials[rootNodeIndex], tmpDWeights, - tmpDFrequencies, - kPaddedPatternCount, kCategoryCount, 0); - } else if (subsetIndex == count - 1) { - kernels->IntegrateLikelihoodsMulti(dIntegrationTmp, dPartials[rootNodeIndex], tmpDWeights, - tmpDFrequencies, - kPaddedPatternCount, kCategoryCount, 1); - } else { - kernels->IntegrateLikelihoodsMulti(dIntegrationTmp, dPartials[rootNodeIndex], tmpDWeights, - tmpDFrequencies, - kPaddedPatternCount, kCategoryCount, 2); - } - } - - - kernels->SumSites1(dIntegrationTmp, dSumLogLikelihood, dPatternWeights, - kPatternCount); - - if (kFlags & BEAGLE_FLAG_COMPUTATION_SYNCH) { - gpu->MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); - - *outSumLogLikelihood = 0.0; - for (int i = 0; i < kSumSitesBlockCount; i++) { - if (hLogLikelihoodsCache[i] != hLogLikelihoodsCache[i]) - returnCode = BEAGLE_ERROR_FLOATING_POINT; - - *outSumLogLikelihood += hLogLikelihoodsCache[i]; - } - } + *outSumLogLikelihood += hLogLikelihoodsCache[i]; } } From c93e01b3afc72b0662d17b1e00f6d3fc5ea8da39 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 25 Jun 2024 18:04:12 -0400 Subject: [PATCH 187/421] Remove some old code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 36 ------------------------ 1 file changed, 36 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c653304e..b54f330a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -689,42 +689,6 @@ int BeagleGPUActionImpl::updatePartials(const int* operation return returnCode; } -//BEAGLE_GPU_TEMPLATE -//int BeagleGPUActionImpl::calculateRootLogLikelihoods(const int* bufferIndices, -// const int* categoryWeightsIndices, -// const int* stateFrequenciesIndices, -// const int* cumulativeScaleIndices, -// int count, -// double* outSumLogLikelihood) { -// -//#ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tEntering BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); -//#endif -// -// int returnCode = BEAGLE_SUCCESS; -// -// if (count == 1) { -// -// } else { -// return BEAGLE_ERROR_NO_IMPLEMENTATION; -// } -// -//#ifdef BEAGLE_DEBUG_VALUES -// Real r = 0; -//fprintf(stderr, "parent = \n"); -//gpu->PrintfDeviceVector(dIntegrationTmp, kPatternCount, r); -//#endif -// -// -//#ifdef BEAGLE_DEBUG_FLOW -// fprintf(stderr, "\tLeaving BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); -//#endif -// -// return returnCode; -//} - - - BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int categoryIndex) { return nodeIndex * kCategoryCount + categoryIndex; From aca251b4f6443bad49cd8e546719e87e2d673a95 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 25 Jun 2024 18:05:15 -0400 Subject: [PATCH 188/421] More printing operations. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index b54f330a..84534621 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -439,6 +439,42 @@ std::ostream& operator<<(std::ostream& o, const SpMatrixDevice& D) return o< +struct asDeviceVec +{ + const Real* data; + int length; + + asDeviceVec(const Real* d, int l):data(d),length(l) { } +}; + +template +std::ostream& operator<<(std::ostream& o, const asDeviceVec& D) +{ + auto H = MemcpyDeviceToHostVector(D.data, D.length); + + o<<"[ "; + for(auto& h: H) + o< +std::ostream& operator<<(std::ostream& o, const std::vector& H) +{ + o<<"[ "; + for(auto& h: H) + o<::createInstance(int tipCount, int partialsBufferCount, From 73130c0e32d758947806e9d8965c94dab3a87c00 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 25 Jun 2024 18:05:33 -0400 Subject: [PATCH 189/421] Use printing operations. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 84534621..979e6f20 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1078,8 +1078,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i } #ifdef BEAGLE_DEBUG_VALUES - fprintf(stderr,"before SumSites1 = \n"); - gpu->PrintfDeviceVector(dIntegrationTmp, kPaddedPatternCount, r); + std::cerr<<"before pattern weights = "< Date: Tue, 25 Jun 2024 18:06:29 -0400 Subject: [PATCH 190/421] First try computing root likelihoods on the host. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 42 +++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 979e6f20..d4991f90 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1052,14 +1052,46 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i scale = 0; #ifdef BEAGLE_DEBUG_VALUES - Real r = 0; - fprintf(stderr,"root partials = \n"); - gpu->PrintfDeviceVector(dPartials[rootNodeIndex], kPaddedPatternCount, r); + std::cerr<<"root partials = "< hColumnProbs(kPatternCount); + +// std::cerr<<"root partials (h) = "<::calculateRootLogLikelihoods(const i // The output (dSumLogLikelihood) needs to be a device pointer. dotProduct((Real*)dSumLogLikelihood, cublasHandle, kPatternCount, (Real*)dIntegrationTmp, (Real*)dPatternWeights); + std::cerr<<"logLikelihood (kernel) = "<MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); From 0b3b50022b9400e385f45b5f78e5925f82382dc2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 12:03:11 -0400 Subject: [PATCH 191/421] Add function to show scaling info. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index d4991f90..d55921f8 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1022,6 +1022,31 @@ double BeagleGPUActionImpl::getDValue(int p, int eigenIndex) return hds[eigenIndex][p]; } +std::ostream& showScalingInfo(std::ostream& o, std::uint64_t kFlags, const int* cumulativeScaleIndices, int kScaleBufferSize) +{ + int scale = 1; + if (kFlags & BEAGLE_FLAG_SCALING_AUTO) + ; + else if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) + ; + else if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE) + ; + else + scale = 0; + + o<<"scale = "<::calculateRootLogLikelihoods(const int* bufferIndices, const int* categoryWeightsIndices, From f0709b32c585ae4673f67dbdfd29ec794e806200 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 12:08:10 -0400 Subject: [PATCH 192/421] Fix likelihood by incorporating scaling factors (log). --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index d55921f8..153775f7 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1084,11 +1084,26 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i auto hStateFrequencies = MemcpyDeviceToHostVector((Real*)dFrequencies[stateFrequenciesIndex], kStateCount); auto hCategoryWeights = MemcpyDeviceToHostVector((Real*)dWeights[categoryWeightsIndex], kCategoryCount); auto hPatternWeights = MemcpyDeviceToHostVector((Real*)dPatternWeights, kPatternCount); + std::vector hScalingFactors; + if (kFlags & BEAGLE_FLAG_SCALING_AUTO) + { + // scaling factor per pattern*category -- see BeagleGPUImpl<>::createInstance + // scaling factor is char?? -- see BeagleGPUImpl<>::createInstance + // scaling factor is int?? -- see KernelIntegrateLikelihoodsAutoScaling + // See kernelIntegrateLikelihoodsAutoScaling in KernelsX.cu + std::cerr<<"BeagleGPUActionImpl< >::calculateRootLogLikelihoods -- FLAG_SCALING_AUTO not implemented!"; + std::abort(); + } + else if (scale) + hScalingFactors = MemcpyDeviceToHostVector((Real*)dCumulativeScalingFactor, kScaleBufferSize); + std::vector hColumnProbs(kPatternCount); +// showScalingInfo(std::cerr, kFlags, cumulativeScaleIndices, kScaleBufferSize); // std::cerr<<"root partials (h) = "<::calculateRootLogLikelihoods(const i Pr += tmp * hCategoryWeights[category]; } Pr = log(Pr); + + if (scale) + { + // see kernelIntegrateLikelihoodsFixedScale in KernelsX.cu + Pr += hScalingFactors[pattern]; + } hColumnProbs[pattern] = Pr; OurResult += Pr*hPatternWeights[pattern]; } @@ -1119,7 +1140,8 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i */ if (scale) { - // See kernelIntegrateLikelihoodsAutoScaling in KernelsX.cu + // if SCALING_AUTO -> See kernelIntegrateLikelihoodsAutoScaling (in KernelsX.cu) + // otherwise -> see kernelIntegrateLikelihoodsFixedScale (in KernelsX.cu) kernels->IntegrateLikelihoodsDynamicScaling(dIntegrationTmp, dPartials[rootNodeIndex], dWeights[categoryWeightsIndex], dFrequencies[stateFrequenciesIndex], From c077523106b3ab7e6f520192c7f246028d42db8c Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 14:54:18 -0400 Subject: [PATCH 193/421] Add pseudocode for rescaling partials. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 153775f7..d324fb69 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -803,6 +803,34 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); + +/* + FIND_MAX_PARTIALS_X_CPU(); + // fPartialsDynamicScalingAccumulate; + if (kFlags & BEAGLE_FLAG_SCALERS_LOG) + { + // kernelPartialsDynamicScalingAccumulateScalersLog + if (max == 0) { + max = 1.0; + scalingFactors[pattern] = 0.0; + } else { + REAL logMax = log(max); + scalingFactors[pattern] = logMax; + if (cumulativeScalingBuffer != 0) + cumulativeScaling[pattern] += logMax; + } + } + else + { + // "kernelPartialsDynamicScalingAccumulate"; + if (max == 0) + max = 1.0; + scalingFactors[pattern] = max; + if (cumulativeScalingBuffer != 0) + cumulativeScaling[pattern] += log(max); + } + SCALE_PARTIALS_X_CPU(); +*/ } if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { From b09cfc9354bd441f071cb4695a512337a20914fb Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 15:02:10 -0400 Subject: [PATCH 194/421] Updated pseudocode --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 49 ++++++++++++++---------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index d324fb69..ca7e1f88 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -804,32 +804,39 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); + + bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; + /* - FIND_MAX_PARTIALS_X_CPU(); - // fPartialsDynamicScalingAccumulate; - if (kFlags & BEAGLE_FLAG_SCALERS_LOG) + for(int pattern = 0; pattern < kPatternCount;pattern++) { - // kernelPartialsDynamicScalingAccumulateScalersLog - if (max == 0) { - max = 1.0; - scalingFactors[pattern] = 0.0; - } else { - REAL logMax = log(max); - scalingFactors[pattern] = logMax; - if (cumulativeScalingBuffer != 0) - cumulativeScaling[pattern] += logMax; - } - } - else - { - // "kernelPartialsDynamicScalingAccumulate"; + FIND_MAX_PARTIALS_X_CPU(); if (max == 0) + { max = 1.0; - scalingFactors[pattern] = max; - if (cumulativeScalingBuffer != 0) - cumulativeScaling[pattern] += log(max); + if (scalers_log) + scalingFactors[pattern] = 0; + else + scalingFactors[pattern] = 1; + } + else + { + if (scalers_log) + { + REAL logMax = log(max); + scalingFactors[pattern] = logMax; + if (cumulativeScalingBuffer != 0) + cumulativeScaling[pattern] += logMax; + } + else + { + scalingFactors[pattern] = max; + if (cumulativeScalingBuffer != 0) + cumulativeScaling[pattern] += log(max); + } + } + SCALE_PARTIALS_X_CPU(); } - SCALE_PARTIALS_X_CPU(); */ } From 0b18cc811a8ddb6c84c834392838a66e09bfc689 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 15:18:26 -0400 Subject: [PATCH 195/421] This code compiles. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 98 +++++++++++++++--------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index ca7e1f88..869c1402 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -797,48 +797,74 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, secondChildPartialIndex, secondChildSubstitutionMatrixIndex); - if (rescale == 1) - { - GPUPtr partials3 = dPartials[destinationPartialIndex]; + if (rescale == 1) + { + GPUPtr partials3 = dPartials[destinationPartialIndex]; - kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, - kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); + auto hostPartials = MemcpyDeviceToHostVector((Real*)partials3, kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + auto hostScalingFactors = MemcpyDeviceToHostVector((Real*)scalingFactors, kPaddedPatternCount); + std::vector hostCumulativeScalingBuffer; + if (cumulativeScalingBuffer) + hostCumulativeScalingBuffer = MemcpyDeviceToHostVector((Real*)cumulativeScalingBuffer, kPaddedPatternCount); + kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, + kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); - bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; -/* - for(int pattern = 0; pattern < kPatternCount;pattern++) - { - FIND_MAX_PARTIALS_X_CPU(); - if (max == 0) + bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; + for(int pattern = 0; pattern < kPatternCount;pattern++) + { + // FIND_MAX_PARTIALS_X_CPU(); + int deltaPartialsByState = pattern * kPaddedPatternCount; + REAL max = 0; + for(int m = 0; m < kCategoryCount; m++) { - max = 1.0; - if (scalers_log) - scalingFactors[pattern] = 0; - else - scalingFactors[pattern] = 1; - } - else + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + REAL iPartial = hostPartials[deltaPartials + i]; + if (iPartial > max) + max = iPartial; + } + } + + if (max == 0) + { + max = 1.0; + if (scalers_log) + hostScalingFactors[pattern] = 0; + else + hostScalingFactors[pattern] = 1; + } + else + { + if (scalers_log) + { + REAL logMax = log(max); + hostScalingFactors[pattern] = logMax; + if (cumulativeScalingBuffer != 0) + hostCumulativeScalingBuffer[pattern] += logMax; + } + else + { + hostScalingFactors[pattern] = max; + if (cumulativeScalingBuffer != 0) + hostCumulativeScalingBuffer[pattern] += log(max); + } + } + + // SCALE_PARTIALS_X_CPU(); + for(int m = 0; m < kCategoryCount; m++) { - if (scalers_log) - { - REAL logMax = log(max); - scalingFactors[pattern] = logMax; - if (cumulativeScalingBuffer != 0) - cumulativeScaling[pattern] += logMax; - } - else - { - scalingFactors[pattern] = max; - if (cumulativeScalingBuffer != 0) - cumulativeScaling[pattern] += log(max); - } - } - SCALE_PARTIALS_X_CPU(); - } -*/ - } + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + hostPartials[deltaPartials + i] /= max; + } + } + } + + } if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { int parScalingIndex = destinationPartialIndex - kTipCount; From c077fff042a16fd0f12697d3475b32b86f0a6cf1 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 15:26:59 -0400 Subject: [PATCH 196/421] Fix computed index. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 869c1402..08bddcbc 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -815,7 +815,7 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, for(int pattern = 0; pattern < kPatternCount;pattern++) { // FIND_MAX_PARTIALS_X_CPU(); - int deltaPartialsByState = pattern * kPaddedPatternCount; + int deltaPartialsByState = pattern * kPaddedStateCount; REAL max = 0; for(int m = 0; m < kCategoryCount; m++) { From d94708d4ef7c760be23cec3c02c50f3ef1ee1a0b Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 15:31:04 -0400 Subject: [PATCH 197/421] Copy the host quantities back to the device. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 08bddcbc..c9bb1b53 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -864,6 +864,15 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, } } + MemcpyHostToDevice( (Real*)partials3, hostPartials.data(), kPaddedStateCount * kPaddedPatternCount * kCategoryCount ); + MemcpyHostToDevice( (Real*)scalingFactors, hostScalingFactors.data(), kPatternCount ); + if (cumulativeScalingBuffer) + MemcpyHostToDevice( (Real*)cumulativeScalingBuffer, hostCumulativeScalingBuffer.data(), kPatternCount ); + + std::cerr<<"rescaled partials (kernel) = "< Date: Wed, 26 Jun 2024 15:38:35 -0400 Subject: [PATCH 198/421] Comment out the kernel to use our host code for calculateRootLogLikelihoods( ) --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c9bb1b53..c918859d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1209,6 +1209,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::cerr<<"site probs (d1) = "< See kernelIntegrateLikelihoodsAutoScaling (in KernelsX.cu) // otherwise -> see kernelIntegrateLikelihoodsFixedScale (in KernelsX.cu) @@ -1225,6 +1226,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i dFrequencies[stateFrequenciesIndex], kPaddedPatternCount, kCategoryCount); } + */ #ifdef BEAGLE_DEBUG_VALUES std::cerr<<"before pattern weights = "< Date: Wed, 26 Jun 2024 15:39:02 -0400 Subject: [PATCH 199/421] Comment out the device kernel to use our host code for rescaling partials. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c918859d..f690c846 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -807,8 +807,8 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, if (cumulativeScalingBuffer) hostCumulativeScalingBuffer = MemcpyDeviceToHostVector((Real*)cumulativeScalingBuffer, kPaddedPatternCount); - kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, - kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); +// kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, +// kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; From adb32f406779b6a0378741ef8f1d8a7cc312a6d6 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 15:39:14 -0400 Subject: [PATCH 200/421] Remove debug prints. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index f690c846..5bfcc3bb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -869,10 +869,10 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, if (cumulativeScalingBuffer) MemcpyHostToDevice( (Real*)cumulativeScalingBuffer, hostCumulativeScalingBuffer.data(), kPatternCount ); - std::cerr<<"rescaled partials (kernel) = "< Date: Wed, 26 Jun 2024 15:49:04 -0400 Subject: [PATCH 201/421] Move rescaling code into its own subroutine. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 + libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 167 ++++++++++++----------- 2 files changed, 89 insertions(+), 80 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 218ba75b..11bef7ac 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -458,6 +458,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int operationCount, int cumulativeScalingIndex); + void rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex); + protected: int kPartialsCacheOffset; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5bfcc3bb..45dcfa2d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -725,6 +725,84 @@ int BeagleGPUActionImpl::updatePartials(const int* operation return returnCode; } +BEAGLE_GPU_TEMPLATE +void BeagleGPUActionImpl::rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex) +{ + +// kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, +// kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); + + auto hostPartials = MemcpyDeviceToHostVector(partials, kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + auto hostScalingFactors = MemcpyDeviceToHostVector(scalingFactors, kPaddedPatternCount); + std::vector hostCumulativeScalingBuffer; + if (cumulativeScalingBuffer) + hostCumulativeScalingBuffer = MemcpyDeviceToHostVector(cumulativeScalingBuffer, kPaddedPatternCount); + + + bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; + for(int pattern = 0; pattern < kPatternCount;pattern++) + { + // FIND_MAX_PARTIALS_X_CPU(); + int deltaPartialsByState = pattern * kPaddedStateCount; + REAL max = 0; + for(int m = 0; m < kCategoryCount; m++) + { + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + REAL iPartial = hostPartials[deltaPartials + i]; + if (iPartial > max) + max = iPartial; + } + } + + if (max == 0) + { + max = 1.0; + if (scalers_log) + hostScalingFactors[pattern] = 0; + else + hostScalingFactors[pattern] = 1; + } + else + { + if (scalers_log) + { + REAL logMax = log(max); + hostScalingFactors[pattern] = logMax; + if (cumulativeScalingBuffer != 0) + hostCumulativeScalingBuffer[pattern] += logMax; + } + else + { + hostScalingFactors[pattern] = max; + if (cumulativeScalingBuffer != 0) + hostCumulativeScalingBuffer[pattern] += log(max); + } + } + + // SCALE_PARTIALS_X_CPU(); + for(int m = 0; m < kCategoryCount; m++) + { + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + hostPartials[deltaPartials + i] /= max; + } + } + } + + MemcpyHostToDevice( partials, hostPartials.data(), kPaddedStateCount * kPaddedPatternCount * kCategoryCount ); + MemcpyHostToDevice( scalingFactors, hostScalingFactors.data(), kPatternCount ); + if (cumulativeScalingBuffer) + MemcpyHostToDevice( cumulativeScalingBuffer, hostCumulativeScalingBuffer.data(), kPatternCount ); + +// std::cerr<<"rescaled partials (kernel) = "<::getPartialIndex(int nodeIndex, int categoryIndex) { return nodeIndex * kCategoryCount + categoryIndex; @@ -756,9 +834,9 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, fprintf(stderr, "\tEntering BeagleGPUActionImpl::upPartials\n"); #endif - GPUPtr cumulativeScalingBuffer = 0; + Real* cumulativeScalingBuffer = 0; if (cumulativeScalingIndex != BEAGLE_OP_NONE) - cumulativeScalingBuffer = dScalingFactors[cumulativeScalingIndex]; + cumulativeScalingBuffer = (Real*)dScalingFactors[cumulativeScalingIndex]; int streamIndex = -1; int waitIndex = -1; @@ -775,22 +853,22 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, const int secondChildSubstitutionMatrixIndex = operations[op * numOps + 6]; int rescale = BEAGLE_OP_NONE; - GPUPtr scalingFactors = (GPUPtr)NULL; + Real* scalingFactors = nullptr; if (kFlags & BEAGLE_FLAG_SCALING_AUTO) { int sIndex = destinationPartialIndex - kTipCount; rescale = 2; - scalingFactors = dScalingFactors[sIndex]; + scalingFactors = (Real*)dScalingFactors[sIndex]; } else if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { rescale = 1; - scalingFactors = dScalingFactors[destinationPartialIndex - kTipCount]; + scalingFactors = (Real*)dScalingFactors[destinationPartialIndex - kTipCount]; } else if ((kFlags & BEAGLE_FLAG_SCALING_MANUAL) && writeScalingIndex >= 0) { rescale = 1; - scalingFactors = dScalingFactors[writeScalingIndex]; + scalingFactors = (Real*)dScalingFactors[writeScalingIndex]; } else if ((kFlags & BEAGLE_FLAG_SCALING_MANUAL) && readScalingIndex >= 0) { rescale = 0; - scalingFactors = dScalingFactors[readScalingIndex]; + scalingFactors = (Real*)dScalingFactors[readScalingIndex]; } calcPartialsPartials(destinationPartialIndex, firstChildPartialIndex, firstChildSubstitutionMatrixIndex, @@ -799,80 +877,9 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, if (rescale == 1) { - GPUPtr partials3 = dPartials[destinationPartialIndex]; - - auto hostPartials = MemcpyDeviceToHostVector((Real*)partials3, kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - auto hostScalingFactors = MemcpyDeviceToHostVector((Real*)scalingFactors, kPaddedPatternCount); - std::vector hostCumulativeScalingBuffer; - if (cumulativeScalingBuffer) - hostCumulativeScalingBuffer = MemcpyDeviceToHostVector((Real*)cumulativeScalingBuffer, kPaddedPatternCount); - -// kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, -// kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); - - - bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; - for(int pattern = 0; pattern < kPatternCount;pattern++) - { - // FIND_MAX_PARTIALS_X_CPU(); - int deltaPartialsByState = pattern * kPaddedStateCount; - REAL max = 0; - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - REAL iPartial = hostPartials[deltaPartials + i]; - if (iPartial > max) - max = iPartial; - } - } - - if (max == 0) - { - max = 1.0; - if (scalers_log) - hostScalingFactors[pattern] = 0; - else - hostScalingFactors[pattern] = 1; - } - else - { - if (scalers_log) - { - REAL logMax = log(max); - hostScalingFactors[pattern] = logMax; - if (cumulativeScalingBuffer != 0) - hostCumulativeScalingBuffer[pattern] += logMax; - } - else - { - hostScalingFactors[pattern] = max; - if (cumulativeScalingBuffer != 0) - hostCumulativeScalingBuffer[pattern] += log(max); - } - } - - // SCALE_PARTIALS_X_CPU(); - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - hostPartials[deltaPartials + i] /= max; - } - } - } - - MemcpyHostToDevice( (Real*)partials3, hostPartials.data(), kPaddedStateCount * kPaddedPatternCount * kCategoryCount ); - MemcpyHostToDevice( (Real*)scalingFactors, hostScalingFactors.data(), kPatternCount ); - if (cumulativeScalingBuffer) - MemcpyHostToDevice( (Real*)cumulativeScalingBuffer, hostCumulativeScalingBuffer.data(), kPatternCount ); + Real* partials3 = (Real*)dPartials[destinationPartialIndex]; -// std::cerr<<"rescaled partials (kernel) = "< Date: Wed, 26 Jun 2024 15:49:10 -0400 Subject: [PATCH 202/421] Remove debug print. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 45dcfa2d..de8ed830 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1243,7 +1243,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i // The output (dSumLogLikelihood) needs to be a device pointer. dotProduct((Real*)dSumLogLikelihood, cublasHandle, kPatternCount, (Real*)dIntegrationTmp, (Real*)dPatternWeights); - std::cerr<<"logLikelihood (kernel) = "<MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); From 5e1ecbc5c168be9c44474fedcb32f37294a04872 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 16:20:12 -0400 Subject: [PATCH 203/421] Comment out debugging code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index de8ed830..527e5bd8 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1207,7 +1207,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i Pr += hScalingFactors[pattern]; } hColumnProbs[pattern] = Pr; - OurResult += Pr*hPatternWeights[pattern]; +// OurResult += Pr*hPatternWeights[pattern]; } MemcpyHostToDevice((Real*)dIntegrationTmp, (Real*)hColumnProbs.data(), kPatternCount); From 173bd8bfc1a686a7210f2c51473409659dce027d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 26 Jun 2024 17:10:14 -0400 Subject: [PATCH 204/421] Add some in-process notes. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 527e5bd8..556d4e42 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1216,6 +1216,24 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::cerr<<"site probs (d1) = "< TMP1 + // pi = 1 x kStateCount partials = kStateCount * kPatternCount * kCategoryCount TMP = 1 x (kPatternCount * kCategoryCount) + // DnMatrixDevice PI(cublasHandle, (Real*)dFrequencies[stateFrequenciesIndex], 1, kPaddedStateCount); // grouped by column + // DnMatrixDevice PARTIALS(cublasHandle, dIntegrationTmp, kPaddedStateCount, kPaddedPatternCount * kCategoryCount); // grouped by column + + // ***STARTHERE*** PROBLEM - We need some memory here to store TMP1! + // Alternatively we MODIFY partials(cat,pat,state) by multiplying by weight(cat) and freq(state), and then just sum it. + // cublas is heavily optimized, so writing our own kernel might not be so easy: https://siboehm.com/articles/22/CUDA-MMM + + // Step 2. Sum over category weights: TMP1 * cats -> columnProbs + // TMP = kPatternCount x kCategoryCount cats = kCategoryCount x 1 columnProbs = kPatternCount x 1 + + // Step 3. Take the log of all the column probabilities. + + // Step 4. Take the sum of the logged column probabilities. + + // Step 5. If cumulativeScalingBuffer, take the sum of the cumulativeScalingBuffer also. + /* if (scale) { // if SCALING_AUTO -> See kernelIntegrateLikelihoodsAutoScaling (in KernelsX.cu) From bd3fce2b1782d2961a27cbc1f86badeba64e93a4 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 1 Jul 2024 18:20:53 -0400 Subject: [PATCH 205/421] Split loop into three sequential parts. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 556d4e42..1413f2f8 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1185,7 +1185,6 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i // We want to sum dRootPartials(category, pattern, state) * dWeights[category] * dFrequencies[state,category] // over (category,state). // dRootPartials(c,p,s) = dRootPartials[c*kPatternCount*kPaddedStateCount + p*kPaddedStateCount + s] - double OurResult = 0; for(int pattern = 0; pattern < kPatternCount; pattern++) { double Pr = 0; @@ -1199,15 +1198,16 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i Pr += tmp * hCategoryWeights[category]; } + hColumnProbs[pattern] = Pr; + } + + for(auto& Pr: hColumnProbs) Pr = log(Pr); + for(int pattern = 0; pattern < kPatternCount; pattern++) + { if (scale) - { - // see kernelIntegrateLikelihoodsFixedScale in KernelsX.cu - Pr += hScalingFactors[pattern]; - } - hColumnProbs[pattern] = Pr; -// OurResult += Pr*hPatternWeights[pattern]; + hColumnProbs[pattern] += hScalingFactors[pattern]; } MemcpyHostToDevice((Real*)dIntegrationTmp, (Real*)hColumnProbs.data(), kPatternCount); From c3e071b74d32fdfdb9724e8371dc008c1f70d174 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 1 Jul 2024 18:21:34 -0400 Subject: [PATCH 206/421] Initialize the host column probs from the device version. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 1413f2f8..913d95a6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1174,7 +1174,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i else if (scale) hScalingFactors = MemcpyDeviceToHostVector((Real*)dCumulativeScalingFactor, kScaleBufferSize); - std::vector hColumnProbs(kPatternCount); + std::vector hColumnProbs = MemcpyDeviceToHostVector(siteProbs, kPatternCount); // showScalingInfo(std::cerr, kFlags, cumulativeScaleIndices, kScaleBufferSize); // std::cerr<<"root partials (h) = "< Date: Mon, 1 Jul 2024 18:39:43 -0400 Subject: [PATCH 207/421] Initial implementation using S/Dgemv --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 50 +++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 913d95a6..5aa432ed 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1157,9 +1157,55 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::cerr<<"root partials = "<::value) { + status = cublasSgemv(cublasHandle, CUBLAS_OP_N, + kPaddedPatternCount, kPaddedStateCount, + weightForCat, + rootPartialsForCat, kPaddedPatternCount, //leading dimension + stateFrequenciesForCat, 1, + &beta, + siteProbs, 1); + } + else + { + status = cublasDgemv(cublasHandle, CUBLAS_OP_N, + kPaddedPatternCount, kPaddedStateCount, + weightForCat, + rootPartialsForCat, kPaddedPatternCount, //leading dimension + stateFrequenciesForCat, 1, + &beta, + siteProbs, 1); + } + if (status != CUBLAS_STATUS_SUCCESS) + { + std::cerr<<"cublas error "<gemv( )DnMatrix<>::operator*=\n"; + exit(1); + } + std::cerr<<" siteProbs(after) = "< hScalingFactors; if (kFlags & BEAGLE_FLAG_SCALING_AUTO) From e85e40c4828f7b8a0409cd0414c24c858ab9a5d0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 1 Jul 2024 18:41:18 -0400 Subject: [PATCH 208/421] Add note. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5aa432ed..43dbd8f9 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1164,6 +1164,8 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i auto hCategoryWeights = MemcpyDeviceToHostVector((Real*)dWeights[categoryWeightsIndex], kCategoryCount); std::cerr<<"hRootPartials = "< Date: Tue, 2 Jul 2024 16:09:21 -0400 Subject: [PATCH 209/421] Reorder host sum. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 43dbd8f9..dd84294b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1230,23 +1230,24 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i // std::cerr<<"category weights (h) = "< Date: Tue, 2 Jul 2024 16:09:33 -0400 Subject: [PATCH 210/421] Transpose the matrix --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index dd84294b..411843d6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1181,7 +1181,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::cerr<<" siteProbs(before) = "<::value) { - status = cublasSgemv(cublasHandle, CUBLAS_OP_N, + status = cublasSgemv(cublasHandle, CUBLAS_OP_T, kPaddedPatternCount, kPaddedStateCount, weightForCat, rootPartialsForCat, kPaddedPatternCount, //leading dimension @@ -1191,7 +1191,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i } else { - status = cublasDgemv(cublasHandle, CUBLAS_OP_N, + status = cublasDgemv(cublasHandle, CUBLAS_OP_T, kPaddedPatternCount, kPaddedStateCount, weightForCat, rootPartialsForCat, kPaddedPatternCount, //leading dimension From 7cc126a114d60fec855194d0c5538d3f71e11ad0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 2 Jul 2024 16:10:32 -0400 Subject: [PATCH 211/421] Remove the CPU code in favor of GPU code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 411843d6..7da37754 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1230,26 +1230,6 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i // std::cerr<<"category weights (h) = "< Date: Tue, 2 Jul 2024 16:14:23 -0400 Subject: [PATCH 212/421] Remove some debug printing. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 7da37754..a6cdd061 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1162,7 +1162,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i auto hRootPartials = MemcpyDeviceToHostVector(rootPartials, kPaddedPatternCount * kPaddedStateCount * kCategoryCount); auto hCategoryWeights = MemcpyDeviceToHostVector((Real*)dWeights[categoryWeightsIndex], kCategoryCount); - std::cerr<<"hRootPartials = "<::calculateRootLogLikelihoods(const i // siteProbs[p] = dWeights[0] * \sum(s) rootPartials(0,p,s) * stateFreqs(s) // siteProbs[p] += dWeights[c] * \sum(s) rootPartials(c,p,s) * stateFreqs(s) - std::cerr<<"category = "<::value) { status = cublasSgemv(cublasHandle, CUBLAS_OP_T, @@ -1204,7 +1201,6 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::cerr<<"cublas error "<gemv( )DnMatrix<>::operator*=\n"; exit(1); } - std::cerr<<" siteProbs(after) = "< Date: Tue, 2 Jul 2024 16:58:39 -0400 Subject: [PATCH 213/421] Fix order of stateCount / patternCount in gemv --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index a6cdd061..8c012bc3 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1179,9 +1179,9 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i cublasStatus_t status; if constexpr (std::is_same::value) { status = cublasSgemv(cublasHandle, CUBLAS_OP_T, - kPaddedPatternCount, kPaddedStateCount, + kPaddedStateCount, kPaddedPatternCount, weightForCat, - rootPartialsForCat, kPaddedPatternCount, //leading dimension + rootPartialsForCat, kPaddedStateCount, //leading dimension stateFrequenciesForCat, 1, &beta, siteProbs, 1); @@ -1189,9 +1189,9 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i else { status = cublasDgemv(cublasHandle, CUBLAS_OP_T, - kPaddedPatternCount, kPaddedStateCount, + kPaddedStateCount, kPaddedPatternCount, weightForCat, - rootPartialsForCat, kPaddedPatternCount, //leading dimension + rootPartialsForCat, kPaddedStateCount, //leading dimension stateFrequenciesForCat, 1, &beta, siteProbs, 1); From 761db5958c0844100a48e18bdf19e79b3e96c208 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 2 Jul 2024 17:26:27 -0400 Subject: [PATCH 214/421] Move gemv wrapper into its own function. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 66 ++++++++++++++---------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 8c012bc3..fbd35687 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1124,6 +1124,37 @@ std::ostream& showScalingInfo(std::ostream& o, std::uint64_t kFlags, const int* return o; } +template +void gemv(cublasHandle_t handle, cublasOperation_t op, int size1, int size2, Real alpha, const Real* mat, int lda, const Real* vec, Real beta, Real* out) +{ + cublasStatus_t status; + + if constexpr (std::is_same::value) { + status = cublasSgemv(handle, op, + size1, size2, + &alpha, + mat, size1, //leading dimension + vec, 1, + &beta, + out, 1); + } + else + { + status = cublasDgemv(handle, op, + size1, size2, + &alpha, + mat, size1, //leading dimension + vec, 1, + &beta, + out, 1); + } + if (status != CUBLAS_STATUS_SUCCESS) + { + std::cerr<<"cublas error "<gemv( )."; + exit(1); + } +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::calculateRootLogLikelihoods(const int* bufferIndices, const int* categoryWeightsIndices, @@ -1170,37 +1201,14 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i { Real* rootPartialsForCat = rootPartials + (category * kPaddedPatternCount * kPaddedStateCount); Real* stateFrequenciesForCat = (Real*)dFrequencies[stateFrequenciesIndex]; // these really should be category-dependent - Real* weightForCat = &hCategoryWeights[category]; + Real weightForCat = hCategoryWeights[category]; Real beta = (category == 0) ? 0.0 : 1.0; // siteProbs[p] = dWeights[0] * \sum(s) rootPartials(0,p,s) * stateFreqs(s) // siteProbs[p] += dWeights[c] * \sum(s) rootPartials(c,p,s) * stateFreqs(s) - cublasStatus_t status; - if constexpr (std::is_same::value) { - status = cublasSgemv(cublasHandle, CUBLAS_OP_T, - kPaddedStateCount, kPaddedPatternCount, - weightForCat, - rootPartialsForCat, kPaddedStateCount, //leading dimension - stateFrequenciesForCat, 1, - &beta, - siteProbs, 1); - } - else - { - status = cublasDgemv(cublasHandle, CUBLAS_OP_T, - kPaddedStateCount, kPaddedPatternCount, - weightForCat, - rootPartialsForCat, kPaddedStateCount, //leading dimension - stateFrequenciesForCat, 1, - &beta, - siteProbs, 1); - } - if (status != CUBLAS_STATUS_SUCCESS) - { - std::cerr<<"cublas error "<gemv( )DnMatrix<>::operator*=\n"; - exit(1); - } + gemv(cublasHandle, CUBLAS_OP_T, kPaddedStateCount, kPaddedPatternCount, + weightForCat, rootPartialsForCat, kPaddedStateCount, stateFrequenciesForCat, beta, siteProbs); } auto hStateFrequencies = MemcpyDeviceToHostVector((Real*)dFrequencies[stateFrequenciesIndex], kStateCount); @@ -1229,6 +1237,12 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i for(auto& Pr: hColumnProbs) Pr = log(Pr); + if constexpr (std::is_same::value) { + } + else + { + } + for(int pattern = 0; pattern < kPatternCount; pattern++) { if (scale) From 6a560b130689f23846df700b4d790e73b9177648 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 3 Jul 2024 11:35:20 -0400 Subject: [PATCH 215/421] Remove commented-out code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 37 ------------------------ 1 file changed, 37 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index fbd35687..8990c46d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1255,43 +1255,6 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::cerr<<"site probs (d1) = "< TMP1 - // pi = 1 x kStateCount partials = kStateCount * kPatternCount * kCategoryCount TMP = 1 x (kPatternCount * kCategoryCount) - // DnMatrixDevice PI(cublasHandle, (Real*)dFrequencies[stateFrequenciesIndex], 1, kPaddedStateCount); // grouped by column - // DnMatrixDevice PARTIALS(cublasHandle, dIntegrationTmp, kPaddedStateCount, kPaddedPatternCount * kCategoryCount); // grouped by column - - // ***STARTHERE*** PROBLEM - We need some memory here to store TMP1! - // Alternatively we MODIFY partials(cat,pat,state) by multiplying by weight(cat) and freq(state), and then just sum it. - // cublas is heavily optimized, so writing our own kernel might not be so easy: https://siboehm.com/articles/22/CUDA-MMM - - // Step 2. Sum over category weights: TMP1 * cats -> columnProbs - // TMP = kPatternCount x kCategoryCount cats = kCategoryCount x 1 columnProbs = kPatternCount x 1 - - // Step 3. Take the log of all the column probabilities. - - // Step 4. Take the sum of the logged column probabilities. - - // Step 5. If cumulativeScalingBuffer, take the sum of the cumulativeScalingBuffer also. - - /* - if (scale) { - // if SCALING_AUTO -> See kernelIntegrateLikelihoodsAutoScaling (in KernelsX.cu) - // otherwise -> see kernelIntegrateLikelihoodsFixedScale (in KernelsX.cu) - kernels->IntegrateLikelihoodsDynamicScaling(dIntegrationTmp, dPartials[rootNodeIndex], - dWeights[categoryWeightsIndex], - dFrequencies[stateFrequenciesIndex], - dCumulativeScalingFactor, - kPaddedPatternCount, - kCategoryCount); - } else { - // See kernelIntegrateLikelihoods in KernelsX.cu - kernels->IntegrateLikelihoods(dIntegrationTmp, dPartials[rootNodeIndex], - dWeights[categoryWeightsIndex], - dFrequencies[stateFrequenciesIndex], - kPaddedPatternCount, kCategoryCount); - } - */ - #ifdef BEAGLE_DEBUG_VALUES std::cerr<<"before pattern weights = "< Date: Wed, 3 Jul 2024 16:25:42 -0400 Subject: [PATCH 216/421] Make dCumulativeScalingFactor into a const Real* --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 8990c46d..8e65770b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1173,14 +1173,14 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i const int categoryWeightsIndex = categoryWeightsIndices[0]; const int stateFrequenciesIndex = stateFrequenciesIndices[0]; - GPUPtr dCumulativeScalingFactor; + const Real* dCumulativeScalingFactor = nullptr; bool scale = 1; if (kFlags & BEAGLE_FLAG_SCALING_AUTO) - dCumulativeScalingFactor = dAccumulatedScalingFactors; + dCumulativeScalingFactor = (const Real*)dAccumulatedScalingFactors; else if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) - dCumulativeScalingFactor = dScalingFactors[bufferIndices[0] - kTipCount]; + dCumulativeScalingFactor = (const Real*)dScalingFactors[bufferIndices[0] - kTipCount]; else if (cumulativeScaleIndices[0] != BEAGLE_OP_NONE) - dCumulativeScalingFactor = dScalingFactors[cumulativeScaleIndices[0]]; + dCumulativeScalingFactor = (const Real*)dScalingFactors[cumulativeScaleIndices[0]]; else scale = 0; @@ -1224,9 +1224,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::abort(); } else if (scale) - hScalingFactors = MemcpyDeviceToHostVector((Real*)dCumulativeScalingFactor, kScaleBufferSize); - - std::vector hColumnProbs = MemcpyDeviceToHostVector(siteProbs, kPatternCount); + hScalingFactors = MemcpyDeviceToHostVector(dCumulativeScalingFactor, kScaleBufferSize); // showScalingInfo(std::cerr, kFlags, cumulativeScaleIndices, kScaleBufferSize); // std::cerr<<"root partials (h) = "<::calculateRootLogLikelihoods(const i // std::cerr<<"category weights (h) = "< hColumnProbs = MemcpyDeviceToHostVector(siteProbs, kPatternCount); + for(auto& Pr: hColumnProbs) Pr = log(Pr); @@ -1249,7 +1249,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i hColumnProbs[pattern] += hScalingFactors[pattern]; } - MemcpyHostToDevice((Real*)dIntegrationTmp, (Real*)hColumnProbs.data(), kPatternCount); + MemcpyHostToDevice(siteProbs, (Real*)hColumnProbs.data(), kPatternCount); /* std::cerr<<"site probs (d1) = "< Date: Wed, 3 Jul 2024 16:27:08 -0400 Subject: [PATCH 217/421] Use cblasaxpy instead of CPU summation. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 47 ++++++++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 8e65770b..767a4e43 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1237,19 +1237,50 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i for(auto& Pr: hColumnProbs) Pr = log(Pr); - if constexpr (std::is_same::value) { - } - else - { - } + MemcpyHostToDevice(siteProbs, (Real*)hColumnProbs.data(), kPatternCount); - for(int pattern = 0; pattern < kPatternCount; pattern++) + if (scale) { - if (scale) + cublasStatus_t status; + if constexpr (std::is_same::value) + { + Real alpha = 1; + status = cublasSaxpy(cublasHandle, kPatternCount, + &alpha, + dCumulativeScalingFactor, 1, + siteProbs, 1); + } + else + { +// std::cerr<<"scaling factors (device) = "<axpy( )."; + exit(1); + } +/* + std::cerr<<"site Probs(host) = "< Date: Wed, 3 Jul 2024 16:27:36 -0400 Subject: [PATCH 218/421] Remove comments and debugging code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 767a4e43..8969ef20 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1252,33 +1252,17 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i } else { -// std::cerr<<"scaling factors (device) = "<axpy( )."; exit(1); } -/* - std::cerr<<"site Probs(host) = "< Date: Wed, 3 Jul 2024 16:30:46 -0400 Subject: [PATCH 219/421] Comment out now-unused host vectors. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 8969ef20..080343c4 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1191,7 +1191,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i Real* siteProbs = (Real*)dIntegrationTmp; Real* rootPartials = (Real*)dPartials[rootNodeIndex]; - auto hRootPartials = MemcpyDeviceToHostVector(rootPartials, kPaddedPatternCount * kPaddedStateCount * kCategoryCount); + // auto hRootPartials = MemcpyDeviceToHostVector(rootPartials, kPaddedPatternCount * kPaddedStateCount * kCategoryCount); auto hCategoryWeights = MemcpyDeviceToHostVector((Real*)dWeights[categoryWeightsIndex], kCategoryCount); // std::cerr<<"hRootPartials = "<::calculateRootLogLikelihoods(const i weightForCat, rootPartialsForCat, kPaddedStateCount, stateFrequenciesForCat, beta, siteProbs); } - auto hStateFrequencies = MemcpyDeviceToHostVector((Real*)dFrequencies[stateFrequenciesIndex], kStateCount); - auto hPatternWeights = MemcpyDeviceToHostVector((Real*)dPatternWeights, kPatternCount); - std::vector hScalingFactors; + // auto hStateFrequencies = MemcpyDeviceToHostVector((Real*)dFrequencies[stateFrequenciesIndex], kStateCount); + // auto hPatternWeights = MemcpyDeviceToHostVector((Real*)dPatternWeights, kPatternCount); if (kFlags & BEAGLE_FLAG_SCALING_AUTO) { // scaling factor per pattern*category -- see BeagleGPUImpl<>::createInstance @@ -1223,8 +1222,10 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::cerr<<"BeagleGPUActionImpl< >::calculateRootLogLikelihoods -- FLAG_SCALING_AUTO not implemented!"; std::abort(); } - else if (scale) - hScalingFactors = MemcpyDeviceToHostVector(dCumulativeScalingFactor, kScaleBufferSize); + +// std::vector hScalingFactors; +// if (scale) +// hScalingFactors = MemcpyDeviceToHostVector(dCumulativeScalingFactor, kScaleBufferSize); // showScalingInfo(std::cerr, kFlags, cumulativeScaleIndices, kScaleBufferSize); // std::cerr<<"root partials (h) = "< Date: Tue, 9 Jul 2024 12:43:47 -0400 Subject: [PATCH 220/421] First working version of doing log on the device. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 45 +--------------- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 7 +-- libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 4 ++ libhmsbeagle/GPU/cuda_ops.cu | 33 ++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 55 ++++++++++++++++++++ 5 files changed, 94 insertions(+), 50 deletions(-) create mode 100644 libhmsbeagle/GPU/cuda_ops.cu create mode 100644 libhmsbeagle/GPU/cuda_ops.h diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 11bef7ac..10e2301a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -41,6 +41,7 @@ #include // cudaMalloc, cudaMemcpy, etc. #include // cusparseSpMV #include +#include "cuda_ops.h" using std::vector; using std::tuple; @@ -109,50 +110,6 @@ Real normPInf(Real* matrix, int nRows, int nCols, cublasHandle_t cublasH) { return std::abs(result); } -template -T* cudaDeviceNew(int n) -{ - T* result; - auto status = cudaMalloc((void**)&result, n*sizeof(T)); - if (status != cudaSuccess) - throw std::runtime_error("cudaMalloc: failed!"); - return result; -} - -template -void MemcpyHostToDevice(T* dptr, const T* hptr, int n) -{ - auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyHostToDevice); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); -} - -template -void MemcpyDeviceToDevice(T* dptr, const T* hptr, int n) -{ - auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyDeviceToDevice); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); -} - -template -void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) -{ - auto status = cudaMemcpy(hptr, dptr, n*sizeof(T), cudaMemcpyDeviceToHost); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); -} - -template -std::vector MemcpyDeviceToHostVector(const T* dptr, int n) -{ - std::vector host_vec(n); - auto status = cudaMemcpy(host_vec.data(), dptr, n*sizeof(T), cudaMemcpyDeviceToHost); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); - return host_vec; -} - template using SpMatrix = Eigen::SparseMatrix; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 080343c4..9f06d92d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1233,12 +1233,7 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i // std::cerr<<"category weights (h) = "< hColumnProbs = MemcpyDeviceToHostVector(siteProbs, kPatternCount); - - for(auto& Pr: hColumnProbs) - Pr = log(Pr); - - MemcpyHostToDevice(siteProbs, (Real*)hColumnProbs.data(), kPatternCount); + cuda_log_vector(siteProbs, kPatternCount); if (scale) { diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt index d3de92ac..06aa922e 100644 --- a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -10,6 +10,9 @@ link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ) +enable_language(CUDA) +set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -O3 -Wno-deprecated-gpu-targets --extended-lambda -std=c++17") + if(WIN32) set(NVCC "nvcc.exe") set(NVCCFLAGS "-O3 -Wno-deprecated-gpu-targets") @@ -45,6 +48,7 @@ add_library(hmsbeagle-action-cuda SHARED ../KernelResource.h ../ActionCUDAPlugin.cpp ../ActionCUDAPlugin.h + ../cuda_ops.cu ../Precision.h ) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu new file mode 100644 index 00000000..273d89fd --- /dev/null +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -0,0 +1,33 @@ +#include "cuda_ops.h" +#include + +void cuda_log_vector(double* v, int length) +{ + thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + + thrust::device_vector V(vdptr, vdptr + length); + + thrust::device_vector Vout = V; + + thrust::transform(V.begin(), V.end(), Vout.begin(), [] __device__ (double x) {return log(x);}); + + double* Vout_ptr = thrust::raw_pointer_cast(&Vout[0]); + + MemcpyDeviceToDevice(v, Vout_ptr, length); +} + +void cuda_log_vector(float* v, int length) +{ + thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + + thrust::device_vector V(vdptr, vdptr + length); + + thrust::device_vector Vout = V; + + thrust::transform(V.begin(), V.end(), Vout.begin(), [] __device__ (float x) {return log(x);}); + + float* Vout_ptr = thrust::raw_pointer_cast(&Vout[0]); + + MemcpyDeviceToDevice(v, Vout_ptr, length); +} + diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h new file mode 100644 index 00000000..8134373f --- /dev/null +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -0,0 +1,55 @@ +#ifndef CUDA_OPS_H +#define CUDA_OPS_H + +#include +#include // cudaMalloc, cudaMemcpy, etc. +#include // for std::runtime_error + +template +T* cudaDeviceNew(int n) +{ + T* result; + auto status = cudaMalloc((void**)&result, n*sizeof(T)); + if (status != cudaSuccess) + throw std::runtime_error("cudaMalloc: failed!"); + return result; +} + +template +void MemcpyHostToDevice(T* dptr, const T* hptr, int n) +{ + auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyHostToDevice); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); +} + +template +void MemcpyDeviceToDevice(T* dptr, const T* hptr, int n) +{ + auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyDeviceToDevice); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); +} + +template +void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) +{ + auto status = cudaMemcpy(hptr, dptr, n*sizeof(T), cudaMemcpyDeviceToHost); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); +} + +template +std::vector MemcpyDeviceToHostVector(const T* dptr, int n) +{ + std::vector host_vec(n); + auto status = cudaMemcpy(host_vec.data(), dptr, n*sizeof(T), cudaMemcpyDeviceToHost); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); + return host_vec; +} + +void cuda_log_vector(double* v, int length); +void cuda_log_vector(float* v, int length); + +#endif From 1ff16d9bafbc84a07de6885f56b035f84d15459b Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 9 Jul 2024 12:56:10 -0400 Subject: [PATCH 221/421] Simplify cuda_log_vector --- libhmsbeagle/GPU/cuda_ops.cu | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 273d89fd..208b1cd8 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -1,19 +1,15 @@ #include "cuda_ops.h" #include +// QUESTION: Why can't we do the transform in-place? + void cuda_log_vector(double* v, int length) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); thrust::device_vector V(vdptr, vdptr + length); - thrust::device_vector Vout = V; - - thrust::transform(V.begin(), V.end(), Vout.begin(), [] __device__ (double x) {return log(x);}); - - double* Vout_ptr = thrust::raw_pointer_cast(&Vout[0]); - - MemcpyDeviceToDevice(v, Vout_ptr, length); + thrust::transform(V.begin(), V.end(), vdptr, [] __device__ (double x) {return log(x);}); } void cuda_log_vector(float* v, int length) @@ -22,12 +18,6 @@ void cuda_log_vector(float* v, int length) thrust::device_vector V(vdptr, vdptr + length); - thrust::device_vector Vout = V; - - thrust::transform(V.begin(), V.end(), Vout.begin(), [] __device__ (float x) {return log(x);}); - - float* Vout_ptr = thrust::raw_pointer_cast(&Vout[0]); - - MemcpyDeviceToDevice(v, Vout_ptr, length); + thrust::transform(V.begin(), V.end(), vdptr, [] __device__ (float x) {return log(x);}); } From c0a41e75c8d8bc5553daa78c0125acb7d2fc9833 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 9 Jul 2024 13:27:58 -0400 Subject: [PATCH 222/421] Add question. --- libhmsbeagle/GPU/cuda_ops.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 208b1cd8..86f1954b 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -7,6 +7,7 @@ void cuda_log_vector(double* v, int length) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + // QUESTION: How slow is allocating the device memory here? thrust::device_vector V(vdptr, vdptr + length); thrust::transform(V.begin(), V.end(), vdptr, [] __device__ (double x) {return log(x);}); @@ -16,6 +17,7 @@ void cuda_log_vector(float* v, int length) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + // QUESTION: How slow is allocating the device memory here? thrust::device_vector V(vdptr, vdptr + length); thrust::transform(V.begin(), V.end(), vdptr, [] __device__ (float x) {return log(x);}); From a4fd32a02ce23bd4dbac55692025d4400f93e4e9 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 10 Jul 2024 10:54:26 -0400 Subject: [PATCH 223/421] Simplify code for scaling. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 38 ++++++++++-------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 9f06d92d..953bda15 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -757,29 +757,21 @@ void BeagleGPUActionImpl::rescalePartials(Real* partials, R } if (max == 0) - { - max = 1.0; - if (scalers_log) - hostScalingFactors[pattern] = 0; - else - hostScalingFactors[pattern] = 1; - } - else - { - if (scalers_log) - { - REAL logMax = log(max); - hostScalingFactors[pattern] = logMax; - if (cumulativeScalingBuffer != 0) - hostCumulativeScalingBuffer[pattern] += logMax; - } - else - { - hostScalingFactors[pattern] = max; - if (cumulativeScalingBuffer != 0) - hostCumulativeScalingBuffer[pattern] += log(max); - } - } + max = 1.0; + + if (scalers_log) + { + REAL logMax = log(max); + hostScalingFactors[pattern] = logMax; + if (cumulativeScalingBuffer != 0) + hostCumulativeScalingBuffer[pattern] += logMax; + } + else + { + hostScalingFactors[pattern] = max; + if (cumulativeScalingBuffer != 0) + hostCumulativeScalingBuffer[pattern] += log(max); + } // SCALE_PARTIALS_X_CPU(); for(int m = 0; m < kCategoryCount; m++) From c9be335c1e0a06b0fab6794b97e096c47747b72f Mon Sep 17 00:00:00 2001 From: yucais Date: Thu, 11 Jul 2024 04:01:51 -0700 Subject: [PATCH 224/421] calculate all operations for reduce within interval --- libhmsbeagle/GPU/BeagleGPUImpl.h | 5 + libhmsbeagle/GPU/BeagleGPUImpl.hpp | 127 ++++++++++++++++++++----- libhmsbeagle/GPU/KernelLauncher.cpp | 15 ++- libhmsbeagle/GPU/KernelLauncher.h | 2 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 111 +++++++++++++-------- 5 files changed, 190 insertions(+), 70 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 860952d3..1d633566 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -222,6 +222,11 @@ class BeagleGPUImpl : public BeagleImpl { GPUPtr dBastaOperationQueue; GPUPtr dCoalescentBuffers; GPUPtr dBastaInterval; + GPUPtr dBastaFlags; + GPUPtr dBastaBlockResE; + GPUPtr dBastaBlockResF; + GPUPtr dBastaBlockResG; + GPUPtr dBastaBlockResH; int kCoalescentBufferLength; int kCoalescentBufferCount; int kBastaIntervalBlockCount; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 088ca9ee..8130fc1c 100644 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -85,7 +85,12 @@ namespace gpu { dBastaOperationQueue = (GPUPtr)NULL; dBastaBuffers = (GPUPtr)NULL; dCoalescentBuffers = (GPUPtr)NULL; - dBastaInterval = (GPUPtr)NULL;; + dBastaInterval = (GPUPtr)NULL; + dBastaFlags = (GPUPtr)NULL; + dBastaBlockResE = (GPUPtr)NULL; + dBastaBlockResF = (GPUPtr)NULL; + dBastaBlockResG = (GPUPtr)NULL; + dBastaBlockResH = (GPUPtr)NULL; dEigenValues = NULL; dEvec = NULL; dIevc = NULL; @@ -259,10 +264,15 @@ BeagleGPUImpl::~BeagleGPUImpl() { gpu->FreeMemory(dBastaF); gpu->FreeMemory(dBastaG); gpu->FreeMemory(dBastaH); + gpu->FreeMemory(dBastaBlockResE); + gpu->FreeMemory(dBastaBlockResF); + gpu->FreeMemory(dBastaBlockResG); + gpu->FreeMemory(dBastaBlockResH); gpu->FreeMemory(dBastaLogL); gpu->FreeMemory(dBastaDistance); gpu->FreeMemory(dBastaOperationQueue); gpu->FreeMemory(dBastaInterval); + gpu->FreeMemory(dBastaFlags); gpu->FreeHostMemory(hPtrQueue); @@ -2248,6 +2258,7 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, dBastaLogL = gpu->AllocateMemory(kBastaIntervalBlockCount * sizeof(Real)); dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); dBastaInterval = gpu->AllocateMemory(kBufferCount * sizeof(Real)); + dBastaFlags = gpu->AllocateMemory(kBufferCount * sizeof(Real)); // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); @@ -2286,7 +2297,7 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations #endif // GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - GPUPtr coalescent = (GPUPtr) NULL; + GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2385,7 +2396,7 @@ int BeagleGPUImpl::updateInnerBastaPartials(const int* opera // gpu->PrintfDeviceVector(matrices1, kMatrixSize, r); // kernels->InnerBastaPartialsCoalescent(partials1, partials2, partials3, // matrices1, matrices2, accumulation1, accumulation2, sizes, coalescent, intervalNumber, 1, child2Index); - // Real r = 0; + //Real r = 0; // fprintf(stderr, "The child1index is: %d\n", child1Index); // fprintf(stderr, "partials1:\n"); // gpu->PrintfDeviceVector(partials1, kPaddedStateCount, r); @@ -2450,18 +2461,56 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; - const int chunkSize = 64; + const int blockSize = 32; // Real* hBastaOperationQueue = (Real*) gpu->CallocHost(sizeof(Real), operationCount * numOps); Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); for (int i = 0; i < operationCount; ++i) { hBastaInterval[i] = (Real) operations[i * numOps + 7]; + // fprintf(stderr, "interval end is: %1.2e\n", hBastaInterval[i]); + } + + Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + for (int i = 0; i < operationCount; ++i) { + if (i == 0 || hBastaInterval[i] != hBastaInterval[i - 1]) { + hBastaFlags[i] = 1; + } else { + hBastaFlags[i] = 0; + } + } + + int numBlocks = (operationCount + blockSize - 1) / blockSize; + // fprintf(stderr, "The BLOCK count is: %d\n", numBlocks); + Real* hBlockSegmentKeysEnd = (Real*) gpu->CallocHost(sizeof(Real), numBlocks); + for (int block = 0; block < numBlocks; ++block) { + int start_index = block * blockSize; + int end_index = std::min(start_index + blockSize - 1, operationCount - 1); + hBlockSegmentKeysEnd[block] = hBastaInterval[end_index]; + // fprintf(stderr, "BLOCK is: %d\n", block); + // fprintf(stderr, "BLOCK end is: %1.2e\n", hBlockSegmentKeysEnd[block]); } + + Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaEres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numBlocks); + Real* hBastaFres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numBlocks); + Real* hBastaGres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numBlocks); + Real* hBastaHres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numBlocks); + + Real r = 0; gpu->MemcpyHostToDevice(dBastaInterval, hBastaInterval, sizeof(Real) * operationCount); - gpu->FreeHostMemory(hBastaInterval); + gpu->MemcpyHostToDevice(dBastaFlags, hBastaFlags, sizeof(Real) * operationCount); + //gpu->PrintfDeviceVector(dBastaFlags, operationCount, r); dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaBlockResE = gpu->AllocateMemory(kPaddedStateCount * numBlocks * sizeof(Real)); + dBastaBlockResF = gpu->AllocateMemory(kPaddedStateCount * numBlocks * sizeof(Real)); + dBastaBlockResG = gpu->AllocateMemory(kPaddedStateCount * numBlocks * sizeof(Real)); + dBastaBlockResH = gpu->AllocateMemory(kPaddedStateCount * numBlocks * sizeof(Real)); + // for (int i = 0; i < operationCount; ++i) { // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; // hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; @@ -2482,21 +2531,13 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // gpu->MemcpyHostToDevice(dBastaOperationQueue, hBastaOperationQueue, sizeof(Real) * operationCount * numOps); // gpu->FreeHostMemory(hBastaOperationQueue); - // Real r = 0; - // fprintf(stderr, "e:\n"); - // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "f:\n"); - // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "g:\n"); - // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "h:\n"); - // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + //Real r = 0; - for (int i = 0; i < operationCount; i += chunkSize) { - // TODO execute in parallel (no race conditions) - const int start = i; - const int end = std::min(operationCount, i + chunkSize); + // for (int i = 0; i < operationCount; i += chunkSize) { + // // TODO execute in parallel (no race conditions) + const int start = 0; + const int end = operationCount; // const int child1PartialIndex = operations[op * numOps + 1]; // const int child2PartialIndex = operations[op * numOps + 3]; @@ -2509,9 +2550,40 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; // Real r = 0; - kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaE, dBastaF, dBastaG, dBastaH, dBastaInterval, numOps, + kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaE, dBastaF, dBastaG, dBastaH, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBastaInterval, dBastaFlags, numOps, start, end); - // fprintf(stderr, "e:\n"); + + gpu->MemcpyDeviceToHost(hBastaE, dBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyDeviceToHost(hBastaF, dBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyDeviceToHost(hBastaG, dBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyDeviceToHost(hBastaH, dBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyDeviceToHost(hBastaEres, dBastaBlockResE, sizeof(Real) * kPaddedStateCount * numBlocks); + gpu->MemcpyDeviceToHost(hBastaFres, dBastaBlockResF, sizeof(Real) * kPaddedStateCount * numBlocks); + gpu->MemcpyDeviceToHost(hBastaGres, dBastaBlockResG, sizeof(Real) * kPaddedStateCount * numBlocks); + gpu->MemcpyDeviceToHost(hBastaHres, dBastaBlockResH, sizeof(Real) * kPaddedStateCount * numBlocks); + + for (int block = 0; block < numBlocks; ++block) { + for (int state = 0; state < kStateCount; ++state) { + int blockInterval = hBlockSegmentKeysEnd[block]; + // fprintf(stderr, "e before: %1.2e\n", hBastaE[blockInterval * kPaddedStateCount + state]); + hBastaE[blockInterval * kPaddedStateCount + state] += hBastaEres[block * kPaddedStateCount + state]; + hBastaF[blockInterval * kPaddedStateCount + state] += hBastaFres[block * kPaddedStateCount + state]; + hBastaG[blockInterval * kPaddedStateCount + state] += hBastaGres[block * kPaddedStateCount + state]; + hBastaH[blockInterval * kPaddedStateCount + state] += hBastaHres[block * kPaddedStateCount + state]; + // fprintf(stderr, "e after: %1.2e\n", hBastaE[blockInterval * kPaddedStateCount + state]); + // fprintf(stderr,"block for eres is: %d\n", block); + // fprintf(stderr,"block interval for eres is: %d\n", blockInterval); + // fprintf(stderr, "eres is: %1.2e\n", hBastaEres[block * kPaddedStateCount + state]); + + } + } + //fprintf(stderr, "3The BLOCK count is: %d\n", numBlocks); + gpu->MemcpyHostToDevice(dBastaE, hBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaF, hBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaG, hBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaH, hBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + + // fprintf(stderr, "e:\n"); // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); // fprintf(stderr, "f:\n"); // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); @@ -2525,7 +2597,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // gpu->PrintfDeviceVector(dPartialsOrigin, 24 * kPaddedStateCount, r); // fprintf(stderr, "operations:\n"); // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); - } + // } // for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { // // TODO execute in parallel (no race conditions) @@ -2573,13 +2645,24 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // fprintf(stderr, "logL:\n"); // gpu->PrintfDeviceVector(dBastaLogL, kBastaIntervalBlockCount, r); // fprintf(stderr, "sizes:\n"); - // gpu->PrintfDeviceVector(sizes, 6, r); + // gpu->PrintfDeviceVector(sizes, 4, r); // fprintf(stderr, "coalescent:\n"); // gpu->PrintfDeviceVector(coalescent, kCoalescentBufferLength, r); gpu->MemcpyDeviceToHost(hBastaLogL, dBastaLogL, sizeof(Real) * kBastaIntervalBlockCount); for (int i = 0; i < kBastaIntervalBlockCount; i++) { out[0] += hBastaLogL[i]; } + gpu->FreeHostMemory(hBastaInterval); + gpu->FreeHostMemory(hBastaFlags); + gpu->FreeHostMemory(hBastaE); + gpu->FreeHostMemory(hBastaF); + gpu->FreeHostMemory(hBastaG); + gpu->FreeHostMemory(hBastaH); + gpu->FreeHostMemory(hBlockSegmentKeysEnd); + gpu->FreeHostMemory(hBastaEres); + gpu->FreeHostMemory(hBastaFres); + gpu->FreeHostMemory(hBastaGres); + gpu->FreeHostMemory(hBastaHres); gpu->FreeHostMemory(hBastaLogL); gpu->FreeHostMemory(hBastaDistance); diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 029e6a24..324acfc7 100644 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -241,8 +241,8 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgBastaPeelingBlock = Dim3Int(kPaddedStateCount, 32); bgBastaPeelingGrid = Dim3Int(1,1); - bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 64); - bgBastaReductionGrid = Dim3Int(1); + bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 32); + bgBastaReductionGrid = Dim3Int(110, 1); bgBastaSumBlock = Dim3Int(128); bgBastaSumGrid = Dim3Int(32); @@ -2477,7 +2477,12 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, GPUPtr f, GPUPtr g, GPUPtr h, + GPUPtr eRes, + GPUPtr fRes, + GPUPtr gRes, + GPUPtr hRes, GPUPtr intervals, + GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end) { @@ -2485,12 +2490,12 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); #endif - int parameterCountV = 7; - int totalParameterCount = 10; + int parameterCountV = 12; + int totalParameterCount = 15; gpu->LaunchKernel(fReduceWithinInterval, bgBastaReductionBlock, bgBastaReductionGrid, parameterCountV, totalParameterCount, - operations, partials, e, f, g, h, intervals, numOps, start, end); + operations, partials, e, f, g, h, eRes, fRes, gRes, hRes, intervals, flags, numOps, start, end); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index b037bbca..bf718145 100644 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -624,7 +624,7 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, // void reduceWithinInterval(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr startPartials1, GPUPtr startPartials2, // GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex, unsigned int renew); - void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end); + void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr intervals, GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end); void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); void SetupKernelBlocksAndGrids(); diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index deb5bc9c..2e775716 100644 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2174,12 +2174,17 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES KW_GLOBAL_VAR REAL* f, KW_GLOBAL_VAR REAL* g, KW_GLOBAL_VAR REAL* h, + KW_GLOBAL_VAR REAL* eRes, + KW_GLOBAL_VAR REAL* fRes, + KW_GLOBAL_VAR REAL* gRes, + KW_GLOBAL_VAR REAL* hRes, KW_GLOBAL_VAR REAL* intervals, + KW_GLOBAL_VAR REAL* flags, int numOps, int start, int end) { -#define SUM_PARTIAL_BLOCK_SIZE_B 64 +#define SUM_PARTIAL_BLOCK_SIZE_B 32 #define SUM_PARTIAL_GRID_SIZE_B 1 int state = KW_LOCAL_ID_0; @@ -2187,12 +2192,15 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES int opBlock = KW_GROUP_ID_0; int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; int opCount = end - start; + // int opCountLast = opCount % SUM_PARTIAL_BLOCK_SIZE_B; + // int blockSize = (SUM_PARTIAL_BLOCK_SIZE_B > opCountLast) ? opCountLast : SUM_PARTIAL_BLOCK_SIZE_B; + int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartialsIntervals1[SUM_PARTIAL_BLOCK_SIZE_B]; - KW_LOCAL_MEM REAL sPartialsIntervals2[SUM_PARTIAL_BLOCK_SIZE_B]; + KW_LOCAL_MEM REAL sSegmentKeys[SUM_PARTIAL_BLOCK_SIZE_B]; + KW_LOCAL_MEM REAL sSegmentFlags[SUM_PARTIAL_BLOCK_SIZE_B]; if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { int op = opNumber + start; @@ -2201,77 +2209,96 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES KW_GLOBAL_VAR REAL* startPartials1 = partials + child1PartialIndex; KW_GLOBAL_VAR REAL* endPartials1 = partials + accumulation1PartialIndex; KW_GLOBAL_VAR REAL* partialIntervals = intervals + start; - - //if (state == 0 && opIdx == 0) { - // printf("start %d\n", start); - //printf("end %d\n", end); - // printf("ops %d\n", child1PartialIndex / PADDED_STATE_COUNT); - // printf("opIdx %d\n", opIdx); - // printf("added0 %1.2e\n", startPartials1[0]); - // printf("added1 %1.2e\n", startPartials1[1]); - // printf("added2 %1.2e\n", startPartials1[2]); - // printf("added3 %1.2e\n", startPartials1[3]); - //} - + sSegmentFlags[opIdx] = flags[op]; + sSegmentKeys[opIdx] = partialIntervals[op]; sPartialsE[opIdx][state] = startPartials1[state]; sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; sPartialsG[opIdx][state] = endPartials1[state]; sPartialsH[opIdx][state] = endPartials1[state] * endPartials1[state]; - sPartialsIntervals1[opIdx] = partialIntervals[opIdx]; - sPartialsIntervals2[opIdx] = partialIntervals[opIdx]; + + } else { sPartialsE[opIdx][state] = 0; sPartialsF[opIdx][state] = 0; sPartialsG[opIdx][state] = 0; sPartialsH[opIdx][state] = 0; - sPartialsIntervals1[opIdx] = -1; - sPartialsIntervals2[opIdx] = -1; + sSegmentKeys[opIdx] = -1; + sSegmentFlags[opIdx] = 0; } KW_LOCAL_FENCE; - int intervalNumber = sPartialsIntervals1[opNumber]; + + int intervalNumber = sSegmentKeys[opIdx]; int y = intervalNumber * PADDED_STATE_COUNT; - for (int s = SUM_PARTIAL_BLOCK_SIZE_B / 2; s > 0; s >>= 1) { - int neighbor = opIdx + s; - if (neighbor < opCount && (sPartialsIntervals1[opIdx] == sPartialsIntervals1[neighbor])) { - sPartialsIntervals1[neighbor] = -1; - sPartialsE[opIdx][state] += sPartialsE[neighbor][state]; - sPartialsF[opIdx][state] += sPartialsF[neighbor][state]; - sPartialsG[opIdx][state] += sPartialsG[neighbor][state]; - sPartialsH[opIdx][state] += sPartialsH[neighbor][state]; + int u = opBlock * PADDED_STATE_COUNT; + for (int stride = 1; stride < blockSize; stride *= 2) { + int k = (opIdx + 1) * 2 * stride - 1; + if (k < blockSize) { + if (sSegmentFlags[k] == 0) { + sPartialsE[k][state] += sPartialsE[k - stride][state]; + sPartialsF[k][state] += sPartialsF[k - stride][state]; + sPartialsG[k][state] += sPartialsG[k - stride][state]; + sPartialsH[k][state] += sPartialsH[k - stride][state]; + sSegmentFlags[k] = sSegmentFlags[k - stride]; + } } KW_LOCAL_FENCE; } - KW_LOCAL_FENCE; - - if ((opIdx == 0 || sPartialsIntervals2[opIdx] != sPartialsIntervals2[opIdx - 1]) && (opNumber < opCount) && (state < PADDED_STATE_COUNT)) { - if (y >= 0) { - e[y + state] += sPartialsE[opIdx][state]; - f[y + state] += sPartialsF[opIdx][state]; - g[y + state] += sPartialsG[opIdx][state]; - h[y + state] += sPartialsH[opIdx][state]; + for (int stride = blockSize / 2; stride > 0; stride /= 2) { + int k = (opIdx + 1) * 2 * stride - 1; + if (k + stride < blockSize) { + if (sSegmentFlags[k + stride] == 0) { + sPartialsE[k + stride][state] += sPartialsE[k][state]; + sPartialsF[k + stride][state] += sPartialsF[k][state]; + sPartialsG[k + stride][state] += sPartialsG[k][state]; + sPartialsH[k + stride][state] += sPartialsH[k][state]; + sSegmentFlags[k + stride] = sSegmentFlags[k]; + } } + KW_LOCAL_FENCE; } - KW_LOCAL_FENCE; + //if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1]) { + //if (sSegmentFlags[opIdx] == 0) { + //sPartialsE[opIdx][state] += sPartialsE[opIdx - 1][state]; + //sPartialsF[opIdx][state] += sPartialsF[opIdx - 1][state]; + //sPartialsG[opIdx][state] += sPartialsG[opIdx - 1][state]; + //sPartialsH[opIdx][state] += sPartialsH[opIdx - 1][state]; + //} + //} + + if (opIdx == blockSize - 1 && (opNumber < opCount -1) || opNumber == opCount -1) { + eRes[u + state] = sPartialsE[opIdx][state]; + fRes[u + state] = sPartialsF[opIdx][state]; + gRes[u + state] = sPartialsG[opIdx][state]; + hRes[u + state] = sPartialsH[opIdx][state]; + } - if (opNumber < opCount && state < PADDED_STATE_COUNT) { + if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1] && (opIdx < blockSize - 1) && (opNumber < opCount -1)) { + if (y >= 0) { + e[y + state] = sPartialsE[opIdx][state]; + f[y + state] = sPartialsF[opIdx][state]; + g[y + state] = sPartialsG[opIdx][state]; + h[y + state] = sPartialsH[opIdx][state]; + } + } + + if (opNumber < opCount && state < PADDED_STATE_COUNT) { int op = opNumber + start; int child2PartialIndex = operations[op * numOps + 3]; int accumulation2PartialIndex = operations[op * numOps + 6]; + KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; + KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; if (child2PartialIndex >= 0) { - KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; - KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; - e[y + state] += startPartials2[state]; f[y + state] += startPartials2[state] * startPartials2[state]; g[y + state] += endPartials2[state]; h[y + state] += endPartials2[state] * endPartials2[state]; } - } + } } KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, From 95f457b1ea3c23b7dbd300888447ba686f24ff9e Mon Sep 17 00:00:00 2001 From: yucais Date: Thu, 11 Jul 2024 04:06:27 -0700 Subject: [PATCH 225/421] move calculations of flags and accumulations to GPU --- libhmsbeagle/GPU/BeagleGPUImpl.h | 1 + libhmsbeagle/GPU/BeagleGPUImpl.hpp | 79 ++++-------------------- libhmsbeagle/GPU/KernelLauncher.cpp | 52 ++++++++++++++++ libhmsbeagle/GPU/KernelLauncher.h | 8 ++- libhmsbeagle/GPU/kernels/kernelsAll.cu | 84 +++++++++++++++++++++++++- 5 files changed, 151 insertions(+), 73 deletions(-) mode change 100644 => 100755 libhmsbeagle/GPU/BeagleGPUImpl.h mode change 100644 => 100755 libhmsbeagle/GPU/BeagleGPUImpl.hpp mode change 100644 => 100755 libhmsbeagle/GPU/KernelLauncher.cpp mode change 100644 => 100755 libhmsbeagle/GPU/KernelLauncher.h mode change 100644 => 100755 libhmsbeagle/GPU/kernels/kernelsAll.cu diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h old mode 100644 new mode 100755 index 1d633566..4e31f1bd --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -214,6 +214,7 @@ class BeagleGPUImpl : public BeagleImpl { GPUPtr dBastaBuffers; GPUPtr dBastaE; + GPUPtr dBlockSegmentKeysEnd; GPUPtr dBastaF; GPUPtr dBastaG; GPUPtr dBastaH; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp old mode 100644 new mode 100755 index 8130fc1c..820d4c0f --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -91,6 +91,7 @@ namespace gpu { dBastaBlockResF = (GPUPtr)NULL; dBastaBlockResG = (GPUPtr)NULL; dBastaBlockResH = (GPUPtr)NULL; + dBlockSegmentKeysEnd= (GPUPtr)NULL; dEigenValues = NULL; dEvec = NULL; dIevc = NULL; @@ -261,6 +262,7 @@ BeagleGPUImpl::~BeagleGPUImpl() { gpu->FreeMemory(dBastaBuffers); gpu->FreeMemory(dBastaE); + gpu->FreeMemory(dBlockSegmentKeysEnd); gpu->FreeMemory(dBastaF); gpu->FreeMemory(dBastaG); gpu->FreeMemory(dBastaH); @@ -2457,7 +2459,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #endif - // GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; @@ -2469,39 +2471,18 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // fprintf(stderr, "interval end is: %1.2e\n", hBastaInterval[i]); } - Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), operationCount); - for (int i = 0; i < operationCount; ++i) { - if (i == 0 || hBastaInterval[i] != hBastaInterval[i - 1]) { - hBastaFlags[i] = 1; - } else { - hBastaFlags[i] = 0; - } - } + gpu->MemcpyHostToDevice(dBastaInterval, hBastaInterval, sizeof(Real) * operationCount); int numBlocks = (operationCount + blockSize - 1) / blockSize; // fprintf(stderr, "The BLOCK count is: %d\n", numBlocks); - Real* hBlockSegmentKeysEnd = (Real*) gpu->CallocHost(sizeof(Real), numBlocks); - for (int block = 0; block < numBlocks; ++block) { - int start_index = block * blockSize; - int end_index = std::min(start_index + blockSize - 1, operationCount - 1); - hBlockSegmentKeysEnd[block] = hBastaInterval[end_index]; - // fprintf(stderr, "BLOCK is: %d\n", block); - // fprintf(stderr, "BLOCK end is: %1.2e\n", hBlockSegmentKeysEnd[block]); - } + dBlockSegmentKeysEnd = gpu->AllocateMemory(numBlocks * sizeof(Real)); - Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaEres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numBlocks); - Real* hBastaFres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numBlocks); - Real* hBastaGres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numBlocks); - Real* hBastaHres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numBlocks); + kernels->preProcessBastaFlags(dBastaInterval, dBastaFlags, dBlockSegmentKeysEnd, operationCount, + numBlocks); Real r = 0; - gpu->MemcpyHostToDevice(dBastaInterval, hBastaInterval, sizeof(Real) * operationCount); - gpu->MemcpyHostToDevice(dBastaFlags, hBastaFlags, sizeof(Real) * operationCount); - //gpu->PrintfDeviceVector(dBastaFlags, operationCount, r); + // gpu->PrintfDeviceVector(dBastaFlags, operationCount, r); + // gpu->PrintfDeviceVector(dBlockSegmentKeysEnd, numBlocks, r); dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); @@ -2553,37 +2534,9 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaE, dBastaF, dBastaG, dBastaH, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBastaInterval, dBastaFlags, numOps, start, end); - gpu->MemcpyDeviceToHost(hBastaE, dBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyDeviceToHost(hBastaF, dBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyDeviceToHost(hBastaG, dBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyDeviceToHost(hBastaH, dBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyDeviceToHost(hBastaEres, dBastaBlockResE, sizeof(Real) * kPaddedStateCount * numBlocks); - gpu->MemcpyDeviceToHost(hBastaFres, dBastaBlockResF, sizeof(Real) * kPaddedStateCount * numBlocks); - gpu->MemcpyDeviceToHost(hBastaGres, dBastaBlockResG, sizeof(Real) * kPaddedStateCount * numBlocks); - gpu->MemcpyDeviceToHost(hBastaHres, dBastaBlockResH, sizeof(Real) * kPaddedStateCount * numBlocks); - - for (int block = 0; block < numBlocks; ++block) { - for (int state = 0; state < kStateCount; ++state) { - int blockInterval = hBlockSegmentKeysEnd[block]; - // fprintf(stderr, "e before: %1.2e\n", hBastaE[blockInterval * kPaddedStateCount + state]); - hBastaE[blockInterval * kPaddedStateCount + state] += hBastaEres[block * kPaddedStateCount + state]; - hBastaF[blockInterval * kPaddedStateCount + state] += hBastaFres[block * kPaddedStateCount + state]; - hBastaG[blockInterval * kPaddedStateCount + state] += hBastaGres[block * kPaddedStateCount + state]; - hBastaH[blockInterval * kPaddedStateCount + state] += hBastaHres[block * kPaddedStateCount + state]; - // fprintf(stderr, "e after: %1.2e\n", hBastaE[blockInterval * kPaddedStateCount + state]); - // fprintf(stderr,"block for eres is: %d\n", block); - // fprintf(stderr,"block interval for eres is: %d\n", blockInterval); - // fprintf(stderr, "eres is: %1.2e\n", hBastaEres[block * kPaddedStateCount + state]); - - } - } - //fprintf(stderr, "3The BLOCK count is: %d\n", numBlocks); - gpu->MemcpyHostToDevice(dBastaE, hBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaF, hBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaG, hBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaH, hBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + kernels->accumulateCarryOut(dBastaE, dBastaF, dBastaG, dBastaH, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBlockSegmentKeysEnd, numBlocks); - // fprintf(stderr, "e:\n"); + // fprintf(stderr, "e:\n"); // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); // fprintf(stderr, "f:\n"); // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); @@ -2653,16 +2606,6 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat out[0] += hBastaLogL[i]; } gpu->FreeHostMemory(hBastaInterval); - gpu->FreeHostMemory(hBastaFlags); - gpu->FreeHostMemory(hBastaE); - gpu->FreeHostMemory(hBastaF); - gpu->FreeHostMemory(hBastaG); - gpu->FreeHostMemory(hBastaH); - gpu->FreeHostMemory(hBlockSegmentKeysEnd); - gpu->FreeHostMemory(hBastaEres); - gpu->FreeHostMemory(hBastaFres); - gpu->FreeHostMemory(hBastaGres); - gpu->FreeHostMemory(hBastaHres); gpu->FreeHostMemory(hBastaLogL); gpu->FreeHostMemory(hBastaDistance); diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp old mode 100644 new mode 100755 index 324acfc7..04f337f4 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -244,6 +244,9 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 32); bgBastaReductionGrid = Dim3Int(110, 1); + bgBastaPreBlock = Dim3Int(32); + bgBastaPreGrid = Dim3Int(110); + bgBastaSumBlock = Dim3Int(128); bgBastaSumGrid = Dim3Int(32); } @@ -416,6 +419,8 @@ void KernelLauncher::LoadKernels() { fInnerBastaPartialsCoalescent = gpu->GetFunction("kernelInnerBastaPartialsCoalescent"); fReduceWithinInterval = gpu->GetFunction("kernelBastaReduceWithinInterval"); fReduceAcrossInterval = gpu->GetFunction("kernelBastaReduceAcrossInterval"); + fPreProcessBastaFlags = gpu->GetFunction("kernelPreProcessBastaFlags"); + fAccumulateCarryOut = gpu->GetFunction("kernelAccumulateCarryOut"); fReorderPatterns = gpu->GetFunction("kernelReorderPatterns"); @@ -2504,6 +2509,53 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, } + void KernelLauncher::preProcessBastaFlags(GPUPtr dBastaInterval, + GPUPtr dBastaFlags, + GPUPtr dBlockSegmentKeysEnd, + unsigned int operationCount, + unsigned int numBlocks) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); +#endif + int parameterCountV = 3; + int totalParameterCount = 5; + + gpu->LaunchKernel(fPreProcessBastaFlags, + bgBastaPreBlock, bgBastaPreGrid, + parameterCountV, totalParameterCount, + dBastaInterval, dBastaFlags, dBlockSegmentKeysEnd, operationCount, numBlocks); +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); +#endif + +} + + + void KernelLauncher::accumulateCarryOut(GPUPtr e, + GPUPtr f, + GPUPtr g, + GPUPtr h, + GPUPtr eRes, + GPUPtr fRes, + GPUPtr gRes, + GPUPtr hRes, + GPUPtr keys, + unsigned int numBlocks) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); +#endif + int parameterCountV = 9; + int totalParameterCount = 10; + + gpu->LaunchKernel(fAccumulateCarryOut, + bgBastaSumBlock, bgBastaSumGrid, + parameterCountV, totalParameterCount, + e, f, g, h, eRes, fRes, gRes, hRes, keys, numBlocks); +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); +#endif + +} void KernelLauncher::reduceAcrossIntervals(GPUPtr e, GPUPtr f, diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h old mode 100644 new mode 100755 index bf718145..968ef321 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -109,7 +109,8 @@ class KernelLauncher { GPUFunction fInnerBastaPartialsCoalescent; GPUFunction fReduceWithinInterval; GPUFunction fReduceAcrossInterval; - + GPUFunction fPreProcessBastaFlags; + GPUFunction fAccumulateCarryOut; GPUFunction fReorderPatterns; Dim3Int bgTransitionProbabilitiesBlock; @@ -136,6 +137,8 @@ class KernelLauncher { Dim3Int bgBastaPeelingGrid; Dim3Int bgBastaReductionBlock; Dim3Int bgBastaReductionGrid; + Dim3Int bgBastaPreBlock; + Dim3Int bgBastaPreGrid; Dim3Int bgBastaSumBlock; Dim3Int bgBastaSumGrid; @@ -626,7 +629,8 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr intervals, GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end); void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); - + void preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, GPUPtr dBlockSegmentKeysEnd, unsigned int operationCount, unsigned int numBlocks); + void accumulateCarryOut(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr keys, unsigned int numBlocks); void SetupKernelBlocksAndGrids(); protected: diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu old mode 100644 new mode 100755 index 2e775716..8c386044 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2168,6 +2168,86 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ //} //} + +KW_GLOBAL_KERNEL void kernelPreProcessBastaFlags(KW_GLOBAL_VAR REAL* KW_RESTRICT intervals, + KW_GLOBAL_VAR REAL* flags, + KW_GLOBAL_VAR REAL* blockEnds, + int operationCount, + int numBlocks) { +#define SUM_PARTIAL_BLOCK_SIZE_B 32 + int opIdx = KW_LOCAL_ID_0; + int opBlock = KW_GROUP_ID_0; + int opNumber = KW_LOCAL_ID_0 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; + int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; + + if (opNumber < operationCount) { + if (opNumber == 0 || intervals[opNumber] != intervals[opNumber - 1]) { + flags[opNumber] = 1; + } else { + flags[opNumber] = 0; + } + } + + KW_LOCAL_FENCE; + + if (opBlock < numBlocks && opIdx == 0) { + int start_index = opBlock * blockSize; + int end_index = min(start_index + blockSize - 1, operationCount - 1); + blockEnds[opBlock] = intervals[end_index]; + } +} + + +KW_GLOBAL_KERNEL void kernelAccumulateCarryOut(KW_GLOBAL_VAR REAL* e, + KW_GLOBAL_VAR REAL* f, + KW_GLOBAL_VAR REAL* g, + KW_GLOBAL_VAR REAL* h, + KW_GLOBAL_VAR REAL* eRes, + KW_GLOBAL_VAR REAL* fRes, + KW_GLOBAL_VAR REAL* gRes, + KW_GLOBAL_VAR REAL* hRes, + KW_GLOBAL_VAR REAL* keys, + int numBlocks) { +#define SUM_INTERVAL_BLOCK_SIZE 32 + int tid = KW_LOCAL_ID_0; + int group = KW_GROUP_ID_0; + int state = tid % PADDED_STATE_COUNT; + int blockIdx = tid / PADDED_STATE_COUNT; + int blockNumber = group * SUM_INTERVAL_BLOCK_SIZE + blockIdx; + int u = state + blockNumber * PADDED_STATE_COUNT; + + KW_LOCAL_MEM REAL sResE[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResF[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResG[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResH[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sSegmentEnds[SUM_INTERVAL_BLOCK_SIZE]; + if (blockNumber < numBlocks && (state < PADDED_STATE_COUNT)) { + sSegmentEnds[blockIdx] = keys[blockNumber]; + sResE[tid] = eRes[u]; + sResF[tid] = fRes[u]; + sResG[tid] = gRes[u]; + sResH[tid] = hRes[u]; + } else { + sSegmentEnds[blockIdx] = -1; + sResE[tid] = 0; + sResF[tid] = 0; + sResG[tid] = 0; + sResH[tid] = 0; + } + + KW_LOCAL_FENCE; + + if (blockNumber < numBlocks && (state < PADDED_STATE_COUNT)) { + int blockInterval = sSegmentEnds[blockIdx]; + atomicAdd(&e[blockInterval * PADDED_STATE_COUNT + state], sResE[tid]); + atomicAdd(&f[blockInterval * PADDED_STATE_COUNT + state], sResF[tid]); + atomicAdd(&g[blockInterval * PADDED_STATE_COUNT + state], sResG[tid]); + atomicAdd(&h[blockInterval * PADDED_STATE_COUNT + state], sResH[tid]); + } + KW_LOCAL_FENCE; +} + + KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, KW_GLOBAL_VAR REAL* KW_RESTRICT partials, KW_GLOBAL_VAR REAL* e, @@ -2215,8 +2295,6 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; sPartialsG[opIdx][state] = endPartials1[state]; sPartialsH[opIdx][state] = endPartials1[state] * endPartials1[state]; - - } else { sPartialsE[opIdx][state] = 0; sPartialsF[opIdx][state] = 0; @@ -2284,7 +2362,7 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES h[y + state] = sPartialsH[opIdx][state]; } } - + if (opNumber < opCount && state < PADDED_STATE_COUNT) { int op = opNumber + start; int child2PartialIndex = operations[op * numOps + 3]; From 858e0bc4fa925a320850f49ebf5e8b6406ca3c85 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 24 Jul 2024 12:27:05 -0400 Subject: [PATCH 226/421] Add first part of GPU rescaling code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 1 + libhmsbeagle/GPU/cuda_ops.cu | 109 ++++++++++++++++++++++- libhmsbeagle/GPU/cuda_ops.h | 3 + 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 953bda15..cf36ace9 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -738,6 +738,7 @@ void BeagleGPUActionImpl::rescalePartials(Real* partials, R if (cumulativeScalingBuffer) hostCumulativeScalingBuffer = MemcpyDeviceToHostVector(cumulativeScalingBuffer, kPaddedPatternCount); + rescalePartialsDevice(partials, scalingFactors, cumulativeScalingBuffer, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; for(int pattern = 0; pattern < kPatternCount;pattern++) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 86f1954b..100ff6fd 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -1,8 +1,15 @@ #include "cuda_ops.h" + #include +#include +#include +#include +#include +#include +#include +#include // QUESTION: Why can't we do the transform in-place? - void cuda_log_vector(double* v, int length) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); @@ -23,3 +30,103 @@ void cuda_log_vector(float* v, int length) thrust::transform(V.begin(), V.end(), vdptr, [] __device__ (float x) {return log(x);}); } +// FIXME: It would be nice to merge the code for the and versions of +// rescalePartialsDevice, but this was somehow causing the program to crash. + +void rescalePartialsDevice(float* partials, float* scalingFactors, float* cumulativeScalingBuffer, + int nStates, int nPatterns, int nCategories) +{ + using namespace thrust::placeholders; + + // 1. Surprisingly, cuBLAS has no operations that reduce (sum,maximize,minimize,etc.) rows or columns. + // It can only reduce an entire dense matrix to a single value. + + // 2. thrust::reduce_by_key is able to reduce regions of a vector down to MULTIPLE values. + // We assign each value a "key" that decides which group it is in. + // Adjacent values with the same key end up in the same group. + // However, non-adjacent values with the same key end up in different groups. + // thrust::reduce_by_key performs the reduction operation on each group. + + // 3. After maximizing over states, we then need to maximize over categories. + + // Implementation notes: + // * ideally we could maximize over states (adjacent memory) and categories (non-adjacent) in one operation. + // * can thrust maximize over states and categories in one operation? + // * the library MatX (https://github.com/NVIDIA/MatX) offers much nicer syntax, but + // thrust is more older and more stable (in 2024) + + // The size of a partials buffer + size_t partials_size = nStates * nPatterns * nCategories; + + // When we maximize over states, we will get this many values. + thrust::device_vector max_tmp(nPatterns * nCategories); + + // A list of 00000..111111..222222........00000... that groups values by pattern. + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / nStates) % nPatterns); + + // maximize over states for each category. + thrust::reduce_by_key( + in_keys_start, // key indices start (group by pattern) + in_keys_start + partials_size, // key indices end + thrust::device_pointer_cast(partials), // values to reduce (category, pattern, state) + thrust::make_discard_iterator(), // key values out + &max_tmp[0], // reduced values out + thrust::equal_to(), // compare keys operation + thrust::maximum() // reduction operation + ); + + // Now we have maxima per (category,pattern) in max_tmp. + // We want to go from (category,pattern) -> pattern. + // 01234...(P-1)01234..(P-1)01234...(P-1). + // We need to maximize over the non-adjacent values for each category. +} + +void rescalePartialsDevice(double* partials, double* scalingFactors, double* cumulativeScalingBuffer, + int nStates, int nPatterns, int nCategories) +{ + using namespace thrust::placeholders; + + // 1. Surprisingly, cuBLAS has no operations that reduce (sum,maximize,minimize,etc.) rows or columns. + // It can only reduce an entire dense matrix to a single value. + + // 2. thrust::reduce_by_key is able to reduce regions of a vector down to MULTIPLE values. + // We assign each value a "key" that decides which group it is in. + // Adjacent values with the same key end up in the same group. + // However, non-adjacent values with the same key end up in different groups. + // thrust::reduce_by_key performs the reduction operation on each group. + + // 3. After maximizing over states, we then need to maximize over categories. + + // Implementation notes: + // * ideally we could maximize over states (adjacent memory) and categories (non-adjacent) in one operation. + // * can thrust maximize over states and categories in one operation? + // * the library MatX (https://github.com/NVIDIA/MatX) offers much nicer syntax, but + // thrust is more older and more stable (in 2024) + + // The size of a partials buffer + size_t partials_size = nStates * nPatterns * nCategories; + + // When we maximize over states, we will get this many values. + thrust::device_vector max_tmp(nPatterns * nCategories); + + // A list of 00000..111111..222222........00000... that groups values by pattern. + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / nStates) % nPatterns); + + // maximize over states for each category. + thrust::reduce_by_key( + // add execution policy thrust::cuda::par_nosync? + in_keys_start, // key indices start (group by pattern) + in_keys_start + partials_size, // key indices end + thrust::device_pointer_cast(partials), // values to reduce (category, pattern, state) + thrust::make_discard_iterator(), // key values out + &max_tmp[0], // reduced values out + thrust::equal_to(), // compare keys operation + thrust::maximum() // reduction operation + ); + + // Now we have maxima per (category,pattern) in max_tmp. + // We want to go from (category,pattern) -> pattern. + // 01234...(P-1)01234..(P-1)01234...(P-1). + // We need to maximize over the non-adjacent values for each category. +} + diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 8134373f..e7a26a7c 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -52,4 +52,7 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); +void rescalePartialsDevice(float*, float*, float*, int, int, int); +void rescalePartialsDevice(double*, double*, double*, int, int, int); + #endif From 1dd4649e67e2dbf1a356be02ef0dbb3d7d15e136 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 24 Jul 2024 23:13:07 -0400 Subject: [PATCH 227/421] Add notes for what to do next. --- libhmsbeagle/GPU/cuda_ops.cu | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 100ff6fd..314e45ae 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -128,5 +128,11 @@ void rescalePartialsDevice(double* partials, double* scalingFactors, double* cum // We want to go from (category,pattern) -> pattern. // 01234...(P-1)01234..(P-1)01234...(P-1). // We need to maximize over the non-adjacent values for each category. + + // Actually, we can probably do this in one operation without allocating a temporary. + // We need to construct a permutation iterator that reorders the input values so that all entries + // for the same category are adjacent. + // Then we can order the keys so that we have groups of size nStates * nCategories. + // auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates*nCategories)) ); } From 4ca852dfee48267540c67260b2b6c978b438bfab Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 29 Jul 2024 14:41:30 -0400 Subject: [PATCH 228/421] Implement maximization over states and categories in one operation. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- libhmsbeagle/GPU/cuda_ops.cu | 93 ++++++++++++++---------- 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index cf36ace9..9837ba28 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -738,7 +738,7 @@ void BeagleGPUActionImpl::rescalePartials(Real* partials, R if (cumulativeScalingBuffer) hostCumulativeScalingBuffer = MemcpyDeviceToHostVector(cumulativeScalingBuffer, kPaddedPatternCount); - rescalePartialsDevice(partials, scalingFactors, cumulativeScalingBuffer, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); + // rescalePartialsDevice(partials, scalingFactors, cumulativeScalingBuffer, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; for(int pattern = 0; pattern < kPatternCount;pattern++) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 314e45ae..9c010aba 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -47,38 +47,48 @@ void rescalePartialsDevice(float* partials, float* scalingFactors, float* cumula // However, non-adjacent values with the same key end up in different groups. // thrust::reduce_by_key performs the reduction operation on each group. - // 3. After maximizing over states, we then need to maximize over categories. + // 3. Thrust can maximize over states (adjacent memory_ and categories (non-adjacent) in one operation + // by changing the order in which entries of the input matrix are visited. All the entries in the + // same group need to be visited sequentially. We can do this using a permutation iterator. // Implementation notes: - // * ideally we could maximize over states (adjacent memory) and categories (non-adjacent) in one operation. - // * can thrust maximize over states and categories in one operation? - // * the library MatX (https://github.com/NVIDIA/MatX) offers much nicer syntax, but - // thrust is more older and more stable (in 2024) + // * the library MatX (https://github.com/NVIDIA/MatX) offers much nicer syntax, but thrust is older + // and more stable (in 2024) // The size of a partials buffer size_t partials_size = nStates * nPatterns * nCategories; - // When we maximize over states, we will get this many values. - thrust::device_vector max_tmp(nPatterns * nCategories); - // A list of 00000..111111..222222........00000... that groups values by pattern. - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / nStates) % nPatterns); + // OK, so we can convert the input index i=[0...partials_size] to (p,c,s) as follows: + // s = i % nStates + // c = (i / nStates) % nCategories + // p = (i / nStates * nCategories) + + // The linear index into the input matrix is: + // j = s + nStates *p + (nStates*nPatterns)*c + + // A list of 00000..111111..222222........(P-1)(P-1)(P-1)(P-1) that groups values by state and categories. + // There should be nPatterns groups. + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates * nCategories))); + + // We need to compute j as a function of i: + // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) + auto in_values_start = thrust::make_permutation_iterator( + thrust::device_pointer_cast(partials), + thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), + (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) + ); - // maximize over states for each category. thrust::reduce_by_key( + // add execution policy thrust::cuda::par_nosync? in_keys_start, // key indices start (group by pattern) in_keys_start + partials_size, // key indices end - thrust::device_pointer_cast(partials), // values to reduce (category, pattern, state) + thrust::device_pointer_cast(partials), // values to reduce (category, pattern, state) thrust::make_discard_iterator(), // key values out - &max_tmp[0], // reduced values out + thrust::device_pointer_cast(cumulativeScalingBuffer), // reduced values out thrust::equal_to(), // compare keys operation - thrust::maximum() // reduction operation + thrust::maximum() // reduction operation ); - - // Now we have maxima per (category,pattern) in max_tmp. - // We want to go from (category,pattern) -> pattern. - // 01234...(P-1)01234..(P-1)01234...(P-1). - // We need to maximize over the non-adjacent values for each category. } void rescalePartialsDevice(double* partials, double* scalingFactors, double* cumulativeScalingBuffer, @@ -95,44 +105,47 @@ void rescalePartialsDevice(double* partials, double* scalingFactors, double* cum // However, non-adjacent values with the same key end up in different groups. // thrust::reduce_by_key performs the reduction operation on each group. - // 3. After maximizing over states, we then need to maximize over categories. + // 3. Thrust can maximize over states (adjacent memory_ and categories (non-adjacent) in one operation + // by changing the order in which entries of the input matrix are visited. All the entries in the + // same group need to be visited sequentially. We can do this using a permutation iterator. // Implementation notes: - // * ideally we could maximize over states (adjacent memory) and categories (non-adjacent) in one operation. - // * can thrust maximize over states and categories in one operation? - // * the library MatX (https://github.com/NVIDIA/MatX) offers much nicer syntax, but - // thrust is more older and more stable (in 2024) + // * the library MatX (https://github.com/NVIDIA/MatX) offers much nicer syntax, but thrust is older + // and more stable (in 2024) // The size of a partials buffer size_t partials_size = nStates * nPatterns * nCategories; - // When we maximize over states, we will get this many values. - thrust::device_vector max_tmp(nPatterns * nCategories); - // A list of 00000..111111..222222........00000... that groups values by pattern. - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / nStates) % nPatterns); + // OK, so we can convert the input index i=[0...partials_size] to (p,c,s) as follows: + // s = i % nStates + // c = (i / nStates) % nCategories + // p = (i / nStates * nCategories) + + // The linear index into the input matrix is: + // j = s + nStates *p + (nStates*nPatterns)*c + + // A list of 00000..111111..222222........(P-1)(P-1)(P-1)(P-1) that groups values by state and categories. + // There should be nPatterns groups. + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates * nCategories))); + + // We need to compute j as a function of i: + // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) + auto in_values_start = thrust::make_permutation_iterator( + thrust::device_pointer_cast(partials), + thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), + (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) + ); - // maximize over states for each category. thrust::reduce_by_key( // add execution policy thrust::cuda::par_nosync? in_keys_start, // key indices start (group by pattern) in_keys_start + partials_size, // key indices end thrust::device_pointer_cast(partials), // values to reduce (category, pattern, state) thrust::make_discard_iterator(), // key values out - &max_tmp[0], // reduced values out + thrust::device_pointer_cast(cumulativeScalingBuffer), // reduced values out thrust::equal_to(), // compare keys operation thrust::maximum() // reduction operation ); - - // Now we have maxima per (category,pattern) in max_tmp. - // We want to go from (category,pattern) -> pattern. - // 01234...(P-1)01234..(P-1)01234...(P-1). - // We need to maximize over the non-adjacent values for each category. - - // Actually, we can probably do this in one operation without allocating a temporary. - // We need to construct a permutation iterator that reorders the input values so that all entries - // for the same category are adjacent. - // Then we can order the keys so that we have groups of size nStates * nCategories. - // auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates*nCategories)) ); } From 9ac404ecabfa2ebc431226952f7a6c554fea0595 Mon Sep 17 00:00:00 2001 From: yucais Date: Tue, 30 Jul 2024 06:58:07 -0700 Subject: [PATCH 229/421] revision to kernel code --- libhmsbeagle/GPU/kernels/kernelsAll.cu | 42 ++++++++++++-------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 8c386044..32b4fbe4 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2265,7 +2265,6 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES int end) { #define SUM_PARTIAL_BLOCK_SIZE_B 32 -#define SUM_PARTIAL_GRID_SIZE_B 1 int state = KW_LOCAL_ID_0; int opIdx = KW_LOCAL_ID_1; @@ -2310,6 +2309,22 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES int intervalNumber = sSegmentKeys[opIdx]; int y = intervalNumber * PADDED_STATE_COUNT; int u = opBlock * PADDED_STATE_COUNT; + + if (opNumber < opCount && state < PADDED_STATE_COUNT) { + int op = opNumber + start; + int child2PartialIndex = operations[op * numOps + 3]; + int accumulation2PartialIndex = operations[op * numOps + 6]; + KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; + KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; + + if (child2PartialIndex >= 0) { + e[y + state] = startPartials2[state]; + f[y + state] = startPartials2[state] * startPartials2[state]; + g[y + state] = endPartials2[state]; + h[y + state] = endPartials2[state] * endPartials2[state]; + } + } + for (int stride = 1; stride < blockSize; stride *= 2) { int k = (opIdx + 1) * 2 * stride - 1; if (k < blockSize) { @@ -2355,28 +2370,11 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES } if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1] && (opIdx < blockSize - 1) && (opNumber < opCount -1)) { - if (y >= 0) { - e[y + state] = sPartialsE[opIdx][state]; - f[y + state] = sPartialsF[opIdx][state]; - g[y + state] = sPartialsG[opIdx][state]; - h[y + state] = sPartialsH[opIdx][state]; - } + e[y + state] += sPartialsE[opIdx][state]; + f[y + state] += sPartialsF[opIdx][state]; + g[y + state] += sPartialsG[opIdx][state]; + h[y + state] += sPartialsH[opIdx][state]; } - - if (opNumber < opCount && state < PADDED_STATE_COUNT) { - int op = opNumber + start; - int child2PartialIndex = operations[op * numOps + 3]; - int accumulation2PartialIndex = operations[op * numOps + 6]; - KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; - KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; - - if (child2PartialIndex >= 0) { - e[y + state] += startPartials2[state]; - f[y + state] += startPartials2[state] * startPartials2[state]; - g[y + state] += endPartials2[state]; - h[y + state] += endPartials2[state] * endPartials2[state]; - } - } } KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, From d7641fdf602a4138cfe4ec234ca9f8ef0af61bc4 Mon Sep 17 00:00:00 2001 From: yucais Date: Wed, 31 Jul 2024 04:08:23 -0700 Subject: [PATCH 230/421] one for loop for updating flags --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 41 ++++++++++++++++++------------ 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 820d4c0f..0932dbd5 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2298,8 +2298,8 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - // GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - GPUPtr coalescent = (GPUPtr) NULL; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + // GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2459,30 +2459,36 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #endif - //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - GPUPtr coalescent = (GPUPtr) NULL; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + // GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; const int blockSize = 32; - // Real* hBastaOperationQueue = (Real*) gpu->CallocHost(sizeof(Real), operationCount * numOps); + Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + int numBlocks = (operationCount + blockSize - 1) / blockSize; + Real* hBlockSegmentKeysEnd = (Real*) gpu->CallocHost(sizeof(Real), numBlocks); + for (int i = 0; i < operationCount; ++i) { hBastaInterval[i] = (Real) operations[i * numOps + 7]; - // fprintf(stderr, "interval end is: %1.2e\n", hBastaInterval[i]); - } + if (i == 0 || hBastaInterval[i] != hBastaInterval[i - 1]) { + hBastaFlags[i] = 1; + } else { + hBastaFlags[i] = 0; + } - gpu->MemcpyHostToDevice(dBastaInterval, hBastaInterval, sizeof(Real) * operationCount); + int block = i / blockSize; + if (i == operationCount - 1 || (i + 1) % blockSize == 0) { + hBlockSegmentKeysEnd[block] = hBastaInterval[i]; + } + } - int numBlocks = (operationCount + blockSize - 1) / blockSize; - // fprintf(stderr, "The BLOCK count is: %d\n", numBlocks); dBlockSegmentKeysEnd = gpu->AllocateMemory(numBlocks * sizeof(Real)); + gpu->MemcpyHostToDevice(dBastaInterval, hBastaInterval, sizeof(Real) * operationCount); + gpu->MemcpyHostToDevice(dBastaFlags, hBastaFlags, sizeof(Real) * operationCount); + gpu->MemcpyHostToDevice(dBlockSegmentKeysEnd, hBlockSegmentKeysEnd, sizeof(Real) * numBlocks); - kernels->preProcessBastaFlags(dBastaInterval, dBastaFlags, dBlockSegmentKeysEnd, operationCount, - numBlocks); - - Real r = 0; - // gpu->PrintfDeviceVector(dBastaFlags, operationCount, r); - // gpu->PrintfDeviceVector(dBlockSegmentKeysEnd, numBlocks, r); dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); @@ -2608,6 +2614,9 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat gpu->FreeHostMemory(hBastaInterval); gpu->FreeHostMemory(hBastaLogL); gpu->FreeHostMemory(hBastaDistance); + gpu->FreeHostMemory(hBastaFlags); + gpu->FreeHostMemory(hBlockSegmentKeysEnd); + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); From f9b5671669c0c91ca4236aca10abff3a355c4b02 Mon Sep 17 00:00:00 2001 From: yucais Date: Wed, 31 Jul 2024 04:12:00 -0700 Subject: [PATCH 231/421] all cpu code for reduce within interval --- libhmsbeagle/GPU/BeagleGPUImpl.h | 1 - libhmsbeagle/GPU/BeagleGPUImpl.hpp | 113 ++++++++++++++----------- libhmsbeagle/GPU/KernelLauncher.cpp | 52 ------------ libhmsbeagle/GPU/KernelLauncher.h | 8 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 93 ++------------------ 5 files changed, 76 insertions(+), 191 deletions(-) mode change 100755 => 100644 libhmsbeagle/GPU/BeagleGPUImpl.h mode change 100755 => 100644 libhmsbeagle/GPU/BeagleGPUImpl.hpp mode change 100755 => 100644 libhmsbeagle/GPU/KernelLauncher.cpp mode change 100755 => 100644 libhmsbeagle/GPU/KernelLauncher.h mode change 100755 => 100644 libhmsbeagle/GPU/kernels/kernelsAll.cu diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h old mode 100755 new mode 100644 index 4e31f1bd..1d633566 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -214,7 +214,6 @@ class BeagleGPUImpl : public BeagleImpl { GPUPtr dBastaBuffers; GPUPtr dBastaE; - GPUPtr dBlockSegmentKeysEnd; GPUPtr dBastaF; GPUPtr dBastaG; GPUPtr dBastaH; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp old mode 100755 new mode 100644 index 0932dbd5..ff8bea5f --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -91,7 +91,6 @@ namespace gpu { dBastaBlockResF = (GPUPtr)NULL; dBastaBlockResG = (GPUPtr)NULL; dBastaBlockResH = (GPUPtr)NULL; - dBlockSegmentKeysEnd= (GPUPtr)NULL; dEigenValues = NULL; dEvec = NULL; dIevc = NULL; @@ -262,7 +261,6 @@ BeagleGPUImpl::~BeagleGPUImpl() { gpu->FreeMemory(dBastaBuffers); gpu->FreeMemory(dBastaE); - gpu->FreeMemory(dBlockSegmentKeysEnd); gpu->FreeMemory(dBastaF); gpu->FreeMemory(dBastaG); gpu->FreeMemory(dBastaH); @@ -608,10 +606,6 @@ int BeagleGPUImpl::createInstance(int tipCount, for (int i = 0; i < kMatrixCount; i++) { dMatrices[i] = gpu->CreateSubPointer(dMatricesOrigin, ptrIncrement*i, ptrIncrement); } - fprintf(stderr, "The matrice increment is: %d\n", ptrIncrement); - fprintf(stderr, "The matrix size is: %d\n", kMatrixSize); - fprintf(stderr, "The state size is: %d\n", kPaddedStateCount); - fprintf(stderr, "The matrix count is: %d\n", kMatrixCount); if (kScaleBufferCount > 0) { if (kFlags & BEAGLE_FLAG_SCALING_AUTO) { dScalingFactors = (GPUPtr*) malloc(sizeof(GPUPtr) * kScaleBufferCount); @@ -706,9 +700,7 @@ int BeagleGPUImpl::createInstance(int tipCount, ptrIncrement = gpu->AlignMemOffset(kPartialsSize * sizeof(Real)); GPUPtr dPartialsTmpOrigin = gpu->AllocateMemory(partialsBufferCountTotal * ptrIncrement); dPartialsOrigin = gpu->CreateSubPointer(dPartialsTmpOrigin, 0, ptrIncrement); - fprintf(stderr, "The increment is: %d\n", ptrIncrement); hPartialsOffsets = (unsigned int*) calloc(sizeof(unsigned int), bufferCountTotal); - fprintf(stderr, "The partial buffer count is: %d\n", partialsBufferCountTotal); kIndexOffsetPat = gpu->AlignMemOffset(kPartialsSize * sizeof(Real)) / sizeof(Real); size_t ptrIncrementStates = gpu->AlignMemOffset(kPaddedPatternCount * sizeof(int)); @@ -2298,7 +2290,7 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; // GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); @@ -2462,42 +2454,65 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; // GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; - const int numOps = BEAGLE_BASTA_OP_COUNT; - const int blockSize = 32; - - Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); - Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), operationCount); - int numBlocks = (operationCount + blockSize - 1) / blockSize; - Real* hBlockSegmentKeysEnd = (Real*) gpu->CallocHost(sizeof(Real), numBlocks); - - for (int i = 0; i < operationCount; ++i) { - hBastaInterval[i] = (Real) operations[i * numOps + 7]; - if (i == 0 || hBastaInterval[i] != hBastaInterval[i - 1]) { - hBastaFlags[i] = 1; - } else { - hBastaFlags[i] = 0; - } - - int block = i / blockSize; - if (i == operationCount - 1 || (i + 1) % blockSize == 0) { - hBlockSegmentKeysEnd[block] = hBastaInterval[i]; - } - } - - dBlockSegmentKeysEnd = gpu->AllocateMemory(numBlocks * sizeof(Real)); - gpu->MemcpyHostToDevice(dBastaInterval, hBastaInterval, sizeof(Real) * operationCount); - gpu->MemcpyHostToDevice(dBastaFlags, hBastaFlags, sizeof(Real) * operationCount); - gpu->MemcpyHostToDevice(dBlockSegmentKeysEnd, hBlockSegmentKeysEnd, sizeof(Real) * numBlocks); + Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaPartials = (Real*) gpu->CallocHost(sizeof(Real), kPartialsBufferCount * kPaddedStateCount); + Real r = 0; + //gpu->PrintfDeviceVector(dBastaFlags, operationCount, r); dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaBlockResE = gpu->AllocateMemory(kPaddedStateCount * numBlocks * sizeof(Real)); - dBastaBlockResF = gpu->AllocateMemory(kPaddedStateCount * numBlocks * sizeof(Real)); - dBastaBlockResG = gpu->AllocateMemory(kPaddedStateCount * numBlocks * sizeof(Real)); - dBastaBlockResH = gpu->AllocateMemory(kPaddedStateCount * numBlocks * sizeof(Real)); + gpu->MemcpyDeviceToHost(hBastaPartials, dPartialsOrigin, sizeof(Real) * kPartialsBufferCount * kPaddedStateCount); + + for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { + // TODO execute in parallel (no race conditions) + const int start = intervalStarts[interval]; + const int end = intervalStarts[interval + 1]; + for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h + + const int numOps = BEAGLE_BASTA_OP_COUNT; + const int child1PartialIndex = hPartialsOffsets[operations[op * numOps + 1]]; + const int child2PartialIndex = hPartialsOffsets[operations[op * numOps + 3]]; + const int accumulation1PartialIndex = hPartialsOffsets[operations[op * numOps + 5]]; + const int accumulation2PartialIndex = hPartialsOffsets[operations[op * numOps + 6]]; + Real* startPartials1 = hBastaPartials + child1PartialIndex; + Real* startPartials2 = hBastaPartials + child2PartialIndex; + Real* endPartials1 = hBastaPartials + accumulation1PartialIndex; + Real* endPartials2 = hBastaPartials + accumulation2PartialIndex; + for (int i = 0; i < kStateCount; ++i) { + Real startP = startPartials1[i]; + hBastaE[interval * kPaddedStateCount + i] += startP; + hBastaF[interval * kPaddedStateCount + i] += startP * startP; + + Real endP = endPartials1[i]; + hBastaG[interval * kPaddedStateCount + i] += endP; + hBastaH[interval * kPaddedStateCount + i] += endP * endP; + } + + if (child2PartialIndex > 0) { + for (int i = 0; i < kStateCount; ++i) { + Real startP = startPartials2[i]; + hBastaE[interval * kPaddedStateCount + i] += startP; + hBastaF[interval * kPaddedStateCount + i] += startP * startP; + + Real endP = endPartials2[i]; + hBastaG[interval * kPaddedStateCount + i] += endP; + hBastaH[interval * kPaddedStateCount + i] += endP * endP; + } + } + + // fprintf(stderr, "startPartials1:\n"); + // gpu->PrintfDeviceVector(startPartials1, kPaddedStateCount, r); + // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + } + } // for (int i = 0; i < operationCount; ++i) { // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; // hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; @@ -2523,8 +2538,6 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // for (int i = 0; i < operationCount; i += chunkSize) { // // TODO execute in parallel (no race conditions) - const int start = 0; - const int end = operationCount; // const int child1PartialIndex = operations[op * numOps + 1]; // const int child2PartialIndex = operations[op * numOps + 3]; @@ -2537,12 +2550,13 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; // Real r = 0; - kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaE, dBastaF, dBastaG, dBastaH, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBastaInterval, dBastaFlags, numOps, - start, end); - kernels->accumulateCarryOut(dBastaE, dBastaF, dBastaG, dBastaH, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBlockSegmentKeysEnd, numBlocks); + gpu->MemcpyHostToDevice(dBastaE, hBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaF, hBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaG, hBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaH, hBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - // fprintf(stderr, "e:\n"); + // fprintf(stderr, "e:\n"); // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); // fprintf(stderr, "f:\n"); // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); @@ -2611,11 +2625,14 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat for (int i = 0; i < kBastaIntervalBlockCount; i++) { out[0] += hBastaLogL[i]; } - gpu->FreeHostMemory(hBastaInterval); + + gpu->FreeHostMemory(hBastaE); + gpu->FreeHostMemory(hBastaF); + gpu->FreeHostMemory(hBastaG); + gpu->FreeHostMemory(hBastaH); gpu->FreeHostMemory(hBastaLogL); gpu->FreeHostMemory(hBastaDistance); - gpu->FreeHostMemory(hBastaFlags); - gpu->FreeHostMemory(hBlockSegmentKeysEnd); + gpu->FreeHostMemory(hBastaPartials); #ifdef BEAGLE_DEBUG_FLOW diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp old mode 100755 new mode 100644 index 04f337f4..324acfc7 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -244,9 +244,6 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 32); bgBastaReductionGrid = Dim3Int(110, 1); - bgBastaPreBlock = Dim3Int(32); - bgBastaPreGrid = Dim3Int(110); - bgBastaSumBlock = Dim3Int(128); bgBastaSumGrid = Dim3Int(32); } @@ -419,8 +416,6 @@ void KernelLauncher::LoadKernels() { fInnerBastaPartialsCoalescent = gpu->GetFunction("kernelInnerBastaPartialsCoalescent"); fReduceWithinInterval = gpu->GetFunction("kernelBastaReduceWithinInterval"); fReduceAcrossInterval = gpu->GetFunction("kernelBastaReduceAcrossInterval"); - fPreProcessBastaFlags = gpu->GetFunction("kernelPreProcessBastaFlags"); - fAccumulateCarryOut = gpu->GetFunction("kernelAccumulateCarryOut"); fReorderPatterns = gpu->GetFunction("kernelReorderPatterns"); @@ -2509,53 +2504,6 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, } - void KernelLauncher::preProcessBastaFlags(GPUPtr dBastaInterval, - GPUPtr dBastaFlags, - GPUPtr dBlockSegmentKeysEnd, - unsigned int operationCount, - unsigned int numBlocks) { -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); -#endif - int parameterCountV = 3; - int totalParameterCount = 5; - - gpu->LaunchKernel(fPreProcessBastaFlags, - bgBastaPreBlock, bgBastaPreGrid, - parameterCountV, totalParameterCount, - dBastaInterval, dBastaFlags, dBlockSegmentKeysEnd, operationCount, numBlocks); -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); -#endif - -} - - - void KernelLauncher::accumulateCarryOut(GPUPtr e, - GPUPtr f, - GPUPtr g, - GPUPtr h, - GPUPtr eRes, - GPUPtr fRes, - GPUPtr gRes, - GPUPtr hRes, - GPUPtr keys, - unsigned int numBlocks) { -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); -#endif - int parameterCountV = 9; - int totalParameterCount = 10; - - gpu->LaunchKernel(fAccumulateCarryOut, - bgBastaSumBlock, bgBastaSumGrid, - parameterCountV, totalParameterCount, - e, f, g, h, eRes, fRes, gRes, hRes, keys, numBlocks); -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); -#endif - -} void KernelLauncher::reduceAcrossIntervals(GPUPtr e, GPUPtr f, diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h old mode 100755 new mode 100644 index 968ef321..bf718145 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -109,8 +109,7 @@ class KernelLauncher { GPUFunction fInnerBastaPartialsCoalescent; GPUFunction fReduceWithinInterval; GPUFunction fReduceAcrossInterval; - GPUFunction fPreProcessBastaFlags; - GPUFunction fAccumulateCarryOut; + GPUFunction fReorderPatterns; Dim3Int bgTransitionProbabilitiesBlock; @@ -137,8 +136,6 @@ class KernelLauncher { Dim3Int bgBastaPeelingGrid; Dim3Int bgBastaReductionBlock; Dim3Int bgBastaReductionGrid; - Dim3Int bgBastaPreBlock; - Dim3Int bgBastaPreGrid; Dim3Int bgBastaSumBlock; Dim3Int bgBastaSumGrid; @@ -629,8 +626,7 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr intervals, GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end); void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); - void preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, GPUPtr dBlockSegmentKeysEnd, unsigned int operationCount, unsigned int numBlocks); - void accumulateCarryOut(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr keys, unsigned int numBlocks); + void SetupKernelBlocksAndGrids(); protected: diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu old mode 100755 new mode 100644 index 32b4fbe4..8fc9f357 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2168,86 +2168,6 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ //} //} - -KW_GLOBAL_KERNEL void kernelPreProcessBastaFlags(KW_GLOBAL_VAR REAL* KW_RESTRICT intervals, - KW_GLOBAL_VAR REAL* flags, - KW_GLOBAL_VAR REAL* blockEnds, - int operationCount, - int numBlocks) { -#define SUM_PARTIAL_BLOCK_SIZE_B 32 - int opIdx = KW_LOCAL_ID_0; - int opBlock = KW_GROUP_ID_0; - int opNumber = KW_LOCAL_ID_0 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; - int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; - - if (opNumber < operationCount) { - if (opNumber == 0 || intervals[opNumber] != intervals[opNumber - 1]) { - flags[opNumber] = 1; - } else { - flags[opNumber] = 0; - } - } - - KW_LOCAL_FENCE; - - if (opBlock < numBlocks && opIdx == 0) { - int start_index = opBlock * blockSize; - int end_index = min(start_index + blockSize - 1, operationCount - 1); - blockEnds[opBlock] = intervals[end_index]; - } -} - - -KW_GLOBAL_KERNEL void kernelAccumulateCarryOut(KW_GLOBAL_VAR REAL* e, - KW_GLOBAL_VAR REAL* f, - KW_GLOBAL_VAR REAL* g, - KW_GLOBAL_VAR REAL* h, - KW_GLOBAL_VAR REAL* eRes, - KW_GLOBAL_VAR REAL* fRes, - KW_GLOBAL_VAR REAL* gRes, - KW_GLOBAL_VAR REAL* hRes, - KW_GLOBAL_VAR REAL* keys, - int numBlocks) { -#define SUM_INTERVAL_BLOCK_SIZE 32 - int tid = KW_LOCAL_ID_0; - int group = KW_GROUP_ID_0; - int state = tid % PADDED_STATE_COUNT; - int blockIdx = tid / PADDED_STATE_COUNT; - int blockNumber = group * SUM_INTERVAL_BLOCK_SIZE + blockIdx; - int u = state + blockNumber * PADDED_STATE_COUNT; - - KW_LOCAL_MEM REAL sResE[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sResF[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sResG[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sResH[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sSegmentEnds[SUM_INTERVAL_BLOCK_SIZE]; - if (blockNumber < numBlocks && (state < PADDED_STATE_COUNT)) { - sSegmentEnds[blockIdx] = keys[blockNumber]; - sResE[tid] = eRes[u]; - sResF[tid] = fRes[u]; - sResG[tid] = gRes[u]; - sResH[tid] = hRes[u]; - } else { - sSegmentEnds[blockIdx] = -1; - sResE[tid] = 0; - sResF[tid] = 0; - sResG[tid] = 0; - sResH[tid] = 0; - } - - KW_LOCAL_FENCE; - - if (blockNumber < numBlocks && (state < PADDED_STATE_COUNT)) { - int blockInterval = sSegmentEnds[blockIdx]; - atomicAdd(&e[blockInterval * PADDED_STATE_COUNT + state], sResE[tid]); - atomicAdd(&f[blockInterval * PADDED_STATE_COUNT + state], sResF[tid]); - atomicAdd(&g[blockInterval * PADDED_STATE_COUNT + state], sResG[tid]); - atomicAdd(&h[blockInterval * PADDED_STATE_COUNT + state], sResH[tid]); - } - KW_LOCAL_FENCE; -} - - KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, KW_GLOBAL_VAR REAL* KW_RESTRICT partials, KW_GLOBAL_VAR REAL* e, @@ -2294,6 +2214,8 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; sPartialsG[opIdx][state] = endPartials1[state]; sPartialsH[opIdx][state] = endPartials1[state] * endPartials1[state]; + + } else { sPartialsE[opIdx][state] = 0; sPartialsF[opIdx][state] = 0; @@ -2370,11 +2292,14 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES } if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1] && (opIdx < blockSize - 1) && (opNumber < opCount -1)) { - e[y + state] += sPartialsE[opIdx][state]; - f[y + state] += sPartialsF[opIdx][state]; - g[y + state] += sPartialsG[opIdx][state]; - h[y + state] += sPartialsH[opIdx][state]; + if (y >= 0) { + e[y + state] += sPartialsE[opIdx][state]; + f[y + state] += sPartialsF[opIdx][state]; + g[y + state] += sPartialsG[opIdx][state]; + h[y + state] += sPartialsH[opIdx][state]; + } } + } KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, From 605a0dbea68f5afbc980b9d13783687bb9ae8b9f Mon Sep 17 00:00:00 2001 From: yucais Date: Wed, 31 Jul 2024 04:28:41 -0700 Subject: [PATCH 232/421] padding (one interval per block) --- libhmsbeagle/GPU/BeagleGPUImpl.h | 1 + libhmsbeagle/GPU/BeagleGPUImpl.hpp | 185 ++++++++++--------------- libhmsbeagle/GPU/KernelLauncher.cpp | 33 ++--- libhmsbeagle/GPU/KernelLauncher.h | 2 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 100 ++++--------- 5 files changed, 117 insertions(+), 204 deletions(-) mode change 100644 => 100755 libhmsbeagle/GPU/BeagleGPUImpl.h mode change 100644 => 100755 libhmsbeagle/GPU/BeagleGPUImpl.hpp mode change 100644 => 100755 libhmsbeagle/GPU/KernelLauncher.cpp mode change 100644 => 100755 libhmsbeagle/GPU/KernelLauncher.h mode change 100644 => 100755 libhmsbeagle/GPU/kernels/kernelsAll.cu diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h old mode 100644 new mode 100755 index 1d633566..4e31f1bd --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -214,6 +214,7 @@ class BeagleGPUImpl : public BeagleImpl { GPUPtr dBastaBuffers; GPUPtr dBastaE; + GPUPtr dBlockSegmentKeysEnd; GPUPtr dBastaF; GPUPtr dBastaG; GPUPtr dBastaH; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp old mode 100644 new mode 100755 index ff8bea5f..de05c197 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2252,8 +2252,7 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, dBastaLogL = gpu->AllocateMemory(kBastaIntervalBlockCount * sizeof(Real)); dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); dBastaInterval = gpu->AllocateMemory(kBufferCount * sizeof(Real)); - dBastaFlags = gpu->AllocateMemory(kBufferCount * sizeof(Real)); - + dBastaFlags = gpu->AllocateMemory(kBufferCount * 32 * sizeof(Real)); // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); #ifdef BEAGLE_DEBUG_FLOW @@ -2290,7 +2289,7 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; // GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); @@ -2454,65 +2453,51 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; // GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; + const int numOps = BEAGLE_BASTA_OP_COUNT; + const int blockSize = 32; + Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + int numBlocks = operationCount; + Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), numBlocks * blockSize); + Real* hBlockSegmentKeysEnd = (Real*) gpu->CallocHost(sizeof(Real), numBlocks); + for (int i = 0; i < numBlocks * blockSize; ++i) { + hBastaFlags[i] = -1; + } - Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaPartials = (Real*) gpu->CallocHost(sizeof(Real), kPartialsBufferCount * kPaddedStateCount); - Real r = 0; - //gpu->PrintfDeviceVector(dBastaFlags, operationCount, r); + int blockIndex = 0; + int threadIdxInBlock = 0; + int currentInterval = hBastaInterval[0]; + int actualBlocks = 0; + for (int i = 0; i < operationCount; ++i) { + hBastaInterval[i] = (Real) operations[i * numOps + 7]; + if (hBastaInterval[i] != currentInterval || threadIdxInBlock == blockSize) { + blockIndex++; + threadIdxInBlock = 0; + currentInterval = hBastaInterval[i]; + } + hBastaFlags[blockIndex * blockSize + threadIdxInBlock] = i; + hBlockSegmentKeysEnd[blockIndex] = hBastaInterval[i]; + threadIdxInBlock++; + + actualBlocks = blockIndex + 1; + } + + gpu->MemcpyHostToDevice(dBastaFlags, hBastaFlags, sizeof(Real) * actualBlocks * blockSize); dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - gpu->MemcpyDeviceToHost(hBastaPartials, dPartialsOrigin, sizeof(Real) * kPartialsBufferCount * kPaddedStateCount); - - for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { - // TODO execute in parallel (no race conditions) - const int start = intervalStarts[interval]; - const int end = intervalStarts[interval + 1]; - for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h - - const int numOps = BEAGLE_BASTA_OP_COUNT; - - const int child1PartialIndex = hPartialsOffsets[operations[op * numOps + 1]]; - const int child2PartialIndex = hPartialsOffsets[operations[op * numOps + 3]]; - const int accumulation1PartialIndex = hPartialsOffsets[operations[op * numOps + 5]]; - const int accumulation2PartialIndex = hPartialsOffsets[operations[op * numOps + 6]]; - Real* startPartials1 = hBastaPartials + child1PartialIndex; - Real* startPartials2 = hBastaPartials + child2PartialIndex; - Real* endPartials1 = hBastaPartials + accumulation1PartialIndex; - Real* endPartials2 = hBastaPartials + accumulation2PartialIndex; - for (int i = 0; i < kStateCount; ++i) { - Real startP = startPartials1[i]; - hBastaE[interval * kPaddedStateCount + i] += startP; - hBastaF[interval * kPaddedStateCount + i] += startP * startP; - - Real endP = endPartials1[i]; - hBastaG[interval * kPaddedStateCount + i] += endP; - hBastaH[interval * kPaddedStateCount + i] += endP * endP; - } - - if (child2PartialIndex > 0) { - for (int i = 0; i < kStateCount; ++i) { - Real startP = startPartials2[i]; - hBastaE[interval * kPaddedStateCount + i] += startP; - hBastaF[interval * kPaddedStateCount + i] += startP * startP; - - Real endP = endPartials2[i]; - hBastaG[interval * kPaddedStateCount + i] += endP; - hBastaH[interval * kPaddedStateCount + i] += endP * endP; - } - } - - // fprintf(stderr, "startPartials1:\n"); - // gpu->PrintfDeviceVector(startPartials1, kPaddedStateCount, r); - // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); - // fprintf(stderr, "h:\n"); - // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - } - } + dBastaBlockResE = gpu->AllocateMemory(kPaddedStateCount * actualBlocks * sizeof(Real)); + dBastaBlockResF = gpu->AllocateMemory(kPaddedStateCount * actualBlocks * sizeof(Real)); + dBastaBlockResG = gpu->AllocateMemory(kPaddedStateCount * actualBlocks * sizeof(Real)); + dBastaBlockResH = gpu->AllocateMemory(kPaddedStateCount * actualBlocks * sizeof(Real)); + Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaEres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * actualBlocks); + Real* hBastaFres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * actualBlocks); + Real* hBastaGres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * actualBlocks); + Real* hBastaHres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * actualBlocks); // for (int i = 0; i < operationCount; ++i) { // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; // hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; @@ -2533,11 +2518,13 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // gpu->MemcpyHostToDevice(dBastaOperationQueue, hBastaOperationQueue, sizeof(Real) * operationCount * numOps); // gpu->FreeHostMemory(hBastaOperationQueue); - //Real r = 0; + Real r = 0; // for (int i = 0; i < operationCount; i += chunkSize) { // // TODO execute in parallel (no race conditions) + const int start = 0; + const int end = operationCount; // const int child1PartialIndex = operations[op * numOps + 1]; // const int child2PartialIndex = operations[op * numOps + 3]; @@ -2550,59 +2537,29 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; // Real r = 0; + kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBastaFlags, numOps, + start, end); - gpu->MemcpyHostToDevice(dBastaE, hBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaF, hBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaG, hBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaH, hBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyDeviceToHost(hBastaEres, dBastaBlockResE, sizeof(Real) * kPaddedStateCount * actualBlocks); + gpu->MemcpyDeviceToHost(hBastaFres, dBastaBlockResF, sizeof(Real) * kPaddedStateCount * actualBlocks); + gpu->MemcpyDeviceToHost(hBastaGres, dBastaBlockResG, sizeof(Real) * kPaddedStateCount * actualBlocks); + gpu->MemcpyDeviceToHost(hBastaHres, dBastaBlockResH, sizeof(Real) * kPaddedStateCount * actualBlocks); - // fprintf(stderr, "e:\n"); - // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "f:\n"); - // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "g:\n"); - // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "h:\n"); - // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - // Real r = 0; - // // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); - // fprintf(stderr, "partials:\n"); - // gpu->PrintfDeviceVector(dPartialsOrigin, 24 * kPaddedStateCount, r); - // fprintf(stderr, "operations:\n"); - // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); - // } - // for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { - // // TODO execute in parallel (no race conditions) - // const int start = intervalStarts[interval]; - // const int end = intervalStarts[interval + 1]; - // int renew = 0; - // for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h - // - // const int numOps = BEAGLE_BASTA_OP_COUNT; - // - // const int child1PartialIndex = operations[op * numOps + 1]; - // const int child2PartialIndex = operations[op * numOps + 3]; - // const int accumulation1PartialIndex = operations[op * numOps + 5]; - // const int accumulation2PartialIndex = operations[op * numOps + 6]; - // const int intervalNumber = operations[op * numOps + 7]; - // GPUPtr startPartials1 = dPartials[child1PartialIndex]; - // GPUPtr startPartials2 = dPartials[child2PartialIndex]; - // GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; - // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; - // Real r = 0; - // kernels->reduceWithinInterval(dBastaE, dBastaF, dBastaG, dBastaH, - // startPartials1, startPartials2, - // endPartials1, endPartials2, - // intervalNumber, child2PartialIndex, renew); - // renew = 1; - // fprintf(stderr, "startPartials1:\n"); - // gpu->PrintfDeviceVector(startPartials1, kPaddedStateCount, r); - // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); - // fprintf(stderr, "h:\n"); - // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - // } + for (int i = 0; i < actualBlocks; ++i) { + int interval = hBlockSegmentKeysEnd[i]; + for (int j = 0; j < kStateCount; ++j) { + hBastaE[interval * kPaddedStateCount + j] += hBastaEres[i * kPaddedStateCount + j]; + hBastaF[interval * kPaddedStateCount + j] += hBastaFres[i * kPaddedStateCount + j]; + hBastaG[interval * kPaddedStateCount + j] += hBastaGres[i * kPaddedStateCount + j]; + hBastaH[interval * kPaddedStateCount + j] += hBastaHres[i * kPaddedStateCount + j]; + } + } + gpu->MemcpyHostToDevice(dBastaE, hBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaF, hBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaG, hBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaH, hBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); Real* hBastaDistance = (Real*) gpu->CallocHost(sizeof(Real), kCoalescentBufferLength); for (int i = 0; i < kCoalescentBufferLength; ++i) { @@ -2625,15 +2582,19 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat for (int i = 0; i < kBastaIntervalBlockCount; i++) { out[0] += hBastaLogL[i]; } - + gpu->FreeHostMemory(hBastaInterval); + gpu->FreeHostMemory(hBastaLogL); + gpu->FreeHostMemory(hBastaDistance); + gpu->FreeHostMemory(hBastaFlags); + gpu->FreeHostMemory(hBlockSegmentKeysEnd); gpu->FreeHostMemory(hBastaE); gpu->FreeHostMemory(hBastaF); gpu->FreeHostMemory(hBastaG); gpu->FreeHostMemory(hBastaH); - gpu->FreeHostMemory(hBastaLogL); - gpu->FreeHostMemory(hBastaDistance); - gpu->FreeHostMemory(hBastaPartials); - + gpu->FreeHostMemory(hBastaEres); + gpu->FreeHostMemory(hBastaFres); + gpu->FreeHostMemory(hBastaGres); + gpu->FreeHostMemory(hBastaHres); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp old mode 100644 new mode 100755 index 324acfc7..7a4cf624 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -242,7 +242,7 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgBastaPeelingGrid = Dim3Int(1,1); bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 32); - bgBastaReductionGrid = Dim3Int(110, 1); + bgBastaReductionGrid = Dim3Int(295, 1); bgBastaSumBlock = Dim3Int(128); bgBastaSumGrid = Dim3Int(32); @@ -2471,31 +2471,26 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, // // } - void KernelLauncher::reduceWithinInterval(GPUPtr operations, - GPUPtr partials, - GPUPtr e, - GPUPtr f, - GPUPtr g, - GPUPtr h, - GPUPtr eRes, - GPUPtr fRes, - GPUPtr gRes, - GPUPtr hRes, - GPUPtr intervals, - GPUPtr flags, - unsigned int numOps, - unsigned int start, - unsigned int end) { +void KernelLauncher::reduceWithinInterval(GPUPtr operations, + GPUPtr partials, + GPUPtr e, + GPUPtr f, + GPUPtr g, + GPUPtr h, + GPUPtr flags, + unsigned int numOps, + unsigned int start, + unsigned int end) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); #endif - int parameterCountV = 12; - int totalParameterCount = 15; + int parameterCountV = 7; + int totalParameterCount = 10; gpu->LaunchKernel(fReduceWithinInterval, bgBastaReductionBlock, bgBastaReductionGrid, parameterCountV, totalParameterCount, - operations, partials, e, f, g, h, eRes, fRes, gRes, hRes, intervals, flags, numOps, start, end); + operations, partials, e, f, g, h, flags, numOps, start, end); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h old mode 100644 new mode 100755 index bf718145..6621f9f4 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -624,7 +624,7 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, // void reduceWithinInterval(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr startPartials1, GPUPtr startPartials2, // GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex, unsigned int renew); - void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr intervals, GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end); + void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end); void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); void SetupKernelBlocksAndGrids(); diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu old mode 100644 new mode 100755 index 8fc9f357..00e82ed9 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2174,11 +2174,6 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES KW_GLOBAL_VAR REAL* f, KW_GLOBAL_VAR REAL* g, KW_GLOBAL_VAR REAL* h, - KW_GLOBAL_VAR REAL* eRes, - KW_GLOBAL_VAR REAL* fRes, - KW_GLOBAL_VAR REAL* gRes, - KW_GLOBAL_VAR REAL* hRes, - KW_GLOBAL_VAR REAL* intervals, KW_GLOBAL_VAR REAL* flags, int numOps, int start, @@ -2190,50 +2185,39 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES int opIdx = KW_LOCAL_ID_1; int opBlock = KW_GROUP_ID_0; int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; + int y = opBlock * PADDED_STATE_COUNT; int opCount = end - start; - // int opCountLast = opCount % SUM_PARTIAL_BLOCK_SIZE_B; - // int blockSize = (SUM_PARTIAL_BLOCK_SIZE_B > opCountLast) ? opCountLast : SUM_PARTIAL_BLOCK_SIZE_B; - int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sSegmentKeys[SUM_PARTIAL_BLOCK_SIZE_B]; - KW_LOCAL_MEM REAL sSegmentFlags[SUM_PARTIAL_BLOCK_SIZE_B]; - if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { - int op = opNumber + start; + int flag = flags[opNumber]; + KW_LOCAL_FENCE; + + if (state < PADDED_STATE_COUNT && flag != -1) { + int op = flag; int child1PartialIndex = operations[op * numOps + 1]; int accumulation1PartialIndex = operations[op * numOps + 5]; + int interval = operations[op * numOps + 7]; KW_GLOBAL_VAR REAL* startPartials1 = partials + child1PartialIndex; KW_GLOBAL_VAR REAL* endPartials1 = partials + accumulation1PartialIndex; - KW_GLOBAL_VAR REAL* partialIntervals = intervals + start; - sSegmentFlags[opIdx] = flags[op]; - sSegmentKeys[opIdx] = partialIntervals[op]; + sPartialsE[opIdx][state] = startPartials1[state]; sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; sPartialsG[opIdx][state] = endPartials1[state]; sPartialsH[opIdx][state] = endPartials1[state] * endPartials1[state]; - - } else { sPartialsE[opIdx][state] = 0; sPartialsF[opIdx][state] = 0; sPartialsG[opIdx][state] = 0; sPartialsH[opIdx][state] = 0; - sSegmentKeys[opIdx] = -1; - sSegmentFlags[opIdx] = 0; } KW_LOCAL_FENCE; - - int intervalNumber = sSegmentKeys[opIdx]; - int y = intervalNumber * PADDED_STATE_COUNT; - int u = opBlock * PADDED_STATE_COUNT; - - if (opNumber < opCount && state < PADDED_STATE_COUNT) { - int op = opNumber + start; + if (state < PADDED_STATE_COUNT && flag != -1) { + int op = flag; int child2PartialIndex = operations[op * numOps + 3]; int accumulation2PartialIndex = operations[op * numOps + 6]; KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; @@ -2247,60 +2231,32 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES } } - for (int stride = 1; stride < blockSize; stride *= 2) { - int k = (opIdx + 1) * 2 * stride - 1; - if (k < blockSize) { - if (sSegmentFlags[k] == 0) { - sPartialsE[k][state] += sPartialsE[k - stride][state]; - sPartialsF[k][state] += sPartialsF[k - stride][state]; - sPartialsG[k][state] += sPartialsG[k - stride][state]; - sPartialsH[k][state] += sPartialsH[k - stride][state]; - sSegmentFlags[k] = sSegmentFlags[k - stride]; - } + for (int s = SUM_PARTIAL_BLOCK_SIZE_B / 2; s > 0; s >>= 1) { + if (opIdx < s && opIdx + s < opCount) { + sPartialsE[opIdx][state] += sPartialsE[opIdx + s][state]; + sPartialsF[opIdx][state] += sPartialsF[opIdx + s][state]; + sPartialsG[opIdx][state] += sPartialsG[opIdx + s][state]; + sPartialsH[opIdx][state] += sPartialsH[opIdx + s][state]; } KW_LOCAL_FENCE; } - for (int stride = blockSize / 2; stride > 0; stride /= 2) { - int k = (opIdx + 1) * 2 * stride - 1; - if (k + stride < blockSize) { - if (sSegmentFlags[k + stride] == 0) { - sPartialsE[k + stride][state] += sPartialsE[k][state]; - sPartialsF[k + stride][state] += sPartialsF[k][state]; - sPartialsG[k + stride][state] += sPartialsG[k][state]; - sPartialsH[k + stride][state] += sPartialsH[k][state]; - sSegmentFlags[k + stride] = sSegmentFlags[k]; - } - } - KW_LOCAL_FENCE; + KW_LOCAL_FENCE; + if (opIdx == 0 && (state < PADDED_STATE_COUNT)) { + e[y + state] += sPartialsE[0][state]; + f[y + state] += sPartialsF[0][state]; + g[y + state] += sPartialsG[0][state]; + h[y + state] += sPartialsH[0][state]; } +} + + + + + - //if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1]) { - //if (sSegmentFlags[opIdx] == 0) { - //sPartialsE[opIdx][state] += sPartialsE[opIdx - 1][state]; - //sPartialsF[opIdx][state] += sPartialsF[opIdx - 1][state]; - //sPartialsG[opIdx][state] += sPartialsG[opIdx - 1][state]; - //sPartialsH[opIdx][state] += sPartialsH[opIdx - 1][state]; - //} - //} - - if (opIdx == blockSize - 1 && (opNumber < opCount -1) || opNumber == opCount -1) { - eRes[u + state] = sPartialsE[opIdx][state]; - fRes[u + state] = sPartialsF[opIdx][state]; - gRes[u + state] = sPartialsG[opIdx][state]; - hRes[u + state] = sPartialsH[opIdx][state]; - } - if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1] && (opIdx < blockSize - 1) && (opNumber < opCount -1)) { - if (y >= 0) { - e[y + state] += sPartialsE[opIdx][state]; - f[y + state] += sPartialsF[opIdx][state]; - g[y + state] += sPartialsG[opIdx][state]; - h[y + state] += sPartialsH[opIdx][state]; - } - } -} KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, KW_GLOBAL_VAR REAL* KW_RESTRICT f, From 999dc7c496a183875b9ec5490aa1d3d4affe2364 Mon Sep 17 00:00:00 2001 From: yucais Date: Wed, 31 Jul 2024 05:00:23 -0700 Subject: [PATCH 233/421] two segmented reduction to handle carry out --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 171 +++++++++++++++--------- libhmsbeagle/GPU/KernelLauncher.cpp | 81 ++++++++++-- libhmsbeagle/GPU/KernelLauncher.h | 10 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 172 ++++++++++++++++++++----- 4 files changed, 330 insertions(+), 104 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index de05c197..b184ed9e 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -91,6 +91,7 @@ namespace gpu { dBastaBlockResF = (GPUPtr)NULL; dBastaBlockResG = (GPUPtr)NULL; dBastaBlockResH = (GPUPtr)NULL; + dBlockSegmentKeysEnd= (GPUPtr)NULL; dEigenValues = NULL; dEvec = NULL; dIevc = NULL; @@ -261,6 +262,7 @@ BeagleGPUImpl::~BeagleGPUImpl() { gpu->FreeMemory(dBastaBuffers); gpu->FreeMemory(dBastaE); + gpu->FreeMemory(dBlockSegmentKeysEnd); gpu->FreeMemory(dBastaF); gpu->FreeMemory(dBastaG); gpu->FreeMemory(dBastaH); @@ -2252,7 +2254,8 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, dBastaLogL = gpu->AllocateMemory(kBastaIntervalBlockCount * sizeof(Real)); dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); dBastaInterval = gpu->AllocateMemory(kBufferCount * sizeof(Real)); - dBastaFlags = gpu->AllocateMemory(kBufferCount * 32 * sizeof(Real)); + dBastaFlags = gpu->AllocateMemory(kBufferCount * sizeof(Real)); + // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); #ifdef BEAGLE_DEBUG_FLOW @@ -2289,8 +2292,8 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - // GPUPtr coalescent = (GPUPtr) NULL; + //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2450,54 +2453,51 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - // GPUPtr coalescent = (GPUPtr) NULL; + //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; const int blockSize = 32; - Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); - int numBlocks = operationCount; - Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), numBlocks * blockSize); - Real* hBlockSegmentKeysEnd = (Real*) gpu->CallocHost(sizeof(Real), numBlocks); - for (int i = 0; i < numBlocks * blockSize; ++i) { - hBastaFlags[i] = -1; - } - int blockIndex = 0; - int threadIdxInBlock = 0; - int currentInterval = hBastaInterval[0]; - int actualBlocks = 0; + Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + int numBlocks = (operationCount + blockSize - 1) / blockSize; + Real* hBastaSubintervals = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + Real* hBastaFinalKeys = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + int currentSubinterval = 0; + int previousInterval = 0; + int numSubinterval = 0; for (int i = 0; i < operationCount; ++i) { hBastaInterval[i] = (Real) operations[i * numOps + 7]; - if (hBastaInterval[i] != currentInterval || threadIdxInBlock == blockSize) { - blockIndex++; - threadIdxInBlock = 0; - currentInterval = hBastaInterval[i]; + if (i == 0 || hBastaInterval[i] != hBastaInterval[i - 1]) { + hBastaFlags[i] = 1; + } else { + hBastaFlags[i] = 0; } - hBastaFlags[blockIndex * blockSize + threadIdxInBlock] = i; - hBlockSegmentKeysEnd[blockIndex] = hBastaInterval[i]; - threadIdxInBlock++; - actualBlocks = blockIndex + 1; + int block = i / blockSize; + if (i > 0 && (hBastaInterval[i] != previousInterval || (i % blockSize == 0 && block > 0))) { + currentSubinterval++; + previousInterval = hBastaInterval[i]; + } + hBastaSubintervals[i] = currentSubinterval; + hBastaFinalKeys[currentSubinterval] = hBastaInterval[i]; } - gpu->MemcpyHostToDevice(dBastaFlags, hBastaFlags, sizeof(Real) * actualBlocks * blockSize); + numSubinterval = currentSubinterval + 1; + + gpu->MemcpyHostToDevice(dBastaInterval, hBastaSubintervals, sizeof(Real) * operationCount); + gpu->MemcpyHostToDevice(dBastaFlags, hBastaFlags, sizeof(Real) * operationCount); + dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaBlockResE = gpu->AllocateMemory(kPaddedStateCount * actualBlocks * sizeof(Real)); - dBastaBlockResF = gpu->AllocateMemory(kPaddedStateCount * actualBlocks * sizeof(Real)); - dBastaBlockResG = gpu->AllocateMemory(kPaddedStateCount * actualBlocks * sizeof(Real)); - dBastaBlockResH = gpu->AllocateMemory(kPaddedStateCount * actualBlocks * sizeof(Real)); - Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaEres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * actualBlocks); - Real* hBastaFres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * actualBlocks); - Real* hBastaGres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * actualBlocks); - Real* hBastaHres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * actualBlocks); + dBastaBlockResE = gpu->AllocateMemory(kPaddedStateCount * numSubinterval * sizeof(Real)); + dBastaBlockResF = gpu->AllocateMemory(kPaddedStateCount * numSubinterval * sizeof(Real)); + dBastaBlockResG = gpu->AllocateMemory(kPaddedStateCount * numSubinterval * sizeof(Real)); + dBastaBlockResH = gpu->AllocateMemory(kPaddedStateCount * numSubinterval * sizeof(Real)); + // for (int i = 0; i < operationCount; ++i) { // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; // hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; @@ -2518,9 +2518,16 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // gpu->MemcpyHostToDevice(dBastaOperationQueue, hBastaOperationQueue, sizeof(Real) * operationCount * numOps); // gpu->FreeHostMemory(hBastaOperationQueue); - Real r = 0; - + //Real r = 0; + Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaEres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); + Real* hBastaFres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); + Real* hBastaGres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); + Real* hBastaHres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); // for (int i = 0; i < operationCount; i += chunkSize) { // // TODO execute in parallel (no race conditions) const int start = 0; @@ -2537,29 +2544,76 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; // Real r = 0; - kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBastaFlags, numOps, - start, end); + kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBastaInterval, dBastaFlags, numOps, + start, end); - gpu->MemcpyDeviceToHost(hBastaEres, dBastaBlockResE, sizeof(Real) * kPaddedStateCount * actualBlocks); - gpu->MemcpyDeviceToHost(hBastaFres, dBastaBlockResF, sizeof(Real) * kPaddedStateCount * actualBlocks); - gpu->MemcpyDeviceToHost(hBastaGres, dBastaBlockResG, sizeof(Real) * kPaddedStateCount * actualBlocks); - gpu->MemcpyDeviceToHost(hBastaHres, dBastaBlockResH, sizeof(Real) * kPaddedStateCount * actualBlocks); + // fprintf(stderr, "e:\n"); + // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "f:\n"); + // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "g:\n"); + // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + // Real r = 0; + // // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); + // fprintf(stderr, "partials:\n"); + // gpu->PrintfDeviceVector(dPartialsOrigin, 24 * kPaddedStateCount, r); + // fprintf(stderr, "operations:\n"); + // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); + // } - for (int i = 0; i < actualBlocks; ++i) { - int interval = hBlockSegmentKeysEnd[i]; - for (int j = 0; j < kStateCount; ++j) { - hBastaE[interval * kPaddedStateCount + j] += hBastaEres[i * kPaddedStateCount + j]; - hBastaF[interval * kPaddedStateCount + j] += hBastaFres[i * kPaddedStateCount + j]; - hBastaG[interval * kPaddedStateCount + j] += hBastaGres[i * kPaddedStateCount + j]; - hBastaH[interval * kPaddedStateCount + j] += hBastaHres[i * kPaddedStateCount + j]; + gpu->MemcpyDeviceToHost(hBastaEres, dBastaBlockResE, sizeof(Real) * kPaddedStateCount * numSubinterval); + gpu->MemcpyDeviceToHost(hBastaFres, dBastaBlockResF, sizeof(Real) * kPaddedStateCount * numSubinterval); + gpu->MemcpyDeviceToHost(hBastaGres, dBastaBlockResG, sizeof(Real) * kPaddedStateCount * numSubinterval); + gpu->MemcpyDeviceToHost(hBastaHres, dBastaBlockResH, sizeof(Real) * kPaddedStateCount * numSubinterval); + + for (int i = 0; i < numSubinterval; ++i) { + int interval = hBastaFinalKeys[i]; + for (int j = 0; j < kStateCount; ++j) { + hBastaE[interval * kPaddedStateCount + j] += hBastaEres[i * kPaddedStateCount + j]; + hBastaF[interval * kPaddedStateCount + j] += hBastaFres[i * kPaddedStateCount + j]; + hBastaG[interval * kPaddedStateCount + j] += hBastaGres[i * kPaddedStateCount + j]; + hBastaH[interval * kPaddedStateCount + j] += hBastaHres[i * kPaddedStateCount + j]; + } } - } - gpu->MemcpyHostToDevice(dBastaE, hBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaF, hBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaG, hBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaH, hBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaE, hBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaF, hBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaG, hBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaH, hBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + // for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { + // // TODO execute in parallel (no race conditions) + // const int start = intervalStarts[interval]; + // const int end = intervalStarts[interval + 1]; + // int renew = 0; + // for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h + // + // const int numOps = BEAGLE_BASTA_OP_COUNT; + // + // const int child1PartialIndex = operations[op * numOps + 1]; + // const int child2PartialIndex = operations[op * numOps + 3]; + // const int accumulation1PartialIndex = operations[op * numOps + 5]; + // const int accumulation2PartialIndex = operations[op * numOps + 6]; + // const int intervalNumber = operations[op * numOps + 7]; + // GPUPtr startPartials1 = dPartials[child1PartialIndex]; + // GPUPtr startPartials2 = dPartials[child2PartialIndex]; + // GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; + // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; + // Real r = 0; + // kernels->reduceWithinInterval(dBastaE, dBastaF, dBastaG, dBastaH, + // startPartials1, startPartials2, + // endPartials1, endPartials2, + // intervalNumber, child2PartialIndex, renew); + // renew = 1; + // fprintf(stderr, "startPartials1:\n"); + // gpu->PrintfDeviceVector(startPartials1, kPaddedStateCount, r); + // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + // } + Real* hBastaDistance = (Real*) gpu->CallocHost(sizeof(Real), kCoalescentBufferLength); for (int i = 0; i < kCoalescentBufferLength; ++i) { @@ -2586,7 +2640,8 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat gpu->FreeHostMemory(hBastaLogL); gpu->FreeHostMemory(hBastaDistance); gpu->FreeHostMemory(hBastaFlags); - gpu->FreeHostMemory(hBlockSegmentKeysEnd); + gpu->FreeHostMemory(hBastaSubintervals); + gpu->FreeHostMemory(hBastaFinalKeys); gpu->FreeHostMemory(hBastaE); gpu->FreeHostMemory(hBastaF); gpu->FreeHostMemory(hBastaG); diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 7a4cf624..d61ecb12 100755 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -242,7 +242,10 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgBastaPeelingGrid = Dim3Int(1,1); bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 32); - bgBastaReductionGrid = Dim3Int(295, 1); + bgBastaReductionGrid = Dim3Int(110, 1); + + bgBastaPreBlock = Dim3Int(32); + bgBastaPreGrid = Dim3Int(110); bgBastaSumBlock = Dim3Int(128); bgBastaSumGrid = Dim3Int(32); @@ -416,6 +419,8 @@ void KernelLauncher::LoadKernels() { fInnerBastaPartialsCoalescent = gpu->GetFunction("kernelInnerBastaPartialsCoalescent"); fReduceWithinInterval = gpu->GetFunction("kernelBastaReduceWithinInterval"); fReduceAcrossInterval = gpu->GetFunction("kernelBastaReduceAcrossInterval"); + fPreProcessBastaFlags = gpu->GetFunction("kernelPreProcessBastaFlags"); + fAccumulateCarryOut = gpu->GetFunction("kernelAccumulateCarryOut"); fReorderPatterns = gpu->GetFunction("kernelReorderPatterns"); @@ -2471,26 +2476,27 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, // // } -void KernelLauncher::reduceWithinInterval(GPUPtr operations, - GPUPtr partials, - GPUPtr e, - GPUPtr f, - GPUPtr g, - GPUPtr h, - GPUPtr flags, - unsigned int numOps, - unsigned int start, - unsigned int end) { + void KernelLauncher::reduceWithinInterval(GPUPtr operations, + GPUPtr partials, + GPUPtr e, + GPUPtr f, + GPUPtr g, + GPUPtr h, + GPUPtr intervals, + GPUPtr flags, + unsigned int numOps, + unsigned int start, + unsigned int end) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); #endif - int parameterCountV = 7; - int totalParameterCount = 10; + int parameterCountV = 8; + int totalParameterCount = 11; gpu->LaunchKernel(fReduceWithinInterval, bgBastaReductionBlock, bgBastaReductionGrid, parameterCountV, totalParameterCount, - operations, partials, e, f, g, h, flags, numOps, start, end); + operations, partials, e, f, g, h, intervals, flags, numOps, start, end); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); @@ -2499,6 +2505,53 @@ void KernelLauncher::reduceWithinInterval(GPUPtr operations, } + void KernelLauncher::preProcessBastaFlags(GPUPtr dBastaInterval, + GPUPtr dBastaFlags, + GPUPtr dBlockSegmentKeysEnd, + unsigned int operationCount, + unsigned int numBlocks) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); +#endif + int parameterCountV = 3; + int totalParameterCount = 5; + + gpu->LaunchKernel(fPreProcessBastaFlags, + bgBastaPreBlock, bgBastaPreGrid, + parameterCountV, totalParameterCount, + dBastaInterval, dBastaFlags, dBlockSegmentKeysEnd, operationCount, numBlocks); +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); +#endif + +} + + + void KernelLauncher::accumulateCarryOut(GPUPtr e, + GPUPtr f, + GPUPtr g, + GPUPtr h, + GPUPtr eRes, + GPUPtr fRes, + GPUPtr gRes, + GPUPtr hRes, + GPUPtr keys, + unsigned int numBlocks) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); +#endif + int parameterCountV = 9; + int totalParameterCount = 10; + + gpu->LaunchKernel(fAccumulateCarryOut, + bgBastaSumBlock, bgBastaSumGrid, + parameterCountV, totalParameterCount, + e, f, g, h, eRes, fRes, gRes, hRes, keys, numBlocks); +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); +#endif + +} void KernelLauncher::reduceAcrossIntervals(GPUPtr e, GPUPtr f, diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index 6621f9f4..23831575 100755 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -109,7 +109,8 @@ class KernelLauncher { GPUFunction fInnerBastaPartialsCoalescent; GPUFunction fReduceWithinInterval; GPUFunction fReduceAcrossInterval; - + GPUFunction fPreProcessBastaFlags; + GPUFunction fAccumulateCarryOut; GPUFunction fReorderPatterns; Dim3Int bgTransitionProbabilitiesBlock; @@ -136,6 +137,8 @@ class KernelLauncher { Dim3Int bgBastaPeelingGrid; Dim3Int bgBastaReductionBlock; Dim3Int bgBastaReductionGrid; + Dim3Int bgBastaPreBlock; + Dim3Int bgBastaPreGrid; Dim3Int bgBastaSumBlock; Dim3Int bgBastaSumGrid; @@ -624,9 +627,10 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, // void reduceWithinInterval(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr startPartials1, GPUPtr startPartials2, // GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex, unsigned int renew); - void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end); + void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr intervals, GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end); void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); - + void preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, GPUPtr dBlockSegmentKeysEnd, unsigned int operationCount, unsigned int numBlocks); + void accumulateCarryOut(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr keys, unsigned int numBlocks); void SetupKernelBlocksAndGrids(); protected: diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 00e82ed9..3940da2f 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2168,12 +2168,93 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ //} //} + +KW_GLOBAL_KERNEL void kernelPreProcessBastaFlags(KW_GLOBAL_VAR REAL* KW_RESTRICT intervals, + KW_GLOBAL_VAR REAL* flags, + KW_GLOBAL_VAR REAL* blockEnds, + int operationCount, + int numBlocks) { +#define SUM_PARTIAL_BLOCK_SIZE_B 32 + int opIdx = KW_LOCAL_ID_0; + int opBlock = KW_GROUP_ID_0; + int opNumber = KW_LOCAL_ID_0 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; + int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; + + if (opNumber < operationCount) { + if (opNumber == 0 || intervals[opNumber] != intervals[opNumber - 1]) { + flags[opNumber] = 1; + } else { + flags[opNumber] = 0; + } + } + + KW_LOCAL_FENCE; + + if (opBlock < numBlocks && opIdx == 0) { + int start_index = opBlock * blockSize; + int end_index = min(start_index + blockSize - 1, operationCount - 1); + blockEnds[opBlock] = intervals[end_index]; + } +} + + +KW_GLOBAL_KERNEL void kernelAccumulateCarryOut(KW_GLOBAL_VAR REAL* e, + KW_GLOBAL_VAR REAL* f, + KW_GLOBAL_VAR REAL* g, + KW_GLOBAL_VAR REAL* h, + KW_GLOBAL_VAR REAL* eRes, + KW_GLOBAL_VAR REAL* fRes, + KW_GLOBAL_VAR REAL* gRes, + KW_GLOBAL_VAR REAL* hRes, + KW_GLOBAL_VAR REAL* keys, + int numBlocks) { +#define SUM_INTERVAL_BLOCK_SIZE 32 + int tid = KW_LOCAL_ID_0; + int group = KW_GROUP_ID_0; + int state = tid % PADDED_STATE_COUNT; + int blockIdx = tid / PADDED_STATE_COUNT; + int blockNumber = group * SUM_INTERVAL_BLOCK_SIZE + blockIdx; + int u = state + blockNumber * PADDED_STATE_COUNT; + + KW_LOCAL_MEM REAL sResE[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResF[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResG[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResH[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sSegmentEnds[SUM_INTERVAL_BLOCK_SIZE]; + if (blockNumber < numBlocks && (state < PADDED_STATE_COUNT)) { + sSegmentEnds[blockIdx] = keys[blockNumber]; + sResE[tid] = eRes[u]; + sResF[tid] = fRes[u]; + sResG[tid] = gRes[u]; + sResH[tid] = hRes[u]; + } else { + sSegmentEnds[blockIdx] = -1; + sResE[tid] = 0; + sResF[tid] = 0; + sResG[tid] = 0; + sResH[tid] = 0; + } + + KW_LOCAL_FENCE; + + if (blockNumber < numBlocks && (state < PADDED_STATE_COUNT)) { + int blockInterval = sSegmentEnds[blockIdx]; + atomicAdd(&e[blockInterval * PADDED_STATE_COUNT + state], sResE[tid]); + atomicAdd(&f[blockInterval * PADDED_STATE_COUNT + state], sResF[tid]); + atomicAdd(&g[blockInterval * PADDED_STATE_COUNT + state], sResG[tid]); + atomicAdd(&h[blockInterval * PADDED_STATE_COUNT + state], sResH[tid]); + } + KW_LOCAL_FENCE; +} + + KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, KW_GLOBAL_VAR REAL* KW_RESTRICT partials, KW_GLOBAL_VAR REAL* e, KW_GLOBAL_VAR REAL* f, KW_GLOBAL_VAR REAL* g, KW_GLOBAL_VAR REAL* h, + KW_GLOBAL_VAR REAL* intervals, KW_GLOBAL_VAR REAL* flags, int numOps, int start, @@ -2185,24 +2266,26 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES int opIdx = KW_LOCAL_ID_1; int opBlock = KW_GROUP_ID_0; int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; - int y = opBlock * PADDED_STATE_COUNT; int opCount = end - start; + // int opCountLast = opCount % SUM_PARTIAL_BLOCK_SIZE_B; + // int blockSize = (SUM_PARTIAL_BLOCK_SIZE_B > opCountLast) ? opCountLast : SUM_PARTIAL_BLOCK_SIZE_B; + int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sSegmentKeys[SUM_PARTIAL_BLOCK_SIZE_B]; + KW_LOCAL_MEM REAL sSegmentFlags[SUM_PARTIAL_BLOCK_SIZE_B]; - int flag = flags[opNumber]; - KW_LOCAL_FENCE; - - if (state < PADDED_STATE_COUNT && flag != -1) { - int op = flag; + if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { + int op = opNumber + start; int child1PartialIndex = operations[op * numOps + 1]; int accumulation1PartialIndex = operations[op * numOps + 5]; - int interval = operations[op * numOps + 7]; KW_GLOBAL_VAR REAL* startPartials1 = partials + child1PartialIndex; KW_GLOBAL_VAR REAL* endPartials1 = partials + accumulation1PartialIndex; - + KW_GLOBAL_VAR REAL* partialIntervals = intervals + start; + sSegmentFlags[opIdx] = flags[op]; + sSegmentKeys[opIdx] = partialIntervals[op]; sPartialsE[opIdx][state] = startPartials1[state]; sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; sPartialsG[opIdx][state] = endPartials1[state]; @@ -2212,12 +2295,18 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES sPartialsF[opIdx][state] = 0; sPartialsG[opIdx][state] = 0; sPartialsH[opIdx][state] = 0; + sSegmentKeys[opIdx] = -1; + sSegmentFlags[opIdx] = 0; } KW_LOCAL_FENCE; - if (state < PADDED_STATE_COUNT && flag != -1) { - int op = flag; + + int intervalNumber = sSegmentKeys[opIdx]; + int y = intervalNumber * PADDED_STATE_COUNT; + + if (opNumber < opCount && state < PADDED_STATE_COUNT) { + int op = opNumber + start; int child2PartialIndex = operations[op * numOps + 3]; int accumulation2PartialIndex = operations[op * numOps + 6]; KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; @@ -2231,32 +2320,57 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES } } - for (int s = SUM_PARTIAL_BLOCK_SIZE_B / 2; s > 0; s >>= 1) { - if (opIdx < s && opIdx + s < opCount) { - sPartialsE[opIdx][state] += sPartialsE[opIdx + s][state]; - sPartialsF[opIdx][state] += sPartialsF[opIdx + s][state]; - sPartialsG[opIdx][state] += sPartialsG[opIdx + s][state]; - sPartialsH[opIdx][state] += sPartialsH[opIdx + s][state]; + for (int stride = 1; stride < blockSize; stride *= 2) { + int k = (opIdx + 1) * 2 * stride - 1; + if (k < blockSize) { + if (sSegmentFlags[k] == 0) { + sPartialsE[k][state] += sPartialsE[k - stride][state]; + sPartialsF[k][state] += sPartialsF[k - stride][state]; + sPartialsG[k][state] += sPartialsG[k - stride][state]; + sPartialsH[k][state] += sPartialsH[k - stride][state]; + sSegmentFlags[k] = sSegmentFlags[k - stride]; + } } KW_LOCAL_FENCE; } - KW_LOCAL_FENCE; - if (opIdx == 0 && (state < PADDED_STATE_COUNT)) { - e[y + state] += sPartialsE[0][state]; - f[y + state] += sPartialsF[0][state]; - g[y + state] += sPartialsG[0][state]; - h[y + state] += sPartialsH[0][state]; + for (int stride = blockSize / 2; stride > 0; stride /= 2) { + int k = (opIdx + 1) * 2 * stride - 1; + if (k + stride < blockSize) { + if (sSegmentFlags[k + stride] == 0) { + sPartialsE[k + stride][state] += sPartialsE[k][state]; + sPartialsF[k + stride][state] += sPartialsF[k][state]; + sPartialsG[k + stride][state] += sPartialsG[k][state]; + sPartialsH[k + stride][state] += sPartialsH[k][state]; + sSegmentFlags[k + stride] = sSegmentFlags[k]; + } + } + KW_LOCAL_FENCE; } -} - - - - - - + //if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1]) { + //if (sSegmentFlags[opIdx] == 0) { + //sPartialsE[opIdx][state] += sPartialsE[opIdx - 1][state]; + //sPartialsF[opIdx][state] += sPartialsF[opIdx - 1][state]; + //sPartialsG[opIdx][state] += sPartialsG[opIdx - 1][state]; + //sPartialsH[opIdx][state] += sPartialsH[opIdx - 1][state]; + //} + //} + + if (opIdx == blockSize - 1 && (opNumber < opCount -1) || opNumber == opCount -1) { + e[y + state] += sPartialsE[opIdx][state]; + f[y + state] += sPartialsF[opIdx][state]; + g[y + state] += sPartialsG[opIdx][state]; + h[y + state] += sPartialsH[opIdx][state]; + } + if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1] && (opIdx < blockSize - 1) && (opNumber < opCount -1)) { + e[y + state] += sPartialsE[opIdx][state]; + f[y + state] += sPartialsF[opIdx][state]; + g[y + state] += sPartialsG[opIdx][state]; + h[y + state] += sPartialsH[opIdx][state]; + } +} KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, KW_GLOBAL_VAR REAL* KW_RESTRICT f, From e17e30420f4b347fbafe794ad11c70b6ad4bd506 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 5 Aug 2024 15:45:32 -0400 Subject: [PATCH 234/421] Start adding generic GPU code for rescalePartials --- libhmsbeagle/GPU/cuda_ops.cu | 13 +++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 3 +++ 2 files changed, 16 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 9c010aba..faea5ba6 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -149,3 +149,16 @@ void rescalePartialsDevice(double* partials, double* scalingFactors, double* cum ); } +void rescalePartials2(int kPaddedPatternCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) +{ + auto start_pattern = thrust::make_counting_iterator(0); + auto rescale_pattern = [] __device__ (int pattern) {}; + thrust::for_each(start_pattern, start_pattern + kPaddedPatternCount, rescale_pattern); +} + +void rescalePartials2(int kPaddedPatternCount, double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) +{ + auto start_pattern = thrust::make_counting_iterator(0); + auto rescale_pattern = [] __device__ (int pattern) {}; + thrust::for_each(start_pattern, start_pattern + kPaddedPatternCount, rescale_pattern); +} diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index e7a26a7c..b857626b 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -55,4 +55,7 @@ void cuda_log_vector(float* v, int length); void rescalePartialsDevice(float*, float*, float*, int, int, int); void rescalePartialsDevice(double*, double*, double*, int, int, int); +void rescalePartials2(int kPaddedPatternCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); +void rescalePartials2(int kPaddedPatternCount, double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex); + #endif From b599b42425b0bfc5b3829673088792bc0b60568d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 5 Aug 2024 15:49:22 -0400 Subject: [PATCH 235/421] Pass in extra dimensions. --- libhmsbeagle/GPU/cuda_ops.cu | 24 +++++++++++++++++++++--- libhmsbeagle/GPU/cuda_ops.h | 6 ++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index faea5ba6..7f8bee2b 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -149,14 +149,32 @@ void rescalePartialsDevice(double* partials, double* scalingFactors, double* cum ); } -void rescalePartials2(int kPaddedPatternCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) +void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) { auto start_pattern = thrust::make_counting_iterator(0); - auto rescale_pattern = [] __device__ (int pattern) {}; + auto rescale_pattern = [=] __device__ (int pattern) { + + // FIND_MAX_PARTIALS_X_CPU(); + int deltaPartialsByState = pattern * kPaddedStateCount; + float max = 0; + for(int m = 0; m < kCategoryCount; m++) + { + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + float iPartial = partials[deltaPartials + i]; + if (iPartial > max) + max = iPartial; + } + } + + }; thrust::for_each(start_pattern, start_pattern + kPaddedPatternCount, rescale_pattern); } -void rescalePartials2(int kPaddedPatternCount, double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) +void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) { auto start_pattern = thrust::make_counting_iterator(0); auto rescale_pattern = [] __device__ (int pattern) {}; diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index b857626b..7552b478 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -55,7 +55,9 @@ void cuda_log_vector(float* v, int length); void rescalePartialsDevice(float*, float*, float*, int, int, int); void rescalePartialsDevice(double*, double*, double*, int, int, int); -void rescalePartials2(int kPaddedPatternCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); -void rescalePartials2(int kPaddedPatternCount, double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex); +void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); +void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex); #endif From 66fd7337026bb4ab977ed7b701cd8a0ace95f05f Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 5 Aug 2024 15:50:32 -0400 Subject: [PATCH 236/421] Add commented-out call to new GPU code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 9837ba28..19b20a08 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -728,6 +728,7 @@ int BeagleGPUActionImpl::updatePartials(const int* operation BEAGLE_GPU_TEMPLATE void BeagleGPUActionImpl::rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex) { +// rescalePartials2(kCategoryCount, kPaddedPatternCount, kPaddedStateCount, partials, scalingFactors, cumulativeScalingBuffer, streamIndex); // kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, // kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); From c18af98ee8e26eaec19998f73154c7a8a07446a0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 5 Aug 2024 15:55:47 -0400 Subject: [PATCH 237/421] Pass in scalers_log also. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 5 ++--- libhmsbeagle/GPU/cuda_ops.cu | 23 ++++++++++++++++++++--- libhmsbeagle/GPU/cuda_ops.h | 4 ++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 19b20a08..bde4ac5b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -728,8 +728,6 @@ int BeagleGPUActionImpl::updatePartials(const int* operation BEAGLE_GPU_TEMPLATE void BeagleGPUActionImpl::rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex) { -// rescalePartials2(kCategoryCount, kPaddedPatternCount, kPaddedStateCount, partials, scalingFactors, cumulativeScalingBuffer, streamIndex); - // kernels->RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, // kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); @@ -739,9 +737,10 @@ void BeagleGPUActionImpl::rescalePartials(Real* partials, R if (cumulativeScalingBuffer) hostCumulativeScalingBuffer = MemcpyDeviceToHostVector(cumulativeScalingBuffer, kPaddedPatternCount); + bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; + // rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, partials, scalingFactors, cumulativeScalingBuffer, streamIndex); // rescalePartialsDevice(partials, scalingFactors, cumulativeScalingBuffer, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); - bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; for(int pattern = 0; pattern < kPatternCount;pattern++) { // FIND_MAX_PARTIALS_X_CPU(); diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 7f8bee2b..614f3d72 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -149,8 +149,8 @@ void rescalePartialsDevice(double* partials, double* scalingFactors, double* cum ); } -void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) { auto start_pattern = thrust::make_counting_iterator(0); auto rescale_pattern = [=] __device__ (int pattern) { @@ -169,11 +169,28 @@ void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedS } } + if (max == 0) + max = 1.0; + + if (scalers_log) + { + float logMax = log(max); + scalingFactors[pattern] = logMax; + if (cumulativeScalingBuffer != 0) + cumulativeScalingBuffer[pattern] += logMax; + } + else + { + scalingFactors[pattern] = max; + if (cumulativeScalingBuffer != 0) + cumulativeScalingBuffer[pattern] += log(max); + } + }; thrust::for_each(start_pattern, start_pattern + kPaddedPatternCount, rescale_pattern); } -void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) { auto start_pattern = thrust::make_counting_iterator(0); diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 7552b478..2247a151 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -55,9 +55,9 @@ void cuda_log_vector(float* v, int length); void rescalePartialsDevice(float*, float*, float*, int, int, int); void rescalePartialsDevice(double*, double*, double*, int, int, int); -void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); -void rescalePartials2(int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex); #endif From c3d54c09158bfd3bfa6caf9508641457eba51f21 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 5 Aug 2024 15:57:48 -0400 Subject: [PATCH 238/421] Add last bit. --- libhmsbeagle/GPU/cuda_ops.cu | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 614f3d72..cf4f6a8a 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -186,7 +186,17 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC cumulativeScalingBuffer[pattern] += log(max); } + // SCALE_PARTIALS_X_CPU(); + for(int m = 0; m < kCategoryCount; m++) + { + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + partials[deltaPartials + i] /= max; + } + } }; + thrust::for_each(start_pattern, start_pattern + kPaddedPatternCount, rescale_pattern); } From 5aa5c5ee2125621793601ba31e67b06b1f9d5027 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 5 Aug 2024 15:59:36 -0400 Subject: [PATCH 239/421] Also add the REAL=double version. --- libhmsbeagle/GPU/cuda_ops.cu | 45 +++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index cf4f6a8a..3a35542b 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -204,6 +204,49 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) { auto start_pattern = thrust::make_counting_iterator(0); - auto rescale_pattern = [] __device__ (int pattern) {}; + auto rescale_pattern = [=] __device__ (int pattern) { + + // FIND_MAX_PARTIALS_X_CPU(); + int deltaPartialsByState = pattern * kPaddedStateCount; + double max = 0; + for(int m = 0; m < kCategoryCount; m++) + { + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + double iPartial = partials[deltaPartials + i]; + if (iPartial > max) + max = iPartial; + } + } + + if (max == 0) + max = 1.0; + + if (scalers_log) + { + double logMax = log(max); + scalingFactors[pattern] = logMax; + if (cumulativeScalingBuffer != 0) + cumulativeScalingBuffer[pattern] += logMax; + } + else + { + scalingFactors[pattern] = max; + if (cumulativeScalingBuffer != 0) + cumulativeScalingBuffer[pattern] += log(max); + } + + // SCALE_PARTIALS_X_CPU(); + for(int m = 0; m < kCategoryCount; m++) + { + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + partials[deltaPartials + i] /= max; + } + } + }; + thrust::for_each(start_pattern, start_pattern + kPaddedPatternCount, rescale_pattern); } From efca74aa5cbe6373dd09892f465c0dd4a90d8c21 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 5 Aug 2024 16:37:37 -0400 Subject: [PATCH 240/421] Try the new rescaling code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 66 +++--------------------- 1 file changed, 6 insertions(+), 60 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index bde4ac5b..a48c5946 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -728,72 +728,18 @@ int BeagleGPUActionImpl::updatePartials(const int* operation BEAGLE_GPU_TEMPLATE void BeagleGPUActionImpl::rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex) { + std::cerr<<"initial partials (kernel) = "<RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, // kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); - auto hostPartials = MemcpyDeviceToHostVector(partials, kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - auto hostScalingFactors = MemcpyDeviceToHostVector(scalingFactors, kPaddedPatternCount); - std::vector hostCumulativeScalingBuffer; - if (cumulativeScalingBuffer) - hostCumulativeScalingBuffer = MemcpyDeviceToHostVector(cumulativeScalingBuffer, kPaddedPatternCount); - bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; - // rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, partials, scalingFactors, cumulativeScalingBuffer, streamIndex); - // rescalePartialsDevice(partials, scalingFactors, cumulativeScalingBuffer, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); - for(int pattern = 0; pattern < kPatternCount;pattern++) - { - // FIND_MAX_PARTIALS_X_CPU(); - int deltaPartialsByState = pattern * kPaddedStateCount; - REAL max = 0; - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - REAL iPartial = hostPartials[deltaPartials + i]; - if (iPartial > max) - max = iPartial; - } - } - - if (max == 0) - max = 1.0; - - if (scalers_log) - { - REAL logMax = log(max); - hostScalingFactors[pattern] = logMax; - if (cumulativeScalingBuffer != 0) - hostCumulativeScalingBuffer[pattern] += logMax; - } - else - { - hostScalingFactors[pattern] = max; - if (cumulativeScalingBuffer != 0) - hostCumulativeScalingBuffer[pattern] += log(max); - } - - // SCALE_PARTIALS_X_CPU(); - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - hostPartials[deltaPartials + i] /= max; - } - } - } - - MemcpyHostToDevice( partials, hostPartials.data(), kPaddedStateCount * kPaddedPatternCount * kCategoryCount ); - MemcpyHostToDevice( scalingFactors, hostScalingFactors.data(), kPatternCount ); - if (cumulativeScalingBuffer) - MemcpyHostToDevice( cumulativeScalingBuffer, hostCumulativeScalingBuffer.data(), kPatternCount ); + // rescalePartialsDevice(partials, scalingFactors, cumulativeScalingBuffer, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); + rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, partials, scalingFactors, cumulativeScalingBuffer, streamIndex); -// std::cerr<<"rescaled partials (kernel) = "< Date: Mon, 12 Aug 2024 13:51:59 -0400 Subject: [PATCH 241/421] Run the existing scaling code on the device. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 8 +- libhmsbeagle/GPU/cuda_ops.cu | 96 +++++++++++++++++++++++- libhmsbeagle/GPU/cuda_ops.h | 4 +- 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index a48c5946..d548902e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -728,18 +728,16 @@ int BeagleGPUActionImpl::updatePartials(const int* operation BEAGLE_GPU_TEMPLATE void BeagleGPUActionImpl::rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex) { - std::cerr<<"initial partials (kernel) = "<RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, // kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; - // rescalePartialsDevice(partials, scalingFactors, cumulativeScalingBuffer, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, partials, scalingFactors, cumulativeScalingBuffer, streamIndex); - - std::cerr<<"rescaled partials (kernel) = "< +struct rescalePartialsDeviceOp +{ + REAL* partials; + REAL* scalingFactors; + REAL* cumulativeScalingBuffer; + + int kPaddedStateCount; + int kPaddedPatternCount; + int kCategoryCount; + + bool scalers_log; + __host__ __device__ void operator()(int pattern) const + { + // FIND_MAX_PARTIALS_X_CPU(); + int deltaPartialsByState = pattern * kPaddedStateCount; + REAL max = 0; + for(int m = 0; m < kCategoryCount; m++) + { + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + REAL iPartial = partials[deltaPartials + i]; + if (iPartial > max) + max = iPartial; + } + } + + if (max == 0) + max = 1.0; + + if (scalers_log) + { + REAL logMax = log(max); + scalingFactors[pattern] = logMax; + if (cumulativeScalingBuffer != 0) + cumulativeScalingBuffer[pattern] += logMax; + } + else + { + scalingFactors[pattern] = max; + if (cumulativeScalingBuffer != 0) + cumulativeScalingBuffer[pattern] += log(max); + } + + // SCALE_PARTIALS_X_CPU(); + for(int m = 0; m < kCategoryCount; m++) + { + int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; + int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; + for(int i = 0; i < kPaddedStateCount; i++) { + partials[deltaPartials + i] /= max; + } + } + } + + rescalePartialsDeviceOp(REAL* r1, REAL* r2, REAL* r3, int i1, int i2, int i3, bool b) + :partials(r1), + scalingFactors(r2), + cumulativeScalingBuffer(r3), + kPaddedStateCount(i1), + kPaddedPatternCount(i2), + kCategoryCount(i3), + scalers_log(b) + { + } + +}; + +void rescalePartialsDevice(float* partials, float* scalingFactors, float* cumulativeScalingBuffer, + int nStates, int nPatterns, int nCategories, bool scalers_log) +{ + auto start = thrust::make_counting_iterator((int)0); + auto end = start + nPatterns; + thrust::for_each(start, end, rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, + nStates, nPatterns, nCategories, scalers_log)); +} + +void rescalePartialsDevice(double* partials, double* scalingFactors, double* cumulativeScalingBuffer, + int nStates, int nPatterns, int nCategories, bool scalers_log) +{ + auto start = thrust::make_counting_iterator((int)0); + auto end = start + nPatterns; + thrust::for_each(start, end, rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, + nStates, nPatterns, nCategories, scalers_log)); +} + + // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. -void rescalePartialsDevice(float* partials, float* scalingFactors, float* cumulativeScalingBuffer, - int nStates, int nPatterns, int nCategories) +void justMaximize(float* partials, float* scalingFactors, float* cumulativeScalingBuffer, + int nStates, int nPatterns, int nCategories) { using namespace thrust::placeholders; @@ -91,8 +179,8 @@ void rescalePartialsDevice(float* partials, float* scalingFactors, float* cumula ); } -void rescalePartialsDevice(double* partials, double* scalingFactors, double* cumulativeScalingBuffer, - int nStates, int nPatterns, int nCategories) +void justMaximize(double* partials, double* scalingFactors, double* cumulativeScalingBuffer, + int nStates, int nPatterns, int nCategories, bool scalers_log) { using namespace thrust::placeholders; diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 2247a151..49397089 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -52,8 +52,8 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); -void rescalePartialsDevice(float*, float*, float*, int, int, int); -void rescalePartialsDevice(double*, double*, double*, int, int, int); +void rescalePartialsDevice(float*, float*, float*, int, int, int, bool); +void rescalePartialsDevice(double*, double*, double*, int, int, int, bool); void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); From b3893c404a46a591c8e2ae40c81e55ccf8a8ad17 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 12 Aug 2024 14:45:10 -0400 Subject: [PATCH 242/421] Don't allocate device memory to do thrust::transform --- libhmsbeagle/GPU/cuda_ops.cu | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 04b76ff5..9df2bbb9 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -9,25 +9,18 @@ #include #include -// QUESTION: Why can't we do the transform in-place? void cuda_log_vector(double* v, int length) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); - // QUESTION: How slow is allocating the device memory here? - thrust::device_vector V(vdptr, vdptr + length); - - thrust::transform(V.begin(), V.end(), vdptr, [] __device__ (double x) {return log(x);}); + thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (double x) {return log(x);}); } void cuda_log_vector(float* v, int length) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); - // QUESTION: How slow is allocating the device memory here? - thrust::device_vector V(vdptr, vdptr + length); - - thrust::transform(V.begin(), V.end(), vdptr, [] __device__ (float x) {return log(x);}); + thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); } template From 36832e83b6b88550568815e13939b1feefc8fee6 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 12 Aug 2024 14:46:23 -0400 Subject: [PATCH 243/421] Add explanatory note. --- libhmsbeagle/GPU/cuda_ops.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 9df2bbb9..1773de8a 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -13,6 +13,7 @@ void cuda_log_vector(double* v, int length) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + // In-place update is accomplished by making the output iterator the same the starting input iterator. thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (double x) {return log(x);}); } @@ -20,6 +21,7 @@ void cuda_log_vector(float* v, int length) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + // In-place update is accomplished by making the output iterator the same the starting input iterator. thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); } From 186c04958f629a6212a81a8559d8313beda8c3cb Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 12 Aug 2024 15:50:58 -0400 Subject: [PATCH 244/421] Actually use the template version. --- libhmsbeagle/GPU/cuda_ops.cu | 130 +++++------------------------------ libhmsbeagle/GPU/cuda_ops.h | 3 - 2 files changed, 16 insertions(+), 117 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 1773de8a..e42c96e2 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -94,24 +94,27 @@ struct rescalePartialsDeviceOp }; -void rescalePartialsDevice(float* partials, float* scalingFactors, float* cumulativeScalingBuffer, - int nStates, int nPatterns, int nCategories, bool scalers_log) +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) { - auto start = thrust::make_counting_iterator((int)0); - auto end = start + nPatterns; - thrust::for_each(start, end, rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, - nStates, nPatterns, nCategories, scalers_log)); + auto start = thrust::make_counting_iterator(0); + auto end = start + kPaddedPatternCount; + + thrust::for_each(start, end, + rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, + kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); } -void rescalePartialsDevice(double* partials, double* scalingFactors, double* cumulativeScalingBuffer, - int nStates, int nPatterns, int nCategories, bool scalers_log) +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) { - auto start = thrust::make_counting_iterator((int)0); - auto end = start + nPatterns; - thrust::for_each(start, end, rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, - nStates, nPatterns, nCategories, scalers_log)); -} + auto start = thrust::make_counting_iterator(0); + auto end = start + kPaddedPatternCount; + thrust::for_each(start, end, + rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, + kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); +} // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. @@ -232,104 +235,3 @@ void justMaximize(double* partials, double* scalingFactors, double* cumulativeSc ); } -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) -{ - auto start_pattern = thrust::make_counting_iterator(0); - auto rescale_pattern = [=] __device__ (int pattern) { - - // FIND_MAX_PARTIALS_X_CPU(); - int deltaPartialsByState = pattern * kPaddedStateCount; - float max = 0; - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - float iPartial = partials[deltaPartials + i]; - if (iPartial > max) - max = iPartial; - } - } - - if (max == 0) - max = 1.0; - - if (scalers_log) - { - float logMax = log(max); - scalingFactors[pattern] = logMax; - if (cumulativeScalingBuffer != 0) - cumulativeScalingBuffer[pattern] += logMax; - } - else - { - scalingFactors[pattern] = max; - if (cumulativeScalingBuffer != 0) - cumulativeScalingBuffer[pattern] += log(max); - } - - // SCALE_PARTIALS_X_CPU(); - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - partials[deltaPartials + i] /= max; - } - } - }; - - thrust::for_each(start_pattern, start_pattern + kPaddedPatternCount, rescale_pattern); -} - -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) -{ - auto start_pattern = thrust::make_counting_iterator(0); - auto rescale_pattern = [=] __device__ (int pattern) { - - // FIND_MAX_PARTIALS_X_CPU(); - int deltaPartialsByState = pattern * kPaddedStateCount; - double max = 0; - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - double iPartial = partials[deltaPartials + i]; - if (iPartial > max) - max = iPartial; - } - } - - if (max == 0) - max = 1.0; - - if (scalers_log) - { - double logMax = log(max); - scalingFactors[pattern] = logMax; - if (cumulativeScalingBuffer != 0) - cumulativeScalingBuffer[pattern] += logMax; - } - else - { - scalingFactors[pattern] = max; - if (cumulativeScalingBuffer != 0) - cumulativeScalingBuffer[pattern] += log(max); - } - - // SCALE_PARTIALS_X_CPU(); - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - partials[deltaPartials + i] /= max; - } - } - }; - - thrust::for_each(start_pattern, start_pattern + kPaddedPatternCount, rescale_pattern); -} diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 49397089..14af290c 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -52,9 +52,6 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); -void rescalePartialsDevice(float*, float*, float*, int, int, int, bool); -void rescalePartialsDevice(double*, double*, double*, int, int, int, bool); - void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, From 15757fec221c690aef1a9c2bb67af65466e59982 Mon Sep 17 00:00:00 2001 From: yucais Date: Fri, 16 Aug 2024 00:47:48 -0700 Subject: [PATCH 245/421] fix bug in peeling for large transition matrix --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 28 +++---- libhmsbeagle/GPU/KernelLauncher.cpp | 8 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 109 ++++++++++++++----------- 3 files changed, 78 insertions(+), 67 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index b184ed9e..bd5d1051 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2247,7 +2247,7 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, kCoalescentBufferLength = bufferLength; kCoalescentBufferCount = bufferCount; - kBastaIntervalBlockCount = bufferLength / 32 + 1; + kBastaIntervalBlockCount = bufferLength / 4 + 1; dCoalescentBuffers = gpu->AllocateMemory(kCoalescentBufferCount * kCoalescentBufferLength * sizeof(Real)); @@ -2292,8 +2292,8 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - GPUPtr coalescent = (GPUPtr) NULL; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + // GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2453,11 +2453,11 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #endif - //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - GPUPtr coalescent = (GPUPtr) NULL; + GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + // GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; - const int blockSize = 32; + const int blockSize = 4; Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), operationCount); @@ -2520,14 +2520,14 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat //Real r = 0; - Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaEres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); - Real* hBastaFres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); - Real* hBastaGres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); - Real* hBastaHres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); + Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaEres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); + Real* hBastaFres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); + Real* hBastaGres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); + Real* hBastaHres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); // for (int i = 0; i < operationCount; i += chunkSize) { // // TODO execute in parallel (no race conditions) const int start = 0; diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index d61ecb12..007bcf55 100755 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -238,11 +238,11 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { //Set up block for basta partials - bgBastaPeelingBlock = Dim3Int(kPaddedStateCount, 32); - bgBastaPeelingGrid = Dim3Int(1,1); + bgBastaPeelingBlock = Dim3Int(kPaddedStateCount, 4); + bgBastaPeelingGrid = Dim3Int(30,1); - bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 32); - bgBastaReductionGrid = Dim3Int(110, 1); + bgBastaReductionBlock = Dim3Int(kPaddedStateCount, 4); + bgBastaReductionGrid = Dim3Int(18162, 1); bgBastaPreBlock = Dim3Int(32); bgBastaPreGrid = Dim3Int(110); diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 3940da2f..359c170a 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -1989,14 +1989,15 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ int numOps, int totalPatterns) { - #define PATTERN_BLOCK_SIZE_B 32 - #define BLOCK_PEELING_SIZE_B PADDED_STATE_COUNT + #define PATTERN_BLOCK_SIZE_B 4 + #define BLOCK_PEELING_SIZE_B 4 int state = KW_LOCAL_ID_0; int patIdx = KW_LOCAL_ID_1; int pattern = __umul24(KW_GROUP_ID_0,PATTERN_BLOCK_SIZE_B) + patIdx; KW_LOCAL_MEM REAL sMatrix1[BLOCK_PEELING_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sMatrix2[BLOCK_PEELING_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartials1[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; int op = pattern + start; @@ -2004,8 +2005,8 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ int desIndex = operations[op * numOps]; int child1PartialIndex = operations[op * numOps + 1]; int child1TransIndex = operations[maxOp * numOps + 2]; - int child2PartialIndex = operations[op * numOps + 3]; - int child2TransIndex = operations[op * numOps + 4]; + int child2PartialIndex = (pattern < totalPatterns) ? operations[op * numOps + 3] : -1; + int child2TransIndex = (pattern < totalPatterns) ? operations[op * numOps + 4] : -1; int accumulation1PartialIndex = operations[op * numOps + 5]; int accumulation2PartialIndex = operations[op * numOps + 6]; @@ -2015,6 +2016,9 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ if (pattern < totalPatterns) { KW_GLOBAL_VAR REAL* KW_RESTRICT partials1 = partials + child1PartialIndex; sPartials1[patIdx][state] = partials1[state]; + //printf("op %d \n", op); + //printf("index %d \n", child1PartialIndex); + //printf("partials1 %1.2e \n", partials1[state]); } else { sPartials1[patIdx][state] = 0; } @@ -2045,61 +2049,68 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ /* copy PADDED_STATE_COUNT*PATTERN_BLOCK_SIZE lengthed partials */ /* These are all coherent global memory reads; checked in Profiler */ - if (child2PartialIndex >= 0) { - KW_LOCAL_MEM REAL sPartials2[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartials2[PATTERN_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - if (pattern < totalPatterns) { - KW_GLOBAL_VAR REAL* KW_RESTRICT partials2 = partials + child2PartialIndex; - sPartials2[patIdx][state] = partials2[state]; - popSizes[state] = sizes[state]; - } else { - sPartials2[patIdx][state] = 0; - popSizes[state] = 0; - } + if (pattern < totalPatterns && child2PartialIndex >= 0) { + KW_GLOBAL_VAR REAL* KW_RESTRICT partials2 = partials + child2PartialIndex; + sPartials2[patIdx][state] = partials2[state]; + popSizes[state] = sizes[state]; + } else { + sPartials2[patIdx][state] = 0; + popSizes[state] = 0; + } - if (patIdx == 0) { - popSizes[state] = sizes[state]; - } + if (patIdx == 0) { + popSizes[state] = sizes[state]; + } - REAL sum2 = 0; + REAL sum2 = 0; - for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE_B) { - for(int j = 0; j < BLOCK_PEELING_SIZE_B; j++) { - FMA(sMatrix1[j][state], sPartials2[patIdx][i + j], sum2); - } - KW_LOCAL_FENCE; + KW_GLOBAL_VAR REAL* KW_RESTRICT matrix2 = matrices + child1TransIndex; + for (int i = 0; i < PADDED_STATE_COUNT; i += BLOCK_PEELING_SIZE_B) { + /* load one row of matrices */ + if (patIdx < BLOCK_PEELING_SIZE_B) { + /* These are all coherent global memory reads. */ + sMatrix2[patIdx][state] = matrix2[patIdx * PADDED_STATE_COUNT + state]; + /* sMatrix now filled with starting in state and ending in i */ + matrix2 += BLOCK_PEELING_SIZE_B * PADDED_STATE_COUNT; } + KW_LOCAL_FENCE; + for(int j = 0; j < BLOCK_PEELING_SIZE_B; j++) { + FMA(sMatrix2[j][state], sPartials2[patIdx][i + j], sum2); + } + KW_LOCAL_FENCE; + } - if (pattern < totalPatterns) { - KW_GLOBAL_VAR REAL* KW_RESTRICT partials3 = partials + desIndex; - KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation1 = partials + accumulation1PartialIndex; - KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation2 = partials + accumulation2PartialIndex; - accumulation1[state] = sum1; - accumulation2[state] = sum2; - if (popSizes[state] > 0) { - partials3[state] = sum1 * sum2 / popSizes[state]; - } else { - partials3[state] = 0; - } - sPartials3[patIdx][state] = partials3[state]; + if (pattern < totalPatterns && child2PartialIndex >= 0) { + KW_GLOBAL_VAR REAL* KW_RESTRICT partials3 = partials + desIndex; + KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation1 = partials + accumulation1PartialIndex; + KW_GLOBAL_VAR REAL* KW_RESTRICT accumulation2 = partials + accumulation2PartialIndex; + accumulation1[state] = sum1; + accumulation2[state] = sum2; + if (popSizes[state] > 0) { + partials3[state] = sum1 * sum2 / popSizes[state]; + } else { + partials3[state] = 0; + } + sPartials3[patIdx][state] = partials3[state]; #ifdef IS_POWER_OF_TWO - // parallelized reduction *** only works for powers-of-2 **** - for (int i = PADDED_STATE_COUNT / 2; i > 0; i >>= 1) { - if (state < i) { + // parallelized reduction *** only works for powers-of-2 **** + for (int i = PADDED_STATE_COUNT / 2; i > 0; i >>= 1) { + if (state < i) { #else - for (int i = SMALLEST_POWER_OF_TWO / 2; i > 0; i >>= 1) { - if (state < i && state + i < PADDED_STATE_COUNT) { + for (int i = SMALLEST_POWER_OF_TWO / 2; i > 0; i >>= 1) { + if (state < i && state + i < PADDED_STATE_COUNT) { #endif // IS_POWER_OF_TWO - sPartials3[patIdx][state] += sPartials3[patIdx][state + i]; - } - KW_LOCAL_FENCE; + sPartials3[patIdx][state] += sPartials3[patIdx][state + i]; } - REAL denominator = sPartials3[patIdx][0]; - partials3[state] = partials3[state] / denominator; - coalescent[intervalNumber] = denominator; - } + KW_LOCAL_FENCE; + } + REAL denominator = sPartials3[patIdx][0]; + partials3[state] = partials3[state] / denominator; + coalescent[intervalNumber] = denominator; } } @@ -2260,7 +2271,7 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES int start, int end) { -#define SUM_PARTIAL_BLOCK_SIZE_B 32 +#define SUM_PARTIAL_BLOCK_SIZE_B 4 int state = KW_LOCAL_ID_0; int opIdx = KW_LOCAL_ID_1; @@ -2382,7 +2393,7 @@ KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RES KW_GLOBAL_VAR REAL* KW_RESTRICT coalescent, int intervalStartsCount) { -#define SUM_INTERVAL_BLOCK_SIZE 32 +#define SUM_INTERVAL_BLOCK_SIZE 4 int intervalCount = intervalStartsCount - 1; int tid = KW_LOCAL_ID_0; From 6197c342e19cc1942afe437907123aa32167962d Mon Sep 17 00:00:00 2001 From: yucais Date: Fri, 16 Aug 2024 01:31:37 -0700 Subject: [PATCH 246/421] one read and write for e, f, g, h and flags --- libhmsbeagle/GPU/BeagleGPUImpl.h | 11 +- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 134 +++++++++++-------------- libhmsbeagle/GPU/KernelLauncher.cpp | 31 +++--- libhmsbeagle/GPU/KernelLauncher.h | 4 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 29 ++++-- 5 files changed, 95 insertions(+), 114 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 4e31f1bd..e45f8973 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -213,21 +213,16 @@ class BeagleGPUImpl : public BeagleImpl { int* hStreamIndices; GPUPtr dBastaBuffers; - GPUPtr dBastaE; + GPUPtr dBastaBlockResMemory; + GPUPtr dBastaFinalResMemory; + GPUPtr dBastaMemory; GPUPtr dBlockSegmentKeysEnd; - GPUPtr dBastaF; - GPUPtr dBastaG; - GPUPtr dBastaH; GPUPtr dBastaLogL; GPUPtr dBastaDistance; GPUPtr dBastaOperationQueue; GPUPtr dCoalescentBuffers; GPUPtr dBastaInterval; GPUPtr dBastaFlags; - GPUPtr dBastaBlockResE; - GPUPtr dBastaBlockResF; - GPUPtr dBastaBlockResG; - GPUPtr dBastaBlockResH; int kCoalescentBufferLength; int kCoalescentBufferCount; int kBastaIntervalBlockCount; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index bd5d1051..233a778a 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -34,6 +34,7 @@ #include "libhmsbeagle/GPU/GPUInterface.h" #include "libhmsbeagle/GPU/Precision.h" #include "BeagleGPUImpl.h" +#include "libhmsbeagle/benchmark/BenchmarkHelper.h" namespace beagle { namespace gpu { @@ -76,21 +77,15 @@ namespace gpu { dMaxScalingFactors = (GPUPtr)NULL; dIndexMaxScalingFactors = (GPUPtr)NULL; - dBastaE = (GPUPtr)NULL; - dBastaF = (GPUPtr)NULL; - dBastaG = (GPUPtr)NULL; - dBastaH = (GPUPtr)NULL; + + dBastaMemory = (GPUPtr)NULL; + dBastaBlockResMemory = (GPUPtr)NULL; dBastaLogL = (GPUPtr)NULL; dBastaDistance = (GPUPtr)NULL; dBastaOperationQueue = (GPUPtr)NULL; dBastaBuffers = (GPUPtr)NULL; dCoalescentBuffers = (GPUPtr)NULL; dBastaInterval = (GPUPtr)NULL; - dBastaFlags = (GPUPtr)NULL; - dBastaBlockResE = (GPUPtr)NULL; - dBastaBlockResF = (GPUPtr)NULL; - dBastaBlockResG = (GPUPtr)NULL; - dBastaBlockResH = (GPUPtr)NULL; dBlockSegmentKeysEnd= (GPUPtr)NULL; dEigenValues = NULL; dEvec = NULL; @@ -261,20 +256,13 @@ BeagleGPUImpl::~BeagleGPUImpl() { gpu->FreeMemory(dBastaBuffers); - gpu->FreeMemory(dBastaE); + gpu->FreeMemory(dBastaBlockResMemory); gpu->FreeMemory(dBlockSegmentKeysEnd); - gpu->FreeMemory(dBastaF); - gpu->FreeMemory(dBastaG); - gpu->FreeMemory(dBastaH); - gpu->FreeMemory(dBastaBlockResE); - gpu->FreeMemory(dBastaBlockResF); - gpu->FreeMemory(dBastaBlockResG); - gpu->FreeMemory(dBastaBlockResH); + gpu->FreeMemory(dBastaMemory); gpu->FreeMemory(dBastaLogL); gpu->FreeMemory(dBastaDistance); gpu->FreeMemory(dBastaOperationQueue); gpu->FreeMemory(dBastaInterval); - gpu->FreeMemory(dBastaFlags); gpu->FreeHostMemory(hPtrQueue); @@ -2384,8 +2372,6 @@ int BeagleGPUImpl::updateInnerBastaPartials(const int* opera // GPUPtr accumulation2 = dPartials[accumulation2PartialIndex]; // Real r = 1; - // fprintf(stderr, "The partial index is: %d\n", child1Index); - // fprintf(stderr, "The matrix index is: %d\n", child1TransMatIndex); // fprintf(stderr, "partial1:\n"); // gpu->PrintfDeviceVector(partials1, kPaddedStateCount, r); // fprintf(stderr, "matice1:\n"); @@ -2451,8 +2437,8 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); #endif - - + //DEBUG_CREATE_TIME(); + //DEBUG_START_TIME(); GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; // GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2460,9 +2446,9 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat const int blockSize = 4; Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); - Real* hBastaFlags = (Real*) gpu->CallocHost(sizeof(Real), operationCount); int numBlocks = (operationCount + blockSize - 1) / blockSize; - Real* hBastaSubintervals = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + Real* hBastaSubintervals = (Real*) gpu->CallocHost(sizeof(Real), 2 * operationCount); + Real* hBastaFlags = hBastaSubintervals + operationCount; Real* hBastaFinalKeys = (Real*) gpu->CallocHost(sizeof(Real), operationCount); int currentSubinterval = 0; int previousInterval = 0; @@ -2483,20 +2469,20 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat hBastaSubintervals[i] = currentSubinterval; hBastaFinalKeys[currentSubinterval] = hBastaInterval[i]; } + //DEBUG_END_TIME(); + + // std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl; + //for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) { + // std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first) + // .count() << " ns" << "\t" << it->second.second << std::endl; + // }; numSubinterval = currentSubinterval + 1; - gpu->MemcpyHostToDevice(dBastaInterval, hBastaSubintervals, sizeof(Real) * operationCount); - gpu->MemcpyHostToDevice(dBastaFlags, hBastaFlags, sizeof(Real) * operationCount); + gpu->MemcpyHostToDevice(dBastaInterval, hBastaSubintervals, sizeof(Real) * 2 * operationCount); - dBastaE = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaF = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaG = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaH = gpu->AllocateMemory(kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaBlockResE = gpu->AllocateMemory(kPaddedStateCount * numSubinterval * sizeof(Real)); - dBastaBlockResF = gpu->AllocateMemory(kPaddedStateCount * numSubinterval * sizeof(Real)); - dBastaBlockResG = gpu->AllocateMemory(kPaddedStateCount * numSubinterval * sizeof(Real)); - dBastaBlockResH = gpu->AllocateMemory(kPaddedStateCount * numSubinterval * sizeof(Real)); + dBastaMemory = gpu->AllocateMemory(4 * kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaBlockResMemory = gpu->AllocateMemory(4 * kPaddedStateCount * numSubinterval * sizeof(Real)); // for (int i = 0; i < operationCount; ++i) { // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; @@ -2520,18 +2506,19 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat //Real r = 0; - Real* hBastaE = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaF = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaG = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaH = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaEres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); - Real* hBastaFres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); - Real* hBastaGres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); - Real* hBastaHres = (Real*) gpu->CallocHost(sizeof(Real), kPaddedStateCount * numSubinterval); + Real* hBastaMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaResMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * numSubinterval); + + // Set the pointers to the correct locations within the block + Real* hBastaE = hBastaMemory; + Real* hBastaF = hBastaE + kPaddedStateCount * kCoalescentBufferLength; + Real* hBastaG = hBastaF + kPaddedStateCount * kCoalescentBufferLength; + Real* hBastaH = hBastaG + kPaddedStateCount * kCoalescentBufferLength; + // for (int i = 0; i < operationCount; i += chunkSize) { - // // TODO execute in parallel (no race conditions) - const int start = 0; - const int end = operationCount; + + const int start1 = 0; + const int end1 = operationCount; // const int child1PartialIndex = operations[op * numOps + 1]; // const int child2PartialIndex = operations[op * numOps + 3]; @@ -2544,8 +2531,8 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; // Real r = 0; - kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResE, dBastaBlockResF, dBastaBlockResG, dBastaBlockResH, dBastaInterval, dBastaFlags, numOps, - start, end); + kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResMemory, dBastaInterval, numOps, + start1, end1, numSubinterval); // fprintf(stderr, "e:\n"); @@ -2564,25 +2551,28 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); // } - gpu->MemcpyDeviceToHost(hBastaEres, dBastaBlockResE, sizeof(Real) * kPaddedStateCount * numSubinterval); - gpu->MemcpyDeviceToHost(hBastaFres, dBastaBlockResF, sizeof(Real) * kPaddedStateCount * numSubinterval); - gpu->MemcpyDeviceToHost(hBastaGres, dBastaBlockResG, sizeof(Real) * kPaddedStateCount * numSubinterval); - gpu->MemcpyDeviceToHost(hBastaHres, dBastaBlockResH, sizeof(Real) * kPaddedStateCount * numSubinterval); - - for (int i = 0; i < numSubinterval; ++i) { - int interval = hBastaFinalKeys[i]; - for (int j = 0; j < kStateCount; ++j) { - hBastaE[interval * kPaddedStateCount + j] += hBastaEres[i * kPaddedStateCount + j]; - hBastaF[interval * kPaddedStateCount + j] += hBastaFres[i * kPaddedStateCount + j]; - hBastaG[interval * kPaddedStateCount + j] += hBastaGres[i * kPaddedStateCount + j]; - hBastaH[interval * kPaddedStateCount + j] += hBastaHres[i * kPaddedStateCount + j]; - } + gpu->MemcpyDeviceToHost(hBastaResMemory, dBastaBlockResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubinterval); + + + // Set the pointers to the correct locations within the block + Real* hBastaEres = hBastaResMemory; + Real* hBastaFres = hBastaEres + kPaddedStateCount * numSubinterval; + Real* hBastaGres = hBastaFres + kPaddedStateCount * numSubinterval; + Real* hBastaHres = hBastaGres + kPaddedStateCount * numSubinterval; + + + gpu->MemcpyDeviceToHost(hBastaResMemory, dBastaBlockResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubinterval); + for (int i = 0; i < numSubinterval; ++i) { + int interval = hBastaFinalKeys[i]; + for (int j = 0; j < kStateCount; ++j) { + hBastaE[interval * kPaddedStateCount + j] += hBastaEres[i * kPaddedStateCount + j]; + hBastaF[interval * kPaddedStateCount + j] += hBastaFres[i * kPaddedStateCount + j]; + hBastaG[interval * kPaddedStateCount + j] += hBastaGres[i * kPaddedStateCount + j]; + hBastaH[interval * kPaddedStateCount + j] += hBastaHres[i * kPaddedStateCount + j]; } + } - gpu->MemcpyHostToDevice(dBastaE, hBastaE, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaF, hBastaF, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaG, hBastaG, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); - gpu->MemcpyHostToDevice(dBastaH, hBastaH, sizeof(Real) * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaMemory, hBastaMemory, sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); // for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { // // TODO execute in parallel (no race conditions) // const int start = intervalStarts[interval]; @@ -2623,8 +2613,8 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat gpu->MemcpyHostToDevice(dBastaDistance, hBastaDistance, sizeof(Real) * kCoalescentBufferLength); - kernels->reduceAcrossIntervals(dBastaE, dBastaF, dBastaG, dBastaH, dBastaDistance, dBastaLogL, sizes, coalescent, - intervalStartsCount); + kernels->reduceAcrossIntervals(dBastaMemory, dBastaDistance, dBastaLogL, sizes, coalescent, + intervalStartsCount, kCoalescentBufferLength); // fprintf(stderr, "The block count is: %d\n", kBastaIntervalBlockCount); // fprintf(stderr, "logL:\n"); // gpu->PrintfDeviceVector(dBastaLogL, kBastaIntervalBlockCount, r); @@ -2639,17 +2629,11 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat gpu->FreeHostMemory(hBastaInterval); gpu->FreeHostMemory(hBastaLogL); gpu->FreeHostMemory(hBastaDistance); - gpu->FreeHostMemory(hBastaFlags); gpu->FreeHostMemory(hBastaSubintervals); gpu->FreeHostMemory(hBastaFinalKeys); - gpu->FreeHostMemory(hBastaE); - gpu->FreeHostMemory(hBastaF); - gpu->FreeHostMemory(hBastaG); - gpu->FreeHostMemory(hBastaH); - gpu->FreeHostMemory(hBastaEres); - gpu->FreeHostMemory(hBastaFres); - gpu->FreeHostMemory(hBastaGres); - gpu->FreeHostMemory(hBastaHres); + gpu->FreeHostMemory(hBastaMemory); + gpu->FreeHostMemory(hBastaResMemory); + #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 007bcf55..dc1678f3 100755 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -248,7 +248,7 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgBastaPreGrid = Dim3Int(110); bgBastaSumBlock = Dim3Int(128); - bgBastaSumGrid = Dim3Int(32); + bgBastaSumGrid = Dim3Int(400); } @@ -2478,25 +2478,22 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, void KernelLauncher::reduceWithinInterval(GPUPtr operations, GPUPtr partials, - GPUPtr e, - GPUPtr f, - GPUPtr g, - GPUPtr h, + GPUPtr dBastaBlockResMemory, GPUPtr intervals, - GPUPtr flags, unsigned int numOps, unsigned int start, - unsigned int end) { + unsigned int end, + unsigned int numSubinterval) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); #endif - int parameterCountV = 8; - int totalParameterCount = 11; + int parameterCountV = 5; + int totalParameterCount = 9; gpu->LaunchKernel(fReduceWithinInterval, bgBastaReductionBlock, bgBastaReductionGrid, parameterCountV, totalParameterCount, - operations, partials, e, f, g, h, intervals, flags, numOps, start, end); + operations, partials, dBastaBlockResMemory, intervals, numOps, start, end, numSubinterval); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); @@ -2553,24 +2550,22 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, } -void KernelLauncher::reduceAcrossIntervals(GPUPtr e, - GPUPtr f, - GPUPtr g, - GPUPtr h, +void KernelLauncher::reduceAcrossIntervals(GPUPtr dBastaMemory, GPUPtr distance, GPUPtr dLogL, const GPUPtr sizes, GPUPtr coalescent, - unsigned int intervalNumber) { + unsigned int intervalNumber, + unsigned int kCoalescentBufferLength) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); #endif - int parameterCountV = 8; - int totalParameterCount = 9; + int parameterCountV = 5; + int totalParameterCount = 7; gpu->LaunchKernel(fReduceAcrossInterval, bgBastaSumBlock, bgBastaSumGrid, parameterCountV, totalParameterCount, - e, f, g, h, distance, dLogL, sizes, coalescent, intervalNumber); + dBastaMemory, distance, dLogL, sizes, coalescent, intervalNumber, kCoalescentBufferLength); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); #endif diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index 23831575..a90d4365 100755 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -627,8 +627,8 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, // void reduceWithinInterval(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr startPartials1, GPUPtr startPartials2, // GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex, unsigned int renew); - void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr intervals, GPUPtr flags, unsigned int numOps, unsigned int start, unsigned int end); - void reduceAcrossIntervals(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber); + void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr dBastaBlockResMemory, GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end, unsigned int numSubinterval); + void reduceAcrossIntervals(GPUPtr dBastaMemory, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber, unsigned int kCoalescentBufferLength); void preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, GPUPtr dBlockSegmentKeysEnd, unsigned int operationCount, unsigned int numBlocks); void accumulateCarryOut(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr keys, unsigned int numBlocks); void SetupKernelBlocksAndGrids(); diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 359c170a..386f1f80 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2261,15 +2261,12 @@ KW_GLOBAL_KERNEL void kernelAccumulateCarryOut(KW_GLOBAL_VAR REAL* e, KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, KW_GLOBAL_VAR REAL* KW_RESTRICT partials, - KW_GLOBAL_VAR REAL* e, - KW_GLOBAL_VAR REAL* f, - KW_GLOBAL_VAR REAL* g, - KW_GLOBAL_VAR REAL* h, + KW_GLOBAL_VAR REAL* dBastaBlockResMemory, KW_GLOBAL_VAR REAL* intervals, - KW_GLOBAL_VAR REAL* flags, int numOps, int start, - int end) { + int end, + int numSubinterval) { #define SUM_PARTIAL_BLOCK_SIZE_B 4 @@ -2281,6 +2278,13 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES // int opCountLast = opCount % SUM_PARTIAL_BLOCK_SIZE_B; // int blockSize = (SUM_PARTIAL_BLOCK_SIZE_B > opCountLast) ? opCountLast : SUM_PARTIAL_BLOCK_SIZE_B; int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; + + KW_GLOBAL_VAR REAL* e = dBastaBlockResMemory; + KW_GLOBAL_VAR REAL* f = e + PADDED_STATE_COUNT * numSubinterval; + KW_GLOBAL_VAR REAL* g = f + PADDED_STATE_COUNT * numSubinterval; + KW_GLOBAL_VAR REAL* h = g + PADDED_STATE_COUNT * numSubinterval; + KW_GLOBAL_VAR REAL* flags = intervals + end; + KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; @@ -2383,15 +2387,13 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES } } -KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT e, - KW_GLOBAL_VAR REAL* KW_RESTRICT f, - KW_GLOBAL_VAR REAL* KW_RESTRICT g, - KW_GLOBAL_VAR REAL* KW_RESTRICT h, +KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT dBastaMemory, KW_GLOBAL_VAR REAL* KW_RESTRICT distance, KW_GLOBAL_VAR REAL* KW_RESTRICT dLogL, KW_GLOBAL_VAR REAL* KW_RESTRICT sizes, KW_GLOBAL_VAR REAL* KW_RESTRICT coalescent, - int intervalStartsCount) { + int intervalStartsCount, + int kCoalescentBufferLength) { #define SUM_INTERVAL_BLOCK_SIZE 4 @@ -2403,6 +2405,11 @@ KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RES int intervalNumber = __umul24(KW_GROUP_ID_0, SUM_INTERVAL_BLOCK_SIZE) + intervalIdx; int u = state + intervalNumber * PADDED_STATE_COUNT; + KW_GLOBAL_VAR REAL* e = dBastaMemory; + KW_GLOBAL_VAR REAL* f = e + PADDED_STATE_COUNT * kCoalescentBufferLength; + KW_GLOBAL_VAR REAL* g = f + PADDED_STATE_COUNT * kCoalescentBufferLength; + KW_GLOBAL_VAR REAL* h = g + PADDED_STATE_COUNT * kCoalescentBufferLength; + KW_LOCAL_MEM REAL sPartials1[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; KW_LOCAL_MEM REAL sPartials2[SUM_INTERVAL_BLOCK_SIZE]; From ae2a8cb2d69f1f04964b1da601a085f0f91b4ffa Mon Sep 17 00:00:00 2001 From: yucais Date: Fri, 16 Aug 2024 01:52:33 -0700 Subject: [PATCH 247/421] typo --- libhmsbeagle/GPU/BeagleGPUImpl.h | 1 - libhmsbeagle/GPU/BeagleGPUImpl.hpp | 1 - libhmsbeagle/GPU/KernelLauncher.cpp | 4 ++-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index e45f8973..679db03a 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -222,7 +222,6 @@ class BeagleGPUImpl : public BeagleImpl { GPUPtr dBastaOperationQueue; GPUPtr dCoalescentBuffers; GPUPtr dBastaInterval; - GPUPtr dBastaFlags; int kCoalescentBufferLength; int kCoalescentBufferCount; int kBastaIntervalBlockCount; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 233a778a..e2cacaad 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2242,7 +2242,6 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, dBastaLogL = gpu->AllocateMemory(kBastaIntervalBlockCount * sizeof(Real)); dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); dBastaInterval = gpu->AllocateMemory(kBufferCount * sizeof(Real)); - dBastaFlags = gpu->AllocateMemory(kBufferCount * sizeof(Real)); // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index dc1678f3..09e1e36c 100755 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -2488,8 +2488,8 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); #endif - int parameterCountV = 5; - int totalParameterCount = 9; + int parameterCountV = 4; + int totalParameterCount = 8; gpu->LaunchKernel(fReduceWithinInterval, bgBastaReductionBlock, bgBastaReductionGrid, parameterCountV, totalParameterCount, From 231625d4720204e3c4cba9bdc239a2ab577cae15 Mon Sep 17 00:00:00 2001 From: yucais Date: Fri, 16 Aug 2024 06:16:07 -0700 Subject: [PATCH 248/421] reorder for benchmark --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 33 ++++++++++++------------------ 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index e2cacaad..43329f1d 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2279,8 +2279,8 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - // GPUPtr coalescent = (GPUPtr) NULL; + //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = (GPUPtr) NULL; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2436,10 +2436,9 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); #endif - //DEBUG_CREATE_TIME(); - //DEBUG_START_TIME(); - GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - // GPUPtr coalescent = (GPUPtr) NULL; + + //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; + GPUPtr coalescent = (GPUPtr) NULL; const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; const int blockSize = 4; @@ -2468,13 +2467,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat hBastaSubintervals[i] = currentSubinterval; hBastaFinalKeys[currentSubinterval] = hBastaInterval[i]; } - //DEBUG_END_TIME(); - // std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl; - //for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) { - // std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first) - // .count() << " ns" << "\t" << it->second.second << std::endl; - // }; numSubinterval = currentSubinterval + 1; @@ -2482,7 +2475,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat dBastaMemory = gpu->AllocateMemory(4 * kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaBlockResMemory = gpu->AllocateMemory(4 * kPaddedStateCount * numSubinterval * sizeof(Real)); - + // for (int i = 0; i < operationCount; ++i) { // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; // hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; @@ -2505,14 +2498,9 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat //Real r = 0; - Real* hBastaMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaResMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * numSubinterval); // Set the pointers to the correct locations within the block - Real* hBastaE = hBastaMemory; - Real* hBastaF = hBastaE + kPaddedStateCount * kCoalescentBufferLength; - Real* hBastaG = hBastaF + kPaddedStateCount * kCoalescentBufferLength; - Real* hBastaH = hBastaG + kPaddedStateCount * kCoalescentBufferLength; + // for (int i = 0; i < operationCount; i += chunkSize) { @@ -2550,9 +2538,14 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); // } + Real* hBastaMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * kCoalescentBufferLength); + Real* hBastaResMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * numSubinterval); gpu->MemcpyDeviceToHost(hBastaResMemory, dBastaBlockResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubinterval); - + Real* hBastaE = hBastaMemory; + Real* hBastaF = hBastaE + kPaddedStateCount * kCoalescentBufferLength; + Real* hBastaG = hBastaF + kPaddedStateCount * kCoalescentBufferLength; + Real* hBastaH = hBastaG + kPaddedStateCount * kCoalescentBufferLength; // Set the pointers to the correct locations within the block Real* hBastaEres = hBastaResMemory; Real* hBastaFres = hBastaEres + kPaddedStateCount * numSubinterval; From 261c73a43527650be1d450a9ea344c13a97a9278 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 19 Aug 2024 16:17:29 -0400 Subject: [PATCH 249/421] Turn scaling on for actiontest by default. Allow option --noscaling to disable scaling. --- examples/actiontest/actiontest.cpp | 38 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/examples/actiontest/actiontest.cpp b/examples/actiontest/actiontest.cpp index 31751c70..3eec8a9f 100644 --- a/examples/actiontest/actiontest.cpp +++ b/examples/actiontest/actiontest.cpp @@ -162,6 +162,11 @@ std::string showFlags(BeagleFlags flags) return o.str(); } +bool has_only_digits(const std::string& s) +{ + return s.find_first_not_of( "0123456789" ) == std::string::npos; +} + int main( int argc, const char* argv[] ) { // print resource list @@ -175,8 +180,6 @@ int main( int argc, const char* argv[] ) } std::cout<<"\n"; - bool scaling = false; - bool doJC = true; bool singlePrecision = false; @@ -194,9 +197,23 @@ int main( int argc, const char* argv[] ) // int rateCategoryCount = 4; int rateCategoryCount = 2; + bool scaling = true; + for(int i=1;i 1 && strcmp(argv[1] , "--gpu") == 0; + int whichDevice = -1; + bool useGpu = false; + for(int i=1;i\n"; std::cerr<<"Flag: --threading\n"; + std::cerr<<"Flag: --noscaling\n"; std::exit(1); } bool useTipStates = false; - int whichDevice = -1; - if (useGpu) - { - useSSE = false; - - if (argc > 2) { - whichDevice = atol(argv[2]); - if (whichDevice < 0) { - whichDevice = -1; - } - } - } + if (useGpu) useSSE = false; BeagleInstanceDetails instDetails; From 2dbe8c7821997c0aa485d73e35a206533595c38d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 19 Aug 2024 16:32:35 -0400 Subject: [PATCH 250/421] Make sure that nSites != nStates --- examples/actiontest/actiontest.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/actiontest/actiontest.cpp b/examples/actiontest/actiontest.cpp index 3eec8a9f..87285991 100644 --- a/examples/actiontest/actiontest.cpp +++ b/examples/actiontest/actiontest.cpp @@ -21,9 +21,9 @@ typedef Eigen::Triplet Triplet; #include "libhmsbeagle/beagle.h" -char *human = (char*)"GAGT"; -char *chimp = (char*)"GAGG"; -char *gorilla = (char*)"AAAT"; +char *human = (char*)"GAGTC"; +char *chimp = (char*)"GAGGT"; +char *gorilla = (char*)"AAATC"; //char *human = (char*)"G"; //char *chimp = (char*)"G"; @@ -653,7 +653,7 @@ int main( int argc, const char* argv[] ) // 1); // count - std::cout<<"logL = "< Date: Mon, 19 Aug 2024 17:21:48 -0400 Subject: [PATCH 251/421] Use the permuted order. --- libhmsbeagle/GPU/cuda_ops.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index e42c96e2..b2c1da93 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -169,7 +169,7 @@ void justMaximize(float* partials, float* scalingFactors, float* cumulativeScali // add execution policy thrust::cuda::par_nosync? in_keys_start, // key indices start (group by pattern) in_keys_start + partials_size, // key indices end - thrust::device_pointer_cast(partials), // values to reduce (category, pattern, state) + in_values_start, // values to reduce (category, pattern, state) thrust::make_discard_iterator(), // key values out thrust::device_pointer_cast(cumulativeScalingBuffer), // reduced values out thrust::equal_to(), // compare keys operation @@ -227,7 +227,7 @@ void justMaximize(double* partials, double* scalingFactors, double* cumulativeSc // add execution policy thrust::cuda::par_nosync? in_keys_start, // key indices start (group by pattern) in_keys_start + partials_size, // key indices end - thrust::device_pointer_cast(partials), // values to reduce (category, pattern, state) + in_values_start, // values to reduce (category, pattern, state) thrust::make_discard_iterator(), // key values out thrust::device_pointer_cast(cumulativeScalingBuffer), // reduced values out thrust::equal_to(), // compare keys operation From 556cdf77b49367ca491ffeb6d4b1907b62310bf0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 19 Aug 2024 17:22:54 -0400 Subject: [PATCH 252/421] justMaximize should write the scalingFactors, not cumulativeScalingBuffer --- libhmsbeagle/GPU/cuda_ops.cu | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index b2c1da93..db023579 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -119,7 +119,7 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. -void justMaximize(float* partials, float* scalingFactors, float* cumulativeScalingBuffer, +void justMaximize(float* partials, float* scalingFactors, int nStates, int nPatterns, int nCategories) { using namespace thrust::placeholders; @@ -171,14 +171,14 @@ void justMaximize(float* partials, float* scalingFactors, float* cumulativeScali in_keys_start + partials_size, // key indices end in_values_start, // values to reduce (category, pattern, state) thrust::make_discard_iterator(), // key values out - thrust::device_pointer_cast(cumulativeScalingBuffer), // reduced values out + thrust::device_pointer_cast(scalingFactors), // reduced values out thrust::equal_to(), // compare keys operation thrust::maximum() // reduction operation ); } -void justMaximize(double* partials, double* scalingFactors, double* cumulativeScalingBuffer, - int nStates, int nPatterns, int nCategories, bool scalers_log) +void justMaximize(double* partials, double* scalingFactors, + int nStates, int nPatterns, int nCategories) { using namespace thrust::placeholders; @@ -229,7 +229,7 @@ void justMaximize(double* partials, double* scalingFactors, double* cumulativeSc in_keys_start + partials_size, // key indices end in_values_start, // values to reduce (category, pattern, state) thrust::make_discard_iterator(), // key values out - thrust::device_pointer_cast(cumulativeScalingBuffer), // reduced values out + thrust::device_pointer_cast(scalingFactors), // reduced values out thrust::equal_to(), // compare keys operation thrust::maximum() // reduction operation ); From 4f0c5e2779a842ddde788d5a74992c160b020ad0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 19 Aug 2024 17:23:04 -0400 Subject: [PATCH 253/421] Fix indent. --- libhmsbeagle/GPU/cuda_ops.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index db023579..55afa56b 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -39,7 +39,7 @@ struct rescalePartialsDeviceOp bool scalers_log; __host__ __device__ void operator()(int pattern) const { - // FIND_MAX_PARTIALS_X_CPU(); + // FIND_MAX_PARTIALS_X_CPU(); int deltaPartialsByState = pattern * kPaddedStateCount; REAL max = 0; for(int m = 0; m < kCategoryCount; m++) From 57d7da178ea39932bb4eca547877210ea78498d2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 19 Aug 2024 17:23:20 -0400 Subject: [PATCH 254/421] Move rescalePartials2 after justMaximize --- libhmsbeagle/GPU/cuda_ops.cu | 50 ++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 55afa56b..94c32cfe 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -94,28 +94,6 @@ struct rescalePartialsDeviceOp }; -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) -{ - auto start = thrust::make_counting_iterator(0); - auto end = start + kPaddedPatternCount; - - thrust::for_each(start, end, - rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, - kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); -} - -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) -{ - auto start = thrust::make_counting_iterator(0); - auto end = start + kPaddedPatternCount; - - thrust::for_each(start, end, - rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, - kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); -} - // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. @@ -235,3 +213,31 @@ void justMaximize(double* partials, double* scalingFactors, ); } +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) +{ + justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); + + auto start = thrust::make_counting_iterator(0); + auto end = start + kPaddedPatternCount; + + thrust::for_each(start, end, + rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, + kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); +} + +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) +{ + justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); + + auto start = thrust::make_counting_iterator(0); + auto end = start + kPaddedPatternCount; + + thrust::for_each(start, end, + rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, + kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); + + thrust::device_vector tmp2(scalingFactors,scalingFactors+std::min(8,kPaddedPatternCount)); +} + From 48b20b878083078d1a01457e77e3db70dde58dc7 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 19 Aug 2024 18:19:09 -0400 Subject: [PATCH 255/421] Calculate the maximum using thrust::reduced_by_key This will probably use a lot more threads. --- libhmsbeagle/GPU/cuda_ops.cu | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 94c32cfe..f79576c2 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -41,33 +41,28 @@ struct rescalePartialsDeviceOp { // FIND_MAX_PARTIALS_X_CPU(); int deltaPartialsByState = pattern * kPaddedStateCount; - REAL max = 0; - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - REAL iPartial = partials[deltaPartials + i]; - if (iPartial > max) - max = iPartial; - } - } + auto& max = scalingFactors[pattern]; if (max == 0) max = 1.0; - if (scalers_log) + if (not cumulativeScalingBuffer) { - REAL logMax = log(max); - scalingFactors[pattern] = logMax; - if (cumulativeScalingBuffer != 0) - cumulativeScalingBuffer[pattern] += logMax; + if (scalers_log) + scalingFactors[pattern] = log(max); } else { - scalingFactors[pattern] = max; - if (cumulativeScalingBuffer != 0) + if (scalers_log) + { + REAL logMax = log(max); + scalingFactors[pattern] = logMax; + cumulativeScalingBuffer[pattern] += logMax; + } + else + { cumulativeScalingBuffer[pattern] += log(max); + } } // SCALE_PARTIALS_X_CPU(); From f724932d456583faa69cf22d937e2bf4cb29dac2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 20 Aug 2024 14:40:45 -0400 Subject: [PATCH 256/421] Simplify log-transform of scaling factors. --- libhmsbeagle/GPU/cuda_ops.cu | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index f79576c2..b25617a9 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -46,25 +46,6 @@ struct rescalePartialsDeviceOp if (max == 0) max = 1.0; - if (not cumulativeScalingBuffer) - { - if (scalers_log) - scalingFactors[pattern] = log(max); - } - else - { - if (scalers_log) - { - REAL logMax = log(max); - scalingFactors[pattern] = logMax; - cumulativeScalingBuffer[pattern] += logMax; - } - else - { - cumulativeScalingBuffer[pattern] += log(max); - } - } - // SCALE_PARTIALS_X_CPU(); for(int m = 0; m < kCategoryCount; m++) { @@ -74,6 +55,11 @@ struct rescalePartialsDeviceOp partials[deltaPartials + i] /= max; } } + + if (scalers_log) max = log(max); + + if (cumulativeScalingBuffer) + cumulativeScalingBuffer[pattern] += (scalers_log)?max:log(max); } rescalePartialsDeviceOp(REAL* r1, REAL* r2, REAL* r3, int i1, int i2, int i3, bool b) From 38831a111f9c75727b6bdfd473c244c267238192 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 20 Aug 2024 14:54:00 -0400 Subject: [PATCH 257/421] Move the s/0/1/ transform out of the thrust::for_each --- libhmsbeagle/GPU/cuda_ops.cu | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index b25617a9..86e0795a 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -8,6 +8,7 @@ #include #include #include +#include void cuda_log_vector(double* v, int length) { @@ -43,9 +44,6 @@ struct rescalePartialsDeviceOp int deltaPartialsByState = pattern * kPaddedStateCount; auto& max = scalingFactors[pattern]; - if (max == 0) - max = 1.0; - // SCALE_PARTIALS_X_CPU(); for(int m = 0; m < kCategoryCount; m++) { @@ -197,28 +195,40 @@ void justMaximize(double* partials, double* scalingFactors, void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) { + // 1. Find maximum partial likelihood -> scalingFactors[pattern] justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); + // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. + thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); + + thrust::transform(partials2, partials2 + kPaddedPatternCount, partials2, [] __device__ (float x) { return (x == 0) ? 1.0 : x;}); + + // 3. Rescale each pattern by scalingFactors[pattern] auto start = thrust::make_counting_iterator(0); auto end = start + kPaddedPatternCount; - thrust::for_each(start, end, - rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, - kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); + thrust::async::for_each(start, end, + rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, + kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); } void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) + double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) { + // 1. Find maximum partial likelihood -> scalingFactors[pattern] justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); + // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. + thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); + + thrust::transform(partials2, partials2 + kPaddedPatternCount, partials2, [] __device__ (double x) { return (x == 0) ? 1.0 : x;}); + + // 3. Rescale each pattern by scalingFactors[pattern] auto start = thrust::make_counting_iterator(0); auto end = start + kPaddedPatternCount; - thrust::for_each(start, end, - rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, - kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); - - thrust::device_vector tmp2(scalingFactors,scalingFactors+std::min(8,kPaddedPatternCount)); + thrust::async::for_each(start, end, + rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, + kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); } From 436ce91013cd2564f5c6e822c9e70372d10e9be5 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 20 Aug 2024 15:23:32 -0400 Subject: [PATCH 258/421] Factor out transformations of scalingFactors and cumulativeScalingBuffer. --- libhmsbeagle/GPU/cuda_ops.cu | 113 ++++++++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 23 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 86e0795a..f07908a2 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -31,13 +31,11 @@ struct rescalePartialsDeviceOp { REAL* partials; REAL* scalingFactors; - REAL* cumulativeScalingBuffer; int kPaddedStateCount; int kPaddedPatternCount; int kCategoryCount; - bool scalers_log; __host__ __device__ void operator()(int pattern) const { // FIND_MAX_PARTIALS_X_CPU(); @@ -53,21 +51,14 @@ struct rescalePartialsDeviceOp partials[deltaPartials + i] /= max; } } - - if (scalers_log) max = log(max); - - if (cumulativeScalingBuffer) - cumulativeScalingBuffer[pattern] += (scalers_log)?max:log(max); } - rescalePartialsDeviceOp(REAL* r1, REAL* r2, REAL* r3, int i1, int i2, int i3, bool b) + rescalePartialsDeviceOp(REAL* r1, REAL* r2, int i1, int i2, int i3) :partials(r1), scalingFactors(r2), - cumulativeScalingBuffer(r3), kPaddedStateCount(i1), kPaddedPatternCount(i2), - kCategoryCount(i3), - scalers_log(b) + kCategoryCount(i3) { } @@ -195,40 +186,116 @@ void justMaximize(double* partials, double* scalingFactors, void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) { + using namespace thrust::placeholders; + +// thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); + thrust::device_ptr scalingFactors2 = thrust::device_pointer_cast(scalingFactors); + // 1. Find maximum partial likelihood -> scalingFactors[pattern] justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. - thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); - - thrust::transform(partials2, partials2 + kPaddedPatternCount, partials2, [] __device__ (float x) { return (x == 0) ? 1.0 : x;}); + thrust::transform(scalingFactors2, scalingFactors2 + kPaddedPatternCount, scalingFactors2, [] __device__ (float x) { return (x == 0) ? 1.0 : x;}); // 3. Rescale each pattern by scalingFactors[pattern] auto start = thrust::make_counting_iterator(0); auto end = start + kPaddedPatternCount; - thrust::async::for_each(start, end, - rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, - kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); + thrust::for_each(start, end, + rescalePartialsDeviceOp(partials, scalingFactors, + kPaddedStateCount, kPaddedPatternCount, kCategoryCount)); + +// std::cerr<<"scalers_log = "< cumulative2 = thrust::device_pointer_cast(cumulativeScalingBuffer); + + if (scalers_log) + { + thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 + scalingFactors2, // in2 + cumulative2, // out + thrust::plus() // operation; + ); + } + else + { + auto logScalingFactors2 = thrust::make_transform_iterator( scalingFactors2, + [] __host__ __device__ (float x) { return log(x); }); + + thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 + logScalingFactors2, // in2 + cumulative2, // out + thrust::plus() // operation; + ); + } + } } void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) { + using namespace thrust::placeholders; + +// thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); + thrust::device_ptr scalingFactors2 = thrust::device_pointer_cast(scalingFactors); + // 1. Find maximum partial likelihood -> scalingFactors[pattern] justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. - thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); - - thrust::transform(partials2, partials2 + kPaddedPatternCount, partials2, [] __device__ (double x) { return (x == 0) ? 1.0 : x;}); + thrust::transform(scalingFactors2, scalingFactors2 + kPaddedPatternCount, scalingFactors2, [] __device__ (double x) { return (x == 0) ? 1.0 : x;}); // 3. Rescale each pattern by scalingFactors[pattern] auto start = thrust::make_counting_iterator(0); auto end = start + kPaddedPatternCount; - thrust::async::for_each(start, end, - rescalePartialsDeviceOp(partials, scalingFactors, cumulativeScalingBuffer, - kPaddedStateCount, kPaddedPatternCount, kCategoryCount, scalers_log)); + thrust::for_each(start, end, + rescalePartialsDeviceOp(partials, scalingFactors, + kPaddedStateCount, kPaddedPatternCount, kCategoryCount)); + +// std::cerr<<"scalers_log = "< cumulative2 = thrust::device_pointer_cast(cumulativeScalingBuffer); + + if (scalers_log) + { + thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 + scalingFactors2, // in2 + cumulative2, // out + thrust::plus() // operation; + ); + } + else + { + auto logScalingFactors2 = thrust::make_transform_iterator( scalingFactors2, + [] __host__ __device__ (double x) { return log(x); }); + + thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 + logScalingFactors2, // in2 + cumulative2, // out + thrust::plus() // operation; + ); + } + } } From c2f841a841ae96b1203771b113de816b8e63b25f Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 20 Aug 2024 15:49:50 -0400 Subject: [PATCH 259/421] Speed up rescaling. --- libhmsbeagle/GPU/cuda_ops.cu | 102 ++++++++++++++++------------------- 1 file changed, 45 insertions(+), 57 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index f07908a2..02f6fbba 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -26,44 +26,6 @@ void cuda_log_vector(float* v, int length) thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); } -template -struct rescalePartialsDeviceOp -{ - REAL* partials; - REAL* scalingFactors; - - int kPaddedStateCount; - int kPaddedPatternCount; - int kCategoryCount; - - __host__ __device__ void operator()(int pattern) const - { - // FIND_MAX_PARTIALS_X_CPU(); - int deltaPartialsByState = pattern * kPaddedStateCount; - auto& max = scalingFactors[pattern]; - - // SCALE_PARTIALS_X_CPU(); - for(int m = 0; m < kCategoryCount; m++) - { - int deltaPartialsByCategory = m * kPaddedStateCount * kPaddedPatternCount; - int deltaPartials = deltaPartialsByCategory + deltaPartialsByState; - for(int i = 0; i < kPaddedStateCount; i++) { - partials[deltaPartials + i] /= max; - } - } - } - - rescalePartialsDeviceOp(REAL* r1, REAL* r2, int i1, int i2, int i3) - :partials(r1), - scalingFactors(r2), - kPaddedStateCount(i1), - kPaddedPatternCount(i2), - kCategoryCount(i3) - { - } - -}; - // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. @@ -188,22 +150,35 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC { using namespace thrust::placeholders; -// thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); + size_t partialsSize = kCategoryCount * kPaddedPatternCount * kPaddedStateCount; + thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); thrust::device_ptr scalingFactors2 = thrust::device_pointer_cast(scalingFactors); - // 1. Find maximum partial likelihood -> scalingFactors[pattern] + // 1. Find maximize partial likelihood -> scalingFactors[pattern] justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. thrust::transform(scalingFactors2, scalingFactors2 + kPaddedPatternCount, scalingFactors2, [] __device__ (float x) { return (x == 0) ? 1.0 : x;}); - // 3. Rescale each pattern by scalingFactors[pattern] - auto start = thrust::make_counting_iterator(0); - auto end = start + kPaddedPatternCount; + // 3. Rescale each pattern by scalingFactors[pattern(index)] + + // pattern = (i/kPaddedStateCount) % kPaddedPatternCount + + // iter_max computes the scaling factor as a function of the index into the partials buffer. + auto iter_max = thrust::make_permutation_iterator( + scalingFactors2, + thrust::make_transform_iterator( + thrust::make_counting_iterator(0), + (_1/kPaddedStateCount) % kPaddedPatternCount + ) + ); - thrust::for_each(start, end, - rescalePartialsDeviceOp(partials, scalingFactors, - kPaddedStateCount, kPaddedPatternCount, kCategoryCount)); + + thrust::transform(partials2, partials2 + partialsSize, // in1 = partials[i] + iter_max, // in2 = scalingFactors[pattern(i)] + partials2, // out + thrust::divides() // operation + ); // std::cerr<<"scalers_log = "<() // operation; + thrust::plus() // operation; ); } else @@ -235,7 +210,7 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 logScalingFactors2, // in2 cumulative2, // out - thrust::plus() // operation; + thrust::plus() // operation; ); } } @@ -246,22 +221,35 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC { using namespace thrust::placeholders; -// thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); + size_t partialsSize = kCategoryCount * kPaddedPatternCount * kPaddedStateCount; + thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); thrust::device_ptr scalingFactors2 = thrust::device_pointer_cast(scalingFactors); - // 1. Find maximum partial likelihood -> scalingFactors[pattern] + // 1. Find maximize partial likelihood -> scalingFactors[pattern] justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. thrust::transform(scalingFactors2, scalingFactors2 + kPaddedPatternCount, scalingFactors2, [] __device__ (double x) { return (x == 0) ? 1.0 : x;}); - // 3. Rescale each pattern by scalingFactors[pattern] - auto start = thrust::make_counting_iterator(0); - auto end = start + kPaddedPatternCount; + // 3. Rescale each pattern by scalingFactors[pattern(index)] + + // pattern = (i/kPaddedStateCount) % kPaddedPatternCount + + // iter_max computes the scaling factor as a function of the index into the partials buffer. + auto iter_max = thrust::make_permutation_iterator( + scalingFactors2, + thrust::make_transform_iterator( + thrust::make_counting_iterator(0), + (_1/kPaddedStateCount) % kPaddedPatternCount + ) + ); + - thrust::for_each(start, end, - rescalePartialsDeviceOp(partials, scalingFactors, - kPaddedStateCount, kPaddedPatternCount, kCategoryCount)); + thrust::transform(partials2, partials2 + partialsSize, // in1 = partials[i] + iter_max, // in2 = scalingFactors[pattern(i)] + partials2, // out + thrust::divides() // operation + ); // std::cerr<<"scalers_log = "< Date: Tue, 20 Aug 2024 16:08:10 -0400 Subject: [PATCH 260/421] Remove (probably) unnecessary cudaDeviceSynchronize() calls. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 1 - libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 10e2301a..9fd1a10f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -105,7 +105,6 @@ Real normPInf(Real* matrix, int nRows, int nCols, cublasHandle_t cublasH) { } else { CUBLAS_CHECK(cublasIdamax(cublasH, nRows * nCols, matrix, 1, &index)); } - cudaDeviceSynchronize(); CHECK_CUDA(cudaMemcpy(&result, matrix + index - 1, sizeof(Real), cudaMemcpyDeviceToHost)) return std::abs(result); } diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index d548902e..505bb092 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -989,7 +989,6 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde } // simpleAction3(partials1Index, edgeIndex1, partials2Index, edgeIndex2); - cudaDeviceSynchronize(); for (int category = 0; category < kCategoryCount; category++) { @@ -1018,8 +1017,6 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde // PrintfDeviceVector(dPartialCache[destPartialIndex], kPaddedStateCount * kPaddedPatternCount, -1, 0, 0); //#endif } - cudaDeviceSynchronize(); - } BEAGLE_GPU_TEMPLATE @@ -1346,7 +1343,7 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int std::cerr<<"dAs["<::simpleAction2(DnMatrixDevice& //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"F = partials operation, FCache:\n"< Date: Tue, 20 Aug 2024 16:32:19 -0400 Subject: [PATCH 261/421] Slight fix to simpleAction2 --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 505bb092..addb4f87 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1446,7 +1446,7 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& // std::cerr<<"dPartials:\n"< Date: Tue, 20 Aug 2024 17:31:02 -0400 Subject: [PATCH 262/421] Add cuda_max( ) funtion that uses thrust. --- libhmsbeagle/GPU/cuda_ops.cu | 12 ++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 3 +++ 2 files changed, 15 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 02f6fbba..6670a827 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -26,6 +26,18 @@ void cuda_log_vector(float* v, int length) thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); } +double cuda_max(double* v, int length) +{ + thrust::device_ptr dptr = thrust::device_pointer_cast(v); + return *(thrust::max_element(dptr, dptr + length)); +} + +float cuda_max(float* v, int length) +{ + thrust::device_ptr dptr = thrust::device_pointer_cast(v); + return *(thrust::max_element(dptr, dptr + length)); +} + // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 14af290c..04481610 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -52,6 +52,9 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); +double cuda_max(double* v, int length); +float cuda_max(float* v, int length); + void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, From 4dc265f5e97f3784f47ec31c90d4ca788a886384 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 20 Aug 2024 17:31:09 -0400 Subject: [PATCH 263/421] Add notes. --- libhmsbeagle/GPU/cuda_ops.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 6670a827..2ceb7da9 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -26,12 +26,14 @@ void cuda_log_vector(float* v, int length) thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); } +// This seems to be slow because it inserts cudaStreamSynchronize() double cuda_max(double* v, int length) { thrust::device_ptr dptr = thrust::device_pointer_cast(v); return *(thrust::max_element(dptr, dptr + length)); } +// This seems to be slow because it inserts cudaStreamSynchronize() float cuda_max(float* v, int length) { thrust::device_ptr dptr = thrust::device_pointer_cast(v); From 117299089724e58a88c8fa1975601a8c3ae9e4b6 Mon Sep 17 00:00:00 2001 From: yucais Date: Thu, 22 Aug 2024 08:31:51 -0700 Subject: [PATCH 264/421] code for accumulation using gpu/change memory allocation/add benchmark --- libhmsbeagle/GPU/BeagleGPUImpl.h | 1 + libhmsbeagle/GPU/BeagleGPUImpl.hpp | 302 ++++++++++++++++--------- libhmsbeagle/GPU/KernelLauncher.cpp | 23 +- libhmsbeagle/GPU/KernelLauncher.h | 2 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 140 ++++++++---- 5 files changed, 300 insertions(+), 168 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 679db03a..e45f8973 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -222,6 +222,7 @@ class BeagleGPUImpl : public BeagleImpl { GPUPtr dBastaOperationQueue; GPUPtr dCoalescentBuffers; GPUPtr dBastaInterval; + GPUPtr dBastaFlags; int kCoalescentBufferLength; int kCoalescentBufferCount; int kBastaIntervalBlockCount; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 43329f1d..166d62e3 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -80,12 +80,14 @@ namespace gpu { dBastaMemory = (GPUPtr)NULL; dBastaBlockResMemory = (GPUPtr)NULL; + dBastaFinalResMemory = (GPUPtr)NULL; dBastaLogL = (GPUPtr)NULL; dBastaDistance = (GPUPtr)NULL; dBastaOperationQueue = (GPUPtr)NULL; dBastaBuffers = (GPUPtr)NULL; dCoalescentBuffers = (GPUPtr)NULL; dBastaInterval = (GPUPtr)NULL; + dBastaFlags = (GPUPtr)NULL; dBlockSegmentKeysEnd= (GPUPtr)NULL; dEigenValues = NULL; dEvec = NULL; @@ -257,12 +259,14 @@ BeagleGPUImpl::~BeagleGPUImpl() { gpu->FreeMemory(dBastaBuffers); gpu->FreeMemory(dBastaBlockResMemory); + gpu->FreeMemory(dBastaFinalResMemory); gpu->FreeMemory(dBlockSegmentKeysEnd); gpu->FreeMemory(dBastaMemory); gpu->FreeMemory(dBastaLogL); gpu->FreeMemory(dBastaDistance); gpu->FreeMemory(dBastaOperationQueue); gpu->FreeMemory(dBastaInterval); + gpu->FreeMemory(dBastaFlags); gpu->FreeHostMemory(hPtrQueue); @@ -2237,12 +2241,16 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, kCoalescentBufferCount = bufferCount; kBastaIntervalBlockCount = bufferLength / 4 + 1; - dCoalescentBuffers = gpu->AllocateMemory(kCoalescentBufferCount * kCoalescentBufferLength * sizeof(Real)); + dCoalescentBuffers = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); dBastaLogL = gpu->AllocateMemory(kBastaIntervalBlockCount * sizeof(Real)); dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); dBastaInterval = gpu->AllocateMemory(kBufferCount * sizeof(Real)); - + dBastaFlags = gpu->AllocateMemory(kBufferCount * sizeof(Real)); + dBastaOperationQueue = gpu->AllocateMemory(kBufferCount * 8 * sizeof(Real)); + dBastaMemory = gpu->AllocateMemory(4 * kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + dBastaBlockResMemory = gpu->AllocateMemory(4 * kPaddedStateCount * kCoalescentBufferLength * kBastaIntervalBlockCount * sizeof(Real)); + dBastaFinalResMemory = gpu->AllocateMemory(4 * kPaddedStateCount * kCoalescentBufferLength * kBastaIntervalBlockCount * sizeof(Real)); // gBastaGradBuffers.resize(4 * kStateCount * kStateCount * kPartialsPaddedStateCount * kCoalescentBufferLength); #ifdef BEAGLE_DEBUG_FLOW @@ -2279,8 +2287,7 @@ int BeagleGPUImpl::updateBastaPartials(const int* operations fprintf(stderr, "\tEntering BeagleGPUImpl::updateBastaPartials\n"); #endif - //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - GPUPtr coalescent = (GPUPtr) NULL; + GPUPtr coalescent = dCoalescentBuffers; // std::fill(coalescent, coalescent + kCoalescentBufferLength, 0); const GPUPtr sizes = dFrequencies[populationSizesIndex]; @@ -2323,7 +2330,6 @@ int BeagleGPUImpl::updateInnerBastaPartials(const int* opera hBastaOperationQueue[i * numOps + 7] = operations[i * numOps + 7]; } - dBastaOperationQueue = gpu->AllocateMemory(operationCount * numOps * sizeof(Real)); gpu->MemcpyHostToDevice(dBastaOperationQueue, hBastaOperationQueue, sizeof(Real) * operationCount * numOps); gpu->FreeHostMemory(hBastaOperationQueue); @@ -2435,14 +2441,12 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat double* out) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); -#endif - - //GPUPtr coalescent = dCoalescentBuffers + kCoalescentBufferLength * coalescentIndex; - GPUPtr coalescent = (GPUPtr) NULL; +#endif + GPUPtr coalescent = dCoalescentBuffers; const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; const int blockSize = 4; - + // auto start1 = std::chrono::high_resolution_clock::now(); Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); int numBlocks = (operationCount + blockSize - 1) / blockSize; Real* hBastaSubintervals = (Real*) gpu->CallocHost(sizeof(Real), 2 * operationCount); @@ -2453,7 +2457,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat int numSubinterval = 0; for (int i = 0; i < operationCount; ++i) { hBastaInterval[i] = (Real) operations[i * numOps + 7]; - if (i == 0 || hBastaInterval[i] != hBastaInterval[i - 1]) { + if (i % blockSize == 0 || hBastaInterval[i] != hBastaInterval[i - 1]) { hBastaFlags[i] = 1; } else { hBastaFlags[i] = 0; @@ -2468,134 +2472,210 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat hBastaFinalKeys[currentSubinterval] = hBastaInterval[i]; } - numSubinterval = currentSubinterval + 1; - gpu->MemcpyHostToDevice(dBastaInterval, hBastaSubintervals, sizeof(Real) * 2 * operationCount); - dBastaMemory = gpu->AllocateMemory(4 * kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - dBastaBlockResMemory = gpu->AllocateMemory(4 * kPaddedStateCount * numSubinterval * sizeof(Real)); - - // for (int i = 0; i < operationCount; ++i) { - // hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; - // hBastaOperationQueue[i * numOps + 1] = hPartialsOffsets[operations[i * numOps + 1]]; - // hBastaOperationQueue[i * numOps + 2] = operations[i * numOps + 2] * kPaddedStateCount * kPaddedStateCount; - // if (operations[i * numOps + 3] >= 0) { - // hBastaOperationQueue[i * numOps + 3] = hPartialsOffsets[operations[i * numOps + 3]]; - // hBastaOperationQueue[i * numOps + 4] = operations[i * numOps + 4] * kPaddedStateCount * kPaddedStateCount; - // } else { - // hBastaOperationQueue[i * numOps + 3] = operations[i * numOps + 3]; - // hBastaOperationQueue[i * numOps + 4] = operations[i * numOps + 4]; - // } - // hBastaOperationQueue[i * numOps + 5] = hPartialsOffsets[operations[i * numOps + 5]]; - // hBastaOperationQueue[i * numOps + 6] = hPartialsOffsets[operations[i * numOps + 6]]; - // hBastaOperationQueue[i * numOps + 7] = operations[i * numOps + 7]; + // auto end1 = std::chrono::high_resolution_clock::now();\ + // std::string key1 = "flag1";\ + // std::map, int>>::iterator it1 = benchmarkDuration.find(key1);\ + // if(it1 != benchmarkDuration.end()){\ + // it1->second.second += 1;\ + // it1->second.first += end1-start1;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key1, std::pair, int>(end1 - start1, 1)));\ // } // - // dBastaOperationQueue = gpu->AllocateMemory(operationCount * numOps * sizeof(Real)); - // gpu->MemcpyHostToDevice(dBastaOperationQueue, hBastaOperationQueue, sizeof(Real) * operationCount * numOps); - // gpu->FreeHostMemory(hBastaOperationQueue); - - //Real r = 0; + // auto start2 = std::chrono::high_resolution_clock::now(); + gpu->MemcpyHostToDevice(dBastaInterval, hBastaSubintervals, sizeof(Real) * 2 * operationCount); - // Set the pointers to the correct locations within the block + // auto end2 = std::chrono::high_resolution_clock::now();\ + // std::string key2 = "copy: flag1";\ + // std::map, int>>::iterator it2 = benchmarkDuration.find(key2);\ + // if(it2 != benchmarkDuration.end()){\ + // it2->second.second += 1;\ + // it2->second.first += end2-start2;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key2, std::pair, int>(end2 - start2, 1)));\ + // } - // for (int i = 0; i < operationCount; i += chunkSize) { + const int start = 0; + const int end = operationCount; - const int start1 = 0; - const int end1 = operationCount; - // const int child1PartialIndex = operations[op * numOps + 1]; - // const int child2PartialIndex = operations[op * numOps + 3]; - // const int accumulation1PartialIndex = operations[op * numOps + 5]; - // const int accumulation2PartialIndex = operations[op * numOps + 6]; - // const int intervalNumber = operations[op * numOps + 7]; - // GPUPtr startPartials1 = dPartials[child1PartialIndex]; - // GPUPtr startPartials2 = dPartials[child2PartialIndex]; - // GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; - // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; - // Real r = 0; + //auto start3 = std::chrono::high_resolution_clock::now(); kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResMemory, dBastaInterval, numOps, - start1, end1, numSubinterval); + start, end, numSubinterval); + //gpu->SynchronizeDevice(); + + // auto end3 = std::chrono::high_resolution_clock::now();\ + // std::string key3 = "kernel1";\ + // std::map, int>>::iterator it3 = benchmarkDuration.find(key3);\ + // if(it3 != benchmarkDuration.end()){\ + // it3->second.second += 1;\ + // it3->second.first += end3-start3;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key3, std::pair, int>(end3 - start3, 1)));\ + // } + // fprintf(stderr, "e:\n"); + // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "f:\n"); + // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "g:\n"); + // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); + // fprintf(stderr, "h:\n"); + // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); + // Real r = 0; + // // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); + // fprintf(stderr, "partials:\n"); + // gpu->PrintfDeviceVector(dPartialsOrigin, 24 * kPaddedStateCount, r); + // fprintf(stderr, "operations:\n"); + // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); +// } + + // auto start4 = std::chrono::high_resolution_clock::now(); + Real* hBastaFinalSubintervals = (Real*) gpu->CallocHost(sizeof(Real), 2 * numSubinterval); + Real* hBastaAccumFinalKeys = (Real*) gpu->CallocHost(sizeof(Real), numSubinterval); + Real* hBastaFinalFlags = hBastaFinalSubintervals + numSubinterval; + int currentSubintervalFinal = 0; + int previousIntervalFinal = 0; + int numSubintervalFinal = 0; + for (int i = 0; i < numSubinterval; ++i) { + if (i % blockSize == 0 || hBastaFinalKeys[i] != hBastaFinalKeys[i - 1]) { + hBastaFinalFlags[i] = 1; + } else { + hBastaFinalFlags[i] = 0; + } - // fprintf(stderr, "e:\n"); - // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "f:\n"); - // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "g:\n"); - // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "h:\n"); - // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - // Real r = 0; - // // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); - // fprintf(stderr, "partials:\n"); - // gpu->PrintfDeviceVector(dPartialsOrigin, 24 * kPaddedStateCount, r); - // fprintf(stderr, "operations:\n"); - // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); + int block = i / blockSize; + if (i > 0 && (hBastaFinalKeys[i] != previousIntervalFinal || (i % blockSize == 0 && block > 0))) { + currentSubintervalFinal++; + previousIntervalFinal = hBastaFinalKeys[i]; + } + hBastaFinalSubintervals[i] = currentSubintervalFinal; + hBastaAccumFinalKeys[currentSubintervalFinal] = hBastaFinalKeys[i]; + } + + + numSubintervalFinal = currentSubintervalFinal + 1; + + // auto end4 = std::chrono::high_resolution_clock::now();\ + // std::string key4 = "flag2";\ + // std::map, int>>::iterator it4 = benchmarkDuration.find(key4);\ + // if(it4 != benchmarkDuration.end()){\ + // it4 ->second.second += 1;\ + // it4 ->second.first += end4-start4;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key4, std::pair, int>(end4 - start4, 1)));\ + // } + + // auto start5 = std::chrono::high_resolution_clock::now(); + + gpu->MemcpyHostToDevice(dBastaFlags, hBastaFinalSubintervals, sizeof(Real) * 2 * numSubinterval); + + // auto end5 = std::chrono::high_resolution_clock::now();\ + // std::string key5 = "copy: flag2";\ + // std::map, int>>::iterator it5 = benchmarkDuration.find(key5);\ + // if(it5 != benchmarkDuration.end()){\ + // it5 ->second.second += 1;\ + // it5 ->second.first += end5-start5;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key5, std::pair, int>(end5 - start5, 1)));\ + // } + + + // auto start6 = std::chrono::high_resolution_clock::now(); + + kernels->accumulateCarryOut(dBastaBlockResMemory, dBastaFinalResMemory, dBastaFlags, numSubinterval, numSubintervalFinal); + //gpu->SynchronizeDevice(); + + // auto end6 = std::chrono::high_resolution_clock::now();\ + // std::string key6 = "kernel2";\ + // std::map, int>>::iterator it6 = benchmarkDuration.find(key6);\ + // if(it6 != benchmarkDuration.end()){\ + // it6 ->second.second += 1;\ + // it6 ->second.first += end6-start6;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key6, std::pair, int>(end6 - start6, 1)));\ + // } + + + //gpu->PrintfDeviceVector(dBastaFlags, 2 * numSubinterval, r); + //gpu->PrintfDeviceVector(dBastaFinalResMemory, kPaddedStateCount * kCoalescentBufferLength, r); + + + + Real* hBastaFinalResMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * numSubintervalFinal); + + Real* hBastaFinalEres = hBastaFinalResMemory; + Real* hBastaFinalFres = hBastaFinalEres + kPaddedStateCount * numSubintervalFinal; + Real* hBastaFinalGres = hBastaFinalFres + kPaddedStateCount * numSubintervalFinal; + Real* hBastaFinalHres = hBastaFinalGres + kPaddedStateCount * numSubintervalFinal; + + // auto start7 = std::chrono::high_resolution_clock::now(); + gpu->MemcpyDeviceToHost(hBastaFinalResMemory, dBastaFinalResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubintervalFinal); + + // auto end7 = std::chrono::high_resolution_clock::now();\ + // std::string key7 = "copy intermediate result to cpu";\ + // std::map, int>>::iterator it7 = benchmarkDuration.find(key7);\ + // if(it7 != benchmarkDuration.end()){\ + // it7 ->second.second += 1;\ + // it7 ->second.first += end7-start7;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key7, std::pair, int>(end7 - start7, 1)));\ // } + // auto start8 = std::chrono::high_resolution_clock::now(); Real* hBastaMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaResMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * numSubinterval); - gpu->MemcpyDeviceToHost(hBastaResMemory, dBastaBlockResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubinterval); + // Set the pointers to the correct locations within the block Real* hBastaE = hBastaMemory; Real* hBastaF = hBastaE + kPaddedStateCount * kCoalescentBufferLength; Real* hBastaG = hBastaF + kPaddedStateCount * kCoalescentBufferLength; Real* hBastaH = hBastaG + kPaddedStateCount * kCoalescentBufferLength; - // Set the pointers to the correct locations within the block - Real* hBastaEres = hBastaResMemory; - Real* hBastaFres = hBastaEres + kPaddedStateCount * numSubinterval; - Real* hBastaGres = hBastaFres + kPaddedStateCount * numSubinterval; - Real* hBastaHres = hBastaGres + kPaddedStateCount * numSubinterval; - - gpu->MemcpyDeviceToHost(hBastaResMemory, dBastaBlockResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubinterval); - for (int i = 0; i < numSubinterval; ++i) { - int interval = hBastaFinalKeys[i]; + for (int i = 0; i < numSubintervalFinal; ++i) { + int interval = hBastaAccumFinalKeys[i]; for (int j = 0; j < kStateCount; ++j) { - hBastaE[interval * kPaddedStateCount + j] += hBastaEres[i * kPaddedStateCount + j]; - hBastaF[interval * kPaddedStateCount + j] += hBastaFres[i * kPaddedStateCount + j]; - hBastaG[interval * kPaddedStateCount + j] += hBastaGres[i * kPaddedStateCount + j]; - hBastaH[interval * kPaddedStateCount + j] += hBastaHres[i * kPaddedStateCount + j]; + hBastaE[interval * kPaddedStateCount + j] += hBastaFinalEres[i * kPaddedStateCount + j]; + hBastaF[interval * kPaddedStateCount + j] += hBastaFinalFres[i * kPaddedStateCount + j]; + hBastaG[interval * kPaddedStateCount + j] += hBastaFinalGres[i * kPaddedStateCount + j]; + hBastaH[interval * kPaddedStateCount + j] += hBastaFinalHres[i * kPaddedStateCount + j]; } } + // auto end8 = std::chrono::high_resolution_clock::now();\ + // std::string key8 = "last cpu accumulation";\ + // std::map, int>>::iterator it8 = benchmarkDuration.find(key8);\ + // if(it8 != benchmarkDuration.end()){\ + // it8 ->second.second += 1;\ + // it8 ->second.first += end8-start8;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key8, std::pair, int>(end8 - start8, 1)));\ + // } + + // auto start9 = std::chrono::high_resolution_clock::now(); + gpu->MemcpyHostToDevice(dBastaMemory, hBastaMemory, sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); - // for (int interval = 0; interval < intervalStartsCount - 1; ++interval) { - // // TODO execute in parallel (no race conditions) - // const int start = intervalStarts[interval]; - // const int end = intervalStarts[interval + 1]; - // int renew = 0; - // for (int op = start; op < end; ++op) { // TODO execute in parallel (has race conditions on e,f,g,h - // - // const int numOps = BEAGLE_BASTA_OP_COUNT; - // - // const int child1PartialIndex = operations[op * numOps + 1]; - // const int child2PartialIndex = operations[op * numOps + 3]; - // const int accumulation1PartialIndex = operations[op * numOps + 5]; - // const int accumulation2PartialIndex = operations[op * numOps + 6]; - // const int intervalNumber = operations[op * numOps + 7]; - // GPUPtr startPartials1 = dPartials[child1PartialIndex]; - // GPUPtr startPartials2 = dPartials[child2PartialIndex]; - // GPUPtr endPartials1 = dPartials[accumulation1PartialIndex]; - // GPUPtr endPartials2 = dPartials[accumulation2PartialIndex]; - // Real r = 0; - // kernels->reduceWithinInterval(dBastaE, dBastaF, dBastaG, dBastaH, - // startPartials1, startPartials2, - // endPartials1, endPartials2, - // intervalNumber, child2PartialIndex, renew); - // renew = 1; - // fprintf(stderr, "startPartials1:\n"); - // gpu->PrintfDeviceVector(startPartials1, kPaddedStateCount, r); - // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); - // fprintf(stderr, "h:\n"); - // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - // } + // auto end9 = std::chrono::high_resolution_clock::now();\ + // std::string key9 = "copy: final result";\ + // std::map, int>>::iterator it9 = benchmarkDuration.find(key9);\ + // if(it9 != benchmarkDuration.end()){\ + // it9 ->second.second += 1;\ + // it9 ->second.first += end9-start9;\ + // } else {\ + // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key9, std::pair, int>(end9 - start9, 1)));\ + // } + // + // std::map, int>>::iterator it;\ + // std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ + // for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ + // std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ + // } Real* hBastaDistance = (Real*) gpu->CallocHost(sizeof(Real), kCoalescentBufferLength); for (int i = 0; i < kCoalescentBufferLength; ++i) { @@ -2623,8 +2703,10 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat gpu->FreeHostMemory(hBastaDistance); gpu->FreeHostMemory(hBastaSubintervals); gpu->FreeHostMemory(hBastaFinalKeys); + gpu->FreeHostMemory(hBastaAccumFinalKeys); gpu->FreeHostMemory(hBastaMemory); - gpu->FreeHostMemory(hBastaResMemory); + gpu->FreeHostMemory(hBastaFinalSubintervals); + gpu->FreeHostMemory(hBastaFinalResMemory); #ifdef BEAGLE_DEBUG_FLOW diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 09e1e36c..ed896f09 100755 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -2524,26 +2524,21 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, } - void KernelLauncher::accumulateCarryOut(GPUPtr e, - GPUPtr f, - GPUPtr g, - GPUPtr h, - GPUPtr eRes, - GPUPtr fRes, - GPUPtr gRes, - GPUPtr hRes, - GPUPtr keys, - unsigned int numBlocks) { + void KernelLauncher::accumulateCarryOut(GPUPtr dBastaBlockResMemory, + GPUPtr dBastaFinalResMemory, + GPUPtr dBastaFlags, + unsigned int numSubinterval, + unsigned int numSubintervalFinal) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); #endif - int parameterCountV = 9; - int totalParameterCount = 10; + int parameterCountV = 3; + int totalParameterCount = 5; gpu->LaunchKernel(fAccumulateCarryOut, - bgBastaSumBlock, bgBastaSumGrid, + bgBastaReductionBlock, bgBastaReductionGrid, parameterCountV, totalParameterCount, - e, f, g, h, eRes, fRes, gRes, hRes, keys, numBlocks); + dBastaBlockResMemory, dBastaFinalResMemory, dBastaFlags, numSubinterval, numSubintervalFinal); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); #endif diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index a90d4365..ba25461f 100755 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -630,7 +630,7 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr dBastaBlockResMemory, GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end, unsigned int numSubinterval); void reduceAcrossIntervals(GPUPtr dBastaMemory, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber, unsigned int kCoalescentBufferLength); void preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, GPUPtr dBlockSegmentKeysEnd, unsigned int operationCount, unsigned int numBlocks); - void accumulateCarryOut(GPUPtr e, GPUPtr f, GPUPtr g, GPUPtr h, GPUPtr eRes, GPUPtr fRes, GPUPtr gRes, GPUPtr hRes, GPUPtr keys, unsigned int numBlocks); + void accumulateCarryOut(GPUPtr dBastaBlockResMemory, GPUPtr dBastaFinalResMemory, GPUPtr dBastaFlags, unsigned int numSubinterval, unsigned int numSubintervalFinal); void SetupKernelBlocksAndGrids(); protected: diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 386f1f80..c1d244e4 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2185,7 +2185,7 @@ KW_GLOBAL_KERNEL void kernelPreProcessBastaFlags(KW_GLOBAL_VAR REAL* KW_RESTRICT KW_GLOBAL_VAR REAL* blockEnds, int operationCount, int numBlocks) { -#define SUM_PARTIAL_BLOCK_SIZE_B 32 +#define SUM_PARTIAL_BLOCK_SIZE_B 4 int opIdx = KW_LOCAL_ID_0; int opBlock = KW_GROUP_ID_0; int opNumber = KW_LOCAL_ID_0 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; @@ -2209,53 +2209,101 @@ KW_GLOBAL_KERNEL void kernelPreProcessBastaFlags(KW_GLOBAL_VAR REAL* KW_RESTRICT } -KW_GLOBAL_KERNEL void kernelAccumulateCarryOut(KW_GLOBAL_VAR REAL* e, - KW_GLOBAL_VAR REAL* f, - KW_GLOBAL_VAR REAL* g, - KW_GLOBAL_VAR REAL* h, - KW_GLOBAL_VAR REAL* eRes, - KW_GLOBAL_VAR REAL* fRes, - KW_GLOBAL_VAR REAL* gRes, - KW_GLOBAL_VAR REAL* hRes, - KW_GLOBAL_VAR REAL* keys, - int numBlocks) { -#define SUM_INTERVAL_BLOCK_SIZE 32 - int tid = KW_LOCAL_ID_0; - int group = KW_GROUP_ID_0; - int state = tid % PADDED_STATE_COUNT; - int blockIdx = tid / PADDED_STATE_COUNT; - int blockNumber = group * SUM_INTERVAL_BLOCK_SIZE + blockIdx; - int u = state + blockNumber * PADDED_STATE_COUNT; - - KW_LOCAL_MEM REAL sResE[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sResF[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sResG[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sResH[SUM_INTERVAL_BLOCK_SIZE * PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sSegmentEnds[SUM_INTERVAL_BLOCK_SIZE]; - if (blockNumber < numBlocks && (state < PADDED_STATE_COUNT)) { - sSegmentEnds[blockIdx] = keys[blockNumber]; - sResE[tid] = eRes[u]; - sResF[tid] = fRes[u]; - sResG[tid] = gRes[u]; - sResH[tid] = hRes[u]; - } else { - sSegmentEnds[blockIdx] = -1; - sResE[tid] = 0; - sResF[tid] = 0; - sResG[tid] = 0; - sResH[tid] = 0; - } +KW_GLOBAL_KERNEL void kernelAccumulateCarryOut(KW_GLOBAL_VAR REAL* dBastaBlockResMemory, + KW_GLOBAL_VAR REAL* dBastaFinalResMemory, + KW_GLOBAL_VAR REAL* intervals, + int numSubinterval, + int numSubintervalFinal) { + +#define SUM_PARTIAL_BLOCK_SIZE_B 4 + int state = KW_LOCAL_ID_0; + int opIdx = KW_LOCAL_ID_1; + int opBlock = KW_GROUP_ID_0; + int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; + int opCount = numSubinterval; + int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; + + KW_GLOBAL_VAR REAL* e = dBastaFinalResMemory; + KW_GLOBAL_VAR REAL* f = e + PADDED_STATE_COUNT * numSubintervalFinal; + KW_GLOBAL_VAR REAL* g = f + PADDED_STATE_COUNT * numSubintervalFinal; + KW_GLOBAL_VAR REAL* h = g + PADDED_STATE_COUNT * numSubintervalFinal; + KW_GLOBAL_VAR REAL* flags = intervals + numSubinterval; + + KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sSegmentKeys[SUM_PARTIAL_BLOCK_SIZE_B]; + KW_LOCAL_MEM REAL sSegmentFlags[SUM_PARTIAL_BLOCK_SIZE_B]; + if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { + int op = opNumber; + int u = op * PADDED_STATE_COUNT; + int m = numSubinterval * PADDED_STATE_COUNT; + sSegmentFlags[opIdx] = flags[op]; + sSegmentKeys[opIdx] = intervals[op]; + sPartialsE[opIdx][state] = dBastaBlockResMemory[u + state]; + sPartialsF[opIdx][state] = dBastaBlockResMemory[m + u + state]; + sPartialsG[opIdx][state] = dBastaBlockResMemory[2 * m + u + state]; + sPartialsH[opIdx][state] = dBastaBlockResMemory[3 * m + u + state]; + + } else { + sPartialsE[opIdx][state] = 0; + sPartialsF[opIdx][state] = 0; + sPartialsG[opIdx][state] = 0; + sPartialsH[opIdx][state] = 0; + sSegmentKeys[opIdx] = -1; + sSegmentFlags[opIdx] = 0; + } + + KW_LOCAL_FENCE; + + + int intervalNumber = sSegmentKeys[opIdx]; + int y = intervalNumber * PADDED_STATE_COUNT; + for (int stride = 1; stride < blockSize; stride *= 2) { + int k = (opIdx + 1) * 2 * stride - 1; + if (k < blockSize) { + if (sSegmentFlags[k] == 0) { + sPartialsE[k][state] += sPartialsE[k - stride][state]; + sPartialsF[k][state] += sPartialsF[k - stride][state]; + sPartialsG[k][state] += sPartialsG[k - stride][state]; + sPartialsH[k][state] += sPartialsH[k - stride][state]; + sSegmentFlags[k] = sSegmentFlags[k - stride]; + } + } KW_LOCAL_FENCE; + } - if (blockNumber < numBlocks && (state < PADDED_STATE_COUNT)) { - int blockInterval = sSegmentEnds[blockIdx]; - atomicAdd(&e[blockInterval * PADDED_STATE_COUNT + state], sResE[tid]); - atomicAdd(&f[blockInterval * PADDED_STATE_COUNT + state], sResF[tid]); - atomicAdd(&g[blockInterval * PADDED_STATE_COUNT + state], sResG[tid]); - atomicAdd(&h[blockInterval * PADDED_STATE_COUNT + state], sResH[tid]); + for (int stride = blockSize / 2; stride > 0; stride /= 2) { + int k = (opIdx + 1) * 2 * stride - 1; + if (k + stride < blockSize) { + if (sSegmentFlags[k + stride] == 0) { + sPartialsE[k + stride][state] += sPartialsE[k][state]; + sPartialsF[k + stride][state] += sPartialsF[k][state]; + sPartialsG[k + stride][state] += sPartialsG[k][state]; + sPartialsH[k + stride][state] += sPartialsH[k][state]; + sSegmentFlags[k + stride] = sSegmentFlags[k]; + } } KW_LOCAL_FENCE; + } + + if (sSegmentKeys[opIdx] != sSegmentKeys[opIdx + 1] && (opIdx < blockSize - 1) && (opNumber < opCount -1)) { + e[y + state] = sPartialsE[opIdx][state]; + f[y + state] = sPartialsF[opIdx][state]; + g[y + state] = sPartialsG[opIdx][state]; + h[y + state] = sPartialsH[opIdx][state]; + } + + if (opIdx == blockSize - 1 && (opNumber < opCount -1) || opNumber == opCount -1) { + e[y + state] = sPartialsE[opIdx][state]; + f[y + state] = sPartialsF[opIdx][state]; + g[y + state] = sPartialsG[opIdx][state]; + h[y + state] = sPartialsH[opIdx][state]; + } + + } @@ -2326,6 +2374,12 @@ KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RES int accumulation2PartialIndex = operations[op * numOps + 6]; KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; + e[y + state] = 0; + f[y + state] = 0; + g[y + state] = 0; + h[y + state] = 0; + + KW_LOCAL_FENCE; if (child2PartialIndex >= 0) { e[y + state] = startPartials2[state]; From 51ecb482ae3f194055a5c8dc87a0bb3e6699bb1b Mon Sep 17 00:00:00 2001 From: yucais Date: Fri, 23 Aug 2024 06:53:58 -0700 Subject: [PATCH 265/421] use atomic add for last accumulation --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 78 +++++++------------------- libhmsbeagle/GPU/KernelLauncher.cpp | 23 ++++++++ libhmsbeagle/GPU/KernelLauncher.h | 2 + libhmsbeagle/GPU/kernels/kernelsAll.cu | 53 +++++++++++++++++ 4 files changed, 99 insertions(+), 57 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 166d62e3..42cc10ea 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2246,7 +2246,7 @@ int BeagleGPUImpl::allocateBastaBuffers(int bufferCount, dBastaLogL = gpu->AllocateMemory(kBastaIntervalBlockCount * sizeof(Real)); dBastaDistance = gpu->AllocateMemory(kCoalescentBufferLength * sizeof(Real)); dBastaInterval = gpu->AllocateMemory(kBufferCount * sizeof(Real)); - dBastaFlags = gpu->AllocateMemory(kBufferCount * sizeof(Real)); + dBastaFlags = gpu->AllocateMemory(3 * kBufferCount * sizeof(Real)); dBastaOperationQueue = gpu->AllocateMemory(kBufferCount * 8 * sizeof(Real)); dBastaMemory = gpu->AllocateMemory(4 * kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); dBastaBlockResMemory = gpu->AllocateMemory(4 * kPaddedStateCount * kCoalescentBufferLength * kBastaIntervalBlockCount * sizeof(Real)); @@ -2536,10 +2536,10 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); // } - // auto start4 = std::chrono::high_resolution_clock::now(); - Real* hBastaFinalSubintervals = (Real*) gpu->CallocHost(sizeof(Real), 2 * numSubinterval); - Real* hBastaAccumFinalKeys = (Real*) gpu->CallocHost(sizeof(Real), numSubinterval); + // auto start4 = std::chrono::high_resolution_clock::now(); + Real* hBastaFinalSubintervals = (Real*) gpu->CallocHost(sizeof(Real), 3 * numSubinterval); Real* hBastaFinalFlags = hBastaFinalSubintervals + numSubinterval; + Real* hBastaAccumFinalKeys = hBastaFinalFlags + numSubinterval; int currentSubintervalFinal = 0; int previousIntervalFinal = 0; int numSubintervalFinal = 0; @@ -2572,9 +2572,9 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key4, std::pair, int>(end4 - start4, 1)));\ // } - // auto start5 = std::chrono::high_resolution_clock::now(); + // auto start5 = std::chrono::high_resolution_clock::now(); - gpu->MemcpyHostToDevice(dBastaFlags, hBastaFinalSubintervals, sizeof(Real) * 2 * numSubinterval); + gpu->MemcpyHostToDevice(dBastaFlags, hBastaFinalSubintervals, sizeof(Real) * 3 * numSubinterval); // auto end5 = std::chrono::high_resolution_clock::now();\ // std::string key5 = "copy: flag2";\ @@ -2585,12 +2585,12 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // } else {\ // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key5, std::pair, int>(end5 - start5, 1)));\ // } - - - // auto start6 = std::chrono::high_resolution_clock::now(); + // + // + // auto start6 = std::chrono::high_resolution_clock::now(); kernels->accumulateCarryOut(dBastaBlockResMemory, dBastaFinalResMemory, dBastaFlags, numSubinterval, numSubintervalFinal); - //gpu->SynchronizeDevice(); + // gpu->SynchronizeDevice(); // auto end6 = std::chrono::high_resolution_clock::now();\ // std::string key6 = "kernel2";\ @@ -2602,24 +2602,18 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key6, std::pair, int>(end6 - start6, 1)));\ // } + // auto start7 = std::chrono::high_resolution_clock::now(); - //gpu->PrintfDeviceVector(dBastaFlags, 2 * numSubinterval, r); - //gpu->PrintfDeviceVector(dBastaFinalResMemory, kPaddedStateCount * kCoalescentBufferLength, r); - - - - Real* hBastaFinalResMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * numSubintervalFinal); + Real* zeroes = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * kCoalescentBufferLength); - Real* hBastaFinalEres = hBastaFinalResMemory; - Real* hBastaFinalFres = hBastaFinalEres + kPaddedStateCount * numSubintervalFinal; - Real* hBastaFinalGres = hBastaFinalFres + kPaddedStateCount * numSubintervalFinal; - Real* hBastaFinalHres = hBastaFinalGres + kPaddedStateCount * numSubintervalFinal; + // Fill with zeroes + gpu->MemcpyHostToDevice(dBastaMemory, zeroes, + sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); - // auto start7 = std::chrono::high_resolution_clock::now(); - gpu->MemcpyDeviceToHost(hBastaFinalResMemory, dBastaFinalResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubintervalFinal); + gpu->FreeHostMemory(zeroes); // auto end7 = std::chrono::high_resolution_clock::now();\ - // std::string key7 = "copy intermediate result to cpu";\ + // std::string key7 = "zero out dbastamemory";\ // std::map, int>>::iterator it7 = benchmarkDuration.find(key7);\ // if(it7 != benchmarkDuration.end()){\ // it7 ->second.second += 1;\ @@ -2629,26 +2623,12 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // } // auto start8 = std::chrono::high_resolution_clock::now(); - Real* hBastaMemory = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * kCoalescentBufferLength); - - // Set the pointers to the correct locations within the block - Real* hBastaE = hBastaMemory; - Real* hBastaF = hBastaE + kPaddedStateCount * kCoalescentBufferLength; - Real* hBastaG = hBastaF + kPaddedStateCount * kCoalescentBufferLength; - Real* hBastaH = hBastaG + kPaddedStateCount * kCoalescentBufferLength; - - for (int i = 0; i < numSubintervalFinal; ++i) { - int interval = hBastaAccumFinalKeys[i]; - for (int j = 0; j < kStateCount; ++j) { - hBastaE[interval * kPaddedStateCount + j] += hBastaFinalEres[i * kPaddedStateCount + j]; - hBastaF[interval * kPaddedStateCount + j] += hBastaFinalFres[i * kPaddedStateCount + j]; - hBastaG[interval * kPaddedStateCount + j] += hBastaFinalGres[i * kPaddedStateCount + j]; - hBastaH[interval * kPaddedStateCount + j] += hBastaFinalHres[i * kPaddedStateCount + j]; - } - } + + kernels->accumulateCarryOutFinal(dBastaFinalResMemory, dBastaMemory, dBastaFlags, numSubinterval, numSubintervalFinal, kCoalescentBufferLength); + // gpu->SynchronizeDevice(); // auto end8 = std::chrono::high_resolution_clock::now();\ - // std::string key8 = "last cpu accumulation";\ + // std::string key8 = "last gpu kernel for accumulation";\ // std::map, int>>::iterator it8 = benchmarkDuration.find(key8);\ // if(it8 != benchmarkDuration.end()){\ // it8 ->second.second += 1;\ @@ -2657,20 +2637,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key8, std::pair, int>(end8 - start8, 1)));\ // } - // auto start9 = std::chrono::high_resolution_clock::now(); - gpu->MemcpyHostToDevice(dBastaMemory, hBastaMemory, sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); - - // auto end9 = std::chrono::high_resolution_clock::now();\ - // std::string key9 = "copy: final result";\ - // std::map, int>>::iterator it9 = benchmarkDuration.find(key9);\ - // if(it9 != benchmarkDuration.end()){\ - // it9 ->second.second += 1;\ - // it9 ->second.first += end9-start9;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key9, std::pair, int>(end9 - start9, 1)));\ - // } - // // std::map, int>>::iterator it;\ // std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ // for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ @@ -2703,10 +2670,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat gpu->FreeHostMemory(hBastaDistance); gpu->FreeHostMemory(hBastaSubintervals); gpu->FreeHostMemory(hBastaFinalKeys); - gpu->FreeHostMemory(hBastaAccumFinalKeys); - gpu->FreeHostMemory(hBastaMemory); gpu->FreeHostMemory(hBastaFinalSubintervals); - gpu->FreeHostMemory(hBastaFinalResMemory); #ifdef BEAGLE_DEBUG_FLOW diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index ed896f09..ed6b89b0 100755 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -421,6 +421,7 @@ void KernelLauncher::LoadKernels() { fReduceAcrossInterval = gpu->GetFunction("kernelBastaReduceAcrossInterval"); fPreProcessBastaFlags = gpu->GetFunction("kernelPreProcessBastaFlags"); fAccumulateCarryOut = gpu->GetFunction("kernelAccumulateCarryOut"); + fAccumulateCarryOutFinal = gpu->GetFunction("kernelAccumulateCarryOutFinal"); fReorderPatterns = gpu->GetFunction("kernelReorderPatterns"); @@ -2543,6 +2544,28 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); #endif +} + + void KernelLauncher::accumulateCarryOutFinal(GPUPtr dBastaFinalResMemory, + GPUPtr dBastaMemory, + GPUPtr dBastaFlags, + unsigned int numSubinterval, + unsigned int numSubintervalFinal, + unsigned int kCoalescentBufferLength) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tEntering KernelLauncher::ReduceAcrossinInterval\n"); +#endif + int parameterCountV = 3; + int totalParameterCount = 6; + + gpu->LaunchKernel(fAccumulateCarryOutFinal, + bgBastaReductionBlock, bgBastaReductionGrid, + parameterCountV, totalParameterCount, + dBastaFinalResMemory, dBastaMemory, dBastaFlags, numSubinterval, numSubintervalFinal, kCoalescentBufferLength); +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceAcrossinInterval\n"); +#endif + } void KernelLauncher::reduceAcrossIntervals(GPUPtr dBastaMemory, diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index ba25461f..80589625 100755 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -111,6 +111,7 @@ class KernelLauncher { GPUFunction fReduceAcrossInterval; GPUFunction fPreProcessBastaFlags; GPUFunction fAccumulateCarryOut; + GPUFunction fAccumulateCarryOutFinal; GPUFunction fReorderPatterns; Dim3Int bgTransitionProbabilitiesBlock; @@ -631,6 +632,7 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, void reduceAcrossIntervals(GPUPtr dBastaMemory, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber, unsigned int kCoalescentBufferLength); void preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, GPUPtr dBlockSegmentKeysEnd, unsigned int operationCount, unsigned int numBlocks); void accumulateCarryOut(GPUPtr dBastaBlockResMemory, GPUPtr dBastaFinalResMemory, GPUPtr dBastaFlags, unsigned int numSubinterval, unsigned int numSubintervalFinal); + void accumulateCarryOutFinal(GPUPtr dBastaFinalResMemory, GPUPtr dBastaMemory, GPUPtr dBastaFlags, unsigned int numSubinterval, unsigned int numSubintervalFinal, unsigned int kCoalescentBufferLength); void SetupKernelBlocksAndGrids(); protected: diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index c1d244e4..857b100b 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2179,6 +2179,59 @@ KW_GLOBAL_KERNEL void kernelInnerBastaPartialsCoalescent(KW_GLOBAL_VAR REAL* KW_ //} //} +KW_GLOBAL_KERNEL void kernelAccumulateCarryOutFinal(KW_GLOBAL_VAR REAL* dBastaFinalResMemory, + KW_GLOBAL_VAR REAL* dBastaMemory, + KW_GLOBAL_VAR REAL* intervals, + int numSubinterval, + int numSubintervalFinal, + int kCoalescentBufferLength) { +#define SUM_PARTIAL_BLOCK_SIZE_B 4 + int state = KW_LOCAL_ID_0; + int opIdx = KW_LOCAL_ID_1; + int opBlock = KW_GROUP_ID_0; + int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; + int opCount = numSubintervalFinal; + int u = opNumber * PADDED_STATE_COUNT; + + KW_GLOBAL_VAR REAL* e = dBastaMemory; + KW_GLOBAL_VAR REAL* f = e + PADDED_STATE_COUNT * kCoalescentBufferLength; + KW_GLOBAL_VAR REAL* g = f + PADDED_STATE_COUNT * kCoalescentBufferLength; + KW_GLOBAL_VAR REAL* h = g + PADDED_STATE_COUNT * kCoalescentBufferLength; + KW_GLOBAL_VAR REAL* keys = intervals + 2 * numSubinterval; + + KW_LOCAL_MEM REAL sResE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sResH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sSegmentKeys[SUM_PARTIAL_BLOCK_SIZE_B]; + + if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { + int m = numSubintervalFinal * PADDED_STATE_COUNT; + sSegmentKeys[opIdx] = keys[opNumber]; + sResE[opIdx][state] = dBastaFinalResMemory[u + state]; + sResF[opIdx][state] = dBastaFinalResMemory[m + u + state]; + sResG[opIdx][state] = dBastaFinalResMemory[2 * m + u + state]; + sResH[opIdx][state] = dBastaFinalResMemory[3 * m + u + state]; + } else { + sSegmentKeys[opIdx] = -1; + sResE[opIdx][state] = 0; + sResF[opIdx][state] = 0; + sResG[opIdx][state]= 0; + sResH[opIdx][state] = 0; + } + + KW_LOCAL_FENCE; + + if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { + int intervalNumber = sSegmentKeys[opIdx]; + int y = intervalNumber * PADDED_STATE_COUNT; + + atomicAdd(&e[y + state], sResE[opIdx][state]); + atomicAdd(&f[y + state], sResF[opIdx][state]); + atomicAdd(&g[y + state], sResG[opIdx][state]); + atomicAdd(&h[y + state], sResH[opIdx][state]); + } +} KW_GLOBAL_KERNEL void kernelPreProcessBastaFlags(KW_GLOBAL_VAR REAL* KW_RESTRICT intervals, KW_GLOBAL_VAR REAL* flags, From f383e767dc9e096b6084b991529e447c965fba24 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 26 Aug 2024 17:41:45 -0400 Subject: [PATCH 266/421] Calculate root likelihoods in a single operation w/o memory transfers. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 35 ++-------- libhmsbeagle/GPU/cuda_ops.cu | 86 ++++++++++++++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 7 ++ 3 files changed, 99 insertions(+), 29 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index addb4f87..89e535b5 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1124,26 +1124,11 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i Real* siteProbs = (Real*)dIntegrationTmp; Real* rootPartials = (Real*)dPartials[rootNodeIndex]; + Real* categoryWeights = (Real*)dWeights[categoryWeightsIndex]; + Real* stateFrequencies = (Real*)dFrequencies[stateFrequenciesIndex]; // these really should be category-dependent - // auto hRootPartials = MemcpyDeviceToHostVector(rootPartials, kPaddedPatternCount * kPaddedStateCount * kCategoryCount); - auto hCategoryWeights = MemcpyDeviceToHostVector((Real*)dWeights[categoryWeightsIndex], kCategoryCount); - // std::cerr<<"hRootPartials = "<::calculateRootLogLikelihoods(const i std::abort(); } -// std::vector hScalingFactors; -// if (scale) -// hScalingFactors = MemcpyDeviceToHostVector(dCumulativeScalingFactor, kScaleBufferSize); - -// showScalingInfo(std::cerr, kFlags, cumulativeScaleIndices, kScaleBufferSize); -// std::cerr<<"root partials (h) = "< +struct Prod3 +{ + __host__ __device__ T operator()(thrust::tuple t) + { + return thrust::get<0>(t) * thrust::get<1>(t) * thrust::get<2>(t); + } +}; + +void sumRootLikelihoods(float* siteProbs, // OUT + float* partials, float* weights, float* frequencies, // INT + int nStates, int nPatterns, int nCategories) +{ +} + + +void sumRootLikelihoods(double* siteProbs, // OUT + double* likelihoods, double* weights, double* frequencies, // INT + int nStates, int nPatterns, int nCategories) +{ + using namespace thrust::placeholders; + + // The size of a partials buffer + size_t partials_size = nStates * nPatterns * nCategories; + + + // OK, so we can convert the input index i=[0...partials_size] to (p,c,s) as follows: + // s = i % nStates + // c = (i / nStates) % nCategories + // p = (i / nStates * nCategories) + + // 1. A list of 00000..111111..222222........(P-1)(P-1)(P-1)(P-1) that groups values by state and categories. + // There should be nPatterns groups. + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates * nCategories))); + + // 2. The linear index into the likelihood matrix is: + // j = s + nStates *p + (nStates*nPatterns)*c + // + // We need to compute j as a function of i: + // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) + auto in_lks_start = thrust::make_permutation_iterator( + thrust::device_pointer_cast(likelihoods), + thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), + (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) + ); + + // 3. The linear index into the weights matrix is just: + // k = c + // Therefore + // k(i) = (i / nStates) % nCategories + + auto in_weights_start = thrust::make_permutation_iterator( + thrust::device_pointer_cast(weights), + thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), + (_1/nStates)%nCategories) + ); + + // 4. The linear index into the states matrix is: + // l = s + // Therefore + // l(i) = (i % nStates) + + auto in_frequencies_start = thrust::make_permutation_iterator( + thrust::device_pointer_cast(frequencies), + thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), + _1 % nStates) + ); + + + auto tuples = thrust::make_zip_iterator(thrust::make_tuple(in_lks_start, in_weights_start, in_frequencies_start)); + + auto in_values_start = thrust::make_transform_iterator(tuples, Prod3()); + + thrust::reduce_by_key( + // add execution policy thrust::cuda::par_nosync? + in_keys_start, // key indices start (group by pattern) + in_keys_start + partials_size, // key indices end + in_values_start, // values to reduce (category, pattern, state) + thrust::make_discard_iterator(), // key values out + thrust::device_pointer_cast(siteProbs), // reduced values out + thrust::equal_to() // compare keys operation + // Default operation is (+) + ); +} + + void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) { diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 04481610..b986b584 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -60,4 +60,11 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex); +void sumRootLikelihoods(float* siteProbs, // OUT + float* partials, float* weights, float* frequencies, // INT + int nStates, int nPatterns, int nCategories); + +void sumRootLikelihoods(double* siteProbs, // OUT + double* partials, double* weights, double* frequencies, // INT + int nStates, int nPatterns, int nCategories); #endif From fbadbdf5043045d85b032df9a468d64e5fef1dbf Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 26 Aug 2024 17:45:29 -0400 Subject: [PATCH 267/421] Also implement sumRootLikelihoods for float (duplicate code). --- libhmsbeagle/GPU/cuda_ops.cu | 69 +++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index b2a98b03..69a03ecc 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -169,9 +169,74 @@ struct Prod3 }; void sumRootLikelihoods(float* siteProbs, // OUT - float* partials, float* weights, float* frequencies, // INT + float* likelihoods, float* weights, float* frequencies, // INT int nStates, int nPatterns, int nCategories) { + using namespace thrust::placeholders; + + // The size of a partials buffer + size_t partials_size = nStates * nPatterns * nCategories; + + + // OK, so we can convert the input index i=[0...partials_size] to (p,c,s) as follows: + // s = i % nStates + // c = (i / nStates) % nCategories + // p = (i / nStates * nCategories) + + // 1. A list of 00000..111111..222222........(P-1)(P-1)(P-1)(P-1) that groups values by state and categories. + // There should be nPatterns groups. + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates * nCategories))); + + // 2. The linear index into the likelihood matrix is: + // j = s + nStates *p + (nStates*nPatterns)*c + // + // We need to compute j as a function of i: + // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) + auto in_lks_start = thrust::make_permutation_iterator( + thrust::device_pointer_cast(likelihoods), + thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), + (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) + ); + + // 3. The linear index into the weights matrix is just: + // k = c + // Therefore + // k(i) = (i / nStates) % nCategories + + auto in_weights_start = thrust::make_permutation_iterator( + thrust::device_pointer_cast(weights), + thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), + (_1/nStates)%nCategories) + ); + + // 4. The linear index into the states matrix is: + // l = s + // Therefore + // l(i) = (i % nStates) + + auto in_frequencies_start = thrust::make_permutation_iterator( + thrust::device_pointer_cast(frequencies), + thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), + _1 % nStates) + ); + + + // 5. Make the sequence of likelihood * categoryWeight * stateFrequency + auto tuples = thrust::make_zip_iterator(thrust::make_tuple(in_lks_start, in_weights_start, in_frequencies_start)); + + auto in_values_start = thrust::make_transform_iterator(tuples, Prod3()); + + // 6. Sum over state and category for each pattern. + thrust::reduce_by_key( + // add execution policy thrust::cuda::par_nosync? + in_keys_start, // key indices start (group by pattern) + in_keys_start + partials_size, // key indices end + in_values_start, // values to reduce (category, pattern, state) + thrust::make_discard_iterator(), // key values out + thrust::device_pointer_cast(siteProbs), // reduced values out + thrust::equal_to() // compare keys operation + // Default operation is (+) + ); } @@ -228,10 +293,12 @@ void sumRootLikelihoods(double* siteProbs, // OUT ); + // 5. Make the sequence of likelihood * categoryWeight * stateFrequency auto tuples = thrust::make_zip_iterator(thrust::make_tuple(in_lks_start, in_weights_start, in_frequencies_start)); auto in_values_start = thrust::make_transform_iterator(tuples, Prod3()); + // 6. Sum over state and category for each pattern. thrust::reduce_by_key( // add execution policy thrust::cuda::par_nosync? in_keys_start, // key indices start (group by pattern) From 4ffc40e1ce72aa07a8dcfaee1dab937b08ea1d0b Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 27 Aug 2024 13:21:49 -0400 Subject: [PATCH 268/421] Simplify/fix the CUDA Inf norm. We were reporting abs(max(...)) instead of max(abs(...)). Using thrust yields the same speed as cublas, but we don't need to pass around the cublasHandle_t. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 15 ++++----------- libhmsbeagle/GPU/cuda_ops.cu | 25 ++++++++++++++++++------- libhmsbeagle/GPU/cuda_ops.h | 4 ++-- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 9fd1a10f..28f5bbb2 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -97,16 +97,9 @@ auto normPInf(const T& matrix) { } template -Real normPInf(Real* matrix, int nRows, int nCols, cublasHandle_t cublasH) { - int index; - Real result; - if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasIsamax(cublasH, nRows * nCols, matrix, 1, &index)); - } else { - CUBLAS_CHECK(cublasIdamax(cublasH, nRows * nCols, matrix, 1, &index)); - } - CHECK_CUDA(cudaMemcpy(&result, matrix + index - 1, sizeof(Real), cudaMemcpyDeviceToHost)) - return std::abs(result); +auto normPInf(Real* matrix, int nRows, int nCols) +{ + return cuda_max_abs(matrix, nRows * nCols); } template @@ -356,7 +349,7 @@ void dotProduct(Real* out, cublasHandle_t handle, int n, Real* v1, Real* v2) template auto normPInf(const DnMatrixDevice& M) { - return normPInf(M.ptr, M.size1, M.size2, M.cublasHandle); + return normPInf(M.ptr, M.size1, M.size2); } diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 69a03ecc..431e5cb2 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -26,18 +26,29 @@ void cuda_log_vector(float* v, int length) thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); } -// This seems to be slow because it inserts cudaStreamSynchronize() -double cuda_max(double* v, int length) +double cuda_max_abs(double* values, int length) { - thrust::device_ptr dptr = thrust::device_pointer_cast(v); - return *(thrust::max_element(dptr, dptr + length)); + thrust::device_ptr values_ptr = thrust::device_pointer_cast(values); + + auto in_ptr = thrust::transform_iterator(values_ptr, [] __host__ __device__ (double x) {return std::abs(x);}); + + // Don't use `*(thrust::max_element(....))` -- it is slower than thrust::reduce. + + // thrust::reduce is similar in speed to cublasI{s,d}amax. + return thrust::reduce(in_ptr, in_ptr + length, 0.0, thrust::maximum()); } // This seems to be slow because it inserts cudaStreamSynchronize() -float cuda_max(float* v, int length) +float cuda_max_abs(float* values, int length) { - thrust::device_ptr dptr = thrust::device_pointer_cast(v); - return *(thrust::max_element(dptr, dptr + length)); + thrust::device_ptr values_ptr = thrust::device_pointer_cast(values); + + auto in_ptr = thrust::transform_iterator(values_ptr, [] __host__ __device__ (float x) {return std::abs(x);}); + + // Don't use `*(thrust::max_element(....))` -- it is slower than thrust::reduce. + + // thrust::reduce is similar in speed to cublasI{s,d}amax. + return thrust::reduce(in_ptr, in_ptr + length, 0.0, thrust::maximum()); } // FIXME: It would be nice to merge the code for the and versions of diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index b986b584..b9de4c97 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -52,8 +52,8 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); -double cuda_max(double* v, int length); -float cuda_max(float* v, int length); +double cuda_max_abs(double* v, int length); +float cuda_max_abs(float* v, int length); void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); From 72e57fd874df702ca672faedd7879ce195ba43f7 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 27 Aug 2024 13:22:49 -0400 Subject: [PATCH 269/421] Remove some debug code to make the code easier to read. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 41 ------------------------ 1 file changed, 41 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 89e535b5..a86ed20f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1389,39 +1389,15 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& auto& F = (*FF)[category]; auto& integrationTmp = (*integrationTmpPtr)[category]; -//#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"Before destP = partials operation, destPCache:\n"<::simpleAction2(DnMatrixDevice& // std::cerr<<"j/m = "<::simpleAction2(DnMatrixDevice& F *= eta; -//#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"F *= eta (eta ="< Date: Tue, 27 Aug 2024 13:28:08 -0400 Subject: [PATCH 270/421] Remove another piece of debug code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index a86ed20f..93ea86bd 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1410,20 +1410,14 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& destP.copyFrom( integrationTmp ); - F += destP; + F += destP; Real c2 = normPInf(destP); -//#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"F += destP, c1 = " <::updateInnerBastaPartials(const int* opera const int numOps = BEAGLE_BASTA_OP_COUNT; const int operationCount = end - begin; - Real* hBastaOperationQueue = (Real*) gpu->CallocHost(sizeof(Real), operationCount * numOps); + for (int i = 0; i < operationCount; ++i) { hBastaOperationQueue[i * numOps] = hPartialsOffsets[operations[i * numOps]]; @@ -2331,7 +2362,7 @@ int BeagleGPUImpl::updateInnerBastaPartials(const int* opera } gpu->MemcpyHostToDevice(dBastaOperationQueue, hBastaOperationQueue, sizeof(Real) * operationCount * numOps); - gpu->FreeHostMemory(hBastaOperationQueue); + // Real r = 0; // fprintf(stderr, "e:\n"); @@ -2446,12 +2477,15 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat const GPUPtr sizes = dFrequencies[populationSizesIndex]; const int numOps = BEAGLE_BASTA_OP_COUNT; const int blockSize = 4; - // auto start1 = std::chrono::high_resolution_clock::now(); - Real* hBastaInterval = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + const int start = 0; + const int end = operationCount; +#ifdef BEAGLE_BENCHMARK + auto start1 = std::chrono::high_resolution_clock::now(); +#endif int numBlocks = (operationCount + blockSize - 1) / blockSize; - Real* hBastaSubintervals = (Real*) gpu->CallocHost(sizeof(Real), 2 * operationCount); + Real* hBastaFlags = hBastaSubintervals + operationCount; - Real* hBastaFinalKeys = (Real*) gpu->CallocHost(sizeof(Real), operationCount); + int currentSubinterval = 0; int previousInterval = 0; int numSubinterval = 0; @@ -2474,70 +2508,59 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat numSubinterval = currentSubinterval + 1; +#ifdef BEAGLE_BENCHMARK + auto end1 = std::chrono::high_resolution_clock::now();\ + std::string key1 = "flag1";\ + std::map, int>>::iterator it1 = benchmarkDuration.find(key1);\ + if(it1 != benchmarkDuration.end()){\ + it1->second.second += 1;\ + it1->second.first += end1-start1;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key1, std::pair, int>(end1 - start1, 1)));\ + } - // auto end1 = std::chrono::high_resolution_clock::now();\ - // std::string key1 = "flag1";\ - // std::map, int>>::iterator it1 = benchmarkDuration.find(key1);\ - // if(it1 != benchmarkDuration.end()){\ - // it1->second.second += 1;\ - // it1->second.first += end1-start1;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key1, std::pair, int>(end1 - start1, 1)));\ - // } - // - // auto start2 = std::chrono::high_resolution_clock::now(); + auto start2 = std::chrono::high_resolution_clock::now(); +#endif gpu->MemcpyHostToDevice(dBastaInterval, hBastaSubintervals, sizeof(Real) * 2 * operationCount); - // auto end2 = std::chrono::high_resolution_clock::now();\ - // std::string key2 = "copy: flag1";\ - // std::map, int>>::iterator it2 = benchmarkDuration.find(key2);\ - // if(it2 != benchmarkDuration.end()){\ - // it2->second.second += 1;\ - // it2->second.first += end2-start2;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key2, std::pair, int>(end2 - start2, 1)));\ - // } +#ifdef BEAGLE_BENCHMARK + auto end2 = std::chrono::high_resolution_clock::now();\ + std::string key2 = "copy: flag1";\ + std::map, int>>::iterator it2 = benchmarkDuration.find(key2);\ + if(it2 != benchmarkDuration.end()){\ + it2->second.second += 1;\ + it2->second.first += end2-start2;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key2, std::pair, int>(end2 - start2, 1)));\ + } - const int start = 0; - const int end = operationCount; - //auto start3 = std::chrono::high_resolution_clock::now(); + + auto start3 = std::chrono::high_resolution_clock::now(); +#endif kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResMemory, dBastaInterval, numOps, start, end, numSubinterval); - //gpu->SynchronizeDevice(); - - // auto end3 = std::chrono::high_resolution_clock::now();\ - // std::string key3 = "kernel1";\ - // std::map, int>>::iterator it3 = benchmarkDuration.find(key3);\ - // if(it3 != benchmarkDuration.end()){\ - // it3->second.second += 1;\ - // it3->second.first += end3-start3;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key3, std::pair, int>(end3 - start3, 1)));\ - // } - - // fprintf(stderr, "e:\n"); - // gpu->PrintfDeviceVector(dBastaE, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "f:\n"); - // gpu->PrintfDeviceVector(dBastaF, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "g:\n"); - // gpu->PrintfDeviceVector(dBastaG, kPaddedStateCount * kCoalescentBufferLength, r); - // fprintf(stderr, "h:\n"); - // gpu->PrintfDeviceVector(dBastaH, kPaddedStateCount * kCoalescentBufferLength, r); - // Real r = 0; - // // fprintf(stderr, "The count is: %d\n", kPaddedStateCount); - // fprintf(stderr, "partials:\n"); - // gpu->PrintfDeviceVector(dPartialsOrigin, 24 * kPaddedStateCount, r); - // fprintf(stderr, "operations:\n"); - // gpu->PrintfDeviceVector(dBastaOperationQueue, operationCount * numOps, r); -// } - - // auto start4 = std::chrono::high_resolution_clock::now(); - Real* hBastaFinalSubintervals = (Real*) gpu->CallocHost(sizeof(Real), 3 * numSubinterval); + +#ifdef BEAGLE_BENCHMARK + gpu->SynchronizeHost(); + + auto end3 = std::chrono::high_resolution_clock::now();\ + std::string key3 = "kernel1";\ + std::map, int>>::iterator it3 = benchmarkDuration.find(key3);\ + if(it3 != benchmarkDuration.end()){\ + it3->second.second += 1;\ + it3->second.first += end3-start3;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key3, std::pair, int>(end3 - start3, 1)));\ + } + + auto start4 = std::chrono::high_resolution_clock::now(); +#endif + Real* hBastaFinalFlags = hBastaFinalSubintervals + numSubinterval; Real* hBastaAccumFinalKeys = hBastaFinalFlags + numSubinterval; int currentSubintervalFinal = 0; @@ -2562,93 +2585,176 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat numSubintervalFinal = currentSubintervalFinal + 1; - // auto end4 = std::chrono::high_resolution_clock::now();\ - // std::string key4 = "flag2";\ - // std::map, int>>::iterator it4 = benchmarkDuration.find(key4);\ - // if(it4 != benchmarkDuration.end()){\ - // it4 ->second.second += 1;\ - // it4 ->second.first += end4-start4;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key4, std::pair, int>(end4 - start4, 1)));\ - // } - // auto start5 = std::chrono::high_resolution_clock::now(); +#ifdef BEAGLE_BENCHMARK + auto end4 = std::chrono::high_resolution_clock::now();\ + std::string key4 = "flag2";\ + std::map, int>>::iterator it4 = benchmarkDuration.find(key4);\ + if(it4 != benchmarkDuration.end()){\ + it4 ->second.second += 1;\ + it4 ->second.first += end4-start4;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key4, std::pair, int>(end4 - start4, 1)));\ + } + + auto start5 = std::chrono::high_resolution_clock::now(); +#endif gpu->MemcpyHostToDevice(dBastaFlags, hBastaFinalSubintervals, sizeof(Real) * 3 * numSubinterval); - // auto end5 = std::chrono::high_resolution_clock::now();\ - // std::string key5 = "copy: flag2";\ - // std::map, int>>::iterator it5 = benchmarkDuration.find(key5);\ - // if(it5 != benchmarkDuration.end()){\ - // it5 ->second.second += 1;\ - // it5 ->second.first += end5-start5;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key5, std::pair, int>(end5 - start5, 1)));\ - // } - // - // - // auto start6 = std::chrono::high_resolution_clock::now(); +#ifdef BEAGLE_BENCHMARK + auto end5 = std::chrono::high_resolution_clock::now();\ + std::string key5 = "copy: flag2";\ + std::map, int>>::iterator it5 = benchmarkDuration.find(key5);\ + if(it5 != benchmarkDuration.end()){\ + it5 ->second.second += 1;\ + it5 ->second.first += end5-start5;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key5, std::pair, int>(end5 - start5, 1)));\ + } + + auto start6 = std::chrono::high_resolution_clock::now(); +#endif kernels->accumulateCarryOut(dBastaBlockResMemory, dBastaFinalResMemory, dBastaFlags, numSubinterval, numSubintervalFinal); - // gpu->SynchronizeDevice(); - // auto end6 = std::chrono::high_resolution_clock::now();\ - // std::string key6 = "kernel2";\ - // std::map, int>>::iterator it6 = benchmarkDuration.find(key6);\ - // if(it6 != benchmarkDuration.end()){\ - // it6 ->second.second += 1;\ - // it6 ->second.first += end6-start6;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key6, std::pair, int>(end6 - start6, 1)));\ - // } +#ifdef BEAGLE_BENCHMARK + gpu->SynchronizeHost();; + + auto end6 = std::chrono::high_resolution_clock::now();\ + std::string key6 = "kernel2";\ + std::map, int>>::iterator it6 = benchmarkDuration.find(key6);\ + if(it6 != benchmarkDuration.end()){\ + it6 ->second.second += 1;\ + it6 ->second.first += end6-start6;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key6, std::pair, int>(end6 - start6, 1)));\ + } + + auto start7 = std::chrono::high_resolution_clock::now(); +#endif + +#ifdef BEAGLE_BASTA_CPU_ACCUM + + Real* hBastaFinalEres = hBastaFinalResMemory; + Real* hBastaFinalFres = hBastaFinalEres + kPaddedStateCount * numSubintervalFinal; + Real* hBastaFinalGres = hBastaFinalFres + kPaddedStateCount * numSubintervalFinal; + Real* hBastaFinalHres = hBastaFinalGres + kPaddedStateCount * numSubintervalFinal; + + + gpu->MemcpyDeviceToHost(hBastaFinalResMemory, dBastaFinalResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubintervalFinal); - // auto start7 = std::chrono::high_resolution_clock::now(); +#ifdef BEAGLE_BENCHMARK + auto end7 = std::chrono::high_resolution_clock::now();\ + std::string key7 = "copy intermediate result to cpu";\ + std::map, int>>::iterator it7 = benchmarkDuration.find(key7);\ + if(it7 != benchmarkDuration.end()){\ + it7 ->second.second += 1;\ + it7 ->second.first += end7-start7;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key7, std::pair, int>(end7 - start7, 1)));\ + } + + auto start8 = std::chrono::high_resolution_clock::now(); +#endif + memset(hBastaMemory, 0, 4 * kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + // Set the pointers to the correct locations within the block + Real* hBastaE = hBastaMemory; + Real* hBastaF = hBastaE + kPaddedStateCount * kCoalescentBufferLength; + Real* hBastaG = hBastaF + kPaddedStateCount * kCoalescentBufferLength; + Real* hBastaH = hBastaG + kPaddedStateCount * kCoalescentBufferLength; + + for (int i = 0; i < numSubintervalFinal; ++i) { + int interval = hBastaAccumFinalKeys[i]; + for (int j = 0; j < kStateCount; ++j) { + hBastaE[interval * kPaddedStateCount + j] += hBastaFinalEres[i * kPaddedStateCount + j]; + hBastaF[interval * kPaddedStateCount + j] += hBastaFinalFres[i * kPaddedStateCount + j]; + hBastaG[interval * kPaddedStateCount + j] += hBastaFinalGres[i * kPaddedStateCount + j]; + hBastaH[interval * kPaddedStateCount + j] += hBastaFinalHres[i * kPaddedStateCount + j]; + } + } + +#ifdef BEAGLE_BENCHMARK + auto end8 = std::chrono::high_resolution_clock::now();\ + std::string key8 = "last cpu accumulation";\ + std::map, int>>::iterator it8 = benchmarkDuration.find(key8);\ + if(it8 != benchmarkDuration.end()){\ + it8 ->second.second += 1;\ + it8 ->second.first += end8-start8;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key8, std::pair, int>(end8 - start8, 1)));\ + } + + auto start9 = std::chrono::high_resolution_clock::now(); +#endif - Real* zeroes = (Real*) gpu->CallocHost(sizeof(Real), 4 * kPaddedStateCount * kCoalescentBufferLength); + gpu->MemcpyHostToDevice(dBastaMemory, hBastaMemory, sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); +#ifdef BEAGLE_BENCHMARK + auto end9 = std::chrono::high_resolution_clock::now();\ + std::string key9 = "copy: final result";\ + std::map, int>>::iterator it9 = benchmarkDuration.find(key9);\ + if(it9 != benchmarkDuration.end()){\ + it9 ->second.second += 1;\ + it9 ->second.first += end9-start9;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key9, std::pair, int>(end9 - start9, 1)));\ + } + + std::map, int>>::iterator it;\ + std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ + for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ + std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ + } +#endif + +#else // Fill with zeroes - gpu->MemcpyHostToDevice(dBastaMemory, zeroes, + gpu->MemcpyHostToDevice(dBastaMemory, hBastazeroes, sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); - gpu->FreeHostMemory(zeroes); - - // auto end7 = std::chrono::high_resolution_clock::now();\ - // std::string key7 = "zero out dbastamemory";\ - // std::map, int>>::iterator it7 = benchmarkDuration.find(key7);\ - // if(it7 != benchmarkDuration.end()){\ - // it7 ->second.second += 1;\ - // it7 ->second.first += end7-start7;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key7, std::pair, int>(end7 - start7, 1)));\ - // } +#ifdef BEAGLE_BENCHMARK + auto end7 = std::chrono::high_resolution_clock::now();\ + std::string key7 = "zero out dbastamemory";\ + std::map, int>>::iterator it7 = benchmarkDuration.find(key7);\ + if(it7 != benchmarkDuration.end()){\ + it7 ->second.second += 1;\ + it7 ->second.first += end7-start7;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key7, std::pair, int>(end7 - start7, 1)));\ + } - // auto start8 = std::chrono::high_resolution_clock::now(); + auto start8 = std::chrono::high_resolution_clock::now(); +#endif kernels->accumulateCarryOutFinal(dBastaFinalResMemory, dBastaMemory, dBastaFlags, numSubinterval, numSubintervalFinal, kCoalescentBufferLength); - // gpu->SynchronizeDevice(); - - // auto end8 = std::chrono::high_resolution_clock::now();\ - // std::string key8 = "last gpu kernel for accumulation";\ - // std::map, int>>::iterator it8 = benchmarkDuration.find(key8);\ - // if(it8 != benchmarkDuration.end()){\ - // it8 ->second.second += 1;\ - // it8 ->second.first += end8-start8;\ - // } else {\ - // benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key8, std::pair, int>(end8 - start8, 1)));\ - // } - - - // std::map, int>>::iterator it;\ - // std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ - // for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ - // std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ - // } - - Real* hBastaDistance = (Real*) gpu->CallocHost(sizeof(Real), kCoalescentBufferLength); + +#ifdef BEAGLE_BENCHMARK + gpu->SynchronizeHost(); + + auto end8 = std::chrono::high_resolution_clock::now();\ + std::string key8 = "last gpu kernel for accumulation";\ + std::map, int>>::iterator it8 = benchmarkDuration.find(key8);\ + if(it8 != benchmarkDuration.end()){\ + it8 ->second.second += 1;\ + it8 ->second.first += end8-start8;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key8, std::pair, int>(end8 - start8, 1)));\ + } + + + std::map, int>>::iterator it;\ + std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ + for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ + std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ + } +#endif +#endif + for (int i = 0; i < kCoalescentBufferLength; ++i) { hBastaDistance[i] = (Real) intervalLengths[i]; } - Real* hBastaLogL = (Real*) gpu->CallocHost(sizeof(Real), kBastaIntervalBlockCount); + gpu->MemcpyHostToDevice(dBastaDistance, hBastaDistance, sizeof(Real) * kCoalescentBufferLength); @@ -2665,13 +2771,6 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat for (int i = 0; i < kBastaIntervalBlockCount; i++) { out[0] += hBastaLogL[i]; } - gpu->FreeHostMemory(hBastaInterval); - gpu->FreeHostMemory(hBastaLogL); - gpu->FreeHostMemory(hBastaDistance); - gpu->FreeHostMemory(hBastaSubintervals); - gpu->FreeHostMemory(hBastaFinalKeys); - gpu->FreeHostMemory(hBastaFinalSubintervals); - #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUImpl::accumulateBastaPartials\n"); From d621fc5ed33cd98774ba8f44c1ecd4cc976ce754 Mon Sep 17 00:00:00 2001 From: yucais Date: Tue, 3 Sep 2024 00:11:27 -0700 Subject: [PATCH 272/421] benchmark option --- CMakeLists.txt | 6 ++++++ libhmsbeagle/GPU/kernels/kernelsAll.cu | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bba46f7a..6a49a4c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ OPTION(BEAGLE_DEBUG_FLOW "Print flow" OFF) OPTION(BEAGLE_DEBUG_VALUES "Print debug values" OFF) OPTION(BEAGLE_DEBUG_SYNCH "Synchronize threads on GPU for timing" OFF) OPTION(BEAGLE_BENCHMARK_ENERGY "Print energy consumption" OFF) +OPTION(BEAGLE_BASTA_CPU_ACCUM "Use CPU for final accumulation for reduction" OFF) IF(BEAGLE_BENCHMARK) ADD_DEFINITIONS(-DBEAGLE_BENCHMARK) @@ -52,6 +53,11 @@ IF(BEAGLE_DEBUG_SYNCH) ADD_DEFINITIONS(-DBEAGLE_DEBUG_SYNCH) ENDIF(BEAGLE_DEBUG_SYNCH) +IF(BEAGLE_BASTA_CPU_ACCUM) + ADD_DEFINITIONS(-DBEAGLE_BASTA_CPU_ACCUM) +ENDIF(BEAGLE_BASTA_CPU_ACCUM) + + add_definitions( -DPACKAGE_BUGREPORT="beagle-dev@googlegroups.com" -DPACKAGE_NAME="${BEAGLE_PACKAGE_NAME}" diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 857b100b..337d86d5 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2529,7 +2529,7 @@ KW_GLOBAL_KERNEL void kernelBastaReduceAcrossInterval(KW_GLOBAL_VAR REAL* KW_RES KW_LOCAL_FENCE; if (tidTotal < intervalCount && (coalescent[u] != 0)) { - sPartials2[tid] = logf(coalescent[u]); + sPartials2[tid] = log(coalescent[u]); } else { sPartials2[tid] = 0; } From ff22e7f819e04e546d95a625bfd85c26a34ef3a1 Mon Sep 17 00:00:00 2001 From: yucais Date: Thu, 12 Sep 2024 11:04:30 -0700 Subject: [PATCH 273/421] serially add up elements in reduceWithinInterval kernel --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 99 ++++++++++++++++++++++++-- libhmsbeagle/GPU/KernelLauncher.cpp | 25 +++++++ libhmsbeagle/GPU/KernelLauncher.h | 2 + libhmsbeagle/GPU/kernels/kernelsAll.cu | 99 ++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index fc0cfc86..2e897985 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -84,11 +84,9 @@ namespace gpu { dBastaLogL = (GPUPtr)NULL; dBastaDistance = (GPUPtr)NULL; dBastaOperationQueue = (GPUPtr)NULL; - dBastaBuffers = (GPUPtr)NULL; dCoalescentBuffers = (GPUPtr)NULL; dBastaInterval = (GPUPtr)NULL; dBastaFlags = (GPUPtr)NULL; - dBlockSegmentKeysEnd= (GPUPtr)NULL; dEigenValues = NULL; dEvec = NULL; dIevc = NULL; @@ -267,10 +265,8 @@ BeagleGPUImpl::~BeagleGPUImpl() { free(hStatesOffsets); - gpu->FreeMemory(dBastaBuffers); gpu->FreeMemory(dBastaBlockResMemory); gpu->FreeMemory(dBastaFinalResMemory); - gpu->FreeMemory(dBlockSegmentKeysEnd); gpu->FreeMemory(dBastaMemory); gpu->FreeMemory(dBastaLogL); gpu->FreeMemory(dBastaDistance); @@ -2537,11 +2533,102 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat } + auto start3 = std::chrono::high_resolution_clock::now(); +#endif +#ifdef BEAGLE_BASTA_SERIAL - auto start3 = std::chrono::high_resolution_clock::now(); + kernels->reduceWithinIntervalSerial(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResMemory, dBastaInterval, numOps, + start, end, numSubinterval); + + Real* hBastaFinalEres = hBastaFinalResMemory; + Real* hBastaFinalFres = hBastaFinalEres + kPaddedStateCount * numSubinterval; + Real* hBastaFinalGres = hBastaFinalFres + kPaddedStateCount * numSubinterval; + Real* hBastaFinalHres = hBastaFinalGres + kPaddedStateCount * numSubinterval; +#ifdef BEAGLE_BENCHMARK + gpu->SynchronizeHost(); + + auto end3 = std::chrono::high_resolution_clock::now();\ + std::string key3 = "kernel1";\ + std::map, int>>::iterator it3 = benchmarkDuration.find(key3);\ + if(it3 != benchmarkDuration.end()){\ + it3->second.second += 1;\ + it3->second.first += end3-start3;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key3, std::pair, int>(end3 - start3, 1)));\ + } + + auto start4 = std::chrono::high_resolution_clock::now(); +#endif + + gpu->MemcpyDeviceToHost(hBastaFinalResMemory, dBastaBlockResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubinterval); + +#ifdef BEAGLE_BENCHMARK + auto end4 = std::chrono::high_resolution_clock::now();\ + std::string key4 = "memcopy intermediate result";\ + std::map, int>>::iterator it4 = benchmarkDuration.find(key4);\ + if(it4 != benchmarkDuration.end()){\ + it4 ->second.second += 1;\ + it4 ->second.first += end4-start4;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key4, std::pair, int>(end4 - start4, 1)));\ + } + + auto start5 = std::chrono::high_resolution_clock::now(); +#endif + + memset(hBastaMemory, 0, 4 * kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); + // Set the pointers to the correct locations within the block + Real* hBastaE = hBastaMemory; + Real* hBastaF = hBastaE + kPaddedStateCount * kCoalescentBufferLength; + Real* hBastaG = hBastaF + kPaddedStateCount * kCoalescentBufferLength; + Real* hBastaH = hBastaG + kPaddedStateCount * kCoalescentBufferLength; + + for (int i = 0; i < numSubinterval; ++i) { + int interval = hBastaFinalKeys[i]; + for (int j = 0; j < kStateCount; ++j) { + hBastaE[interval * kPaddedStateCount + j] += hBastaFinalEres[i * kPaddedStateCount + j]; + hBastaF[interval * kPaddedStateCount + j] += hBastaFinalFres[i * kPaddedStateCount + j]; + hBastaG[interval * kPaddedStateCount + j] += hBastaFinalGres[i * kPaddedStateCount + j]; + hBastaH[interval * kPaddedStateCount + j] += hBastaFinalHres[i * kPaddedStateCount + j]; + } + } + +#ifdef BEAGLE_BENCHMARK + auto end5 = std::chrono::high_resolution_clock::now();\ + std::string key5 = "cpu accumulation: final result";\ + std::map, int>>::iterator it5 = benchmarkDuration.find(key5);\ + if(it5 != benchmarkDuration.end()){\ + it5 ->second.second += 1;\ + it5 ->second.first += end5-start5;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key5, std::pair, int>(end5 - start5, 1)));\ + } + + + auto start6 = std::chrono::high_resolution_clock::now(); #endif + gpu->MemcpyHostToDevice(dBastaMemory, hBastaMemory, sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); + +#ifdef BEAGLE_BENCHMARK + auto end6 = std::chrono::high_resolution_clock::now();\ + std::string key6 = "memcopy back final result";\ + std::map, int>>::iterator it6 = benchmarkDuration.find(key6);\ + if(it6 != benchmarkDuration.end()){\ + it6 ->second.second += 1;\ + it6 ->second.first += end6-start6;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key6, std::pair, int>(end6 - start6, 1)));\ + } + + std::map, int>>::iterator it;\ + std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ + for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ + std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ + } +#endif +#else kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResMemory, dBastaInterval, numOps, start, end, numSubinterval); @@ -2750,7 +2837,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat } #endif #endif - +#endif for (int i = 0; i < kCoalescentBufferLength; ++i) { hBastaDistance[i] = (Real) intervalLengths[i]; } diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index ed6b89b0..734f30ae 100755 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -418,6 +418,7 @@ void KernelLauncher::LoadKernels() { fInnerBastaPartialsCoalescent = gpu->GetFunction("kernelInnerBastaPartialsCoalescent"); fReduceWithinInterval = gpu->GetFunction("kernelBastaReduceWithinInterval"); + fReduceWithinIntervalSerial = gpu->GetFunction("kernelBastaReduceWithinIntervalSerial"); fReduceAcrossInterval = gpu->GetFunction("kernelBastaReduceAcrossInterval"); fPreProcessBastaFlags = gpu->GetFunction("kernelPreProcessBastaFlags"); fAccumulateCarryOut = gpu->GetFunction("kernelAccumulateCarryOut"); @@ -2502,6 +2503,30 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, } + void KernelLauncher::reduceWithinIntervalSerial(GPUPtr operations, + GPUPtr partials, + GPUPtr dBastaMemory, + GPUPtr intervals, + unsigned int numOps, + unsigned int start, + unsigned int end, + unsigned int kCoalescentBufferLength) { +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tEntering KernelLauncher::ReduceWithinInterval\n"); +#endif + + int parameterCountV = 4; + int totalParameterCount = 8; + gpu->LaunchKernel(fReduceWithinIntervalSerial, + bgBastaReductionBlock, bgBastaReductionGrid, + parameterCountV, totalParameterCount, + operations, partials, dBastaMemory, intervals, numOps, start, end, kCoalescentBufferLength); + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); +#endif + +} void KernelLauncher::preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index 80589625..b2c32429 100755 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -108,6 +108,7 @@ class KernelLauncher { GPUFunction fInnerBastaPartialsCoalescent; GPUFunction fReduceWithinInterval; + GPUFunction fReduceWithinIntervalSerial; GPUFunction fReduceAcrossInterval; GPUFunction fPreProcessBastaFlags; GPUFunction fAccumulateCarryOut; @@ -629,6 +630,7 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, // GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex, unsigned int renew); void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr dBastaBlockResMemory, GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end, unsigned int numSubinterval); + void reduceWithinIntervalSerial(GPUPtr operations, GPUPtr partials, GPUPtr dBastaMemory, GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end, unsigned int kCoalescentBufferLength); void reduceAcrossIntervals(GPUPtr dBastaMemory, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber, unsigned int kCoalescentBufferLength); void preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, GPUPtr dBlockSegmentKeysEnd, unsigned int operationCount, unsigned int numBlocks); void accumulateCarryOut(GPUPtr dBastaBlockResMemory, GPUPtr dBastaFinalResMemory, GPUPtr dBastaFlags, unsigned int numSubinterval, unsigned int numSubintervalFinal); diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index 337d86d5..e92bbea3 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2360,6 +2360,105 @@ KW_GLOBAL_KERNEL void kernelAccumulateCarryOut(KW_GLOBAL_VAR REAL* dBastaBlockRe } +KW_GLOBAL_KERNEL void kernelBastaReduceWithinIntervalSerial(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, + KW_GLOBAL_VAR REAL* KW_RESTRICT partials, + KW_GLOBAL_VAR REAL* dBastaBlockResMemory, + KW_GLOBAL_VAR REAL* intervals, + int numOps, + int start, + int end, + int numSubinterval) { + +#define SUM_PARTIAL_BLOCK_SIZE_B 4 + + int state = KW_LOCAL_ID_0; + int opIdx = KW_LOCAL_ID_1; + int opBlock = KW_GROUP_ID_0; + int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; + int opCount = end - start; + int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; + + KW_GLOBAL_VAR REAL* e = dBastaBlockResMemory; + KW_GLOBAL_VAR REAL* f = e + PADDED_STATE_COUNT * numSubinterval; + KW_GLOBAL_VAR REAL* g = f + PADDED_STATE_COUNT * numSubinterval; + KW_GLOBAL_VAR REAL* h = g + PADDED_STATE_COUNT * numSubinterval; + + KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sPartialsH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; + KW_LOCAL_MEM REAL sSegmentKeys[SUM_PARTIAL_BLOCK_SIZE_B]; + + if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { + int op = opNumber + start; + int child1PartialIndex = operations[op * numOps + 1]; + int accumulation1PartialIndex = operations[op * numOps + 5]; + KW_GLOBAL_VAR REAL* startPartials1 = partials + child1PartialIndex; + KW_GLOBAL_VAR REAL* endPartials1 = partials + accumulation1PartialIndex; + KW_GLOBAL_VAR REAL* partialIntervals = intervals + start; + sSegmentKeys[opIdx] = partialIntervals[op]; + sPartialsE[opIdx][state] = startPartials1[state]; + sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; + sPartialsG[opIdx][state] = endPartials1[state]; + sPartialsH[opIdx][state] = endPartials1[state] * endPartials1[state]; + } else { + sPartialsE[opIdx][state] = 0; + sPartialsF[opIdx][state] = 0; + sPartialsG[opIdx][state] = 0; + sPartialsH[opIdx][state] = 0; + sSegmentKeys[opIdx] = -1; + } + + KW_LOCAL_FENCE; + + int intervalNumber = sSegmentKeys[opIdx]; + int y = intervalNumber * PADDED_STATE_COUNT; + + if (opNumber < opCount && state < PADDED_STATE_COUNT) { + int op = opNumber + start; + int child2PartialIndex = operations[op * numOps + 3]; + int accumulation2PartialIndex = operations[op * numOps + 6]; + KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; + KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; + e[y + state] = 0; + f[y + state] = 0; + g[y + state] = 0; + h[y + state] = 0; + + KW_LOCAL_FENCE; + + if (child2PartialIndex >= 0) { + e[y + state] = startPartials2[state]; + f[y + state] = startPartials2[state] * startPartials2[state]; + g[y + state] = endPartials2[state]; + h[y + state] = endPartials2[state] * endPartials2[state]; + } + } + + if (opNumber < opCount && state < PADDED_STATE_COUNT) { + int current_key = sSegmentKeys[opIdx]; + int previous_key = sSegmentKeys[opIdx - 1]; + + if (opIdx == 0 || (opIdx != 0) && current_key != previous_key) { + for (int i = opIdx + 1; i < blockSize && sSegmentKeys[i] == current_key; i++) { + sPartialsE[opIdx][state] += sPartialsE[i][state]; + sPartialsF[opIdx][state] += sPartialsF[i][state]; + sPartialsG[opIdx][state] += sPartialsG[i][state]; + sPartialsH[opIdx][state] += sPartialsH[i][state]; + } + + KW_LOCAL_FENCE; + + e[y + state] += sPartialsE[opIdx][state]; + f[y + state] += sPartialsF[opIdx][state]; + g[y + state] += sPartialsG[opIdx][state]; + h[y + state] += sPartialsH[opIdx][state]; + + } + } + +} + KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, KW_GLOBAL_VAR REAL* KW_RESTRICT partials, KW_GLOBAL_VAR REAL* dBastaBlockResMemory, From e1252f6228adc2986d3a73c72ad83227cd663607 Mon Sep 17 00:00:00 2001 From: yucais Date: Fri, 20 Sep 2024 14:51:11 -0700 Subject: [PATCH 274/421] update serial reduction --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 141 +++++++++---------------- libhmsbeagle/GPU/KernelLauncher.cpp | 7 +- libhmsbeagle/GPU/KernelLauncher.h | 2 +- libhmsbeagle/GPU/kernels/kernelsAll.cu | 132 ++++++++--------------- 4 files changed, 93 insertions(+), 189 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 2e897985..6c1b243e 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -2466,6 +2466,7 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat const int populationSizesIndex, int coalescentIndex, double* out) { +//#define BEAGLE_BASTA_SERIAL #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUImpl::accumulateBastaPartials\n"); #endif @@ -2475,10 +2476,55 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat const int blockSize = 4; const int start = 0; const int end = operationCount; + #ifdef BEAGLE_BENCHMARK auto start1 = std::chrono::high_resolution_clock::now(); #endif - int numBlocks = (operationCount + blockSize - 1) / blockSize; + +#ifdef BEAGLE_BASTA_SERIAL + + gpu->MemcpyHostToDevice(dBastaMemory, hBastazeroes, + sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); + +#ifdef BEAGLE_BENCHMARK + auto end1 = std::chrono::high_resolution_clock::now();\ + std::string key1 = "zero out bastamemory";\ + std::map, int>>::iterator it1 = benchmarkDuration.find(key1);\ + if(it1 != benchmarkDuration.end()){\ + it1->second.second += 1;\ + it1->second.first += end1-start1;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key1, std::pair, int>(end1 - start1, 1)));\ + } + + auto start2 = std::chrono::high_resolution_clock::now(); +#endif + + kernels->reduceWithinIntervalSerial(dBastaOperationQueue, dPartialsOrigin, dBastaMemory, numOps, + start, end, kCoalescentBufferLength); + +#ifdef BEAGLE_BENCHMARK + gpu->SynchronizeHost(); + + auto end2 = std::chrono::high_resolution_clock::now();\ + std::string key2 = "kernel1";\ + std::map, int>>::iterator it2 = benchmarkDuration.find(key2);\ + if(it2 != benchmarkDuration.end()){\ + it2->second.second += 1;\ + it2->second.first += end2-start2;\ + } else {\ + benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key2, std::pair, int>(end2 - start2, 1)));\ + } + + std::map, int>>::iterator it;\ + std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ + for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ + std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ + } + +#endif + +#else Real* hBastaFlags = hBastaSubintervals + operationCount; @@ -2536,99 +2582,6 @@ int BeagleGPUImpl::accumulateBastaPartials(const int* operat auto start3 = std::chrono::high_resolution_clock::now(); #endif -#ifdef BEAGLE_BASTA_SERIAL - - kernels->reduceWithinIntervalSerial(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResMemory, dBastaInterval, numOps, - start, end, numSubinterval); - - Real* hBastaFinalEres = hBastaFinalResMemory; - Real* hBastaFinalFres = hBastaFinalEres + kPaddedStateCount * numSubinterval; - Real* hBastaFinalGres = hBastaFinalFres + kPaddedStateCount * numSubinterval; - Real* hBastaFinalHres = hBastaFinalGres + kPaddedStateCount * numSubinterval; -#ifdef BEAGLE_BENCHMARK - gpu->SynchronizeHost(); - - auto end3 = std::chrono::high_resolution_clock::now();\ - std::string key3 = "kernel1";\ - std::map, int>>::iterator it3 = benchmarkDuration.find(key3);\ - if(it3 != benchmarkDuration.end()){\ - it3->second.second += 1;\ - it3->second.first += end3-start3;\ - } else {\ - benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key3, std::pair, int>(end3 - start3, 1)));\ - } - - auto start4 = std::chrono::high_resolution_clock::now(); -#endif - - gpu->MemcpyDeviceToHost(hBastaFinalResMemory, dBastaBlockResMemory, sizeof(Real) * 4 * kPaddedStateCount * numSubinterval); - -#ifdef BEAGLE_BENCHMARK - auto end4 = std::chrono::high_resolution_clock::now();\ - std::string key4 = "memcopy intermediate result";\ - std::map, int>>::iterator it4 = benchmarkDuration.find(key4);\ - if(it4 != benchmarkDuration.end()){\ - it4 ->second.second += 1;\ - it4 ->second.first += end4-start4;\ - } else {\ - benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key4, std::pair, int>(end4 - start4, 1)));\ - } - - auto start5 = std::chrono::high_resolution_clock::now(); -#endif - - memset(hBastaMemory, 0, 4 * kPaddedStateCount * kCoalescentBufferLength * sizeof(Real)); - // Set the pointers to the correct locations within the block - Real* hBastaE = hBastaMemory; - Real* hBastaF = hBastaE + kPaddedStateCount * kCoalescentBufferLength; - Real* hBastaG = hBastaF + kPaddedStateCount * kCoalescentBufferLength; - Real* hBastaH = hBastaG + kPaddedStateCount * kCoalescentBufferLength; - - for (int i = 0; i < numSubinterval; ++i) { - int interval = hBastaFinalKeys[i]; - for (int j = 0; j < kStateCount; ++j) { - hBastaE[interval * kPaddedStateCount + j] += hBastaFinalEres[i * kPaddedStateCount + j]; - hBastaF[interval * kPaddedStateCount + j] += hBastaFinalFres[i * kPaddedStateCount + j]; - hBastaG[interval * kPaddedStateCount + j] += hBastaFinalGres[i * kPaddedStateCount + j]; - hBastaH[interval * kPaddedStateCount + j] += hBastaFinalHres[i * kPaddedStateCount + j]; - } - } - -#ifdef BEAGLE_BENCHMARK - auto end5 = std::chrono::high_resolution_clock::now();\ - std::string key5 = "cpu accumulation: final result";\ - std::map, int>>::iterator it5 = benchmarkDuration.find(key5);\ - if(it5 != benchmarkDuration.end()){\ - it5 ->second.second += 1;\ - it5 ->second.first += end5-start5;\ - } else {\ - benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key5, std::pair, int>(end5 - start5, 1)));\ - } - - - auto start6 = std::chrono::high_resolution_clock::now(); -#endif - - gpu->MemcpyHostToDevice(dBastaMemory, hBastaMemory, sizeof(Real) * 4 * kPaddedStateCount * kCoalescentBufferLength); - -#ifdef BEAGLE_BENCHMARK - auto end6 = std::chrono::high_resolution_clock::now();\ - std::string key6 = "memcopy back final result";\ - std::map, int>>::iterator it6 = benchmarkDuration.find(key6);\ - if(it6 != benchmarkDuration.end()){\ - it6 ->second.second += 1;\ - it6 ->second.first += end6-start6;\ - } else {\ - benchmarkDuration.insert(benchmarkDuration.end(), std::pair, int>>(key6, std::pair, int>(end6 - start6, 1)));\ - } - - std::map, int>>::iterator it;\ - std::cerr << "\nBENCHMARKS: " << __FILE__ << "\n" << "Function\tTime" << std::endl;\ - for (it = benchmarkDuration.begin(); it != benchmarkDuration.end(); it++) {\ - std::cerr << it->first << "\t" << std::chrono::duration_cast(it->second.first).count() << " ns" << "\t" << it->second.second << std::endl;\ - } -#endif -#else kernels->reduceWithinInterval(dBastaOperationQueue, dPartialsOrigin, dBastaBlockResMemory, dBastaInterval, numOps, start, end, numSubinterval); diff --git a/libhmsbeagle/GPU/KernelLauncher.cpp b/libhmsbeagle/GPU/KernelLauncher.cpp index 734f30ae..efc6e419 100755 --- a/libhmsbeagle/GPU/KernelLauncher.cpp +++ b/libhmsbeagle/GPU/KernelLauncher.cpp @@ -245,7 +245,7 @@ void KernelLauncher::SetupKernelBlocksAndGrids() { bgBastaReductionGrid = Dim3Int(18162, 1); bgBastaPreBlock = Dim3Int(32); - bgBastaPreGrid = Dim3Int(110); + bgBastaPreGrid = Dim3Int(1); bgBastaSumBlock = Dim3Int(128); bgBastaSumGrid = Dim3Int(400); @@ -2506,7 +2506,6 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, void KernelLauncher::reduceWithinIntervalSerial(GPUPtr operations, GPUPtr partials, GPUPtr dBastaMemory, - GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end, @@ -2518,9 +2517,9 @@ void KernelLauncher::InnerBastaPartialsCoalescent(GPUPtr partials, int parameterCountV = 4; int totalParameterCount = 8; gpu->LaunchKernel(fReduceWithinIntervalSerial, - bgBastaReductionBlock, bgBastaReductionGrid, + bgBastaPreBlock, bgBastaPreGrid, parameterCountV, totalParameterCount, - operations, partials, dBastaMemory, intervals, numOps, start, end, kCoalescentBufferLength); + operations, partials, dBastaMemory, numOps, start, end, kCoalescentBufferLength); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\t\tLeaving KernelLauncher::ReduceWithinInterval\n"); diff --git a/libhmsbeagle/GPU/KernelLauncher.h b/libhmsbeagle/GPU/KernelLauncher.h index b2c32429..8ceb990a 100755 --- a/libhmsbeagle/GPU/KernelLauncher.h +++ b/libhmsbeagle/GPU/KernelLauncher.h @@ -630,7 +630,7 @@ GPUPtr operations, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNumber, // GPUPtr endPartials1, GPUPtr endPartials2, unsigned int intervalNUmber, unsigned int child2PartialIndex, unsigned int renew); void reduceWithinInterval(GPUPtr operations, GPUPtr partials, GPUPtr dBastaBlockResMemory, GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end, unsigned int numSubinterval); - void reduceWithinIntervalSerial(GPUPtr operations, GPUPtr partials, GPUPtr dBastaMemory, GPUPtr intervals, unsigned int numOps, unsigned int start, unsigned int end, unsigned int kCoalescentBufferLength); + void reduceWithinIntervalSerial(GPUPtr operations, GPUPtr partials, GPUPtr dBastaMemory, unsigned int numOps, unsigned int start, unsigned int end, unsigned int kCoalescentBufferLength); void reduceAcrossIntervals(GPUPtr dBastaMemory, GPUPtr distance, GPUPtr dLogL, GPUPtr sizes, GPUPtr coalescent, unsigned int intervalNUmber, unsigned int kCoalescentBufferLength); void preProcessBastaFlags(GPUPtr dBastaInterval, GPUPtr dBastaFlags, GPUPtr dBlockSegmentKeysEnd, unsigned int operationCount, unsigned int numBlocks); void accumulateCarryOut(GPUPtr dBastaBlockResMemory, GPUPtr dBastaFinalResMemory, GPUPtr dBastaFlags, unsigned int numSubinterval, unsigned int numSubintervalFinal); diff --git a/libhmsbeagle/GPU/kernels/kernelsAll.cu b/libhmsbeagle/GPU/kernels/kernelsAll.cu index e92bbea3..6dfed614 100755 --- a/libhmsbeagle/GPU/kernels/kernelsAll.cu +++ b/libhmsbeagle/GPU/kernels/kernelsAll.cu @@ -2362,101 +2362,53 @@ KW_GLOBAL_KERNEL void kernelAccumulateCarryOut(KW_GLOBAL_VAR REAL* dBastaBlockRe KW_GLOBAL_KERNEL void kernelBastaReduceWithinIntervalSerial(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, KW_GLOBAL_VAR REAL* KW_RESTRICT partials, - KW_GLOBAL_VAR REAL* dBastaBlockResMemory, - KW_GLOBAL_VAR REAL* intervals, + KW_GLOBAL_VAR REAL* dBastaMemory, int numOps, int start, int end, - int numSubinterval) { - -#define SUM_PARTIAL_BLOCK_SIZE_B 4 - - int state = KW_LOCAL_ID_0; - int opIdx = KW_LOCAL_ID_1; - int opBlock = KW_GROUP_ID_0; - int opNumber = KW_LOCAL_ID_1 + opBlock * SUM_PARTIAL_BLOCK_SIZE_B; - int opCount = end - start; - int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; - - KW_GLOBAL_VAR REAL* e = dBastaBlockResMemory; - KW_GLOBAL_VAR REAL* f = e + PADDED_STATE_COUNT * numSubinterval; - KW_GLOBAL_VAR REAL* g = f + PADDED_STATE_COUNT * numSubinterval; - KW_GLOBAL_VAR REAL* h = g + PADDED_STATE_COUNT * numSubinterval; - - KW_LOCAL_MEM REAL sPartialsE[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartialsF[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartialsG[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sPartialsH[SUM_PARTIAL_BLOCK_SIZE_B][PADDED_STATE_COUNT]; - KW_LOCAL_MEM REAL sSegmentKeys[SUM_PARTIAL_BLOCK_SIZE_B]; - - if (opNumber < opCount && (state < PADDED_STATE_COUNT)) { - int op = opNumber + start; - int child1PartialIndex = operations[op * numOps + 1]; - int accumulation1PartialIndex = operations[op * numOps + 5]; - KW_GLOBAL_VAR REAL* startPartials1 = partials + child1PartialIndex; - KW_GLOBAL_VAR REAL* endPartials1 = partials + accumulation1PartialIndex; - KW_GLOBAL_VAR REAL* partialIntervals = intervals + start; - sSegmentKeys[opIdx] = partialIntervals[op]; - sPartialsE[opIdx][state] = startPartials1[state]; - sPartialsF[opIdx][state] = startPartials1[state] * startPartials1[state]; - sPartialsG[opIdx][state] = endPartials1[state]; - sPartialsH[opIdx][state] = endPartials1[state] * endPartials1[state]; - } else { - sPartialsE[opIdx][state] = 0; - sPartialsF[opIdx][state] = 0; - sPartialsG[opIdx][state] = 0; - sPartialsH[opIdx][state] = 0; - sSegmentKeys[opIdx] = -1; - } - - KW_LOCAL_FENCE; - - int intervalNumber = sSegmentKeys[opIdx]; - int y = intervalNumber * PADDED_STATE_COUNT; - - if (opNumber < opCount && state < PADDED_STATE_COUNT) { - int op = opNumber + start; - int child2PartialIndex = operations[op * numOps + 3]; - int accumulation2PartialIndex = operations[op * numOps + 6]; - KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; - KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; - e[y + state] = 0; - f[y + state] = 0; - g[y + state] = 0; - h[y + state] = 0; - - KW_LOCAL_FENCE; - - if (child2PartialIndex >= 0) { - e[y + state] = startPartials2[state]; - f[y + state] = startPartials2[state] * startPartials2[state]; - g[y + state] = endPartials2[state]; - h[y + state] = endPartials2[state] * endPartials2[state]; - } - } - - if (opNumber < opCount && state < PADDED_STATE_COUNT) { - int current_key = sSegmentKeys[opIdx]; - int previous_key = sSegmentKeys[opIdx - 1]; - - if (opIdx == 0 || (opIdx != 0) && current_key != previous_key) { - for (int i = opIdx + 1; i < blockSize && sSegmentKeys[i] == current_key; i++) { - sPartialsE[opIdx][state] += sPartialsE[i][state]; - sPartialsF[opIdx][state] += sPartialsF[i][state]; - sPartialsG[opIdx][state] += sPartialsG[i][state]; - sPartialsH[opIdx][state] += sPartialsH[i][state]; - } - - KW_LOCAL_FENCE; - - e[y + state] += sPartialsE[opIdx][state]; - f[y + state] += sPartialsF[opIdx][state]; - g[y + state] += sPartialsG[opIdx][state]; - h[y + state] += sPartialsH[opIdx][state]; - + int kCoalescentBufferLength) { + int state = KW_LOCAL_ID_0; + int opCount = end - start; + int blockSize = SUM_PARTIAL_BLOCK_SIZE_B; + + KW_GLOBAL_VAR REAL* e = dBastaMemory; + KW_GLOBAL_VAR REAL* f = e + PADDED_STATE_COUNT * kCoalescentBufferLength; + KW_GLOBAL_VAR REAL* g = f + PADDED_STATE_COUNT * kCoalescentBufferLength; + KW_GLOBAL_VAR REAL* h = g + PADDED_STATE_COUNT * kCoalescentBufferLength; + + + if (state < PADDED_STATE_COUNT) { + for (int opIdx = 0; opIdx < opCount; ++opIdx) { + int op = start + opIdx; + int intervalNumber = operations[op * numOps + 7]; + int child1PartialIndex = operations[op * numOps + 1]; + int accumulation1PartialIndex = operations[op * numOps + 5]; + int u = intervalNumber * PADDED_STATE_COUNT + state; + KW_GLOBAL_VAR REAL* startPartials1 = partials + child1PartialIndex; + KW_GLOBAL_VAR REAL* endPartials1 = partials + accumulation1PartialIndex; + + REAL partialE = startPartials1[state]; + REAL partialF = partialE * partialE; + REAL partialG = endPartials1[state]; + REAL partialH = partialG * partialG; + + int child2PartialIndex = operations[op * numOps + 3]; + int accumulation2PartialIndex = operations[op * numOps + 6]; + if (child2PartialIndex >= 0) { + KW_GLOBAL_VAR REAL* startPartials2 = partials + child2PartialIndex; + KW_GLOBAL_VAR REAL* endPartials2 = partials + accumulation2PartialIndex; + partialE += startPartials2[state]; + partialF += startPartials2[state] * startPartials2[state]; + partialG += endPartials2[state]; + partialH += endPartials2[state] * endPartials2[state]; } - } + e[u] += partialE; + f[u] += partialF; + g[u] += partialG; + h[u] += partialH; + } + } } KW_GLOBAL_KERNEL void kernelBastaReduceWithinInterval(KW_GLOBAL_VAR REAL* KW_RESTRICT operations, From 49403baa9b4ab9afabe2bc32c82c0757023030cb Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 24 Oct 2024 19:18:24 -0500 Subject: [PATCH 275/421] JNI edits --- java/beagle/BeagleJNIImpl.java | 2 +- java/beagle/GeneralBeagleImpl.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/java/beagle/BeagleJNIImpl.java b/java/beagle/BeagleJNIImpl.java index bad23d4e..c31924d2 100644 --- a/java/beagle/BeagleJNIImpl.java +++ b/java/beagle/BeagleJNIImpl.java @@ -15,7 +15,7 @@ public class BeagleJNIImpl implements Beagle { - private int instance = -1; + protected int instance = -1; private InstanceDetails details = new InstanceDetails(); public BeagleJNIImpl(int tipCount, diff --git a/java/beagle/GeneralBeagleImpl.java b/java/beagle/GeneralBeagleImpl.java index ae63b98b..b37367a8 100644 --- a/java/beagle/GeneralBeagleImpl.java +++ b/java/beagle/GeneralBeagleImpl.java @@ -229,6 +229,9 @@ public void setStateFrequencies(final int stateFrequenciesIndex, final double[] System.arraycopy(stateFrequencies, 0, this.stateFrequencies[stateFrequenciesIndex], 0, stateCount); } + public void setSparseMatrix(int maatrixIndex, final int[] inRowIndices, final int[] inColIndices, final double[] inValues, int numNonZeros) { + }; + public void setCategoryWeights(final int categoryWeightsIndex, final double[] categoryWeights) { System.arraycopy(categoryWeights, 0, this.categoryWeights[categoryWeightsIndex], 0, categoryCount); } From 134bc77cf089c607f6f9454c7b8c5acec601ffd2 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 24 Oct 2024 19:42:25 -0500 Subject: [PATCH 276/421] fix java flag --- java/beagle/BeagleFlag.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/beagle/BeagleFlag.java b/java/beagle/BeagleFlag.java index 8695b761..f8e9bd9b 100644 --- a/java/beagle/BeagleFlag.java +++ b/java/beagle/BeagleFlag.java @@ -44,7 +44,7 @@ public enum BeagleFlag { PREORDER_TRANSPOSE_MANUAL((long) 1 << 30, "Pre-order transition matrices passed to BEAGLE have been transposed"), PREORDER_TRANSPOSE_AUTO((long) 1 << 31, "Automatically transpose pre-order transition matrices"), - PREORDER_TRANSPOSE_LOW_MEMORY(1L << 32, "Use low-memory automatic transposition of pre-order transition matrices"); + PREORDER_TRANSPOSE_LOW_MEMORY(1L << 32, "Use low-memory automatic transposition of pre-order transition matrices"), BEAGLE_FLAG_COMPUTATION_ACTION((long) 1 << 40, "Action computation"); From 045a878654a8453a5cbd2b9ec61914bc0ca6985a Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 12 Nov 2024 11:56:42 -0500 Subject: [PATCH 277/421] Clarify comment. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 93ea86bd..464e54df 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -100,7 +100,7 @@ Real normest1(const SpMatrix& A, int p, int t=2, int itmax=5) for(int k=1; k<=itmax; k++) { // std::cerr<<"iter "< Date: Tue, 12 Nov 2024 12:16:04 -0500 Subject: [PATCH 278/421] Fix indentation. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 61 ++++++++++++++------------ 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 28f5bbb2..c47329fd 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -137,10 +137,10 @@ struct DnMatrixDevice void copyFrom(const DnMatrixDevice& D) { - assert(order == D.order); - assert(size1 == D.size1); - assert(size2 == D.size2); - MemcpyDeviceToDevice(ptr, D.ptr, size()); + assert(order == D.order); + assert(size1 == D.size1); + assert(size2 == D.size2); + MemcpyDeviceToDevice(ptr, D.ptr, size()); } // Disallow copying -- only one object can "own" the descriptor. @@ -148,13 +148,13 @@ struct DnMatrixDevice // Allow moving. DnMatrixDevice& operator=(DnMatrixDevice&& D) noexcept { - std::swap(cublasHandle, D.cublasHandle); - std::swap(descr, D.descr); - std::swap(ptr, D.ptr); - std::swap(size1, D.size1); - std::swap(size2, D.size2); - std::swap(order, D.order); - return *this; + std::swap(cublasHandle, D.cublasHandle); + std::swap(descr, D.descr); + std::swap(ptr, D.ptr); + std::swap(size1, D.size1); + std::swap(size2, D.size2); + std::swap(order, D.order); + return *this; } DnMatrixDevice& operator*=(Real d) @@ -200,32 +200,32 @@ struct DnMatrixDevice // Allow moving. DnMatrixDevice(DnMatrixDevice&& D) noexcept { - operator=(std::move(D)); + operator=(std::move(D)); } DnMatrixDevice() = default; DnMatrixDevice(cublasHandle_t cb, Real* p, size_t s1, size_t s2, cusparseOrder_t o = CUSPARSE_ORDER_COL) - :cublasHandle(cb), ptr(p), size1(s1), size2(s2), order(o) + :cublasHandle(cb), ptr(p), size1(s1), size2(s2), order(o) { - auto status = (order == CUSPARSE_ORDER_COL) - ? cusparseCreateDnMat(&descr, size1, size2, size1, ptr, DataType, CUSPARSE_ORDER_COL) - : cusparseCreateDnMat(&descr, size1, size2, size2, ptr, DataType, CUSPARSE_ORDER_ROW); + auto status = (order == CUSPARSE_ORDER_COL) + ? cusparseCreateDnMat(&descr, size1, size2, size1, ptr, DataType, CUSPARSE_ORDER_COL) + : cusparseCreateDnMat(&descr, size1, size2, size2, ptr, DataType, CUSPARSE_ORDER_ROW); - if (status != CUSPARSE_STATUS_SUCCESS) - { - std::cerr<<"cusparseCreateDnMat( ) failed!"; - std::exit(1); - } + if (status != CUSPARSE_STATUS_SUCCESS) + { + std::cerr<<"cusparseCreateDnMat( ) failed!"; + std::exit(1); + } } ~DnMatrixDevice() { - // This class does not own the memory, so does not call cudaFree. - if (descr) - { - cusparseDestroyDnMat(descr); - descr = nullptr; - } + // This class does not own the memory, so does not call cudaFree. + if (descr) + { + cusparseDestroyDnMat(descr); + descr = nullptr; + } } }; @@ -257,6 +257,9 @@ struct SpMatrixDevice std::abort(); } + int rows() const {return size1;} + int cols() const {return size2;} + // Disallow copying -- only one object can "own" the descriptor. SpMatrixDevice& operator=(const SpMatrixDevice&) = delete; // Allow moving. @@ -338,11 +341,11 @@ void dotProduct(Real* out, cublasHandle_t handle, int n, Real* v1, Real* v2) { if constexpr (std::is_same::value) { - cublasSdot(handle, n, v1, 1, v2, 1, out); + cublasSdot(handle, n, v1, 1, v2, 1, out); } else { - cublasDdot(handle, n, v1, 1, v2, 1, out); + cublasDdot(handle, n, v1, 1, v2, 1, out); } }; From 8349f35a91010ce22b20e0945b6978630a5aacf8 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 12 Nov 2024 16:47:59 -0500 Subject: [PATCH 279/421] First code for GPU L1 norms --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 87 ++++++++++++++++++++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 76 ++++++++------------- 2 files changed, 114 insertions(+), 49 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index c47329fd..f03cab67 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -217,6 +217,10 @@ struct DnMatrixDevice std::exit(1); } } + DnMatrixDevice(cublasHandle_t cb, size_t s1, size_t s2, cusparseOrder_t o = CUSPARSE_ORDER_COL) + :DnMatrixDevice(cb, cudaDeviceNew(s1*s2), s1, s2, o) + { + } ~DnMatrixDevice() { @@ -355,6 +359,88 @@ auto normPInf(const DnMatrixDevice& M) return normPInf(M.ptr, M.size1, M.size2); } +template +int spMM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSpMatDescr_t A, cusparseDnMatDescr_t B, Real beta, void*& buffer, size_t& buffersize) +{ + size_t new_buffersize; + CHECK_CUSPARSE(cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A, B, &beta, C, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &new_buffersize)); + + if(new_buffersize > buffersize) + { + CHECK_CUDA(cudaFree(buffer)); + CHECK_CUDA(cudaMalloc(&buffer, buffersize)); + buffersize = new_buffersize; + } + + // integrationTmp = alpha * A * destP + CHECK_CUSPARSE(cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A, B, &beta, C, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, buffer)); //row-major layout provides higher performance (?) + return 0; +} + +template +int spMM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) +{ + return spMM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); +} + +// Make the temporary variables part of a data structure that represents to normest1 problem. +// We don't want to allocate memory on the GPU every time we run this. +template +struct GPUnormest1 +{ + int p = 0; + int n = 0; + int t = 0; + int itmax = 0; + DnMatrixDevice X; + DnMatrixDevice Y; + void* buffer = nullptr; + size_t buffer_size = 0; + + Real operator()(const SpMatrixDevice& A) + { + std::cerr<<"n = "<(Y, 1, A, X, 0, buffer, buffer_size); + // X = Y + X.copyFrom(Y); + } + } + + return 0; + } + + GPUnormest1(cublasHandle_t cb, int p_, int n_, int t_=2, int itmax_=5) + :p(p_), n(n_), t(t_), itmax(itmax_), X(cb,n,t), Y(cb,n,t) + { + assert(p >= 0); + assert(t != 0); // negative means t = n + assert(itmax >= 1); + + // Handle t too large + t = std::min(n,t); + + // Interpret negative t as t == n + if (t < 0) t = n; + } +}; + + //template //using MapType = DnMatrix; @@ -471,6 +557,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector> dIntegrationTmpLeft; std::vector> dIntegrationTmpRight; std::vector> dAs; + std::vector> L1normForPower; std::vector integrationLeftBufferSize; std::vector dIntegrationLeftBuffer; std::vector integrationRightBufferSize; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 464e54df..3efda812 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -519,6 +519,12 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dBsCsrValuesCache.resize(kEigenDecompCount); dACscValuesCache.resize(kEigenDecompCount * kCategoryCount * 2); dAs = std::vector>(kEigenDecompCount * kCategoryCount * 2); + for(int p=0; p<=getPMax()+1; p++) + { + int t = 5; + L1normForPower.push_back(GPUnormest1(cublasHandle, p, kStateCount, t)); + } + currentCacheNNZs = std::vector(kEigenDecompCount, kPaddedStateCount); for (int i = 0; i < kEigenDecompCount; i++) { dBsCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); @@ -1289,26 +1295,26 @@ int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int MemcpyDeviceToDevice(dACscValuesCache[matrixIndex1], dBsCsrValuesCache[hEigenMaps[edgeIndex1]], currentCacheNNZs[hEigenMaps[edgeIndex1]]); MemcpyDeviceToDevice(dACscValuesCache[matrixIndex2], dBsCsrValuesCache[hEigenMaps[edgeIndex2]], currentCacheNNZs[hEigenMaps[edgeIndex2]]); - auto format = transpose ? sparseFormat::csc : sparseFormat::csr; - - dAs[matrixIndex1] = SpMatrixDevice(cublasHandle, cusparseHandle, - kPaddedStateCount, kPaddedStateCount, - currentCacheNNZs[hEigenMaps[edgeIndex1]], - dACscValuesCache[matrixIndex1], - dBsCsrColumnsCache[hEigenMaps[edgeIndex1]], - dBsCsrOffsetsCache[hEigenMaps[edgeIndex1]], - format); - - dAs[matrixIndex2] = SpMatrixDevice(cublasHandle, cusparseHandle, - kPaddedStateCount, kPaddedStateCount, - currentCacheNNZs[hEigenMaps[edgeIndex2]], - dACscValuesCache[matrixIndex2], - dBsCsrColumnsCache[hEigenMaps[edgeIndex2]], - dBsCsrOffsetsCache[hEigenMaps[edgeIndex2]], - format); - - dAs[matrixIndex1] *= edgeMultiplier1; //Check if this is asynchronous with the following - dAs[matrixIndex2] *= edgeMultiplier2; + auto format = transpose ? sparseFormat::csc : sparseFormat::csr; + + dAs[matrixIndex1] = SpMatrixDevice(cublasHandle, cusparseHandle, + kPaddedStateCount, kPaddedStateCount, + currentCacheNNZs[hEigenMaps[edgeIndex1]], + dACscValuesCache[matrixIndex1], + dBsCsrColumnsCache[hEigenMaps[edgeIndex1]], + dBsCsrOffsetsCache[hEigenMaps[edgeIndex1]], + format); + + dAs[matrixIndex2] = SpMatrixDevice(cublasHandle, cusparseHandle, + kPaddedStateCount, kPaddedStateCount, + currentCacheNNZs[hEigenMaps[edgeIndex2]], + dACscValuesCache[matrixIndex2], + dBsCsrColumnsCache[hEigenMaps[edgeIndex2]], + dBsCsrOffsetsCache[hEigenMaps[edgeIndex2]], + format); + + dAs[matrixIndex1] *= edgeMultiplier1; //Check if this is asynchronous with the following + dAs[matrixIndex2] *= edgeMultiplier2; #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"category = "<::cacheAMatrices(int edgeIndex1, int std::cerr<<"edgeMultiplierIndex1 = "< -int spMM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSpMatDescr_t A, cusparseDnMatDescr_t B, Real beta, void*& buffer, size_t& buffersize) -{ - size_t new_buffersize; - CHECK_CUSPARSE(cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A, B, &beta, C, DataType, - CUSPARSE_SPMM_ALG_DEFAULT, &new_buffersize)); - - if(new_buffersize > buffersize) - { - CHECK_CUDA(cudaFree(buffer)); - CHECK_CUDA(cudaMalloc(&buffer, buffersize)); - buffersize = new_buffersize; - } - - // integrationTmp = alpha * A * destP - CHECK_CUSPARSE(cusparseSpMM(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A, B, &beta, C, DataType, - CUSPARSE_SPMM_ALG_DEFAULT, buffer)); //row-major layout provides higher performance (?) - return 0; -} - -template -int spMM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) -{ - return spMM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); -} - BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& destP, const DnMatrixDevice& inPartials, int edgeIndex, int category, int matrixIndex, bool left, bool transpose) { const double tol = pow(2.0, -53.0); From 732b7d629178102c31a5677182715b15afcaaa86 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 12 Nov 2024 16:48:06 -0500 Subject: [PATCH 280/421] Debugging code --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 1 + libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index f03cab67..721f47b6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -42,6 +42,7 @@ #include // cusparseSpMV #include #include "cuda_ops.h" +#include using std::vector; using std::tuple; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 3efda812..292b7c52 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1662,6 +1662,9 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, int pMax = getPMax(); for(int p=0;p <= pMax+1; p++) { + L1normForPower[p]( dAs[matrixIndex] ); + std::cerr<<"p = "< Date: Tue, 19 Nov 2024 13:36:13 -0500 Subject: [PATCH 281/421] Remove one crash, but find another. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 292b7c52..09bba76b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1659,11 +1659,20 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, hds[matrixIndex].clear(); + auto dB = SpMatrixDevice(cublasHandle, cusparseHandle, + kPaddedStateCount, kPaddedStateCount, + currentNNZ, + dBsCsrValuesCache[matrixIndex], + dBsCsrColumnsCache[matrixIndex], + dBsCsrOffsetsCache[matrixIndex], + sparseFormat::csr); + int pMax = getPMax(); for(int p=0;p <= pMax+1; p++) { - L1normForPower[p]( dAs[matrixIndex] ); - std::cerr<<"p = "<::setSparseMatrix(int matrixIndex, // equation 3.7 in Al-Mohy and Higham hds[matrixIndex].push_back(pow(approx_norm, 1.0 / Real(p) ) ); } + std::cerr<<"Finished: pMax = "< Date: Tue, 19 Nov 2024 16:33:05 -0500 Subject: [PATCH 282/421] Fix error message. --- libhmsbeagle/GPU/cuda_ops.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index b9de4c97..cd684bcb 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -28,7 +28,7 @@ void MemcpyDeviceToDevice(T* dptr, const T* hptr, int n) { auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyDeviceToDevice); if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); + throw std::runtime_error("cudaMemcpy(Device->Device): failed!"); } template From fc8813a8f62b234c045bb6b4f2c1ba234997ac79 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 19 Nov 2024 17:03:41 -0500 Subject: [PATCH 283/421] Fix memory bug that was making everything crash. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 721f47b6..e21c132f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -371,7 +371,7 @@ int spMM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSp if(new_buffersize > buffersize) { CHECK_CUDA(cudaFree(buffer)); - CHECK_CUDA(cudaMalloc(&buffer, buffersize)); + CHECK_CUDA(cudaMalloc(&buffer, new_buffersize)); buffersize = new_buffersize; } From 9f61db94a9dd334d15ecf8465ac94d1da15932ce Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 19 Nov 2024 17:10:12 -0500 Subject: [PATCH 284/421] Write informative error string. --- libhmsbeagle/GPU/cuda_ops.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index cd684bcb..69b43953 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -11,7 +11,7 @@ T* cudaDeviceNew(int n) T* result; auto status = cudaMalloc((void**)&result, n*sizeof(T)); if (status != cudaSuccess) - throw std::runtime_error("cudaMalloc: failed!"); + throw std::runtime_error(std::string("cudaMalloc: ") + std::string(cudaGetErrorString(status))); return result; } @@ -20,7 +20,7 @@ void MemcpyHostToDevice(T* dptr, const T* hptr, int n) { auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyHostToDevice); if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Host->Device): failed!"); + throw std::runtime_error("cudaMemcpy(Host->Device): " + std::string(cudaGetErrorString(status))); } template @@ -28,7 +28,7 @@ void MemcpyDeviceToDevice(T* dptr, const T* hptr, int n) { auto status = cudaMemcpy(dptr, hptr, n*sizeof(T), cudaMemcpyDeviceToDevice); if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Device->Device): failed!"); + throw std::runtime_error("cudaMemcpy(Device->Device): " + std::string(cudaGetErrorString(status))); } template @@ -36,7 +36,7 @@ void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) { auto status = cudaMemcpy(hptr, dptr, n*sizeof(T), cudaMemcpyDeviceToHost); if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpy(Device->Host): failed!"); + throw std::runtime_error(std::string("cudaMemcpy(Device->Host): ") + std::string(cudaGetErrorString(status))); } template From 0767b1bf0d0cc9f21c78c8dfef21b81c0fa0a4c3 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 26 Nov 2024 13:52:32 -0500 Subject: [PATCH 285/421] Add thrust routine to initialize X matrix. --- libhmsbeagle/GPU/cuda_ops.cu | 44 ++++++++++++++++++++++++++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 5 ++++ 2 files changed, 49 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 431e5cb2..a2a3578a 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -9,6 +9,7 @@ #include #include #include +#include void cuda_log_vector(double* v, int length) { @@ -465,3 +466,46 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC } } +void initialize_norm_x_matrix(float* data, int n, int m) +{ + auto out_ptr = thrust::device_pointer_cast(data); + auto indices = thrust::counting_iterator(0); + auto initialize = [n,m] __host__ __device__ (int i) + { + thrust::default_random_engine rng; + thrust::uniform_real_distribution dist(0, 1); + + if (n==0) return 1.0/n; + rng.discard(i); + if (dist(rng) > 0.5) + return 1.0/n; + else + return -1.0/n; + }; + + thrust::transform(indices, indices + n*m, // in + out_ptr, // out + initialize); +} + +void initialize_norm_x_matrix(double* data, int n, int m) +{ + auto out_ptr = thrust::device_pointer_cast(data); + auto indices = thrust::counting_iterator(0); + auto initialize = [n,m] __host__ __device__ (int i) + { + thrust::default_random_engine rng; + thrust::uniform_real_distribution dist(0, 1); + + if (n==0) return 1.0/n; + rng.discard(i); + if (dist(rng) > 0.5) + return 1.0/n; + else + return -1.0/n; + }; + + thrust::transform(indices, indices + n*m, // in + out_ptr, // out + initialize); +} diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 69b43953..8fe9757d 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -67,4 +67,9 @@ void sumRootLikelihoods(float* siteProbs, // OUT void sumRootLikelihoods(double* siteProbs, // OUT double* partials, double* weights, double* frequencies, // INT int nStates, int nPatterns, int nCategories); + + +void initialize_norm_x_matrix(float* data, int n, int m); +void initialize_norm_x_matrix(double* data, int n, int m); + #endif From 1837e2f7d20918a893f3659eb3111f6e7ccf11b7 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 26 Nov 2024 13:53:47 -0500 Subject: [PATCH 286/421] Use the CUDA/thrust routine to initialize the matrix. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index e21c132f..bc66a379 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -410,6 +410,10 @@ struct GPUnormest1 assert(A.rows() == A.cols()); assert(A.cols() == n); + // X[0][j] = 1 + // X[i][j] = if (uniform(0,1)<0.5) +1 else -1 + initialize_norm_x_matrix(X.ptr, n, t); + for(int k=1; k<=itmax; k++) { // std::cerr<<"iter "< Date: Tue, 26 Nov 2024 15:13:42 -0500 Subject: [PATCH 287/421] Add cuda_max_l1_norm --- libhmsbeagle/GPU/cuda_ops.cu | 48 ++++++++++++++++++++++++++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 3 +++ 2 files changed, 51 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index a2a3578a..a4af3d60 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -52,6 +52,54 @@ float cuda_max_abs(float* values, int length) return thrust::reduce(in_ptr, in_ptr + length, 0.0, thrust::maximum()); } +float cuda_max_l1_norm(float* values, int n, int t, float* buffer_) +{ + using namespace thrust::placeholders; + + + // 1. First sum the absolute values in each column and place the results into buffer + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); + + auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values), + [] __host__ __device__ (float x) {return std::abs(x);} ); + + auto buffer = thrust::device_pointer_cast(buffer_); + + thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) + in_values_start, // values to reduce (with abs applied) + thrust::make_discard_iterator(), // key values out + buffer, // reduced values out + thrust::equal_to() // compare-keys operation + ); // summation is the default operation. + + // 2. Second maximize over the column sums and return the highest. + return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); +} + +double cuda_max_l1_norm(double* values, int n, int t, double* buffer_) +{ + using namespace thrust::placeholders; + + + // 1. First sum the absolute values in each column and place the results into buffer + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); + + auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values), + [] __host__ __device__ (double x) {return std::abs(x);} ); + + auto buffer = thrust::device_pointer_cast(buffer_); + + thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) + in_values_start, // values to reduce (with abs applied) + thrust::make_discard_iterator(), // key values out + buffer, // reduced values out + thrust::equal_to() // compare-keys operation + ); // summation is the default operation. + + // 2. Second maximize over the column sums and return the highest. + return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); +} + // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 8fe9757d..d04da0b5 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -72,4 +72,7 @@ void sumRootLikelihoods(double* siteProbs, // OUT void initialize_norm_x_matrix(float* data, int n, int m); void initialize_norm_x_matrix(double* data, int n, int m); +float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); +double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); + #endif From d3af6330d5d24f3f63caf2763d625539cea42ae0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 26 Nov 2024 15:13:55 -0500 Subject: [PATCH 288/421] Add cuda_sign_div_vector( ). --- libhmsbeagle/GPU/cuda_ops.cu | 16 ++++++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 3 +++ 2 files changed, 19 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index a4af3d60..1893a9b6 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -27,6 +27,22 @@ void cuda_log_vector(float* v, int length) thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); } +void cuda_sign_div_vector(double* v, int n, int t) +{ + thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + + // In-place update is accomplished by making the output iterator the same the starting input iterator. + thrust::transform(vdptr, vdptr + n*t, vdptr, [n] __device__ (double x) -> double {return (x<0)?-1.0/n:1.0/n;}); +} + +void cuda_sign_div_vector(float* v, int n, int t) +{ + thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + + // In-place update is accomplished by making the output iterator the same the starting input iterator. + thrust::transform(vdptr, vdptr + n*t, vdptr, [n] __device__ (float x) ->float {return (x<0)?-1.0/n:1.0/n;}); +} + double cuda_max_abs(double* values, int length) { thrust::device_ptr values_ptr = thrust::device_pointer_cast(values); diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index d04da0b5..176d4193 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -52,6 +52,9 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); +void cuda_sign_div_vector(double* v, int n, int t); +void cuda_sign_div_vector(float* v, int n, int t); + double cuda_max_abs(double* v, int length); float cuda_max_abs(float* v, int length); From 8bc97efb27b6f343e229152f92bd92745e61d4d9 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 26 Nov 2024 15:18:17 -0500 Subject: [PATCH 289/421] Try and compute the norm. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index bc66a379..06a7abac 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -401,7 +401,10 @@ struct GPUnormest1 DnMatrixDevice Y; void* buffer = nullptr; size_t buffer_size = 0; - + + // Temporary storage for cuda_max_l1_norm + Real* buffer2 = nullptr; + Real operator()(const SpMatrixDevice& A) { std::cerr<<"n = "<(t); } }; From 8f8b92fcee24dc369a19496288947133fe96970f Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 26 Nov 2024 15:18:28 -0500 Subject: [PATCH 290/421] Also print t. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 06a7abac..8809c64a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -407,7 +407,7 @@ struct GPUnormest1 Real operator()(const SpMatrixDevice& A) { - std::cerr<<"n = "< Date: Mon, 2 Dec 2024 16:55:35 -0500 Subject: [PATCH 291/421] Fix initializing X matrix. --- libhmsbeagle/GPU/cuda_ops.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 1893a9b6..19a893de 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -539,7 +539,7 @@ void initialize_norm_x_matrix(float* data, int n, int m) thrust::default_random_engine rng; thrust::uniform_real_distribution dist(0, 1); - if (n==0) return 1.0/n; + if (i 0.5) return 1.0/n; @@ -561,7 +561,7 @@ void initialize_norm_x_matrix(double* data, int n, int m) thrust::default_random_engine rng; thrust::uniform_real_distribution dist(0, 1); - if (n==0) return 1.0/n; + if (i 0.5) return 1.0/n; From 09b4dcde6bf267dcb87e5e0f0ad8a9f016f09d90 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 2 Dec 2024 16:56:10 -0500 Subject: [PATCH 292/421] Move byRow matrix printers to header. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 225 ++++++++++++++++++++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 217 ---------------------- 2 files changed, 221 insertions(+), 221 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 8809c64a..6623003f 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -382,6 +382,223 @@ int spMM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSp return 0; } +// Create an io-manipulator so that we can write std::cerr< +class byRowDense +{ + const DnMatrixDevice& D; + bool multiline = false; +public: + + friend std::ostream& operator<<(std::ostream& o, const byRowDense& pr) + { + auto& D = pr.D; + bool multiline = pr.multiline; + + auto hPtr = MemcpyDeviceToHostVector(D.ptr, D.size()); + + if (not multiline) o<<"Rows ["; + for(int row=0;row& d, bool m=false):D(d),multiline(m) {} +}; + +// Create an io-manipulator so that we can write std::cerr< +class byColDense +{ + const DnMatrixDevice& D; +public: + + friend std::ostream& operator<<(std::ostream& o, const byColDense& pr) + { + auto& D = pr.D; + + auto hPtr = MemcpyDeviceToHostVector(D.ptr, D.size()); + + o<<"Cols[ "; + for(int col=0;col& d):D(d) {} +}; + +std::ostream& operator<<(std::ostream& o, sparseFormat f) +{ + if (f == sparseFormat::csr) + o<<"csr"; + else if (f == sparseFormat::csc) + o<<"csc"; + else + o<<"sparseFormat=unknown"; + return o; +} + +// Create an io-manipulator so that we can write std::cerr< +class byRowSparse +{ + const SpMatrixDevice& S; + bool multiline = false; +public: + + friend std::ostream& operator<<(std::ostream& o, const byRowSparse& pr) + { + auto& S = pr.S; + bool multiline = pr.multiline; + + auto values = MemcpyDeviceToHostVector(S.values, S.num_non_zeros); + auto inner = MemcpyDeviceToHostVector(S.inner, S.num_non_zeros); + auto offsets = MemcpyDeviceToHostVector(S.offsets, S.outer_dim_size() + 1); + + std::vector< std::vector > D(S.size1, std::vector(S.size2, 0)); + for(int o=0; o< S.outer_dim_size();o++) + { + for(int index=offsets[o];index& s, bool m = false):S(s),multiline(m) {} +}; + +template +byRowDense byRow(const DnMatrixDevice& D, bool m = false) +{ + return byRowDense(D, m); +} + +template +byRowSparse byRow(const SpMatrixDevice& D, bool m = false) +{ + return byRowSparse(D, m); +} + +template +byColDense byCol(const DnMatrixDevice& D) +{ + return byColDense(D); +} + +template +std::ostream& operator<<(std::ostream& o, const DnMatrixDevice& D) +{ + return o< +std::ostream& operator<<(std::ostream& o, const SpMatrixDevice& D) +{ + return o< +struct asDeviceVec +{ + const Real* data; + int length; + + asDeviceVec(const Real* d, int l):data(d),length(l) { } +}; + +template +std::ostream& operator<<(std::ostream& o, const asDeviceVec& D) +{ + auto H = MemcpyDeviceToHostVector(D.data, D.length); + + o<<"[ "; + for(auto& h: H) + o< +std::ostream& operator<<(std::ostream& o, const std::vector& H) +{ + o<<"[ "; + for(auto& h: H) + o< int spMM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) { @@ -412,15 +629,17 @@ struct GPUnormest1 // A is (n,n); assert(A.rows() == A.cols()); assert(A.cols() == n); + std::cerr<<"A = "<= 1); - // Handle t too large - t = std::min(n,t); - // Interpret negative t as t == n if (t < 0) t = n; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 09bba76b..28e4b9cb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -258,223 +258,6 @@ long long BeagleGPUActionImpl::getFlags() return flags; } -// Create an io-manipulator so that we can write std::cerr< -class byRowDense -{ - const DnMatrixDevice& D; - bool multiline = false; -public: - - friend std::ostream& operator<<(std::ostream& o, const byRowDense& pr) - { - auto& D = pr.D; - bool multiline = pr.multiline; - - auto hPtr = MemcpyDeviceToHostVector(D.ptr, D.size()); - - if (not multiline) o<<"Rows[ "; - for(int row=0;row& d, bool m=false):D(d),multiline(m) {} -}; - -// Create an io-manipulator so that we can write std::cerr< -class byColDense -{ - const DnMatrixDevice& D; -public: - - friend std::ostream& operator<<(std::ostream& o, const byColDense& pr) - { - auto& D = pr.D; - - auto hPtr = MemcpyDeviceToHostVector(D.ptr, D.size()); - - o<<"Cols[ "; - for(int col=0;col& d):D(d) {} -}; - -std::ostream& operator<<(std::ostream& o, sparseFormat f) -{ - if (f == sparseFormat::csr) - o<<"csr"; - else if (f == sparseFormat::csc) - o<<"csc"; - else - o<<"sparseFormat=unknown"; - return o; -} - -// Create an io-manipulator so that we can write std::cerr< -class byRowSparse -{ - const SpMatrixDevice& S; - bool multiline = false; -public: - - friend std::ostream& operator<<(std::ostream& o, const byRowSparse& pr) - { - auto& S = pr.S; - bool multiline = pr.multiline; - - auto values = MemcpyDeviceToHostVector(S.values, S.num_non_zeros); - auto inner = MemcpyDeviceToHostVector(S.inner, S.num_non_zeros); - auto offsets = MemcpyDeviceToHostVector(S.offsets, S.outer_dim_size() + 1); - - std::vector< std::vector > D(S.size1, std::vector(S.size2, 0)); - for(int o=0; o< S.outer_dim_size();o++) - { - for(int index=offsets[o];index& s, bool m = false):S(s),multiline(m) {} -}; - -template -byRowDense byRow(const DnMatrixDevice& D, bool m = false) -{ - return byRowDense(D, m); -} - -template -byRowSparse byRow(const SpMatrixDevice& D, bool m = false) -{ - return byRowSparse(D, m); -} - -template -byColDense byCol(const DnMatrixDevice& D) -{ - return byColDense(D); -} - -template -std::ostream& operator<<(std::ostream& o, const DnMatrixDevice& D) -{ - return o< -std::ostream& operator<<(std::ostream& o, const SpMatrixDevice& D) -{ - return o< -struct asDeviceVec -{ - const Real* data; - int length; - - asDeviceVec(const Real* d, int l):data(d),length(l) { } -}; - -template -std::ostream& operator<<(std::ostream& o, const asDeviceVec& D) -{ - auto H = MemcpyDeviceToHostVector(D.data, D.length); - - o<<"[ "; - for(auto& h: H) - o< -std::ostream& operator<<(std::ostream& o, const std::vector& H) -{ - o<<"[ "; - for(auto& h: H) - o<::createInstance(int tipCount, int partialsBufferCount, From 8e65e2365d623a4a9950a5626d2ad7856df705ae Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 2 Dec 2024 16:56:33 -0500 Subject: [PATCH 293/421] Log norm values. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 28e4b9cb..314f178a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1451,14 +1451,20 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, sparseFormat::csr); int pMax = getPMax(); + auto tmp = hBs[matrixIndex]; + tmp.setIdentity(); for(int p=0;p <= pMax+1; p++) { std::cerr<<"START: p = "< Date: Mon, 2 Dec 2024 17:11:14 -0500 Subject: [PATCH 294/421] Print before/after. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 6623003f..12e5ad41 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -635,11 +635,11 @@ struct GPUnormest1 // X[i][j] = if (uniform(0,1)<0.5) +1 else -1 initialize_norm_x_matrix(X.ptr, n, t); - std::cerr<<"X0 = "< Date: Tue, 3 Dec 2024 10:43:49 -0500 Subject: [PATCH 295/421] Start adding missing stage (3) of the algorithm. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 52 ++++++++++++++++++++---- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 5 ++- libhmsbeagle/GPU/cuda_ops.cu | 4 +- libhmsbeagle/GPU/cuda_ops.h | 4 +- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 12e5ad41..1f58627a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -382,6 +382,40 @@ int spMM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSp return 0; } +template +int spMM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) +{ + return spMM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); +} + +template +int spMTM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSpMatDescr_t A, cusparseDnMatDescr_t B, Real beta, void*& buffer, size_t& buffersize) +{ + size_t new_buffersize; + CHECK_CUSPARSE(cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A, B, &beta, C, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &new_buffersize)); + + if(new_buffersize > buffersize) + { + CHECK_CUDA(cudaFree(buffer)); + CHECK_CUDA(cudaMalloc(&buffer, new_buffersize)); + buffersize = new_buffersize; + } + + // integrationTmp = alpha * A * destP + CHECK_CUSPARSE(cusparseSpMM(handle, CUSPARSE_OPERATION_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A, B, &beta, C, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, buffer)); //row-major layout provides higher performance (?) + return 0; +} + +template +int spMTM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) +{ + return spMTM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); +} + // Create an io-manipulator so that we can write std::cerr< class byRowDense @@ -599,11 +633,6 @@ std::ostream& operator<<(std::ostream& o, const std::vector& H) -template -int spMM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) -{ - return spMM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); -} // Make the temporary variables part of a data structure that represents to normest1 problem. // We don't want to allocate memory on the GPU every time we run this. @@ -653,8 +682,17 @@ struct GPUnormest1 norm = cuda_max_l1_norm(X.ptr, n, t, buffer2); std::cerr<<"A^p*X = "<(Y, 1, A, X, 0, buffer, buffer_size); + } return norm; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 314f178a..403ca2cf 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -99,13 +99,16 @@ Real normest1(const SpMatrix& A, int p, int t=2, int itmax=5) for(int k=1; k<=itmax; k++) { - // std::cerr<<"iter "< est_old or k == 2) { // Note that j is in [0,t-1], but indices[j] is in [0,n-1]. diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 19a893de..64d9b153 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -27,7 +27,7 @@ void cuda_log_vector(float* v, int length) thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); } -void cuda_sign_div_vector(double* v, int n, int t) +void cuda_sign_vector(double* v, int n, int t) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); @@ -35,7 +35,7 @@ void cuda_sign_div_vector(double* v, int n, int t) thrust::transform(vdptr, vdptr + n*t, vdptr, [n] __device__ (double x) -> double {return (x<0)?-1.0/n:1.0/n;}); } -void cuda_sign_div_vector(float* v, int n, int t) +void cuda_sign_vector(float* v, int n, int t) { thrust::device_ptr vdptr = thrust::device_pointer_cast(v); diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 176d4193..aa6225a0 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -52,8 +52,8 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); -void cuda_sign_div_vector(double* v, int n, int t); -void cuda_sign_div_vector(float* v, int n, int t); +void cuda_sign_vector(double* v, int n, int t); +void cuda_sign_vector(float* v, int n, int t); double cuda_max_abs(double* v, int length); float cuda_max_abs(float* v, int length); From 89103a0674da98906ac02be6bd0e5faf3cc4fccb Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:09:59 -0500 Subject: [PATCH 296/421] Correct dimensions for Z and h in the comments of normest1 --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 403ca2cf..ef9d76ab 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -169,9 +169,10 @@ Real normest1(const SpMatrix& A, int p, int t=2, int itmax=5) } // (3) of Algorithm 2.4 - Z = A.transpose() * S; // (t,n) * (n,t) -> (t,t) + Z = A.transpose() * S; // (n,n) * (n,t) -> (n,t) - h = Z.cwiseAbs().rowwise().maxCoeff(); + // Maximize across each the t entries in each row of Z. + h = Z.cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) // (4) of Algorithm 2.4 if (k >= 2 and h.maxCoeff() == h[ind_best]) @@ -181,7 +182,7 @@ Real normest1(const SpMatrix& A, int p, int t=2, int itmax=5) // According to Algorithm 2.4, we should exit here. // However, continuing until we find a different reason to exit - // seems to providegreater accuracy. + // seems to provide greater accuracy. // return est; } From 87384af68f6d127f3d043c415edd6c1cf89a45b1 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:11:21 -0500 Subject: [PATCH 297/421] Get rowwise maximum using CUDA for column-major matrix. --- libhmsbeagle/GPU/cuda_ops.cu | 46 ++++++++++++++++++++++++++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 6 +++++ 2 files changed, 52 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 64d9b153..e9566a9d 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -116,6 +116,52 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer_) return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); } +void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr) +{ + using namespace thrust::placeholders; + // We assume that the matrix has dimensions (n,t) and is column-major. + + // 1. First sum the absolute values in each column and place the results into buffer + // Using _1 % n here should group by row, yielding 0 1 2 3 ...(n-1) 0 1 2 3 ... (n-1) ... + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 % n)); + + auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values_ptr), + [] __host__ __device__ (float x) {return std::abs(x);} ); + + auto out = thrust::device_pointer_cast(out_ptr); + + thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) + in_values_start, // values to reduce (with abs applied) + thrust::make_discard_iterator(), // key values out + out, // reduced values out + thrust::equal_to(), // compare-keys operation + thrust::maximum() // summation is the default operation. + ); +} + +void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr) +{ + using namespace thrust::placeholders; + // We assume that the matrix has dimensions (n,t) and is column-major. + + // 1. First sum the absolute values in each column and place the results into buffer + // Using _1 % n here should group by row, yielding 0 1 2 3 ...(n-1) 0 1 2 3 ... (n-1) ... + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 % n)); + + auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values_ptr), + [] __host__ __device__ (double x) {return std::abs(x);} ); + + auto out = thrust::device_pointer_cast(out_ptr); + + thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) + in_values_start, // values to reduce (with abs applied) + thrust::make_discard_iterator(), // key values out + out, // reduced values out + thrust::equal_to(), // compare-keys operation + thrust::maximum() // summation is the default operation. + ); +} + // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index aa6225a0..a7dead17 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -52,12 +52,18 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); +// (n,t) -> (n,t) void cuda_sign_vector(double* v, int n, int t); void cuda_sign_vector(float* v, int n, int t); +// sum.abs over (n,t) -> (1,t), then max over (1,t) -> (1,1). Matrix assumed to be column-major. double cuda_max_abs(double* v, int length); float cuda_max_abs(float* v, int length); +// max over (n,t) -> (n,1). Matrix assumed to be column-major. +void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr); +void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr); + void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, From 24311a0c521349c7bcd78cdc4e2ade25a438cfca Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:11:40 -0500 Subject: [PATCH 298/421] Construct the h vector using the GPU. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 1f58627a..43e5ff73 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -643,10 +643,12 @@ struct GPUnormest1 int n = 0; int t = 0; int itmax = 0; - DnMatrixDevice X; - DnMatrixDevice Y; + DnMatrixDevice X; // (n,t) + DnMatrixDevice Y; // (n,t) + DnMatrixDevice h; // (n,1) void* buffer = nullptr; size_t buffer_size = 0; + int* indices = nullptr; // Temporary storage for cuda_max_l1_norm Real* buffer2 = nullptr; @@ -692,6 +694,16 @@ struct GPUnormest1 // (3) of Algorithm 2.4 // Z = A^T * S spMTM(Y, 1, A, X, 0, buffer, buffer_size); + + // h[0,j] = max(i) abs(Z(i,j)) + cuda_rowwise_max_abs(Y.ptr, t, t, h.ptr); + + // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. + // We don't do this, because finding a different reason to exit + // seems to provide greater accuracy. + + // + // cuda_get_sorted_indices(h.ptr, t, indices); } @@ -699,7 +711,7 @@ struct GPUnormest1 } GPUnormest1(cublasHandle_t cb, int p_, int n_, int t_=2, int itmax_=5) - :p(p_), n(n_), t(t_), itmax(itmax_), X(cb,n,t), Y(cb,n,t) + :p(p_), n(n_), t(t_), itmax(itmax_), X(cb,n,t), Y(cb,n,t), h(cb,n,1) { assert(p >= 0); assert(t != 0); // negative means t = n @@ -709,6 +721,8 @@ struct GPUnormest1 if (t < 0) t = n; buffer2 = cudaDeviceNew(t); + + indices = cudaDeviceNew(t); } }; From 882638189466d7c6652128b8a20de97f8eb1c080 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:28:35 -0500 Subject: [PATCH 299/421] Make the destructor call cudaFree --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 50 +++++++++++++++++++++----- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 43e5ff73..d258e660 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -646,13 +646,35 @@ struct GPUnormest1 DnMatrixDevice X; // (n,t) DnMatrixDevice Y; // (n,t) DnMatrixDevice h; // (n,1) + + // Sorted matrix dimensions by approximate norm. + int* indices = nullptr; // (n) + + // Used for spMM void* buffer = nullptr; size_t buffer_size = 0; - int* indices = nullptr; // Temporary storage for cuda_max_l1_norm Real* buffer2 = nullptr; + GPUnormest1& operator=(const GPUnormest1&) = delete; + GPUnormest1& operator=(GPUnormest1&& g) + { + std::swap(p, g.p); + std::swap(n, g.n); + std::swap(t, g.t); + std::swap(X, g.X); + std::swap(Y, g.Y); + std::swap(h, g.h); + + // ensure that (*this) and (g) don't both own the buffers! + std::swap(buffer, g.buffer); + std::swap(buffer_size, g.buffer_size); + std::swap(indices, g.indices); + std::swap(buffer2, g.buffer2); + } + + Real operator()(const SpMatrixDevice& A) { std::cerr<<"n = "< (n,n) // (2) Replace parallel entries with random +1/-1 entries. // Skipping this part for now because it involves a lot of branching logic @@ -693,23 +715,28 @@ struct GPUnormest1 // (3) of Algorithm 2.4 // Z = A^T * S - spMTM(Y, 1, A, X, 0, buffer, buffer_size); + spMTM(Y, 1, A, X, 0, buffer, buffer_size); // (n,n) * (n,t) -> (n,t) // h[0,j] = max(i) abs(Z(i,j)) - cuda_rowwise_max_abs(Y.ptr, t, t, h.ptr); + cuda_rowwise_max_abs(Y.ptr, t, t, h.ptr); // (n,t) -> (n,1) // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. // We don't do this, because finding a different reason to exit // seems to provide greater accuracy. - // - // cuda_get_sorted_indices(h.ptr, t, indices); - + // cuda_get_sorted_indices(h.ptr, n, indices); } return norm; } + GPUnormest1(GPUnormest1&& g) + { + operator=(std::move(g)); + } + + GPUnormest1(const GPUnormest1&) = delete; + GPUnormest1(cublasHandle_t cb, int p_, int n_, int t_=2, int itmax_=5) :p(p_), n(n_), t(t_), itmax(itmax_), X(cb,n,t), Y(cb,n,t), h(cb,n,1) { @@ -722,7 +749,14 @@ struct GPUnormest1 buffer2 = cudaDeviceNew(t); - indices = cudaDeviceNew(t); + indices = cudaDeviceNew(n); + } + + ~GPUnormest1() + { + cudaFree(buffer); + cudaFree(buffer2); + cudaFree(indices); } }; From 1eb30d137dcfd1f2e311a34189b377383c3fd1d7 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:32:18 -0500 Subject: [PATCH 300/421] Add cudaDeviceDelete, to handle possible errors in cudaFree. --- libhmsbeagle/GPU/cuda_ops.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index a7dead17..3603bc29 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -15,6 +15,14 @@ T* cudaDeviceNew(int n) return result; } +template +void cudaDeviceDelete(T* buffer) +{ + auto status = cudaFree(buffer); + if (status != cudaSuccess) + throw std::runtime_error(std::string("cudaFree: ") + std::string(cudaGetErrorString(status))); +} + template void MemcpyHostToDevice(T* dptr, const T* hptr, int n) { From 28fc158a1e64f1b02c919526b0fa5b30ea736edd Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:32:30 -0500 Subject: [PATCH 301/421] Fix missing return statement. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index d258e660..c8c92db6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -672,6 +672,8 @@ struct GPUnormest1 std::swap(buffer_size, g.buffer_size); std::swap(indices, g.indices); std::swap(buffer2, g.buffer2); + + return *this; } From c982b9faa66940afef8f6be5ea0063e063005273 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:32:40 -0500 Subject: [PATCH 302/421] Use cudaDeviceDelete( ). --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index c8c92db6..ccf93696 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -756,9 +756,9 @@ struct GPUnormest1 ~GPUnormest1() { - cudaFree(buffer); - cudaFree(buffer2); - cudaFree(indices); + cudaDeviceDelete(buffer); + cudaDeviceDelete(buffer2); + cudaDeviceDelete(indices); } }; From 4d7bb48573a1ff355d66a639d3a9a2d865760330 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:50:14 -0500 Subject: [PATCH 303/421] Sort the indices by the vector elements. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 +- libhmsbeagle/GPU/cuda_ops.cu | 29 ++++++++++++++++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 18 +++++++++------- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index ccf93696..11971c46 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -726,7 +726,7 @@ struct GPUnormest1 // We don't do this, because finding a different reason to exit // seems to provide greater accuracy. - // cuda_get_sorted_indices(h.ptr, n, indices); + cuda_sort_indices_by_vector(h.ptr, n, indices); } return norm; diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index e9566a9d..4c76b226 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -10,6 +10,7 @@ #include #include #include +#include void cuda_log_vector(double* v, int length) { @@ -162,6 +163,34 @@ void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr) ); } +void cuda_sort_indices_by_vector(const float* values_ptr, int n, int* indices_ptr) +{ + auto indices = thrust::device_pointer_cast(indices_ptr); + auto values = thrust::device_pointer_cast(values_ptr); + + // 1. Initialize the indices to [0, n-1] + thrust::counting_iterator iter(0); + thrust::copy(iter, iter + n, indices); + + // 2. Sort the indices by values + thrust::sort(indices, indices+n, [values] __host__ __device__ (int idx1, int idx2) {return values[idx1] > values[idx2];}); +} + + +void cuda_sort_indices_by_vector(const double* values_ptr, int n, int* indices_ptr) +{ + auto indices = thrust::device_pointer_cast(indices_ptr); + auto values = thrust::device_pointer_cast(values_ptr); + + // 1. Initialize the indices to [0, n-1] + thrust::counting_iterator iter(0); + thrust::copy(iter, iter + n, indices); + + // 2. Sort the indices by values + thrust::sort(indices, indices+n, [values] __host__ __device__ (int idx1, int idx2) {return values[idx1] > values[idx2];}); +} + + // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 3603bc29..c7c094e8 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -72,6 +72,16 @@ float cuda_max_abs(float* v, int length); void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr); void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr); + +void initialize_norm_x_matrix(float* data, int n, int m); +void initialize_norm_x_matrix(double* data, int n, int m); + +float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); +double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); + +void cuda_sort_indices_by_vector(const float* values_ptr, int n, int* indices_ptr); +void cuda_sort_indices_by_vector(const double* values_ptr, int n, int* indices_ptr); + void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, @@ -84,12 +94,4 @@ void sumRootLikelihoods(float* siteProbs, // OUT void sumRootLikelihoods(double* siteProbs, // OUT double* partials, double* weights, double* frequencies, // INT int nStates, int nPatterns, int nCategories); - - -void initialize_norm_x_matrix(float* data, int n, int m); -void initialize_norm_x_matrix(double* data, int n, int m); - -float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); -double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); - #endif From 1c290cdecd406651ecf5425ff7bb74304a60d893 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 15:56:30 -0500 Subject: [PATCH 304/421] Fill the X matrix with zeros. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 4 ++++ libhmsbeagle/GPU/cuda_ops.cu | 10 ++++++++++ libhmsbeagle/GPU/cuda_ops.h | 2 ++ 3 files changed, 16 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 11971c46..aec43acd 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -726,7 +726,11 @@ struct GPUnormest1 // We don't do this, because finding a different reason to exit // seems to provide greater accuracy. + // Rank the dimensions by the size of their approximate norm in h, in decreasing order. cuda_sort_indices_by_vector(h.ptr, n, indices); + + // Zero the X matrix. + cuda_fill_vector(X.ptr, n*t, 0); // (n,t) -> (n,t) } return norm; diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 4c76b226..731e5274 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -163,6 +163,16 @@ void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr) ); } +void cuda_fill_vector(float* buffer, int n, double x) +{ + thrust::fill_n(thrust::device_pointer_cast(buffer), n, x); +} + +void cuda_fill_vector(double* buffer, int n, double x) +{ + thrust::fill_n(thrust::device_pointer_cast(buffer), n, x); +} + void cuda_sort_indices_by_vector(const float* values_ptr, int n, int* indices_ptr) { auto indices = thrust::device_pointer_cast(indices_ptr); diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index c7c094e8..2652848a 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -72,6 +72,8 @@ float cuda_max_abs(float* v, int length); void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr); void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr); +void cuda_fill_vector(float* buffer, int n, double x); +void cuda_fill_vector(double* buffer, int n, double x); void initialize_norm_x_matrix(float* data, int n, int m); void initialize_norm_x_matrix(double* data, int n, int m); From 9d7b02dd0e7a935ca6c582de9fa9c044b5a97fe3 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 16:16:00 -0500 Subject: [PATCH 305/421] Also copy itmax. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index aec43acd..b14a86af 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -663,6 +663,7 @@ struct GPUnormest1 std::swap(p, g.p); std::swap(n, g.n); std::swap(t, g.t); + std::swap(itmax, g.itmax); std::swap(X, g.X); std::swap(Y, g.Y); std::swap(h, g.h); From 466dedb178a4510583d10299852fbd084a38338a Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 16:19:53 -0500 Subject: [PATCH 306/421] GPU version seems to work. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 9 ++++++--- libhmsbeagle/GPU/cuda_ops.cu | 21 +++++++++++++++++++++ libhmsbeagle/GPU/cuda_ops.h | 3 +++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index b14a86af..711f407e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -732,6 +732,9 @@ struct GPUnormest1 // Zero the X matrix. cuda_fill_vector(X.ptr, n*t, 0); // (n,t) -> (n,t) + + // Set X(i,indices[i]) = 1 for i in [0,t-1] + cuda_set_indices(X.ptr, n, t, indices); } return norm; @@ -745,14 +748,14 @@ struct GPUnormest1 GPUnormest1(const GPUnormest1&) = delete; GPUnormest1(cublasHandle_t cb, int p_, int n_, int t_=2, int itmax_=5) - :p(p_), n(n_), t(t_), itmax(itmax_), X(cb,n,t), Y(cb,n,t), h(cb,n,1) + :p(p_), n(n_), t(std::min(t_,n)), itmax(itmax_), X(cb,n,t), Y(cb,n,t), h(cb,n,1) { assert(p >= 0); - assert(t != 0); // negative means t = n + assert(t > 0); // negative means t = n assert(itmax >= 1); // Interpret negative t as t == n - if (t < 0) t = n; + // if (t < 0) t = n; buffer2 = cudaDeviceNew(t); diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 731e5274..701b2bd4 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -200,6 +200,27 @@ void cuda_sort_indices_by_vector(const double* values_ptr, int n, int* indices_p thrust::sort(indices, indices+n, [values] __host__ __device__ (int idx1, int idx2) {return values[idx1] > values[idx2];}); } +void cuda_set_indices(float* x_ptr, int n, int t, const int* indices_ptr) +{ + // 1. Initialize the indices to [0, n-1] + thrust::counting_iterator iter(0); + auto x = thrust::device_pointer_cast(x_ptr); + auto indices = thrust::device_pointer_cast(indices_ptr); + + thrust::for_each(iter, iter+t, [=] __host__ __device__ (int i) { x[n*i + indices[i]] = 1.0; }); +} + +// Set X(i,indices[i]) = 1 for i in [0,t-1] +void cuda_set_indices(double* x_ptr, int n, int t, const int* indices_ptr) +{ + // 1. Initialize the indices to [0, n-1] + thrust::counting_iterator iter(0); + auto x = thrust::device_pointer_cast(x_ptr); + auto indices = thrust::device_pointer_cast(indices_ptr); + + thrust::for_each(iter, iter+t, [=] __host__ __device__ (int i) { x[n*i + indices[i]] = 1.0; }); +} + // FIXME: It would be nice to merge the code for the and versions of // rescalePartialsDevice, but this was somehow causing the program to crash. diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 2652848a..6f5827cd 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -84,6 +84,9 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); void cuda_sort_indices_by_vector(const float* values_ptr, int n, int* indices_ptr); void cuda_sort_indices_by_vector(const double* values_ptr, int n, int* indices_ptr); +void cuda_set_indices(float* x, int n, int t, const int* indices); +void cuda_set_indices(double* x, int n, int t, const int* indices); + void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, From c46b7e81e31bc87ec743016246fc01a5f6e81546 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Dec 2024 16:58:17 -0500 Subject: [PATCH 307/421] Remove debugging print statements. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 14 +++++++------- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 711f407e..19807264 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -680,12 +680,12 @@ struct GPUnormest1 Real operator()(const SpMatrixDevice& A) { - std::cerr<<"n = "< (n,n) @@ -722,7 +722,7 @@ struct GPUnormest1 // h[0,j] = max(i) abs(Z(i,j)) cuda_rowwise_max_abs(Y.ptr, t, t, h.ptr); // (n,t) -> (n,1) - + // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. // We don't do this, because finding a different reason to exit // seems to provide greater accuracy. diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index ef9d76ab..e04eaf06 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -99,15 +99,15 @@ Real normest1(const SpMatrix& A, int p, int t=2, int itmax=5) for(int k=1; k<=itmax; k++) { - std::cerr<<"iter "< est_old or k == 2) { From 54c7abe1caf644961f4ae0dea2a8a241bfc8b602 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 9 Dec 2024 16:03:12 -0500 Subject: [PATCH 308/421] Use kPaddedStateCount instead of kStateCount --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index e04eaf06..616662f1 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -309,7 +309,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, for(int p=0; p<=getPMax()+1; p++) { int t = 5; - L1normForPower.push_back(GPUnormest1(cublasHandle, p, kStateCount, t)); + L1normForPower.push_back(GPUnormest1(cublasHandle, p, kPaddedStateCount, t)); } currentCacheNNZs = std::vector(kEigenDecompCount, kPaddedStateCount); From a893e9e510f5458e61d5050c818b61d0362668aa Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 9 Dec 2024 16:10:25 -0500 Subject: [PATCH 309/421] Stop calculating tmp^p. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 616662f1..a4f6e850 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1456,7 +1456,6 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, int pMax = getPMax(); auto tmp = hBs[matrixIndex]; - tmp.setIdentity(); for(int p=0;p <= pMax+1; p++) { std::cerr<<"START: p = "<::setSparseMatrix(int matrixIndex, int t = 5; Real approx_norm = normest1( hBs[matrixIndex], p, t); std::cerr<<" CPU_value = "< Date: Mon, 9 Dec 2024 16:10:35 -0500 Subject: [PATCH 310/421] Try switching to xcode 16 --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de696a3a..1d96a6bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,8 +16,8 @@ jobs: ubuntu-gcc-11, ubuntu-gcc-12, ubuntu-clang-15, - macos-xcode-14-intel, - macos-xcode-14-arm, + macos-xcode-16-intel, + macos-xcode-16-arm, windows ] @@ -50,17 +50,17 @@ jobs: buildtype: "release" arch: "linux64" - - name: macos-xcode-14-intel + - name: macos-xcode-16-intel os: macos-12 compiler: xcode - version: "14" + version: "16" buildtype: "release" arch: "mac-intel64" - - name: macos-xcode-14-arm + - name: macos-xcode-16-arm os: macos-14 compiler: xcode - version: "14" + version: "16" buildtype: "release" arch: "mac-arm64" From 9847ef3a36636c3b93f19cff7632f7ec41b05e40 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 9 Dec 2024 16:17:03 -0500 Subject: [PATCH 311/421] Try switching windows build to ubuntu 24.04 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1d96a6bf..dc93faeb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: arch: "mac-arm64" - name: windows - os: ubuntu-22.04 + os: ubuntu-24.04 compiler: mingw version: "N/A" buildtype: "release" From b882e6c5cc3f89d81b8b80a612ebbf68e12cd575 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 9 Dec 2024 16:18:09 -0500 Subject: [PATCH 312/421] Switch mac into build to MacOs-13 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc93faeb..4c47523b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: arch: "linux64" - name: macos-xcode-16-intel - os: macos-12 + os: macos-13 compiler: xcode version: "16" buildtype: "release" From 4c77bd8d3fc32490d06c8acb841f429619be1342 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 9 Dec 2024 16:19:28 -0500 Subject: [PATCH 313/421] Use XCode 15 for MacOs 13 --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c47523b..74e4729a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: ubuntu-gcc-11, ubuntu-gcc-12, ubuntu-clang-15, - macos-xcode-16-intel, + macos-xcode-15-intel, macos-xcode-16-arm, windows ] @@ -50,10 +50,10 @@ jobs: buildtype: "release" arch: "linux64" - - name: macos-xcode-16-intel + - name: macos-xcode-15-intel os: macos-13 compiler: xcode - version: "16" + version: "15" buildtype: "release" arch: "mac-intel64" From eba02fed031db0ebb8238ed460a4f3100ce84abd Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 9 Dec 2024 16:29:15 -0500 Subject: [PATCH 314/421] Use the GPU norm for |A^p| --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index a4f6e850..65516ab1 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1458,19 +1458,16 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, auto tmp = hBs[matrixIndex]; for(int p=0;p <= pMax+1; p++) { - std::cerr<<"START: p = "< Date: Mon, 16 Dec 2024 13:29:23 -0500 Subject: [PATCH 315/421] Add early-exit from L1 norm code. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 19807264..5192d3ab 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -706,7 +706,11 @@ struct GPUnormest1 } // get largest L1 norm - norm = cuda_max_l1_norm(X.ptr, n, t, buffer2); + double new_norm = cuda_max_l1_norm(X.ptr, n, t, buffer2); + if (new_norm <= norm and k >= 2) + break; + + norm = std::max(norm,new_norm); // std::cerr<<"A^p*X = "< (n,1) - + // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. // We don't do this, because finding a different reason to exit // seems to provide greater accuracy. From b125eaf70b24688e65fec32820afa58f08c593b1 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 16 Dec 2024 13:46:46 -0500 Subject: [PATCH 316/421] Don't compute norms for early exit. --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 65516ab1..520da22a 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -1164,7 +1164,9 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& const Real one = 1; for (int i = 0; i < s; i++) { +#ifdef ACTION_EARLY_EXIT_NORM Real c1 = normPInf(destP); +#endif for (int j = 1; j < m + 1; j++) { //#ifdef BEAGLE_DEBUG_FLOW @@ -1177,9 +1179,11 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& F += destP; +#ifdef ACTION_EARLY_EXIT_NORM Real c2 = normPInf(destP); if (c1 + c2 <= tol * normPInf(F)) break; c1 = c2; +#endif } F *= eta; From 5550607fcb93fa13a22e13bbc2a1f1fbdbb7d572 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 17 Dec 2024 17:40:50 -0600 Subject: [PATCH 317/421] fix cmake cuda error for recent versions --- libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt index 06aa922e..d736d97e 100644 --- a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -10,9 +10,6 @@ link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ) -enable_language(CUDA) -set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -O3 -Wno-deprecated-gpu-targets --extended-lambda -std=c++17") - if(WIN32) set(NVCC "nvcc.exe") set(NVCCFLAGS "-O3 -Wno-deprecated-gpu-targets") @@ -20,11 +17,20 @@ if(WIN32) set(CMD_NAME createCUDAKernels.bat) else(WIN32) set(NVCC "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc") + message(STATUS "NVCC = ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc") set(NVCCFLAGS "-O3 -Wno-deprecated-gpu-targets -ccbin ${CMAKE_CXX_COMPILER} -m64 -D_POSIX_C_SOURCE -std=c++11") set(NVCC_INC "-I${CMAKE_SOURCE_DIR}") + set(CMAKE_CUDA_COMPILER ${NVCC}) set(CMD_NAME make_cuda_kernels.sh) endif(WIN32) + +set(CMAKE_CUDA_ARCHITECTURES "native") +enable_language(CUDA) +set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -O3 -Wno-deprecated-gpu-targets --extended-lambda -std=c++17") + + + add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/../kernels/BeagleCUDA_kernels.h COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/../kernels/${CMD_NAME} ${NVCC} "${NVCCFLAGS}" "${NVCC_INC}" From 9e0224520ab43a43ebacf93e15fe3701fe91f762 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 17 Dec 2024 17:49:17 -0600 Subject: [PATCH 318/421] do not really need "native" it seems --- libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt index d736d97e..c6fbc845 100644 --- a/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMake_Action/CMakeLists.txt @@ -17,15 +17,11 @@ if(WIN32) set(CMD_NAME createCUDAKernels.bat) else(WIN32) set(NVCC "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc") - message(STATUS "NVCC = ${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc") set(NVCCFLAGS "-O3 -Wno-deprecated-gpu-targets -ccbin ${CMAKE_CXX_COMPILER} -m64 -D_POSIX_C_SOURCE -std=c++11") set(NVCC_INC "-I${CMAKE_SOURCE_DIR}") - set(CMAKE_CUDA_COMPILER ${NVCC}) set(CMD_NAME make_cuda_kernels.sh) endif(WIN32) - -set(CMAKE_CUDA_ARCHITECTURES "native") enable_language(CUDA) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -O3 -Wno-deprecated-gpu-targets --extended-lambda -std=c++17") From cbc2286b328c17d64d6e69b2e07cb43113d44bd6 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 19 Dec 2024 10:35:24 -0600 Subject: [PATCH 319/421] unrolling categories result in slower calculation because of different s, m values for diffferent categories --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 8 + libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 252 +++++++++++++---------- libhmsbeagle/GPU/cuda_ops.h | 9 + 3 files changed, 156 insertions(+), 113 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 5192d3ab..358c449c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -144,6 +144,14 @@ struct DnMatrixDevice MemcpyDeviceToDevice(ptr, D.ptr, size()); } + void copyFromAsync(const DnMatrixDevice& D) + { + assert(order == D.order); + assert(size1 == D.size1); + assert(size2 == D.size2); + MemcpyDeviceToDevice(ptr, D.ptr, size()); + } + // Disallow copying -- only one object can "own" the descriptor. DnMatrixDevice& operator=(const DnMatrixDevice&) = delete; // Allow moving. diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 520da22a..f155d08b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -765,11 +765,6 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde for (int category = 0; category < kCategoryCount; category++) { - const int partial1Index = getPartialIndex(partials1Index, category); - const int partial2Index = getPartialIndex(partials2Index, category); - - const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); - const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; @@ -781,7 +776,7 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde edgeIndex2, category, matrixIndex2, false, false); } -// simpleAction3(partials1Index, edgeIndex1, partials2Index, edgeIndex2); + // simpleAction3(partials1Index, edgeIndex1, partials2Index, edgeIndex2); for (int category = 0; category < kCategoryCount; category++) { @@ -1169,9 +1164,14 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& #endif for (int j = 1; j < m + 1; j++) { -//#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"j/m = "<(integrationTmp, t / ((Real) s * j), A, destP, 0, integrationBuffer[category], integrationBufferSize[category]); @@ -1179,6 +1179,13 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& F += destP; +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"After destP = \n" << destP << "\n"; + std::cerr<<"Action2 Step 2, integrationTmp =\n"<::simpleAction3(int partialsIndex1, i const double t = 1.0; const int nCol = kPaddedStateCount * kPaddedPatternCount; + + int m_max = 0, s_max = 0; for (int category = 0; category < kCategoryCount; category++) { const int edgeMultiplierIndex1 = edgeIndex1 * kCategoryCount + category; const double edgeMultiplier1 = hEdgeMultipliers[edgeMultiplierIndex1]; msCache[category] = getStatistics2(t, nCol, edgeMultiplier1, hEigenMaps[edgeIndex1]); auto [m1, s1] = msCache[category]; - etaCache[category] = exp(t * hMuBs[hEigenMaps[edgeIndex1]] * edgeMultiplier1 / (Real) s1); if (m_max < m1) m_max = m1; if (s_max < s1) s_max = s1; @@ -1220,153 +1228,171 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i const double edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; msCache[kCategoryCount + category] = getStatistics2(t, nCol, edgeMultiplier2, hEigenMaps[edgeIndex2]); auto [m2, s2] = msCache[kCategoryCount + category]; - etaCache[kCategoryCount + category] = exp(t * hMuBs[hEigenMaps[edgeIndex2]] * edgeMultiplier2 / (Real) s2); if (m_max < m2) m_max = m2; if (s_max < s2) s_max = s2; } +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"s_max = "<(msCache[i])<<", "<(msCache[i])<<"), "; - std::cerr<<"("<(msCache[i + kCategoryCount])<<", "<(msCache[i + kCategoryCount])<<"), "; - } -#endif + auto& destPLeft = getPartialsCacheWrapper(partialsIndex1, category); + auto& destPRight = getPartialsCacheWrapper(partialsIndex2, category); - const Real zero = 0; - const Real one = 1; - for (int i = 0; i < s_max; i++) { + auto& inPartialsLeft = getPartialsWrapper(partialsIndex1, category); + auto& inPartialsRight = getPartialsWrapper(partialsIndex2, category); - for (int category = 0; category < kCategoryCount; category++) { - const Real c1Left = normPInf(dPartialsWrapper[getPartialIndex(partialsIndex1, category)]); - c1Cache[category] = c1Left; - const Real c1Right = normPInf(dPartialsWrapper[getPartialIndex(partialsIndex2, category)]); - c1Cache[kCategoryCount + category] = c1Right; + destPLeft.copyFromAsync( inPartialsLeft ); + destPRight.copyFromAsync( inPartialsRight ); + + FLeft.copyFromAsync( inPartialsLeft ); + FRight.copyFromAsync( inPartialsRight ); - etaCache[category] = (i < std::get<1>(msCache[category])) * (etaCache[category] - 1) + 1; - etaCache[kCategoryCount + category] = (i < std::get<1>(msCache[kCategoryCount + category])) * (etaCache[kCategoryCount + category] - 1) + 1; - } - std::fill(integrationMultipliers.begin(), integrationMultipliers.end(), 1.0); #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"s/s_max = "<(dIntegrationTmpLeft[category], alphaCache[category], dAs[hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex1, category)], 0, dIntegrationLeftBuffer[category], integrationLeftBufferSize[category]); - spMM(dIntegrationTmpRight[category], alphaCache[kCategoryCount + category], dAs[hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category], dPartialsWrapper[getPartialCacheIndex(partialsIndex2, category)], 0, dIntegrationRightBuffer[category], integrationRightBufferSize[category]); + auto& destPLeft = getPartialsCacheWrapper(partialsIndex1, category); + auto& destPRight = getPartialsCacheWrapper(partialsIndex2, category); - } - cudaDeviceSynchronize(); + spMM(dIntegrationTmpLeft[category], t / ((Real) s_max * j), ALeft, destPLeft, 0, dIntegrationLeftBuffer[category], integrationLeftBufferSize[category]); + spMM(dIntegrationTmpRight[category], t / ((Real) s_max * j), ARight, destPRight, 0, dIntegrationRightBuffer[category], integrationRightBufferSize[category]); #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"integrationTmp = alpha * A * destP = " <::value) { - CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeft[0].ptr, 1, dFLeft[0].ptr, 1)); - CUBLAS_CHECK(cublasSaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRight[0].ptr, 1, dFRight[0].ptr, 1)); - } else { - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpLeft[0].ptr, 1, dFLeft[0].ptr, 1)); - CUBLAS_CHECK(cublasDaxpy(cublasHandle, kPaddedStateCount * kPaddedPatternCount * kCategoryCount, &one, dIntegrationTmpRight[0].ptr, 1, dFRight[0].ptr, 1)); - } + // std::cerr<<"ARight =\n" << ARight << "\n"; + // std::cerr<<"destPRight = \n" << destPRight << "\n"; + // std::cerr<<"dIntegrationTmpRight = \n" << dIntegrationTmpRight[category] << "\n"; +#endif + } cudaDeviceSynchronize(); + + for (int category = 0; category < kCategoryCount; category++) { - c2Cache[category] = normPInf(dIntegrationTmpLeft[category]); - c2Cache[kCategoryCount + category] = normPInf(dIntegrationTmpRight[category]); - if (c1Cache[category] + c2Cache[category] < tol * normPInf(dFLeft[category])) { - integrationMultipliers[category] = 0; - } - if (c1Cache[kCategoryCount + category] + c2Cache[kCategoryCount + category] < tol * normPInf(dFRight[category])) { - integrationMultipliers[kCategoryCount + category] = 0; - } - c1Cache[category] = c2Cache[category]; - c1Cache[kCategoryCount + category] = c2Cache[kCategoryCount + category]; - } + auto& FLeft = dFLeft[category]; + auto& FRight = dFRight[category]; + + auto& destPLeft = getPartialsCacheWrapper(partialsIndex1, category); + auto& destPRight = getPartialsCacheWrapper(partialsIndex2, category); #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"dFLeftCache = " <::value) { - CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[category], dFLeft[category].ptr, 1)); - CUBLAS_CHECK(cublasSscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[kCategoryCount + category], dFRight[category].ptr, 1)); - } else { - CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[category], dFLeft[category].ptr, 1)); - CUBLAS_CHECK(cublasDscal(cublasHandle, kPaddedStateCount * kPaddedPatternCount, &etaCache[kCategoryCount + category], dFRight[category].ptr, 1)); - } + + const int edgeMultiplierIndex1 = edgeIndex1 * kCategoryCount + category; + const double edgeMultiplier1 = hEdgeMultipliers[edgeMultiplierIndex1]; + const Real eta1 = exp(etaConst1 * edgeMultiplier1); + const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; + const double edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; + const Real eta2 = exp(etaConst2 * edgeMultiplier2); + + auto& FLeft = dFLeft[category]; + auto& FRight = dFRight[category]; + + FLeft *= eta1; + FRight *= eta2; + } cudaDeviceSynchronize(); -// destP = F; - CHECK_CUDA(cudaMemcpyAsync(dPartialsWrapper[getPartialCacheIndex(partialsIndex1, 0)].ptr, dFLeft[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) - CHECK_CUDA(cudaMemcpyAsync(dPartialsWrapper[getPartialCacheIndex(partialsIndex2, 0)].ptr, dFRight[0].ptr, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * kCategoryCount, cudaMemcpyDeviceToDevice)) + + for (int category = 0; category < kCategoryCount; category++) { + + + auto& FLeft = dFLeft[category]; + auto& FRight = dFRight[category]; + + auto& destPLeft = getPartialsCacheWrapper(partialsIndex1, category); + auto& destPRight = getPartialsCacheWrapper(partialsIndex2, category); + + destPLeft.copyFromAsync( FLeft ); + destPRight.copyFromAsync( FRight ); + } cudaDeviceSynchronize(); } return BEAGLE_SUCCESS; } + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPrePartials(bool byPartition, const int *operations, diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 6f5827cd..dfb083d1 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -39,6 +39,15 @@ void MemcpyDeviceToDevice(T* dptr, const T* hptr, int n) throw std::runtime_error("cudaMemcpy(Device->Device): " + std::string(cudaGetErrorString(status))); } +template +void MemcpyDeviceToDeviceAsync(T* dptr, const T* hptr, int n) +{ + auto status = cudaMemcpyAsync(dptr, hptr, n*sizeof(T), cudaMemcpyDeviceToDevice); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpyAsync(Device->Device): " + std::string(cudaGetErrorString(status))); +} + + template void MemcpyDeviceToHost(T* hptr, const T* dptr, int n) { From 0a5376e73bc5608b080496d76ee56b6138166958 Mon Sep 17 00:00:00 2001 From: xji3 Date: Thu, 19 Dec 2024 17:56:27 -0600 Subject: [PATCH 320/421] another GPU L1 norm calculation, not working yet --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 92 +++++++++++++++++++++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 7 +- libhmsbeagle/GPU/cuda_ops.cu | 19 +++++ libhmsbeagle/GPU/cuda_ops.h | 3 + 4 files changed, 117 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 358c449c..4693f2ef 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -685,7 +685,6 @@ struct GPUnormest1 return *this; } - Real operator()(const SpMatrixDevice& A) { // std::cerr<<"n = "< +struct GPUnormest2 +{ + int p = 0; + int n = 0; + Real const one = Real(1); + DnMatrixDevice Y; + + DnMatrixDevice Ones; // (n,1) + + SpMatrixDevice Aabs; // abs(A) + + // Used for spMM + void* buffer = nullptr; + size_t buffer_size = 0; + + + GPUnormest2& operator=(const GPUnormest2&) = delete; + GPUnormest2& operator=(GPUnormest2&& g) + { + std::swap(p, g.p); + std::swap(n, g.n); + std::swap(Ones, g.Ones); + std::swap(Aabs, g.Aabs); + + // ensure that (*this) and (g) don't both own the buffers! + std::swap(buffer, g.buffer); + std::swap(buffer_size, g.buffer_size); + + return *this; + } + + Real operator()(const SpMatrixDevice& A) + { + // std::cerr<<"n = "<(Y, one, Aabs, Ones, 0, buffer, buffer_size); + + // Step 3, find max + int index = 0; + double norm; + + CUBLAS_CHECK(cublasIdamax(A.cublasHandle, n, Y.ptr, one, &index)); + + cudaMemcpy(&norm, Y.ptr + index -1, sizeof(Real), cudaMemcpyDeviceToHost); + return norm; + } + + + GPUnormest2(GPUnormest2&& g) + { + operator=(std::move(g)); + } + + GPUnormest2(const GPUnormest2&) = delete; + + // SpMatrixDevice(cublasHandle_t h1, cusparseHandle_t h2, int s1, int s2, int n, Real* v, int* c, int* o, sparseFormat f) + + GPUnormest2(cublasHandle_t cb, cusparseHandle_t cs, int p_, int n_) + :p(p_), n(n_), Y(cb, n, 1), Aabs(cb, cs, n, n, 0, cudaDeviceNew(n * n), cudaDeviceNew(n * n), cudaDeviceNew(n + 1), sparseFormat::csr), Ones(cb, n, 1, CUSPARSE_ORDER_COL) + { + assert(p >= 0); + + cudaMemset(Ones.ptr, Real(1), Ones.size1*Ones.size2*sizeof(Real)); + + buffer = cudaDeviceNew(n * n); + buffer_size = n * n; + + } + + ~GPUnormest2() + { + cudaDeviceDelete(buffer); + } +}; + //template @@ -899,7 +987,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector> dIntegrationTmpLeft; std::vector> dIntegrationTmpRight; std::vector> dAs; - std::vector> L1normForPower; + std::vector> L1normForPower; std::vector integrationLeftBufferSize; std::vector dIntegrationLeftBuffer; std::vector integrationRightBufferSize; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index f155d08b..39a8743e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -308,8 +308,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dAs = std::vector>(kEigenDecompCount * kCategoryCount * 2); for(int p=0; p<=getPMax()+1; p++) { - int t = 5; - L1normForPower.push_back(GPUnormest1(cublasHandle, p, kPaddedStateCount, t)); + // int t = 5; + // L1normForPower.push_back(GPUnormest1(cublasHandle, p, kPaddedStateCount, t)); + L1normForPower.push_back(GPUnormest2(cublasHandle, cusparseHandle, p, kPaddedStateCount)); } currentCacheNNZs = std::vector(kEigenDecompCount, kPaddedStateCount); @@ -1011,6 +1012,8 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub assert( edgeMultiplier >= 0 ); assert( eigenIndex >= 0); + std::cerr<<"Calculating statistics\n"; + if (t * hB1Norms[eigenIndex] == 0.0) return {0, 1}; diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 701b2bd4..e9a235c2 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -11,6 +11,7 @@ #include #include #include +#include void cuda_log_vector(double* v, int length) { @@ -117,6 +118,24 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer_) return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); } +float cuda_vec_abs(float *values, int n, float *results) +{ + using namespace thrust::placeholders; + + thrust::transform(values, values + n, results, [] __host__ __device__ (float x) {return std::abs(x);}); + + return 0; +} + +double cuda_vec_abs(double* values, int n, double* results) +{ + using namespace thrust::placeholders; + + thrust::transform(values, values + n, results, [] __host__ __device__ (double x) {return std::abs(x);}); + + return 0; +} + void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr) { using namespace thrust::placeholders; diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index dfb083d1..a2d537c8 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -90,6 +90,9 @@ void initialize_norm_x_matrix(double* data, int n, int m); float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); +float cuda_vec_abs(float *values, int n, float *results); +double cuda_vec_abs(double* values, int n, double *results); + void cuda_sort_indices_by_vector(const float* values_ptr, int n, int* indices_ptr); void cuda_sort_indices_by_vector(const double* values_ptr, int n, int* indices_ptr); From 65dc5a3f48a719d8c82200e6d951282340e2737d Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 20 Dec 2024 16:09:21 -0600 Subject: [PATCH 321/421] another L1 norm --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 39 ++++++++++++++------ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 2 - libhmsbeagle/GPU/CMake_Action/CMakeLists.txt | 2 + libhmsbeagle/GPU/cuda_ops.cu | 36 ++++++++++++++++-- libhmsbeagle/GPU/cuda_ops.h | 3 ++ 5 files changed, 66 insertions(+), 16 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 4693f2ef..b2ab6bc7 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -316,6 +316,10 @@ struct SpMatrixDevice operator=(std::move(D)); } + void setValues(Real *newValues) { + values = newValues; + } + SpMatrixDevice() = default; SpMatrixDevice(cublasHandle_t h1, cusparseHandle_t h2, int s1, int s2, int n, Real* v, int* c, int* o, sparseFormat f) :cublasHandle(h1), cusparseHandle(h2), size1(s1), size2(s2), num_non_zeros(n), values(v), inner(c), offsets(o), format(f) @@ -789,16 +793,19 @@ struct GPUnormest2 int p = 0; int n = 0; Real const one = Real(1); - DnMatrixDevice Y; + Real const zero = Real(0); + DnMatrixDevice Ones; // (n,1) - SpMatrixDevice Aabs; // abs(A) + DnMatrixDevice Y; // Used for spMM void* buffer = nullptr; size_t buffer_size = 0; + Real *valuesBuffer = nullptr; + GPUnormest2& operator=(const GPUnormest2&) = delete; GPUnormest2& operator=(GPUnormest2&& g) @@ -806,11 +813,12 @@ struct GPUnormest2 std::swap(p, g.p); std::swap(n, g.n); std::swap(Ones, g.Ones); - std::swap(Aabs, g.Aabs); + std::swap(Y, g.Y); // ensure that (*this) and (g) don't both own the buffers! std::swap(buffer, g.buffer); std::swap(buffer_size, g.buffer_size); + std::swap(valuesBuffer, g.valuesBuffer); return *this; } @@ -825,20 +833,30 @@ struct GPUnormest2 //std::cerr<<"A = "< #include #include -#include void cuda_log_vector(double* v, int length) { @@ -118,11 +117,37 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer_) return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); } +float cuda_vec_fill(float* values, int length, float fill) { + using namespace thrust::placeholders; + + thrust::device_ptr valuesPtr = thrust::device_pointer_cast(values); + thrust::fill(valuesPtr, valuesPtr + length, fill); + + return 0; +} + +double cuda_vec_fill(double* values, int length, double fill) { + using namespace thrust::placeholders; + + thrust::device_ptr valuesPtr = thrust::device_pointer_cast(values); + thrust::fill(valuesPtr, valuesPtr + length, fill); + + return 0; +} + + float cuda_vec_abs(float *values, int n, float *results) { using namespace thrust::placeholders; - thrust::transform(values, values + n, results, [] __host__ __device__ (float x) {return std::abs(x);}); + + thrust::device_ptr valuesPtr = thrust::device_pointer_cast(values); + thrust::device_ptr resultsPtr = thrust::device_pointer_cast(results); + + thrust::transform(valuesPtr, valuesPtr + n, resultsPtr, [] __device__ (float x) {return abs(x);}); + + + // thrust::transform(values, values + n, results, [] __host__ __device__ (float x) {return std::abs(x);}); return 0; } @@ -131,7 +156,12 @@ double cuda_vec_abs(double* values, int n, double* results) { using namespace thrust::placeholders; - thrust::transform(values, values + n, results, [] __host__ __device__ (double x) {return std::abs(x);}); + // thrust::transform(values, values + n, results, [] __host__ __device__ (double x) {return std::abs(x);}); + + thrust::device_ptr valuesPtr = thrust::device_pointer_cast(values); + thrust::device_ptr resultsPtr = thrust::device_pointer_cast(results); + + thrust::transform(valuesPtr, valuesPtr + n, resultsPtr, [] __device__ (double x) {return abs(x);}); return 0; } diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index a2d537c8..4c36c6d9 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -90,6 +90,9 @@ void initialize_norm_x_matrix(double* data, int n, int m); float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); +float cuda_vec_fill(float* values, int length, float fill); +double cuda_vec_fill(double* values, int length, double fill); + float cuda_vec_abs(float *values, int n, float *results); double cuda_vec_abs(double* values, int n, double *results); From db477bb0933f0e1ac649e71d0deb16f23b6b0374 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 14 Jan 2025 16:53:38 -0500 Subject: [PATCH 322/421] Switch back to original GPUnormest1 for now. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 +- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index b2ab6bc7..847eccb0 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -1004,7 +1004,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector> dIntegrationTmpLeft; std::vector> dIntegrationTmpRight; std::vector> dAs; - std::vector> L1normForPower; + std::vector> L1normForPower; std::vector integrationLeftBufferSize; std::vector dIntegrationLeftBuffer; std::vector integrationRightBufferSize; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 63e40550..cd67734b 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -308,9 +308,11 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dAs = std::vector>(kEigenDecompCount * kCategoryCount * 2); for(int p=0; p<=getPMax()+1; p++) { - // int t = 5; - // L1normForPower.push_back(GPUnormest1(cublasHandle, p, kPaddedStateCount, t)); - L1normForPower.push_back(GPUnormest2(cublasHandle, cusparseHandle, p, kPaddedStateCount)); + int t = 5; + L1normForPower.push_back(GPUnormest1(cublasHandle, p, kPaddedStateCount, t)); + +// This is very fast, but computes |A| instead of |A^p| +// L1normForPower.push_back(GPUnormest2(cublasHandle, cusparseHandle, p, kPaddedStateCount)); } currentCacheNNZs = std::vector(kEigenDecompCount, kPaddedStateCount); From 67683915f335acaa3ae098601e33bfd6c996caf6 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 14 Jan 2025 16:54:11 -0500 Subject: [PATCH 323/421] Add draft for cuda_max_l1_norm_to_device( ) --- libhmsbeagle/GPU/cuda_ops.cu | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index e0faea61..b1503ae6 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -1,6 +1,7 @@ #include "cuda_ops.h" #include +#include #include #include #include @@ -117,6 +118,38 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer_) return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); } +void cuda_max_l1_norm_to_device(double* values, int n, int t, double* buffer_, double* out_) +{ + using namespace thrust::placeholders; + + // 1. First sum the absolute values in each column and place the results into buffer + auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); + + auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values), + [] __host__ __device__ (double x) {return std::abs(x);} ); + + auto buffer = thrust::device_pointer_cast(buffer_); + + thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) + in_values_start, // values to reduce (with abs applied) + thrust::make_discard_iterator(), // key values out + buffer, // reduced values out + thrust::equal_to() // compare-keys operation + ); // summation is the default operation. + + // 2. Second maximize over the column sums and return the highest. + auto out = thrust::device_pointer_cast(buffer); + + auto same_keys_start = thrust::make_constant_iterator(0); + + thrust::reduce_by_key(same_keys_start, same_keys_start + t, // key indices (all same group) + buffer, // values to maximize + thrust::make_discard_iterator(), // key values out + out, // maximized value out + thrust::equal_to(), // compare-keys operation + thrust::maximum()); // maximize, not sum +} + float cuda_vec_fill(float* values, int length, float fill) { using namespace thrust::placeholders; From 93c50d5d66f9789c97a9b792c46ea74787e374f9 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 21 Jan 2025 16:14:10 -0500 Subject: [PATCH 324/421] Share code between float and double versions in cuda_ops --- libhmsbeagle/GPU/cuda_ops.cu | 111 +++++++++++++++++++++++------------ libhmsbeagle/GPU/cuda_ops.h | 6 ++ 2 files changed, 78 insertions(+), 39 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index b1503ae6..3e94547c 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -13,68 +13,93 @@ #include #include -void cuda_log_vector(double* v, int length) -{ - thrust::device_ptr vdptr = thrust::device_pointer_cast(v); +using thrust::device_ptr; +using thrust::device_pointer_cast; - // In-place update is accomplished by making the output iterator the same the starting input iterator. - thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (double x) {return log(x);}); +template +void cuda_log_vector(device_ptr v, int length) +{ + thrust::transform(v, v + length, v, [] __device__ (T x) {return log(x);}); } void cuda_log_vector(float* v, int length) { - thrust::device_ptr vdptr = thrust::device_pointer_cast(v); + cuda_log_vector(device_pointer_cast(v), length); +} +void cuda_log_vector(double* v, int length) +{ + cuda_log_vector(device_pointer_cast(v), length); +} + + +template +void cuda_sign_vector(device_ptr v, int n, int t) +{ // In-place update is accomplished by making the output iterator the same the starting input iterator. - thrust::transform(vdptr, vdptr + length, vdptr, [] __device__ (float x) {return log(x);}); + thrust::transform(v, v + n*t, v, [n] __device__ (T x) -> T {return (x<0)?-1.0/n:1.0/n;}); } void cuda_sign_vector(double* v, int n, int t) { - thrust::device_ptr vdptr = thrust::device_pointer_cast(v); - - // In-place update is accomplished by making the output iterator the same the starting input iterator. - thrust::transform(vdptr, vdptr + n*t, vdptr, [n] __device__ (double x) -> double {return (x<0)?-1.0/n:1.0/n;}); + cuda_sign_vector(device_pointer_cast(v), n, t); } void cuda_sign_vector(float* v, int n, int t) { - thrust::device_ptr vdptr = thrust::device_pointer_cast(v); - - // In-place update is accomplished by making the output iterator the same the starting input iterator. - thrust::transform(vdptr, vdptr + n*t, vdptr, [n] __device__ (float x) ->float {return (x<0)?-1.0/n:1.0/n;}); + cuda_sign_vector(device_pointer_cast(v), n, t); } -double cuda_max_abs(double* values, int length) +template +// This seems to be slow because it inserts cudaStreamSynchronize() +T cuda_max_abs(device_ptr values, int length) { - thrust::device_ptr values_ptr = thrust::device_pointer_cast(values); - - auto in_ptr = thrust::transform_iterator(values_ptr, [] __host__ __device__ (double x) {return std::abs(x);}); + auto in_ptr = thrust::transform_iterator(values, [] __host__ __device__ (T x) {return std::abs(x);}); // Don't use `*(thrust::max_element(....))` -- it is slower than thrust::reduce. // thrust::reduce is similar in speed to cublasI{s,d}amax. - return thrust::reduce(in_ptr, in_ptr + length, 0.0, thrust::maximum()); + // using thrust::reduce requires a device-to-host-memcpy, so it is slow! + return thrust::reduce(in_ptr, in_ptr + length, 0.0, thrust::maximum()); } -// This seems to be slow because it inserts cudaStreamSynchronize() float cuda_max_abs(float* values, int length) { - thrust::device_ptr values_ptr = thrust::device_pointer_cast(values); + return cuda_max_abs(device_pointer_cast(values), length); +} - auto in_ptr = thrust::transform_iterator(values_ptr, [] __host__ __device__ (float x) {return std::abs(x);}); +double cuda_max_abs(double* values, int length) +{ + return cuda_max_abs(device_pointer_cast(values), length); +} - // Don't use `*(thrust::max_element(....))` -- it is slower than thrust::reduce. +template +void cuda_max(device_ptr values, int length, device_ptr out) +{ + auto same_keys_start = thrust::make_constant_iterator(0); - // thrust::reduce is similar in speed to cublasI{s,d}amax. - return thrust::reduce(in_ptr, in_ptr + length, 0.0, thrust::maximum()); + thrust::reduce_by_key(same_keys_start, same_keys_start + length, // key indices (all same group) + values, // values to maximize + thrust::make_discard_iterator(), // key values out + out, // maximized value out + thrust::equal_to(), // compare-keys operation + thrust::maximum()); // maximize, not sum +} + +void cuda_max(float* values, int length, float* out) +{ + cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out)); +} + +void cuda_max(double* values, int length, double* out) +{ + cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out)); } float cuda_max_l1_norm(float* values, int n, int t, float* buffer_) { using namespace thrust::placeholders; - // 1. First sum the absolute values in each column and place the results into buffer auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); @@ -118,7 +143,12 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer_) return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); } -void cuda_max_l1_norm_to_device(double* values, int n, int t, double* buffer_, double* out_) +template +void cuda_max_l1_norm(device_ptr values, + int n, + int t, + device_ptr buffer, + device_ptr out) { using namespace thrust::placeholders; @@ -126,9 +156,7 @@ void cuda_max_l1_norm_to_device(double* values, int n, int t, double* buffer_, d auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values), - [] __host__ __device__ (double x) {return std::abs(x);} ); - - auto buffer = thrust::device_pointer_cast(buffer_); + [] __host__ __device__ (T x) {return std::abs(x);} ); thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) in_values_start, // values to reduce (with abs applied) @@ -138,16 +166,21 @@ void cuda_max_l1_norm_to_device(double* values, int n, int t, double* buffer_, d ); // summation is the default operation. // 2. Second maximize over the column sums and return the highest. - auto out = thrust::device_pointer_cast(buffer); + cuda_max(buffer, t, out); +} - auto same_keys_start = thrust::make_constant_iterator(0); +void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out) +{ + using namespace thrust; - thrust::reduce_by_key(same_keys_start, same_keys_start + t, // key indices (all same group) - buffer, // values to maximize - thrust::make_discard_iterator(), // key values out - out, // maximized value out - thrust::equal_to(), // compare-keys operation - thrust::maximum()); // maximize, not sum + cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out)); +} + +void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out) +{ + using namespace thrust; + + cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out)); } float cuda_vec_fill(float* values, int length, float fill) { diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 4c36c6d9..6ed2a278 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -77,6 +77,9 @@ void cuda_sign_vector(float* v, int n, int t); double cuda_max_abs(double* v, int length); float cuda_max_abs(float* v, int length); +void cuda_max(double* v, int length, double* out); +void cuda_max(float* v, int length, float* out); + // max over (n,t) -> (n,1). Matrix assumed to be column-major. void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr); void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr); @@ -90,6 +93,9 @@ void initialize_norm_x_matrix(double* data, int n, int m); float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); +void cuda_max_l1_norm(float* values, int n, int t, float* buffer, double* out); +void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out); + float cuda_vec_fill(float* values, int length, float fill); double cuda_vec_fill(double* values, int length, double fill); From 94b4a7cd0bad6efbd1fb98a6034ae8c052fff485 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 21 Jan 2025 16:21:03 -0500 Subject: [PATCH 325/421] Unify more code. --- libhmsbeagle/GPU/cuda_ops.cu | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 3e94547c..00c133b5 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -96,7 +96,8 @@ void cuda_max(double* values, int length, double* out) cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out)); } -float cuda_max_l1_norm(float* values, int n, int t, float* buffer_) +template +T cuda_max_l1_norm(device_ptr values, int n, int t, device_ptr buffer) { using namespace thrust::placeholders; @@ -104,9 +105,7 @@ float cuda_max_l1_norm(float* values, int n, int t, float* buffer_) auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values), - [] __host__ __device__ (float x) {return std::abs(x);} ); - - auto buffer = thrust::device_pointer_cast(buffer_); + [] __host__ __device__ (T x) {return std::abs(x);} ); thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) in_values_start, // values to reduce (with abs applied) @@ -119,28 +118,14 @@ float cuda_max_l1_norm(float* values, int n, int t, float* buffer_) return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); } -double cuda_max_l1_norm(double* values, int n, int t, double* buffer_) +float cuda_max_l1_norm(float* values, int n, int t, float* buffer) { - using namespace thrust::placeholders; - - - // 1. First sum the absolute values in each column and place the results into buffer - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); - - auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values), - [] __host__ __device__ (double x) {return std::abs(x);} ); - - auto buffer = thrust::device_pointer_cast(buffer_); - - thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) - in_values_start, // values to reduce (with abs applied) - thrust::make_discard_iterator(), // key values out - buffer, // reduced values out - thrust::equal_to() // compare-keys operation - ); // summation is the default operation. + return cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer)); +} - // 2. Second maximize over the column sums and return the highest. - return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); +double cuda_max_l1_norm(double* values, int n, int t, double* buffer) +{ + return cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer)); } template From 26ea0a8e928c80d4f8add51ed2039dbcc8b1df56 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 21 Jan 2025 16:27:24 -0500 Subject: [PATCH 326/421] Unduplicate more code. --- libhmsbeagle/GPU/cuda_ops.cu | 53 ++++++++++++------------------------ libhmsbeagle/GPU/cuda_ops.h | 8 +++--- 2 files changed, 22 insertions(+), 39 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 00c133b5..e09de404 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -168,53 +168,36 @@ void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out) cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out)); } -float cuda_vec_fill(float* values, int length, float fill) { - using namespace thrust::placeholders; - - thrust::device_ptr valuesPtr = thrust::device_pointer_cast(values); - thrust::fill(valuesPtr, valuesPtr + length, fill); - - return 0; -} -double cuda_vec_fill(double* values, int length, double fill) { +template +void cuda_vec_fill(device_ptr values, int length, T fill) { using namespace thrust::placeholders; - thrust::device_ptr valuesPtr = thrust::device_pointer_cast(values); - thrust::fill(valuesPtr, valuesPtr + length, fill); + thrust::fill(values, values + length, fill); +} - return 0; +void cuda_vec_fill(float* values, int length, float fill) { + cuda_vec_fill(device_pointer_cast(values), length, fill); } +void cuda_vec_fill(double* values, int length, double fill) { + cuda_vec_fill(device_pointer_cast(values), length, fill); +} -float cuda_vec_abs(float *values, int n, float *results) +template +void cuda_vec_abs(device_ptr values, int n, device_ptr results) { - using namespace thrust::placeholders; - - - thrust::device_ptr valuesPtr = thrust::device_pointer_cast(values); - thrust::device_ptr resultsPtr = thrust::device_pointer_cast(results); - - thrust::transform(valuesPtr, valuesPtr + n, resultsPtr, [] __device__ (float x) {return abs(x);}); - - - // thrust::transform(values, values + n, results, [] __host__ __device__ (float x) {return std::abs(x);}); - - return 0; + thrust::transform(values, values + n, results, [] __device__ (T x) {return abs(x);}); } -double cuda_vec_abs(double* values, int n, double* results) +void cuda_vec_abs(float* values, int n, float* results) { - using namespace thrust::placeholders; - - // thrust::transform(values, values + n, results, [] __host__ __device__ (double x) {return std::abs(x);}); - - thrust::device_ptr valuesPtr = thrust::device_pointer_cast(values); - thrust::device_ptr resultsPtr = thrust::device_pointer_cast(results); - - thrust::transform(valuesPtr, valuesPtr + n, resultsPtr, [] __device__ (double x) {return abs(x);}); + cuda_vec_abs(device_pointer_cast(values), n, device_pointer_cast(results)); +} - return 0; +void cuda_vec_abs(double* values, int n, double* results) +{ + cuda_vec_abs(device_pointer_cast(values), n, device_pointer_cast(results)); } void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr) diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 6ed2a278..c7333caf 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -96,11 +96,11 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); void cuda_max_l1_norm(float* values, int n, int t, float* buffer, double* out); void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out); -float cuda_vec_fill(float* values, int length, float fill); -double cuda_vec_fill(double* values, int length, double fill); +void cuda_vec_fill(float* values, int length, float fill); +void cuda_vec_fill(double* values, int length, double fill); -float cuda_vec_abs(float *values, int n, float *results); -double cuda_vec_abs(double* values, int n, double *results); +void cuda_vec_abs(float *values, int n, float *results); +void cuda_vec_abs(double* values, int n, double *results); void cuda_sort_indices_by_vector(const float* values_ptr, int n, int* indices_ptr); void cuda_sort_indices_by_vector(const double* values_ptr, int n, int* indices_ptr); From 9c82e5a3726d8a78473bd750441dc74820421e07 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 21 Jan 2025 16:37:06 -0500 Subject: [PATCH 327/421] Unduplicate initialize_norm_x_matrix --- libhmsbeagle/GPU/cuda_ops.cu | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index e09de404..5c387c7e 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -719,14 +719,15 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC } } -void initialize_norm_x_matrix(float* data, int n, int m) + +template +void initialize_norm_x_matrix(device_ptr out_ptr, int n, int m) { - auto out_ptr = thrust::device_pointer_cast(data); auto indices = thrust::counting_iterator(0); auto initialize = [n,m] __host__ __device__ (int i) { thrust::default_random_engine rng; - thrust::uniform_real_distribution dist(0, 1); + thrust::uniform_real_distribution dist(0, 1); if (i(0); - auto initialize = [n,m] __host__ __device__ (int i) - { - thrust::default_random_engine rng; - thrust::uniform_real_distribution dist(0, 1); - - if (i 0.5) - return 1.0/n; - else - return -1.0/n; - }; + initialize_norm_x_matrix(device_pointer_cast(data), n, m); +} - thrust::transform(indices, indices + n*m, // in - out_ptr, // out - initialize); +void initialize_norm_x_matrix(double* data, int n, int m) +{ + initialize_norm_x_matrix(device_pointer_cast(data), n, m); } From 28abcb11300bdb74851b11ac494f0f7e6fa8d724 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 21 Jan 2025 16:40:43 -0500 Subject: [PATCH 328/421] Unduplicate code for justMaximize( ). --- libhmsbeagle/GPU/cuda_ops.cu | 77 ++++++++---------------------------- 1 file changed, 16 insertions(+), 61 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 5c387c7e..d48be52a 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -304,11 +304,8 @@ void cuda_set_indices(double* x_ptr, int n, int t, const int* indices_ptr) thrust::for_each(iter, iter+t, [=] __host__ __device__ (int i) { x[n*i + indices[i]] = 1.0; }); } - -// FIXME: It would be nice to merge the code for the and versions of -// rescalePartialsDevice, but this was somehow causing the program to crash. - -void justMaximize(float* partials, float* scalingFactors, +template +void justMaximize(device_ptr partials, device_ptr scalingFactors, int nStates, int nPatterns, int nCategories) { using namespace thrust::placeholders; @@ -349,7 +346,7 @@ void justMaximize(float* partials, float* scalingFactors, // We need to compute j as a function of i: // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) auto in_values_start = thrust::make_permutation_iterator( - thrust::device_pointer_cast(partials), + partials, thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) ); @@ -360,68 +357,26 @@ void justMaximize(float* partials, float* scalingFactors, in_keys_start + partials_size, // key indices end in_values_start, // values to reduce (category, pattern, state) thrust::make_discard_iterator(), // key values out - thrust::device_pointer_cast(scalingFactors), // reduced values out + scalingFactors, // reduced values out thrust::equal_to(), // compare keys operation - thrust::maximum() // reduction operation + thrust::maximum() // reduction operation ); } -void justMaximize(double* partials, double* scalingFactors, +void justMaximize(float* partials, float* scalingFactors, int nStates, int nPatterns, int nCategories) { - using namespace thrust::placeholders; - - // 1. Surprisingly, cuBLAS has no operations that reduce (sum,maximize,minimize,etc.) rows or columns. - // It can only reduce an entire dense matrix to a single value. - - // 2. thrust::reduce_by_key is able to reduce regions of a vector down to MULTIPLE values. - // We assign each value a "key" that decides which group it is in. - // Adjacent values with the same key end up in the same group. - // However, non-adjacent values with the same key end up in different groups. - // thrust::reduce_by_key performs the reduction operation on each group. - - // 3. Thrust can maximize over states (adjacent memory_ and categories (non-adjacent) in one operation - // by changing the order in which entries of the input matrix are visited. All the entries in the - // same group need to be visited sequentially. We can do this using a permutation iterator. - - // Implementation notes: - // * the library MatX (https://github.com/NVIDIA/MatX) offers much nicer syntax, but thrust is older - // and more stable (in 2024) - - // The size of a partials buffer - size_t partials_size = nStates * nPatterns * nCategories; - - - // OK, so we can convert the input index i=[0...partials_size] to (p,c,s) as follows: - // s = i % nStates - // c = (i / nStates) % nCategories - // p = (i / nStates * nCategories) - - // The linear index into the input matrix is: - // j = s + nStates *p + (nStates*nPatterns)*c - - // A list of 00000..111111..222222........(P-1)(P-1)(P-1)(P-1) that groups values by state and categories. - // There should be nPatterns groups. - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates * nCategories))); - - // We need to compute j as a function of i: - // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) - auto in_values_start = thrust::make_permutation_iterator( - thrust::device_pointer_cast(partials), - thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), - (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) - ); + justMaximize(device_pointer_cast(partials), + device_pointer_cast(scalingFactors), + nStates, nPatterns, nCategories); +} - thrust::reduce_by_key( - // add execution policy thrust::cuda::par_nosync? - in_keys_start, // key indices start (group by pattern) - in_keys_start + partials_size, // key indices end - in_values_start, // values to reduce (category, pattern, state) - thrust::make_discard_iterator(), // key values out - thrust::device_pointer_cast(scalingFactors), // reduced values out - thrust::equal_to(), // compare keys operation - thrust::maximum() // reduction operation - ); +void justMaximize(double* partials, double* scalingFactors, + int nStates, int nPatterns, int nCategories) +{ + justMaximize(device_pointer_cast(partials), + device_pointer_cast(scalingFactors), + nStates, nPatterns, nCategories); } template From bce6cdae09a28e82090cb3a198bc9584c3d3b52e Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 21 Jan 2025 16:45:04 -0500 Subject: [PATCH 329/421] Unduplicate sumRootLikelihoods --- libhmsbeagle/GPU/cuda_ops.cu | 95 +++++++++--------------------------- 1 file changed, 24 insertions(+), 71 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index d48be52a..1d66aa14 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -388,8 +388,9 @@ struct Prod3 } }; -void sumRootLikelihoods(float* siteProbs, // OUT - float* likelihoods, float* weights, float* frequencies, // INT +template +void sumRootLikelihoods(device_ptr siteProbs, // OUT + device_ptr likelihoods, device_ptr weights, device_ptr frequencies, // IN int nStates, int nPatterns, int nCategories) { using namespace thrust::placeholders; @@ -413,7 +414,7 @@ void sumRootLikelihoods(float* siteProbs, // OUT // We need to compute j as a function of i: // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) auto in_lks_start = thrust::make_permutation_iterator( - thrust::device_pointer_cast(likelihoods), + likelihoods, thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) ); @@ -424,7 +425,7 @@ void sumRootLikelihoods(float* siteProbs, // OUT // k(i) = (i / nStates) % nCategories auto in_weights_start = thrust::make_permutation_iterator( - thrust::device_pointer_cast(weights), + weights, thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), (_1/nStates)%nCategories) ); @@ -435,7 +436,7 @@ void sumRootLikelihoods(float* siteProbs, // OUT // l(i) = (i % nStates) auto in_frequencies_start = thrust::make_permutation_iterator( - thrust::device_pointer_cast(frequencies), + frequencies, thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), _1 % nStates) ); @@ -453,82 +454,34 @@ void sumRootLikelihoods(float* siteProbs, // OUT in_keys_start + partials_size, // key indices end in_values_start, // values to reduce (category, pattern, state) thrust::make_discard_iterator(), // key values out - thrust::device_pointer_cast(siteProbs), // reduced values out + siteProbs, // reduced values out thrust::equal_to() // compare keys operation // Default operation is (+) ); } -void sumRootLikelihoods(double* siteProbs, // OUT - double* likelihoods, double* weights, double* frequencies, // INT +void sumRootLikelihoods(float* siteProbs, // OUT + float* likelihoods, float* weights, float* frequencies, // INT int nStates, int nPatterns, int nCategories) { - using namespace thrust::placeholders; - - // The size of a partials buffer - size_t partials_size = nStates * nPatterns * nCategories; - - - // OK, so we can convert the input index i=[0...partials_size] to (p,c,s) as follows: - // s = i % nStates - // c = (i / nStates) % nCategories - // p = (i / nStates * nCategories) - - // 1. A list of 00000..111111..222222........(P-1)(P-1)(P-1)(P-1) that groups values by state and categories. - // There should be nPatterns groups. - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates * nCategories))); - - // 2. The linear index into the likelihood matrix is: - // j = s + nStates *p + (nStates*nPatterns)*c - // - // We need to compute j as a function of i: - // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) - auto in_lks_start = thrust::make_permutation_iterator( - thrust::device_pointer_cast(likelihoods), - thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), - (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) - ); - - // 3. The linear index into the weights matrix is just: - // k = c - // Therefore - // k(i) = (i / nStates) % nCategories - - auto in_weights_start = thrust::make_permutation_iterator( - thrust::device_pointer_cast(weights), - thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), - (_1/nStates)%nCategories) - ); - - // 4. The linear index into the states matrix is: - // l = s - // Therefore - // l(i) = (i % nStates) - - auto in_frequencies_start = thrust::make_permutation_iterator( - thrust::device_pointer_cast(frequencies), - thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), - _1 % nStates) - ); - - - // 5. Make the sequence of likelihood * categoryWeight * stateFrequency - auto tuples = thrust::make_zip_iterator(thrust::make_tuple(in_lks_start, in_weights_start, in_frequencies_start)); + sumRootLikelihoods(device_pointer_cast(siteProbs), + device_pointer_cast(likelihoods), + device_pointer_cast(weights), + device_pointer_cast(frequencies), + nStates, nPatterns, nCategories); +} - auto in_values_start = thrust::make_transform_iterator(tuples, Prod3()); - // 6. Sum over state and category for each pattern. - thrust::reduce_by_key( - // add execution policy thrust::cuda::par_nosync? - in_keys_start, // key indices start (group by pattern) - in_keys_start + partials_size, // key indices end - in_values_start, // values to reduce (category, pattern, state) - thrust::make_discard_iterator(), // key values out - thrust::device_pointer_cast(siteProbs), // reduced values out - thrust::equal_to() // compare keys operation - // Default operation is (+) - ); +void sumRootLikelihoods(double* siteProbs, // OUT + double* likelihoods, double* weights, double* frequencies, // INT + int nStates, int nPatterns, int nCategories) +{ + sumRootLikelihoods(device_pointer_cast(siteProbs), + device_pointer_cast(likelihoods), + device_pointer_cast(weights), + device_pointer_cast(frequencies), + nStates, nPatterns, nCategories); } From d8a6999534cdc13e26fae1b28787c062f3392e4e Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 21 Jan 2025 16:51:25 -0500 Subject: [PATCH 330/421] Unduplicate rescalePartials2( ). --- libhmsbeagle/GPU/cuda_ops.cu | 124 ++++++++++------------------------- 1 file changed, 33 insertions(+), 91 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 1d66aa14..381264e6 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -484,21 +484,19 @@ void sumRootLikelihoods(double* siteProbs, // OUT nStates, nPatterns, nCategories); } - +template void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) + device_ptr partials, device_ptr scalingFactors, device_ptr cumulativeScalingBuffer, int streamIndex) { using namespace thrust::placeholders; size_t partialsSize = kCategoryCount * kPaddedPatternCount * kPaddedStateCount; - thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); - thrust::device_ptr scalingFactors2 = thrust::device_pointer_cast(scalingFactors); // 1. Find maximize partial likelihood -> scalingFactors[pattern] justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. - thrust::transform(scalingFactors2, scalingFactors2 + kPaddedPatternCount, scalingFactors2, [] __device__ (float x) { return (x == 0) ? 1.0 : x;}); + thrust::transform(scalingFactors, scalingFactors + kPaddedPatternCount, scalingFactors, [] __device__ (T x) { return (x == 0) ? 1.0 : x;}); // 3. Rescale each pattern by scalingFactors[pattern(index)] @@ -506,7 +504,7 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC // iter_max computes the scaling factor as a function of the index into the partials buffer. auto iter_max = thrust::make_permutation_iterator( - scalingFactors2, + scalingFactors, thrust::make_transform_iterator( thrust::make_counting_iterator(0), (_1/kPaddedStateCount) % kPaddedPatternCount @@ -514,117 +512,61 @@ void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternC ); - thrust::transform(partials2, partials2 + partialsSize, // in1 = partials[i] - iter_max, // in2 = scalingFactors[pattern(i)] - partials2, // out - thrust::divides() // operation + thrust::transform(partials, partials + partialsSize, // in1 = partials[i] + iter_max, // in2 = scalingFactors[pattern(i)] + partials, // out + thrust::divides() // operation ); // std::cerr<<"scalers_log = "< cumulative2 = thrust::device_pointer_cast(cumulativeScalingBuffer); - if (scalers_log) { - thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 - scalingFactors2, // in2 - cumulative2, // out - thrust::plus() // operation; + thrust::transform(cumulativeScalingBuffer, cumulativeScalingBuffer + kPaddedPatternCount, // in1 + scalingFactors, // in2 + cumulativeScalingBuffer, // out + thrust::plus() // operation; ); } else { - auto logScalingFactors2 = thrust::make_transform_iterator( scalingFactors2, - [] __host__ __device__ (float x) { return log(x); }); + auto logScalingFactors2 = thrust::make_transform_iterator( scalingFactors, + [] __host__ __device__ (T x) { return log(x); }); - thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 - logScalingFactors2, // in2 - cumulative2, // out - thrust::plus() // operation; + thrust::transform(cumulativeScalingBuffer, cumulativeScalingBuffer + kPaddedPatternCount, // in1 + logScalingFactors2, // in2 + cumulativeScalingBuffer, // out + thrust::plus() // operation; ); } } } -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) { - using namespace thrust::placeholders; - - size_t partialsSize = kCategoryCount * kPaddedPatternCount * kPaddedStateCount; - thrust::device_ptr partials2 = thrust::device_pointer_cast(partials); - thrust::device_ptr scalingFactors2 = thrust::device_pointer_cast(scalingFactors); - - // 1. Find maximize partial likelihood -> scalingFactors[pattern] - justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); - - // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. - thrust::transform(scalingFactors2, scalingFactors2 + kPaddedPatternCount, scalingFactors2, [] __device__ (double x) { return (x == 0) ? 1.0 : x;}); - - // 3. Rescale each pattern by scalingFactors[pattern(index)] - - // pattern = (i/kPaddedStateCount) % kPaddedPatternCount - - // iter_max computes the scaling factor as a function of the index into the partials buffer. - auto iter_max = thrust::make_permutation_iterator( - scalingFactors2, - thrust::make_transform_iterator( - thrust::make_counting_iterator(0), - (_1/kPaddedStateCount) % kPaddedPatternCount - ) - ); - - - thrust::transform(partials2, partials2 + partialsSize, // in1 = partials[i] - iter_max, // in2 = scalingFactors[pattern(i)] - partials2, // out - thrust::divides() // operation - ); - -// std::cerr<<"scalers_log = "< cumulative2 = thrust::device_pointer_cast(cumulativeScalingBuffer); + rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, + device_pointer_cast(partials), device_pointer_cast(scalingFactors), device_pointer_cast(cumulativeScalingBuffer), + streamIndex); +} - if (scalers_log) - { - thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 - scalingFactors2, // in2 - cumulative2, // out - thrust::plus() // operation; - ); - } - else - { - auto logScalingFactors2 = thrust::make_transform_iterator( scalingFactors2, - [] __host__ __device__ (double x) { return log(x); }); - thrust::transform(cumulative2, cumulative2 + kPaddedPatternCount, // in1 - logScalingFactors2, // in2 - cumulative2, // out - thrust::plus() // operation; - ); - } - } +void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, + double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) +{ + rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, + device_pointer_cast(partials), device_pointer_cast(scalingFactors), device_pointer_cast(cumulativeScalingBuffer), + streamIndex); } From abec137b587e06ece32d32cf9c675f17afe0e5a0 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 27 Jan 2025 16:51:16 -0500 Subject: [PATCH 331/421] Add additional buffers to struct GPUnormest1 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 847eccb0..e220d6eb 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -669,6 +669,12 @@ struct GPUnormest1 // Temporary storage for cuda_max_l1_norm Real* buffer2 = nullptr; + // Space to store the norms for the different iterations. + Real* buffer3 = nullptr; + + // Space to store a result + Real* buffer4 = nullptr; + GPUnormest1& operator=(const GPUnormest1&) = delete; GPUnormest1& operator=(GPUnormest1&& g) { @@ -685,6 +691,8 @@ struct GPUnormest1 std::swap(buffer_size, g.buffer_size); std::swap(indices, g.indices); std::swap(buffer2, g.buffer2); + std::swap(buffer3, g.buffer3); + std::swap(buffer4, g.buffer4); return *this; } @@ -774,6 +782,10 @@ struct GPUnormest1 buffer2 = cudaDeviceNew(t); + buffer3 = cudaDeviceNew(itmax); + + buffer4 = cudaDeviceNew(1); + indices = cudaDeviceNew(n); } @@ -781,6 +793,8 @@ struct GPUnormest1 { cudaDeviceDelete(buffer); cudaDeviceDelete(buffer2); + cudaDeviceDelete(buffer3); + cudaDeviceDelete(buffer4); cudaDeviceDelete(indices); } }; From b60986b2894b19e324f61ba1413d53ebf562a190 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Feb 2025 17:26:03 -0500 Subject: [PATCH 332/421] Reimplement cuda_max using cub::DeviceReduce::Max( ) --- libhmsbeagle/GPU/cuda_ops.cu | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 381264e6..2f872a40 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -1,5 +1,7 @@ #include "cuda_ops.h" +#include + #include #include #include @@ -73,17 +75,27 @@ double cuda_max_abs(double* values, int length) return cuda_max_abs(device_pointer_cast(values), length); } +// QUESTION: Why do cub::DeviceReduce and thrust::reduce need to perform allocation? +// That seems like a big problem. template void cuda_max(device_ptr values, int length, device_ptr out) { - auto same_keys_start = thrust::make_constant_iterator(0); + auto values_ptr = thrust::raw_pointer_cast(values); + auto out_ptr = thrust::raw_pointer_cast(out); + + // 1. Get size of temporary allocation, if any. + void* d_temp_storage = nullptr; + size_t temp_storage_bytes = 0; + cub::DeviceReduce::Max(d_temp_storage, temp_storage_bytes, values_ptr, out_ptr, length); + + // 2. Do temporary allocation, if needed. + + // Using a thrust::device_vector should ensure (i) no allocation for 0 butes and (ii) automatic deallocation if needed. + thrust::device_vector temp_storage(temp_storage_bytes); + d_temp_storage = thrust::raw_pointer_cast(temp_storage.data()); - thrust::reduce_by_key(same_keys_start, same_keys_start + length, // key indices (all same group) - values, // values to maximize - thrust::make_discard_iterator(), // key values out - out, // maximized value out - thrust::equal_to(), // compare-keys operation - thrust::maximum()); // maximize, not sum + // 3. Do the reduction. + cub::DeviceReduce::Max(d_temp_storage, temp_storage_bytes, values_ptr, out_ptr, length); } void cuda_max(float* values, int length, float* out) From 6c4ea043ca23ae7486d242e827c87e26202b8667 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Feb 2025 15:57:12 -0500 Subject: [PATCH 333/421] Add cuda_scratch_space object. --- libhmsbeagle/GPU/cuda_ops.cu | 120 +++++++++++++++++++++++++++++++++-- libhmsbeagle/GPU/cuda_ops.h | 30 +++++++-- 2 files changed, 139 insertions(+), 11 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 2f872a40..247cb8d8 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -108,6 +108,15 @@ void cuda_max(double* values, int length, double* out) cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out)); } +struct SumAbs +{ + template + __device__ __forceinline__ T operator()(const T& a, const T& b) const + { + return std::abs(a) + std::abs(b); + } +}; + template T cuda_max_l1_norm(device_ptr values, int n, int t, device_ptr buffer) { @@ -140,13 +149,93 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer) return cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer)); } + +void cuda_scratch_space::discard() +{ + if (device_ptr) + { + cudaDeviceDelete(device_ptr); + device_ptr = 0; + size = 0; + } + else + { + assert(device_ptr == nullptr); + assert(size == 0); + } +} + +void cuda_scratch_space::fit(size_t n) +{ + if (n > size) + { + std::cerr<<"cuda_scratch_space::fit( ): reallocating from "< void cuda_max_l1_norm(device_ptr values, int n, int t, device_ptr buffer, - device_ptr out) -{ + device_ptr out, + cuda_scratch_space& scratch) +{ + // cudaStreamSynchronize(); + + // Run reduction + int num_segments = t; + thrust::device_vector d_offsets = {0, n, 2*n}; // {0, n, 2n, 3n ,,,,} + auto d_offsets_it = thrust::raw_pointer_cast(d_offsets.data()); + + SumAbs sum_abs_op; + T initial_value = 0; + + // Determine temporary device storage requirements + size_t temp_storage_bytes; + + cub::DeviceSegmentedReduce::Reduce( + nullptr, + temp_storage_bytes, + values, + buffer, + num_segments, + d_offsets_it, + d_offsets_it + 1, + sum_abs_op, + initial_value); + + std::cerr<<"cuda_max_l1_norm: wants "< values, buffer, // reduced values out thrust::equal_to() // compare-keys operation ); // summation is the default operation. +*/ + // cudaStreamSynchronize(); // 2. Second maximize over the column sums and return the highest. cuda_max(buffer, t, out); + + // cudaStreamSynchronize(); } -void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out) +void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out, cuda_scratch_space& scratch) { using namespace thrust; - cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out)); + cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out), scratch); } -void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out) +void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch) { using namespace thrust; - cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out)); + cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out), scratch); +} + + +void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out) +{ + cuda_scratch_space scratch; + + cuda_max_l1_norm(values, n, t, buffer, out, scratch); +} + +void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out) +{ + cuda_scratch_space scratch; + + cuda_max_l1_norm(values, n, t, buffer, out, scratch); } diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index c7333caf..f9487504 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -5,16 +5,22 @@ #include // cudaMalloc, cudaMemcpy, etc. #include // for std::runtime_error -template -T* cudaDeviceNew(int n) + +inline void* cudaMallocWrapped(int n) { - T* result; - auto status = cudaMalloc((void**)&result, n*sizeof(T)); + void* result; + auto status = cudaMalloc((void**)&result, n); if (status != cudaSuccess) - throw std::runtime_error(std::string("cudaMalloc: ") + std::string(cudaGetErrorString(status))); + throw std::runtime_error(std::string("cudaMalloc: ") + std::string(cudaGetErrorString(status))); return result; } +template +T* cudaDeviceNew(int n) +{ + return (T*)cudaMallocWrapped(n*sizeof(T)); +} + template void cudaDeviceDelete(T* buffer) { @@ -66,6 +72,17 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) return host_vec; } +struct cuda_scratch_space +{ + void* device_ptr = nullptr; + size_t size = 0; + + void discard(); + void fit(size_t n); + + ~cuda_scratch_space(); +}; + void cuda_log_vector(double* v, int length); void cuda_log_vector(float* v, int length); @@ -96,6 +113,9 @@ double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); void cuda_max_l1_norm(float* values, int n, int t, float* buffer, double* out); void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out); +void cuda_max_l1_norm(float* values, int n, int t, float* buffer, double* out, cuda_scratch_space& scratch); +void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch); + void cuda_vec_fill(float* values, int length, float fill); void cuda_vec_fill(double* values, int length, double fill); From a404ab64d43662f169cc4111aa572be538332b35 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Feb 2025 16:34:59 -0500 Subject: [PATCH 334/421] Also use cuda_scratch_space in cuda_max --- libhmsbeagle/GPU/cuda_ops.cu | 21 +++++++++------------ libhmsbeagle/GPU/cuda_ops.h | 4 ++-- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 247cb8d8..a82f7669 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -78,7 +78,7 @@ double cuda_max_abs(double* values, int length) // QUESTION: Why do cub::DeviceReduce and thrust::reduce need to perform allocation? // That seems like a big problem. template -void cuda_max(device_ptr values, int length, device_ptr out) +void cuda_max(device_ptr values, int length, device_ptr out, cuda_scratch_space& scratch) { auto values_ptr = thrust::raw_pointer_cast(values); auto out_ptr = thrust::raw_pointer_cast(out); @@ -86,26 +86,23 @@ void cuda_max(device_ptr values, int length, device_ptr out) // 1. Get size of temporary allocation, if any. void* d_temp_storage = nullptr; size_t temp_storage_bytes = 0; - cub::DeviceReduce::Max(d_temp_storage, temp_storage_bytes, values_ptr, out_ptr, length); + cub::DeviceReduce::Max(nullptr, temp_storage_bytes, values_ptr, out_ptr, length); // 2. Do temporary allocation, if needed. - - // Using a thrust::device_vector should ensure (i) no allocation for 0 butes and (ii) automatic deallocation if needed. - thrust::device_vector temp_storage(temp_storage_bytes); - d_temp_storage = thrust::raw_pointer_cast(temp_storage.data()); + scratch.fit(temp_storage_bytes); // 3. Do the reduction. - cub::DeviceReduce::Max(d_temp_storage, temp_storage_bytes, values_ptr, out_ptr, length); + cub::DeviceReduce::Max(scratch.device_ptr, temp_storage_bytes, values_ptr, out_ptr, length); } -void cuda_max(float* values, int length, float* out) +void cuda_max(float* values, int length, float* out, cuda_scratch_space& scratch) { - cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out)); + cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out), scratch); } -void cuda_max(double* values, int length, double* out) +void cuda_max(double* values, int length, double* out, cuda_scratch_space& scratch) { - cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out)); + cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out), scratch); } struct SumAbs @@ -254,7 +251,7 @@ void cuda_max_l1_norm(device_ptr values, // cudaStreamSynchronize(); // 2. Second maximize over the column sums and return the highest. - cuda_max(buffer, t, out); + cuda_max(buffer, t, out, scratch); // cudaStreamSynchronize(); } diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index f9487504..0b329122 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -94,8 +94,8 @@ void cuda_sign_vector(float* v, int n, int t); double cuda_max_abs(double* v, int length); float cuda_max_abs(float* v, int length); -void cuda_max(double* v, int length, double* out); -void cuda_max(float* v, int length, float* out); +void cuda_max(double* v, int length, double* out, cuda_scratch_space& scratch); +void cuda_max(float* v, int length, float* out, cuda_scratch_space& scratch); // max over (n,t) -> (n,1). Matrix assumed to be column-major. void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr); From 49ec20304169281322a3bbd0ca8c7997999f15ab Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Feb 2025 18:19:10 -0500 Subject: [PATCH 335/421] Don't allocate memory for offsets. --- libhmsbeagle/GPU/cuda_ops.cu | 51 +++++++++++------------------------- libhmsbeagle/GPU/cuda_ops.h | 7 ++--- 2 files changed, 18 insertions(+), 40 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index a82f7669..9e8e4af3 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -105,15 +105,6 @@ void cuda_max(double* values, int length, double* out, cuda_scratch_space& scrat cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out), scratch); } -struct SumAbs -{ - template - __device__ __forceinline__ T operator()(const T& a, const T& b) const - { - return std::abs(a) + std::abs(b); - } -}; - template T cuda_max_l1_norm(device_ptr values, int n, int t, device_ptr buffer) { @@ -184,20 +175,28 @@ cuda_scratch_space::~cuda_scratch_space() } +struct SumAbs +{ + template + __device__ __forceinline__ T operator()(const T& a, const T& b) const + { + return std::abs(a) + std::abs(b); + } +}; + template void cuda_max_l1_norm(device_ptr values, int n, int t, device_ptr buffer, device_ptr out, - cuda_scratch_space& scratch) + cuda_scratch_space& scratch, + thrust::device_ptr d_offsets_it) { // cudaStreamSynchronize(); // Run reduction int num_segments = t; - thrust::device_vector d_offsets = {0, n, 2*n}; // {0, n, 2n, 3n ,,,,} - auto d_offsets_it = thrust::raw_pointer_cast(d_offsets.data()); SumAbs sum_abs_op; T initial_value = 0; @@ -216,14 +215,12 @@ void cuda_max_l1_norm(device_ptr values, sum_abs_op, initial_value); - std::cerr<<"cuda_max_l1_norm: wants "< values, // cudaStreamSynchronize(); } -void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out, cuda_scratch_space& scratch) +void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out, cuda_scratch_space& scratch, int* offsets) { using namespace thrust; - cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out), scratch); + cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out), scratch, device_pointer_cast(offsets)); } -void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch) +void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch, int* offsets) { using namespace thrust; - cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out), scratch); -} - - -void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out) -{ - cuda_scratch_space scratch; - - cuda_max_l1_norm(values, n, t, buffer, out, scratch); + cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out), scratch, device_pointer_cast(offsets)); } -void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out) -{ - cuda_scratch_space scratch; - - cuda_max_l1_norm(values, n, t, buffer, out, scratch); -} - - template void cuda_vec_fill(device_ptr values, int length, T fill) { using namespace thrust::placeholders; diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 0b329122..33865f95 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -110,11 +110,8 @@ void initialize_norm_x_matrix(double* data, int n, int m); float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); -void cuda_max_l1_norm(float* values, int n, int t, float* buffer, double* out); -void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out); - -void cuda_max_l1_norm(float* values, int n, int t, float* buffer, double* out, cuda_scratch_space& scratch); -void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch); +void cuda_max_l1_norm(float* values, int n, int t, float* buffer, double* out, cuda_scratch_space& scratch, int* offsets); +void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch, int* offsets); void cuda_vec_fill(float* values, int length, float fill); void cuda_vec_fill(double* values, int length, double fill); From 43aa56b2bc4b904a5dbf59b92b64adfe62d205ab Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Feb 2025 18:19:24 -0500 Subject: [PATCH 336/421] Add GPUnormest3 --- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index cd67734b..20829a08 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -309,7 +309,8 @@ int BeagleGPUActionImpl::createInstance(int tipCount, for(int p=0; p<=getPMax()+1; p++) { int t = 5; - L1normForPower.push_back(GPUnormest1(cublasHandle, p, kPaddedStateCount, t)); + int itmax = 2; + L1normForPower.push_back({cublasHandle, p, kPaddedStateCount, t, 2}); // This is very fast, but computes |A| instead of |A^p| // L1normForPower.push_back(GPUnormest2(cublasHandle, cusparseHandle, p, kPaddedStateCount)); From 6e810a181191e352d18ca0d005726602c0bd337e Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Feb 2025 18:56:27 -0500 Subject: [PATCH 337/421] Don't call cudaFree if the buffer is nullptr. --- libhmsbeagle/GPU/cuda_ops.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 33865f95..611538ba 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -24,9 +24,12 @@ T* cudaDeviceNew(int n) template void cudaDeviceDelete(T* buffer) { - auto status = cudaFree(buffer); - if (status != cudaSuccess) - throw std::runtime_error(std::string("cudaFree: ") + std::string(cudaGetErrorString(status))); + if (buffer) + { + auto status = cudaFree(buffer); + if (status != cudaSuccess) + throw std::runtime_error(std::string("cudaFree: ") + std::string(cudaGetErrorString(status))); + } } template From 8cae1ba53e8ec9d3bcfebb4e39bf5f8628d66544 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Feb 2025 18:57:24 -0500 Subject: [PATCH 338/421] Don't call cudaFree unnecessarily. --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index e220d6eb..091f6bf5 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -382,8 +382,8 @@ int spMM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSp if(new_buffersize > buffersize) { - CHECK_CUDA(cudaFree(buffer)); - CHECK_CUDA(cudaMalloc(&buffer, new_buffersize)); + cudaDeviceDelete(buffer); + buffer = cudaMallocWrapped(new_buffersize); buffersize = new_buffersize; } @@ -410,8 +410,8 @@ int spMTM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseS if(new_buffersize > buffersize) { - CHECK_CUDA(cudaFree(buffer)); - CHECK_CUDA(cudaMalloc(&buffer, new_buffersize)); + cudaDeviceDelete(buffer); + buffer = cudaMallocWrapped(new_buffersize); buffersize = new_buffersize; } From 858154c1c1fe8b02a33009b4e91cde4ed98e261a Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Feb 2025 18:58:07 -0500 Subject: [PATCH 339/421] Add GPUnormest3 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 161 +++++++++++++++++++++++++ 1 file changed, 161 insertions(+) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 091f6bf5..920b93ce 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -901,6 +901,167 @@ struct GPUnormest2 } }; +// Make the temporary variables part of a data structure that represents to normest1 problem. +// We don't want to allocate memory on the GPU every time we run this. +template +struct GPUnormest3 +{ + int p = 0; + int n = 0; + int t = 0; + int itmax = 0; + DnMatrixDevice X; // (n,t) + DnMatrixDevice Y; // (n,t) + DnMatrixDevice h; // (n,1) + + // Sorted matrix dimensions by approximate norm. + int* indices = nullptr; // (n) + + // Used for spMM + void* buffer = nullptr; + size_t buffer_size = 0; + + // Temporary storage for cuda_max_l1_norm + Real* buffer2 = nullptr; + + // Space to store the norms for the different iterations. + Real* buffer3 = nullptr; + + // Space to store a result + Real* buffer4 = nullptr; + + cuda_scratch_space scratch; + + int* offsets = nullptr; + + GPUnormest3& operator=(const GPUnormest3&) = delete; + GPUnormest3& operator=(GPUnormest3&& g) + { + std::swap(p, g.p); + std::swap(n, g.n); + std::swap(t, g.t); + std::swap(itmax, g.itmax); + std::swap(X, g.X); + std::swap(Y, g.Y); + std::swap(h, g.h); + + // ensure that (*this) and (g) don't both own the buffers! + std::swap(buffer, g.buffer); + std::swap(buffer_size, g.buffer_size); + std::swap(indices, g.indices); + std::swap(buffer2, g.buffer2); + std::swap(buffer3, g.buffer3); + std::swap(buffer4, g.buffer4); + std::swap(scratch, g.scratch); + std::swap(offsets, g.offsets); + + return *this; + } + + Real operator()(const SpMatrixDevice& A) + { + // std::cerr<<"n = "<(Y, 1, A, X, 0, buffer, buffer_size); + // X = Y + X.copyFrom(Y); + } + + // get the L1 norm for this iterations -> buffer3[k] + cuda_max_l1_norm(X.ptr, n, 1, buffer2, buffer3 + (k-1), scratch, offsets); + + // S = sign(X) + cuda_sign_vector(X.ptr, n, t); // (n,n) -> (n,n) + + // (2) Replace parallel entries with random +1/-1 entries. + // Skipping this part for now because it involves a lot of branching logic + // and might be slow for the GPU. + + // (3) of Algorithm 2.4 + // Z = A^T * S + spMTM(Y, 1, A, X, 0, buffer, buffer_size); // (n,n) * (n,t) -> (n,t) + + // h[0,j] = max(i) abs(Z(i,j)) + cuda_rowwise_max_abs(Y.ptr, t, t, h.ptr); // (n,t) -> (n,1) + + // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. + // We don't do this, because finding a different reason to exit + // seems to provide greater accuracy. + + // Rank the dimensions by the size of their approximate norm in h, in decreasing order. + cuda_sort_indices_by_vector(h.ptr, n, indices); + + // Zero the X matrix. + cuda_fill_vector(X.ptr, n*t, 0); // (n,t) -> (n,t) + + // Set X(i,indices[i]) = 1 for i in [0,t-1] + cuda_set_indices(X.ptr, n, t, indices); + } + + return cuda_max_abs(buffer3, itmax); + } + + GPUnormest3(GPUnormest3&& g) + { + operator=(std::move(g)); + } + + GPUnormest3(const GPUnormest3&) = delete; + + GPUnormest3(cublasHandle_t cb, int p_, int n_, int t_=2, int itmax_=5) + :p(p_), n(n_), t(std::min(t_,n)), itmax(itmax_), X(cb,n,t), Y(cb,n,t), h(cb,n,1) + { + assert(p >= 0); + assert(t > 0); // negative means t = n + assert(itmax >= 1); + + // Interpret negative t as t == n + // if (t < 0) t = n; + + buffer2 = cudaDeviceNew(t); + + buffer3 = cudaDeviceNew(itmax); + + buffer4 = cudaDeviceNew(1); + + indices = cudaDeviceNew(n); + + offsets = cudaDeviceNew(t+1); + + std::vector h_offsets(t+1); + for(int i=0;i From dac7168e534a7599331b13ff59476b6445caeb98 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Feb 2025 18:58:15 -0500 Subject: [PATCH 340/421] Use GPUnormest3 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 920b93ce..081e4e62 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -1179,7 +1179,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl std::vector> dIntegrationTmpLeft; std::vector> dIntegrationTmpRight; std::vector> dAs; - std::vector> L1normForPower; + std::vector> L1normForPower; std::vector integrationLeftBufferSize; std::vector dIntegrationLeftBuffer; std::vector integrationRightBufferSize; From 8b1120ab6a3159541afa5cf1378af898e2a65202 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 4 Mar 2025 14:32:54 -0500 Subject: [PATCH 341/421] Don't run find_package(CUDA) if BUILD_CUDA=OFF --- libhmsbeagle/GPU/CMakeLists.txt | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/GPU/CMakeLists.txt b/libhmsbeagle/GPU/CMakeLists.txt index 22a18789..eb237c9c 100644 --- a/libhmsbeagle/GPU/CMakeLists.txt +++ b/libhmsbeagle/GPU/CMakeLists.txt @@ -1,6 +1,6 @@ -find_package(OpenCL) if(BUILD_OPENCL) + find_package(OpenCL) # set(OpenCL_INCLUDE_DIRS /opt/rocm/opencl/include) # set(OpenCL_LIBRARIES /opt/rocm/opencl/lib/libOpenCL.so) if(OpenCL_FOUND OR (OpenCL_INCLUDE_DIRS AND OpenCL_LIBRARIES)) @@ -10,11 +10,13 @@ if(BUILD_OPENCL) endif(OpenCL_FOUND OR (OpenCL_INCLUDE_DIRS AND OpenCL_LIBRARIES)) endif(BUILD_OPENCL) -find_package(CUDA) -if(CUDA_FOUND AND BUILD_CUDA) - message(STATUS "CUDA Includes: ${CUDA_INCLUDE_DIRS}") - add_subdirectory("CMake_CUDA") -endif(CUDA_FOUND AND BUILD_CUDA) +if(BUILD_CUDA) + find_package(CUDA) + if(CUDA_FOUND) + message(STATUS "CUDA Includes: ${CUDA_INCLUDE_DIRS}") + add_subdirectory("CMake_CUDA") + endif(CUDA_FOUND) +endif(BUILD_CUDA) if(CUDA_FOUND AND BUILD_ACTION) message(STATUS "CUDA Includes: ${CUDA_INCLUDE_DIRS}") From 9c7b6735e0932eff98b5763b6a2ca8e669142bcb Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Apr 2025 14:56:03 -0700 Subject: [PATCH 342/421] Print size of failed allocation. --- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 922d7fc4..7aa8fd8e 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -5851,6 +5851,7 @@ void* BeagleCPUImpl::mallocAligned(size_t size) { int res; res = posix_memalign(&ptr, align, size); if (res != 0) { + std::cerr<<"Aligned malloc for "< Date: Mon, 28 Apr 2025 14:23:20 -0400 Subject: [PATCH 343/421] Only allocate BASTA memory if BASTA is enabled. --- CMakeLists.txt | 12 ++++++++---- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a4853b6..9643635b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,6 +154,14 @@ if(BUILD_ACTION) # add_definitions(-DBEAGLE_DEBUG_FLOW) endif(BUILD_ACTION) +if (BUILD_BASTA) + IF(BEAGLE_DEBUG_KERNELS) + add_subdirectory(tests/basta) + ENDIF(BEAGLE_DEBUG_KERNELS) + + add_definitions(-DBEAGLE_BEASTA) +endif(BUILD_BASTA) + link_directories( ${PROJECT_SOURCE_DIR}/libhmsbeagle ${PROJECT_SOURCE_DIR}/libhmsbeagle/CPU @@ -231,10 +239,6 @@ enable_testing() add_subdirectory(examples) -IF(BEAGLE_DEBUG_KERNELS) - add_subdirectory(tests/basta) -ENDIF(BEAGLE_DEBUG_KERNELS) - set(CPACK_PACKAGE_VERSION "${BEAGLE_VERSION}") set(CPACK_PACKAGE_MAJOR "${BEAGLE_VERSION_MAJOR}") set(CPACK_PACKAGE_MINOR "${BEAGLE_VERSION_MINOR}") diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 7aa8fd8e..a5cc9b81 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -133,6 +133,7 @@ BeagleCPUImpl::~BeagleCPUImpl() { } free(gTransitionMatrices); +#ifdef BEAGLE_BASTA for (int i = 0; i < kStateCount; i++) { for (int j = 0; j < kStateCount; j++) { for (int k = 0; k < kMatrixCount; k++) { @@ -176,6 +177,7 @@ BeagleCPUImpl::~BeagleCPUImpl() { if (tempGrad[i] != NULL) free(tempGrad[i]); } +#endif /* BEAGLE_BASTA */ for(unsigned int i=0; i::createInstance(int tipCount, if (gPartials == NULL) throw std::bad_alloc(); +#ifdef BEAGLE_BASTA // TODO: deconstrucor should free all of these gPartialsGrad = (REALTYPE****) malloc(sizeof(REALTYPE***) * kStateCount); for (int i = 0; i < kStateCount; i++) { @@ -453,6 +456,7 @@ int BeagleCPUImpl::createInstance(int tipCount, if (tempGrad[i] == NULL) throw std::bad_alloc(); } +#endif /* BEAGLE_BASTA */ gStateFrequencies = (REALTYPE**) calloc(sizeof(REALTYPE*), kEigenDecompCount); if (gStateFrequencies == NULL) @@ -533,6 +537,7 @@ int BeagleCPUImpl::createInstance(int tipCount, throw std::bad_alloc(); } +#ifdef BEAGLE_BASTA // TODO: deconstrucor should free all of these gTransitionMatricesGrad = (REALTYPE****) malloc(sizeof(REALTYPE***) * kStateCount); for (int i = 0; i < kStateCount; i++) { @@ -550,6 +555,7 @@ int BeagleCPUImpl::createInstance(int tipCount, } } } +#endif integrationTmp = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPatternCount * kStateCount); firstDerivTmp = (REALTYPE*) malloc(sizeof(REALTYPE) * kPatternCount * kStateCount); From bac49b51966c5c8624a3cb77e48d70bbf02beba3 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 6 May 2025 20:27:42 -0500 Subject: [PATCH 344/421] action test file update --- examples/actiontest/actiontest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/actiontest/actiontest.cpp b/examples/actiontest/actiontest.cpp index 87285991..9959ebf6 100644 --- a/examples/actiontest/actiontest.cpp +++ b/examples/actiontest/actiontest.cpp @@ -749,6 +749,8 @@ int main( int argc, const char* argv[] ) beagleSetDifferentialMatrix(instance, 4, scaledQ.data()); beagleSetDifferentialMatrix(instance, 5, scaledQ2.data()); + beagleSetSparseDifferentialMatrix(instance, 4, rowIndices, colIndices, eval, 16); + std::vector firstBuffer(nPatterns * 5 * 2); // Get both numerator and denominator std::vector sumBuffer(5); From 22c8416f31dcd86bbd1efdcbfc92a9f6549100b5 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 6 May 2025 20:28:59 -0500 Subject: [PATCH 345/421] add new set sparse differential matrix for action --- libhmsbeagle/BeagleImpl.h | 6 ++ libhmsbeagle/CPU/BeagleCPUActionImpl.h | 23 +++++++- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 70 +++++++++++++++++++++++- libhmsbeagle/CPU/BeagleCPUImpl.h | 6 ++ libhmsbeagle/CPU/BeagleCPUImpl.hpp | 10 ++++ libhmsbeagle/GPU/BeagleGPUImpl.h | 6 ++ libhmsbeagle/GPU/BeagleGPUImpl.hpp | 9 +++ libhmsbeagle/beagle.cpp | 15 +++++ libhmsbeagle/beagle.h | 23 +++++++- 9 files changed, 164 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/BeagleImpl.h b/libhmsbeagle/BeagleImpl.h index ae02b909..104cd025 100755 --- a/libhmsbeagle/BeagleImpl.h +++ b/libhmsbeagle/BeagleImpl.h @@ -79,6 +79,12 @@ class BeagleImpl const double* values, int numNonZeros) = 0; + virtual int setSparseDifferentialMatrix(int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros) = 0; + virtual int setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies) = 0; diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index f4736e16..f3d5dd66 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -80,7 +80,11 @@ namespace beagle { using BeagleCPUImpl::gCategoryRates; using BeagleCPUImpl::gScaleBuffers; using BeagleCPUImpl::kFlags; -// SpMatrix** gScaledQs; + using BeagleCPUImpl::grandNumeratorDerivTmp; + using BeagleCPUImpl::grandDenominatorDerivTmp; + using BeagleCPUImpl::gTransitionMatrices; + + // SpMatrix** gScaledQs; int kPartialsCacheOffset; // using BeagleCPUImpl::gStateFrequencies; // using BeagleCPUImpl::gTipStates; @@ -200,6 +204,12 @@ namespace beagle { const double* values, int numNonZeros); + virtual int setSparseDifferentialMatrix(int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros); + virtual int updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, const int* firstDerivativeIndices, @@ -229,6 +239,17 @@ namespace beagle { int startPattern, int endPattern); + void calcEdgeLogDerivativesPartials(const double *postOrderPartial, + const double *preOrderPartial, + const int firstDerivativeIndex, + const int secondDerivativeIndex, + const double *categoryRates, + const double *categoryWeights, + const int scalingFactorsIndex, + double *siteLogLikelihoods, + double *outLogFirstDerivatives, + double *outLogDiagonalSecondDerivatives); + // Return (m,s) std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, int eigenIndex) const; diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index fbfe0733..e7600c1a 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -325,8 +325,8 @@ namespace beagle { preferenceFlags, requirementFlags); kPartialsCacheOffset = partialsBufferCount + compactBufferCount; - gInstantaneousMatrices.resize(eigenDecompositionCount); - for (int i = 0; i < eigenDecompositionCount; i++) + gInstantaneousMatrices.resize(eigenDecompositionCount + matrixCount); + for (int i = 0; i < eigenDecompositionCount + matrixCount; i++) gInstantaneousMatrices[i] = SpMatrix(kStateCount, kStateCount); gBs.resize(eigenDecompositionCount); gMuBs.resize(eigenDecompositionCount); @@ -664,6 +664,24 @@ namespace beagle { return BEAGLE_SUCCESS; } + BEAGLE_CPU_ACTION_TEMPLATE + int BeagleCPUActionImpl::setSparseDifferentialMatrix(int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros) { + + + std::vector tripletList; + for (int i = 0; i < numNonZeros; i++) { + tripletList.push_back(Triplet(rowIndices[i], colIndices[i], values[i])); + } + + gInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); + + return BEAGLE_SUCCESS; + } + BEAGLE_CPU_ACTION_TEMPLATE int BeagleCPUActionImpl::updateTransitionMatrices(int eigenIndex, const int* probabilityIndices, @@ -744,6 +762,54 @@ namespace beagle { } } + BEAGLE_CPU_ACTION_TEMPLATE + void BeagleCPUActionImpl::calcEdgeLogDerivativesPartials(const double *postOrderPartial, + const double *preOrderPartial, + const int firstDerivativeIndex, + const int secondDerivativeIndex, + const double *categoryRates, + const double *categoryWeights, + const int scalingFactorsIndex, + double *siteLogLikelihoods, + double *outLogFirstDerivatives, + double *outLogDiagonalSecondDerivatives) { + std::cout<<"here?"<::simpleAction2(MapType &destP, MapType &partials, int edgeIndex, diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index 612c7a62..1e3e437e 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -242,6 +242,12 @@ class BeagleCPUImpl : public BeagleImpl { const double* values, int numNonZeros); + int setSparseDifferentialMatrix(int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros); + int setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies); diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index a5cc9b81..ac3f03ca 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -915,6 +915,16 @@ int BeagleCPUImpl::setSparseMatrix(int matrixIndex, return BEAGLE_ERROR_NO_IMPLEMENTATION; } +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::setSparseDifferentialMatrix(int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros) { + + return BEAGLE_ERROR_NO_IMPLEMENTATION; +} + BEAGLE_CPU_TEMPLATE int BeagleCPUImpl::setCategoryRates(const double* inCategoryRates) { int categoryRatesIndex=0; diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.h b/libhmsbeagle/GPU/BeagleGPUImpl.h index 144b52f2..a11c2c72 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUImpl.h @@ -289,6 +289,12 @@ class BeagleGPUImpl : public BeagleImpl { const int* colIndices, const double* values, int numNonZeros); + + int setSparseDifferentialMatrix(int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros); int setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies); diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 2a32baee..9ae9bf34 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -1201,6 +1201,15 @@ int BeagleGPUImpl::setSparseMatrix(int matrixIndex, return BEAGLE_ERROR_NO_IMPLEMENTATION; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUImpl::setSparseDifferentialMatrix(int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros) { + return BEAGLE_ERROR_NO_IMPLEMENTATION; +} + BEAGLE_GPU_TEMPLATE int BeagleGPUImpl::setStateFrequencies(int stateFrequenciesIndex, const double* inStateFrequencies) { diff --git a/libhmsbeagle/beagle.cpp b/libhmsbeagle/beagle.cpp index d1402d36..76a05920 100755 --- a/libhmsbeagle/beagle.cpp +++ b/libhmsbeagle/beagle.cpp @@ -949,6 +949,21 @@ int beagleSetSparseMatrix(int instance, return returnValue; } +int beagleSetSparseDifferentialMatrix(int instance, + int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros) { + DEBUG_START_TIME(); + beagle::BeagleImpl *beagleInstance = beagle::getBeagleInstance(instance); + if (beagleInstance == NULL) + return BEAGLE_ERROR_UNINITIALIZED_INSTANCE; + int returnValue = beagleInstance->setSparseDifferentialMatrix(matrixIndex, rowIndices, colIndices, values, numNonZeros); + DEBUG_END_TIME(); + return returnValue; +} + int beagleSetStateFrequencies(int instance, int stateFrequenciesIndex, const double* inStateFrequencies) { diff --git a/libhmsbeagle/beagle.h b/libhmsbeagle/beagle.h index cbbb7cc3..80e3a278 100755 --- a/libhmsbeagle/beagle.h +++ b/libhmsbeagle/beagle.h @@ -541,7 +541,7 @@ BEAGLE_DLLEXPORT int beagleSetEigenDecomposition(int instance, const double* inEigenValues); /** - * @brief Set a sparse matrix using non-zero entries + * @brief Set a sparse generator matrix with majorization using non-zero entries * * This function sets a sparse matrix in an instance buffer. * @@ -561,6 +561,27 @@ BEAGLE_DLLEXPORT int beagleSetSparseMatrix(int instance, const double* values, int numNonZeros); +/** + * @brief Set a sparse differential matrix using non-zero entries + * + * This function sets a sparse matrix in an instance buffer. + * + * @param instance Instance number (input) + * @param matrixIndex Index of eigen-decomposition buffer (input) + * @param rowIndices List of row indices of non-zero entries (input) + * @param colIndices List of column indices of non-zero entries (input) + * @param values List of non-zero entry values + * @param numNonZeros Number of non-zero entries + * + * @return error code + */ +BEAGLE_DLLEXPORT int beagleSetSparseDifferentialMatrix(int instance, + int matrixIndex, + const int *rowIndices, + const int *colIndices, + const double *values, + int numNonZeros); + /** * @brief Set a state frequency buffer * From 269c56bc32f06f5ac2a2f2203d45e364ad833b0c Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 6 May 2025 22:13:14 -0500 Subject: [PATCH 346/421] intermediate commit: not compile yet --- libhmsbeagle/CPU/BeagleCPUActionImpl.h | 46 ++++-- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 178 +++++++++++++++++------ 2 files changed, 164 insertions(+), 60 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index f3d5dd66..d930ec43 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -64,8 +64,8 @@ namespace beagle { protected: using BeagleCPUImpl::kTipCount; // using BeagleCPUImpl::integrationTmp; -// using BeagleCPUImpl::firstDerivTmp; -// using BeagleCPUImpl::secondDerivTmp; + using BeagleCPUImpl::firstDerivTmp; + using BeagleCPUImpl::secondDerivTmp; using BeagleCPUImpl::kPatternCount; using BeagleCPUImpl::kPaddedPatternCount; using BeagleCPUImpl::kExtraPatterns; @@ -78,11 +78,12 @@ namespace beagle { using BeagleCPUImpl::kEigenDecompCount; using BeagleCPUImpl::gPartials; using BeagleCPUImpl::gCategoryRates; + using BeagleCPUImpl::gCategoryWeights; using BeagleCPUImpl::gScaleBuffers; using BeagleCPUImpl::kFlags; using BeagleCPUImpl::grandNumeratorDerivTmp; using BeagleCPUImpl::grandDenominatorDerivTmp; - using BeagleCPUImpl::gTransitionMatrices; +// using BeagleCPUImpl::gTransitionMatrices; // SpMatrix** gScaledQs; int kPartialsCacheOffset; @@ -182,6 +183,24 @@ namespace beagle { int operationCount, int cumulativeScalingIndex); + virtual void accumulateDerivatives(double* outDerivatives, + double* outSumDerivatives, + double* outSumSquaredDerivatives); + + virtual void resetDerivativeTemporaries(); + + virtual int calcEdgeLogDerivatives(const int *postBufferIndices, + const int *preBufferIndices, + const int *firstDerivativeIndices, + const int *secondDerivativeIndices, + const int *categoryWeightsIndices, + const int *categoryRatesIndices, + const int *cumulativeScaleIndices, + int count, + double *siteLogLikelihoods, + double *outLogFirstDerivatives, + double *outLogDiagonalSecondDerivatives); + inline MapType partialsMap(int index, int category, int startPattern, int endPattern); inline MapType partialsMap(int index, int category); @@ -193,6 +212,17 @@ namespace beagle { // virtual int getPaddedPatternsModulus(); private: + void calcEdgeLogDerivativesPartials(const int postOrderPartialIndex, + const int preOrderPartialIndex, + const int firstDerivativeIndex, + const int secondDerivativeIndex, + const double *categoryRates, + const double *categoryWeights, + const int scalingFactorsIndex, + double *siteLogLikelihoods, + double *outLogFirstDerivatives, + double *outLogDiagonalSecondDerivatives); + virtual int setEigenDecomposition(int eigenIndex, const double *inEigenVectors, const double *inInverseEigenVectors, @@ -239,16 +269,6 @@ namespace beagle { int startPattern, int endPattern); - void calcEdgeLogDerivativesPartials(const double *postOrderPartial, - const double *preOrderPartial, - const int firstDerivativeIndex, - const int secondDerivativeIndex, - const double *categoryRates, - const double *categoryWeights, - const int scalingFactorsIndex, - double *siteLogLikelihoods, - double *outLogFirstDerivatives, - double *outLogDiagonalSecondDerivatives); // Return (m,s) std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index e7600c1a..dce13368 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -564,6 +564,137 @@ namespace beagle { return BEAGLE_SUCCESS; } + BEAGLE_CPU_ACTION_TEMPLATE + void BeagleCPUActionImpl::calcEdgeLogDerivativesPartials(const int postOrderPartialIndex, + const int preOrderPartialIndex, + const int firstDerivativeIndex, + const int secondDerivativeIndex, + const double *categoryRates, + const double *categoryWeights, + const int scalingFactorsIndex, + double *outDerivativesForNode, + double *outSumDerivativesForNode, + double *outSumSquaredDerivativesForNode) + { + + +// const REALTYPE *firstDerivMatrix = gTransitionMatrices[firstDerivativeIndex]; +// +// for (int category = 0; category < kCategoryCount; category++) { +// const REALTYPE weight = categoryWeights[category]; +// +// for (int pattern = 0; pattern < kPatternCount; pattern++) { +// +// int w = category * kMatrixSize; +// +// const int patternIndex = category * kPatternCount + pattern; +// const int v = patternIndex * kPartialsPaddedStateCount; +// +// REALTYPE numerator = 0.0; +// REALTYPE denominator = 0.0; +// +// for (int k = 0; k < kStateCount; k++) { +// +// REALTYPE sumOverEndState = 0.0; +// for (int j = 0; j < kStateCount; j++) { +// sumOverEndState += firstDerivMatrix[w] +// * postOrderPartial[v + j]; // fix padded index +// w++; +// } +// w += T_PAD; +// +// numerator += sumOverEndState * preOrderPartial[v + k]; +// denominator += postOrderPartial[v + k] * preOrderPartial[v + k]; +// } +// +// grandNumeratorDerivTmp[pattern] += weight * numerator; +// grandDenominatorDerivTmp[pattern] += weight * denominator; +// } +// } + + + auto destNumeratorDrivTmp = MapType(grandNumeratorDerivTmp, kStateCount, kPatternCount); + auto destDenominatorDrivTmp = MapType(grandDenominatorDerivTmp, kStateCount, kPatternCount); + auto destFirstDerivTmp = MapType(firstDerivTmp, kStateCount, kPatternCount); + MapType destSecondDerivTmp = MapType(secondDerivTmp, kStateCount, kPatternCount); + SpMatrix differentialMatrix = gInstantaneousMatrices[firstDerivativeIndex]; + + + + for (int category = 0; category < kCategoryCount; category++) { + const double weight = categoryWeights[category]; + + MapType postOrderPartial = partialsMap(postOrderPartialIndex, category, 0, kPatternCount); + MapType preOrderPartial = partialsMap(preOrderPartialIndex, category, 0, kPatternCount); + + const double categoryMultiplier = weight * categoryRates[category]; + + destFirstDerivTmp = differentialMatrix * postOrderPartial; + + destFirstDerivTmp.cwiseProduct(preOrderPartial); + + destSecondDerivTmp = postOrderPartial.template cwiseProduct(preOrderPartial); + + destNumeratorDrivTmp += destFirstDerivTmp.colwise().sum() * categoryMultiplier; + destDenominatorDrivTmp += destSecondDerivTmp.colwise().sum() * categoryMultiplier; + } + } + + BEAGLE_CPU_ACTION_TEMPLATE + int BeagleCPUActionImpl::calcEdgeLogDerivatives(const int *postBufferIndices, + const int *preBufferIndices, + const int *firstDerivativeIndices, + const int *secondDerivativeIndices, + const int *categoryWeightsIndices, + const int *categoryRatesIndices, + const int *cumulativeScaleIndices, + int count, + double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives) { + + int returnCode = BEAGLE_SUCCESS; + + const int secondDerivativeIndex = BEAGLE_OP_NONE; + const double *categoryRates = gCategoryRates[categoryRatesIndices[0]]; // TODO Generalize + const double *categoryWeights = gCategoryWeights[categoryWeightsIndices[0]]; // TODO Generalize + + for (int nodeNum = 0; nodeNum < count; nodeNum++) { + + const int preOrderPartialIndex = preBufferIndices[nodeNum]; + const int postOrderPartialindex = postBufferIndices[nodeNum]; + + + const int firstDerivativeIndex = firstDerivativeIndices[nodeNum]; + const int scalingFactorsIndex = -1; // cumulativeScaleIndices[nodeNum]; + + const int patternOffset = nodeNum * kPatternCount; + double* outDerivativesForNode = (outDerivatives == NULL) ? + NULL : outDerivatives + patternOffset; + double* outSumDerivativesForNode = (outSumDerivatives == NULL) ? + NULL : outSumDerivatives + nodeNum; + double* outSumSquaredDerivativesForNode = (outSumSquaredDerivatives == NULL) ? + NULL : outSumSquaredDerivatives + nodeNum; + + resetDerivativeTemporaries(); + + calcEdgeLogDerivativesPartials(postOrderPartialindex, preOrderPartialIndex, firstDerivativeIndex, + secondDerivativeIndex, categoryRates, categoryWeights, + scalingFactorsIndex, + outDerivativesForNode, + outSumDerivativesForNode, + outSumSquaredDerivativesForNode); + + accumulateDerivatives(outDerivativesForNode, + outSumDerivativesForNode, + outSumSquaredDerivativesForNode); + + } + + return returnCode; + + } + BEAGLE_CPU_ACTION_TEMPLATE int BeagleCPUActionImpl::setTipStates(int tipIndex, const int* inStates) { @@ -762,53 +893,6 @@ namespace beagle { } } - BEAGLE_CPU_ACTION_TEMPLATE - void BeagleCPUActionImpl::calcEdgeLogDerivativesPartials(const double *postOrderPartial, - const double *preOrderPartial, - const int firstDerivativeIndex, - const int secondDerivativeIndex, - const double *categoryRates, - const double *categoryWeights, - const int scalingFactorsIndex, - double *siteLogLikelihoods, - double *outLogFirstDerivatives, - double *outLogDiagonalSecondDerivatives) { - std::cout<<"here?"< Date: Tue, 6 May 2025 22:20:19 -0500 Subject: [PATCH 347/421] minor format fix --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index dce13368..f4a2700f 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -616,7 +616,7 @@ namespace beagle { auto destNumeratorDrivTmp = MapType(grandNumeratorDerivTmp, kStateCount, kPatternCount); auto destDenominatorDrivTmp = MapType(grandDenominatorDerivTmp, kStateCount, kPatternCount); auto destFirstDerivTmp = MapType(firstDerivTmp, kStateCount, kPatternCount); - MapType destSecondDerivTmp = MapType(secondDerivTmp, kStateCount, kPatternCount); + auto destSecondDerivTmp = MapType(secondDerivTmp, kStateCount, kPatternCount); SpMatrix differentialMatrix = gInstantaneousMatrices[firstDerivativeIndex]; @@ -633,7 +633,7 @@ namespace beagle { destFirstDerivTmp.cwiseProduct(preOrderPartial); - destSecondDerivTmp = postOrderPartial.template cwiseProduct(preOrderPartial); + destSecondDerivTmp = postOrderPartial.cwiseProduct(preOrderPartial); destNumeratorDrivTmp += destFirstDerivTmp.colwise().sum() * categoryMultiplier; destDenominatorDrivTmp += destSecondDerivTmp.colwise().sum() * categoryMultiplier; From c295cdbd29c7a5ea7c219885c4b33192012b7589 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 6 May 2025 22:21:39 -0500 Subject: [PATCH 348/421] dimension fix --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index f4a2700f..8369cc1d 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -613,8 +613,8 @@ namespace beagle { // } - auto destNumeratorDrivTmp = MapType(grandNumeratorDerivTmp, kStateCount, kPatternCount); - auto destDenominatorDrivTmp = MapType(grandDenominatorDerivTmp, kStateCount, kPatternCount); + auto destNumeratorDrivTmp = MapType(grandNumeratorDerivTmp, 1, kPatternCount); + auto destDenominatorDrivTmp = MapType(grandDenominatorDerivTmp, 1, kPatternCount); auto destFirstDerivTmp = MapType(firstDerivTmp, kStateCount, kPatternCount); auto destSecondDerivTmp = MapType(secondDerivTmp, kStateCount, kPatternCount); SpMatrix differentialMatrix = gInstantaneousMatrices[firstDerivativeIndex]; From cd80ab4034db6ff7d0cd82b6e85e132d466554e3 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 6 May 2025 22:50:32 -0500 Subject: [PATCH 349/421] let it compile --- libhmsbeagle/CPU/BeagleCPUActionImpl.h | 10 +++------- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index d930ec43..6f8a59d8 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -63,7 +63,7 @@ namespace beagle { protected: using BeagleCPUImpl::kTipCount; -// using BeagleCPUImpl::integrationTmp; + using BeagleCPUImpl::integrationTmp; using BeagleCPUImpl::firstDerivTmp; using BeagleCPUImpl::secondDerivTmp; using BeagleCPUImpl::kPatternCount; @@ -83,6 +83,8 @@ namespace beagle { using BeagleCPUImpl::kFlags; using BeagleCPUImpl::grandNumeratorDerivTmp; using BeagleCPUImpl::grandDenominatorDerivTmp; + using BeagleCPUImpl::accumulateDerivatives; + using BeagleCPUImpl::resetDerivativeTemporaries; // using BeagleCPUImpl::gTransitionMatrices; // SpMatrix** gScaledQs; @@ -183,12 +185,6 @@ namespace beagle { int operationCount, int cumulativeScalingIndex); - virtual void accumulateDerivatives(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); - - virtual void resetDerivativeTemporaries(); - virtual int calcEdgeLogDerivatives(const int *postBufferIndices, const int *preBufferIndices, const int *firstDerivativeIndices, diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 8369cc1d..846f511a 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -620,7 +620,6 @@ namespace beagle { SpMatrix differentialMatrix = gInstantaneousMatrices[firstDerivativeIndex]; - for (int category = 0; category < kCategoryCount; category++) { const double weight = categoryWeights[category]; From c333bed307a9b66aa186c6de68564d801817860f Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 6 May 2025 22:58:22 -0500 Subject: [PATCH 350/421] getting numbers --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 846f511a..29dfb50a 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -655,7 +655,7 @@ namespace beagle { int returnCode = BEAGLE_SUCCESS; const int secondDerivativeIndex = BEAGLE_OP_NONE; - const double *categoryRates = gCategoryRates[categoryRatesIndices[0]]; // TODO Generalize + const double *categoryRates = gCategoryRates[0]; // TODO Generalize const double *categoryWeights = gCategoryWeights[categoryWeightsIndices[0]]; // TODO Generalize for (int nodeNum = 0; nodeNum < count; nodeNum++) { From c09addfb789407ef496596610ceba64f0e4ff490 Mon Sep 17 00:00:00 2001 From: xji3 Date: Tue, 6 May 2025 23:03:01 -0500 Subject: [PATCH 351/421] clean a bit --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 37 ------------------------ 1 file changed, 37 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 29dfb50a..45a2563b 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -576,43 +576,6 @@ namespace beagle { double *outSumDerivativesForNode, double *outSumSquaredDerivativesForNode) { - - -// const REALTYPE *firstDerivMatrix = gTransitionMatrices[firstDerivativeIndex]; -// -// for (int category = 0; category < kCategoryCount; category++) { -// const REALTYPE weight = categoryWeights[category]; -// -// for (int pattern = 0; pattern < kPatternCount; pattern++) { -// -// int w = category * kMatrixSize; -// -// const int patternIndex = category * kPatternCount + pattern; -// const int v = patternIndex * kPartialsPaddedStateCount; -// -// REALTYPE numerator = 0.0; -// REALTYPE denominator = 0.0; -// -// for (int k = 0; k < kStateCount; k++) { -// -// REALTYPE sumOverEndState = 0.0; -// for (int j = 0; j < kStateCount; j++) { -// sumOverEndState += firstDerivMatrix[w] -// * postOrderPartial[v + j]; // fix padded index -// w++; -// } -// w += T_PAD; -// -// numerator += sumOverEndState * preOrderPartial[v + k]; -// denominator += postOrderPartial[v + k] * preOrderPartial[v + k]; -// } -// -// grandNumeratorDerivTmp[pattern] += weight * numerator; -// grandDenominatorDerivTmp[pattern] += weight * denominator; -// } -// } - - auto destNumeratorDrivTmp = MapType(grandNumeratorDerivTmp, 1, kPatternCount); auto destDenominatorDrivTmp = MapType(grandDenominatorDerivTmp, 1, kPatternCount); auto destFirstDerivTmp = MapType(firstDerivTmp, kStateCount, kPatternCount); From e66d792820e0ce35faa5384830784b1f12f1826d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 7 May 2025 07:03:36 -0400 Subject: [PATCH 352/421] Remove ubuntu-20.04, as it is too old. --- .github/workflows/build.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 74e4729a..2731102e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,7 @@ jobs: name: [ubuntu-gcc-10, ubuntu-gcc-11, ubuntu-gcc-12, + ubuntu-gcc-13, ubuntu-clang-15, macos-xcode-15-intel, macos-xcode-16-arm, @@ -23,7 +24,7 @@ jobs: include: - name: ubuntu-gcc-10 - os: ubuntu-20.04 + os: ubuntu-22.04 compiler: gcc version: "10" buildtype: "release" @@ -43,6 +44,13 @@ jobs: buildtype: "release" arch: "linux64" + - name: ubuntu-gcc-13 + os: ubuntu-24.04 + compiler: gcc + version: "12" + buildtype: "release" + arch: "linux64" + - name: ubuntu-clang-15 os: ubuntu-22.04 compiler: clang From c8f2488a020214cf22e2979792688acba477eb60 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 7 May 2025 07:08:19 -0400 Subject: [PATCH 353/421] Update ccache bit. --- .github/workflows/build.yml | 39 ++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2731102e..fe784b82 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,6 +58,13 @@ jobs: buildtype: "release" arch: "linux64" + - name: ubuntu-clang-17 + os: ubuntu-24.04 + compiler: clang + version: "17" + buildtype: "release" + arch: "linux64" + - name: macos-xcode-15-intel os: macos-13 compiler: xcode @@ -86,6 +93,16 @@ jobs: with: python-version: 3.12 + # Caches for different branches are isolated, so we don't need to put the branch name into the key. + # The total size for all caches in a repository is 5Gb. + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.name }} + restore-keys: | + ${{ matrix.name }} + - name: Install (Linux) if: runner.os == 'Linux' && matrix.name != 'windows' run: | @@ -138,28 +155,6 @@ jobs: # Caches for different branches are isolated, so we don't need to put the branch name into the key. # The total size for all caches in a repository is 5Gb. - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - run: | - ccache --set-config=cache_dir=$HOME/.ccache - - if [ "${{ runner.os }}" = "Linux" ]; then - stamp=$(date '+%s') - else - stamp=$(gdate '+%s') - fi - echo "${stamp}" - echo "timestamp=${stamp}" >> $GITHUB_OUTPUT - - - name: ccache cache files -# uses: actions/cache@v2 - uses: pat-s/always-upload-cache@v3.0.11 - with: - path: ~/.ccache - key: ${{ matrix.name }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} - restore-keys: | - ${{ matrix.name }}-ccache- - - name: Configure run: | if [ "${{ matrix.cxxflags }}" != "" ] ; then From 14a57ead50a0aee0f64346e16b6aed7d93172407 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 7 May 2025 07:09:26 -0400 Subject: [PATCH 354/421] Add missing matrix name. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe784b82..08f2fff6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,7 @@ jobs: ubuntu-gcc-12, ubuntu-gcc-13, ubuntu-clang-15, + ubuntu-clang-17, macos-xcode-15-intel, macos-xcode-16-arm, windows From 74133caa33a572599bcfe466742249de8c134d9b Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 7 May 2025 07:13:55 -0400 Subject: [PATCH 355/421] Fix bug and warning, I think. --- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index 9ae9bf34..b9aef636 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -657,7 +657,7 @@ int BeagleGPUImpl::createInstance(int tipCount, } } - if(!preferenceFlags & BEAGLE_FLAG_COMPUTATION_ACTION) { + if(! (preferenceFlags & BEAGLE_FLAG_COMPUTATION_ACTION)) { ptrIncrement = gpu->AlignMemOffset(kMatrixSize * sizeof(Real)); kEvecOffset = ptrIncrement/sizeof(Real); GPUPtr dEvecOrigin = gpu->AllocateMemory(kEigenDecompCount * ptrIncrement); From 0b0420b7fa63956f1b843923aaa3edaf1dd02727 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 7 May 2025 09:35:47 -0500 Subject: [PATCH 356/421] temp commit for debug prints --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 45a2563b..9236a701 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -582,6 +582,8 @@ namespace beagle { auto destSecondDerivTmp = MapType(secondDerivTmp, kStateCount, kPatternCount); SpMatrix differentialMatrix = gInstantaneousMatrices[firstDerivativeIndex]; + std::cerr<<"dQ = " << differentialMatrix << std::endl; + for (int category = 0; category < kCategoryCount; category++) { const double weight = categoryWeights[category]; @@ -591,14 +593,27 @@ namespace beagle { const double categoryMultiplier = weight * categoryRates[category]; + std::cerr<<"Category = " << category << " ; multiplier = " << weight << " * " << categoryRates[category] << " = " << categoryMultiplier << std::endl; + destFirstDerivTmp = differentialMatrix * postOrderPartial; - destFirstDerivTmp.cwiseProduct(preOrderPartial); + std::cerr<<"Post-order partial = " << std::endl << postOrderPartial << std::endl << "Pre-order partial = " << std::endl << preOrderPartial << std::endl; + std::cerr<<"Differential matrix = " << differentialMatrix << std::endl; + std::cerr<<"Qp = " << std::endl << destFirstDerivTmp << std::endl; + + destFirstDerivTmp = destFirstDerivTmp.cwiseProduct(preOrderPartial); + + std::cerr<<"q'Qp = " << std::endl << destFirstDerivTmp << std::endl; destSecondDerivTmp = postOrderPartial.cwiseProduct(preOrderPartial); + std::cerr<<"q'p = " << std::endl << destSecondDerivTmp << std::endl; + destNumeratorDrivTmp += destFirstDerivTmp.colwise().sum() * categoryMultiplier; + + std::cerr<<"colSum(q'Qp)rw = " << std::endl << destNumeratorDrivTmp << std::endl; destDenominatorDrivTmp += destSecondDerivTmp.colwise().sum() * categoryMultiplier; + std::cerr<<"colSum(q'p)rw = " << std::endl << destDenominatorDrivTmp << std::endl; } } @@ -640,6 +655,8 @@ namespace beagle { resetDerivativeTemporaries(); + std::cerr<<"Node = " << nodeNum << std::endl; + calcEdgeLogDerivativesPartials(postOrderPartialindex, preOrderPartialIndex, firstDerivativeIndex, secondDerivativeIndex, categoryRates, categoryWeights, scalingFactorsIndex, @@ -772,6 +789,8 @@ namespace beagle { gInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); + std::cout<<"Checking matrix: "< Date: Wed, 7 May 2025 09:43:36 -0500 Subject: [PATCH 357/421] correct edge derivative numbers --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 9236a701..045572e5 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -582,7 +582,7 @@ namespace beagle { auto destSecondDerivTmp = MapType(secondDerivTmp, kStateCount, kPatternCount); SpMatrix differentialMatrix = gInstantaneousMatrices[firstDerivativeIndex]; - std::cerr<<"dQ = " << differentialMatrix << std::endl; +// std::cerr<<"dQ = " << differentialMatrix << std::endl; for (int category = 0; category < kCategoryCount; category++) { @@ -593,27 +593,27 @@ namespace beagle { const double categoryMultiplier = weight * categoryRates[category]; - std::cerr<<"Category = " << category << " ; multiplier = " << weight << " * " << categoryRates[category] << " = " << categoryMultiplier << std::endl; +// std::cerr<<"Category = " << category << " ; multiplier = " << weight << " * " << categoryRates[category] << " = " << categoryMultiplier << std::endl; destFirstDerivTmp = differentialMatrix * postOrderPartial; - std::cerr<<"Post-order partial = " << std::endl << postOrderPartial << std::endl << "Pre-order partial = " << std::endl << preOrderPartial << std::endl; - std::cerr<<"Differential matrix = " << differentialMatrix << std::endl; - std::cerr<<"Qp = " << std::endl << destFirstDerivTmp << std::endl; +// std::cerr<<"Post-order partial = " << std::endl << postOrderPartial << std::endl << "Pre-order partial = " << std::endl << preOrderPartial << std::endl; +// std::cerr<<"Differential matrix = " << differentialMatrix << std::endl; +// std::cerr<<"Qp = " << std::endl << destFirstDerivTmp << std::endl; destFirstDerivTmp = destFirstDerivTmp.cwiseProduct(preOrderPartial); - std::cerr<<"q'Qp = " << std::endl << destFirstDerivTmp << std::endl; +// std::cerr<<"q'Qp = " << std::endl << destFirstDerivTmp << std::endl; destSecondDerivTmp = postOrderPartial.cwiseProduct(preOrderPartial); - std::cerr<<"q'p = " << std::endl << destSecondDerivTmp << std::endl; +// std::cerr<<"q'p = " << std::endl << destSecondDerivTmp << std::endl; destNumeratorDrivTmp += destFirstDerivTmp.colwise().sum() * categoryMultiplier; - std::cerr<<"colSum(q'Qp)rw = " << std::endl << destNumeratorDrivTmp << std::endl; - destDenominatorDrivTmp += destSecondDerivTmp.colwise().sum() * categoryMultiplier; - std::cerr<<"colSum(q'p)rw = " << std::endl << destDenominatorDrivTmp << std::endl; +// std::cerr<<"colSum(q'Qp)rw = " << std::endl << destNumeratorDrivTmp << std::endl; + destDenominatorDrivTmp += destSecondDerivTmp.colwise().sum() * weight; +// std::cerr<<"colSum(q'p)rw = " << std::endl << destDenominatorDrivTmp << std::endl; } } @@ -655,7 +655,7 @@ namespace beagle { resetDerivativeTemporaries(); - std::cerr<<"Node = " << nodeNum << std::endl; +// std::cerr<<"Node = " << nodeNum << std::endl; calcEdgeLogDerivativesPartials(postOrderPartialindex, preOrderPartialIndex, firstDerivativeIndex, secondDerivativeIndex, categoryRates, categoryWeights, From 763f74b629d663902bb344afb5202c7c767b5ee3 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 7 May 2025 10:34:44 -0500 Subject: [PATCH 358/421] JNI interface update for new setSparseDifferentialMatrix function --- java/beagle/Beagle.java | 22 +- java/beagle/BeagleJNIImpl.java | 8 + java/beagle/BeagleJNIWrapper.java | 7 + java/beagle/GeneralBeagleImpl.java | 5 +- java/beagle_BeagleJNIWrapper.h | 405 +++++++++++++++++++++ libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h | 104 +++--- 6 files changed, 500 insertions(+), 51 deletions(-) create mode 100644 java/beagle_BeagleJNIWrapper.h diff --git a/java/beagle/Beagle.java b/java/beagle/Beagle.java index ad009ae8..eaa2ce57 100644 --- a/java/beagle/Beagle.java +++ b/java/beagle/Beagle.java @@ -282,19 +282,37 @@ void setEigenDecomposition( * * This function copies a sparse matrix into a instance buffer. * - * @param maatrixIndex Index of sparse matrix buffer (input) + * @param matrixIndex Index of sparse matrix buffer (input) * @param inRowIndices Row indices of non-zero entries (input) * @param inColIndices Column indices of non-zero entries (input) * @param inValues Vector of non-zero values (input) * @param numNonZeros Number of non-zero entries (input) */ void setSparseMatrix( - int maatrixIndex, + int matrixIndex, final int[] inRowIndices, final int[] inColIndices, final double[] inValues, int numNonZeros); + /** + * Set a sparse differential matrix + * + * This function copies a sparse matrix into a instance buffer. + * + * @param matrixIndex Index of sparse matrix buffer (input) + * @param inRowIndices Row indices of non-zero entries (input) + * @param inColIndices Column indices of non-zero entries (input) + * @param inValues Vector of non-zero values (input) + * @param numNonZeros Number of non-zero entries (input) + */ + void setSparseDifferentialMatrix( + int matrixIndex, + final int[] inRowIndices, + final int[] inColIndices, + final double[] inValues, + int numNonZeros); + /** * Set a set of state frequences. These will probably correspond to an * eigen-system. diff --git a/java/beagle/BeagleJNIImpl.java b/java/beagle/BeagleJNIImpl.java index c31924d2..a0b72f03 100644 --- a/java/beagle/BeagleJNIImpl.java +++ b/java/beagle/BeagleJNIImpl.java @@ -149,6 +149,14 @@ public void setSparseMatrix(int matrixIndex, int[] inRowIndices, int[] inColIndi } } + @Override + public void setSparseDifferentialMatrix(int matrixIndex, int[] inRowIndices, int[] inColIndices, double[] inValues, int numNonZeros) { + int errCode = BeagleJNIWrapper.INSTANCE.setSparseDifferentialMatrix(instance, matrixIndex, inRowIndices, inColIndices, inValues, numNonZeros); + if (errCode != 0) { + throw new BeagleException("setSparseDifferentialMatrix", errCode); + } + } + public void setStateFrequencies(int stateFrequenciesIndex, final double[] stateFrequencies) { int errCode = BeagleJNIWrapper.INSTANCE.setStateFrequencies(instance, diff --git a/java/beagle/BeagleJNIWrapper.java b/java/beagle/BeagleJNIWrapper.java index 60207f01..6286336e 100644 --- a/java/beagle/BeagleJNIWrapper.java +++ b/java/beagle/BeagleJNIWrapper.java @@ -97,6 +97,13 @@ public native int setSparseMatrix(int instance, final double[] values, int numNonZeros); + public native int setSparseDifferentialMatrix(int instance, + int matrixIndex, + final int[] rowIndices, + final int[] colIndices, + final double[] values, + int numNonZeros); + public native int setStateFrequencies(int instance, int stateFrequenciesIndex, final double[] stateFrequencies); diff --git a/java/beagle/GeneralBeagleImpl.java b/java/beagle/GeneralBeagleImpl.java index b37367a8..844d839a 100644 --- a/java/beagle/GeneralBeagleImpl.java +++ b/java/beagle/GeneralBeagleImpl.java @@ -229,7 +229,10 @@ public void setStateFrequencies(final int stateFrequenciesIndex, final double[] System.arraycopy(stateFrequencies, 0, this.stateFrequencies[stateFrequenciesIndex], 0, stateCount); } - public void setSparseMatrix(int maatrixIndex, final int[] inRowIndices, final int[] inColIndices, final double[] inValues, int numNonZeros) { + public void setSparseMatrix(int maatrixIndex, final int[] inRowIndices, final int[] inColIndices, final double[] inValues, int numNonZeros) { + }; + + public void setSparseDifferentialMatrix(int maatrixIndex, final int[] inRowIndices, final int[] inColIndices, final double[] inValues, int numNonZeros) { }; public void setCategoryWeights(final int categoryWeightsIndex, final double[] categoryWeights) { diff --git a/java/beagle_BeagleJNIWrapper.h b/java/beagle_BeagleJNIWrapper.h new file mode 100644 index 00000000..78897b7e --- /dev/null +++ b/java/beagle_BeagleJNIWrapper.h @@ -0,0 +1,405 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class beagle_BeagleJNIWrapper */ + +#ifndef _Included_beagle_BeagleJNIWrapper +#define _Included_beagle_BeagleJNIWrapper +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: beagle_BeagleJNIWrapper + * Method: getVersion + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_beagle_BeagleJNIWrapper_getVersion + (JNIEnv *, jobject); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: getCitation + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_beagle_BeagleJNIWrapper_getCitation + (JNIEnv *, jobject); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: getResourceList + * Signature: ()[Lbeagle/ResourceDetails; + */ +JNIEXPORT jobjectArray JNICALL Java_beagle_BeagleJNIWrapper_getResourceList + (JNIEnv *, jobject); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: getBenchmarkedResourceList + * Signature: (IIIII[IIJJIIIJ)[Lbeagle/BenchmarkedResourceDetails; + */ +JNIEXPORT jobjectArray JNICALL Java_beagle_BeagleJNIWrapper_getBenchmarkedResourceList + (JNIEnv *, jobject, jint, jint, jint, jint, jint, jintArray, jint, jlong, jlong, jint, jint, jint, jlong); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: createInstance + * Signature: (IIIIIIIII[IIJJLbeagle/InstanceDetails;)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_createInstance + (JNIEnv *, jobject, jint, jint, jint, jint, jint, jint, jint, jint, jint, jintArray, jint, jlong, jlong, jobject); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: finalize + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_finalize + (JNIEnv *, jobject, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setCPUThreadCount + * Signature: (II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCPUThreadCount + (JNIEnv *, jobject, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setPatternWeights + * Signature: (I[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPatternWeights + (JNIEnv *, jobject, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setPatternPartitions + * Signature: (II[I)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPatternPartitions + (JNIEnv *, jobject, jint, jint, jintArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setTipStates + * Signature: (II[I)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTipStates + (JNIEnv *, jobject, jint, jint, jintArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: getTipStates + * Signature: (II[I)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getTipStates + (JNIEnv *, jobject, jint, jint, jintArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setTipPartials + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTipPartials + (JNIEnv *, jobject, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setRootPrePartials + * Signature: (I[I[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setRootPrePartials + (JNIEnv *, jobject, jint, jintArray, jintArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setPartials + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPartials + (JNIEnv *, jobject, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: getPartials + * Signature: (III[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getPartials + (JNIEnv *, jobject, jint, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: getLogScaleFactors + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getLogScaleFactors + (JNIEnv *, jobject, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setEigenDecomposition + * Signature: (II[D[D[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setEigenDecomposition + (JNIEnv *, jobject, jint, jint, jdoubleArray, jdoubleArray, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setSparseMatrix + * Signature: (II[I[I[DI)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setSparseMatrix + (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jdoubleArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setSparseDifferentialMatrix + * Signature: (II[I[I[DI)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setSparseDifferentialMatrix + (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jdoubleArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setStateFrequencies + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setStateFrequencies + (JNIEnv *, jobject, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setCategoryWeights + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryWeights + (JNIEnv *, jobject, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setCategoryRates + * Signature: (I[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryRates + (JNIEnv *, jobject, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setCategoryRatesWithIndex + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryRatesWithIndex + (JNIEnv *, jobject, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setTransitionMatrix + * Signature: (II[DD)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTransitionMatrix + (JNIEnv *, jobject, jint, jint, jdoubleArray, jdouble); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setDifferentialMatrix + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setDifferentialMatrix + (JNIEnv *, jobject, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: getTransitionMatrix + * Signature: (II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getTransitionMatrix + (JNIEnv *, jobject, jint, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: convolveTransitionMatrices + * Signature: (I[I[I[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_convolveTransitionMatrices + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: addTransitionMatrices + * Signature: (I[I[I[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_addTransitionMatrices + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: transposeTransitionMatrices + * Signature: (I[I[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_transposeTransitionMatrices + (JNIEnv *, jobject, jint, jintArray, jintArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: updateTransitionMatrices + * Signature: (II[I[I[I[DI)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updateTransitionMatrices + (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jintArray, jdoubleArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: updateTransitionMatricesWithMultipleModels + * Signature: (I[I[I[I[I[I[DI)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updateTransitionMatricesWithMultipleModels + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jintArray, jdoubleArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: updatePrePartials + * Signature: (I[III)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePrePartials + (JNIEnv *, jobject, jint, jintArray, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: updatePrePartialsByPartition + * Signature: (I[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePrePartialsByPartition + (JNIEnv *, jobject, jint, jintArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: updatePartials + * Signature: (I[III)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePartials + (JNIEnv *, jobject, jint, jintArray, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: updatePartialsByPartition + * Signature: (I[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePartialsByPartition + (JNIEnv *, jobject, jint, jintArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: waitForPartials + * Signature: (I[II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_waitForPartials + (JNIEnv *, jobject, jint, jintArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: accumulateScaleFactors + * Signature: (I[III)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_accumulateScaleFactors + (JNIEnv *, jobject, jint, jintArray, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: accumulateScaleFactorsByPartition + * Signature: (I[IIII)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_accumulateScaleFactorsByPartition + (JNIEnv *, jobject, jint, jintArray, jint, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: removeScaleFactors + * Signature: (I[III)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_removeScaleFactors + (JNIEnv *, jobject, jint, jintArray, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: removeScaleFactorsByPartition + * Signature: (I[IIII)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_removeScaleFactorsByPartition + (JNIEnv *, jobject, jint, jintArray, jint, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: resetScaleFactors + * Signature: (II)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_resetScaleFactors + (JNIEnv *, jobject, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: resetScaleFactorsByPartition + * Signature: (III)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_resetScaleFactorsByPartition + (JNIEnv *, jobject, jint, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: copyScaleFactors + * Signature: (III)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_copyScaleFactors + (JNIEnv *, jobject, jint, jint, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: calculateRootLogLikelihoods + * Signature: (I[I[I[I[II[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateRootLogLikelihoods + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: calculateRootLogLikelihoodsByPartition + * Signature: (I[I[I[I[I[III[D[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateRootLogLikelihoodsByPartition + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jintArray, jint, jint, jdoubleArray, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: getSiteLogLikelihoods + * Signature: (I[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getSiteLogLikelihoods + (JNIEnv *, jobject, jint, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: calculateEdgeDifferentials + * Signature: (I[I[I[I[II[D[D[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateEdgeDifferentials + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jint, jdoubleArray, jdoubleArray, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: calculateCrossProductDifferentials + * Signature: (I[I[I[I[I[DI[D[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateCrossProductDifferentials + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jdoubleArray, jint, jdoubleArray, jdoubleArray); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: calculateEdgeDerivative + * Signature: (I[I[II[I[IIII[II[D[D)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateEdgeDerivative + (JNIEnv *, jobject, jint, jintArray, jintArray, jint, jintArray, jintArray, jint, jint, jint, jintArray, jint, jdoubleArray, jdoubleArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h b/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h index b362f377..1cdbeb37 100644 --- a/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h +++ b/libhmsbeagle/JNI/beagle_BeagleJNIWrapper.h @@ -13,7 +13,7 @@ extern "C" { * Signature: ()Ljava/lang/String; */ JNIEXPORT jstring JNICALL Java_beagle_BeagleJNIWrapper_getVersion - (JNIEnv *, jobject); + (JNIEnv *, jobject); /* * Class: beagle_BeagleJNIWrapper @@ -21,7 +21,7 @@ JNIEXPORT jstring JNICALL Java_beagle_BeagleJNIWrapper_getVersion * Signature: ()Ljava/lang/String; */ JNIEXPORT jstring JNICALL Java_beagle_BeagleJNIWrapper_getCitation - (JNIEnv *, jobject); + (JNIEnv *, jobject); /* * Class: beagle_BeagleJNIWrapper @@ -29,7 +29,7 @@ JNIEXPORT jstring JNICALL Java_beagle_BeagleJNIWrapper_getCitation * Signature: ()[Lbeagle/ResourceDetails; */ JNIEXPORT jobjectArray JNICALL Java_beagle_BeagleJNIWrapper_getResourceList - (JNIEnv *, jobject); + (JNIEnv *, jobject); /* * Class: beagle_BeagleJNIWrapper @@ -37,7 +37,7 @@ JNIEXPORT jobjectArray JNICALL Java_beagle_BeagleJNIWrapper_getResourceList * Signature: (IIIII[IIJJIIIJ)[Lbeagle/BenchmarkedResourceDetails; */ JNIEXPORT jobjectArray JNICALL Java_beagle_BeagleJNIWrapper_getBenchmarkedResourceList - (JNIEnv *, jobject, jint, jint, jint, jint, jint, jintArray, jint, jlong, jlong, jint, jint, jint, jlong); + (JNIEnv *, jobject, jint, jint, jint, jint, jint, jintArray, jint, jlong, jlong, jint, jint, jint, jlong); /* * Class: beagle_BeagleJNIWrapper @@ -45,7 +45,7 @@ JNIEXPORT jobjectArray JNICALL Java_beagle_BeagleJNIWrapper_getBenchmarkedResour * Signature: (IIIIIIIII[IIJJLbeagle/InstanceDetails;)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_createInstance - (JNIEnv *, jobject, jint, jint, jint, jint, jint, jint, jint, jint, jint, jintArray, jint, jlong, jlong, jobject); + (JNIEnv *, jobject, jint, jint, jint, jint, jint, jint, jint, jint, jint, jintArray, jint, jlong, jlong, jobject); /* * Class: beagle_BeagleJNIWrapper @@ -53,7 +53,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_createInstance * Signature: (I)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_finalize - (JNIEnv *, jobject, jint); + (JNIEnv *, jobject, jint); /* * Class: beagle_BeagleJNIWrapper @@ -61,7 +61,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_finalize * Signature: (II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCPUThreadCount - (JNIEnv *, jobject, jint, jint); + (JNIEnv *, jobject, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -69,7 +69,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCPUThreadCount * Signature: (I[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPatternWeights - (JNIEnv *, jobject, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -77,7 +77,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPatternWeights * Signature: (II[I)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPatternPartitions - (JNIEnv *, jobject, jint, jint, jintArray); + (JNIEnv *, jobject, jint, jint, jintArray); /* * Class: beagle_BeagleJNIWrapper @@ -85,7 +85,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPatternPartitions * Signature: (II[I)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTipStates - (JNIEnv *, jobject, jint, jint, jintArray); + (JNIEnv *, jobject, jint, jint, jintArray); /* * Class: beagle_BeagleJNIWrapper @@ -93,7 +93,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTipStates * Signature: (II[I)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getTipStates - (JNIEnv *, jobject, jint, jint, jintArray); + (JNIEnv *, jobject, jint, jint, jintArray); /* * Class: beagle_BeagleJNIWrapper @@ -101,7 +101,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getTipStates * Signature: (II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTipPartials - (JNIEnv *, jobject, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -109,7 +109,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTipPartials * Signature: (I[I[II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setRootPrePartials - (JNIEnv *, jobject, jint, jintArray, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jintArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -117,7 +117,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setRootPrePartials * Signature: (II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPartials - (JNIEnv *, jobject, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -125,7 +125,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setPartials * Signature: (III[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getPartials - (JNIEnv *, jobject, jint, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -133,7 +133,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getPartials * Signature: (II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getLogScaleFactors - (JNIEnv *, jobject, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -141,7 +141,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getLogScaleFactors * Signature: (II[D[D[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setEigenDecomposition - (JNIEnv *, jobject, jint, jint, jdoubleArray, jdoubleArray, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray, jdoubleArray, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -149,7 +149,15 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setEigenDecomposition * Signature: (II[I[I[DI)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setSparseMatrix - (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jdoubleArray, jint); + (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jdoubleArray, jint); + +/* + * Class: beagle_BeagleJNIWrapper + * Method: setSparseDifferentialMatrix + * Signature: (II[I[I[DI)I + */ +JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setSparseDifferentialMatrix + (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jdoubleArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -157,7 +165,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setSparseMatrix * Signature: (II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setStateFrequencies - (JNIEnv *, jobject, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -165,7 +173,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setStateFrequencies * Signature: (II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryWeights - (JNIEnv *, jobject, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -173,7 +181,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryWeights * Signature: (I[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryRates - (JNIEnv *, jobject, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -181,7 +189,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryRates * Signature: (II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryRatesWithIndex - (JNIEnv *, jobject, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -189,7 +197,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setCategoryRatesWithIndex * Signature: (II[DD)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTransitionMatrix - (JNIEnv *, jobject, jint, jint, jdoubleArray, jdouble); + (JNIEnv *, jobject, jint, jint, jdoubleArray, jdouble); /* * Class: beagle_BeagleJNIWrapper @@ -197,7 +205,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setTransitionMatrix * Signature: (II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setDifferentialMatrix - (JNIEnv *, jobject, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -205,7 +213,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_setDifferentialMatrix * Signature: (II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getTransitionMatrix - (JNIEnv *, jobject, jint, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -213,7 +221,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getTransitionMatrix * Signature: (I[I[I[II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_convolveTransitionMatrices - (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -221,7 +229,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_convolveTransitionMatrices * Signature: (I[I[I[II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_addTransitionMatrices - (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -229,7 +237,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_addTransitionMatrices * Signature: (I[I[II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_transposeTransitionMatrices - (JNIEnv *, jobject, jint, jintArray, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jintArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -237,7 +245,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_transposeTransitionMatrices * Signature: (II[I[I[I[DI)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updateTransitionMatrices - (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jintArray, jdoubleArray, jint); + (JNIEnv *, jobject, jint, jint, jintArray, jintArray, jintArray, jdoubleArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -245,7 +253,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updateTransitionMatrices * Signature: (I[I[I[I[I[I[DI)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updateTransitionMatricesWithMultipleModels - (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jintArray, jdoubleArray, jint); + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jintArray, jdoubleArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -253,7 +261,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updateTransitionMatricesWith * Signature: (I[III)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePrePartials - (JNIEnv *, jobject, jint, jintArray, jint, jint); + (JNIEnv *, jobject, jint, jintArray, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -261,7 +269,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePrePartials * Signature: (I[II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePrePartialsByPartition - (JNIEnv *, jobject, jint, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -269,7 +277,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePrePartialsByPartition * Signature: (I[III)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePartials - (JNIEnv *, jobject, jint, jintArray, jint, jint); + (JNIEnv *, jobject, jint, jintArray, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -277,7 +285,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePartials * Signature: (I[II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePartialsByPartition - (JNIEnv *, jobject, jint, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -285,7 +293,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_updatePartialsByPartition * Signature: (I[II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_waitForPartials - (JNIEnv *, jobject, jint, jintArray, jint); + (JNIEnv *, jobject, jint, jintArray, jint); /* * Class: beagle_BeagleJNIWrapper @@ -293,7 +301,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_waitForPartials * Signature: (I[III)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_accumulateScaleFactors - (JNIEnv *, jobject, jint, jintArray, jint, jint); + (JNIEnv *, jobject, jint, jintArray, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -301,7 +309,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_accumulateScaleFactors * Signature: (I[IIII)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_accumulateScaleFactorsByPartition - (JNIEnv *, jobject, jint, jintArray, jint, jint, jint); + (JNIEnv *, jobject, jint, jintArray, jint, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -309,7 +317,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_accumulateScaleFactorsByPart * Signature: (I[III)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_removeScaleFactors - (JNIEnv *, jobject, jint, jintArray, jint, jint); + (JNIEnv *, jobject, jint, jintArray, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -317,7 +325,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_removeScaleFactors * Signature: (I[IIII)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_removeScaleFactorsByPartition - (JNIEnv *, jobject, jint, jintArray, jint, jint, jint); + (JNIEnv *, jobject, jint, jintArray, jint, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -325,7 +333,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_removeScaleFactorsByPartitio * Signature: (II)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_resetScaleFactors - (JNIEnv *, jobject, jint, jint); + (JNIEnv *, jobject, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -333,7 +341,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_resetScaleFactors * Signature: (III)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_resetScaleFactorsByPartition - (JNIEnv *, jobject, jint, jint, jint); + (JNIEnv *, jobject, jint, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -341,7 +349,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_resetScaleFactorsByPartition * Signature: (III)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_copyScaleFactors - (JNIEnv *, jobject, jint, jint, jint); + (JNIEnv *, jobject, jint, jint, jint); /* * Class: beagle_BeagleJNIWrapper @@ -349,7 +357,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_copyScaleFactors * Signature: (I[I[I[I[II[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateRootLogLikelihoods - (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -357,7 +365,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateRootLogLikelihoods * Signature: (I[I[I[I[I[III[D[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateRootLogLikelihoodsByPartition - (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jintArray, jint, jint, jdoubleArray, jdoubleArray); + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jintArray, jint, jint, jdoubleArray, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -365,7 +373,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateRootLogLikelihoodsB * Signature: (I[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getSiteLogLikelihoods - (JNIEnv *, jobject, jint, jdoubleArray); + (JNIEnv *, jobject, jint, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -373,7 +381,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_getSiteLogLikelihoods * Signature: (I[I[I[I[II[D[D[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateEdgeDifferentials - (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jint, jdoubleArray, jdoubleArray, jdoubleArray); + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jint, jdoubleArray, jdoubleArray, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -381,7 +389,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateEdgeDifferentials * Signature: (I[I[I[I[I[DI[D[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateCrossProductDifferentials - (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jdoubleArray, jint, jdoubleArray, jdoubleArray); + (JNIEnv *, jobject, jint, jintArray, jintArray, jintArray, jintArray, jdoubleArray, jint, jdoubleArray, jdoubleArray); /* * Class: beagle_BeagleJNIWrapper @@ -389,7 +397,7 @@ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateCrossProductDiffere * Signature: (I[I[II[I[IIII[II[D[D)I */ JNIEXPORT jint JNICALL Java_beagle_BeagleJNIWrapper_calculateEdgeDerivative - (JNIEnv *, jobject, jint, jintArray, jintArray, jint, jintArray, jintArray, jint, jint, jint, jintArray, jint, jdoubleArray, jdoubleArray); + (JNIEnv *, jobject, jint, jintArray, jintArray, jint, jintArray, jintArray, jint, jint, jint, jintArray, jint, jdoubleArray, jdoubleArray); #ifdef __cplusplus } From 7239127b7dd92b7b56c7601b0104fcf1aed03514 Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 7 May 2025 13:35:48 -0500 Subject: [PATCH 359/421] update JNI interface --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 2 +- libhmsbeagle/JNI/beagle_BeagleJNIWrapper.cpp | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 045572e5..1d27d3ed 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -789,7 +789,7 @@ namespace beagle { gInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); - std::cout<<"Checking matrix: "<GetIntArrayElements(inRowIndices, NULL); + jint *colIndices = env->GetIntArrayElements(inColIndices, NULL); + jdouble *values = env->GetDoubleArrayElements(inValues, NULL); + + jint errCode = (jint)beagleSetSparseDifferentialMatrix(instance, matrixIndex, (int *)rowIndices, (int *)colIndices, (double *)values, numNonZeros); + + env->ReleaseIntArrayElements(inRowIndices, rowIndices, JNI_ABORT); + env->ReleaseIntArrayElements(inColIndices, colIndices, JNI_ABORT); + env->ReleaseDoubleArrayElements(inValues, values, JNI_ABORT); + + return errCode; +} + /* * Class: beagle_BeagleJNIWrapper * Method: setStateFrequencies From a187219c584372a097046832bb5030583e14d6cb Mon Sep 17 00:00:00 2001 From: xji3 Date: Wed, 14 May 2025 19:00:23 -0500 Subject: [PATCH 360/421] let action pre-order partial update multi-threading again --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 1d27d3ed..70d33f23 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -474,8 +474,12 @@ namespace beagle { if (cumulativeScaleIndex != BEAGLE_OP_NONE) cumulativeScaleBuffer = gScaleBuffers[cumulativeScaleIndex]; + int numOps = BEAGLE_OP_COUNT; + if (byPartition) + numOps = BEAGLE_PARTITION_OP_COUNT; + + for (int op = 0; op < operationCount; op++) { - int numOps = BEAGLE_OP_COUNT; // create a list of partial likelihood update operations // the order is [dest, destScaling, source1, matrix1, source2, matrix2] @@ -862,15 +866,15 @@ namespace beagle { int edgeIndex2, int startPattern, int endPattern) { - memset(gIntegrationTmp, 0, (kPatternCount * kStateCount * kCategoryCount)*sizeof(double)); + memset(gIntegrationTmp, 0, (kPatternCount * kStateCount * kCategoryCount) * sizeof(double)); for (int category = 0; category < kCategoryCount; category++) { - auto partialCache2 = partialsCacheMap(partials2Index, category, startPattern, endPattern); - auto partials1 = partialsMap(partials1Index, category, startPattern, endPattern); - auto destP = partialsMap(destPIndex, category, startPattern, endPattern); + auto partialCache2 = partialsCacheMap(partials2Index, category, startPattern, endPattern); + auto partials1 = partialsMap(partials1Index, category, startPattern, endPattern); + auto destP = partialsMap(destPIndex, category, startPattern, endPattern); gMappedIntegrationTmp[category] = partialCache2.cwiseProduct(partials1); - simpleAction2(destP, gMappedIntegrationTmp[category], edgeIndex1, category, true); + simpleAction2(destP, gMappedIntegrationTmp[category], edgeIndex1, category, true); } } From 30341ee936b73b31f3c8408bffccf8fe39298742 Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 16 May 2025 13:51:55 -0500 Subject: [PATCH 361/421] try enable multi threading for CPU pre-order update --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 15 ++++-- libhmsbeagle/CPU/BeagleCPUImpl.h | 3 ++ libhmsbeagle/CPU/BeagleCPUImpl.hpp | 63 ++++++++++++++++++++++-- 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 70d33f23..dd532d8e 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -866,15 +866,24 @@ namespace beagle { int edgeIndex2, int startPattern, int endPattern) { - memset(gIntegrationTmp, 0, (kPatternCount * kStateCount * kCategoryCount) * sizeof(double)); +// memset(gIntegrationTmp, 0, (kPatternCount * kStateCount * kCategoryCount) * sizeof(double)); for (int category = 0; category < kCategoryCount; category++) { +// memset(gIntegrationTmp + category * kPaddedPatternCount * kStateCount + startPattern * kStateCount, 0, (endPattern - startPattern) * kStateCount * sizeof(double)); + auto partialCache2 = partialsCacheMap(partials2Index, category, startPattern, endPattern); auto partials1 = partialsMap(partials1Index, category, startPattern, endPattern); auto destP = partialsMap(destPIndex, category, startPattern, endPattern); - gMappedIntegrationTmp[category] = partialCache2.cwiseProduct(partials1); - simpleAction2(destP, gMappedIntegrationTmp[category], edgeIndex1, category, true); + auto integrationMap = MapType(gIntegrationTmp + category * kPaddedPatternCount * kStateCount + startPattern * kStateCount, kStateCount, endPattern - startPattern); +// memset(integrationMap.data(), 0, (endPattern - startPattern) * kStateCount * sizeof(double)); + integrationMap = partialCache2; + integrationMap = integrationMap.cwiseProduct(partials1); +// integrationMap = partialCache2.cwiseProduct(partials1); + simpleAction2(destP, integrationMap, edgeIndex1, category, true); + +// gMappedIntegrationTmp[category] = partialCache2.cwiseProduct(partials1); +// simpleAction2(destP, gMappedIntegrationTmp[category], edgeIndex1, category, true); } } diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index 1e3e437e..83304059 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -633,6 +633,9 @@ class BeagleCPUImpl : public BeagleImpl { virtual int upPartialsByPartitionAsync(const int* operations, int operationCount); + virtual int upPrePartialsByPartitionAsync(const int* operations, + int operationCount); + virtual int reorderPatternsByPartition(); virtual void calcStatesStates(REALTYPE* destP, diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index ac3f03ca..e212ebb4 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -1508,8 +1508,21 @@ int BeagleCPUImpl::updatePrePartials(const int *operations, int cumulativeScaleIndex) { int returnCode = BEAGLE_ERROR_GENERAL; - bool byPartition = false; - returnCode = upPrePartials(byPartition, operations, count, cumulativeScaleIndex); + if (kAutoPartitioningEnabled) { + autoPartitionPartialsOperations(operations, + gAutoPartitionOperations, + count, + cumulativeScaleIndex); + count *= kPartitionCount; + returnCode = upPrePartialsByPartitionAsync((const int*) gAutoPartitionOperations, + count); + } else { + bool byPartition = false; + returnCode = upPartials(byPartition, + operations, + count, + cumulativeScaleIndex); + } return returnCode; } @@ -2492,9 +2505,8 @@ int BeagleCPUImpl::updatePrePartialsByPartition(const int* o int returnCode = BEAGLE_ERROR_GENERAL; if (kThreadingEnabled) { -// TODO -// returnCode = upPrePartialsByPartitionAsync(operations, -// count); + returnCode = upPrePartialsByPartitionAsync(operations, + count); } else { bool byPartition = true; returnCode = upPrePartials(byPartition, @@ -2567,6 +2579,47 @@ int BeagleCPUImpl::upPartialsByPartitionAsync(const int* ope return BEAGLE_SUCCESS; } +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::upPrePartialsByPartitionAsync(const int* operations, + int count) { + + int numOps = BEAGLE_PARTITION_OP_COUNT; + + memset(gThreadOpCounts, 0, sizeof(int) * kNumThreads); + + for (int i=0; i threadTask( + std::bind(&BeagleCPUImpl::upPrePartials, this, + true, + (const int*) gThreadOperations[i], + gThreadOpCounts[i], + BEAGLE_OP_NONE)); + + gFutures[i] = threadTask.get_future(); + threadData* td = &gThreads[i]; + + std::unique_lock l(td->m); + td->jobs.push(std::move(threadTask)); + l.unlock(); + + gThreads[i].cv.notify_one(); + } + + for (int i=0; i::upPartials(bool byPartition, const int* operations, From a15aff57a434ad887f8c1d9bceaa7207915df6c4 Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 16 May 2025 13:57:36 -0500 Subject: [PATCH 362/421] not working yet --- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index e212ebb4..5512306f 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -1508,21 +1508,21 @@ int BeagleCPUImpl::updatePrePartials(const int *operations, int cumulativeScaleIndex) { int returnCode = BEAGLE_ERROR_GENERAL; - if (kAutoPartitioningEnabled) { - autoPartitionPartialsOperations(operations, - gAutoPartitionOperations, - count, - cumulativeScaleIndex); - count *= kPartitionCount; - returnCode = upPrePartialsByPartitionAsync((const int*) gAutoPartitionOperations, - count); - } else { +// if (kAutoPartitioningEnabled) { +// autoPartitionPartialsOperations(operations, +// gAutoPartitionOperations, +// count, +// cumulativeScaleIndex); +// count *= kPartitionCount; +// returnCode = upPrePartialsByPartitionAsync((const int*) gAutoPartitionOperations, +// count); +// } else { bool byPartition = false; returnCode = upPartials(byPartition, operations, count, cumulativeScaleIndex); - } +// } return returnCode; } From ee36b46850596300e24e348bd66f80c03cf0563e Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 16 May 2025 15:31:01 -0500 Subject: [PATCH 363/421] action CPU multi-threading actually works, need to fix non-action code --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 6 +----- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index dd532d8e..ea72b736 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -869,17 +869,13 @@ namespace beagle { // memset(gIntegrationTmp, 0, (kPatternCount * kStateCount * kCategoryCount) * sizeof(double)); for (int category = 0; category < kCategoryCount; category++) { -// memset(gIntegrationTmp + category * kPaddedPatternCount * kStateCount + startPattern * kStateCount, 0, (endPattern - startPattern) * kStateCount * sizeof(double)); auto partialCache2 = partialsCacheMap(partials2Index, category, startPattern, endPattern); auto partials1 = partialsMap(partials1Index, category, startPattern, endPattern); auto destP = partialsMap(destPIndex, category, startPattern, endPattern); auto integrationMap = MapType(gIntegrationTmp + category * kPaddedPatternCount * kStateCount + startPattern * kStateCount, kStateCount, endPattern - startPattern); -// memset(integrationMap.data(), 0, (endPattern - startPattern) * kStateCount * sizeof(double)); - integrationMap = partialCache2; - integrationMap = integrationMap.cwiseProduct(partials1); -// integrationMap = partialCache2.cwiseProduct(partials1); + integrationMap = partialCache2.cwiseProduct(partials1); simpleAction2(destP, integrationMap, edgeIndex1, category, true); // gMappedIntegrationTmp[category] = partialCache2.cwiseProduct(partials1); diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 5512306f..e212ebb4 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -1508,21 +1508,21 @@ int BeagleCPUImpl::updatePrePartials(const int *operations, int cumulativeScaleIndex) { int returnCode = BEAGLE_ERROR_GENERAL; -// if (kAutoPartitioningEnabled) { -// autoPartitionPartialsOperations(operations, -// gAutoPartitionOperations, -// count, -// cumulativeScaleIndex); -// count *= kPartitionCount; -// returnCode = upPrePartialsByPartitionAsync((const int*) gAutoPartitionOperations, -// count); -// } else { + if (kAutoPartitioningEnabled) { + autoPartitionPartialsOperations(operations, + gAutoPartitionOperations, + count, + cumulativeScaleIndex); + count *= kPartitionCount; + returnCode = upPrePartialsByPartitionAsync((const int*) gAutoPartitionOperations, + count); + } else { bool byPartition = false; returnCode = upPartials(byPartition, operations, count, cumulativeScaleIndex); -// } + } return returnCode; } From 433b1c87b3b1fc880c1c5bc5d2a4140efe2051ff Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 19 May 2025 16:32:38 -0500 Subject: [PATCH 364/421] non-action pre order update mutli-threaded now --- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index e212ebb4..cd40ff55 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -5669,11 +5669,12 @@ void BeagleCPUImpl::calcPrePartialsPartials(REALTYPE* destP, int stateCountModFour = (kStateCount / 4) * 4; REALTYPE* tmpdestPtr = destP; //clean up the partial first, set every entry to 0 - std::fill(tmpdestPtr, tmpdestPtr + kPartialsSize, 0); +// std::fill(tmpdestPtr, tmpdestPtr + kPartialsSize, 0); #pragma omp parallel for num_threads(kCategoryCount) for (int l = 0; l < kCategoryCount; l++) { - int v = l*kPartialsPaddedStateCount*kPatternCount + kPartialsPaddedStateCount*startPattern; + int v = l*kPartialsPaddedStateCount*kPaddedPatternCount + kPartialsPaddedStateCount*startPattern; + std::fill(destP + v, destP + l*kPartialsPaddedStateCount*kPaddedPatternCount + kPartialsPaddedStateCount*endPattern, 0); int matrixOffset = l*kMatrixSize; const REALTYPE* partials1Ptr = &partials1[v]; const REALTYPE* partials2Ptr = &partials2[v]; @@ -5745,11 +5746,12 @@ void BeagleCPUImpl::calcPrePartialsStates(REALTYPE* destP, int stateCountModFour = (kStateCount / 4) * 4; REALTYPE* tmpdestPtr = destP; //clean up the partial first, set every entry to 0 - std::fill(tmpdestPtr, tmpdestPtr + kPartialsSize, 0); +// std::fill(tmpdestPtr, tmpdestPtr + kPartialsSize, 0); #pragma omp parallel for num_threads(kCategoryCount) for (int l = 0; l < kCategoryCount; l++) { - int v = l*kPartialsPaddedStateCount*kPatternCount + kPartialsPaddedStateCount*startPattern; + int v = l*kPartialsPaddedStateCount*kPaddedPatternCount + kPartialsPaddedStateCount*startPattern; + std::fill(destP + v, destP + l*kPartialsPaddedStateCount*kPaddedPatternCount + kPartialsPaddedStateCount*endPattern, 0); int matrixOffset = l*kMatrixSize; const REALTYPE* partials1Ptr = &partials1[v]; From f99a6fe1a832f97af764ac4454593f8e015e3ee1 Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 2 Jun 2025 11:54:49 -0500 Subject: [PATCH 365/421] make edge derivative reduction multi-threaded --- libhmsbeagle/CPU/BeagleCPU4StateImpl.h | 36 +- libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp | 6 +- libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h | 25 +- libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp | 43 +-- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 4 +- libhmsbeagle/CPU/BeagleCPUImpl.h | 41 ++- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 366 ++++++++++++++++---- 7 files changed, 389 insertions(+), 132 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPU4StateImpl.h b/libhmsbeagle/CPU/BeagleCPU4StateImpl.h index 1de8b804..d3e3c4fc 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateImpl.h +++ b/libhmsbeagle/CPU/BeagleCPU4StateImpl.h @@ -111,25 +111,27 @@ class BeagleCPU4StateImpl : public BeagleCPUImpl { int endPattern); virtual void calcEdgeLogDerivativesStates(const int *tipStates, - const REALTYPE *preOrderPartial, - const int firstDerivativeIndex, - const int secondDerivativeIndex, - const double *categoryRates, - const REALTYPE *categoryWeights, - double *outDerivatives, - double *outSumDerivatives, - double *outSumSquaredDerivatives); + const REALTYPE *preOrderPartial, + const int firstDerivativeIndex, + const int secondDerivativeIndex, + const double *categoryRates, + const REALTYPE *categoryWeights, + double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int cacheOffset); virtual void calcEdgeLogDerivativesPartials(const REALTYPE *postOrderPartial, - const REALTYPE *preOrderPartial, - const int firstDerivativeIndex, - const int secondDerivativeIndex, - const double *categoryRates, - const REALTYPE *categoryWeights, - const int scalingFactorsIndex, - double *outDerivatives, - double *outSumDerivatives, - double *outSumSquaredDerivatives); + const REALTYPE *preOrderPartial, + const int firstDerivativeIndex, + const int secondDerivativeIndex, + const double *categoryRates, + const REALTYPE *categoryWeights, + const int scalingFactorsIndex, + double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int cacheOffset); virtual void calcCrossProductsStates(const int *tipStates, const REALTYPE *preOrderPartial, diff --git a/libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp b/libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp index 14883c02..f4d4c756 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPU4StateImpl.hpp @@ -1008,7 +1008,8 @@ void BeagleCPU4StateImpl::calcEdgeLogDerivativesStates(const const REALTYPE *categoryWeights, double *outDerivatives, double *outSumDerivatives, - double *outSumSquaredDerivatives) { + double *outSumSquaredDerivatives, + int cacheOffset) { for (int category = 0; category < kCategoryCount; category++) { @@ -1047,7 +1048,8 @@ void BeagleCPU4StateImpl::calcEdgeLogDerivativesPartials(con const int scalingFactorsIndex, double *outDerivatives, double *outSumDerivatives, - double *outSumSquaredDerivatives) { + double *outSumSquaredDerivatives, + int cacheOffset) { const REALTYPE* transMatrix = gTransitionMatrices[firstDerivativeIndex]; diff --git a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h index 37e13390..2dc5a2c9 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h +++ b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.h @@ -165,26 +165,27 @@ class BeagleCPU4StateSSEImpl : public BeagleCPU4StateIm protected: virtual int getPaddedPatternsModulus(); - virtual void accumulateDerivatives(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); + virtual void accumulateDerivatives(double *outDerivatives, double *outSumDerivatives, + double *outSumSquaredDerivatives, int offset); private: template - void accumulateDerivativesDispatch1(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); + void + accumulateDerivativesDispatch1(double *outDerivatives, double *outSumDerivatives, double *outSumSquaredDerivatives, + int offset); template - void accumulateDerivativesDispatch2(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); + void accumulateDerivativesDispatch2(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset); template - void accumulateDerivativesImpl(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); + void accumulateDerivativesImpl(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset); virtual void calcStatesStates(double* destP, const int* states1, diff --git a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp index 8f04af87..34d4a86f 100644 --- a/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPU4StateSSEImpl.hpp @@ -639,10 +639,10 @@ void BeagleCPU4StateSSEImpl::calcCrossProductsStates(co } BEAGLE_CPU_4_SSE_TEMPLATE template -void BeagleCPU4StateSSEImpl::accumulateDerivativesImpl( - double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives) { +void BeagleCPU4StateSSEImpl::accumulateDerivativesImpl(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset) { V_Real vSum = VEC_SETZERO(); V_Real vSumSquared = VEC_SETZERO(); @@ -703,46 +703,47 @@ void BeagleCPU4StateSSEImpl::accumulateDerivativesImpl( } BEAGLE_CPU_4_SSE_TEMPLATE template -void BeagleCPU4StateSSEImpl::accumulateDerivativesDispatch2( - double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives) { +void BeagleCPU4StateSSEImpl::accumulateDerivativesDispatch2(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset) { if (outSumSquaredDerivatives == NULL) { accumulateDerivativesImpl( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } else { accumulateDerivativesImpl( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } } BEAGLE_CPU_4_SSE_TEMPLATE template -void BeagleCPU4StateSSEImpl::accumulateDerivativesDispatch1( - double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives) { +void BeagleCPU4StateSSEImpl::accumulateDerivativesDispatch1(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset) { if (outSumDerivatives == NULL) { accumulateDerivativesDispatch2( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } else { accumulateDerivativesDispatch2( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } } BEAGLE_CPU_4_SSE_TEMPLATE -void BeagleCPU4StateSSEImpl::accumulateDerivatives(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives) { +void BeagleCPU4StateSSEImpl::accumulateDerivatives(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset) { if (outDerivatives == NULL) { accumulateDerivativesDispatch1( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } else { accumulateDerivativesDispatch1( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } } diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index ea72b736..734d52c0 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -657,7 +657,7 @@ namespace beagle { double* outSumSquaredDerivativesForNode = (outSumSquaredDerivatives == NULL) ? NULL : outSumSquaredDerivatives + nodeNum; - resetDerivativeTemporaries(); + resetDerivativeTemporaries(0); // std::cerr<<"Node = " << nodeNum << std::endl; @@ -670,7 +670,7 @@ namespace beagle { accumulateDerivatives(outDerivativesForNode, outSumDerivativesForNode, - outSumSquaredDerivativesForNode); + outSumSquaredDerivativesForNode, 0); } diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.h b/libhmsbeagle/CPU/BeagleCPUImpl.h index 83304059..5478e40b 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUImpl.h @@ -559,6 +559,13 @@ class BeagleCPUImpl : public BeagleImpl { int count, int cumulativeScaleIndex); + + virtual int calcEdgeLogDerivativesByAutoPartitionAsync(const int *operations, + int count, + double *siteLogLikelihoods, + double *outLogFirstDerivatives, + double *outLogDiagonalSecondDerivatives); + virtual int calcEdgeLogDerivatives(const int *postBufferIndices, const int *preBufferIndices, const int *firstDerivativeIndices, @@ -571,16 +578,15 @@ class BeagleCPUImpl : public BeagleImpl { double *outLogFirstDerivatives, double *outLogDiagonalSecondDerivatives); - virtual void calcEdgeLogDerivativesStates(const int *tipStates, - const REALTYPE *preOrderPartial, + virtual void calcEdgeLogDerivativesStates(const int *tipStates, const REALTYPE *preOrderPartial, const int firstDerivativeIndex, const int secondDerivativeIndex, const double *categoryRates, const REALTYPE *categoryWeights, -// const REALTYPE *cumulativeScaleBuffer, double *siteLogLikelihoods, double *outLogFirstDerivatives, - double *outLogDiagonalSecondDerivatives); + double *outLogDiagonalSecondDerivatives, + int cacheOffset); virtual void calcEdgeLogDerivativesPartials(const REALTYPE *postOrderPartial, const REALTYPE *preOrderPartial, @@ -589,10 +595,10 @@ class BeagleCPUImpl : public BeagleImpl { const double *categoryRates, const REALTYPE *categoryWeights, const int scalingFactorsIndex, -// const REALTYPE *cumulativeScaleBuffer, double *siteLogLikelihoods, double *outLogFirstDerivatives, - double *outLogDiagonalSecondDerivatives); + double *outLogDiagonalSecondDerivatives, + int cacheOffset); virtual int calcCrossProducts(const int *postBufferIndices, const int *preBufferIndices, @@ -619,11 +625,10 @@ class BeagleCPUImpl : public BeagleImpl { double *outCrossProducts, double *outSumSquaredDerivatives); - virtual void resetDerivativeTemporaries(); + virtual void resetDerivativeTemporaries(int offset); - virtual void accumulateDerivatives(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); + virtual void accumulateDerivatives(double *outDerivatives, double *outSumDerivatives, + double *outSumSquaredDerivatives, int offset); virtual void autoPartitionPartialsOperations(const int* operations, int* partitionOperations, @@ -891,19 +896,17 @@ class BeagleCPUImpl : public BeagleImpl { private: template - void accumulateDerivativesDispatch1(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); + void + accumulateDerivativesDispatch1(double *outDerivatives, double *outSumDerivatives, double *outSumSquaredDerivatives, + int offset); template - void accumulateDerivativesDispatch2(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); + void accumulateDerivativesDispatch2(double *outDerivatives, double *outSumDerivatives, + double *outSumSquaredDerivatives, int offset); template - void accumulateDerivativesImpl(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives); + void accumulateDerivativesImpl(double *outDerivatives, double *outSumDerivatives, + double *outSumSquaredDerivatives, int offset); }; BEAGLE_CPU_FACTORY_TEMPLATE diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index cd40ff55..a9567c43 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -968,6 +968,8 @@ int BeagleCPUImpl::setPatternPartitions(int partitionCount, assert(inPatternPartitions != 0L); kPartitionCount = partitionCount; + grandDenominatorDerivTmp = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPaddedPatternCount * kPartitionCount); + grandNumeratorDerivTmp = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPaddedPatternCount * kPartitionCount); if (!kPartitionsInitialised) { gPatternPartitions = (int*) malloc(sizeof(int) * kPatternCount); @@ -2450,13 +2452,161 @@ int BeagleCPUImpl::calculateEdgeDerivatives(const int *postB double *outDerivatives, double *outSumDerivatives, double *outSumSquaredDerivatives) { - return calcEdgeLogDerivatives( - postBufferIndices, preBufferIndices, - derivativeMatrixIndices, NULL, - categoryWeightsIndices,categoryRatesIndices, cumulativeScaleIndices, - count, - outDerivatives, - outSumDerivatives, outSumSquaredDerivatives); + int returnCode = BEAGLE_ERROR_GENERAL; + + if (kAutoPartitioningEnabled && kPartitionCount > 1) { + + + + int numOps = BEAGLE_OP_COUNT; + int numOpsP = BEAGLE_PARTITION_OP_COUNT; + + int perThreadCount = (count - (count % kPartitionCount)) / kPartitionCount; + +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"Mark 0" << ", Original count = "<< count << ", perThreadCount = " << perThreadCount < threadTask( + std::bind(&BeagleCPUImpl::calcEdgeLogDerivativesByAutoPartitionAsync, this, + (const int*) gThreadOperations[i], + gThreadOpCounts[i], + outDerivatives, + outSumDerivatives, + outSumSquaredDerivatives)); + + gFutures[i] = threadTask.get_future(); + threadData* td = &gThreads[i]; + + std::unique_lock l(td->m); + td->jobs.push(std::move(threadTask)); + l.unlock(); + + gThreads[i].cv.notify_one(); + } + + for (int i=0; i::calcCrossProducts(const int *postBufferIn return returnCode; } +BEAGLE_CPU_TEMPLATE +int BeagleCPUImpl::calcEdgeLogDerivativesByAutoPartitionAsync(const int *operations, + int count, + double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives) { + + int returnCode = BEAGLE_SUCCESS; + + int numOps = BEAGLE_OP_COUNT; + int numOpsP = BEAGLE_PARTITION_OP_COUNT; + + const int secondDerivativeIndex = BEAGLE_OP_NONE; + const double *categoryRates = NULL; // gCategoryRates[categoryRatesIndices[0]]; // TODO Generalize + const REALTYPE *categoryWeights = gCategoryWeights[operations[3]]; // TODO Generalize + + + + for (int nodeNum = 0; nodeNum < count; nodeNum++) { + + const REALTYPE *preOrderPartial = gPartials[operations[nodeNum * numOpsP + 1]]; + const int *tipStates = gTipStates[operations[nodeNum * numOpsP]]; + + const int firstDerivativeIndex = operations[nodeNum * numOpsP + 2]; + const int scalingFactorsIndex = -1; // cumulativeScaleIndices[nodeNum]; + + const int patternOffset = operations[nodeNum * numOpsP + 4] * kPatternCount; + const int threadOffset = operations[nodeNum * numOpsP + 5] * kPatternCount; + + +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"Job = " << operations[nodeNum * numOpsP + 4] <::calcEdgeLogDerivatives(const int *postBufferIndices, const int *preBufferIndices, @@ -3098,30 +3342,30 @@ int BeagleCPUImpl::calcEdgeLogDerivatives(const int *postBuf double* outSumSquaredDerivativesForNode = (outSumSquaredDerivatives == NULL) ? NULL : outSumSquaredDerivatives + nodeNum; - resetDerivativeTemporaries(); + resetDerivativeTemporaries(0); if (tipStates != NULL) { calcEdgeLogDerivativesStates(tipStates, preOrderPartial, firstDerivativeIndex, - secondDerivativeIndex, categoryRates, categoryWeights, - outDerivativesForNode, - outSumDerivativesForNode, - outSumSquaredDerivativesForNode); + secondDerivativeIndex, categoryRates, categoryWeights, + outDerivativesForNode, + outSumDerivativesForNode, + outSumSquaredDerivativesForNode, 0); } else { const REALTYPE *postOrderPartial = gPartials[postBufferIndices[nodeNum]]; calcEdgeLogDerivativesPartials(postOrderPartial, preOrderPartial, firstDerivativeIndex, - secondDerivativeIndex, categoryRates, categoryWeights, - scalingFactorsIndex, - outDerivativesForNode, - outSumDerivativesForNode, - outSumSquaredDerivativesForNode); + secondDerivativeIndex, categoryRates, categoryWeights, + scalingFactorsIndex, + outDerivativesForNode, + outSumDerivativesForNode, + outSumSquaredDerivativesForNode, 0); } accumulateDerivatives(outDerivativesForNode, - outSumDerivativesForNode, - outSumSquaredDerivativesForNode); + outSumDerivativesForNode, + outSumSquaredDerivativesForNode, 0); } @@ -3129,16 +3373,16 @@ int BeagleCPUImpl::calcEdgeLogDerivatives(const int *postBuf } BEAGLE_CPU_TEMPLATE template -void BeagleCPUImpl::accumulateDerivativesImpl( - double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives) { +void BeagleCPUImpl::accumulateDerivativesImpl(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset) { REALTYPE sum = 0.0; REALTYPE sumSquared = 0.0; for (int k = 0; k < kPatternCount; k++) { - REALTYPE derivative = grandNumeratorDerivTmp[k] / grandDenominatorDerivTmp[k]; + REALTYPE derivative = grandNumeratorDerivTmp[offset + k] / grandDenominatorDerivTmp[offset + k]; if (DoDerivatives) { outDerivatives[k] = derivative; } @@ -3160,65 +3404,69 @@ void BeagleCPUImpl::accumulateDerivativesImpl( } BEAGLE_CPU_TEMPLATE template -void BeagleCPUImpl::accumulateDerivativesDispatch2( - double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives) { +void +BeagleCPUImpl::accumulateDerivativesDispatch2(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset) { if (outSumSquaredDerivatives == NULL) { accumulateDerivativesImpl( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } else { accumulateDerivativesImpl( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } } BEAGLE_CPU_TEMPLATE template -void BeagleCPUImpl::accumulateDerivativesDispatch1( - double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives) { +void +BeagleCPUImpl::accumulateDerivativesDispatch1(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset) { if (outSumDerivatives == NULL) { accumulateDerivativesDispatch2( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } else { accumulateDerivativesDispatch2( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } } BEAGLE_CPU_TEMPLATE -void BeagleCPUImpl::resetDerivativeTemporaries() { - std::fill(grandNumeratorDerivTmp, grandNumeratorDerivTmp + kPaddedPatternCount, 0); - std::fill(grandDenominatorDerivTmp, grandDenominatorDerivTmp + kPaddedPatternCount, 0); +void BeagleCPUImpl::resetDerivativeTemporaries(int offset) { + std::fill(grandNumeratorDerivTmp + offset, grandNumeratorDerivTmp + offset + kPaddedPatternCount, 0); + std::fill(grandDenominatorDerivTmp + offset, grandDenominatorDerivTmp + offset + kPaddedPatternCount, 0); } BEAGLE_CPU_TEMPLATE -void BeagleCPUImpl::accumulateDerivatives(double* outDerivatives, - double* outSumDerivatives, - double* outSumSquaredDerivatives) { +void BeagleCPUImpl::accumulateDerivatives(double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int offset) { if (outDerivatives == NULL) { accumulateDerivativesDispatch1( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } else { accumulateDerivativesDispatch1( - outDerivatives, outSumDerivatives, outSumSquaredDerivatives); + outDerivatives, outSumDerivatives, outSumSquaredDerivatives, offset); } } BEAGLE_CPU_TEMPLATE -void BeagleCPUImpl::calcEdgeLogDerivativesStates(const int *tipStates, - const REALTYPE *preOrderPartial, - const int firstDerivativeIndex, - const int secondDerivativeIndex, - const double *categoryRates, - const REALTYPE *categoryWeights, - double *outDerivatives, - double *outSumDerivatives, - double *outSumSquaredDerivatives) { +void +BeagleCPUImpl::calcEdgeLogDerivativesStates(const int *tipStates, const REALTYPE *preOrderPartial, + const int firstDerivativeIndex, + const int secondDerivativeIndex, + const double *categoryRates, + const REALTYPE *categoryWeights, + double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives, + int cacheOffset) { const REALTYPE *firstDerivMatrix = gTransitionMatrices[firstDerivativeIndex]; @@ -3239,8 +3487,8 @@ void BeagleCPUImpl::calcEdgeLogDerivativesStates(const int * preOrderPartial[patternIndex * kPartialsPaddedStateCount + k]; } - grandNumeratorDerivTmp[pattern] += categoryWeights[category] * numerator; - grandDenominatorDerivTmp[pattern] += categoryWeights[category] * denominator; + grandNumeratorDerivTmp[cacheOffset + pattern] += categoryWeights[category] * numerator; + grandDenominatorDerivTmp[cacheOffset + pattern] += categoryWeights[category] * denominator; } } } @@ -3371,10 +3619,10 @@ void BeagleCPUImpl::calcEdgeLogDerivativesPartials(const REA const double *categoryRates, const REALTYPE *categoryWeights, const int scalingFactorsIndex, -// const REALTYPE *cumulativeScaleBuffer, double *outDerivatives, double *outSumDerivatives, - double *outSumSquaredDerivatives) { + double *outSumSquaredDerivatives, + int cacheOffset) { const REALTYPE *firstDerivMatrix = gTransitionMatrices[firstDerivativeIndex]; @@ -3405,8 +3653,8 @@ void BeagleCPUImpl::calcEdgeLogDerivativesPartials(const REA denominator += postOrderPartial[v + k] * preOrderPartial[v + k]; } - grandNumeratorDerivTmp[pattern] += weight * numerator; - grandDenominatorDerivTmp[pattern] += weight * denominator; + grandNumeratorDerivTmp[cacheOffset + pattern] += weight * numerator; + grandDenominatorDerivTmp[cacheOffset + pattern] += weight * denominator; } } } From 0246298a18e18a085d9866093dce6c2d91a4925c Mon Sep 17 00:00:00 2001 From: xji3 Date: Mon, 2 Jun 2025 14:52:16 -0500 Subject: [PATCH 366/421] make action edge derivative reduction multi-threaded --- libhmsbeagle/CPU/BeagleCPUActionImpl.h | 9 +- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 107 +++++++++++++++++++++-- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 29 +++--- 3 files changed, 117 insertions(+), 28 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index 6f8a59d8..1d648773 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -217,7 +217,14 @@ namespace beagle { const int scalingFactorsIndex, double *siteLogLikelihoods, double *outLogFirstDerivatives, - double *outLogDiagonalSecondDerivatives); + double *outLogDiagonalSecondDerivatives, + int offset); + + int calcEdgeLogDerivativesByAutoPartitionAsync(const int *operations, + int count, + double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives); virtual int setEigenDecomposition(int eigenIndex, const double *inEigenVectors, diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 734d52c0..d41b5938 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -578,26 +578,31 @@ namespace beagle { const int scalingFactorsIndex, double *outDerivativesForNode, double *outSumDerivativesForNode, - double *outSumSquaredDerivativesForNode) + double *outSumSquaredDerivativesForNode, + int offset) { - auto destNumeratorDrivTmp = MapType(grandNumeratorDerivTmp, 1, kPatternCount); - auto destDenominatorDrivTmp = MapType(grandDenominatorDerivTmp, 1, kPatternCount); - auto destFirstDerivTmp = MapType(firstDerivTmp, kStateCount, kPatternCount); - auto destSecondDerivTmp = MapType(secondDerivTmp, kStateCount, kPatternCount); + + auto destNumeratorDrivTmp = MapType(grandNumeratorDerivTmp + offset * kPatternCount , 1, kPatternCount); + auto destDenominatorDrivTmp = MapType(grandDenominatorDerivTmp + offset * kPatternCount, 1, kPatternCount); + auto destFirstDerivTmp = MapType(firstDerivTmp + offset * kPatternCount * kStateCount, kStateCount, kPatternCount); + auto destSecondDerivTmp = MapType(secondDerivTmp + offset * kPatternCount * kStateCount, kStateCount, kPatternCount); SpMatrix differentialMatrix = gInstantaneousMatrices[firstDerivativeIndex]; -// std::cerr<<"dQ = " << differentialMatrix << std::endl; +// std::cerr<<"dQ = " << differentialMatrix << ", offset = "<< offset << std::endl; +// std::cerr<< "offset = "<< offset << std::endl; for (int category = 0; category < kCategoryCount; category++) { const double weight = categoryWeights[category]; +// std::cerr<<"Category = " << category << " ; multiplier = " << weight << " * " << categoryRates[category] << std::endl; + MapType postOrderPartial = partialsMap(postOrderPartialIndex, category, 0, kPatternCount); MapType preOrderPartial = partialsMap(preOrderPartialIndex, category, 0, kPatternCount); const double categoryMultiplier = weight * categoryRates[category]; -// std::cerr<<"Category = " << category << " ; multiplier = " << weight << " * " << categoryRates[category] << " = " << categoryMultiplier << std::endl; +// std::cerr<< " = " << categoryMultiplier << std::endl; destFirstDerivTmp = differentialMatrix * postOrderPartial; @@ -619,6 +624,91 @@ namespace beagle { destDenominatorDrivTmp += destSecondDerivTmp.colwise().sum() * weight; // std::cerr<<"colSum(q'p)rw = " << std::endl << destDenominatorDrivTmp << std::endl; } +// std::cerr<< "reduction finished"<< std::endl; + } + + BEAGLE_CPU_ACTION_TEMPLATE + int BeagleCPUActionImpl::calcEdgeLogDerivativesByAutoPartitionAsync(const int *operations, + int count, + double *outDerivatives, + double *outSumDerivatives, + double *outSumSquaredDerivatives) { + + int returnCode = BEAGLE_SUCCESS; + + int numOps = BEAGLE_OP_COUNT; + int numOpsP = BEAGLE_PARTITION_OP_COUNT; + + const int secondDerivativeIndex = BEAGLE_OP_NONE; + const double *categoryRates = gCategoryRates[0]; // TODO Generalize + const double *categoryWeights = gCategoryWeights[operations[3]]; // TODO Generalize + + + + for (int nodeNum = 0; nodeNum < count; nodeNum++) { + + const int preOrderPartialIndex = operations[nodeNum * numOpsP + 1]; + const int postOrderPartialindex = operations[nodeNum * numOpsP]; + + + const int firstDerivativeIndex = operations[nodeNum * numOpsP + 2]; + const int scalingFactorsIndex = -1; // cumulativeScaleIndices[nodeNum]; + + const int patternOffset = operations[nodeNum * numOpsP + 4] * kPatternCount; + const int threadOffset = operations[nodeNum * numOpsP + 5] * kPatternCount; + +#ifdef BEAGLE_DEBUG_FLOW + std::cerr<<"Job = " << operations[nodeNum * numOpsP + 4] <::setPatternPartitions(int partitionCount, kPartitionCount = partitionCount; grandDenominatorDerivTmp = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPaddedPatternCount * kPartitionCount); grandNumeratorDerivTmp = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPaddedPatternCount * kPartitionCount); + firstDerivTmp = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPaddedPatternCount * kStateCount * kPartitionCount); + secondDerivTmp = (REALTYPE*) mallocAligned(sizeof(REALTYPE) * kPaddedPatternCount * kStateCount * kPartitionCount); + if (!kPartitionsInitialised) { gPatternPartitions = (int*) malloc(sizeof(int) * kPatternCount); @@ -2505,32 +2508,20 @@ int BeagleCPUImpl::calculateEdgeDerivatives(const int *postB for (int j=0; j normest1_all(const SpMatrix& A, int pMax, int t=2, int itmax=5) +{ + std::vector norms(pMax+1, 1.0); + + for(int p=0; p<=pMax; p++) + { + norms[p] = normest1(A, p, t, itmax); + } + + return norms; +} + +std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int itmax=5) +{ + assert(pMax >= 0); + assert(t != 0); // negative means t = n + assert(itmax >= 1); + + // A is (n,n); + assert(A.rows() == A.cols()); + int n = A.cols(); + + // Handle t too large + t = std::min(n,t); + + // Interpret negative t as t == n + if (t < 0) t = n; + + std::vector norms(pMax+1, 1.0); + + for(int p=0; p<=pMax; p++) + { + if (p == 0) + { + norms[p] = 1.0; + continue; + } + + // Defer to normP1 if p=1 and n is small or we want an exact answer. + if (p == 1 and (n <= 4 or t == n)) + { + norms[p] = normP1(A); + continue; + } + + // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. + MatrixXd X(n,t); + // We choose the first column to be all 1s. + X.col(0).setOnes(); + // The other columns have randomly chosen {-1,+1} entries. + X = X.unaryExpr( &random_plus_minus_1_func ); + // Divide by n so that the norm of each column is 1. + X /= n; + + // 3. + std::vector ind_hist(n,0); + std::vector indices(n,0); + int ind_best = -1; + double est_old = 0; + MatrixXd S = MatrixXd::Ones(n,t); + MatrixXd S_old = MatrixXd::Ones(n,t); + MatrixXi prodS(t,t); + MatrixXd Y(n,t); + MatrixXd Z(n,t); + Eigen::VectorXd h(n); + + std::optional result; + + for(int k=1; k<=itmax and not result; k++) + { + // std::cerr<<"iter "< est_old or k == 2) + { + // Note that j is in [0,t-1], but indices[j] is in [0,n-1]. + ind_best = indices[j]; + // w = Y.col(ind_best); + } + // std::cerr<<" est = "<= 2) + { + // std::cerr<<" The new estimate ("<= est_old); + + // S = sign(Y), 0.0 -> 1.0 + S = Y.unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); + + // prodS is (t,t) + prodS = (S_old.transpose() * S).matrix().cwiseAbs().cast() ; + + // (2) If each columns in S is parallel to SOME column of S_old + if (prodS.colwise().maxCoeff().sum() == n * t and k >= 2) + { + // std::cerr<<" All columns of S parallel to S_old\n"; + result = est; + break; + } + + if (t > 1) + { + // If S(j) is parallel to S_old(i), replace S(j) with random column + for(int j=0;j() ; + for(int i=0;i (n,t) + + // Maximize across each the t entries in each row of Z. + h = Z.cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) + + // (4) of Algorithm 2.4 + if (k >= 2 and h.maxCoeff() == h[ind_best]) + { + // std::cerr<<" The best column ("< h[j];}); + + // (5) of Algorithm 2.4 + int n_found = 0; + for(int i=0;i= 2); + // std::cerr<<" All columns were found in the column history.\n"; + result = est; + break; + } + + // find the first t indices that are not in ind_hist + int l=0; + for(int i=0;i(t, indices.size()); + + X = MatrixXd::Zero(n, tmax); + for(int j=0; j < tmax; j++) + X(indices[j], j) = 1; // X(:,j) = e(indices[j]) + + for(int i: indices) + ind_hist[i] = true; + + S_old = S; + } + + norms[p] = result.value(); + } + + auto sep_norms = normest1_all(A,pMax,t,itmax); + for(int p=0;p approx_norms = normest1_merged( gBs[eigenIndex], pMax+1, t); - // equation 3.7 in Al-Mohy and Higham - ds[eigenIndex].push_back( pow( approx_norm, 1.0/double(p) ) ); - } + // equation 3.7 in Al-Mohy and Higham + for(int p=0;p <= pMax+1; p++) + ds[eigenIndex].push_back( pow( approx_norms[p], 1.0/double(p) ) ); // gSimpleActions[eigenIndex]->setInstantaneousMatrix(tripletList); // gSimpleActions[eigenIndex]->fireMatrixChanged(); @@ -858,13 +1069,12 @@ namespace beagle { ds[matrixIndex].clear(); int pMax = getPMax(); + int t = 5; + auto approx_norms = normest1_merged( gBs[matrixIndex], pMax+1, t); for(int p=0;p <= pMax+1; p++) { - int t = 5; - double approx_norm = normest1( gBs[matrixIndex], p, t); - // equation 3.7 in Al-Mohy and Higham - ds[matrixIndex].push_back( pow( approx_norm, 1.0/double(p) ) ); + ds[matrixIndex].push_back( pow( approx_norms[p], 1.0/double(p) ) ); } return BEAGLE_SUCCESS; From 6ebc30b79a987177d0b57a809198c3774d9c4fde Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 15:47:35 -0400 Subject: [PATCH 369/421] Remove early exit based on prodS --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 38 ------------------------ 1 file changed, 38 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index bd563727..9b21f10e 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -330,7 +330,6 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it double est_old = 0; MatrixXd S = MatrixXd::Ones(n,t); MatrixXd S_old = MatrixXd::Ones(n,t); - MatrixXi prodS(t,t); MatrixXd Y(n,t); MatrixXd Z(n,t); Eigen::VectorXd h(n); @@ -370,43 +369,6 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it // S = sign(Y), 0.0 -> 1.0 S = Y.unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); - // prodS is (t,t) - prodS = (S_old.transpose() * S).matrix().cwiseAbs().cast() ; - - // (2) If each columns in S is parallel to SOME column of S_old - if (prodS.colwise().maxCoeff().sum() == n * t and k >= 2) - { - // std::cerr<<" All columns of S parallel to S_old\n"; - result = est; - break; - } - - if (t > 1) - { - // If S(j) is parallel to S_old(i), replace S(j) with random column - for(int j=0;j() ; - for(int i=0;i (n,t) From 40a60444939c101f70847ed99fd4faa092307433 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 15:49:57 -0400 Subject: [PATCH 370/421] Remove unused S_old. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 9b21f10e..dcdffd3d 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -329,7 +329,6 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it int ind_best = -1; double est_old = 0; MatrixXd S = MatrixXd::Ones(n,t); - MatrixXd S_old = MatrixXd::Ones(n,t); MatrixXd Y(n,t); MatrixXd Z(n,t); Eigen::VectorXd h(n); @@ -375,19 +374,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it // Maximize across each the t entries in each row of Z. h = Z.cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) - // (4) of Algorithm 2.4 - if (k >= 2 and h.maxCoeff() == h[ind_best]) - { - // std::cerr<<" The best column ("< normest1_merged(const SpMatrix& A, int pMax, int t=2, int it for(int i: indices) ind_hist[i] = true; - - S_old = S; } norms[p] = result.value(); From 4032df0c040e22952181aa3a3d3e918d16941cd6 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 15:57:05 -0400 Subject: [PATCH 371/421] Make X into a vector of matrices. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index dcdffd3d..e8a4c77a 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -299,6 +299,8 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector norms(pMax+1, 1.0); + std::vector X(pMax+1); + for(int p=0; p<=pMax; p++) { if (p == 0) @@ -315,13 +317,13 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it } // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. - MatrixXd X(n,t); + X[p].resize(n,t); // We choose the first column to be all 1s. - X.col(0).setOnes(); + X[p].col(0).setOnes(); // The other columns have randomly chosen {-1,+1} entries. - X = X.unaryExpr( &random_plus_minus_1_func ); + X[p] = X[p].unaryExpr( &random_plus_minus_1_func ); // Divide by n so that the norm of each column is 1. - X /= n; + X[p] /= n; // 3. std::vector ind_hist(n,0); @@ -338,7 +340,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it for(int k=1; k<=itmax and not result; k++) { // std::cerr<<"iter "< normest1_merged(const SpMatrix& A, int pMax, int t=2, int it int tmax = std::min(t, indices.size()); - X = MatrixXd::Zero(n, tmax); + X[p] = MatrixXd::Zero(n, tmax); for(int j=0; j < tmax; j++) - X(indices[j], j) = 1; // X(:,j) = e(indices[j]) + X[p](indices[j], j) = 1; // X(:,j) = e(indices[j]) for(int i: indices) ind_hist[i] = true; From bcddfa21f8919ee81b20fc11ecd4e2e4c8a19fbb Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 16:03:01 -0400 Subject: [PATCH 372/421] Move Y, Z, S, and h out of the inner loop. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index e8a4c77a..c90af730 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -300,6 +300,10 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector norms(pMax+1, 1.0); std::vector X(pMax+1); + std::vector Y(pMax+1, MatrixXd(n,t)); + std::vector Z(pMax+1, MatrixXd(n,t)); + std::vector S(pMax+1, MatrixXd::Ones(n,t)); + std::vector h(pMax+1, Eigen::VectorXd(n)); for(int p=0; p<=pMax; p++) { @@ -330,27 +334,23 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector indices(n,0); int ind_best = -1; double est_old = 0; - MatrixXd S = MatrixXd::Ones(n,t); - MatrixXd Y(n,t); - MatrixXd Z(n,t); - Eigen::VectorXd h(n); std::optional result; for(int k=1; k<=itmax and not result; k++) { // std::cerr<<"iter "< est_old or k == 2) { // Note that j is in [0,t-1], but indices[j] is in [0,n-1]. ind_best = indices[j]; - // w = Y.col(ind_best); + // w = Y[p].col(ind_best); } // std::cerr<<" est = "< normest1_merged(const SpMatrix& A, int pMax, int t=2, int it assert(est >= est_old); - // S = sign(Y), 0.0 -> 1.0 - S = Y.unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); + // S[p] = sign(Y[p]), 0.0 -> 1.0 + S[p] = Y[p].unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); // (3) of Algorithm 2.4 - Z = A.transpose() * S; // (n,n) * (n,t) -> (n,t) + Z[p] = A.transpose() * S[p]; // (n,n) * (n,t) -> (n,t) - // Maximize across each the t entries in each row of Z. - h = Z.cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) + // Maximize across each the t entries in each row of Z[p]. + h[p] = Z[p].cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) // Sort dimensions based on h indices.resize(n); @@ -382,7 +382,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it indices[i] = i; // reorder idx so that the highest values of h[indices[i]] come first. - std::sort(indices.begin(), indices.end(), [&](int i,int j) {return h[i] > h[j];}); + std::sort(indices.begin(), indices.end(), [&](int i,int j) {return h[p][i] > h[p][j];}); // (5) of Algorithm 2.4 int n_found = 0; From 406616d0bc69c2f315027b1cf71db535bd9a0e0a Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 16:11:22 -0400 Subject: [PATCH 373/421] Move other quantities out of the loop. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 48 ++++++++++++------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index c90af730..05d1d8b5 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -304,6 +304,10 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector Z(pMax+1, MatrixXd(n,t)); std::vector S(pMax+1, MatrixXd::Ones(n,t)); std::vector h(pMax+1, Eigen::VectorXd(n)); + std::vector ind_best(pMax+1, -1); + std::vector est_old(pMax+1,0); + std::vector> indices(pMax+1,std::vector(n,0)); + std::vector> ind_hist(pMax+1, std::vector(n,0)); for(int p=0; p<=pMax; p++) { @@ -330,10 +334,6 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it X[p] /= n; // 3. - std::vector ind_hist(n,0); - std::vector indices(n,0); - int ind_best = -1; - double est_old = 0; std::optional result; @@ -346,26 +346,26 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it auto [est, j] = ArgNormP1(Y[p]); - if (est > est_old or k == 2) + if (est > est_old[p] or k == 2) { // Note that j is in [0,t-1], but indices[j] is in [0,n-1]. - ind_best = indices[j]; + ind_best[p] = indices[p][j]; // w = Y[p].col(ind_best); } // std::cerr<<" est = "<= 2) + if (est < est_old[p] and k >= 2) { // std::cerr<<" The new estimate ("<= est_old); + assert(est >= est_old[p]); // S[p] = sign(Y[p]), 0.0 -> 1.0 S[p] = Y[p].unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); @@ -377,17 +377,17 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it h[p] = Z[p].cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) // Sort dimensions based on h - indices.resize(n); + indices[p].resize(n); for(int i=0;i h[p][j];}); + std::sort(indices[p].begin(), indices[p].end(), [&](int i,int j) {return h[p][i] > h[p][j];}); // (5) of Algorithm 2.4 int n_found = 0; for(int i=0;i normest1_merged(const SpMatrix& A, int pMax, int t=2, int it // find the first t indices that are not in ind_hist int l=0; - for(int i=0;i(t, indices.size()); + int tmax = std::min(t, indices[p].size()); X[p] = MatrixXd::Zero(n, tmax); for(int j=0; j < tmax; j++) - X[p](indices[j], j) = 1; // X(:,j) = e(indices[j]) + X[p](indices[p][j], j) = 1; // X(:,j) = e(indices[j]) - for(int i: indices) - ind_hist[i] = true; + for(int i: indices[p]) + ind_hist[p][i] = true; } norms[p] = result.value(); From 5bc8cd2942f5dce374068d5bc06ea5a9fdee0f41 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 16:18:39 -0400 Subject: [PATCH 374/421] Split the loop into two pieces. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 05d1d8b5..509cfc35 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -299,7 +299,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector norms(pMax+1, 1.0); - std::vector X(pMax+1); + std::vector X(pMax+1, MatrixXd(n,t)); std::vector Y(pMax+1, MatrixXd(n,t)); std::vector Z(pMax+1, MatrixXd(n,t)); std::vector S(pMax+1, MatrixXd::Ones(n,t)); @@ -325,15 +325,26 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it } // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. - X[p].resize(n,t); // We choose the first column to be all 1s. X[p].col(0).setOnes(); // The other columns have randomly chosen {-1,+1} entries. X[p] = X[p].unaryExpr( &random_plus_minus_1_func ); // Divide by n so that the norm of each column is 1. X[p] /= n; + } - // 3. + for(int p=0; p<=pMax; p++) + { + if (p == 0) + { + continue; + } + + // Defer to normP1 if p=1 and n is small or we want an exact answer. + if (p == 1 and (n <= 4 or t == n)) + { + continue; + } std::optional result; From 89caaf742f37b0fbae9fe6f7bd65dfc8b9437289 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 16:19:26 -0400 Subject: [PATCH 375/421] Simplify exit condition. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 28 ++---------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 509cfc35..4ca1aa6a 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -309,21 +309,8 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector> indices(pMax+1,std::vector(n,0)); std::vector> ind_hist(pMax+1, std::vector(n,0)); - for(int p=0; p<=pMax; p++) + for(int p=1; p<=pMax; p++) { - if (p == 0) - { - norms[p] = 1.0; - continue; - } - - // Defer to normP1 if p=1 and n is small or we want an exact answer. - if (p == 1 and (n <= 4 or t == n)) - { - norms[p] = normP1(A); - continue; - } - // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. // We choose the first column to be all 1s. X[p].col(0).setOnes(); @@ -333,19 +320,8 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it X[p] /= n; } - for(int p=0; p<=pMax; p++) + for(int p=1; p<=pMax; p++) { - if (p == 0) - { - continue; - } - - // Defer to normP1 if p=1 and n is small or we want an exact answer. - if (p == 1 and (n <= 4 or t == n)) - { - continue; - } - std::optional result; for(int k=1; k<=itmax and not result; k++) From 575d10fd2f48e6833ec071ab6fa0a3d1e0823d46 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 16:22:43 -0400 Subject: [PATCH 376/421] Make separate result variables. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 4ca1aa6a..6d4ed2e1 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -308,6 +308,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector est_old(pMax+1,0); std::vector> indices(pMax+1,std::vector(n,0)); std::vector> ind_hist(pMax+1, std::vector(n,0)); + std::vector> result(pMax+1); for(int p=1; p<=pMax; p++) { @@ -322,9 +323,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it for(int p=1; p<=pMax; p++) { - std::optional result; - - for(int k=1; k<=itmax and not result; k++) + for(int k=1; k<=itmax and not result[p]; k++) { // std::cerr<<"iter "< normest1_merged(const SpMatrix& A, int pMax, int t=2, int it if (est < est_old[p] and k >= 2) { // std::cerr<<" The new estimate ("< normest1_merged(const SpMatrix& A, int pMax, int t=2, int it { assert(k >= 2); // std::cerr<<" All columns were found in the column history.\n"; - result = est; + result[p] = est; break; } @@ -407,13 +406,14 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it for(int i: indices[p]) ind_hist[p][i] = true; } - - norms[p] = result.value(); } auto sep_norms = normest1_all(A,pMax,t,itmax); - for(int p=0;p Date: Mon, 14 Jul 2025 16:25:06 -0400 Subject: [PATCH 377/421] Remove early exit condition. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 6d4ed2e1..7fae4388 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -323,7 +323,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it for(int p=1; p<=pMax; p++) { - for(int k=1; k<=itmax and not result[p]; k++) + for(int k=1; k<=itmax; k++) { // std::cerr<<"iter "< Date: Mon, 14 Jul 2025 16:30:35 -0400 Subject: [PATCH 378/421] Remove some exit conditions. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 7fae4388..aa8b97b4 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -297,7 +297,8 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it // Interpret negative t as t == n if (t < 0) t = n; - std::vector norms(pMax+1, 1.0); + std::vector norms(pMax+1, 0); + norms[0] = 1.0; std::vector X(pMax+1, MatrixXd(n,t)); std::vector Y(pMax+1, MatrixXd(n,t)); @@ -308,7 +309,6 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector est_old(pMax+1,0); std::vector> indices(pMax+1,std::vector(n,0)); std::vector> ind_hist(pMax+1, std::vector(n,0)); - std::vector> result(pMax+1); for(int p=1; p<=pMax; p++) { @@ -332,6 +332,8 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it auto [est, j] = ArgNormP1(Y[p]); + norms[p] = std::max(norms[p], est); + if (est > est_old[p] or k == 2) { // Note that j is in [0,t-1], but indices[j] is in [0,n-1]. @@ -341,14 +343,6 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it // std::cerr<<" est = "<= 2) - { - // std::cerr<<" The new estimate ("<= est_old[p]); @@ -376,14 +370,6 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it if (ind_hist[p][indices[p][i]]) n_found++; - if (n_found == t) - { - assert(k >= 2); - // std::cerr<<" All columns were found in the column history.\n"; - result[p] = est; - break; - } - // find the first t indices that are not in ind_hist int l=0; for(int i=0;i normest1_merged(const SpMatrix& A, int pMax, int t=2, int it auto sep_norms = normest1_all(A,pMax,t,itmax); for(int p=1;p Date: Mon, 14 Jul 2025 16:31:48 -0400 Subject: [PATCH 379/421] Interchange loop order. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index aa8b97b4..a99e8f57 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -321,9 +321,9 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it X[p] /= n; } - for(int p=1; p<=pMax; p++) + for(int k=1; k<=itmax; k++) { - for(int k=1; k<=itmax; k++) + for(int p=1; p<=pMax; p++) { // std::cerr<<"iter "< Date: Mon, 14 Jul 2025 16:49:29 -0400 Subject: [PATCH 380/421] Make S, Z, and h just temp vars. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 29 ++++++++++++------------ 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index a99e8f57..cd48a7d7 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -300,11 +300,12 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it std::vector norms(pMax+1, 0); norms[0] = 1.0; + MatrixXd S(n,t); + MatrixXd Z(n,t); + Eigen::VectorXd h(n); + std::vector X(pMax+1, MatrixXd(n,t)); std::vector Y(pMax+1, MatrixXd(n,t)); - std::vector Z(pMax+1, MatrixXd(n,t)); - std::vector S(pMax+1, MatrixXd::Ones(n,t)); - std::vector h(pMax+1, Eigen::VectorXd(n)); std::vector ind_best(pMax+1, -1); std::vector est_old(pMax+1,0); std::vector> indices(pMax+1,std::vector(n,0)); @@ -321,6 +322,10 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it X[p] /= n; } + std::vector all_indices; + all_indices.resize(n); + std::vector all_ind_hist(n,false); + for(int k=1; k<=itmax; k++) { for(int p=1; p<=pMax; p++) @@ -347,14 +352,14 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it assert(est >= est_old[p]); - // S[p] = sign(Y[p]), 0.0 -> 1.0 - S[p] = Y[p].unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); + // S = sign(Y[p]), 0.0 -> 1.0 + S = Y[p].unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); // (3) of Algorithm 2.4 - Z[p] = A.transpose() * S[p]; // (n,n) * (n,t) -> (n,t) + Z = A.transpose() * S; // (n,n) * (n,t) -> (n,t) - // Maximize across each the t entries in each row of Z[p]. - h[p] = Z[p].cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) + // Maximize across each the t entries in each row of Z. + h = Z.cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) // Sort dimensions based on h indices[p].resize(n); @@ -362,13 +367,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it indices[p][i] = i; // reorder idx so that the highest values of h[indices[i]] come first. - std::sort(indices[p].begin(), indices[p].end(), [&](int i,int j) {return h[p][i] > h[p][j];}); - - // (5) of Algorithm 2.4 - int n_found = 0; - for(int i=0;i h[j];}); // find the first t indices that are not in ind_hist int l=0; From 075e950e56dc7a6862f0933fa6feb2c358f7f512 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 16:49:41 -0400 Subject: [PATCH 381/421] Construct the X values for the next generation separately. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index cd48a7d7..fb34ffac 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -382,6 +382,10 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it indices[p].resize( std::min(l,t) ); assert(not indices[p].empty()); + } + + for(int p=1;p<=pMax;p++) + { int tmax = std::min(t, indices[p].size()); X[p] = MatrixXd::Zero(n, tmax); From 4f35b8aa4304963f7e4b7e8fd561695e60365c97 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 22:45:32 -0400 Subject: [PATCH 382/421] Use a single X for all values of p. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 50 +++++++++++++++--------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index fb34ffac..91ffdcbc 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -304,23 +304,20 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it MatrixXd Z(n,t); Eigen::VectorXd h(n); - std::vector X(pMax+1, MatrixXd(n,t)); + MatrixXd X(n,t); std::vector Y(pMax+1, MatrixXd(n,t)); std::vector ind_best(pMax+1, -1); std::vector est_old(pMax+1,0); std::vector> indices(pMax+1,std::vector(n,0)); std::vector> ind_hist(pMax+1, std::vector(n,0)); - for(int p=1; p<=pMax; p++) - { - // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. - // We choose the first column to be all 1s. - X[p].col(0).setOnes(); - // The other columns have randomly chosen {-1,+1} entries. - X[p] = X[p].unaryExpr( &random_plus_minus_1_func ); - // Divide by n so that the norm of each column is 1. - X[p] /= n; - } + // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. + // We choose the first column to be all 1s. + X.col(0).setOnes(); + // The other columns have randomly chosen {-1,+1} entries. + X = X.unaryExpr( &random_plus_minus_1_func ); + // Divide by n so that the norm of each column is 1. + X /= n; std::vector all_indices; all_indices.resize(n); @@ -339,10 +336,12 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it norms[p] = std::max(norms[p], est); + if (k == itmax) break; + if (est > est_old[p] or k == 2) { // Note that j is in [0,t-1], but indices[j] is in [0,n-1]. - ind_best[p] = indices[p][j]; + ind_best[p] = all_indices[j]; // w = Y[p].col(ind_best); } // std::cerr<<" est = "< normest1_merged(const SpMatrix& A, int pMax, int t=2, int it } + // Combine indices from different values of p + all_indices.clear(); for(int p=1;p<=pMax;p++) { - int tmax = std::min(t, indices[p].size()); - - X[p] = MatrixXd::Zero(n, tmax); - for(int j=0; j < tmax; j++) - X[p](indices[p][j], j) = 1; // X(:,j) = e(indices[j]) - for(int i: indices[p]) + { ind_hist[p][i] = true; + if (not all_ind_hist[i]) + { + all_indices.push_back(i); + all_ind_hist[i] = true; + } + } + } + + std::cerr<<"k = "< 0) + { + X = MatrixXd::Zero(n, tmax); + for(int j=0; j < tmax; j++) + X(all_indices[j], j) = 1; // X(:,j) = e(indices[j]) } } From 932963b40678e6129ea0991945e488bb77ff024c Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 22:46:43 -0400 Subject: [PATCH 383/421] Compute Y[p] from Y[p-1]. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 91ffdcbc..c3c9d4cc 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -328,9 +328,10 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it for(int p=1; p<=pMax; p++) { // std::cerr<<"iter "< Date: Mon, 14 Jul 2025 22:49:38 -0400 Subject: [PATCH 384/421] Use a single value of Y across P. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index c3c9d4cc..918cf440 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -305,7 +305,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it Eigen::VectorXd h(n); MatrixXd X(n,t); - std::vector Y(pMax+1, MatrixXd(n,t)); + MatrixXd Y(n,t); std::vector ind_best(pMax+1, -1); std::vector est_old(pMax+1,0); std::vector> indices(pMax+1,std::vector(n,0)); @@ -329,11 +329,11 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it { // std::cerr<<"iter "< normest1_merged(const SpMatrix& A, int pMax, int t=2, int it assert(est >= est_old[p]); // S = sign(Y[p]), 0.0 -> 1.0 - S = Y[p].unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); + S = Y.unaryExpr([](const double& x) {return (x>=0) ? 1.0 : -1.0 ;}); // (3) of Algorithm 2.4 Z = A.transpose() * S; // (n,n) * (n,t) -> (n,t) From 76673da91b03dc5b9f0d3bc40bedd76e963a28be Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 23:02:31 -0400 Subject: [PATCH 385/421] If there are no more X columns to try, then we're done. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 918cf440..bdb827d2 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -323,7 +323,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it all_indices.resize(n); std::vector all_ind_hist(n,false); - for(int k=1; k<=itmax; k++) + for(int k=1; k<=itmax and X.cols() > 0; k++) { for(int p=1; p<=pMax; p++) { From 4c34edaeecf894420f340ad31880d575c036fc11 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 14 Jul 2025 23:12:11 -0400 Subject: [PATCH 386/421] Fix incorrect break in loop. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index bdb827d2..d933b084 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -337,7 +337,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it norms[p] = std::max(norms[p], est); - if (k == itmax) break; + if (k == itmax) continue; if (est > est_old[p] or k == 2) { From ed0dac80f8ca511e38ca2164017239ab470be768 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 21 Jul 2025 17:49:41 -0400 Subject: [PATCH 387/421] Remove debug logging. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index d933b084..9e73a245 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -281,8 +281,11 @@ std::vector normest1_all(const SpMatrix& A, int pMax, int t=2, int itmax return norms; } -std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int itmax=5) +std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int itmax=2) { + // nstd::cerr<<"--BEGIN--\n"; + // auto sep_norms = normest1_all(A,pMax,t,itmax); + assert(pMax >= 0); assert(t != 0); // negative means t = n assert(itmax >= 1); @@ -399,7 +402,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it } } - std::cerr<<"k = "< normest1_merged(const SpMatrix& A, int pMax, int t=2, int it } } - auto sep_norms = normest1_all(A,pMax,t,itmax); - for(int p=1;p approx_norms = normest1_merged( gBs[eigenIndex], pMax+1, t); + std::vector approx_norms = normest1_merged( gBs[eigenIndex], pMax+1); // equation 3.7 in Al-Mohy and Higham for(int p=0;p <= pMax+1; p++) @@ -1008,8 +1010,7 @@ namespace beagle { ds[matrixIndex].clear(); int pMax = getPMax(); - int t = 5; - auto approx_norms = normest1_merged( gBs[matrixIndex], pMax+1, t); + auto approx_norms = normest1_merged( gBs[matrixIndex], pMax+1); for(int p=0;p <= pMax+1; p++) { // equation 3.7 in Al-Mohy and Higham From ddc7f5a879beec9e4105c7a81581b4339f24e93f Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 21 Jul 2025 17:50:49 -0400 Subject: [PATCH 388/421] Don't make the last partition extra-big. This tends to occur when the number of patterns is not very big (e.g. 90) and the number of threads is relatively prime (i.e. 7). --- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index f60e93fc..62d4c062 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -601,9 +601,9 @@ int BeagleCPUImpl::createInstance(int tipCount, partitionCount = hardwareThreads/2; } int* patternPartitions = (int*) malloc(sizeof(int) * kPatternCount); - int partitionSize = kPatternCount/partitionCount; + for (int i=0; i partitionCount - 1) sitePartition = partitionCount - 1; patternPartitions[i] = sitePartition; @@ -678,9 +678,8 @@ int BeagleCPUImpl::setCPUThreadCount(int threadCount) { } int* patternPartitions = (int*) malloc(sizeof(int) * kPatternCount); - int partitionSize = kPatternCount/partitionCount; for (int i=0; i partitionCount - 1) sitePartition = partitionCount - 1; patternPartitions[i] = sitePartition; From 3e8543b789294b15e930bcb38b02dd86aa7693ad Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 12 Aug 2025 11:24:23 -0400 Subject: [PATCH 389/421] Further simplify and fix assignment of patterns to threads. This fixes a (probably small) memory leak too. --- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 62d4c062..caa4a9ef 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -600,15 +600,12 @@ int BeagleCPUImpl::createInstance(int tipCount, if (partitionCount > hardwareThreads/2) { partitionCount = hardwareThreads/2; } - int* patternPartitions = (int*) malloc(sizeof(int) * kPatternCount); - for (int i=0; i partitionCount - 1) - sitePartition = partitionCount - 1; - patternPartitions[i] = sitePartition; + std::vector partitionForPattern(kPatternCount); + for (int p=0; p::setCPUThreadCount(int threadCount) { partitionCount = threadCount; } - int* patternPartitions = (int*) malloc(sizeof(int) * kPatternCount); - for (int i=0; i partitionCount - 1) - sitePartition = partitionCount - 1; - patternPartitions[i] = sitePartition; + std::vector partitionForPattern(kPatternCount); + for (int p=0; p Date: Tue, 19 Aug 2025 13:48:15 -0400 Subject: [PATCH 390/421] Update test script --- .github/workflows/build.yml | 102 ++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 46 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 08f2fff6..e1e2c07b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,82 +12,89 @@ jobs: build: strategy: matrix: - name: [ubuntu-gcc-10, - ubuntu-gcc-11, - ubuntu-gcc-12, + name: [ubuntu-gcc-12, ubuntu-gcc-13, - ubuntu-clang-15, + ubuntu-gcc-14, + ubuntu-gcc-15, + ubuntu-gcc-15-debug, ubuntu-clang-17, + ubuntu-clang-19, macos-xcode-15-intel, macos-xcode-16-arm, windows ] include: - - name: ubuntu-gcc-10 + + - name: ubuntu-gcc-12 os: ubuntu-22.04 compiler: gcc - version: "10" - buildtype: "release" + version: "12" arch: "linux64" - - name: ubuntu-gcc-11 - os: ubuntu-22.04 + - name: ubuntu-gcc-13 + os: ubuntu-24.04 compiler: gcc - version: "11" - buildtype: "release" + version: "12" arch: "linux64" - - name: ubuntu-gcc-12 - os: ubuntu-22.04 + - name: ubuntu-gcc-14 + os: ubuntu-24.04 compiler: gcc version: "12" - buildtype: "release" arch: "linux64" - - name: ubuntu-gcc-13 + - name: ubuntu-gcc-15 os: ubuntu-24.04 compiler: gcc version: "12" - buildtype: "release" + buildtype: "Release" arch: "linux64" + image: ubuntu:25.04 - - name: ubuntu-clang-15 - os: ubuntu-22.04 - compiler: clang - version: "15" - buildtype: "release" + - name: ubuntu-gcc-15-debug + os: ubuntu-24.04 + compiler: gcc + version: "12" + buildtype: "Debug" arch: "linux64" + image: ubuntu:25.04 - name: ubuntu-clang-17 os: ubuntu-24.04 compiler: clang version: "17" - buildtype: "release" + buildtype: "Release" + arch: "linux64" + + - name: ubuntu-clang-19 + os: ubuntu-24.04 + compiler: clang + version: "19" + buildtype: "Release" arch: "linux64" - name: macos-xcode-15-intel os: macos-13 compiler: xcode version: "15" - buildtype: "release" arch: "mac-intel64" - name: macos-xcode-16-arm os: macos-14 compiler: xcode version: "16" - buildtype: "release" arch: "mac-arm64" - name: windows os: ubuntu-24.04 compiler: mingw version: "N/A" - buildtype: "release" arch: "win64" runs-on: ${{ matrix.os }} + container: + image: ${{ matrix.image || '' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -97,16 +104,22 @@ jobs: # Caches for different branches are isolated, so we don't need to put the branch name into the key. # The total size for all caches in a repository is 5Gb. - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ matrix.name }} - restore-keys: | - ${{ matrix.name }} + - name: Set CMake build type + run: | + CMAKE_ARGS="" + if [ -n "${{ matrix.buildtype }}" ] ; then + CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${{ matrix.buildtype }}" + fi + echo "CMAKE_ARGS=${CMAKE_ARGS}" >> $GITHUB_ENV - - name: Install (Linux) + - name: Install utilities (Linux) if: runner.os == 'Linux' && matrix.name != 'windows' run: | + # If we're running in a docker image, we are already root + if [ -n "${{ matrix.image }}" ] ; then + apt-get update && apt-get install -y sudo build-essential git cmake + fi + sudo apt-get update sudo apt-get install -y cmake ninja-build ccache pkg-config sudo apt-get install -y openjdk-17-jdk libeigen3-dev @@ -123,21 +136,17 @@ jobs: sudo apt-get install -y clang-${{ matrix.version }} echo "CC=ccache clang-${{ matrix.version }}" >> $GITHUB_ENV echo "CXX=ccache clang++-${{ matrix.version }}" >> $GITHUB_ENV - echo "CMAKE_ARGS=-DBUILD_OPENMP=OFF" >> $GITHUB_ENV + echo "CMAKE_ARGS=${CMAKE_ARGS} -DBUILD_OPENMP=OFF" >> $GITHUB_ENV fi - - name: Install (Linux -> Windows [cross]) + - name: Install utilities (Linux -> Windows [cross]) if: matrix.name == 'windows' run: | - # For mingw/gcc-12. - cat /etc/apt/sources.list - sudo sed -i 's/jammy/lunar/g' /etc/apt/sources.list - sudo apt-get update sudo apt-get install -y cmake ninja-build ccache pkg-config sudo apt-get install -y dos2unix g++-mingw-w64 mingw-w64-tools wine64 zstd - echo "CMAKE_ARGS=-DBUILD_ACTION=OFF -DBUILD_JNI=OFF -DBUILD_OPENMP=OFF -DCMAKE_TOOLCHAIN_FILE=scripts/mingw-w64-x86_64.cmake" >> $GITHUB_ENV + echo "CMAKE_ARGS=${CMAKE_ARGS} -DBUILD_ACTION=OFF -DBUILD_JNI=OFF -DBUILD_OPENMP=OFF -DCMAKE_TOOLCHAIN_FILE=scripts/mingw-w64-x86_64.cmake" >> $GITHUB_ENV - name: Select XCode version (macOS) if: runner.os == 'macOS' @@ -145,22 +154,23 @@ jobs: with: xcode-version: ${{ matrix.version }} - - name: Install (macOS) + - name: Install utilities (macOS) if: runner.os == 'macOS' run: | brew install cmake ninja pkg-config ccache coreutils eigen openjdk echo "CC=ccache clang" >> $GITHUB_ENV echo "CXX=ccache clang++" >> $GITHUB_ENV - echo "CMAKE_ARGS=-DBUILD_OPENMP=OFF" >> $GITHUB_ENV + echo "CMAKE_ARGS=${CMAKE_ARGS} -DBUILD_OPENMP=OFF" >> $GITHUB_ENV - # Caches for different branches are isolated, so we don't need to put the branch name into the key. - # The total size for all caches in a repository is 5Gb. + - name: Set up ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.name }} + restore-keys: | + ${{ matrix.name }} - name: Configure run: | - if [ "${{ matrix.cxxflags }}" != "" ] ; then - CPPARGS="$CPPARGS ${{ matrix.cxxflags }}" - fi mkdir build cd build cmake .. -G Ninja -DCMAKE_INSTALL_PREFIX=$HOME/local ${CMAKE_ARGS} From 61bba63ca5acc3458eb49fcc2a37bf0869e3e1b9 Mon Sep 17 00:00:00 2001 From: xji3 Date: Fri, 29 Aug 2025 11:30:27 -0500 Subject: [PATCH 391/421] upPrePartials.. --- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index caa4a9ef..85f6b4c6 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -1516,10 +1516,10 @@ int BeagleCPUImpl::updatePrePartials(const int *operations, count); } else { bool byPartition = false; - returnCode = upPartials(byPartition, - operations, - count, - cumulativeScaleIndex); + returnCode = upPrePartials(byPartition, + operations, + count, + cumulativeScaleIndex); } return returnCode; From ddf3d9c80911de542a1889c05f11573b59f1ac98 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 8 Sep 2025 09:35:35 -0400 Subject: [PATCH 392/421] Fix mac arm test. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e1e2c07b..57904daa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -157,7 +157,7 @@ jobs: - name: Install utilities (macOS) if: runner.os == 'macOS' run: | - brew install cmake ninja pkg-config ccache coreutils eigen openjdk + brew install ninja pkg-config ccache coreutils eigen openjdk echo "CC=ccache clang" >> $GITHUB_ENV echo "CXX=ccache clang++" >> $GITHUB_ENV echo "CMAKE_ARGS=${CMAKE_ARGS} -DBUILD_OPENMP=OFF" >> $GITHUB_ENV From 75dcb55365e98d64fb44aa2b002278db84bf50c4 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 8 Sep 2025 09:35:35 -0400 Subject: [PATCH 393/421] Update ci.yaml to use native linux-arm runner --- .github/workflows/ci.yaml | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c6afdcaf..04f651bc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,10 +3,10 @@ name: CI on: push: branches: - - master + - "*" pull_request: branches: - - master + - "*" jobs: build: @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y cmake file + sudo apt-get install -y cmake file pkgconf libeigen3-dev - name: Build run: | @@ -32,28 +32,21 @@ jobs: build-aarch64: name: Build on Linux aarch64 - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - name: Checkout uses: actions/checkout@v4 + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake file pkgconf libeigen3-dev + - name: Build - uses: uraimo/run-on-arch-action@v2 - with: - arch: aarch64 - distro: ubuntu20.04 - githubToken: ${{ github.token }} - dockerRunArgs: | - --volume "${PWD}:/beagle-lib" - install: | - apt-get update -q -y - apt-get install -q -y cmake gcc g++ openjdk-11-jdk file - run: | - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-arm64 - cd /beagle-lib - mkdir build - cd build - cmake .. - make -j - file libhmsbeagle/libhmsbeagle.so.* | grep aarch64 + run: | + mkdir build + cd build + cmake .. + make -j + file libhmsbeagle/libhmsbeagle.so.* | grep aarch64 From 0f26ad216e2793117555a464aa23bb97eabfbb5c Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 8 Sep 2025 09:35:35 -0400 Subject: [PATCH 394/421] Fix compiler versions. --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57904daa..c11e0200 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,7 @@ name: Build and test on: push: branches: - - "*" + - "*" pull_request: branches: - "*" @@ -35,19 +35,19 @@ jobs: - name: ubuntu-gcc-13 os: ubuntu-24.04 compiler: gcc - version: "12" + version: "13" arch: "linux64" - name: ubuntu-gcc-14 os: ubuntu-24.04 compiler: gcc - version: "12" + version: "14" arch: "linux64" - name: ubuntu-gcc-15 os: ubuntu-24.04 compiler: gcc - version: "12" + version: "14" buildtype: "Release" arch: "linux64" image: ubuntu:25.04 @@ -55,7 +55,7 @@ jobs: - name: ubuntu-gcc-15-debug os: ubuntu-24.04 compiler: gcc - version: "12" + version: "15" buildtype: "Debug" arch: "linux64" image: ubuntu:25.04 From c93dc472d5cc16e42bfd6f7a9639bb0f950102a3 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 8 Sep 2025 09:35:35 -0400 Subject: [PATCH 395/421] Use native linux-arm runner --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c11e0200..c1bec92f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,7 @@ jobs: ubuntu-gcc-14, ubuntu-gcc-15, ubuntu-gcc-15-debug, +# ubuntu-gcc-14-arm, ubuntu-clang-17, ubuntu-clang-19, macos-xcode-15-intel, @@ -60,6 +61,13 @@ jobs: arch: "linux64" image: ubuntu:25.04 +# "Failed to obtain BEAGLE instance" on some tests. +# - name: ubuntu-gcc-14-arm +# os: ubuntu-24.04-arm +# compiler: gcc +# version: "14" +# arch: "linux64" + - name: ubuntu-clang-17 os: ubuntu-24.04 compiler: clang From 524c0a825d870da692cad16558a2139acadbd976 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 9 Sep 2025 15:57:19 -0400 Subject: [PATCH 396/421] Simplify merged_norms by changing std::sort to just an argmax --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 38 +++++++----------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 9e73a245..b35c41ed 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -311,8 +311,7 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it MatrixXd Y(n,t); std::vector ind_best(pMax+1, -1); std::vector est_old(pMax+1,0); - std::vector> indices(pMax+1,std::vector(n,0)); - std::vector> ind_hist(pMax+1, std::vector(n,0)); + vector new_indices(pMax+1); // (0) Choose starting matrix X that is (n,t) with columns of unit 1-norm. // We choose the first column to be all 1s. @@ -364,41 +363,24 @@ std::vector normest1_merged(const SpMatrix& A, int pMax, int t=2, int it // Maximize across each the t entries in each row of Z. h = Z.cwiseAbs().rowwise().maxCoeff(); // (n,t) -> (n,1) - // Sort dimensions based on h - indices[p].resize(n); - for(int i=0;i h[j];}); - - // find the first t indices that are not in ind_hist - int l=0; - for(int i=0;i i; + for(int l=0;l h[i.value()])) + i = l; + new_indices[p] = i.value(); } // Combine indices from different values of p all_indices.clear(); for(int p=1;p<=pMax;p++) { - for(int i: indices[p]) + int i = new_indices[p]; + if (not all_ind_hist[i]) { - ind_hist[p][i] = true; - if (not all_ind_hist[i]) - { - all_indices.push_back(i); - all_ind_hist[i] = true; - } + all_indices.push_back(i); + all_ind_hist[i] = true; } } From b751d5956d2c698d77731afd6dd3db65eae900f9 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 9 Sep 2025 16:45:08 -0400 Subject: [PATCH 397/421] Add a function to compute the true norms (merged). --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index b35c41ed..bba69ebc 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -85,6 +85,29 @@ double random_plus_minus_1_func(double x) return -1; } +std::vector true_norm_merged(const SpMatrix& A, int pMax) +{ + assert(pMax >= 0); + + std::vector norms(pMax+1, 0); + norms[0] = 1.0; + + // A is (n,n); + assert(A.rows() == A.cols()); + int n = A.cols(); + + MatrixXd Y = MatrixXd::Identity(n,n); + + for(int p=1;p<=pMax;p++) + { + Y = A*Y; + auto [est, j] = ArgNormP1(Y); + norms[p] = est; + } + + return norms; +} + // Algorithm 2.4 from Higham and Tisseur (2000), A BLOCK ALGORITHM FOR MATRIX 1-NORM ESTIMATION, // WITH AN APPLICATION TO 1-NORM PSEUDOSPECTRA. // See OneNormEst in https://eprints.maths.manchester.ac.uk/2195/1/thesis-main.pdf From 9e8df82d68cf48d235db965ac92943d0d1d3f909 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 13 Oct 2025 14:48:44 -0400 Subject: [PATCH 398/421] Add comment about condition 3.13 --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index bba69ebc..77cafe9e 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -1318,7 +1318,10 @@ namespace beagle { const bool conditionFragment313 = gB1Norms[eigenIndex] * edgeMultiplier <= 2.0 * theta / ((double) nCol * mMax) * pMax * (pMax + 3); // using l = 1 as in equation 3.13 + // BDR: l is equivalent to 't' in normest1. So maybe we should use (l=1,t=1) or (l=2,t=2). if (conditionFragment313) { + // If this condition is true, it is cheaper to use ||A|| directly if we do NOT calculate the d-values. + // The condition assumes that we are only using the d-values for one branch though. for (auto& [thisM, thetaM]: thetaConstants) { const double thisS = ceil(gB1Norms[eigenIndex] * edgeMultiplier / thetaM); if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { From 4f8aaf4d0a6a9f5012f8fa1b78a924f363a5e0af Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 3 Nov 2025 16:21:14 -0500 Subject: [PATCH 399/421] Enable color in compiler errors --- CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9643635b..cd633e3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,6 @@ set(BEAGLE_VERSION_PATCH "2") set(GENERIC_API_VERSION "1") - set(BEAGLE_VERSION "${BEAGLE_VERSION_MAJOR}.${BEAGLE_VERSION_MINOR}.${BEAGLE_VERSION_PATCH}") set(BEAGLE_PLUGIN_VERSION "${BEAGLE_VERSION_MAJOR}${BEAGLE_VERSION_MINOR}") @@ -76,6 +75,13 @@ if(NOT DEFINED CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo) endif() +# Make erros and warnings more readable by enabling colors in the compiler output +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + add_compile_options(-fdiagnostics-color=always) +elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + add_compile_options(-fcolor-diagnostics) +endif() + ## flags for standard library #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++") SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") From 3198cb8b9daf4305198e13e3f394c12755710287 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 21 Oct 2025 15:03:53 -0400 Subject: [PATCH 400/421] Switch back to the non-merged norms. --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 6 ++++-- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 77cafe9e..79be8209 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -971,7 +971,8 @@ namespace beagle { ds[eigenIndex].clear(); int pMax = getPMax(); - std::vector approx_norms = normest1_merged( gBs[eigenIndex], pMax+1); + std::vector approx_norms = normest1_all( gBs[eigenIndex], pMax+1); + // std::vector approx_norms = normest1_merged( gBs[eigenIndex], pMax+1); // equation 3.7 in Al-Mohy and Higham for(int p=0;p <= pMax+1; p++) @@ -1015,7 +1016,8 @@ namespace beagle { ds[matrixIndex].clear(); int pMax = getPMax(); - auto approx_norms = normest1_merged( gBs[matrixIndex], pMax+1); + auto approx_norms = normest1_all( gBs[matrixIndex], pMax+1); + // auto approx_norms = normest1_merged( gBs[matrixIndex], pMax+1); for(int p=0;p <= pMax+1; p++) { // equation 3.7 in Al-Mohy and Higham diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index 85f6b4c6..f6c44c47 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -378,7 +378,7 @@ int BeagleCPUImpl::createInstance(int tipCount, if (preferenceFlags & BEAGLE_FLAG_COMPUTATION_ACTION) gEigenDecomposition = nullptr; - else if (kFlags & BEAGLE_FLAG_EIGEN_COMPLEX) + else if (bool(kFlags & BEAGLE_FLAG_EIGEN_COMPLEX) || kStateCount > 256) gEigenDecomposition = new EigenDecompositionSquare(kEigenDecompCount, kStateCount,kCategoryCount,kFlags); else From 42fbb9abfb94f4ff87c80c5cdd1a94296562b79b Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 10 Nov 2025 14:56:42 -0500 Subject: [PATCH 401/421] Don't crash in the destructor when initialization fails. --- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUImpl.hpp b/libhmsbeagle/CPU/BeagleCPUImpl.hpp index f6c44c47..7a54c8d5 100755 --- a/libhmsbeagle/CPU/BeagleCPUImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUImpl.hpp @@ -121,14 +121,14 @@ BeagleCPUImpl::~BeagleCPUImpl() { // which is TEMP_SCRATCH_PARTIAL twice. for(unsigned int i=0; i::~BeagleCPUImpl() { for(unsigned int i=0; i::~BeagleCPUImpl() { free(gScaleBuffers[0]); } else { for(unsigned int i=0; i Date: Mon, 10 Nov 2025 14:56:42 -0500 Subject: [PATCH 402/421] Print a more helpful message when EigenDecompositionCube tries to allocate too much memory. --- libhmsbeagle/CPU/EigenDecompositionCube.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libhmsbeagle/CPU/EigenDecompositionCube.hpp b/libhmsbeagle/CPU/EigenDecompositionCube.hpp index c9647ddd..3782d1e5 100644 --- a/libhmsbeagle/CPU/EigenDecompositionCube.hpp +++ b/libhmsbeagle/CPU/EigenDecompositionCube.hpp @@ -41,7 +41,10 @@ EigenDecompositionCube::EigenDecompositionCube(int dec size_t mem = ((size_t)kStateCount) * kStateCount * kStateCount; gCMatrices[i] = (REALTYPE*) malloc(sizeof(REALTYPE) * mem); if (gCMatrices[i] == NULL) - throw std::bad_alloc(); + { + std::cerr<<"BEAGLE: EigenDecompositionCube: memory allocation failed for "< Date: Mon, 10 Nov 2025 14:56:42 -0500 Subject: [PATCH 403/421] Print message if an exception is thrown in createInstance --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 9 ++++-- libhmsbeagle/CPU/BeagleCPUImpl.hpp | 9 ++++-- libhmsbeagle/CPU/BeagleCPUSSEImpl.hpp | 39 +++++++++++++++++------- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 79be8209..dd1a0da4 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -1421,9 +1421,14 @@ namespace beagle { delete impl; return NULL; } + catch(const std::exception& e) + { + std::cerr<<"BEAGLE: exception in createInstance: "<::createImpl(int tip if (*errorCode == BEAGLE_SUCCESS) { return impl; } + } + catch(const std::exception& e) + { + std::cerr<<"BEAGLE: exception in createInstance: "<::createImpl(int try { - if (impl->createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, + *errorCode = + impl->createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, patternCount, eigenBufferCount, matrixBufferCount, categoryCount,scaleBufferCount, resourceNumber, pluginResourceNumber, - preferenceFlags, requirementFlags) == 0) + preferenceFlags, requirementFlags); + + if (*errorCode == BEAGLE_SUCCESS) + { return impl; + } + } + catch(const std::exception& e) + { + std::cerr<<"BEAGLE: exception in createInstance: "<* impl = new BeagleCPUSSEImpl(); try { - if (impl->createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, + *errorCode = + impl->createInstance(tipCount, partialsBufferCount, compactBufferCount, stateCount, patternCount, eigenBufferCount, matrixBufferCount, categoryCount,scaleBufferCount, resourceNumber, pluginResourceNumber, - preferenceFlags, requirementFlags) == 0) + preferenceFlags, requirementFlags); + + + if (*errorCode == BEAGLE_SUCCESS) return impl; } + catch(const std::exception& e) + { + std::cerr<<"BEAGLE: exception in createInstance: "< Date: Wed, 12 Nov 2025 09:53:06 -0500 Subject: [PATCH 404/421] Make auto-download of Eigen work --- CMakeLists.txt | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cd633e3a..546dbef9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.9) +cmake_minimum_required(VERSION 3.18) project(BEAGLE) set(BEAGLE_PACKAGE_NAME "libhmsbeagle") @@ -143,16 +143,30 @@ if(BUILD_ACTION) include(FetchContent) set(EIGEN_VERSION 3.4) find_package(Eigen3 ${EIGEN_VERSION} QUIET) - if(NOT EIGEN3_FOUND) - message(" Eigen3: library not found - downloading.") + if(NOT Eigen3_FOUND) + message(STATUS "Eigen3 not found — downloading.") + set(BUILD_TESTING OFF CACHE INTERNAL "") - FetchContent_Declare(eigen - GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git - GIT_TAG ${EIGEN_VERSION} - GIT_SHALLOW ON) + + FetchContent_Declare( + eigen + GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git + GIT_TAG ${EIGEN_VERSION} + GIT_SHALLOW ON + SOURCE_SUBDIR "NONEXISTANT_DIR" + ) + FetchContent_MakeAvailable(eigen) + message(STATUS " Eigen3: downloaded.") + unset(BUILD_TESTING CACHE) - message(" Eigen3: downloaded.") + + # Create our own Eigen3::Eigen target instead of running the CMakeLists.txt from Eigen, + # which is creates an uninstallable target and so fails to configure. + add_library(Eigen3::Eigen INTERFACE IMPORTED) + target_include_directories(Eigen3::Eigen INTERFACE + ${eigen_SOURCE_DIR} + ) endif() set(CMAKE_CXX_STANDARD 17) From b6affc5510047219c4d1d0c8273a13aba390bf61 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Mar 2026 17:25:30 -0500 Subject: [PATCH 405/421] Compute the d-values on demand --- libhmsbeagle/CPU/BeagleCPUActionImpl.h | 2 +- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 31 +++++++++++------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index 1d648773..9cbab8da 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -101,7 +101,7 @@ namespace beagle { std::vector gB1Norms; std::vector gEigenMaps; std::vector gEdgeMultipliers; - std::vector> ds; + mutable std::vector> ds; SpMatrix identity; MapType* gMappedIntegrationTmp; // MapType* gMappedLeftPartialTmp; diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index dd1a0da4..2bcc77c4 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -970,14 +970,6 @@ namespace beagle { ds[eigenIndex].clear(); - int pMax = getPMax(); - std::vector approx_norms = normest1_all( gBs[eigenIndex], pMax+1); - // std::vector approx_norms = normest1_merged( gBs[eigenIndex], pMax+1); - - // equation 3.7 in Al-Mohy and Higham - for(int p=0;p <= pMax+1; p++) - ds[eigenIndex].push_back( pow( approx_norms[p], 1.0/double(p) ) ); - // gSimpleActions[eigenIndex]->setInstantaneousMatrix(tripletList); // gSimpleActions[eigenIndex]->fireMatrixChanged(); #ifdef BEAGLE_DEBUG_FLOW @@ -1015,15 +1007,6 @@ namespace beagle { ds[matrixIndex].clear(); - int pMax = getPMax(); - auto approx_norms = normest1_all( gBs[matrixIndex], pMax+1); - // auto approx_norms = normest1_merged( gBs[matrixIndex], pMax+1); - for(int p=0;p <= pMax+1; p++) - { - // equation 3.7 in Al-Mohy and Higham - ds[matrixIndex].push_back( pow( approx_norms[p], 1.0/double(p) ) ); - } - return BEAGLE_SUCCESS; } @@ -1367,6 +1350,20 @@ namespace beagle { BEAGLE_CPU_ACTION_TEMPLATE double BeagleCPUActionImpl::getDValue(int p, int eigenIndex) const { + int pMax = getPMax(); + + assert(p >= 0 and p <= pMax+1); + + // If the d-value is not computed, then compute and cache it. + if (p >= ds[eigenIndex].size()) + { + for(int i=ds[eigenIndex].size();i<=p;i++) + { + double approx_norm = normest1( gBs[eigenIndex], i); + ds[eigenIndex].push_back( pow(approx_norm, 1.0/double(i)) ); + } + } + assert(p >= 0 and p < ds[eigenIndex].size()); return ds[eigenIndex][p]; From b31195d02040e77b2d58c7b21ca7cda1732ea232 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 3 Mar 2026 17:49:37 -0500 Subject: [PATCH 406/421] Update github actions --- .github/workflows/build.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c1bec92f..9abecf27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,8 +20,9 @@ jobs: # ubuntu-gcc-14-arm, ubuntu-clang-17, ubuntu-clang-19, - macos-xcode-15-intel, - macos-xcode-16-arm, + macos-intel, + macos-xcode16-arm, + macos-xcode26-arm, windows ] @@ -82,18 +83,24 @@ jobs: buildtype: "Release" arch: "linux64" - - name: macos-xcode-15-intel - os: macos-13 + - name: macos-intel + os: macos-15-intel compiler: xcode - version: "15" + version: "16" arch: "mac-intel64" - - name: macos-xcode-16-arm - os: macos-14 + - name: macos-xcode16-arm + os: macos-latest compiler: xcode version: "16" arch: "mac-arm64" + - name: macos-xcode26-arm + os: macos-latest + compiler: xcode + version: "26" + arch: "mac-arm64" + - name: windows os: ubuntu-24.04 compiler: mingw From caa58664d0092dfdf2a5b18f8fd7982bd68529d2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 16 Mar 2026 19:42:14 -0400 Subject: [PATCH 407/421] Compute the d-values in a single-threaded fashion. --- libhmsbeagle/CPU/BeagleCPUActionImpl.h | 2 ++ libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 26 ++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index 9cbab8da..0c4eb554 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -35,6 +35,7 @@ #include #include #include +#include #define T_PAD_DEFAULT 1 // Pad transition matrix rows with an extra 1.0 for ambiguous characters #define P_PAD_DEFAULT 0 // No partials padding necessary for non-SSE implementations @@ -102,6 +103,7 @@ namespace beagle { std::vector gEigenMaps; std::vector gEdgeMultipliers; mutable std::vector> ds; + mutable std::shared_mutex ds_mutex; SpMatrix identity; MapType* gMappedIntegrationTmp; // MapType* gMappedLeftPartialTmp; diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 2bcc77c4..8ddb8f60 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -1350,21 +1350,25 @@ namespace beagle { BEAGLE_CPU_ACTION_TEMPLATE double BeagleCPUActionImpl::getDValue(int p, int eigenIndex) const { - int pMax = getPMax(); - - assert(p >= 0 and p <= pMax+1); - - // If the d-value is not computed, then compute and cache it. - if (p >= ds[eigenIndex].size()) + // 1. Try to read with a SHARED lock (multiple readers allowed) { - for(int i=ds[eigenIndex].size();i<=p;i++) - { - double approx_norm = normest1( gBs[eigenIndex], i); - ds[eigenIndex].push_back( pow(approx_norm, 1.0/double(i)) ); + std::shared_lock read_lock(ds_mutex); + if (p < ds[eigenIndex].size()) { + return ds[eigenIndex][p]; } } - assert(p >= 0 and p < ds[eigenIndex].size()); + // 2. If not found, upgrade to an EXCLUSIVE lock (only one writer) + std::unique_lock write_lock(ds_mutex); + + // Double-check: another thread might have finished the work + // while we were waiting for the write_lock. + if (p >= ds[eigenIndex].size()) { + for(int i = ds[eigenIndex].size(); i <= p; i++) { + double approx_norm = normest1(gBs[eigenIndex], i); + ds[eigenIndex].push_back(pow(approx_norm, 1.0/double(i))); + } + } return ds[eigenIndex][p]; } From 7907d39d8607f7f32998e0366bbc517446197590 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 23 Mar 2026 21:28:17 -0400 Subject: [PATCH 408/421] Compute cheaper (s,m) for comparison --- libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 33 ++++++++++++++---------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 8ddb8f60..d9323df4 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -1294,26 +1294,33 @@ namespace beagle { if (t * gB1Norms[eigenIndex] == 0.0) return {0, 1}; - int bestM = INT_MAX; - double bestS = INT_MAX; // Not all the values of s can fit in a 32-bit int. - const double theta = thetaConstants.at(mMax); const double pMax = getPMax(); // pMax is the largest positive integer such that p*(p-1) <= mMax + 1 - const bool conditionFragment313 = gB1Norms[eigenIndex] * edgeMultiplier <= 2.0 * theta / ((double) nCol * mMax) * pMax * (pMax + 3); + + // If this condition is true, it is cheaper to use ||A|| directly if we do NOT calculate the d-values. + // The condition assumes that we are only using the d-values for one branch though. + int bestM1 = INT_MAX; + double bestS1 = INT_MAX; + for (auto& [thisM, thetaM]: thetaConstants) { + const double thisS = ceil(gB1Norms[eigenIndex] * edgeMultiplier / thetaM); + if (bestM1 == INT_MAX or ((double) thisM) * thisS < bestM1 * bestS1) { + bestS1 = thisS; + bestM1 = thisM; + } + } + + int bestM = INT_MAX; + double bestS = INT_MAX; // Not all the values of s can fit in a 32-bit int. + // using l = 1 as in equation 3.13 + int l=1; + const bool conditionFragment313 = gB1Norms[eigenIndex] * edgeMultiplier <= 2.0 * l * theta / ((double) nCol * mMax) * pMax * (pMax + 3); // BDR: l is equivalent to 't' in normest1. So maybe we should use (l=1,t=1) or (l=2,t=2). if (conditionFragment313) { - // If this condition is true, it is cheaper to use ||A|| directly if we do NOT calculate the d-values. - // The condition assumes that we are only using the d-values for one branch though. - for (auto& [thisM, thetaM]: thetaConstants) { - const double thisS = ceil(gB1Norms[eigenIndex] * edgeMultiplier / thetaM); - if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { - bestS = thisS; - bestM = thisM; - } - } + bestM = bestM1; + bestS = bestS1; } else { for (int p = 2; p < pMax; p++) { for (int thisM = p * (p - 1) - 1; thisM < mMax + 1; thisM++) { From df15374a017f4639645c0ebdbede506e47352be8 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 24 Mar 2026 17:20:41 -0400 Subject: [PATCH 409/421] Stop computing d[p] when we've done too much work --- libhmsbeagle/CPU/BeagleCPUActionImpl.h | 3 + libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 110 +++++++++++++++-------- 2 files changed, 74 insertions(+), 39 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index 0c4eb554..ac0101ce 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -64,6 +64,7 @@ namespace beagle { protected: using BeagleCPUImpl::kTipCount; + using BeagleCPUImpl::kNumThreads; using BeagleCPUImpl::integrationTmp; using BeagleCPUImpl::firstDerivTmp; using BeagleCPUImpl::secondDerivTmp; @@ -281,6 +282,8 @@ namespace beagle { double getDValue(int p, int eigenIndex) const; + bool hasDValue(int p, int eigenIndex) const; + double getPMax() const; }; diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index d9323df4..2bc2db4e 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -1294,52 +1294,76 @@ namespace beagle { if (t * gB1Norms[eigenIndex] == 0.0) return {0, 1}; - const double theta = thetaConstants.at(mMax); + // pMax is the largest positive integer such that pMax*(pMax-1) <= mMax + 1 + // Al Mohy & Higham picked pMax = 8, mMax = 8*(8-1)-1 = 55 const double pMax = getPMax(); - // pMax is the largest positive integer such that p*(p-1) <= mMax + 1 - - // If this condition is true, it is cheaper to use ||A|| directly if we do NOT calculate the d-values. - // The condition assumes that we are only using the d-values for one branch though. - int bestM1 = INT_MAX; - double bestS1 = INT_MAX; + // 1. What are the best values for (s,m) based only on the ||A||? + // In some sense this is the result for p=1. + int bestM = INT_MAX; + double bestS = INT_MAX; // Not all the values of s can fit in a 32-bit int. for (auto& [thisM, thetaM]: thetaConstants) { const double thisS = ceil(gB1Norms[eigenIndex] * edgeMultiplier / thetaM); - if (bestM1 == INT_MAX or ((double) thisM) * thisS < bestM1 * bestS1) { - bestS1 = thisS; - bestM1 = thisM; + if (bestM == INT_MAX or ((double) thisM) * thisS < bestM * bestS) { + bestS = thisS; + bestM = thisM; + } + } + int bestM1 = bestM; + double bestS1 = bestS; // Not all the values of s can fit in a 32-bit int. + + // BDR: l is called 't' in normest1. Right now it is 2. + int l=2; + int nColPerThread = ceil(nCol / std::max(1,kNumThreads)); + + // Condition 3.13 in the paper: + // + // gB1Norms[eigenIndex] * edgeMultiplier / thetaM_max * mMax * nCol <= 4.0 * l * pMax * (pMax + 3) / 2; + // + // is shorthand for + // + // bestM1 * bestS1 * nCol <= 4.0 * l * pMax * (pMax + 3) / 2; + // + // This specifies that the amount of work computing ALL the ds (on the rhs) exceeds the work + // computing the action with no ds (lhs). + // + // However, + // * what if we compute just a FEW ds? The first ones help the most, and are also cheapest. + // * after all the ds have been computed, it would be silly not to use them. + // * speed is affected by the number of columns PER THREAD, so use that instead. + // + // Previously we always computed all the ds, and then refused to use them if condition 3.13 was met. + + int workComputingDs = 0; + for (int p = 2; p <= pMax; p++) + { + if (not hasDValue(p, eigenIndex)) + workComputingDs += 4.0 * l * p; + + if (not hasDValue(p+1, eigenIndex)) + workComputingDs += 4.0 * l * (p+1); + + // Stop computing D values before it ends up being more expensive to compute them than to compute the action. + if (bestM * bestS * nColPerThread < workComputingDs) + break; + + for (int thisM = p * (p - 1) - 1; thisM < mMax + 1; thisM++) { + auto it = thetaConstants.find(thisM); + if (it != thetaConstants.end()) { + // equation 3.7 in Al-Mohy and Higham + const double dValueP = getDValue(p, eigenIndex); + const double dValuePPlusOne = getDValue(p + 1, eigenIndex); + const double alpha = std::max(dValueP, dValuePPlusOne) * edgeMultiplier; + // part of equation 3.10 + const double thisS = ceil(alpha / thetaConstants.at(thisM)); + if (bestM == INT_MAX or ((double) thisM) * thisS < bestM * bestS) { + bestS = thisS; + bestM = thisM; + } + } } } - int bestM = INT_MAX; - double bestS = INT_MAX; // Not all the values of s can fit in a 32-bit int. - - // using l = 1 as in equation 3.13 - int l=1; - const bool conditionFragment313 = gB1Norms[eigenIndex] * edgeMultiplier <= 2.0 * l * theta / ((double) nCol * mMax) * pMax * (pMax + 3); - // BDR: l is equivalent to 't' in normest1. So maybe we should use (l=1,t=1) or (l=2,t=2). - if (conditionFragment313) { - bestM = bestM1; - bestS = bestS1; - } else { - for (int p = 2; p < pMax; p++) { - for (int thisM = p * (p - 1) - 1; thisM < mMax + 1; thisM++) { - auto it = thetaConstants.find(thisM); - if (it != thetaConstants.end()) { - // equation 3.7 in Al-Mohy and Higham - const double dValueP = getDValue(p, eigenIndex); - const double dValuePPlusOne = getDValue(p + 1, eigenIndex); - const double alpha = std::max(dValueP, dValuePPlusOne) * edgeMultiplier; - // part of equation 3.10 - const double thisS = ceil(alpha / thetaConstants.at(thisM)); - if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { - bestS = thisS; - bestM = thisM; - } - } - } - } - } bestS = std::max(std::min(bestS, INT_MAX), 1.0); assert( bestS >= 1 ); assert( bestS <= INT_MAX ); @@ -1380,6 +1404,14 @@ namespace beagle { return ds[eigenIndex][p]; } + BEAGLE_CPU_ACTION_TEMPLATE + bool BeagleCPUActionImpl::hasDValue(int p, int eigenIndex) const + { + // 1. Try to read with a SHARED lock (multiple readers allowed) + std::shared_lock read_lock(ds_mutex); + return (p < ds[eigenIndex].size()); + } + BEAGLE_CPU_ACTION_TEMPLATE const char* BeagleCPUActionImpl::getName() { return getBeagleCPUActionName(); From d57985b9925c7d19a965a74cb4a82384e434e572 Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Sat, 24 Oct 2026 16:55:29 -0500 Subject: [PATCH 410/421] GPU action: correctness and robustness fixes (Phase 0) - Build with CUDA 13: drop the removed thrust/async/for_each.h include, include and explicitly. - Double precision: sumRootLikelihoods, justMaximize and cuda_max_l1_norm reduced in float; template them on T (logL now matches CPU action). - Multi-GPU: GPUInterface records the device ordinal and the action plugin calls cudaSetDevice before runtime/cuBLAS/cuSPARSE/Thrust work, so devices other than 0 work. - Block 1-norm estimator (GPUnormest3): correct row-wise max kernel (the reduce_by_key version could not reduce rows and wrote 25 elements), 1-norm over all t columns, Z = (A^T)^p S instead of A^T S, column history as in normest1, skip the unused final-iteration tail. BEAGLE_DEBUG_NORMEST prints GPU vs CPU vs exact ||B^p||_1. - (m,s): nCol is the number of patterns; since all d_p are precomputed, always refine the ||B||_1 choice with them, as the CPU code does. - Leaks: keep spMM buffer reallocations, free old CSR buffers on nnz change, free estimator buffers, fix scratch-space double free on move. - Memory: in action mode the base class skips the dense S x S transition matrices and the large pinned matrix cache; two scratch buffers replace the per-node copy of every partials buffer. S=16384 no longer OOMs. Dense-only entry points (pre-order, partitions, edge likelihoods, transition-matrix get/set) return BEAGLE_ERROR_NO_IMPLEMENTATION instead of computing on unallocated or unset matrices. Validated on A100: GPU logL matches the CPU action plugin to ~1e-15 relative on the MM (M=1/8/32) and IGC workloads; timings within ~2.5% of before, setQ slightly faster. Co-Authored-By: Claude Opus 5.5 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 82 ++++++++-- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 186 ++++++++++++++--------- libhmsbeagle/GPU/BeagleGPUImpl.hpp | 20 ++- libhmsbeagle/GPU/GPUInterface.h | 6 + libhmsbeagle/GPU/GPUInterfaceCUDA.cpp | 4 +- libhmsbeagle/GPU/cuda_ops.cu | 89 ++++++----- libhmsbeagle/GPU/cuda_ops.h | 9 +- 7 files changed, 264 insertions(+), 132 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 081e4e62..bf50185d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -743,8 +743,8 @@ struct GPUnormest1 // Z = A^T * S spMTM(Y, 1, A, X, 0, buffer, buffer_size); // (n,n) * (n,t) -> (n,t) - // h[0,j] = max(i) abs(Z(i,j)) - cuda_rowwise_max_abs(Y.ptr, t, t, h.ptr); // (n,t) -> (n,1) + // h[i] = max(j) abs(Z(i,j)) + cuda_rowwise_max_abs(Y.ptr, n, t, h.ptr); // (n,t) -> (n,1) // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. // We don't do this, because finding a different reason to exit @@ -898,6 +898,9 @@ struct GPUnormest2 ~GPUnormest2() { cudaDeviceDelete(buffer); + cudaDeviceDelete(valuesBuffer); + cudaDeviceDelete(Ones.ptr); + cudaDeviceDelete(Y.ptr); } }; @@ -934,6 +937,9 @@ struct GPUnormest3 int* offsets = nullptr; + // Rows already used as unit columns of X (ind_hist in Algorithm 2.4). + int* visited = nullptr; // (n) + GPUnormest3& operator=(const GPUnormest3&) = delete; GPUnormest3& operator=(GPUnormest3&& g) { @@ -952,8 +958,10 @@ struct GPUnormest3 std::swap(buffer2, g.buffer2); std::swap(buffer3, g.buffer3); std::swap(buffer4, g.buffer4); - std::swap(scratch, g.scratch); + std::swap(scratch.device_ptr, g.scratch.device_ptr); + std::swap(scratch.size, g.scratch.size); std::swap(offsets, g.offsets); + std::swap(visited, g.visited); return *this; } @@ -971,6 +979,8 @@ struct GPUnormest3 // X[i][j] = if (uniform(0,1)<0.5) +1 else -1 initialize_norm_x_matrix(X.ptr, n, t); + cudaMemset(visited, 0, n * sizeof(int)); + for(int k=1; k<=itmax; k++) { // std::cerr<<"iter "< buffer3[k] - cuda_max_l1_norm(X.ptr, n, 1, buffer2, buffer3 + (k-1), scratch, offsets); + // get the largest column L1 norm of A^p * X for this iteration -> buffer3[k-1] + cuda_max_l1_norm(X.ptr, n, t, buffer2, buffer3 + (k-1), scratch, offsets); + + // The rest only chooses X for the next iteration. + if (k == itmax) break; // S = sign(X) - cuda_sign_vector(X.ptr, n, t); // (n,n) -> (n,n) + cuda_sign_vector(X.ptr, n, t); // (n,t) -> (n,t) // (2) Replace parallel entries with random +1/-1 entries. // Skipping this part for now because it involves a lot of branching logic // and might be slow for the GPU. // (3) of Algorithm 2.4 - // Z = A^T * S - spMTM(Y, 1, A, X, 0, buffer, buffer_size); // (n,n) * (n,t) -> (n,t) + // Z = (A^T)^p * S + for(int i=0;i(Y, 1, A, X, 0, buffer, buffer_size); // (n,n) * (n,t) -> (n,t) + X.copyFrom(Y); + } - // h[0,j] = max(i) abs(Z(i,j)) - cuda_rowwise_max_abs(Y.ptr, t, t, h.ptr); // (n,t) -> (n,1) + // h[i] = max(j) abs(Z(i,j)) + cuda_rowwise_max_abs(X.ptr, n, t, h.ptr); // (n,t) -> (n,1) // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. // We don't do this, because finding a different reason to exit // seems to provide greater accuracy. + // (5) of Algorithm 2.4 - Only consider rows that have not been used as unit columns yet. + // (If fewer than t remain, used rows are picked again, which only repeats work.) + cuda_mask_visited(h.ptr, visited, n); + // Rank the dimensions by the size of their approximate norm in h, in decreasing order. cuda_sort_indices_by_vector(h.ptr, n, indices); // Zero the X matrix. cuda_fill_vector(X.ptr, n*t, 0); // (n,t) -> (n,t) - // Set X(i,indices[i]) = 1 for i in [0,t-1] + // Set X(indices[j],j) = 1 for j in [0,t-1] cuda_set_indices(X.ptr, n, t, indices); + cuda_mark_visited(visited, indices, t); } return cuda_max_abs(buffer3, itmax); @@ -1045,6 +1067,8 @@ struct GPUnormest3 offsets = cudaDeviceNew(t+1); + visited = cudaDeviceNew(n); + std::vector h_offsets(t+1); for(int i=0;i void rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex); + // Action instances have no dense transition matrices (see createInstance), so these are not available. + // (The base versions of the first three call the dense, non-virtual upPartials/upPrePartials.) + int updatePrePartials(const int*, int, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int updatePartialsByPartition(const int*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int updatePrePartialsByPartition(const int*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int setEigenDecomposition(int, const double*, const double*, const double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int setTransitionMatrix(int, const double*, double) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int setTransitionMatrices(const int*, const double*, const double*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int getTransitionMatrix(int, double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int convolveTransitionMatrices(const int*, const int*, const int*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int transposeTransitionMatrices(const int*, const int*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int updateTransitionMatricesWithModelCategories(int*, const int*, const int*, const int*, const double*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int updateTransitionMatricesWithMultipleModels(const int*, const int*, const int*, const int*, const int*, const double*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int calculateEdgeLogLikelihoods(const int*, const int*, const int*, const int*, const int*, const int*, const int*, const int*, + int, double*, double*, double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int calculateEdgeLogLikelihoodsByPartition(const int*, const int*, const int*, const int*, const int*, const int*, const int*, const int*, + const int*, int, int, double*, double*, double*, double*, double*, double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + int calculateEdgeDerivatives(const int*, const int*, const int*, const int*, const int*, const int*, + int, double*, double*, double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + protected: - int kPartialsCacheOffset; std::vector> hInstantaneousMatrices; std::vector> hBs; SpMatrix hIdentity; @@ -1174,6 +1221,9 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // std::vector dInstantaneousMatrices; std::vector> dPartialsWrapper; + // Per category: the action of each child's edge on its partials (exp(tQ) * partials). + std::vector> dDestPLeft; + std::vector> dDestPRight; std::vector> dFLeft; std::vector> dFRight; std::vector> dIntegrationTmpLeft; @@ -1210,6 +1260,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::kStateCount; using BeagleGPUImpl::kEigenDecompCount; using BeagleGPUImpl::kCategoryCount; + using BeagleGPUImpl::kMatrixCount; using BeagleGPUImpl::kPaddedStateCount; @@ -1269,14 +1320,13 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int count); double getPMax() const; - int getPartialIndex(int nodeIndex, int categoryIndex); + // Make CUDA runtime, cuBLAS, cuSPARSE and Thrust calls target this instance's device. + void setCudaDevice(); - int getPartialCacheIndex(int nodeIndex, int categoryIndex); + int getPartialIndex(int nodeIndex, int categoryIndex); DnMatrixDevice& getPartialsWrapper(int nodeIndex, int categoryIndex); - DnMatrixDevice& getPartialsCacheWrapper(int nodeIndex, int categoryIndex); - void calcPartialsPartials(int destPIndex, int partials1Index, int edgeIndex1, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 20829a08..b9fd610d 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -282,10 +282,15 @@ int BeagleGPUActionImpl::createInstance(int tipCount, fprintf(stderr, "\tEntering BeagleGPUActionImpl::createInstance\n"); #endif - BeagleGPUImpl::createInstance(tipCount, partialsBufferCount * 2, 0, stateCount, patternCount, + // The action flag makes the base class skip its dense (S x S) eigen and transition-matrix buffers. + int returnCode = BeagleGPUImpl::createInstance(tipCount, partialsBufferCount, 0, stateCount, patternCount, eigenDecompositionCount, matrixCount, categoryCount, scaleBufferCount, globalResourceNumber, pluginResourceNumber, - preferenceFlags, requirementFlags); - kPartialsCacheOffset = partialsBufferCount; + preferenceFlags | BEAGLE_FLAG_COMPUTATION_ACTION, requirementFlags); + if (returnCode != BEAGLE_SUCCESS) + return returnCode; + + setCudaDevice(); + hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); hIdentity.setIdentity(); hInstantaneousMatrices.resize(kEigenDecompCount); @@ -310,7 +315,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, { int t = 5; int itmax = 2; - L1normForPower.push_back({cublasHandle, p, kPaddedStateCount, t, 2}); + L1normForPower.push_back({cublasHandle, p, kPaddedStateCount, t, itmax}); // This is very fast, but computes |A| instead of |A^p| // L1normForPower.push_back(GPUnormest2(cublasHandle, cusparseHandle, p, kPaddedStateCount)); @@ -337,6 +342,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } } + Real* dDestPLeftPtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + Real* dDestPRightPtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); + Real* dLeftCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); Real* dRightCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); @@ -348,6 +356,10 @@ int BeagleGPUActionImpl::createInstance(int tipCount, integrationRightBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); + dDestPLeft.clear(); + + dDestPRight.clear(); + dFLeft.clear(); dFRight.clear(); @@ -360,6 +372,9 @@ int BeagleGPUActionImpl::createInstance(int tipCount, { size_t offset = kPaddedStateCount * kPaddedPatternCount * category; + dDestPLeft.emplace_back(cublasHandle, dDestPLeftPtr + offset, kPaddedStateCount, kPaddedPatternCount); + dDestPRight.emplace_back(cublasHandle, dDestPRightPtr + offset, kPaddedStateCount, kPaddedPatternCount); + dFLeft.emplace_back(cublasHandle, dLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); dFRight.emplace_back(cublasHandle, dRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); @@ -373,8 +388,8 @@ int BeagleGPUActionImpl::createInstance(int tipCount, CHECK_CUDA(cudaMalloc(&dIntegrationRightBuffer[category], integrationRightBufferSize[category])) } - hEigenMaps.resize(kPartialsCacheOffset); - hEdgeMultipliers.resize(kPartialsCacheOffset * kCategoryCount); + hEigenMaps.resize(kMatrixCount); + hEdgeMultipliers.resize(kMatrixCount * kCategoryCount); msCache = std::vector>(2 * kCategoryCount, tuple{0, 0}); etaCache = std::vector(2 * kCategoryCount, 0); @@ -389,6 +404,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, BEAGLE_GPU_TEMPLATE BeagleGPUActionImpl::~BeagleGPUActionImpl() { + cudaSetDevice(gpu->GetCudaDeviceOrdinal()); for (int i = 0; i < kCategoryCount; i++) { cudaFree(dIntegrationLeftBuffer[i]); @@ -396,6 +412,8 @@ BeagleGPUActionImpl::~BeagleGPUActionImpl() } // The memory for each matrix group was allocation in a single block. + cudaFree(dDestPLeft[0].ptr); + cudaFree(dDestPRight[0].ptr); cudaFree(dFLeft[0].ptr); cudaFree(dFRight[0].ptr); cudaFree(dIntegrationTmpLeft[0].ptr); @@ -509,6 +527,8 @@ int BeagleGPUActionImpl::updatePartials(const int* operation fprintf(stderr, "\tEntering BeagleGPUActionImpl::updatePartials\n"); #endif + setCudaDevice(); + bool byPartition = false; int returnCode = upPartials(byPartition, operations, @@ -538,13 +558,18 @@ void BeagleGPUActionImpl::rescalePartials(Real* partials, R } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int categoryIndex) { - return nodeIndex * kCategoryCount + categoryIndex; +void BeagleGPUActionImpl::setCudaDevice() +{ + // GPUInterface uses the driver API and pops its context after each call, so without this + // runtime-API work would go to device 0. + auto status = cudaSetDevice(gpu->GetCudaDeviceOrdinal()); + if (status != cudaSuccess) + throw std::runtime_error(std::string("cudaSetDevice: ") + cudaGetErrorString(status)); } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::getPartialCacheIndex(int nodeIndex, int categoryIndex) { - return kPartialsCacheOffset * kCategoryCount + getPartialIndex(nodeIndex, categoryIndex); +int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int categoryIndex) { + return nodeIndex * kCategoryCount + categoryIndex; } BEAGLE_GPU_TEMPLATE @@ -552,11 +577,6 @@ DnMatrixDevice& BeagleGPUActionImpl::getPartialsWrappe return dPartialsWrapper[getPartialIndex(nodeIndex, categoryIndex)]; } -BEAGLE_GPU_TEMPLATE -DnMatrixDevice& BeagleGPUActionImpl::getPartialsCacheWrapper(int nodeIndex, int categoryIndex) { - return dPartialsWrapper[getPartialCacheIndex(nodeIndex, categoryIndex)]; -} - BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPartials(bool byPartition, const int *operations, @@ -773,10 +793,10 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; - simpleAction2(getPartialsCacheWrapper(partials1Index, category), getPartialsWrapper(partials1Index, category), + simpleAction2(dDestPLeft[category], getPartialsWrapper(partials1Index, category), edgeIndex1, category, matrixIndex1, true, false); - simpleAction2(getPartialsCacheWrapper(partials2Index, category), getPartialsWrapper(partials2Index, category), + simpleAction2(dDestPRight[category], getPartialsWrapper(partials2Index, category), edgeIndex2, category, matrixIndex2, false, false); } @@ -786,17 +806,14 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde { const int destPartialIndex = getPartialIndex(destPIndex, category); - const int partial1CacheIndex = getPartialCacheIndex(partials1Index, category); - const int partial2CacheIndex = getPartialCacheIndex(partials2Index, category); - // element-wise multiply if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialsWrapper[partial1CacheIndex].ptr, kPaddedStateCount * kPaddedPatternCount, - dPartialsWrapper[partial2CacheIndex].ptr, 1, dPartialsWrapper[destPartialIndex].ptr, kPaddedStateCount * kPaddedPatternCount)); + CUBLAS_CHECK(cublasSdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dDestPLeft[category].ptr, kPaddedStateCount * kPaddedPatternCount, + dDestPRight[category].ptr, 1, dPartialsWrapper[destPartialIndex].ptr, kPaddedStateCount * kPaddedPatternCount)); } else { - CUBLAS_CHECK(cublasDdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dPartialsWrapper[partial1CacheIndex].ptr, kPaddedStateCount * kPaddedPatternCount, - dPartialsWrapper[partial2CacheIndex].ptr, 1, dPartialsWrapper[destPartialIndex].ptr, kPaddedStateCount * kPaddedPatternCount)); + CUBLAS_CHECK(cublasDdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dDestPLeft[category].ptr, kPaddedStateCount * kPaddedPatternCount, + dDestPRight[category].ptr, 1, dPartialsWrapper[destPartialIndex].ptr, kPaddedStateCount * kPaddedPatternCount)); } //#ifdef BEAGLE_DEBUG_FLOW @@ -893,6 +910,8 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i fprintf(stderr, "\tEntering BeagleGPUActionImpl::calculateRootLogLikelihoods\n"); #endif + setCudaDevice(); + int returnCode = BEAGLE_SUCCESS; const int rootNodeIndex = bufferIndices[0]; @@ -1021,35 +1040,34 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub int bestM = INT_MAX; double bestS = INT_MAX; // Not all the values of s can fit in a 32-bit int. - const double theta = thetaConstants.at(mMax); const double pMax = getPMax(); // pMax is the largest positive integer such that p*(p-1) <= mMax + 1 - const bool conditionFragment313 = hB1Norms[eigenIndex] * edgeMultiplier <= 2.0 * theta / ((double) nCol * mMax) * pMax * (pMax + 3); - // using l = 1 as in equation 3.13 - if (conditionFragment313) { - for (auto& [thisM, thetaM]: thetaConstants) { - const double thisS = ceil(hB1Norms[eigenIndex] * edgeMultiplier / thetaM); - if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { - bestS = thisS; - bestM = thisM; - } + // 1. The best (m,s) based only on ||B||_1 (in some sense the result for p=1). + for (auto& [thisM, thetaM]: thetaConstants) { + const double thisS = ceil(hB1Norms[eigenIndex] * edgeMultiplier / thetaM); + if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { + bestS = thisS; + bestM = thisM; } - } else { - for (int p = 2; p < pMax; p++) { - for (int thisM = p * (p - 1) - 1; thisM < mMax + 1; thisM++) { - auto it = thetaConstants.find(thisM); - if (it != thetaConstants.end()) { - // equation 3.7 in Al-Mohy and Higham - const double dValueP = getDValue(p, eigenIndex); - const double dValuePPlusOne = getDValue(p + 1, eigenIndex); - const double alpha = std::max(dValueP, dValuePPlusOne) * edgeMultiplier; - // part of equation 3.10 - const double thisS = ceil(alpha / thetaConstants.at(thisM)); - if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { - bestS = thisS; - bestM = thisM; - } + } + + // 2. Improve on it with the d_p. Condition 3.13 of Al-Mohy and Higham only decides whether computing + // the d_p is worth it; setSparseMatrix( ) has already computed all of them, so always use them + // (as the CPU implementation does once they are available). + for (int p = 2; p <= pMax; p++) { + for (int thisM = p * (p - 1) - 1; thisM < mMax + 1; thisM++) { + auto it = thetaConstants.find(thisM); + if (it != thetaConstants.end()) { + // equation 3.7 in Al-Mohy and Higham + const double dValueP = getDValue(p, eigenIndex); + const double dValuePPlusOne = getDValue(p + 1, eigenIndex); + const double alpha = std::max(dValueP, dValuePPlusOne) * edgeMultiplier; + // part of equation 3.10 + const double thisS = ceil(alpha / thetaConstants.at(thisM)); + if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { + bestS = thisS; + bestM = thisM; } } } @@ -1120,7 +1138,7 @@ BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& destP, const DnMatrixDevice& inPartials, int edgeIndex, int category, int matrixIndex, bool left, bool transpose) { const double tol = pow(2.0, -53.0); const double t = 1.0; - const int nCol = kPaddedStateCount * kPaddedPatternCount; + const int nCol = kPaddedPatternCount; const int edgeMultiplierIndex = edgeIndex * kCategoryCount + category; const double edgeMultiplier = hEdgeMultipliers[edgeMultiplierIndex]; @@ -1137,21 +1155,24 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& auto& A = dAs[matrixIndex]; vector>* FF; vector>* integrationTmpPtr; - std::vector integrationBuffer; - std::vector integrationBufferSize; + // Pointers, not copies: spMM may reallocate the buffer, and the new one must be kept. + std::vector* integrationBufferPtr; + std::vector* integrationBufferSizePtr; if (left) { FF = &dFLeft; integrationTmpPtr = &dIntegrationTmpLeft; - integrationBuffer = dIntegrationLeftBuffer; - integrationBufferSize = integrationLeftBufferSize; + integrationBufferPtr = &dIntegrationLeftBuffer; + integrationBufferSizePtr = &integrationLeftBufferSize; } else { FF = &dFRight; integrationTmpPtr = &dIntegrationTmpRight; - integrationBuffer = dIntegrationRightBuffer; - integrationBufferSize = integrationRightBufferSize; + integrationBufferPtr = &dIntegrationRightBuffer; + integrationBufferSizePtr = &integrationRightBufferSize; } auto& F = (*FF)[category]; auto& integrationTmp = (*integrationTmpPtr)[category]; + auto& integrationBuffer = (*integrationBufferPtr)[category]; + auto& integrationBufferSize = (*integrationBufferSizePtr)[category]; destP.copyFrom( inPartials ); @@ -1177,7 +1198,7 @@ int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& #endif - spMM(integrationTmp, t / ((Real) s * j), A, destP, 0, integrationBuffer[category], integrationBufferSize[category]); + spMM(integrationTmp, t / ((Real) s * j), A, destP, 0, integrationBuffer, integrationBufferSize); destP.copyFrom( integrationTmp ); @@ -1215,7 +1236,7 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i int partialsIndex2, int edgeIndex2) { const double tol = pow(2.0, -53.0); const double t = 1.0; - const int nCol = kPaddedStateCount * kPaddedPatternCount; + const int nCol = kPaddedPatternCount; @@ -1246,8 +1267,8 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i auto& FLeft = dFLeft[category]; auto& FRight = dFRight[category]; - auto& destPLeft = getPartialsCacheWrapper(partialsIndex1, category); - auto& destPRight = getPartialsCacheWrapper(partialsIndex2, category); + auto& destPLeft = dDestPLeft[category]; + auto& destPRight = dDestPRight[category]; auto& inPartialsLeft = getPartialsWrapper(partialsIndex1, category); auto& inPartialsRight = getPartialsWrapper(partialsIndex2, category); @@ -1298,8 +1319,8 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i auto& FLeft = dFLeft[category]; auto& FRight = dFRight[category]; - auto& destPLeft = getPartialsCacheWrapper(partialsIndex1, category); - auto& destPRight = getPartialsCacheWrapper(partialsIndex2, category); + auto& destPLeft = dDestPLeft[category]; + auto& destPRight = dDestPRight[category]; spMM(dIntegrationTmpLeft[category], t / ((Real) s_max * j), ALeft, destPLeft, 0, dIntegrationLeftBuffer[category], integrationLeftBufferSize[category]); spMM(dIntegrationTmpRight[category], t / ((Real) s_max * j), ARight, destPRight, 0, dIntegrationRightBuffer[category], integrationRightBufferSize[category]); @@ -1326,8 +1347,8 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i auto& FLeft = dFLeft[category]; auto& FRight = dFRight[category]; - auto& destPLeft = getPartialsCacheWrapper(partialsIndex1, category); - auto& destPRight = getPartialsCacheWrapper(partialsIndex2, category); + auto& destPLeft = dDestPLeft[category]; + auto& destPRight = dDestPRight[category]; #ifdef BEAGLE_DEBUG_FLOW cudaDeviceSynchronize(); @@ -1384,8 +1405,8 @@ int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, i auto& FLeft = dFLeft[category]; auto& FRight = dFRight[category]; - auto& destPLeft = getPartialsCacheWrapper(partialsIndex1, category); - auto& destPRight = getPartialsCacheWrapper(partialsIndex2, category); + auto& destPLeft = dDestPLeft[category]; + auto& destPRight = dDestPRight[category]; destPLeft.copyFromAsync( FLeft ); destPRight.copyFromAsync( FRight ); @@ -1403,7 +1424,8 @@ int BeagleGPUActionImpl::upPrePartials(bool byPartition, int operationCount, int cumulativeScalingIndex) { - return BeagleGPUImpl::upPrePartials(byPartition, operations, operationCount, cumulativeScalingIndex); + // Not yet implemented for action; the dense fallback would read transition matrices that are never formed. + return BEAGLE_ERROR_NO_IMPLEMENTATION; } BEAGLE_GPU_TEMPLATE @@ -1413,6 +1435,8 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, const Real* values, int numNonZeros) { + setCudaDevice(); + std::vector> tripletList; for (int i = 0; i < numNonZeros; i++) { tripletList.push_back(Triplet(rowIndices[i], colIndices[i], values[i])); @@ -1451,11 +1475,13 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, const int currentNNZ = hBs[matrixIndex].nonZeros(); if (currentCacheNNZs[matrixIndex] != currentNNZ) { currentCacheNNZs[matrixIndex] = currentNNZ; + cudaDeviceDelete(dBsCsrColumnsCache[matrixIndex]); + cudaDeviceDelete(dBsCsrValuesCache[matrixIndex]); dBsCsrColumnsCache[matrixIndex] = cudaDeviceNew(currentNNZ); dBsCsrValuesCache[matrixIndex] = cudaDeviceNew(currentNNZ); - for (int category = 0; category < kCategoryCount; category++) { - dACscValuesCache[matrixIndex * kCategoryCount * 2 + category] = cudaDeviceNew(currentNNZ); - dACscValuesCache[matrixIndex * kCategoryCount * 2 + kCategoryCount + category] = cudaDeviceNew(currentNNZ); + for (int j = 0; j < kCategoryCount * 2; j++) { + cudaDeviceDelete(dACscValuesCache[matrixIndex * kCategoryCount * 2 + j]); + dACscValuesCache[matrixIndex * kCategoryCount * 2 + j] = cudaDeviceNew(currentNNZ); } } @@ -1490,14 +1516,28 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, int pMax = getPMax(); auto tmp = hBs[matrixIndex]; +#ifdef BEAGLE_DEBUG_NORMEST + DnMatrix denseB = hBs[matrixIndex]; + DnMatrix denseBp = DnMatrix::Identity(kPaddedStateCount, kPaddedStateCount); +#endif for(int p=0;p <= pMax+1; p++) { // GPU Norm Real approx_norm = L1normForPower[p]( dB ); -// // CPU Norm -// int t = 5; -// Real approx_norm_CPU = normest1( hBs[matrixIndex], p, t); +#ifdef BEAGLE_DEBUG_NORMEST + // Compare with the CPU estimator and, for small S, the exact ||B^p||_1. + Real approx_norm_CPU = normest1( hBs[matrixIndex], p, 5); + double exact_norm = -1; + if (kPaddedStateCount <= 4096) + { + if (p > 0) denseBp = denseB * denseBp; + exact_norm = normP1(denseBp); + } + std::fprintf(stderr, "normest matrix=%d S=%d p=%d gpu=%.6e cpu=%.6e exact=%.6e gpu/exact=%.4f cpu/exact=%.4f\n", + matrixIndex, kPaddedStateCount, p, (double)approx_norm, (double)approx_norm_CPU, exact_norm, + approx_norm / exact_norm, approx_norm_CPU / exact_norm); +#endif // equation 3.7 in Al-Mohy and Higham hds[matrixIndex].push_back(pow(approx_norm, 1.0 / Real(p) ) ); diff --git a/libhmsbeagle/GPU/BeagleGPUImpl.hpp b/libhmsbeagle/GPU/BeagleGPUImpl.hpp index b9aef636..42ed0e35 100755 --- a/libhmsbeagle/GPU/BeagleGPUImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUImpl.hpp @@ -146,7 +146,8 @@ BeagleGPUImpl::~BeagleGPUImpl() { } // TODO: free subpointers - gpu->FreeMemory(dMatrices[0]); // TODO Should try: gpu->FreeMemory(dMatricesOrigin); instead of above + if (kMatrixCount > 0 && dMatrices[0]) + gpu->FreeMemory(dMatrices[0]); // TODO Should try: gpu->FreeMemory(dMatricesOrigin); instead of above if(dEigenValues) { gpu->FreeMemory(dEigenValues[0]); // TODO Here is where my Mac / Intel-GPU are throwing bad-exception gpu->FreeMemory(dEvec[0]); // TODO Should be save and then release just d*Origin? @@ -600,6 +601,9 @@ int BeagleGPUImpl::createInstance(int tipCount, int hMatrixCacheSize = kMatrixSize * kCategoryCount * BEAGLE_CACHED_MATRICES_COUNT; if ((2 * kMatrixSize + kEigenValuesSize) > hMatrixCacheSize) hMatrixCacheSize = 2 * kMatrixSize + kEigenValuesSize; + // Action-based instances never form dense transition matrices. + if (preferenceFlags & BEAGLE_FLAG_COMPUTATION_ACTION) + hMatrixCacheSize = kEigenValuesSize; hLogLikelihoodsCache = (Real*) gpu->MallocHost(kPatternCount * sizeof(Real)); hMatrixCache = (Real*) gpu->CallocHost(hMatrixCacheSize, sizeof(Real)); @@ -616,13 +620,19 @@ int BeagleGPUImpl::createInstance(int tipCount, dWeights = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); dFrequencies = (GPUPtr*) calloc(sizeof(GPUPtr),kEigenDecompCount); - dMatrices = (GPUPtr*) malloc(sizeof(GPUPtr) * kMatrixCount); + // Fill with 0s so 'free' does not choke if unallocated + dMatrices = (GPUPtr*) calloc(sizeof(GPUPtr), kMatrixCount); size_t ptrIncrement = gpu->AlignMemOffset(kMatrixSize * kCategoryCount * sizeof(Real)); kIndexOffsetMat = ptrIncrement/sizeof(Real); - dMatricesOrigin = gpu->AllocateMemory(kMatrixCount * ptrIncrement); - for (int i = 0; i < kMatrixCount; i++) { - dMatrices[i] = gpu->CreateSubPointer(dMatricesOrigin, ptrIncrement*i, ptrIncrement); + if (preferenceFlags & BEAGLE_FLAG_COMPUTATION_ACTION) { + // O(S^2) per matrix: exhausts device memory for large state spaces, and is unused by action. + dMatricesOrigin = (GPUPtr) NULL; + } else { + dMatricesOrigin = gpu->AllocateMemory(kMatrixCount * ptrIncrement); + for (int i = 0; i < kMatrixCount; i++) { + dMatrices[i] = gpu->CreateSubPointer(dMatricesOrigin, ptrIncrement*i, ptrIncrement); + } } if (kScaleBufferCount > 0) { if (kFlags & BEAGLE_FLAG_SCALING_AUTO) { diff --git a/libhmsbeagle/GPU/GPUInterface.h b/libhmsbeagle/GPU/GPUInterface.h index cc8a3e13..a1883e39 100644 --- a/libhmsbeagle/GPU/GPUInterface.h +++ b/libhmsbeagle/GPU/GPUInterface.h @@ -66,6 +66,7 @@ class GPUInterface { int numStreams; #ifdef CUDA CUdevice cudaDevice; + int cudaDeviceOrdinal; CUcontext cudaContext; CUmodule cudaModule; CUstream* cudaStreams; @@ -90,6 +91,11 @@ class GPUInterface { int GetDeviceCount(); +#ifdef CUDA + // Ordinal of the device chosen by SetDevice( ), for code that uses the CUDA runtime API. + int GetCudaDeviceOrdinal() { return cudaDeviceOrdinal; } +#endif + void SetDevice(int deviceNumber, int paddedStateCount, int categoryCount, diff --git a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp index b499a33d..c152ab22 100644 --- a/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp +++ b/libhmsbeagle/GPU/GPUInterfaceCUDA.cpp @@ -116,6 +116,7 @@ GPUInterface::GPUInterface() { #endif cudaDevice = (CUdevice) 0; + cudaDeviceOrdinal = 0; cudaContext = NULL; cudaModule = NULL; cudaStreams = NULL; @@ -252,7 +253,8 @@ void GPUInterface::SetDevice(int deviceNumber, int paddedStateCount, int categor fprintf(stderr,"\t\t\tEntering GPUInterface::SetDevice\n"); #endif - SAFE_CUDA(cuDeviceGet(&cudaDevice, (*resourceMap)[deviceNumber])); + cudaDeviceOrdinal = (*resourceMap)[deviceNumber]; + SAFE_CUDA(cuDeviceGet(&cudaDevice, cudaDeviceOrdinal)); // unsigned int ctxFlags = CU_CTX_SCHED_AUTO; diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 9e8e4af3..355471cc 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -2,6 +2,8 @@ #include +#include + #include #include #include @@ -11,7 +13,7 @@ #include #include #include -#include +#include #include #include @@ -124,7 +126,7 @@ T cuda_max_l1_norm(device_ptr values, int n, int t, device_ptr buffer) ); // summation is the default operation. // 2. Second maximize over the column sums and return the highest. - return thrust::reduce(buffer, buffer+t, 0.0, thrust::maximum()); + return thrust::reduce(buffer, buffer+t, T(0), thrust::maximum()); } float cuda_max_l1_norm(float* values, int n, int t, float* buffer) @@ -298,50 +300,65 @@ void cuda_vec_abs(double* values, int n, double* results) cuda_vec_abs(device_pointer_cast(values), n, device_pointer_cast(results)); } -void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr) +// out[i] = max_j |values(i,j)| for a column-major (n,t) matrix. +// (A reduce_by_key with keys i % n cannot do this: equal keys are never adjacent in column-major order.) +template +__global__ void rowwise_max_abs_kernel(const T* values, int n, int t, T* out) { - using namespace thrust::placeholders; - // We assume that the matrix has dimensions (n,t) and is column-major. + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; - // 1. First sum the absolute values in each column and place the results into buffer - // Using _1 % n here should group by row, yielding 0 1 2 3 ...(n-1) 0 1 2 3 ... (n-1) ... - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 % n)); - - auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values_ptr), - [] __host__ __device__ (float x) {return std::abs(x);} ); + T m = 0; + for (int j = 0; j < t; j++) + m = max(m, abs(values[i + (size_t)j * n])); + out[i] = m; +} - auto out = thrust::device_pointer_cast(out_ptr); +template +void cuda_rowwise_max_abs_impl(const T* values_ptr, int n, int t, T* out_ptr) +{ + const int blockSize = 256; + rowwise_max_abs_kernel<<<(n + blockSize - 1) / blockSize, blockSize>>>(values_ptr, n, t, out_ptr); +} - thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) - in_values_start, // values to reduce (with abs applied) - thrust::make_discard_iterator(), // key values out - out, // reduced values out - thrust::equal_to(), // compare-keys operation - thrust::maximum() // summation is the default operation. - ); +void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr) +{ + cuda_rowwise_max_abs_impl(values_ptr, n, t, out_ptr); } void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr) { - using namespace thrust::placeholders; - // We assume that the matrix has dimensions (n,t) and is column-major. + cuda_rowwise_max_abs_impl(values_ptr, n, t, out_ptr); +} - // 1. First sum the absolute values in each column and place the results into buffer - // Using _1 % n here should group by row, yielding 0 1 2 3 ...(n-1) 0 1 2 3 ... (n-1) ... - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 % n)); +// h[i] = -1 for rows already used as unit columns, so they sort last and are not chosen again. +template +__global__ void mask_visited_kernel(T* h, const int* visited, int n) +{ + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n and visited[i]) h[i] = -1; +} + +// visited[indices[j]] = 1 for j in [0,t-1] +__global__ void mark_visited_kernel(int* visited, const int* indices, int t) +{ + int j = blockIdx.x * blockDim.x + threadIdx.x; + if (j < t) visited[indices[j]] = 1; +} - auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values_ptr), - [] __host__ __device__ (double x) {return std::abs(x);} ); +void cuda_mask_visited(float* h, const int* visited, int n) +{ + mask_visited_kernel<<<(n + 255) / 256, 256>>>(h, visited, n); +} - auto out = thrust::device_pointer_cast(out_ptr); +void cuda_mask_visited(double* h, const int* visited, int n) +{ + mask_visited_kernel<<<(n + 255) / 256, 256>>>(h, visited, n); +} - thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) - in_values_start, // values to reduce (with abs applied) - thrust::make_discard_iterator(), // key values out - out, // reduced values out - thrust::equal_to(), // compare-keys operation - thrust::maximum() // summation is the default operation. - ); +void cuda_mark_visited(int* visited, const int* indices, int t) +{ + mark_visited_kernel<<<(t + 255) / 256, 256>>>(visited, indices, t); } void cuda_fill_vector(float* buffer, int n, double x) @@ -457,7 +474,7 @@ void justMaximize(device_ptr partials, device_ptr scalingFactors, thrust::make_discard_iterator(), // key values out scalingFactors, // reduced values out thrust::equal_to(), // compare keys operation - thrust::maximum() // reduction operation + thrust::maximum() // reduction operation ); } @@ -543,7 +560,7 @@ void sumRootLikelihoods(device_ptr siteProbs, // OUT // 5. Make the sequence of likelihood * categoryWeight * stateFrequency auto tuples = thrust::make_zip_iterator(thrust::make_tuple(in_lks_start, in_weights_start, in_frequencies_start)); - auto in_values_start = thrust::make_transform_iterator(tuples, Prod3()); + auto in_values_start = thrust::make_transform_iterator(tuples, Prod3()); // 6. Sum over state and category for each pattern. thrust::reduce_by_key( diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 611538ba..6646696b 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -104,6 +104,13 @@ void cuda_max(float* v, int length, float* out, cuda_scratch_space& scratch); void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr); void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr); +// h[i] = -1 where visited[i] != 0 +void cuda_mask_visited(float* h, const int* visited, int n); +void cuda_mask_visited(double* h, const int* visited, int n); + +// visited[indices[j]] = 1 for j in [0,t-1] +void cuda_mark_visited(int* visited, const int* indices, int t); + void cuda_fill_vector(float* buffer, int n, double x); void cuda_fill_vector(double* buffer, int n, double x); @@ -113,7 +120,7 @@ void initialize_norm_x_matrix(double* data, int n, int m); float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); -void cuda_max_l1_norm(float* values, int n, int t, float* buffer, double* out, cuda_scratch_space& scratch, int* offsets); +void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out, cuda_scratch_space& scratch, int* offsets); void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch, int* offsets); void cuda_vec_fill(float* values, int length, float fill); From fd47d6ef64691e80a1c6ece91845cf88cd6a92ff Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Sat, 24 Oct 2026 18:11:48 -0500 Subject: [PATCH 411/421] GPU action: remove per-term and per-node overhead (Phase 1) - Stream: all action work (cuBLAS, cuSPARSE, kernels, copies) runs on the GPUInterface's stream 0, where the base class issues its own copies and kernels. For S > 4 that is a created blocking stream, which is not ordered with a second stream; the old host-synchronous Thrust calls hid this race (e.g. the async device-to-host copy of the logL). - Per node: one kernel for the child product, per-pattern max, reciprocal rescale and log/cumulative scalers; the root likelihood is two kernels (per-pattern log likelihood, deterministic weighted sum). No Thrust and no host synchronization on the hot path. - Taylor loop: A = edgeMultiplier * B is never formed (the multiplier goes into alpha; cacheAMatrices and its copies are gone), terms alternate between two buffers shared by both children, F accumulates in place, and F *= eta is folded into the last term. The SpMM buffer is sized once per Q. For small work (nnz * columns < 3e6) one fused column-major kernel per term replaces cuSPARSE's partition + beta fill + SpMM + axpy. - setSparseMatrix: one batched block 1-norm estimator for all powers (chained products, top-t kernel, B^T via csr2csc, one copy to the host): ~50 instead of ~700 GPU operations. B = Q - mu*I is built in CSR directly from the triplets instead of through Eigen. - Tip partials: update the cuSPARSE descriptor when the pointer changes. A100, vs Phase 0 (ms per evaluation incl. setQ): MM M=8, 64 patterns 6.0 -> 1.3; IGC, 1 pattern 16.7 -> 4.8; IGC, 4,096 patterns 1,136 -> 1,034; setQ 3.5-8.1 -> 0.24-2.1. logL matches the CPU action plugin to <= 1.5e-14 relative on the MM/IGC workloads. Co-Authored-By: Claude Opus 5.5 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 759 ++++++++------------ libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 714 +++++-------------- libhmsbeagle/GPU/GPUInterface.h | 4 + libhmsbeagle/GPU/cuda_ops.cu | 847 ++++++++--------------- libhmsbeagle/GPU/cuda_ops.h | 110 ++- 5 files changed, 799 insertions(+), 1635 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index bf50185d..d0779a5c 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -123,6 +123,88 @@ template cudaDataType DataType; template <> cudaDataType DataType = CUDA_R_32F; template <> cudaDataType DataType = CUDA_R_64F; +// A CSR matrix on the host. +template +struct HostCsr +{ + std::vector offsets; + std::vector columns; + std::vector values; + + int nonZeros() const {return (int) values.size();} +}; + +// B = Q - mu*I as an (n,n) CSR matrix with sorted columns, built directly from Q's triplets. Duplicate +// triplets are summed, as Eigen's setFromTriplets( ) does, and every row gets a diagonal entry. +// Two stable counting sorts (by column, then by row) leave each row sorted, in O(nnz + n) for any input +// order. (Going through Eigen took ~2 ms for 75K non-zeros.) +template +void buildShiftedCsr(int n, const int* rows, const int* cols, const Real* values, int nnz, Real mu, HostCsr& B) +{ + const int m = nnz + n; // the triplets, plus (i, i, -mu) for each row + auto rowOf = [&](int k) {return (k < nnz) ? rows[k] : k - nnz;}; + auto colOf = [&](int k) {return (k < nnz) ? cols[k] : k - nnz;}; + auto valueOf = [&](int k) {return (k < nnz) ? values[k] : -mu;}; + + // 1. Order the entries by column. + std::vector start(n + 1, 0); + for (int k = 0; k < m; k++) + start[colOf(k) + 1]++; + for (int j = 0; j < n; j++) + start[j + 1] += start[j]; + std::vector byColumn(m); + for (int k = 0; k < m; k++) + byColumn[start[colOf(k)]++] = k; + + // 2. Stably order them by row, so each row is sorted by column. + B.offsets.assign(n + 1, 0); + for (int k = 0; k < m; k++) + B.offsets[rowOf(k) + 1]++; + for (int i = 0; i < n; i++) + B.offsets[i + 1] += B.offsets[i]; + std::vector next(B.offsets.begin(), B.offsets.end() - 1); + B.columns.resize(m); + B.values.resize(m); + for (int k: byColumn) + { + const int a = next[rowOf(k)]++; + B.columns[a] = colOf(k); + B.values[a] = valueOf(k); + } + + // 3. Sum duplicates, compacting in place. + int out = 0; + for (int i = 0; i < n; i++) + { + const int begin = B.offsets[i]; + const int end = B.offsets[i + 1]; + B.offsets[i] = out; + for (int a = begin; a < end; a++) + { + if (out > B.offsets[i] and B.columns[out - 1] == B.columns[a]) + B.values[out - 1] += B.values[a]; + else + { + B.columns[out] = B.columns[a]; + B.values[out++] = B.values[a]; + } + } + } + B.offsets[n] = out; + B.columns.resize(out); + B.values.resize(out); +} + +// max_j sum_i |B(i,j)| +template +double normP1(int n, const HostCsr& B) +{ + std::vector sums(n, 0.0); + for (int k = 0; k < B.nonZeros(); k++) + sums[B.columns[k]] += std::abs(B.values[k]); + return n ? *std::max_element(sums.begin(), sums.end()) : 0.0; +} + template struct DnMatrixDevice { @@ -144,12 +226,14 @@ struct DnMatrixDevice MemcpyDeviceToDevice(ptr, D.ptr, size()); } - void copyFromAsync(const DnMatrixDevice& D) + void copyFromAsync(const DnMatrixDevice& D, cudaStream_t stream) { assert(order == D.order); assert(size1 == D.size1); assert(size2 == D.size2); - MemcpyDeviceToDevice(ptr, D.ptr, size()); + auto status = cudaMemcpyAsync(ptr, D.ptr, byte_size(), cudaMemcpyDeviceToDevice, stream); + if (status != cudaSuccess) + throw std::runtime_error("cudaMemcpyAsync(Device->Device): " + std::string(cudaGetErrorString(status))); } // Disallow copying -- only one object can "own" the descriptor. @@ -204,6 +288,38 @@ struct DnMatrixDevice return *this; } + // Point at different memory; the cuSPARSE descriptor must follow, or SpMM reads the old buffer. + void setPointer(Real* p) + { + ptr = p; + if (descr and cusparseDnMatSetValues(descr, p) != CUSPARSE_STATUS_SUCCESS) + { + std::cerr<<"cusparseDnMatSetValues( ) failed!"; + std::exit(1); + } + } + + // (*this) = a * (*this) + b * D, in one pass. + void axpby(Real a, Real b, const DnMatrixDevice& D) + { + cublasStatus_t status; + assert(D.size1 == size1); + assert(D.size2 == size2); + assert(order == CUSPARSE_ORDER_COL and D.order == CUSPARSE_ORDER_COL); + // geam allows C to alias A when A is not transposed and lda == ldc. + if constexpr (std::is_same::value) { + status = cublasSgeam(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_N, size1, size2, &a, ptr, size1, &b, D.ptr, size1, ptr, size1); + } else { + status = cublasDgeam(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_N, size1, size2, &a, ptr, size1, &b, D.ptr, size1, ptr, size1); + } + + if (status != CUBLAS_STATUS_SUCCESS) + { + std::cerr<<"cublas error "<::axpby\n"; + exit(1); + } + } + // Disallow copying. DnMatrixDevice(const DnMatrixDevice&) = delete; // Allow moving. @@ -400,32 +516,29 @@ int spMM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, con return spMM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); } +// C = alpha * A * B with a work buffer already sized by spMMBufferSize( ). template -int spMTM(cusparseHandle_t handle, cusparseDnMatDescr_t C, Real alpha, cusparseSpMatDescr_t A, cusparseDnMatDescr_t B, Real beta, void*& buffer, size_t& buffersize) +int spMMPrepared(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, void* buffer) { - size_t new_buffersize; - CHECK_CUSPARSE(cusparseSpMM_bufferSize(handle, CUSPARSE_OPERATION_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A, B, &beta, C, DataType, - CUSPARSE_SPMM_ALG_DEFAULT, &new_buffersize)); - - if(new_buffersize > buffersize) - { - cudaDeviceDelete(buffer); - buffer = cudaMallocWrapped(new_buffersize); - buffersize = new_buffersize; - } - - // integrationTmp = alpha * A * destP - CHECK_CUSPARSE(cusparseSpMM(handle, CUSPARSE_OPERATION_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A, B, &beta, C, DataType, - CUSPARSE_SPMM_ALG_DEFAULT, buffer)); //row-major layout provides higher performance (?) + Real beta = 0; + CHECK_CUSPARSE(cusparseSpMM(A.cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A.descr, B.descr, &beta, C.descr, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, buffer)); return 0; } template -int spMTM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, Real beta, void*& buffer, size_t& buffersize) +size_t spMMBufferSize(const DnMatrixDevice& C, const SpMatrixDevice& A, const DnMatrixDevice& B) { - return spMTM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); + Real alpha = 1; + Real beta = 0; + size_t size = 0; + auto status = cusparseSpMM_bufferSize(A.cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, + &alpha, A.descr, B.descr, &beta, C.descr, DataType, + CUSPARSE_SPMM_ALG_DEFAULT, &size); + if (status != CUSPARSE_STATUS_SUCCESS) + throw std::runtime_error(std::string("cusparseSpMM_bufferSize: ") + cusparseGetErrorString(status)); + return size; } // Create an io-manipulator so that we can write std::cerr<& H) -// Make the temporary variables part of a data structure that represents to normest1 problem. -// We don't want to allocate memory on the GPU every time we run this. +// Block 1-norm estimates of ||B^p||_1 for p = 2 .. pTop at once, for the d_p of equation 3.7 in Al-Mohy and +// Higham. This is Algorithm 2.4 of Higham and Tisseur (2000) with 2 iterations and t columns (without +// resampling parallel columns), but the products for all p are chained, so it takes about 3*pTop launches +// of at most (pTop-1)*t columns instead of O(pTop^2) separate SpMMs: +// 1. Y_p = B^p X0 for p = 1..pTop: one chain, each step reading the previous block. +// 2. Z_p = (B^T)^p sign(Y_p): start from [sign(Y_pTop)]; each step multiplies all blocks so far by B^T +// and appends the next sign block, so sign(Y_p) is multiplied exactly p times. +// 3. X_p = the t unit vectors e_i with the largest max_j |Z_p(i,j)|. +// 4. B^p X_p, by the same appending chain with B. +// The estimate for p is the largest column 1-norm of Y_p and of B^p X_p. Everything stays on the +// device until one copy of the pTop+1 results. template -struct GPUnormest1 +struct BlockNormEstimator { - int p = 0; int n = 0; int t = 0; - int itmax = 0; - DnMatrixDevice X; // (n,t) - DnMatrixDevice Y; // (n,t) - DnMatrixDevice h; // (n,1) - - // Sorted matrix dimensions by approximate norm. - int* indices = nullptr; // (n) - - // Used for spMM - void* buffer = nullptr; - size_t buffer_size = 0; - - // Temporary storage for cuda_max_l1_norm - Real* buffer2 = nullptr; - - // Space to store the norms for the different iterations. - Real* buffer3 = nullptr; - - // Space to store a result - Real* buffer4 = nullptr; - - GPUnormest1& operator=(const GPUnormest1&) = delete; - GPUnormest1& operator=(GPUnormest1&& g) + int pTop = 0; + + Real* X0 = nullptr; // (n, t) + Real* Y = nullptr; // (n, pTop*t): block p-1 is B^p X0 + Real* chain[2] = {nullptr, nullptr}; // (n, (pTop-1)*t) each + Real* sums1 = nullptr; // pTop*t column sums of Y + Real* sums2 = nullptr; // (pTop-1)*t column sums of step 4 + int* unitRows = nullptr; // (pTop-1)*t + Real* norms = nullptr; // pTop+1 + + // B^T in CSR (that is, B in CSC) + int* btOffsets = nullptr; + int* btColumns = nullptr; + Real* btValues = nullptr; + int btCapacity = 0; + void* transposeBuffer = nullptr; + size_t transposeBufferSize = 0; + + void allocate(int n_, int t_, int pTop_) { - std::swap(p, g.p); - std::swap(n, g.n); - std::swap(t, g.t); - std::swap(itmax, g.itmax); - std::swap(X, g.X); - std::swap(Y, g.Y); - std::swap(h, g.h); - - // ensure that (*this) and (g) don't both own the buffers! - std::swap(buffer, g.buffer); - std::swap(buffer_size, g.buffer_size); - std::swap(indices, g.indices); - std::swap(buffer2, g.buffer2); - std::swap(buffer3, g.buffer3); - std::swap(buffer4, g.buffer4); - - return *this; + n = n_; + t = std::min(t_, n); + pTop = pTop_; + assert(t <= 32); // cuda_select_unit_columns( ) + X0 = cudaDeviceNew(n * t); + Y = cudaDeviceNew(n * t * pTop); + chain[0] = cudaDeviceNew(n * t * (pTop - 1)); + chain[1] = cudaDeviceNew(n * t * (pTop - 1)); + sums1 = cudaDeviceNew(t * pTop); + sums2 = cudaDeviceNew(t * (pTop - 1)); + unitRows = cudaDeviceNew(t * (pTop - 1)); + norms = cudaDeviceNew(pTop + 1); + btOffsets = cudaDeviceNew(n + 1); + initialize_norm_x_matrix(X0, n, t); } - Real operator()(const SpMatrixDevice& A) + // Returns ||B^p||_1 estimates for p = 2..pTop at index p. (Indices 0 and 1 are not set.) + std::vector estimate(const SpMatrixDevice& B, cudaStream_t stream) { - // std::cerr<<"n = "<(Y, 1, A, X, 0, buffer, buffer_size); - // X = Y - X.copyFrom(Y); - } - - // get largest L1 norm - double new_norm = cuda_max_l1_norm(X.ptr, n, t, buffer2); - if (new_norm <= norm and k >= 2) - break; - - norm = std::max(norm,new_norm); - // std::cerr<<"A^p*X = "< (n,n) - - // (2) Replace parallel entries with random +1/-1 entries. - // Skipping this part for now because it involves a lot of branching logic - // and might be slow for the GPU. - - // (3) of Algorithm 2.4 - // Z = A^T * S - spMTM(Y, 1, A, X, 0, buffer, buffer_size); // (n,n) * (n,t) -> (n,t) - - // h[i] = max(j) abs(Z(i,j)) - cuda_rowwise_max_abs(Y.ptr, n, t, h.ptr); // (n,t) -> (n,1) - - // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. - // We don't do this, because finding a different reason to exit - // seems to provide greater accuracy. - - // Rank the dimensions by the size of their approximate norm in h, in decreasing order. - cuda_sort_indices_by_vector(h.ptr, n, indices); - - // Zero the X matrix. - cuda_fill_vector(X.ptr, n*t, 0); // (n,t) -> (n,t) + assert(B.rows() == n and B.cols() == n); + const int nnz = B.num_non_zeros; + const int nBlocks = pTop - 1; + const size_t blockSize = (size_t) n * t; - // Set X(i,indices[i]) = 1 for i in [0,t-1] - cuda_set_indices(X.ptr, n, t, indices); + // B^T + if (nnz > btCapacity) + { + cudaDeviceDelete(btColumns); + cudaDeviceDelete(btValues); + btColumns = cudaDeviceNew(nnz); + btValues = cudaDeviceNew(nnz); + btCapacity = nnz; } - - return norm; - } - - GPUnormest1(GPUnormest1&& g) - { - operator=(std::move(g)); - } - - GPUnormest1(const GPUnormest1&) = delete; - - GPUnormest1(cublasHandle_t cb, int p_, int n_, int t_=2, int itmax_=5) - :p(p_), n(n_), t(std::min(t_,n)), itmax(itmax_), X(cb,n,t), Y(cb,n,t), h(cb,n,1) - { - assert(p >= 0); - assert(t > 0); // negative means t = n - assert(itmax >= 1); - - // Interpret negative t as t == n - // if (t < 0) t = n; - - buffer2 = cudaDeviceNew(t); - - buffer3 = cudaDeviceNew(itmax); - - buffer4 = cudaDeviceNew(1); - - indices = cudaDeviceNew(n); - } - - ~GPUnormest1() - { - cudaDeviceDelete(buffer); - cudaDeviceDelete(buffer2); - cudaDeviceDelete(buffer3); - cudaDeviceDelete(buffer4); - cudaDeviceDelete(indices); - } -}; - -// Make the temporary variables part of a data structure that represents to normest1 problem. -// We don't want to allocate memory on the GPU every time we run this. -template -struct GPUnormest2 -{ - int p = 0; - int n = 0; - Real const one = Real(1); - Real const zero = Real(0); - - - DnMatrixDevice Ones; // (n,1) - - DnMatrixDevice Y; - - // Used for spMM - void* buffer = nullptr; - size_t buffer_size = 0; - - Real *valuesBuffer = nullptr; - - - GPUnormest2& operator=(const GPUnormest2&) = delete; - GPUnormest2& operator=(GPUnormest2&& g) - { - std::swap(p, g.p); - std::swap(n, g.n); - std::swap(Ones, g.Ones); - std::swap(Y, g.Y); - - // ensure that (*this) and (g) don't both own the buffers! - std::swap(buffer, g.buffer); - std::swap(buffer_size, g.buffer_size); - std::swap(valuesBuffer, g.valuesBuffer); - - return *this; - } - - Real operator()(const SpMatrixDevice& A) - { - // std::cerr<<"n = "<, CUSPARSE_ACTION_NUMERIC, + CUSPARSE_INDEX_BASE_ZERO, CUSPARSE_CSR2CSC_ALG1, &size)); + if (size > transposeBufferSize or transposeBuffer == nullptr) { - // std::cerr<<"iter "<(Y, 1, A, X, 0, buffer, buffer_size); - // X = Y - X.copyFrom(Y); - } - - // get the largest column L1 norm of A^p * X for this iteration -> buffer3[k-1] - cuda_max_l1_norm(X.ptr, n, t, buffer2, buffer3 + (k-1), scratch, offsets); - - // The rest only chooses X for the next iteration. - if (k == itmax) break; - - // S = sign(X) - cuda_sign_vector(X.ptr, n, t); // (n,t) -> (n,t) - - // (2) Replace parallel entries with random +1/-1 entries. - // Skipping this part for now because it involves a lot of branching logic - // and might be slow for the GPU. - - // (3) of Algorithm 2.4 - // Z = (A^T)^p * S - for(int i=0;i(Y, 1, A, X, 0, buffer, buffer_size); // (n,n) * (n,t) -> (n,t) - X.copyFrom(Y); - } - - // h[i] = max(j) abs(Z(i,j)) - cuda_rowwise_max_abs(X.ptr, n, t, h.ptr); // (n,t) -> (n,1) - - // (4) of Algorithm 2.4 - If we don't find a new best dimension, exit early. - // We don't do this, because finding a different reason to exit - // seems to provide greater accuracy. + cudaDeviceDelete(transposeBuffer); + transposeBufferSize = std::max(size, 1); + transposeBuffer = cudaMallocWrapped(transposeBufferSize); + } + checkCusparse(cusparseCsr2cscEx2(B.cusparseHandle, n, n, nnz, B.values, B.offsets, B.inner, + btValues, btOffsets, btColumns, DataType, CUSPARSE_ACTION_NUMERIC, + CUSPARSE_INDEX_BASE_ZERO, CUSPARSE_CSR2CSC_ALG1, transposeBuffer)); - // (5) of Algorithm 2.4 - Only consider rows that have not been used as unit columns yet. - // (If fewer than t remain, used rows are picked again, which only repeats work.) - cuda_mask_visited(h.ptr, visited, n); + // 1. Y_p = B^p X0 + for (int p = 1; p <= pTop; p++) + { + const Real* x = (p == 1) ? X0 : Y + (p - 2) * blockSize; + cuda_norm_chain_step(n, B.offsets, B.inner, B.values, x, Y + (p - 1) * blockSize, t, 0, (Real*)nullptr, nullptr, stream); + } + cuda_column_abs_sums(Y, n, pTop * t, sums1, stream); - // Rank the dimensions by the size of their approximate norm in h, in decreasing order. - cuda_sort_indices_by_vector(h.ptr, n, indices); + // 2. Z_p = (B^T)^p sign(Y_p), in block pTop - p. + Real* in = chain[0]; + Real* out = chain[1]; + cuda_norm_chain_step(n, btOffsets, btColumns, btValues, (Real*)nullptr, in, 0, t, Y + (pTop - 1) * blockSize, nullptr, stream); + for (int k = 1; k <= pTop; k++) + { + const bool append = (k < nBlocks); + const Real* sign = append ? Y + (pTop - k - 1) * blockSize : nullptr; + cuda_norm_chain_step(n, btOffsets, btColumns, btValues, in, out, std::min(k, nBlocks) * t, append ? t : 0, sign, nullptr, stream); + std::swap(in, out); + } - // Zero the X matrix. - cuda_fill_vector(X.ptr, n*t, 0); // (n,t) -> (n,t) + // 3. Unit vectors for the second iteration. + cuda_select_unit_columns(in, n, t, nBlocks, unitRows, stream); - // Set X(indices[j],j) = 1 for j in [0,t-1] - cuda_set_indices(X.ptr, n, t, indices); - cuda_mark_visited(visited, indices, t); + // 4. B^p X_p, in block pTop - p. + cuda_norm_chain_step(n, B.offsets, B.inner, B.values, (Real*)nullptr, out, 0, t, (Real*)nullptr, unitRows, stream); + std::swap(in, out); + for (int k = 1; k <= pTop; k++) + { + const bool append = (k < nBlocks); + cuda_norm_chain_step(n, B.offsets, B.inner, B.values, in, out, std::min(k, nBlocks) * t, append ? t : 0, + (Real*)nullptr, append ? unitRows + k * t : nullptr, stream); + std::swap(in, out); } + cuda_column_abs_sums(in, n, nBlocks * t, sums2, stream); - return cuda_max_abs(buffer3, itmax); - } + cuda_combine_norm_estimates(sums1, sums2, t, pTop, norms, stream); - GPUnormest3(GPUnormest3&& g) - { - operator=(std::move(g)); + std::vector result(pTop + 1, 0); + auto status = cudaMemcpyAsync(result.data(), norms, (pTop + 1) * sizeof(Real), cudaMemcpyDeviceToHost, stream); + if (status == cudaSuccess) + status = cudaStreamSynchronize(stream); + if (status != cudaSuccess) + throw std::runtime_error(std::string("BlockNormEstimator: ") + cudaGetErrorString(status)); + return result; } - GPUnormest3(const GPUnormest3&) = delete; - - GPUnormest3(cublasHandle_t cb, int p_, int n_, int t_=2, int itmax_=5) - :p(p_), n(n_), t(std::min(t_,n)), itmax(itmax_), X(cb,n,t), Y(cb,n,t), h(cb,n,1) + static void checkCusparse(cusparseStatus_t status) { - assert(p >= 0); - assert(t > 0); // negative means t = n - assert(itmax >= 1); - - // Interpret negative t as t == n - // if (t < 0) t = n; - - buffer2 = cudaDeviceNew(t); - - buffer3 = cudaDeviceNew(itmax); - - buffer4 = cudaDeviceNew(1); - - indices = cudaDeviceNew(n); - - offsets = cudaDeviceNew(t+1); - - visited = cudaDeviceNew(n); - - std::vector h_offsets(t+1); - for(int i=0;i //using MapType = DnMatrix; @@ -1146,8 +970,6 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int operationCount, int cumulativeScalingIndex); - void rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex); - // Action instances have no dense transition matrices (see createInstance), so these are not available. // (The base versions of the first three call the dense, non-virtual upPartials/upPrePartials.) int updatePrePartials(const int*, int, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } @@ -1170,9 +992,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl protected: - std::vector> hInstantaneousMatrices; - std::vector> hBs; - SpMatrix hIdentity; + // Per eigen index: B = Q - mu*I. + std::vector> hBs; std::vector hMuBs; std::vector hB1Norms; const int mMax = 55; @@ -1222,38 +1043,32 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // std::vector dInstantaneousMatrices; std::vector> dPartialsWrapper; // Per category: the action of each child's edge on its partials (exp(tQ) * partials). + // These hold the Taylor sum F while it is accumulated. std::vector> dDestPLeft; std::vector> dDestPRight; - std::vector> dFLeft; - std::vector> dFRight; - std::vector> dIntegrationTmpLeft; - std::vector> dIntegrationTmpRight; - std::vector> dAs; - std::vector> L1normForPower; - std::vector integrationLeftBufferSize; - std::vector dIntegrationLeftBuffer; - std::vector integrationRightBufferSize; - std::vector dIntegrationRightBuffer; + // Per category: two buffers that alternate as the input and output of each Taylor term. + std::vector> dTerms[2]; + // Per eigen index: B = Q - mu*I on the device. The edge multiplier goes into the SpMM alpha. + std::vector> dBs; + void* dSpMMBuffer = nullptr; + size_t spMMBufferBytes = 0; + BlockNormEstimator normEstimator; + + // All work is issued on the GPUInterface's stream 0, so it is ordered with the base class's + // copies and kernels (two different blocking streams would not be). See setCudaDevice( ). + cudaStream_t stream = nullptr; std::vector dBsCsrOffsetsCache; std::vector dBsCsrColumnsCache; std::vector dBsCsrValuesCache; - std::vector dACscValuesCache; std::vector currentCacheNNZs; - cublasHandle_t cublasHandle; - cusparseHandle_t cusparseHandle; + cublasHandle_t cublasHandle = nullptr; + cusparseHandle_t cusparseHandle = nullptr; std::vector hEigenMaps; std::vector hEdgeMultipliers; - std::vector> msCache; - std::vector etaCache; - std::vector c1Cache; - std::vector c2Cache; - std::vector alphaCache; - std::vector integrationMultipliers; - using BeagleGPUImpl::kTipCount; using BeagleGPUImpl::kPartialsBufferCount; @@ -1320,25 +1135,25 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int count); double getPMax() const; - // Make CUDA runtime, cuBLAS, cuSPARSE and Thrust calls target this instance's device. + // Make CUDA runtime, cuBLAS, cuSPARSE and Thrust calls target this instance's device, + // and cuBLAS/cuSPARSE use the GPUInterface's stream. void setCudaDevice(); int getPartialIndex(int nodeIndex, int categoryIndex); DnMatrixDevice& getPartialsWrapper(int nodeIndex, int categoryIndex); + // scalingFactors == nullptr means no rescaling. void calcPartialsPartials(int destPIndex, int partials1Index, int edgeIndex1, int partials2Index, - int edgeIndex2); - - int simpleAction2(DnMatrixDevice& destP, const DnMatrixDevice& inPartials, int edgeIndex, int category, int matrixIndex, bool left, bool transpose); - - int simpleAction3(int partialsIndex1, int edgeIndex1, - int partialsIndex2, int edgeIndex2); + int edgeIndex2, + Real* scalingFactors, + Real* cumulativeScalingBuffer); - int cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose); + // F = exp(t * edgeMultiplier * Q) * inPartials + int simpleAction2(DnMatrixDevice& F, const DnMatrixDevice& inPartials, int edgeIndex, int category); std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, int eigenIndex) const; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index b9fd610d..e0b31bf8 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -291,45 +291,27 @@ int BeagleGPUActionImpl::createInstance(int tipCount, setCudaDevice(); - hIdentity = SpMatrix(kPaddedStateCount, kPaddedStateCount); - hIdentity.setIdentity(); - hInstantaneousMatrices.resize(kEigenDecompCount); hBs.resize(kEigenDecompCount); - for (int i = 0; i < kEigenDecompCount; i++) { - hInstantaneousMatrices[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); - hBs[i] = SpMatrix(kPaddedStateCount, kPaddedStateCount); - } hMuBs.resize(kEigenDecompCount); hB1Norms.resize(kEigenDecompCount); hds.resize(kEigenDecompCount); - CUBLAS_CHECK(cublasCreate(&cublasHandle)); //TODO: destroyer: CUBLAS_CHECK(cublasDestroy(cublasHandle)); + CUBLAS_CHECK(cublasCreate(&cublasHandle)); CHECK_CUSPARSE(cusparseCreate(&cusparseHandle)); + setCudaDevice(); dBsCsrOffsetsCache.resize(kEigenDecompCount); dBsCsrColumnsCache.resize(kEigenDecompCount); dBsCsrValuesCache.resize(kEigenDecompCount); - dACscValuesCache.resize(kEigenDecompCount * kCategoryCount * 2); - dAs = std::vector>(kEigenDecompCount * kCategoryCount * 2); - for(int p=0; p<=getPMax()+1; p++) - { - int t = 5; - int itmax = 2; - L1normForPower.push_back({cublasHandle, p, kPaddedStateCount, t, itmax}); - -// This is very fast, but computes |A| instead of |A^p| -// L1normForPower.push_back(GPUnormest2(cublasHandle, cusparseHandle, p, kPaddedStateCount)); - } + dBs = std::vector>(kEigenDecompCount); + // getStatistics2( ) uses d_p for p = 2 .. pMax+1. + normEstimator.allocate(kPaddedStateCount, 5, getPMax() + 1); currentCacheNNZs = std::vector(kEigenDecompCount, kPaddedStateCount); for (int i = 0; i < kEigenDecompCount; i++) { dBsCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); dBsCsrColumnsCache[i] = cudaDeviceNew(currentCacheNNZs[i]); dBsCsrValuesCache[i] = cudaDeviceNew(currentCacheNNZs[i]); - for (int j = 0; j < kCategoryCount; j++) { - dACscValuesCache[i * kCategoryCount * 2 + 2 * j] = cudaDeviceNew(currentCacheNNZs[i]); - dACscValuesCache[i * kCategoryCount * 2 + 2 * j + 1] = cudaDeviceNew(currentCacheNNZs[i]); - } } @@ -342,31 +324,16 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } } - Real* dDestPLeftPtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - Real* dDestPRightPtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - - Real* dLeftCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - Real* dRightCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - - Real* dIntegrationTmpLeftCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - Real* dIntegrationTmpRightCachePtr = cudaDeviceNew(kPaddedStateCount * kPaddedPatternCount * kCategoryCount); - - integrationLeftBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); - dIntegrationLeftBuffer = std::vector(kCategoryCount, nullptr); - integrationRightBufferSize = std::vector(kCategoryCount, kPaddedStateCount * kPaddedPatternCount); - dIntegrationRightBuffer = std::vector(kCategoryCount, nullptr); + const size_t partialsSize = (size_t) kPaddedStateCount * kPaddedPatternCount * kCategoryCount; + Real* dDestPLeftPtr = cudaDeviceNew(partialsSize); + Real* dDestPRightPtr = cudaDeviceNew(partialsSize); + // The two children are processed one after the other, so they share the term buffers. + Real* dTermPtrs[2] = {cudaDeviceNew(partialsSize), cudaDeviceNew(partialsSize)}; dDestPLeft.clear(); - dDestPRight.clear(); - - dFLeft.clear(); - - dFRight.clear(); - - dIntegrationTmpLeft.clear(); - - dIntegrationTmpRight.clear(); + dTerms[0].clear(); + dTerms[1].clear(); for (int category = 0; category < kCategoryCount; category++) { @@ -375,29 +342,13 @@ int BeagleGPUActionImpl::createInstance(int tipCount, dDestPLeft.emplace_back(cublasHandle, dDestPLeftPtr + offset, kPaddedStateCount, kPaddedPatternCount); dDestPRight.emplace_back(cublasHandle, dDestPRightPtr + offset, kPaddedStateCount, kPaddedPatternCount); - dFLeft.emplace_back(cublasHandle, dLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); - dFRight.emplace_back(cublasHandle, dRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); - - dIntegrationTmpLeft.emplace_back(cublasHandle, dIntegrationTmpLeftCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); - dIntegrationTmpRight.emplace_back(cublasHandle, dIntegrationTmpRightCachePtr + offset, kPaddedStateCount, kPaddedPatternCount); - } - - for (int category = 0; category < kCategoryCount; category++) { - - CHECK_CUDA(cudaMalloc(&dIntegrationLeftBuffer[category], integrationLeftBufferSize[category])) - CHECK_CUDA(cudaMalloc(&dIntegrationRightBuffer[category], integrationRightBufferSize[category])) + for (int k = 0; k < 2; k++) + dTerms[k].emplace_back(cublasHandle, dTermPtrs[k] + offset, kPaddedStateCount, kPaddedPatternCount); } hEigenMaps.resize(kMatrixCount); hEdgeMultipliers.resize(kMatrixCount * kCategoryCount); - msCache = std::vector>(2 * kCategoryCount, tuple{0, 0}); - etaCache = std::vector(2 * kCategoryCount, 0); - c1Cache.resize(2 * kCategoryCount); - c2Cache.resize(2 * kCategoryCount); - alphaCache.resize(2 * kCategoryCount); - integrationMultipliers = std::vector(kCategoryCount * 2, 1); - return BEAGLE_SUCCESS; } @@ -405,28 +356,20 @@ BEAGLE_GPU_TEMPLATE BeagleGPUActionImpl::~BeagleGPUActionImpl() { cudaSetDevice(gpu->GetCudaDeviceOrdinal()); + cudaStreamSynchronize(stream); - for (int i = 0; i < kCategoryCount; i++) { - cudaFree(dIntegrationLeftBuffer[i]); - cudaFree(dIntegrationRightBuffer[i]); - } + cudaFree(dSpMMBuffer); // The memory for each matrix group was allocation in a single block. cudaFree(dDestPLeft[0].ptr); cudaFree(dDestPRight[0].ptr); - cudaFree(dFLeft[0].ptr); - cudaFree(dFRight[0].ptr); - cudaFree(dIntegrationTmpLeft[0].ptr); - cudaFree(dIntegrationTmpRight[0].ptr); + cudaFree(dTerms[0][0].ptr); + cudaFree(dTerms[1][0].ptr); for (int i = 0; i < kEigenDecompCount; i++) { cudaFree(dBsCsrOffsetsCache[i]); cudaFree(dBsCsrColumnsCache[i]); cudaFree(dBsCsrValuesCache[i]); - for (int j = 0; j < kCategoryCount * 2; j++) { - cudaFree(dACscValuesCache[i * kCategoryCount * 2 + j]); -// cusparseDestroySpMat(dAs[i * kCategoryCount * 2 + i]); - } } cublasDestroy(cublasHandle); cusparseDestroy(cusparseHandle); @@ -486,7 +429,7 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const BeagleGPUImpl::setTipPartials(tipIndex, inPartials); for (int category = 0; category < kCategoryCount; category++) { - dPartialsWrapper[getPartialIndex(tipIndex, category)].ptr = (Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category; + dPartialsWrapper[getPartialIndex(tipIndex, category)].setPointer((Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category); } @@ -505,7 +448,7 @@ int BeagleGPUActionImpl::setPartials(int tipIndex, const Rea BeagleGPUImpl::setPartials(tipIndex, inPartials); for (int category = 0; category < kCategoryCount; category++) { - dPartialsWrapper[getPartialIndex(tipIndex, category)].ptr = (Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category; + dPartialsWrapper[getPartialIndex(tipIndex, category)].setPointer((Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category); } @@ -542,21 +485,6 @@ int BeagleGPUActionImpl::updatePartials(const int* operation return returnCode; } -BEAGLE_GPU_TEMPLATE -void BeagleGPUActionImpl::rescalePartials(Real* partials, Real* scalingFactors, Real* cumulativeScalingBuffer, int streamIndex) -{ -// std::cerr<<"initial partials (kernel) = "<RescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, -// kPaddedPatternCount, kCategoryCount, 0, streamIndex, -1); - - bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; - // rescalePartialsDevice(partials, scalingFactors, cumulativeScalingBuffer, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); - rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, partials, scalingFactors, cumulativeScalingBuffer, streamIndex); -// std::cerr<<"rescaled partials (kernel) = "<::setCudaDevice() { @@ -565,6 +493,15 @@ void BeagleGPUActionImpl::setCudaDevice() auto status = cudaSetDevice(gpu->GetCudaDeviceOrdinal()); if (status != cudaSuccess) throw std::runtime_error(std::string("cudaSetDevice: ") + cudaGetErrorString(status)); + + auto beagleStream = (cudaStream_t) gpu->GetCudaStream(); + if (cublasHandle and beagleStream != stream) + { + stream = beagleStream; + CUBLAS_CHECK(cublasSetStream(cublasHandle, stream)); + if (cusparseSetStream(cusparseHandle, stream) != CUSPARSE_STATUS_SUCCESS) + throw std::runtime_error("cusparseSetStream failed"); + } } BEAGLE_GPU_TEMPLATE @@ -592,9 +529,6 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, if (cumulativeScalingIndex != BEAGLE_OP_NONE) cumulativeScalingBuffer = (Real*)dScalingFactors[cumulativeScalingIndex]; - int streamIndex = -1; - int waitIndex = -1; - for (int op = 0; op < operationCount; op++) { const int numOps = BEAGLE_OP_COUNT; @@ -626,15 +560,8 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, } calcPartialsPartials(destinationPartialIndex, firstChildPartialIndex, firstChildSubstitutionMatrixIndex, - secondChildPartialIndex, secondChildSubstitutionMatrixIndex); - - - if (rescale == 1) - { - Real* partials3 = (Real*)dPartials[destinationPartialIndex]; - - rescalePartials(partials3, scalingFactors, cumulativeScalingBuffer, streamIndex); - } + secondChildPartialIndex, secondChildSubstitutionMatrixIndex, + (rescale == 1) ? scalingFactors : nullptr, cumulativeScalingBuffer); if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { int parScalingIndex = destinationPartialIndex - kTipCount; @@ -783,49 +710,22 @@ void BeagleGPUActionImpl::calcPartialsPartials(int destPInde int partials1Index, int edgeIndex1, int partials2Index, - int edgeIndex2) { - - cacheAMatrices(edgeIndex1, edgeIndex2, false); + int edgeIndex2, + Real* scalingFactors, + Real* cumulativeScalingBuffer) { for (int category = 0; category < kCategoryCount; category++) { + simpleAction2(dDestPLeft[category], getPartialsWrapper(partials1Index, category), edgeIndex1, category); - const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; - const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; - - simpleAction2(dDestPLeft[category], getPartialsWrapper(partials1Index, category), - edgeIndex1, category, matrixIndex1, true, false); - - simpleAction2(dDestPRight[category], getPartialsWrapper(partials2Index, category), - edgeIndex2, category, matrixIndex2, false, false); + simpleAction2(dDestPRight[category], getPartialsWrapper(partials2Index, category), edgeIndex2, category); } - // simpleAction3(partials1Index, edgeIndex1, partials2Index, edgeIndex2); - - for (int category = 0; category < kCategoryCount; category++) - { - const int destPartialIndex = getPartialIndex(destPIndex, category); - - // element-wise multiply - if constexpr (std::is_same::value) { - CUBLAS_CHECK(cublasSdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dDestPLeft[category].ptr, kPaddedStateCount * kPaddedPatternCount, - dDestPRight[category].ptr, 1, dPartialsWrapper[destPartialIndex].ptr, kPaddedStateCount * kPaddedPatternCount)); - - } else { - CUBLAS_CHECK(cublasDdgmm(cublasHandle, CUBLAS_SIDE_LEFT, kPaddedStateCount * kPaddedPatternCount, 1, dDestPLeft[category].ptr, kPaddedStateCount * kPaddedPatternCount, - dDestPRight[category].ptr, 1, dPartialsWrapper[destPartialIndex].ptr, kPaddedStateCount * kPaddedPatternCount)); - - } -//#ifdef BEAGLE_DEBUG_FLOW -// std::cerr<<"Checking p_parent = p_1 * p_2, parent index = "<::createInstance @@ -953,61 +843,29 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i std::abort(); } - // 2. Compute log(pr[pattern]) - cuda_log_vector(siteProbs, kPatternCount); - - // 3. Add scaling factors back in. - if (scale) - { - cublasStatus_t status; - if constexpr (std::is_same::value) - { - Real alpha = 1; - status = cublasSaxpy(cublasHandle, kPatternCount, - &alpha, - dCumulativeScalingFactor, 1, - siteProbs, 1); - } - else - { - Real alpha = 1; - status = cublasDaxpy(cublasHandle, kPatternCount, - &alpha, - dCumulativeScalingFactor, 1, - siteProbs, 1); - } - if (status != CUBLAS_STATUS_SUCCESS) - { - std::cerr<<"cublas error "<axpy( )."; - exit(1); - } - } - + Real* siteLogLikelihoods = (Real*)dIntegrationTmp; + Real* rootPartials = (Real*)dPartials[rootNodeIndex]; + Real* categoryWeights = (Real*)dWeights[categoryWeightsIndex]; + Real* stateFrequencies = (Real*)dFrequencies[stateFrequenciesIndex]; // these really should be category-dependent -/* - std::cerr<<"site probs (d1) = "<MemcpyDeviceToHost(hLogLikelihoodsCache, dSumLogLikelihood, sizeof(Real) * kSumSitesBlockCount); - - *outSumLogLikelihood = 0.0; - for (int i = 0; i < kSumSitesBlockCount; i++) { - if (hLogLikelihoodsCache[i] != hLogLikelihoodsCache[i]) - returnCode = BEAGLE_ERROR_FLOATING_POINT; + CHECK_CUDA(cudaMemcpyAsync(hLogLikelihoodsCache, (Real*)dSumLogLikelihood, sizeof(Real), cudaMemcpyDeviceToHost, stream)); + CHECK_CUDA(cudaStreamSynchronize(stream)); - *outSumLogLikelihood += hLogLikelihoodsCache[i]; - } + *outSumLogLikelihood = hLogLikelihoodsCache[0]; + if (*outSumLogLikelihood != *outSumLogLikelihood) + returnCode = BEAGLE_ERROR_FLOATING_POINT; } #ifdef BEAGLE_DEBUG_VALUES @@ -1086,144 +944,94 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::cacheAMatrices(int edgeIndex1, int edgeIndex2, bool transpose) -{ - for (int category = 0; category < kCategoryCount; category++) { - const int matrixIndex1 = hEigenMaps[edgeIndex1] * kCategoryCount * 2 + category; - const int edgeMultiplierIndex1 = edgeIndex1 * kCategoryCount + category; - const Real edgeMultiplier1 = hEdgeMultipliers[edgeMultiplierIndex1]; - - const int matrixIndex2 = hEigenMaps[edgeIndex2] * kCategoryCount * 2 + kCategoryCount + category; - const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; - const Real edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; - - MemcpyDeviceToDevice(dACscValuesCache[matrixIndex1], dBsCsrValuesCache[hEigenMaps[edgeIndex1]], currentCacheNNZs[hEigenMaps[edgeIndex1]]); - MemcpyDeviceToDevice(dACscValuesCache[matrixIndex2], dBsCsrValuesCache[hEigenMaps[edgeIndex2]], currentCacheNNZs[hEigenMaps[edgeIndex2]]); - auto format = transpose ? sparseFormat::csc : sparseFormat::csr; - - dAs[matrixIndex1] = SpMatrixDevice(cublasHandle, cusparseHandle, - kPaddedStateCount, kPaddedStateCount, - currentCacheNNZs[hEigenMaps[edgeIndex1]], - dACscValuesCache[matrixIndex1], - dBsCsrColumnsCache[hEigenMaps[edgeIndex1]], - dBsCsrOffsetsCache[hEigenMaps[edgeIndex1]], - format); - - dAs[matrixIndex2] = SpMatrixDevice(cublasHandle, cusparseHandle, - kPaddedStateCount, kPaddedStateCount, - currentCacheNNZs[hEigenMaps[edgeIndex2]], - dACscValuesCache[matrixIndex2], - dBsCsrColumnsCache[hEigenMaps[edgeIndex2]], - dBsCsrOffsetsCache[hEigenMaps[edgeIndex2]], - format); - - dAs[matrixIndex1] *= edgeMultiplier1; //Check if this is asynchronous with the following - dAs[matrixIndex2] *= edgeMultiplier2; - -#ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"category = "<::simpleAction2(DnMatrixDevice& destP, const DnMatrixDevice& inPartials, int edgeIndex, int category, int matrixIndex, bool left, bool transpose) { +int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& F, const DnMatrixDevice& inPartials, int edgeIndex, int category) { const double tol = pow(2.0, -53.0); const double t = 1.0; const int nCol = kPaddedPatternCount; const int edgeMultiplierIndex = edgeIndex * kCategoryCount + category; + const int eigenIndex = hEigenMaps[edgeIndex]; const double edgeMultiplier = hEdgeMultipliers[edgeMultiplierIndex]; - auto [m, s] = getStatistics2(t, nCol, edgeMultiplier, hEigenMaps[edgeIndex]); + auto [m, s] = getStatistics2(t, nCol, edgeMultiplier, eigenIndex); #ifdef BEAGLE_DEBUG_FLOW std::cerr << "\n\nsimpleAction2: m = " << m << " s = " << s << " t = " << t << " nCol = " << nCol << " edgeMultiplier = " << edgeMultiplier << std::endl; - std::cerr << " edgeIndex = " << edgeIndex << " category = " << category << " matrixIndex = "<(integrationTmp, t / ((Real) s * j), A, destP, 0, integrationBuffer, integrationBufferSize); - - destP.copyFrom( integrationTmp ); + auto& y = dTerms[j % 2][category]; + const Real alpha = edgeMultiplier * t / ((Real) s * j); - F += destP; + // y = alpha * B * x; F += y, folding in F *= eta after the last term. + const Real scale = (j < m) ? 1 : eta; + if (fusedTerms) + cuda_csr_taylor_term(kPaddedStateCount, kPaddedPatternCount, B.offsets, B.inner, B.values, alpha, + x->ptr, y.ptr, F.ptr, scale, stream); + else + { + spMMPrepared(y, alpha, B, *x, dSpMMBuffer); + if (j < m) + F += y; + else + F.axpby(eta, eta, y); + } + scaled = (j == m); #ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"After destP = \n" << destP << "\n"; - std::cerr<<"Action2 Step 2, integrationTmp =\n"<::simpleAction2(DnMatrixDevice& } -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::simpleAction3(int partialsIndex1, int edgeIndex1, - int partialsIndex2, int edgeIndex2) { - const double tol = pow(2.0, -53.0); - const double t = 1.0; - const int nCol = kPaddedPatternCount; - - - - int m_max = 0, s_max = 0; - for (int category = 0; category < kCategoryCount; category++) { - const int edgeMultiplierIndex1 = edgeIndex1 * kCategoryCount + category; - const double edgeMultiplier1 = hEdgeMultipliers[edgeMultiplierIndex1]; - msCache[category] = getStatistics2(t, nCol, edgeMultiplier1, hEigenMaps[edgeIndex1]); - auto [m1, s1] = msCache[category]; - if (m_max < m1) m_max = m1; - if (s_max < s1) s_max = s1; - - const int edgeMultiplierIndex2 = edgeIndex2 * kCategoryCount + category; - const double edgeMultiplier2 = hEdgeMultipliers[edgeMultiplierIndex2]; - msCache[kCategoryCount + category] = getStatistics2(t, nCol, edgeMultiplier2, hEigenMaps[edgeIndex2]); - auto [m2, s2] = msCache[kCategoryCount + category]; - if (m_max < m2) m_max = m2; - if (s_max < s2) s_max = s2; - } - -#ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"s_max = "<::upPrePartials(bool byPartition, const int *operations, @@ -1437,40 +1058,27 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, { setCudaDevice(); - std::vector> tripletList; - for (int i = 0; i < numNonZeros; i++) { - tripletList.push_back(Triplet(rowIndices[i], colIndices[i], values[i])); - } - hInstantaneousMatrices[matrixIndex].setFromTriplets(tripletList.begin(), tripletList.end()); - -#ifdef BEAGLE_DEBUG_FLOW - - std::cerr<<"CPU instantaneous matrix "<::setSparseMatrix(int matrixIndex, cudaDeviceDelete(dBsCsrValuesCache[matrixIndex]); dBsCsrColumnsCache[matrixIndex] = cudaDeviceNew(currentNNZ); dBsCsrValuesCache[matrixIndex] = cudaDeviceNew(currentNNZ); - for (int j = 0; j < kCategoryCount * 2; j++) { - cudaDeviceDelete(dACscValuesCache[matrixIndex * kCategoryCount * 2 + j]); - dACscValuesCache[matrixIndex * kCategoryCount * 2 + j] = cudaDeviceNew(currentNNZ); - } } - MemcpyHostToDevice(dBsCsrOffsetsCache[matrixIndex], hBs[matrixIndex].outerIndexPtr(), kPaddedStateCount + 1); - MemcpyHostToDevice(dBsCsrColumnsCache[matrixIndex], hBs[matrixIndex].innerIndexPtr(), currentNNZ); - MemcpyHostToDevice(dBsCsrValuesCache[matrixIndex], hBs[matrixIndex].valuePtr(), currentNNZ); + MemcpyHostToDevice(dBsCsrOffsetsCache[matrixIndex], hBs[matrixIndex].offsets.data(), kPaddedStateCount + 1); + MemcpyHostToDevice(dBsCsrColumnsCache[matrixIndex], hBs[matrixIndex].columns.data(), currentNNZ); + MemcpyHostToDevice(dBsCsrValuesCache[matrixIndex], hBs[matrixIndex].values.data(), currentNNZ); #ifdef BEAGLE_DEBUG_FLOW @@ -1504,44 +1108,54 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, #endif - hds[matrixIndex].clear(); + dBs[matrixIndex] = SpMatrixDevice(cublasHandle, cusparseHandle, + kPaddedStateCount, kPaddedStateCount, + currentNNZ, + dBsCsrValuesCache[matrixIndex], + dBsCsrColumnsCache[matrixIndex], + dBsCsrOffsetsCache[matrixIndex], + sparseFormat::csr); + auto& dB = dBs[matrixIndex]; - auto dB = SpMatrixDevice(cublasHandle, cusparseHandle, - kPaddedStateCount, kPaddedStateCount, - currentNNZ, - dBsCsrValuesCache[matrixIndex], - dBsCsrColumnsCache[matrixIndex], - dBsCsrOffsetsCache[matrixIndex], - sparseFormat::csr); - - int pMax = getPMax(); - auto tmp = hBs[matrixIndex]; -#ifdef BEAGLE_DEBUG_NORMEST - DnMatrix denseB = hBs[matrixIndex]; - DnMatrix denseBp = DnMatrix::Identity(kPaddedStateCount, kPaddedStateCount); -#endif - for(int p=0;p <= pMax+1; p++) + // Size the work buffer for the Taylor-term SpMMs once per matrix, not on every call. + size_t newSpMMBufferSize = spMMBufferSize(dTerms[0][0], dB, dTerms[1][0]); + if (newSpMMBufferSize > spMMBufferBytes or dSpMMBuffer == nullptr) { - // GPU Norm - Real approx_norm = L1normForPower[p]( dB ); + CHECK_CUDA(cudaStreamSynchronize(stream)); + cudaDeviceDelete(dSpMMBuffer); + spMMBufferBytes = std::max(newSpMMBufferSize, 1); + dSpMMBuffer = cudaMallocWrapped(spMMBufferBytes); + } + + // equation 3.7 in Al-Mohy and Higham: d_p = ||B^p||_1^(1/p). d_1 is known exactly; d_0 is not used. + const int pTop = getPMax() + 1; + auto norms = normEstimator.estimate(dB, stream); + hds[matrixIndex].assign(pTop + 1, 1.0); + hds[matrixIndex][1] = hB1Norms[matrixIndex]; + for (int p = 2; p <= pTop; p++) + hds[matrixIndex][p] = pow(norms[p], 1.0 / Real(p)); #ifdef BEAGLE_DEBUG_NORMEST - // Compare with the CPU estimator and, for small S, the exact ||B^p||_1. - Real approx_norm_CPU = normest1( hBs[matrixIndex], p, 5); + // Compare with the CPU estimator and, for small S, the exact ||B^p||_1. + auto& hB = hBs[matrixIndex]; + SpMatrix sparseB = Eigen::Map>(kPaddedStateCount, kPaddedStateCount, hB.nonZeros(), + hB.offsets.data(), hB.columns.data(), hB.values.data()); + DnMatrix denseB = sparseB; + DnMatrix denseBp = denseB; + for (int p = 2; p <= pTop; p++) + { + Real approx_norm_CPU = normest1( sparseB, p, 5); double exact_norm = -1; if (kPaddedStateCount <= 4096) { - if (p > 0) denseBp = denseB * denseBp; + denseBp = denseB * denseBp; exact_norm = normP1(denseBp); } std::fprintf(stderr, "normest matrix=%d S=%d p=%d gpu=%.6e cpu=%.6e exact=%.6e gpu/exact=%.4f cpu/exact=%.4f\n", - matrixIndex, kPaddedStateCount, p, (double)approx_norm, (double)approx_norm_CPU, exact_norm, - approx_norm / exact_norm, approx_norm_CPU / exact_norm); -#endif - - // equation 3.7 in Al-Mohy and Higham - hds[matrixIndex].push_back(pow(approx_norm, 1.0 / Real(p) ) ); + matrixIndex, kPaddedStateCount, p, (double)norms[p], (double)approx_norm_CPU, exact_norm, + norms[p] / exact_norm, approx_norm_CPU / exact_norm); } +#endif //#ifdef BEAGLE_DEBUG_FLOW // std::cerr<<"Setting host matrix: "< - -#include - #include -#include #include #include -#include -#include -#include #include #include -#include #include -#include using thrust::device_ptr; using thrust::device_pointer_cast; -template -void cuda_log_vector(device_ptr v, int length) -{ - thrust::transform(v, v + length, v, [] __device__ (T x) {return log(x);}); -} - -void cuda_log_vector(float* v, int length) -{ - cuda_log_vector(device_pointer_cast(v), length); -} - -void cuda_log_vector(double* v, int length) -{ - cuda_log_vector(device_pointer_cast(v), length); -} - - -template -void cuda_sign_vector(device_ptr v, int n, int t) -{ - // In-place update is accomplished by making the output iterator the same the starting input iterator. - thrust::transform(v, v + n*t, v, [n] __device__ (T x) -> T {return (x<0)?-1.0/n:1.0/n;}); -} - -void cuda_sign_vector(double* v, int n, int t) -{ - cuda_sign_vector(device_pointer_cast(v), n, t); -} - -void cuda_sign_vector(float* v, int n, int t) -{ - cuda_sign_vector(device_pointer_cast(v), n, t); -} - template // This seems to be slow because it inserts cudaStreamSynchronize() T cuda_max_abs(device_ptr values, int length) @@ -77,642 +33,439 @@ double cuda_max_abs(double* values, int length) return cuda_max_abs(device_pointer_cast(values), length); } -// QUESTION: Why do cub::DeviceReduce and thrust::reduce need to perform allocation? -// That seems like a big problem. template -void cuda_max(device_ptr values, int length, device_ptr out, cuda_scratch_space& scratch) +void initialize_norm_x_matrix(device_ptr out_ptr, int n, int m) { - auto values_ptr = thrust::raw_pointer_cast(values); - auto out_ptr = thrust::raw_pointer_cast(out); - - // 1. Get size of temporary allocation, if any. - void* d_temp_storage = nullptr; - size_t temp_storage_bytes = 0; - cub::DeviceReduce::Max(nullptr, temp_storage_bytes, values_ptr, out_ptr, length); + auto indices = thrust::counting_iterator(0); + auto initialize = [n,m] __host__ __device__ (int i) + { + thrust::default_random_engine rng; + thrust::uniform_real_distribution dist(0, 1); - // 2. Do temporary allocation, if needed. - scratch.fit(temp_storage_bytes); + if (i 0.5) + return 1.0/n; + else + return -1.0/n; + }; - // 3. Do the reduction. - cub::DeviceReduce::Max(scratch.device_ptr, temp_storage_bytes, values_ptr, out_ptr, length); + thrust::transform(indices, indices + n*m, // in + out_ptr, // out + initialize); } -void cuda_max(float* values, int length, float* out, cuda_scratch_space& scratch) +void initialize_norm_x_matrix(float* data, int n, int m) { - cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out), scratch); + initialize_norm_x_matrix(device_pointer_cast(data), n, m); } -void cuda_max(double* values, int length, double* out, cuda_scratch_space& scratch) +void initialize_norm_x_matrix(double* data, int n, int m) { - cuda_max(thrust::device_pointer_cast(values), length, thrust::device_pointer_cast(out), scratch); + initialize_norm_x_matrix(device_pointer_cast(data), n, m); } -template -T cuda_max_l1_norm(device_ptr values, int n, int t, device_ptr buffer) -{ - using namespace thrust::placeholders; - // 1. First sum the absolute values in each column and place the results into buffer - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); +// --------------------------------------------------------------------------------------------------- +// Kernels for the per-node product/rescale and the root likelihood. They replace the Thrust +// reduce_by_key chains, which allocate temporary memory and block the host on every call. +// +// Partials are stored as [category][pattern][state] (state fastest), with the padded counts. +// One block handles one pattern. - auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values), - [] __host__ __device__ (T x) {return std::abs(x);} ); - - thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) - in_values_start, // values to reduce (with abs applied) - thrust::make_discard_iterator(), // key values out - buffer, // reduced values out - thrust::equal_to() // compare-keys operation - ); // summation is the default operation. - - // 2. Second maximize over the column sums and return the highest. - return thrust::reduce(buffer, buffer+t, T(0), thrust::maximum()); -} - -float cuda_max_l1_norm(float* values, int n, int t, float* buffer) +// With few patterns there are few blocks, so give each block more threads. +inline int patternBlockSize(int nPatterns) { - return cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer)); + return (nPatterns >= 256) ? 128 : 1024; } -double cuda_max_l1_norm(double* values, int n, int t, double* buffer) +// Reduce v over the block with op; identity must be neutral for op. Every thread gets the result. +template +__device__ T block_reduce(T v, T identity, Op op) { - return cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer)); -} - + __shared__ T warpResults[32]; -void cuda_scratch_space::discard() -{ - if (device_ptr) - { - cudaDeviceDelete(device_ptr); - device_ptr = 0; - size = 0; - } - else - { - assert(device_ptr == nullptr); - assert(size == 0); - } -} + for (int offset = 16; offset > 0; offset /= 2) + v = op(v, __shfl_down_sync(0xffffffff, v, offset)); -void cuda_scratch_space::fit(size_t n) -{ - if (n > size) - { - std::cerr<<"cuda_scratch_space::fit( ): reallocating from "< 0; offset /= 2) + v = op(v, __shfl_down_sync(0xffffffff, v, offset)); + if (lane == 0) warpResults[0] = v; } -} + __syncthreads(); -cuda_scratch_space::~cuda_scratch_space() -{ - discard(); + return warpResults[0]; } +struct MaxOp { template __device__ T operator()(T a, T b) const { return max(a, b); } }; +struct SumOp { template __device__ T operator()(T a, T b) const { return a + b; } }; -struct SumAbs -{ - template - __device__ __forceinline__ T operator()(const T& a, const T& b) const - { - return std::abs(a) + std::abs(b); - } -}; - +// parent = left * right (elementwise). If scalingFactors is non-null, also divide each pattern by its +// largest entry m (1 if all are 0), store m (or log m) in scalingFactors, and add log m to cumulative. template -void cuda_max_l1_norm(device_ptr values, - int n, - int t, - device_ptr buffer, - device_ptr out, - cuda_scratch_space& scratch, - thrust::device_ptr d_offsets_it) -{ - // cudaStreamSynchronize(); - - // Run reduction - int num_segments = t; - - SumAbs sum_abs_op; - T initial_value = 0; - - // Determine temporary device storage requirements - size_t temp_storage_bytes; - - cub::DeviceSegmentedReduce::Reduce( - nullptr, - temp_storage_bytes, - values, - buffer, - num_segments, - d_offsets_it, - d_offsets_it + 1, - sum_abs_op, - initial_value); - - scratch.fit(temp_storage_bytes); - - // Run reduction - cub::DeviceSegmentedReduce::Reduce( - scratch.device_ptr, - temp_storage_bytes, - values, - buffer, - num_segments, - d_offsets_it, - d_offsets_it + 1, - sum_abs_op, - initial_value); - -/* - using namespace thrust::placeholders; - - // 1. First sum the absolute values in each column and place the results into buffer - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int)0), (_1 / n)); - - auto in_values_start = thrust::transform_iterator(thrust::device_pointer_cast(values), - [] __host__ __device__ (T x) {return std::abs(x);} ); - - thrust::reduce_by_key(in_keys_start, in_keys_start + n*t, // key indices (group by column) - in_values_start, // values to reduce (with abs applied) - thrust::make_discard_iterator(), // key values out - buffer, // reduced values out - thrust::equal_to() // compare-keys operation - ); // summation is the default operation. -*/ - // cudaStreamSynchronize(); - - // 2. Second maximize over the column sums and return the highest. - cuda_max(buffer, t, out, scratch); - - // cudaStreamSynchronize(); -} - -void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out, cuda_scratch_space& scratch, int* offsets) +__global__ void partials_product_kernel(const T* left, const T* right, T* parent, + int nStates, int nPatterns, int nCategories, + T* scalingFactors, T* cumulative, bool scalersLog) { - using namespace thrust; + const int pattern = blockIdx.x; + const size_t categoryStride = (size_t)nStates * nPatterns; + const size_t offset = (size_t)pattern * nStates; - cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out), scratch, device_pointer_cast(offsets)); -} - -void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch, int* offsets) -{ - using namespace thrust; - - cuda_max_l1_norm(device_pointer_cast(values), n, t, device_pointer_cast(buffer), device_pointer_cast(out), scratch, device_pointer_cast(offsets)); -} - -template -void cuda_vec_fill(device_ptr values, int length, T fill) { - using namespace thrust::placeholders; + if (not scalingFactors) + { + for (int c = 0; c < nCategories; c++) + for (int s = threadIdx.x; s < nStates; s += blockDim.x) + { + const size_t i = c * categoryStride + offset + s; + parent[i] = left[i] * right[i]; + } + return; + } - thrust::fill(values, values + length, fill); -} + T m = 0; + for (int c = 0; c < nCategories; c++) + for (int s = threadIdx.x; s < nStates; s += blockDim.x) + { + const size_t i = c * categoryStride + offset + s; + m = max(m, left[i] * right[i]); + } + m = block_reduce(m, T(0), MaxOp()); + if (m == 0) m = 1; -void cuda_vec_fill(float* values, int length, float fill) { - cuda_vec_fill(device_pointer_cast(values), length, fill); -} + const T inverse = 1 / m; + for (int c = 0; c < nCategories; c++) + for (int s = threadIdx.x; s < nStates; s += blockDim.x) + { + const size_t i = c * categoryStride + offset + s; + parent[i] = left[i] * right[i] * inverse; + } -void cuda_vec_fill(double* values, int length, double fill) { - cuda_vec_fill(device_pointer_cast(values), length, fill); + if (threadIdx.x == 0) + { + const T logM = log(m); + scalingFactors[pattern] = scalersLog ? logM : m; + if (cumulative) cumulative[pattern] += logM; + } } template -void cuda_vec_abs(device_ptr values, int n, device_ptr results) +void cuda_partials_product_impl(const T* left, const T* right, T* parent, int nStates, int nPatterns, int nCategories, + T* scalingFactors, T* cumulative, bool scalersLog, cudaStream_t stream) { - thrust::transform(values, values + n, results, [] __device__ (T x) {return abs(x);}); + partials_product_kernel<<>>(left, right, parent, nStates, nPatterns, nCategories, + scalingFactors, cumulative, scalersLog); } -void cuda_vec_abs(float* values, int n, float* results) +void cuda_partials_product(const float* left, const float* right, float* parent, int nStates, int nPatterns, int nCategories, + float* scalingFactors, float* cumulative, bool scalersLog, cudaStream_t stream) { - cuda_vec_abs(device_pointer_cast(values), n, device_pointer_cast(results)); + cuda_partials_product_impl(left, right, parent, nStates, nPatterns, nCategories, scalingFactors, cumulative, scalersLog, stream); } -void cuda_vec_abs(double* values, int n, double* results) +void cuda_partials_product(const double* left, const double* right, double* parent, int nStates, int nPatterns, int nCategories, + double* scalingFactors, double* cumulative, bool scalersLog, cudaStream_t stream) { - cuda_vec_abs(device_pointer_cast(values), n, device_pointer_cast(results)); + cuda_partials_product_impl(left, right, parent, nStates, nPatterns, nCategories, scalingFactors, cumulative, scalersLog, stream); } -// out[i] = max_j |values(i,j)| for a column-major (n,t) matrix. -// (A reduce_by_key with keys i % n cannot do this: equal keys are never adjacent in column-major order.) +// siteLogLikelihoods[p] = log( sum_{c,s} root(c,p,s) * categoryWeights[c] * frequencies[s] ) + cumulativeScaling[p] template -__global__ void rowwise_max_abs_kernel(const T* values, int n, int t, T* out) +__global__ void root_site_log_likelihoods_kernel(const T* root, const T* categoryWeights, const T* frequencies, + const T* cumulativeScaling, int nStates, int nPaddedPatterns, int nCategories, + T* siteLogLikelihoods) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= n) return; - - T m = 0; - for (int j = 0; j < t; j++) - m = max(m, abs(values[i + (size_t)j * n])); - out[i] = m; -} + const int pattern = blockIdx.x; + const size_t categoryStride = (size_t)nStates * nPaddedPatterns; + const size_t offset = (size_t)pattern * nStates; -template -void cuda_rowwise_max_abs_impl(const T* values_ptr, int n, int t, T* out_ptr) -{ - const int blockSize = 256; - rowwise_max_abs_kernel<<<(n + blockSize - 1) / blockSize, blockSize>>>(values_ptr, n, t, out_ptr); -} + T sum = 0; + for (int c = 0; c < nCategories; c++) + { + T categorySum = 0; + for (int s = threadIdx.x; s < nStates; s += blockDim.x) + categorySum += root[c * categoryStride + offset + s] * frequencies[s]; + sum += categorySum * categoryWeights[c]; + } + sum = block_reduce(sum, T(0), SumOp()); -void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr) -{ - cuda_rowwise_max_abs_impl(values_ptr, n, t, out_ptr); + if (threadIdx.x == 0) + siteLogLikelihoods[pattern] = log(sum) + (cumulativeScaling ? cumulativeScaling[pattern] : T(0)); } -void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr) +// out[0] = sum_i values[i] * weights[i], using a single block (deterministic). +template +__global__ void weighted_sum_kernel(const T* values, const T* weights, int n, T* out) { - cuda_rowwise_max_abs_impl(values_ptr, n, t, out_ptr); + T sum = 0; + for (int i = threadIdx.x; i < n; i += blockDim.x) + sum += values[i] * weights[i]; + sum = block_reduce(sum, T(0), SumOp()); + if (threadIdx.x == 0) out[0] = sum; } -// h[i] = -1 for rows already used as unit columns, so they sort last and are not chosen again. template -__global__ void mask_visited_kernel(T* h, const int* visited, int n) +void cuda_root_log_likelihoods_impl(const T* root, const T* categoryWeights, const T* frequencies, const T* cumulativeScaling, + const T* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, + T* siteLogLikelihoods, T* sumLogLikelihood, cudaStream_t stream) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i < n and visited[i]) h[i] = -1; + root_site_log_likelihoods_kernel<<>>(root, categoryWeights, frequencies, cumulativeScaling, + nStates, nPaddedPatterns, nCategories, siteLogLikelihoods); + weighted_sum_kernel<<<1, 1024, 0, stream>>>(siteLogLikelihoods, patternWeights, nPatterns, sumLogLikelihood); } -// visited[indices[j]] = 1 for j in [0,t-1] -__global__ void mark_visited_kernel(int* visited, const int* indices, int t) +void cuda_root_log_likelihoods(const float* root, const float* categoryWeights, const float* frequencies, const float* cumulativeScaling, + const float* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, + float* siteLogLikelihoods, float* sumLogLikelihood, cudaStream_t stream) { - int j = blockIdx.x * blockDim.x + threadIdx.x; - if (j < t) visited[indices[j]] = 1; + cuda_root_log_likelihoods_impl(root, categoryWeights, frequencies, cumulativeScaling, patternWeights, + nStates, nPaddedPatterns, nPatterns, nCategories, siteLogLikelihoods, sumLogLikelihood, stream); } -void cuda_mask_visited(float* h, const int* visited, int n) +void cuda_root_log_likelihoods(const double* root, const double* categoryWeights, const double* frequencies, const double* cumulativeScaling, + const double* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, + double* siteLogLikelihoods, double* sumLogLikelihood, cudaStream_t stream) { - mask_visited_kernel<<<(n + 255) / 256, 256>>>(h, visited, n); + cuda_root_log_likelihoods_impl(root, categoryWeights, frequencies, cumulativeScaling, patternWeights, + nStates, nPaddedPatterns, nPatterns, nCategories, siteLogLikelihoods, sumLogLikelihood, stream); } -void cuda_mask_visited(double* h, const int* visited, int n) -{ - mask_visited_kernel<<<(n + 255) / 256, 256>>>(h, visited, n); -} -void cuda_mark_visited(int* visited, const int* indices, int t) +// One Taylor term on column-major (nStates, nColumns) matrices, in a single pass: +// y = alpha * A * x; F = scale * (F + y) +// One thread per (row, column), consecutive threads on consecutive rows. Used when the work is small: +// it replaces cuSPARSE's partition, beta=0 fill and SpMM kernels plus the axpy. +template +__global__ void csr_taylor_term_kernel(int nStates, int nColumns, const int* __restrict__ rowOffsets, + const int* __restrict__ columns, const T* __restrict__ values, T alpha, + const T* __restrict__ x, T* __restrict__ y, T* __restrict__ F, T scale) { - mark_visited_kernel<<<(t + 255) / 256, 256>>>(visited, indices, t); -} + const size_t index = blockIdx.x * (size_t)blockDim.x + threadIdx.x; + if (index >= (size_t)nStates * nColumns) return; + const int row = index % nStates; + const T* xColumn = x + (index - row); -void cuda_fill_vector(float* buffer, int n, double x) -{ - thrust::fill_n(thrust::device_pointer_cast(buffer), n, x); + T sum = 0; + for (int k = rowOffsets[row]; k < rowOffsets[row + 1]; k++) + sum += values[k] * __ldg(xColumn + columns[k]); + sum *= alpha; + + y[index] = sum; + F[index] = scale * (F[index] + sum); } -void cuda_fill_vector(double* buffer, int n, double x) +template +void cuda_csr_taylor_term_impl(int nStates, int nColumns, const int* rowOffsets, const int* columns, const T* values, T alpha, + const T* x, T* y, T* F, T scale, cudaStream_t stream) { - thrust::fill_n(thrust::device_pointer_cast(buffer), n, x); + const size_t n = (size_t)nStates * nColumns; + const int blockSize = 256; + csr_taylor_term_kernel<<<(n + blockSize - 1) / blockSize, blockSize, 0, stream>>>(nStates, nColumns, rowOffsets, columns, values, + alpha, x, y, F, scale); } -void cuda_sort_indices_by_vector(const float* values_ptr, int n, int* indices_ptr) +void cuda_csr_taylor_term(int nStates, int nColumns, const int* rowOffsets, const int* columns, const float* values, float alpha, + const float* x, float* y, float* F, float scale, cudaStream_t stream) { - auto indices = thrust::device_pointer_cast(indices_ptr); - auto values = thrust::device_pointer_cast(values_ptr); - - // 1. Initialize the indices to [0, n-1] - thrust::counting_iterator iter(0); - thrust::copy(iter, iter + n, indices); - - // 2. Sort the indices by values - thrust::sort(indices, indices+n, [values] __host__ __device__ (int idx1, int idx2) {return values[idx1] > values[idx2];}); + cuda_csr_taylor_term_impl(nStates, nColumns, rowOffsets, columns, values, alpha, x, y, F, scale, stream); } - -void cuda_sort_indices_by_vector(const double* values_ptr, int n, int* indices_ptr) +void cuda_csr_taylor_term(int nStates, int nColumns, const int* rowOffsets, const int* columns, const double* values, double alpha, + const double* x, double* y, double* F, double scale, cudaStream_t stream) { - auto indices = thrust::device_pointer_cast(indices_ptr); - auto values = thrust::device_pointer_cast(values_ptr); - - // 1. Initialize the indices to [0, n-1] - thrust::counting_iterator iter(0); - thrust::copy(iter, iter + n, indices); - - // 2. Sort the indices by values - thrust::sort(indices, indices+n, [values] __host__ __device__ (int idx1, int idx2) {return values[idx1] > values[idx2];}); + cuda_csr_taylor_term_impl(nStates, nColumns, rowOffsets, columns, values, alpha, x, y, F, scale, stream); } -void cuda_set_indices(float* x_ptr, int n, int t, const int* indices_ptr) -{ - // 1. Initialize the indices to [0, n-1] - thrust::counting_iterator iter(0); - auto x = thrust::device_pointer_cast(x_ptr); - auto indices = thrust::device_pointer_cast(indices_ptr); - thrust::for_each(iter, iter+t, [=] __host__ __device__ (int i) { x[n*i + indices[i]] = 1.0; }); -} +// --------------------------------------------------------------------------------------------------- +// Block 1-norm estimation of ||B^p||_1 for all p at once (Higham & Tisseur Algorithm 2.4, 2 iterations). +// All matrices are column-major with n rows. See BlockNormEstimator in BeagleGPUActionImpl.h. -// Set X(i,indices[i]) = 1 for i in [0,t-1] -void cuda_set_indices(double* x_ptr, int n, int t, const int* indices_ptr) +// y(:, 0:nApplied) = A * x(:, 0:nApplied), then nAppend more columns of y are set: +// from signSource (n, nAppend): y(i, c) = +1 if signSource(i, c) >= 0, else -1 +// from unitRows (nAppend): y(i, c) = 1 if i == unitRows[c], else 0 +template +__global__ void norm_chain_step_kernel(int n, const int* __restrict__ rowOffsets, const int* __restrict__ columns, + const T* __restrict__ values, const T* __restrict__ x, T* __restrict__ y, + int nApplied, int nAppend, const T* __restrict__ signSource, const int* __restrict__ unitRows) { - // 1. Initialize the indices to [0, n-1] - thrust::counting_iterator iter(0); - auto x = thrust::device_pointer_cast(x_ptr); - auto indices = thrust::device_pointer_cast(indices_ptr); + const size_t index = blockIdx.x * (size_t)blockDim.x + threadIdx.x; + if (index >= (size_t)n * (nApplied + nAppend)) return; + const int row = index % n; + const int column = index / n; - thrust::for_each(iter, iter+t, [=] __host__ __device__ (int i) { x[n*i + indices[i]] = 1.0; }); + if (column < nApplied) + { + const T* xColumn = x + (size_t)column * n; + T sum = 0; + for (int k = rowOffsets[row]; k < rowOffsets[row + 1]; k++) + sum += values[k] * xColumn[columns[k]]; + y[index] = sum; + } + else + { + const int c = column - nApplied; + if (signSource) + y[index] = (signSource[(size_t)c * n + row] >= 0) ? T(1) : T(-1); + else + y[index] = (row == unitRows[c]) ? T(1) : T(0); + } } template -void justMaximize(device_ptr partials, device_ptr scalingFactors, - int nStates, int nPatterns, int nCategories) +void cuda_norm_chain_step_impl(int n, const int* rowOffsets, const int* columns, const T* values, const T* x, T* y, + int nApplied, int nAppend, const T* signSource, const int* unitRows, cudaStream_t stream) { - using namespace thrust::placeholders; - - // 1. Surprisingly, cuBLAS has no operations that reduce (sum,maximize,minimize,etc.) rows or columns. - // It can only reduce an entire dense matrix to a single value. - - // 2. thrust::reduce_by_key is able to reduce regions of a vector down to MULTIPLE values. - // We assign each value a "key" that decides which group it is in. - // Adjacent values with the same key end up in the same group. - // However, non-adjacent values with the same key end up in different groups. - // thrust::reduce_by_key performs the reduction operation on each group. - - // 3. Thrust can maximize over states (adjacent memory_ and categories (non-adjacent) in one operation - // by changing the order in which entries of the input matrix are visited. All the entries in the - // same group need to be visited sequentially. We can do this using a permutation iterator. - - // Implementation notes: - // * the library MatX (https://github.com/NVIDIA/MatX) offers much nicer syntax, but thrust is older - // and more stable (in 2024) - - // The size of a partials buffer - size_t partials_size = nStates * nPatterns * nCategories; - - - // OK, so we can convert the input index i=[0...partials_size] to (p,c,s) as follows: - // s = i % nStates - // c = (i / nStates) % nCategories - // p = (i / nStates * nCategories) - - // The linear index into the input matrix is: - // j = s + nStates *p + (nStates*nPatterns)*c - - // A list of 00000..111111..222222........(P-1)(P-1)(P-1)(P-1) that groups values by state and categories. - // There should be nPatterns groups. - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates * nCategories))); - - // We need to compute j as a function of i: - // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) - auto in_values_start = thrust::make_permutation_iterator( - partials, - thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), - (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) - ); - - thrust::reduce_by_key( - // add execution policy thrust::cuda::par_nosync? - in_keys_start, // key indices start (group by pattern) - in_keys_start + partials_size, // key indices end - in_values_start, // values to reduce (category, pattern, state) - thrust::make_discard_iterator(), // key values out - scalingFactors, // reduced values out - thrust::equal_to(), // compare keys operation - thrust::maximum() // reduction operation - ); + const size_t total = (size_t)n * (nApplied + nAppend); + const int blockSize = 256; + norm_chain_step_kernel<<<(total + blockSize - 1) / blockSize, blockSize, 0, stream>>>(n, rowOffsets, columns, values, x, y, + nApplied, nAppend, signSource, unitRows); } -void justMaximize(float* partials, float* scalingFactors, - int nStates, int nPatterns, int nCategories) +void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const float* values, const float* x, float* y, + int nApplied, int nAppend, const float* signSource, const int* unitRows, cudaStream_t stream) { - justMaximize(device_pointer_cast(partials), - device_pointer_cast(scalingFactors), - nStates, nPatterns, nCategories); + cuda_norm_chain_step_impl(n, rowOffsets, columns, values, x, y, nApplied, nAppend, signSource, unitRows, stream); } -void justMaximize(double* partials, double* scalingFactors, - int nStates, int nPatterns, int nCategories) +void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const double* values, const double* x, double* y, + int nApplied, int nAppend, const double* signSource, const int* unitRows, cudaStream_t stream) { - justMaximize(device_pointer_cast(partials), - device_pointer_cast(scalingFactors), - nStates, nPatterns, nCategories); + cuda_norm_chain_step_impl(n, rowOffsets, columns, values, x, y, nApplied, nAppend, signSource, unitRows, stream); } +// sums[c] = sum_i |x(i, c)|, one block per column. template -struct Prod3 +__global__ void column_abs_sums_kernel(const T* x, int n, T* sums) { - __host__ __device__ T operator()(thrust::tuple t) - { - return thrust::get<0>(t) * thrust::get<1>(t) * thrust::get<2>(t); - } -}; + const T* column = x + (size_t)blockIdx.x * n; + T sum = 0; + for (int i = threadIdx.x; i < n; i += blockDim.x) + sum += abs(column[i]); + sum = block_reduce(sum, T(0), SumOp()); + if (threadIdx.x == 0) sums[blockIdx.x] = sum; +} template -void sumRootLikelihoods(device_ptr siteProbs, // OUT - device_ptr likelihoods, device_ptr weights, device_ptr frequencies, // IN - int nStates, int nPatterns, int nCategories) +void cuda_column_abs_sums_impl(const T* x, int n, int nColumns, T* sums, cudaStream_t stream) { - using namespace thrust::placeholders; - - // The size of a partials buffer - size_t partials_size = nStates * nPatterns * nCategories; - - - // OK, so we can convert the input index i=[0...partials_size] to (p,c,s) as follows: - // s = i % nStates - // c = (i / nStates) % nCategories - // p = (i / nStates * nCategories) - - // 1. A list of 00000..111111..222222........(P-1)(P-1)(P-1)(P-1) that groups values by state and categories. - // There should be nPatterns groups. - auto in_keys_start = thrust::make_transform_iterator(thrust::make_counting_iterator((int) 0), (_1 / (nStates * nCategories))); - - // 2. The linear index into the likelihood matrix is: - // j = s + nStates *p + (nStates*nPatterns)*c - // - // We need to compute j as a function of i: - // j(i) = (i % nStates) + nStates*(i/(nStates * nCategories)) + (nStates*nPatterns)*((i/nStates) % nCategories) - auto in_lks_start = thrust::make_permutation_iterator( - likelihoods, - thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), - (_1 % nStates) + nStates*(_1/(nStates*nCategories)) + nStates*nPatterns*((_1/nStates)%nCategories)) - ); - - // 3. The linear index into the weights matrix is just: - // k = c - // Therefore - // k(i) = (i / nStates) % nCategories - - auto in_weights_start = thrust::make_permutation_iterator( - weights, - thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), - (_1/nStates)%nCategories) - ); - - // 4. The linear index into the states matrix is: - // l = s - // Therefore - // l(i) = (i % nStates) - - auto in_frequencies_start = thrust::make_permutation_iterator( - frequencies, - thrust::make_transform_iterator( thrust::make_counting_iterator((int)0), - _1 % nStates) - ); - - - // 5. Make the sequence of likelihood * categoryWeight * stateFrequency - auto tuples = thrust::make_zip_iterator(thrust::make_tuple(in_lks_start, in_weights_start, in_frequencies_start)); - - auto in_values_start = thrust::make_transform_iterator(tuples, Prod3()); - - // 6. Sum over state and category for each pattern. - thrust::reduce_by_key( - // add execution policy thrust::cuda::par_nosync? - in_keys_start, // key indices start (group by pattern) - in_keys_start + partials_size, // key indices end - in_values_start, // values to reduce (category, pattern, state) - thrust::make_discard_iterator(), // key values out - siteProbs, // reduced values out - thrust::equal_to() // compare keys operation - // Default operation is (+) - ); + column_abs_sums_kernel<<>>(x, n, sums); } - -void sumRootLikelihoods(float* siteProbs, // OUT - float* likelihoods, float* weights, float* frequencies, // INT - int nStates, int nPatterns, int nCategories) +void cuda_column_abs_sums(const float* x, int n, int nColumns, float* sums, cudaStream_t stream) { - sumRootLikelihoods(device_pointer_cast(siteProbs), - device_pointer_cast(likelihoods), - device_pointer_cast(weights), - device_pointer_cast(frequencies), - nStates, nPatterns, nCategories); + cuda_column_abs_sums_impl(x, n, nColumns, sums, stream); } - -void sumRootLikelihoods(double* siteProbs, // OUT - double* likelihoods, double* weights, double* frequencies, // INT - int nStates, int nPatterns, int nCategories) +void cuda_column_abs_sums(const double* x, int n, int nColumns, double* sums, cudaStream_t stream) { - sumRootLikelihoods(device_pointer_cast(siteProbs), - device_pointer_cast(likelihoods), - device_pointer_cast(weights), - device_pointer_cast(frequencies), - nStates, nPatterns, nCategories); + cuda_column_abs_sums_impl(x, n, nColumns, sums, stream); } +// For each block b of t columns of Z: h(i) = max_j |Z_b(i, j)|; unitRows[b*t + r] = the row with the +// r-th largest h (ties go to the lower row). One thread block per column block. template -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - device_ptr partials, device_ptr scalingFactors, device_ptr cumulativeScalingBuffer, int streamIndex) +__global__ void select_unit_columns_kernel(const T* Z, int n, int t, int* unitRows) { - using namespace thrust::placeholders; - - size_t partialsSize = kCategoryCount * kPaddedPatternCount * kPaddedStateCount; - - // 1. Find maximize partial likelihood -> scalingFactors[pattern] - justMaximize(partials, scalingFactors, kPaddedStateCount, kPaddedPatternCount, kCategoryCount); - - // 2. Transform scalingfactors[pattern] -> 1 if it equals 0. - thrust::transform(scalingFactors, scalingFactors + kPaddedPatternCount, scalingFactors, [] __device__ (T x) { return (x == 0) ? 1.0 : x;}); - - // 3. Rescale each pattern by scalingFactors[pattern(index)] - - // pattern = (i/kPaddedStateCount) % kPaddedPatternCount + __shared__ T bestValue[1024]; + __shared__ int bestRow[1024]; + __shared__ int chosen[32]; - // iter_max computes the scaling factor as a function of the index into the partials buffer. - auto iter_max = thrust::make_permutation_iterator( - scalingFactors, - thrust::make_transform_iterator( - thrust::make_counting_iterator(0), - (_1/kPaddedStateCount) % kPaddedPatternCount - ) - ); + const T* block = Z + (size_t)blockIdx.x * n * t; - - thrust::transform(partials, partials + partialsSize, // in1 = partials[i] - iter_max, // in2 = scalingFactors[pattern(i)] - partials, // out - thrust::divides() // operation - ); - -// std::cerr<<"scalers_log = "<() // operation; - ); + bool used = false; + for (int q = 0; q < r; q++) + used = used or (chosen[q] == i); + if (used) continue; + + T h = 0; + for (int j = 0; j < t; j++) + h = max(h, abs(block[(size_t)j * n + i])); + if (h > value) { value = h; row = i; } } - else + bestValue[threadIdx.x] = value; + bestRow[threadIdx.x] = row; + __syncthreads(); + + for (int stride = blockDim.x / 2; stride > 0; stride /= 2) { - auto logScalingFactors2 = thrust::make_transform_iterator( scalingFactors, - [] __host__ __device__ (T x) { return log(x); }); - - thrust::transform(cumulativeScalingBuffer, cumulativeScalingBuffer + kPaddedPatternCount, // in1 - logScalingFactors2, // in2 - cumulativeScalingBuffer, // out - thrust::plus() // operation; - ); + if (threadIdx.x < stride) + { + const int other = threadIdx.x + stride; + if (bestValue[other] > bestValue[threadIdx.x] or + (bestValue[other] == bestValue[threadIdx.x] and bestRow[other] < bestRow[threadIdx.x])) + { + bestValue[threadIdx.x] = bestValue[other]; + bestRow[threadIdx.x] = bestRow[other]; + } + } + __syncthreads(); } + + if (threadIdx.x == 0) + { + chosen[r] = bestRow[0]; + unitRows[blockIdx.x * t + r] = bestRow[0]; + } + __syncthreads(); } } -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex) +template +void cuda_select_unit_columns_impl(const T* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream) { - rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, - device_pointer_cast(partials), device_pointer_cast(scalingFactors), device_pointer_cast(cumulativeScalingBuffer), - streamIndex); + select_unit_columns_kernel<<>>(Z, n, t, unitRows); } - -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex) +void cuda_select_unit_columns(const float* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream) { - rescalePartials2(scalers_log, kCategoryCount, kPaddedPatternCount, kPaddedStateCount, - device_pointer_cast(partials), device_pointer_cast(scalingFactors), device_pointer_cast(cumulativeScalingBuffer), - streamIndex); + cuda_select_unit_columns_impl(Z, n, t, nBlocks, unitRows, stream); } +void cuda_select_unit_columns(const double* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream) +{ + cuda_select_unit_columns_impl(Z, n, t, nBlocks, unitRows, stream); +} +// norms[p] = max over the t columns for p in both iterations, for p = 2 .. pTop: +// iteration 1: block p-1 of sums1; iteration 2: block pTop-p of sums2. template -void initialize_norm_x_matrix(device_ptr out_ptr, int n, int m) +__global__ void combine_norm_estimates_kernel(const T* sums1, const T* sums2, int t, int pTop, T* norms) { - auto indices = thrust::counting_iterator(0); - auto initialize = [n,m] __host__ __device__ (int i) - { - thrust::default_random_engine rng; - thrust::uniform_real_distribution dist(0, 1); - - if (i 0.5) - return 1.0/n; - else - return -1.0/n; - }; + const int p = threadIdx.x + 2; + if (p > pTop) return; + T norm = 0; + for (int j = 0; j < t; j++) + norm = max(norm, max(sums1[(p - 1) * t + j], sums2[(pTop - p) * t + j])); + norms[p] = norm; +} - thrust::transform(indices, indices + n*m, // in - out_ptr, // out - initialize); +template +void cuda_combine_norm_estimates_impl(const T* sums1, const T* sums2, int t, int pTop, T* norms, cudaStream_t stream) +{ + combine_norm_estimates_kernel<<<1, 32, 0, stream>>>(sums1, sums2, t, pTop, norms); } -void initialize_norm_x_matrix(float* data, int n, int m) +void cuda_combine_norm_estimates(const float* sums1, const float* sums2, int t, int pTop, float* norms, cudaStream_t stream) { - initialize_norm_x_matrix(device_pointer_cast(data), n, m); + cuda_combine_norm_estimates_impl(sums1, sums2, t, pTop, norms, stream); } -void initialize_norm_x_matrix(double* data, int n, int m) +void cuda_combine_norm_estimates(const double* sums1, const double* sums2, int t, int pTop, double* norms, cudaStream_t stream) { - initialize_norm_x_matrix(device_pointer_cast(data), n, m); + cuda_combine_norm_estimates_impl(sums1, sums2, t, pTop, norms, stream); } diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 6646696b..4e23d81a 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -75,76 +75,54 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) return host_vec; } -struct cuda_scratch_space -{ - void* device_ptr = nullptr; - size_t size = 0; - - void discard(); - void fit(size_t n); - - ~cuda_scratch_space(); -}; - -void cuda_log_vector(double* v, int length); -void cuda_log_vector(float* v, int length); - -// (n,t) -> (n,t) -void cuda_sign_vector(double* v, int n, int t); -void cuda_sign_vector(float* v, int n, int t); - -// sum.abs over (n,t) -> (1,t), then max over (1,t) -> (1,1). Matrix assumed to be column-major. +// max_i |v[i]| (synchronizes with the host). double cuda_max_abs(double* v, int length); float cuda_max_abs(float* v, int length); -void cuda_max(double* v, int length, double* out, cuda_scratch_space& scratch); -void cuda_max(float* v, int length, float* out, cuda_scratch_space& scratch); - -// max over (n,t) -> (n,1). Matrix assumed to be column-major. -void cuda_rowwise_max_abs(float* values_ptr, int n, int t, float* out_ptr); -void cuda_rowwise_max_abs(double* values_ptr, int n, int t, double* out_ptr); - -// h[i] = -1 where visited[i] != 0 -void cuda_mask_visited(float* h, const int* visited, int n); -void cuda_mask_visited(double* h, const int* visited, int n); - -// visited[indices[j]] = 1 for j in [0,t-1] -void cuda_mark_visited(int* visited, const int* indices, int t); - -void cuda_fill_vector(float* buffer, int n, double x); -void cuda_fill_vector(double* buffer, int n, double x); - +// The starting matrix X (n, t) of the block 1-norm estimator: first column 1/n, others random +-1/n. void initialize_norm_x_matrix(float* data, int n, int m); void initialize_norm_x_matrix(double* data, int n, int m); -float cuda_max_l1_norm(float* values, int n, int t, float* buffer_); -double cuda_max_l1_norm(double* values, int n, int t, double* buffer_); - -void cuda_max_l1_norm(float* values, int n, int t, float* buffer, float* out, cuda_scratch_space& scratch, int* offsets); -void cuda_max_l1_norm(double* values, int n, int t, double* buffer, double* out, cuda_scratch_space& scratch, int* offsets); - -void cuda_vec_fill(float* values, int length, float fill); -void cuda_vec_fill(double* values, int length, double fill); - -void cuda_vec_abs(float *values, int n, float *results); -void cuda_vec_abs(double* values, int n, double *results); - -void cuda_sort_indices_by_vector(const float* values_ptr, int n, int* indices_ptr); -void cuda_sort_indices_by_vector(const double* values_ptr, int n, int* indices_ptr); - -void cuda_set_indices(float* x, int n, int t, const int* indices); -void cuda_set_indices(double* x, int n, int t, const int* indices); - -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - float* partials, float* scalingFactors, float* cumulativeScalingBuffer, int streamIndex); -void rescalePartials2(bool scalers_log, int kCategoryCount, int kPaddedPatternCount, int kPaddedStateCount, - double* partials, double* scalingFactors, double* cumulativeScalingBuffer, int streamIndex); - -void sumRootLikelihoods(float* siteProbs, // OUT - float* partials, float* weights, float* frequencies, // INT - int nStates, int nPatterns, int nCategories); - -void sumRootLikelihoods(double* siteProbs, // OUT - double* partials, double* weights, double* frequencies, // INT - int nStates, int nPatterns, int nCategories); +// Block 1-norm estimation, see BlockNormEstimator in BeagleGPUActionImpl.h. +// y(:, 0:nApplied) = A x(:, 0:nApplied) for column-major y, x with n rows and A in CSR; then nAppend more +// columns of y are set to sign(signSource) if signSource is non-null, otherwise to unit vectors e(unitRows[c]). +void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const float* values, const float* x, float* y, + int nApplied, int nAppend, const float* signSource, const int* unitRows, cudaStream_t stream); +void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const double* values, const double* x, double* y, + int nApplied, int nAppend, const double* signSource, const int* unitRows, cudaStream_t stream); + +// sums[c] = sum_i |x(i, c)| for the nColumns columns of x (n rows). +void cuda_column_abs_sums(const float* x, int n, int nColumns, float* sums, cudaStream_t stream); +void cuda_column_abs_sums(const double* x, int n, int nColumns, double* sums, cudaStream_t stream); + +// For each of nBlocks blocks of t columns of Z (n rows): the t rows with the largest max_j |Z(i,j)| -> unitRows[b*t ...]. +void cuda_select_unit_columns(const float* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream); +void cuda_select_unit_columns(const double* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream); + +// norms[p] for p = 2..pTop: the largest column sum for p in either iteration (see the kernel). +void cuda_combine_norm_estimates(const float* sums1, const float* sums2, int t, int pTop, float* norms, cudaStream_t stream); +void cuda_combine_norm_estimates(const double* sums1, const double* sums2, int t, int pTop, double* norms, cudaStream_t stream); + +// parent = left * right for (nStates, nPatterns, nCategories) partials. If scalingFactors is non-null, also +// rescale each pattern by its largest entry m, write m (log m if scalersLog) to scalingFactors[pattern], +// and add log m to cumulative[pattern] if cumulative is non-null. +void cuda_partials_product(const float* left, const float* right, float* parent, int nStates, int nPatterns, int nCategories, + float* scalingFactors, float* cumulative, bool scalersLog, cudaStream_t stream); +void cuda_partials_product(const double* left, const double* right, double* parent, int nStates, int nPatterns, int nCategories, + double* scalingFactors, double* cumulative, bool scalersLog, cudaStream_t stream); + +// siteLogLikelihoods[p] = log(sum_{c,s} root * categoryWeights[c] * frequencies[s]) + cumulativeScaling[p] (if non-null), +// sumLogLikelihood[0] = sum_p patternWeights[p] * siteLogLikelihoods[p]. +void cuda_root_log_likelihoods(const float* root, const float* categoryWeights, const float* frequencies, const float* cumulativeScaling, + const float* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, + float* siteLogLikelihoods, float* sumLogLikelihood, cudaStream_t stream); +void cuda_root_log_likelihoods(const double* root, const double* categoryWeights, const double* frequencies, const double* cumulativeScaling, + const double* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, + double* siteLogLikelihoods, double* sumLogLikelihood, cudaStream_t stream); + +// One Taylor term on column-major (nStates, nColumns) matrices: y = alpha * A * x; F = scale * (F + y), A in CSR. +void cuda_csr_taylor_term(int nStates, int nColumns, const int* rowOffsets, const int* columns, const float* values, float alpha, + const float* x, float* y, float* F, float scale, cudaStream_t stream); +void cuda_csr_taylor_term(int nStates, int nColumns, const int* rowOffsets, const int* columns, const double* values, double alpha, + const double* x, double* y, double* F, double scale, cudaStream_t stream); #endif From bd7078e4b755d3525bcd701c3eeb7b71c995285a Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Sat, 24 Oct 2026 18:11:51 -0500 Subject: [PATCH 412/421] CI: run the Xcode 16 job on macos-15 macos-latest no longer has Xcode 16, so setup-xcode failed and fail-fast cancelled every other job. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9abecf27..fda20321 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,7 +90,7 @@ jobs: arch: "mac-intel64" - name: macos-xcode16-arm - os: macos-latest + os: macos-15 compiler: xcode version: "16" arch: "mac-arm64" From a5fbb789fe2aae3358c7736939ff8fbbf5ad3712 Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Sat, 24 Oct 2026 18:15:38 -0500 Subject: [PATCH 413/421] CI: build without -march=native ccache keys on the literal -march=native flag, not on the runner's CPU, so objects built on one runner were reused on another with an older CPU and synthetictest died with 'Illegal instruction' (ubuntu-gcc-13). Make BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH an option (still ON by default) and turn it off in the build-and-test workflow. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 4 +++- CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fda20321..703cef24 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,7 +121,9 @@ jobs: - name: Set CMake build type run: | - CMAKE_ARGS="" + # Runners have different CPUs, and ccache would reuse -march=native objects across them + # ("Illegal instruction"), so build for the baseline architecture. + CMAKE_ARGS="-DBEAGLE_OPTIMIZE_FOR_NATIVE_ARCH=OFF" if [ -n "${{ matrix.buildtype }}" ] ; then CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${{ matrix.buildtype }}" fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 546dbef9..d9e08142 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ set(BEAGLE_PLUGIN_VERSION "${BEAGLE_VERSION_MAJOR}${BEAGLE_VERSION_MINOR}") message(STATUS "BEAGLE_VERSION = ${BEAGLE_VERSION}") message(STATUS "BEAGLE_PLUGIN_VERSION = ${BEAGLE_PLUGIN_VERSION}") -set(BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH true) +option(BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH "Compile with -march=native (not portable to other CPUs)" ON) option(BUILD_OPENCL "Build beagle with OpenCL library" ON) option(BUILD_CUDA "Build beagle with CUDA library" ON) From 0556a0f5d42339fd7e41b538f2a2942345e24323 Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Sat, 24 Oct 2026 18:18:34 -0500 Subject: [PATCH 414/421] CI: key the ccache cache on the runner CPU instead of dropping -march=native Turning -march=native off breaks the SSE plugin (_mm_dp_pd needs SSE4.1), so this reverts the previous commit. The 'Illegal instruction' came from ccache reusing -march=native objects built on a runner with a newer CPU; including the CPU model in the cache key prevents that. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 21 ++++++++++++++++----- CMakeLists.txt | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 703cef24..96dcd109 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,9 +121,7 @@ jobs: - name: Set CMake build type run: | - # Runners have different CPUs, and ccache would reuse -march=native objects across them - # ("Illegal instruction"), so build for the baseline architecture. - CMAKE_ARGS="-DBEAGLE_OPTIMIZE_FOR_NATIVE_ARCH=OFF" + CMAKE_ARGS="" if [ -n "${{ matrix.buildtype }}" ] ; then CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${{ matrix.buildtype }}" fi @@ -179,12 +177,25 @@ jobs: echo "CXX=ccache clang++" >> $GITHUB_ENV echo "CMAKE_ARGS=${CMAKE_ARGS} -DBUILD_OPENMP=OFF" >> $GITHUB_ENV + # The build uses -march=native, but ccache keys on the literal flag, not on the CPU. Without the + # CPU in the cache key, objects built on one runner get reused on a runner with an older CPU + # ("Illegal instruction"). + - name: Identify CPU (for the ccache key) + shell: bash + run: | + if [ "$RUNNER_OS" = "macOS" ]; then + cpu=$(sysctl -n machdep.cpu.brand_string) + else + cpu=$(grep -m1 "model name" /proc/cpuinfo || uname -m) + fi + echo "CPU_KEY=$(echo "$cpu" | cksum | cut -d' ' -f1)" >> $GITHUB_ENV + - name: Set up ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ matrix.name }} + key: ${{ matrix.name }}-${{ env.CPU_KEY }} restore-keys: | - ${{ matrix.name }} + ${{ matrix.name }}-${{ env.CPU_KEY }} - name: Configure run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index d9e08142..546dbef9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ set(BEAGLE_PLUGIN_VERSION "${BEAGLE_VERSION_MAJOR}${BEAGLE_VERSION_MINOR}") message(STATUS "BEAGLE_VERSION = ${BEAGLE_VERSION}") message(STATUS "BEAGLE_PLUGIN_VERSION = ${BEAGLE_PLUGIN_VERSION}") -option(BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH "Compile with -march=native (not portable to other CPUs)" ON) +set(BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH true) option(BUILD_OPENCL "Build beagle with OpenCL library" ON) option(BUILD_CUDA "Build beagle with CUDA library" ON) From d372e30b9bbb5fc55402234965979a9858558c07 Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Sat, 24 Oct 2026 23:39:04 -0500 Subject: [PATCH 415/421] GPU action Phase 2: row-major partials, Horner term kernels, level batching - Store action partials pattern-fastest, [category][state][pattern]. setTipPartials/setPartials upload a transposed copy; getPartials transposes back. The node product/rescale and root kernels read the new layout. - Replace cuSPARSE SpMM + axpy with one custom kernel per Taylor term, evaluating the polynomial by Horner's rule (3 memory passes per term). - Wide kernel (>= 32 patterns): one warp per row, 1-4 columns per lane, grid ordered so the blocks in flight share a column panel. - Narrow kernel (< 32 patterns): lanes cover (group, pattern), with the groups' work buffers interleaved so gathers coalesce. - Batch all categories of both children, and all independent operations of a dependency level, into one launch per term. Each group keeps its own (m, s). The products of a level are batched too. - Override accumulateScaleFactors/removeScaleFactors: the base kernels read past the scaling buffers when patterns are not padded to their block size, which the action layout never does. A100, ms per evaluation (original -> now): IGC 4,096 patterns 1107 -> 227, IGC 64 32.1 -> 5.8, MM M=32 3,052 patterns 152 -> 34.8, MM M=8 64 patterns 6.6 -> 0.75. logL matches the CPU action plugin to <= 1.5e-14; compute-sanitizer memcheck/racecheck/synccheck clean. Co-Authored-By: Claude Opus 5.5 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 146 +++---- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 531 ++++++++++++++--------- libhmsbeagle/GPU/cuda_ops.cu | 506 +++++++++++++++------ libhmsbeagle/GPU/cuda_ops.h | 100 ++++- 4 files changed, 856 insertions(+), 427 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index d0779a5c..022ebc61 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -226,16 +226,6 @@ struct DnMatrixDevice MemcpyDeviceToDevice(ptr, D.ptr, size()); } - void copyFromAsync(const DnMatrixDevice& D, cudaStream_t stream) - { - assert(order == D.order); - assert(size1 == D.size1); - assert(size2 == D.size2); - auto status = cudaMemcpyAsync(ptr, D.ptr, byte_size(), cudaMemcpyDeviceToDevice, stream); - if (status != cudaSuccess) - throw std::runtime_error("cudaMemcpyAsync(Device->Device): " + std::string(cudaGetErrorString(status))); - } - // Disallow copying -- only one object can "own" the descriptor. DnMatrixDevice& operator=(const DnMatrixDevice&) = delete; // Allow moving. @@ -288,38 +278,6 @@ struct DnMatrixDevice return *this; } - // Point at different memory; the cuSPARSE descriptor must follow, or SpMM reads the old buffer. - void setPointer(Real* p) - { - ptr = p; - if (descr and cusparseDnMatSetValues(descr, p) != CUSPARSE_STATUS_SUCCESS) - { - std::cerr<<"cusparseDnMatSetValues( ) failed!"; - std::exit(1); - } - } - - // (*this) = a * (*this) + b * D, in one pass. - void axpby(Real a, Real b, const DnMatrixDevice& D) - { - cublasStatus_t status; - assert(D.size1 == size1); - assert(D.size2 == size2); - assert(order == CUSPARSE_ORDER_COL and D.order == CUSPARSE_ORDER_COL); - // geam allows C to alias A when A is not transposed and lda == ldc. - if constexpr (std::is_same::value) { - status = cublasSgeam(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_N, size1, size2, &a, ptr, size1, &b, D.ptr, size1, ptr, size1); - } else { - status = cublasDgeam(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_N, size1, size2, &a, ptr, size1, &b, D.ptr, size1, ptr, size1); - } - - if (status != CUBLAS_STATUS_SUCCESS) - { - std::cerr<<"cublas error "<::axpby\n"; - exit(1); - } - } - // Disallow copying. DnMatrixDevice(const DnMatrixDevice&) = delete; // Allow moving. @@ -516,31 +474,6 @@ int spMM(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, con return spMM(A.cusparseHandle, C.descr, alpha, A.descr, B.descr, beta, buffer, buffersize); } -// C = alpha * A * B with a work buffer already sized by spMMBufferSize( ). -template -int spMMPrepared(DnMatrixDevice& C, Real alpha, const SpMatrixDevice& A, const DnMatrixDevice& B, void* buffer) -{ - Real beta = 0; - CHECK_CUSPARSE(cusparseSpMM(A.cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A.descr, B.descr, &beta, C.descr, DataType, - CUSPARSE_SPMM_ALG_DEFAULT, buffer)); - return 0; -} - -template -size_t spMMBufferSize(const DnMatrixDevice& C, const SpMatrixDevice& A, const DnMatrixDevice& B) -{ - Real alpha = 1; - Real beta = 0; - size_t size = 0; - auto status = cusparseSpMM_bufferSize(A.cusparseHandle, CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, - &alpha, A.descr, B.descr, &beta, C.descr, DataType, - CUSPARSE_SPMM_ALG_DEFAULT, &size); - if (status != CUSPARSE_STATUS_SUCCESS) - throw std::runtime_error(std::string("cusparseSpMM_bufferSize: ") + cusparseGetErrorString(status)); - return size; -} - // Create an io-manipulator so that we can write std::cerr< class byRowDense @@ -960,6 +893,10 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int setPartials(int bufferIndex, const Real* inPartials); + int getPartials(int bufferIndex, + int scaleIndex, + double* outPartials) override; + int setSparseMatrix(int matrixIndex, const int* rowIndices, const int* colIndices, @@ -970,6 +907,11 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int operationCount, int cumulativeScalingIndex); + // The base versions read past the end of the scaling buffers unless the pattern count is padded to their + // block size, which the action layout does not do. + int accumulateScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex) override; + int removeScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex) override; + // Action instances have no dense transition matrices (see createInstance), so these are not available. // (The base versions of the first three call the dense, non-virtual upPartials/upPrePartials.) int updatePrePartials(const int*, int, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } @@ -1040,20 +982,23 @@ class BeagleGPUActionImpl : public BeagleGPUImpl {55, 9.9}, }; -// std::vector dInstantaneousMatrices; - std::vector> dPartialsWrapper; - // Per category: the action of each child's edge on its partials (exp(tQ) * partials). - // These hold the Taylor sum F while it is accumulated. - std::vector> dDestPLeft; - std::vector> dDestPRight; - // Per category: two buffers that alternate as the input and output of each Taylor term. - std::vector> dTerms[2]; - // Per eigen index: B = Q - mu*I on the device. The edge multiplier goes into the SpMM alpha. + // Per eigen index: B = Q - mu*I on the device. The edge multiplier goes into the term coefficients. std::vector> dBs; - void* dSpMMBuffer = nullptr; - size_t spMMBufferBytes = 0; BlockNormEstimator normEstimator; + // Partials are stored pattern-fastest, [category][state][pattern] (see cuda_ops.h); hPartialsTransposed + // converts to and from BEAGLE's [category][pattern][state]. + std::vector hPartialsTransposed; + // Independent operations (e.g. one level of the tree) are computed together, up to maxBatchOperations. + int maxBatchOperations = 1; + // The actions of both children's edges, [operation][child][category] blocks of (kPaddedStateCount, kPaddedPatternCount). + Real* dActionResults = nullptr; + // 2 or 3 Horner work matrices; each holds one (kPaddedStateCount, kPaddedPatternCount) block per operation, + // child and category (see calcPartialsPartials( )). + Real* dActionScratch = nullptr; + // Per-pattern partial results of the product and root kernels. + Real* dReductionScratch = nullptr; + // All work is issued on the GPUInterface's stream 0, so it is ordered with the base class's // copies and kernels (two different blocking streams would not be). See setCudaDevice( ). cudaStream_t stream = nullptr; @@ -1076,6 +1021,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::kEigenDecompCount; using BeagleGPUImpl::kCategoryCount; using BeagleGPUImpl::kMatrixCount; + using BeagleGPUImpl::kPartialsSize; using BeagleGPUImpl::kPaddedStateCount; @@ -1085,6 +1031,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::kMatrixSize; using BeagleGPUImpl::kernels; using BeagleGPUImpl::dScalingFactors; + using BeagleGPUImpl::kScaleBufferCount; using BeagleGPUImpl::dPartials; using BeagleGPUImpl::dPartialsOrigin; using BeagleGPUImpl::hCategoryRates; @@ -1105,6 +1052,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl using BeagleGPUImpl::dMaxScalingFactors; using BeagleGPUImpl::dIndexMaxScalingFactors; + int addScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex, bool subtract); + int upPartials(bool byPartition, const int *operations, int operationCount, @@ -1139,21 +1088,38 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // and cuBLAS/cuSPARSE use the GPUInterface's stream. void setCudaDevice(); - int getPartialIndex(int nodeIndex, int categoryIndex); + // Copy BEAGLE-layout partials (for one category if tipPartials, else for all) into dPartials[bufferIndex]. + void uploadPartials(int bufferIndex, const Real* inPartials, bool tipPartials); - DnMatrixDevice& getPartialsWrapper(int nodeIndex, int categoryIndex); + struct PartialsOperation + { + int destination; + int children[2]; + int edges[2]; + Real* scalingFactors; // nullptr means no rescaling + }; + + // The operations must be independent: no operation reads or writes another one's destination. + void calcPartialsPartials(const std::vector& operations, Real* cumulativeScalingBuffer); + + // out = exp(multiplier * Q_eigenIndex) * in for one (kPaddedStateCount, kPaddedPatternCount) matrix. + // in is not modified; scratch is 2 or 3 matrices of work space with row stride scratchStride. + struct ActionGroup + { + const Real* in; + Real* out; + Real* scratch[3]; + int scratchCount; + int scratchStride; + int eigenIndex; + double multiplier; + }; - // scalingFactors == nullptr means no rescaling. - void calcPartialsPartials(int destPIndex, - int partials1Index, - int edgeIndex1, - int partials2Index, - int edgeIndex2, - Real* scalingFactors, - Real* cumulativeScalingBuffer); + // With fewer than 32 patterns, the Horner work blocks are interleaved (see calcPartialsPartials( )). + bool interleavedScratch() const { return kPaddedPatternCount < 32; } - // F = exp(t * edgeMultiplier * Q) * inPartials - int simpleAction2(DnMatrixDevice& F, const DnMatrixDevice& inPartials, int edgeIndex, int category); + // Computes all the actions, batching the Taylor terms of the groups that share Q into one launch each. + void computeActions(const std::vector& groups); std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, int eigenIndex) const; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index e0b31bf8..5c26fd1e 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -315,36 +315,15 @@ int BeagleGPUActionImpl::createInstance(int tipCount, } - dPartialsWrapper.resize(kPartialsBufferCount * kCategoryCount); - for (int i = 0; i < kPartialsBufferCount; i++) { - for (int category = 0; category < kCategoryCount; category++) { - auto ptr = (Real *) gpu->CreateSubPointer(dPartialsOrigin, sizeof(Real) * kPaddedStateCount * kPaddedPatternCount * (kCategoryCount * i + category), sizeof(Real) * kPaddedStateCount * kPaddedPatternCount); - auto mat = DnMatrixDevice(cublasHandle, ptr, kPaddedStateCount, kPaddedPatternCount); - dPartialsWrapper[i * kCategoryCount + category] = std::move(mat); - } - } - const size_t partialsSize = (size_t) kPaddedStateCount * kPaddedPatternCount * kCategoryCount; - Real* dDestPLeftPtr = cudaDeviceNew(partialsSize); - Real* dDestPRightPtr = cudaDeviceNew(partialsSize); - // The two children are processed one after the other, so they share the term buffers. - Real* dTermPtrs[2] = {cudaDeviceNew(partialsSize), cudaDeviceNew(partialsSize)}; - - dDestPLeft.clear(); - dDestPRight.clear(); - dTerms[0].clear(); - dTerms[1].clear(); - - for (int category = 0; category < kCategoryCount; category++) - { - size_t offset = kPaddedStateCount * kPaddedPatternCount * category; - - dDestPLeft.emplace_back(cublasHandle, dDestPLeftPtr + offset, kPaddedStateCount, kPaddedPatternCount); - dDestPRight.emplace_back(cublasHandle, dDestPRightPtr + offset, kPaddedStateCount, kPaddedPatternCount); - - for (int k = 0; k < 2; k++) - dTerms[k].emplace_back(cublasHandle, dTermPtrs[k] + offset, kPaddedStateCount, kPaddedPatternCount); - } + // Batching operations only pays off when each is small, so cap the extra memory at about 64 MB. + // (2 results and 2 or 3 scratch blocks per child and category of each operation.) + const int scratchCount = interleavedScratch() ? 3 : 2; + maxBatchOperations = std::max(1, std::min(kMaxPartialsProducts, (64 << 20) / ((2 + 2 * scratchCount) * partialsSize * sizeof(Real)))); + dActionResults = cudaDeviceNew(2 * partialsSize * maxBatchOperations); + dActionScratch = cudaDeviceNew(2 * scratchCount * partialsSize * maxBatchOperations); + const int reductionRows = kCategoryCount * kPaddedStateCount; + dReductionScratch = cudaDeviceNew((size_t) cuda_row_chunk_count(reductionRows, kPaddedPatternCount) * kPaddedPatternCount * maxBatchOperations); hEigenMaps.resize(kMatrixCount); hEdgeMultipliers.resize(kMatrixCount * kCategoryCount); @@ -358,13 +337,9 @@ BeagleGPUActionImpl::~BeagleGPUActionImpl() cudaSetDevice(gpu->GetCudaDeviceOrdinal()); cudaStreamSynchronize(stream); - cudaFree(dSpMMBuffer); - - // The memory for each matrix group was allocation in a single block. - cudaFree(dDestPLeft[0].ptr); - cudaFree(dDestPRight[0].ptr); - cudaFree(dTerms[0][0].ptr); - cudaFree(dTerms[1][0].ptr); + cudaFree(dActionResults); + cudaFree(dActionScratch); + cudaFree(dReductionScratch); for (int i = 0; i < kEigenDecompCount; i++) { cudaFree(dBsCsrOffsetsCache[i]); @@ -420,18 +395,32 @@ int BeagleGPUActionImpl::setTipStates(int tipIndex, const in std::abort(); } +BEAGLE_GPU_TEMPLATE +void BeagleGPUActionImpl::uploadPartials(int bufferIndex, const Real* inPartials, bool tipPartials) +{ + // BEAGLE's layout is [category][pattern][state]; tip partials have one category. Padding stays 0. + hPartialsTransposed.assign(kPartialsSize, 0); + for (int category = 0; category < kCategoryCount; category++) { + const Real* in = inPartials + (tipPartials ? 0 : (size_t) category * kPatternCount * kStateCount); + Real* out = hPartialsTransposed.data() + (size_t) category * kPaddedStateCount * kPaddedPatternCount; + for (int pattern = 0; pattern < kPatternCount; pattern++) + for (int state = 0; state < kStateCount; state++) + out[(size_t) state * kPaddedPatternCount + pattern] = in[(size_t) pattern * kStateCount + state]; + } + gpu->MemcpyHostToDevice(dPartials[bufferIndex], hPartialsTransposed.data(), sizeof(Real) * kPartialsSize); +} + BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setTipPartials(int tipIndex, const Real* inPartials) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUActionImpl::setTipPartials\n"); #endif - BeagleGPUImpl::setTipPartials(tipIndex, inPartials); - - for (int category = 0; category < kCategoryCount; category++) { - dPartialsWrapper[getPartialIndex(tipIndex, category)].setPointer((Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category); - } - + // The base class assigns the tip's device buffer; then overwrite it in the action layout. + int returnCode = BeagleGPUImpl::setTipPartials(tipIndex, inPartials); + if (returnCode != BEAGLE_SUCCESS) + return returnCode; + uploadPartials(tipIndex, inPartials, true); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setTipPartials\n"); @@ -441,16 +430,15 @@ int BeagleGPUActionImpl::setTipPartials(int tipIndex, const } BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::setPartials(int tipIndex, const Real* inPartials) { +int BeagleGPUActionImpl::setPartials(int bufferIndex, const Real* inPartials) { #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tEntering BeagleGPUActionImpl::setPartials\n"); #endif - BeagleGPUImpl::setPartials(tipIndex, inPartials); - for (int category = 0; category < kCategoryCount; category++) { - dPartialsWrapper[getPartialIndex(tipIndex, category)].setPointer((Real*) dPartials[tipIndex] + kPaddedStateCount * kPaddedPatternCount * category); - } - + int returnCode = BeagleGPUImpl::setPartials(bufferIndex, inPartials); + if (returnCode != BEAGLE_SUCCESS) + return returnCode; + uploadPartials(bufferIndex, inPartials, false); #ifdef BEAGLE_DEBUG_FLOW fprintf(stderr, "\tLeaving BeagleGPUActionImpl::setPartials\n"); @@ -459,6 +447,23 @@ int BeagleGPUActionImpl::setPartials(int tipIndex, const Rea return BEAGLE_SUCCESS; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::getPartials(int bufferIndex, int scaleIndex, double* outPartials) { + if (bufferIndex < 0 || bufferIndex >= kPartialsBufferCount) + return BEAGLE_ERROR_OUT_OF_RANGE; + + hPartialsTransposed.resize(kPartialsSize); + gpu->MemcpyDeviceToHost(hPartialsTransposed.data(), dPartials[bufferIndex], sizeof(Real) * kPartialsSize); + for (int category = 0; category < kCategoryCount; category++) { + const Real* in = hPartialsTransposed.data() + (size_t) category * kPaddedStateCount * kPaddedPatternCount; + double* out = outPartials + (size_t) category * kPatternCount * kStateCount; + for (int pattern = 0; pattern < kPatternCount; pattern++) + for (int state = 0; state < kStateCount; state++) + out[(size_t) pattern * kStateCount + state] = in[(size_t) state * kPaddedPatternCount + pattern]; + } + return BEAGLE_SUCCESS; +} + BEAGLE_GPU_TEMPLATE @@ -485,6 +490,45 @@ int BeagleGPUActionImpl::updatePartials(const int* operation return returnCode; } +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::accumulateScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex) +{ + return addScaleFactors(scalingIndices, count, cumulativeScalingIndex, false); +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::removeScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex) +{ + return addScaleFactors(scalingIndices, count, cumulativeScalingIndex, true); +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::addScaleFactors(const int* scalingIndices, int count, int cumulativeScalingIndex, bool subtract) +{ + // Automatic and dynamic scaling keep extra state in the base class. + if (kFlags & (BEAGLE_FLAG_SCALING_AUTO | BEAGLE_FLAG_SCALING_DYNAMIC)) { + return subtract ? BeagleGPUImpl::removeScaleFactors(scalingIndices, count, cumulativeScalingIndex) + : BeagleGPUImpl::accumulateScaleFactors(scalingIndices, count, cumulativeScalingIndex); + } + if (cumulativeScalingIndex < 0 || cumulativeScalingIndex >= kScaleBufferCount) + return BEAGLE_ERROR_OUT_OF_RANGE; + + setCudaDevice(); + const bool scalersLog = (kFlags & BEAGLE_FLAG_SCALERS_LOG) != 0; + ScaleFactorBatch batch; + for (int n = 0; n < count; n++) { + if (scalingIndices[n] < 0 || scalingIndices[n] >= kScaleBufferCount) + return BEAGLE_ERROR_OUT_OF_RANGE; + batch.factors[batch.count++] = (const Real*) dScalingFactors[scalingIndices[n]]; + if (batch.count == 64 or n == count - 1) { + cuda_accumulate_scale_factors(batch, kPaddedPatternCount, (Real*) dScalingFactors[cumulativeScalingIndex], scalersLog, + subtract, stream); + batch.count = 0; + } + } + return BEAGLE_SUCCESS; +} + BEAGLE_GPU_TEMPLATE void BeagleGPUActionImpl::setCudaDevice() { @@ -504,16 +548,6 @@ void BeagleGPUActionImpl::setCudaDevice() } } -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::getPartialIndex(int nodeIndex, int categoryIndex) { - return nodeIndex * kCategoryCount + categoryIndex; -} - -BEAGLE_GPU_TEMPLATE -DnMatrixDevice& BeagleGPUActionImpl::getPartialsWrapper(int nodeIndex, int categoryIndex) { - return dPartialsWrapper[getPartialIndex(nodeIndex, categoryIndex)]; -} - BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::upPartials(bool byPartition, const int *operations, @@ -529,6 +563,16 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, if (cumulativeScalingIndex != BEAGLE_OP_NONE) cumulativeScalingBuffer = (Real*)dScalingFactors[cumulativeScalingIndex]; + // Operations are grouped into levels: each goes to the earliest level after every earlier operation that + // writes a buffer it reads, or reads or writes a buffer (partials or scaling) it writes. The operations of + // a level are independent, so they can be computed together. + std::vector parsed(operationCount); + std::vector levels(operationCount); + std::vector writeLevel(kPartialsBufferCount, -1); + std::vector readLevel(kPartialsBufferCount, -1); + std::map scalingWriteLevel; + int levelCount = 0; + for (int op = 0; op < operationCount; op++) { const int numOps = BEAGLE_OP_COUNT; @@ -559,25 +603,64 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, scalingFactors = (Real*)dScalingFactors[readScalingIndex]; } - calcPartialsPartials(destinationPartialIndex, firstChildPartialIndex, firstChildSubstitutionMatrixIndex, - secondChildPartialIndex, secondChildSubstitutionMatrixIndex, - (rescale == 1) ? scalingFactors : nullptr, cumulativeScalingBuffer); + PartialsOperation& operation = parsed[op]; + operation.destination = destinationPartialIndex; + operation.children[0] = firstChildPartialIndex; + operation.children[1] = secondChildPartialIndex; + operation.edges[0] = firstChildSubstitutionMatrixIndex; + operation.edges[1] = secondChildSubstitutionMatrixIndex; + operation.scalingFactors = (rescale == 1) ? scalingFactors : nullptr; + + int level = std::max(writeLevel[operation.destination], readLevel[operation.destination]) + 1; + for (int child: operation.children) + level = std::max(level, writeLevel[child] + 1); + if (operation.scalingFactors) { + auto it = scalingWriteLevel.find(operation.scalingFactors); + if (it != scalingWriteLevel.end()) + level = std::max(level, it->second + 1); + scalingWriteLevel[operation.scalingFactors] = level; + } + writeLevel[operation.destination] = level; + for (int child: operation.children) + readLevel[child] = std::max(readLevel[child], level); + levels[op] = level; + levelCount = std::max(levelCount, level + 1); + } + std::vector batch; + auto flush = [&]() { + if (batch.empty()) + return; + calcPartialsPartials(batch, cumulativeScalingBuffer); if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { - int parScalingIndex = destinationPartialIndex - kTipCount; - int child1ScalingIndex = firstChildPartialIndex - kTipCount; - int child2ScalingIndex = secondChildPartialIndex - kTipCount; - if (child1ScalingIndex >= 0 && child2ScalingIndex >= 0) { - int scalingIndices[2] = {child1ScalingIndex, child2ScalingIndex}; - BeagleGPUImpl::accumulateScaleFactors(scalingIndices, 2, parScalingIndex); - } else if (child1ScalingIndex >= 0) { - int scalingIndices[1] = {child1ScalingIndex}; - BeagleGPUImpl::accumulateScaleFactors(scalingIndices, 1, parScalingIndex); - } else if (child2ScalingIndex >= 0) { - int scalingIndices[1] = {child2ScalingIndex}; - BeagleGPUImpl::accumulateScaleFactors(scalingIndices, 1, parScalingIndex); + for (const auto& operation: batch) { + int parScalingIndex = operation.destination - kTipCount; + int child1ScalingIndex = operation.children[0] - kTipCount; + int child2ScalingIndex = operation.children[1] - kTipCount; + if (child1ScalingIndex >= 0 && child2ScalingIndex >= 0) { + int scalingIndices[2] = {child1ScalingIndex, child2ScalingIndex}; + accumulateScaleFactors(scalingIndices, 2, parScalingIndex); + } else if (child1ScalingIndex >= 0) { + int scalingIndices[1] = {child1ScalingIndex}; + accumulateScaleFactors(scalingIndices, 1, parScalingIndex); + } else if (child2ScalingIndex >= 0) { + int scalingIndices[1] = {child2ScalingIndex}; + accumulateScaleFactors(scalingIndices, 1, parScalingIndex); + } } } + batch.clear(); + }; + + for (int level = 0; level < levelCount; level++) { + for (int op = 0; op < operationCount; op++) { + if (levels[op] != level) + continue; + if ((int) batch.size() == maxBatchOperations) + flush(); + batch.push_back(parsed[op]); + } + flush(); } #ifdef BEAGLE_DEBUG_SYNCH @@ -706,26 +789,188 @@ int BeagleGPUActionImpl::PrintfDeviceVector(cusparseDnMatDes } BEAGLE_GPU_TEMPLATE -void BeagleGPUActionImpl::calcPartialsPartials(int destPIndex, - int partials1Index, - int edgeIndex1, - int partials2Index, - int edgeIndex2, - Real* scalingFactors, +void BeagleGPUActionImpl::calcPartialsPartials(const std::vector& operations, Real* cumulativeScalingBuffer) { - - for (int category = 0; category < kCategoryCount; category++) - { - simpleAction2(dDestPLeft[category], getPartialsWrapper(partials1Index, category), edgeIndex1, category); - - simpleAction2(dDestPRight[category], getPartialsWrapper(partials2Index, category), edgeIndex2, category); + const size_t matrixSize = (size_t) kPaddedStateCount * kPaddedPatternCount; + // With few patterns, the groups' blocks of the 3 scratch matrices are side by side (coalescing the gathers + // across groups); otherwise each group uses 2 contiguous blocks. See computeActions( ). + const size_t scratchMatrixSize = (size_t) 2 * kCategoryCount * maxBatchOperations * matrixSize; + const bool interleaved = interleavedScratch(); + const size_t scratchStride = interleaved ? (size_t) 2 * kCategoryCount * maxBatchOperations * kPaddedPatternCount : kPaddedPatternCount; + + std::vector groups; + for (size_t op = 0; op < operations.size(); op++) { + for (int child = 0; child < 2; child++) { + const int edgeIndex = operations[op].edges[child]; + for (int category = 0; category < kCategoryCount; category++) { + const size_t g = (op * 2 + child) * kCategoryCount + category; + ActionGroup group; + group.in = (const Real*) dPartials[operations[op].children[child]] + category * matrixSize; + group.out = dActionResults + g * matrixSize; + group.scratchCount = interleaved ? 3 : 2; + for (int k = 0; k < group.scratchCount; k++) + group.scratch[k] = dActionScratch + k * scratchMatrixSize + g * (interleaved ? kPaddedPatternCount : matrixSize); + group.scratchStride = scratchStride; + group.eigenIndex = hEigenMaps[edgeIndex]; + group.multiplier = hEdgeMultipliers[edgeIndex * kCategoryCount + category]; + groups.push_back(group); + } + } } + computeActions(groups); // parent = left * right for all categories, then (optionally) rescale each pattern. bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; - cuda_partials_product(dDestPLeft[0].ptr, dDestPRight[0].ptr, (Real*)dPartials[destPIndex], - kPaddedStateCount, kPaddedPatternCount, kCategoryCount, - scalingFactors, cumulativeScalingBuffer, scalers_log, stream); + PartialsProductBatch products; + for (size_t op = 0; op < operations.size(); op++) { + const Real* left = dActionResults + op * 2 * kCategoryCount * matrixSize; + auto& product = products.products[products.count++]; + product.left = left; + product.right = left + kCategoryCount * matrixSize; + product.parent = (Real*)dPartials[operations[op].destination]; + product.scalingFactors = operations[op].scalingFactors; + } + cuda_partials_products(products, kCategoryCount * kPaddedStateCount, kPaddedPatternCount, + cumulativeScalingBuffer, scalers_log, dReductionScratch, stream); +} + +BEAGLE_GPU_TEMPLATE +void BeagleGPUActionImpl::computeActions(const std::vector& groups) +{ + const size_t matrixSize = (size_t) kPaddedStateCount * kPaddedPatternCount; + + // exp(A) b = eta^s T_m(A/s)^s b, with A = multiplier * B (B = Q - mu I) and eta = exp(multiplier * mu / s). + // Each of the s steps evaluates b <- eta T_m(A/s) b by Horner's rule: + // w = b; for j = m .. 1: w = b + A/(s j) w; then b = eta w. + // A term reads w and b and writes a third matrix; the outputs of a step alternate between `result` (written + // by the last term, j = 1) and `other`. + // - With 3 scratch matrices, result and other are the two that are not b, except that the last term of the + // last step writes out. So the gathers read only scratch, whose groups are side by side. + // - With 2, out is also work space: the last step's result is out, and earlier steps write their results + // to scratch, so the last step's b is never out. This keeps the working set smaller (L2). + struct Schedule + { + const ActionGroup* group; + int nCols; + int m; + int s; + Real eta; + int step; + int term; + const Real* b; + const Real* w; + Real* result; + Real* other; + + int stride(const Real* matrix) const + { + for (int k = 0; k < group->scratchCount; k++) + if (matrix == group->scratch[k]) + return group->scratchStride; + return nCols; + } + + void startStep() + { + Real* free[2] = {nullptr, nullptr}; + int n = 0; + for (int k = 0; k < group->scratchCount; k++) + if (group->scratch[k] != b and n < 2) + free[n++] = group->scratch[k]; + if (group->scratchCount == 3) { + result = free[0]; + other = free[1]; + } else if (step == s - 1) { + result = group->out; + other = free[0]; + } else { + result = free[0]; + other = (n == 1) ? group->out : free[1]; + } + w = b; + term = m; + } + + Real* output() const + { + if (group->scratchCount == 3 and step == s - 1) + return (term == 1) ? group->out : (term % 2 == 0) ? result : other; + return (term % 2 == 1) ? result : other; + } + }; + + std::vector done(groups.size(), false); + for (size_t first = 0; first < groups.size(); first++) { + if (done[first]) + continue; + + // All the groups that share this group's B run in lockstep: one launch per term. + const int eigenIndex = groups[first].eigenIndex; + const auto& B = dBs[eigenIndex]; + std::vector schedules; + for (size_t i = first; i < groups.size(); i++) { + if (done[i] or groups[i].eigenIndex != eigenIndex) + continue; + done[i] = true; + const ActionGroup& group = groups[i]; + auto [m, s] = getStatistics2(1.0, kPaddedPatternCount, group.multiplier, eigenIndex); + if (m == 0) { + auto status = cudaMemcpyAsync(group.out, group.in, matrixSize * sizeof(Real), cudaMemcpyDeviceToDevice, stream); + if (status != cudaSuccess) + throw std::runtime_error(std::string("cudaMemcpyAsync: ") + cudaGetErrorString(status)); + continue; + } + Schedule schedule; + schedule.group = &group; + schedule.nCols = kPaddedPatternCount; + schedule.m = m; + schedule.s = s; + schedule.eta = exp(hMuBs[eigenIndex] * group.multiplier / s); + schedule.step = 0; + schedule.b = group.in; + schedule.startStep(); + schedules.push_back(schedule); + +#ifdef BEAGLE_DEBUG_FLOW + std::cerr << "action: m = " << m << " s = " << s << " multiplier = " << group.multiplier << " eigenIndex = " << eigenIndex << std::endl; +#endif + } + + for (bool active = true; active;) { + active = false; + TaylorTermBatch batch; + for (auto& schedule: schedules) { + if (schedule.step == schedule.s) + continue; + active = true; + + const int j = schedule.term; + Real* out = schedule.output(); + auto& term = batch.groups[batch.count++]; + term.x = schedule.w; + term.b = schedule.b; + term.out = out; + term.alpha = schedule.group->multiplier / ((Real) schedule.s * j); + term.scale = (j == 1) ? schedule.eta : 1; + term.ldx = schedule.stride(term.x); + term.ldb = schedule.stride(term.b); + term.ldOut = schedule.stride(term.out); + + if (batch.count == kMaxTaylorGroups) { + cuda_taylor_horner_terms(kPaddedStateCount, kPaddedPatternCount, B.offsets, B.inner, B.values, batch, stream); + batch.count = 0; + } + + schedule.w = out; + if (--schedule.term == 0) { + schedule.b = out; + if (++schedule.step < schedule.s) + schedule.startStep(); + } + } + cuda_taylor_horner_terms(kPaddedStateCount, kPaddedPatternCount, B.offsets, B.inner, B.values, batch, stream); + } + } } BEAGLE_GPU_TEMPLATE @@ -848,12 +1093,12 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i Real* categoryWeights = (Real*)dWeights[categoryWeightsIndex]; Real* stateFrequencies = (Real*)dFrequencies[stateFrequenciesIndex]; // these really should be category-dependent - // siteLogLikelihoods[pattern] = log( sum_{category,state} likelihood(category,pattern,state) * categoryWeights[category] * statesFrequencies[state] ) + // siteLogLikelihoods[pattern] = log( sum_{category,state} likelihood(category,state,pattern) * categoryWeights[category] * statesFrequencies[state] ) // + scaling factor // dSumLogLikelihood[0] = sum_{pattern} patternWeights[pattern] * siteLogLikelihoods[pattern] cuda_root_log_likelihoods(rootPartials, categoryWeights, stateFrequencies, scale ? dCumulativeScalingFactor : nullptr, (Real*)dPatternWeights, kPaddedStateCount, kPaddedPatternCount, kPatternCount, kCategoryCount, - siteLogLikelihoods, (Real*)dSumLogLikelihood, stream); + dReductionScratch, siteLogLikelihoods, (Real*)dSumLogLikelihood, stream); #ifdef BEAGLE_DEBUG_VALUES std::cerr<<"site log likelihoods = "< BeagleGPUActionImpl::getStatistics2(doub return {m,s}; } -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::simpleAction2(DnMatrixDevice& F, const DnMatrixDevice& inPartials, int edgeIndex, int category) { - const double tol = pow(2.0, -53.0); - const double t = 1.0; - const int nCol = kPaddedPatternCount; - const int edgeMultiplierIndex = edgeIndex * kCategoryCount + category; - const int eigenIndex = hEigenMaps[edgeIndex]; - - const double edgeMultiplier = hEdgeMultipliers[edgeMultiplierIndex]; - - auto [m, s] = getStatistics2(t, nCol, edgeMultiplier, eigenIndex); - - -#ifdef BEAGLE_DEBUG_FLOW - std::cerr << "\n\nsimpleAction2: m = " << m << " s = " << s << " t = " << t << " nCol = " << nCol << " edgeMultiplier = " << edgeMultiplier << std::endl; - std::cerr << " edgeIndex = " << edgeIndex << " category = " << category << std::endl; -#endif - - // A = edgeMultiplier * B, so A is never formed: the multiplier goes into the SpMM alpha. - auto& B = dBs[eigenIndex]; - - // For small problems one fused kernel per term beats cuSPARSE's 3 kernels + axpy, which are - // launch-bound there. (A100: 7 vs 21 us per term for IGC at 1 column; cuSPARSE wins from - // about nnz * columns = 3e6, e.g. 53 vs 67 us for MM M=32 at 64 columns.) - const bool fusedTerms = (double) B.num_non_zeros * kPaddedPatternCount < 3e6; - - // F accumulates the Taylor sum in place. Each term is written to one of the two term buffers and - // is the input of the next term, so no copies are needed. - F.copyFromAsync( inPartials, stream ); - - const Real eta = exp(t * hMuBs[eigenIndex] * edgeMultiplier / (Real) s); - - for (int i = 0; i < s; i++) { - - // The first term of each step reads the current F. - const DnMatrixDevice* x = (i == 0) ? &inPartials : &F; - bool scaled = false; - - // The fused kernel updates F while other threads read x, so x must not be F. - // (Term j=1 writes dTerms[1], so dTerms[0] is free here.) - if (fusedTerms and x == &F) - { - dTerms[0][category].copyFromAsync(F, stream); - x = &dTerms[0][category]; - } - -#ifdef ACTION_EARLY_EXIT_NORM - Real c1 = normPInf(*x); -#endif - - for (int j = 1; j < m + 1; j++) { - - auto& y = dTerms[j % 2][category]; - const Real alpha = edgeMultiplier * t / ((Real) s * j); - - // y = alpha * B * x; F += y, folding in F *= eta after the last term. - const Real scale = (j < m) ? 1 : eta; - if (fusedTerms) - cuda_csr_taylor_term(kPaddedStateCount, kPaddedPatternCount, B.offsets, B.inner, B.values, alpha, - x->ptr, y.ptr, F.ptr, scale, stream); - else - { - spMMPrepared(y, alpha, B, *x, dSpMMBuffer); - if (j < m) - F += y; - else - F.axpby(eta, eta, y); - } - scaled = (j == m); - -#ifdef BEAGLE_DEBUG_FLOW - std::cerr<<"j/m = "<::upPrePartials(bool byPartition, const int *operations, @@ -1117,16 +1266,6 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, sparseFormat::csr); auto& dB = dBs[matrixIndex]; - // Size the work buffer for the Taylor-term SpMMs once per matrix, not on every call. - size_t newSpMMBufferSize = spMMBufferSize(dTerms[0][0], dB, dTerms[1][0]); - if (newSpMMBufferSize > spMMBufferBytes or dSpMMBuffer == nullptr) - { - CHECK_CUDA(cudaStreamSynchronize(stream)); - cudaDeviceDelete(dSpMMBuffer); - spMMBufferBytes = std::max(newSpMMBufferSize, 1); - dSpMMBuffer = cudaMallocWrapped(spMMBufferBytes); - } - // equation 3.7 in Al-Mohy and Higham: d_p = ||B^p||_1^(1/p). d_1 is known exactly; d_0 is not used. const int pTop = getPMax() + 1; auto norms = normEstimator.estimate(dB, stream); diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 570d5968..297cfe10 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -1,5 +1,8 @@ #include "cuda_ops.h" +#include +#include + #include #include #include @@ -67,19 +70,13 @@ void initialize_norm_x_matrix(double* data, int n, int m) // --------------------------------------------------------------------------------------------------- -// Kernels for the per-node product/rescale and the root likelihood. They replace the Thrust -// reduce_by_key chains, which allocate temporary memory and block the host on every call. +// Kernels for the Taylor terms, the per-node product/rescale and the root likelihood. // -// Partials are stored as [category][pattern][state] (state fastest), with the padded counts. -// One block handles one pattern. - -// With few patterns there are few blocks, so give each block more threads. -inline int patternBlockSize(int nPatterns) -{ - return (nPatterns >= 256) ? 128 : 1024; -} +// Action partials are stored pattern-fastest: [category][state][pattern], i.e. a (nCategories*nStates) +// x nPatterns row-major matrix, so a CSR row of Q gathers whole rows of x (coalesced over patterns). // Reduce v over the block with op; identity must be neutral for op. Every thread gets the result. +// Only for one-dimensional blocks. template __device__ T block_reduce(T v, T identity, Op op) { @@ -109,179 +106,442 @@ __device__ T block_reduce(T v, T identity, Op op) struct MaxOp { template __device__ T operator()(T a, T b) const { return max(a, b); } }; struct SumOp { template __device__ T operator()(T a, T b) const { return a + b; } }; -// parent = left * right (elementwise). If scalingFactors is non-null, also divide each pattern by its -// largest entry m (1 if all are 0), store m (or log m) in scalingFactors, and add log m to cumulative. -template -__global__ void partials_product_kernel(const T* left, const T* right, T* parent, - int nStates, int nPatterns, int nCategories, - T* scalingFactors, T* cumulative, bool scalersLog) +static int nextPowerOfTwo(int n, int cap) { - const int pattern = blockIdx.x; - const size_t categoryStride = (size_t)nStates * nPatterns; - const size_t offset = (size_t)pattern * nStates; + int p = 1; + while (p < n and p < cap) p *= 2; + return p; +} - if (not scalingFactors) +// One step w <- scale * (b + alpha * A * x) of Horner's rule for the Taylor polynomial, for each group of the +// batch. A is (nRows, nRows) in CSR; x, b and out are (nRows, nCols) row-major. +// +// For nCols >= 32: blockIdx.z is the group, the 32 lanes of a warp cover 32 adjacent columns of one row, and +// each thread does CPT columns 32 apart. Blocks are ordered rows-fastest, so the blocks in flight read the +// same column panel of x and it stays in L2. +template +__global__ void __launch_bounds__(256) taylor_horner_term_kernel(int nRows, int nCols, const int* __restrict__ rowOffsets, + const int* __restrict__ columns, const T* __restrict__ values, + const TaylorTermBatch batch) +{ + const TaylorTermGroup& g = batch.groups[blockIdx.z]; + const int row = blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32; + const int col = blockIdx.y * (32 * CPT) + threadIdx.x % 32; + if (row >= nRows) return; + + const int kBegin = rowOffsets[row]; + const int kEnd = rowOffsets[row + 1]; + const T* b = g.b + (size_t)row * g.ldb + col; + T* out = g.out + (size_t)row * g.ldOut + col; + const T* x = g.x + col; + T acc[CPT]; + T own[CPT]; + + if (col + (CPT - 1) * 32 < nCols) { - for (int c = 0; c < nCategories; c++) - for (int s = threadIdx.x; s < nStates; s += blockDim.x) - { - const size_t i = c * categoryStride + offset + s; - parent[i] = left[i] * right[i]; - } - return; +#pragma unroll + for (int q = 0; q < CPT; q++) + { + own[q] = b[q * 32]; + acc[q] = 0; + } + for (int k = kBegin; k < kEnd; k++) + { + const T v = values[k]; + const T* xRow = x + (size_t)columns[k] * g.ldx; +#pragma unroll + for (int q = 0; q < CPT; q++) + acc[q] += v * __ldg(xRow + q * 32); + } +#pragma unroll + for (int q = 0; q < CPT; q++) + out[q * 32] = g.scale * (own[q] + g.alpha * acc[q]); } - - T m = 0; - for (int c = 0; c < nCategories; c++) - for (int s = threadIdx.x; s < nStates; s += blockDim.x) + else + { +#pragma unroll + for (int q = 0; q < CPT; q++) { - const size_t i = c * categoryStride + offset + s; - m = max(m, left[i] * right[i]); + own[q] = (col + q * 32 < nCols) ? b[q * 32] : T(0); + acc[q] = 0; } - m = block_reduce(m, T(0), MaxOp()); - if (m == 0) m = 1; - - const T inverse = 1 / m; - for (int c = 0; c < nCategories; c++) - for (int s = threadIdx.x; s < nStates; s += blockDim.x) + for (int k = kBegin; k < kEnd; k++) { - const size_t i = c * categoryStride + offset + s; - parent[i] = left[i] * right[i] * inverse; + const T v = values[k]; + const T* xRow = x + (size_t)columns[k] * g.ldx; +#pragma unroll + for (int q = 0; q < CPT; q++) + if (col + q * 32 < nCols) + acc[q] += v * __ldg(xRow + q * 32); } +#pragma unroll + for (int q = 0; q < CPT; q++) + if (col + q * 32 < nCols) + out[q * 32] = g.scale * (own[q] + g.alpha * acc[q]); + } +} + +// For nCols < 32: the columns of all groups are numbered v = group * nCols + column, and LANES consecutive +// lanes share a row and cover LANES consecutive v. So the lanes share each (column, value) load of A, and +// when the groups' matrices are stored side by side the gathers of x are coalesced across groups too. +template +__global__ void __launch_bounds__(256) taylor_horner_term_narrow_kernel(int nRows, int nCols, const int* __restrict__ rowOffsets, + const int* __restrict__ columns, const T* __restrict__ values, + const TaylorTermBatch batch) +{ + // The lanes read different groups, which shared memory serves without serializing like parameter space. + __shared__ TaylorTermGroup groups[kMaxTaylorGroups]; + for (int i = threadIdx.x; i < batch.count; i += blockDim.x) + groups[i] = batch.groups[i]; + __syncthreads(); + + const int lane = threadIdx.x % 32; + const int row = (blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32) * (32 / LANES) + lane / LANES; + const int v = blockIdx.y * LANES + lane % LANES; + if (row >= nRows or v >= batch.count * nCols) return; + + const TaylorTermGroup& g = groups[v / nCols]; + const int col = v % nCols; + const T* x = g.x + col; + const T own = g.b[(size_t)row * g.ldb + col]; + T acc = 0; + for (int k = rowOffsets[row]; k < rowOffsets[row + 1]; k++) + acc += values[k] * __ldg(x + (size_t)columns[k] * g.ldx); + g.out[(size_t)row * g.ldOut + col] = g.scale * (own + g.alpha * acc); +} + +template +void launch_taylor_horner_terms(int nRows, int nCols, const int* rowOffsets, const int* columns, const T* values, + const TaylorTermBatch& batch, cudaStream_t stream) +{ + const int threads = 256; + const dim3 grid((nRows + threads / 32 - 1) / (threads / 32), (nCols + 32 * CPT - 1) / (32 * CPT), batch.count); + taylor_horner_term_kernel<<>>(nRows, nCols, rowOffsets, columns, values, batch); +} - if (threadIdx.x == 0) +template +void launch_taylor_horner_terms_narrow(int nRows, int nCols, const int* rowOffsets, const int* columns, const T* values, + const TaylorTermBatch& batch, cudaStream_t stream) +{ + const int threads = 256; + const int rowsPerBlock = (threads / 32) * (32 / LANES); + const dim3 grid((nRows + rowsPerBlock - 1) / rowsPerBlock, (batch.count * nCols + LANES - 1) / LANES); + taylor_horner_term_narrow_kernel<<>>(nRows, nCols, rowOffsets, columns, values, batch); +} + +// Columns per thread for nCols >= 32; BEAGLE_ACTION_CPT (1, 2 or 4) overrides it for tuning. +static int columnsPerThread(int nCols) +{ + static const int forced = getenv("BEAGLE_ACTION_CPT") ? atoi(getenv("BEAGLE_ACTION_CPT")) : 0; + if (forced == 1 or forced == 2 or forced == 4) + return forced; + return (nCols >= 256) ? 4 : (nCols >= 64) ? 2 : 1; +} + +template +void cuda_taylor_horner_terms_impl(int nRows, int nCols, const int* rowOffsets, const int* columns, const T* values, + const TaylorTermBatch& batch, cudaStream_t stream) +{ + if (batch.count == 0) return; + if (nCols >= 32) { - const T logM = log(m); - scalingFactors[pattern] = scalersLog ? logM : m; - if (cumulative) cumulative[pattern] += logM; + switch (columnsPerThread(nCols)) + { + case 4: launch_taylor_horner_terms(nRows, nCols, rowOffsets, columns, values, batch, stream); break; + case 2: launch_taylor_horner_terms(nRows, nCols, rowOffsets, columns, values, batch, stream); break; + default: launch_taylor_horner_terms(nRows, nCols, rowOffsets, columns, values, batch, stream); break; + } + return; + } + switch (nextPowerOfTwo(batch.count * nCols, 32)) + { + case 1: launch_taylor_horner_terms_narrow(nRows, nCols, rowOffsets, columns, values, batch, stream); break; + case 2: launch_taylor_horner_terms_narrow(nRows, nCols, rowOffsets, columns, values, batch, stream); break; + case 4: launch_taylor_horner_terms_narrow(nRows, nCols, rowOffsets, columns, values, batch, stream); break; + case 8: launch_taylor_horner_terms_narrow(nRows, nCols, rowOffsets, columns, values, batch, stream); break; + case 16: launch_taylor_horner_terms_narrow(nRows, nCols, rowOffsets, columns, values, batch, stream); break; + default: launch_taylor_horner_terms_narrow(nRows, nCols, rowOffsets, columns, values, batch, stream); break; } } -template -void cuda_partials_product_impl(const T* left, const T* right, T* parent, int nStates, int nPatterns, int nCategories, - T* scalingFactors, T* cumulative, bool scalersLog, cudaStream_t stream) +void cuda_taylor_horner_terms(int nRows, int nCols, const int* rowOffsets, const int* columns, const float* values, + const TaylorTermBatch& batch, cudaStream_t stream) { - partials_product_kernel<<>>(left, right, parent, nStates, nPatterns, nCategories, - scalingFactors, cumulative, scalersLog); + cuda_taylor_horner_terms_impl(nRows, nCols, rowOffsets, columns, values, batch, stream); } -void cuda_partials_product(const float* left, const float* right, float* parent, int nStates, int nPatterns, int nCategories, - float* scalingFactors, float* cumulative, bool scalersLog, cudaStream_t stream) +void cuda_taylor_horner_terms(int nRows, int nCols, const int* rowOffsets, const int* columns, const double* values, + const TaylorTermBatch& batch, cudaStream_t stream) { - cuda_partials_product_impl(left, right, parent, nStates, nPatterns, nCategories, scalingFactors, cumulative, scalersLog, stream); + cuda_taylor_horner_terms_impl(nRows, nCols, rowOffsets, columns, values, batch, stream); } -void cuda_partials_product(const double* left, const double* right, double* parent, int nStates, int nPatterns, int nCategories, - double* scalingFactors, double* cumulative, bool scalersLog, cudaStream_t stream) +// The per-pattern reductions over the rows of an (nRows, nPatterns) row-major matrix use blocks of +// patternsPerBlock x (256 / patternsPerBlock) threads, and split the rows into chunks so that there are +// enough blocks. Each chunk writes one partial result per pattern; a second kernel combines them. +struct RowChunks +{ + int patternsPerBlock; + int rowsPerBlock; + int patternBlocks; + int rowsPerChunk; + int chunks; +}; + +static RowChunks rowChunks(int nRows, int nPatterns) { - cuda_partials_product_impl(left, right, parent, nStates, nPatterns, nCategories, scalingFactors, cumulative, scalersLog, stream); + RowChunks c; + c.patternsPerBlock = nextPowerOfTwo(nPatterns, 32); + c.rowsPerBlock = 256 / c.patternsPerBlock; + c.patternBlocks = (nPatterns + c.patternsPerBlock - 1) / c.patternsPerBlock; + const int targetBlocks = 512; + int chunks = (targetBlocks + c.patternBlocks - 1) / c.patternBlocks; + chunks = std::max(1, std::min({chunks, nRows / (8 * c.rowsPerBlock), 128})); + c.rowsPerChunk = (nRows + chunks - 1) / chunks; + c.chunks = (nRows + c.rowsPerChunk - 1) / c.rowsPerChunk; + return c; } -// siteLogLikelihoods[p] = log( sum_{c,s} root(c,p,s) * categoryWeights[c] * frequencies[s] ) + cumulativeScaling[p] +int cuda_row_chunk_count(int nRows, int nPatterns) +{ + return rowChunks(nRows, nPatterns).chunks; +} + +// Reduce v over threadIdx.y (blockDim.y a power of two); the result is valid in the threads with threadIdx.y == 0. +template +__device__ T column_reduce(T v, Op op) +{ + __shared__ T shared[256]; + const int i = threadIdx.y * blockDim.x + threadIdx.x; + shared[i] = v; + __syncthreads(); + for (int stride = blockDim.y / 2; stride > 0; stride /= 2) + { + if (threadIdx.y < stride) + shared[i] = op(shared[i], shared[i + stride * blockDim.x]); + __syncthreads(); + } + return shared[threadIdx.x]; +} + +// parent = left * right for each product of the batch (blockIdx.y) template -__global__ void root_site_log_likelihoods_kernel(const T* root, const T* categoryWeights, const T* frequencies, - const T* cumulativeScaling, int nStates, int nPaddedPatterns, int nCategories, - T* siteLogLikelihoods) +__global__ void partials_product_kernel(const PartialsProductBatch batch, size_t n) { - const int pattern = blockIdx.x; - const size_t categoryStride = (size_t)nStates * nPaddedPatterns; - const size_t offset = (size_t)pattern * nStates; + const PartialsProduct& product = batch.products[blockIdx.y]; + for (size_t i = blockIdx.x * (size_t)blockDim.x + threadIdx.x; i < n; i += (size_t)gridDim.x * blockDim.x) + product.parent[i] = product.left[i] * product.right[i]; +} - T sum = 0; - for (int c = 0; c < nCategories; c++) +// chunkMax[product, chunk, p] = max over the chunk's rows r of left[r, p] * right[r, p] (product = blockIdx.z) +template +__global__ void partials_product_max_kernel(const PartialsProductBatch batch, int nRows, int nPatterns, int rowsPerChunk, T* chunkMax) +{ + const PartialsProduct& product = batch.products[blockIdx.z]; + const int p = blockIdx.x * blockDim.x + threadIdx.x; + const int rowEnd = min(nRows, (blockIdx.y + 1) * rowsPerChunk); + T m = 0; + if (p < nPatterns and product.scalingFactors) + for (int r = blockIdx.y * rowsPerChunk + threadIdx.y; r < rowEnd; r += blockDim.y) + { + const size_t i = (size_t)r * nPatterns + p; + m = max(m, product.left[i] * product.right[i]); + } + m = column_reduce(m, MaxOp()); + if (threadIdx.y == 0 and p < nPatterns) + chunkMax[((size_t)blockIdx.z * gridDim.y + blockIdx.y) * nPatterns + p] = m; +} + +// parent = left * right / m, where m is the largest entry of the pattern (1 if all are 0, or without rescaling). +template +__global__ void partials_product_scale_kernel(const PartialsProductBatch batch, int nRows, int nPatterns, int rowsPerChunk, + const T* chunkMax, bool scalersLog) +{ + const PartialsProduct& product = batch.products[blockIdx.z]; + const int p = blockIdx.x * blockDim.x + threadIdx.x; + + // One row of threads combines the chunks' maxima for the block's patterns. + __shared__ T patternMax[32]; + if (threadIdx.y == 0) { - T categorySum = 0; - for (int s = threadIdx.x; s < nStates; s += blockDim.x) - categorySum += root[c * categoryStride + offset + s] * frequencies[s]; - sum += categorySum * categoryWeights[c]; + T m = 0; + if (p < nPatterns) + for (int c = 0; c < gridDim.y; c++) + m = max(m, chunkMax[((size_t)blockIdx.z * gridDim.y + c) * nPatterns + p]); + patternMax[threadIdx.x] = (m == 0) ? T(1) : m; + } + __syncthreads(); + if (p >= nPatterns) return; + const T m = patternMax[threadIdx.x]; + const T inverse = 1 / m; + + const int rowEnd = min(nRows, (blockIdx.y + 1) * rowsPerChunk); + for (int r = blockIdx.y * rowsPerChunk + threadIdx.y; r < rowEnd; r += blockDim.y) + { + const size_t i = (size_t)r * nPatterns + p; + product.parent[i] = product.left[i] * product.right[i] * inverse; } - sum = block_reduce(sum, T(0), SumOp()); - if (threadIdx.x == 0) - siteLogLikelihoods[pattern] = log(sum) + (cumulativeScaling ? cumulativeScaling[pattern] : T(0)); + if (product.scalingFactors and blockIdx.y == 0 and threadIdx.y == 0) + product.scalingFactors[p] = scalersLog ? log(m) : m; } -// out[0] = sum_i values[i] * weights[i], using a single block (deterministic). +// cumulative[p] += log m of each product in turn (the order of separate calls, so the rounding is the same). template -__global__ void weighted_sum_kernel(const T* values, const T* weights, int n, T* out) +__global__ void accumulate_scaling_kernel(const PartialsProductBatch batch, int nPatterns, T* cumulative, bool scalersLog) { - T sum = 0; - for (int i = threadIdx.x; i < n; i += blockDim.x) - sum += values[i] * weights[i]; - sum = block_reduce(sum, T(0), SumOp()); - if (threadIdx.x == 0) out[0] = sum; + const int p = blockIdx.x * blockDim.x + threadIdx.x; + if (p >= nPatterns) return; + T sum = cumulative[p]; + for (int k = 0; k < batch.count; k++) + { + const T* factors = batch.products[k].scalingFactors; + if (factors) + sum += scalersLog ? factors[p] : log(factors[p]); + } + cumulative[p] = sum; } template -void cuda_root_log_likelihoods_impl(const T* root, const T* categoryWeights, const T* frequencies, const T* cumulativeScaling, - const T* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, - T* siteLogLikelihoods, T* sumLogLikelihood, cudaStream_t stream) +void cuda_partials_products_impl(const PartialsProductBatch& batch, int nRows, int nPatterns, T* cumulative, bool scalersLog, + T* scratch, cudaStream_t stream) { - root_site_log_likelihoods_kernel<<>>(root, categoryWeights, frequencies, cumulativeScaling, - nStates, nPaddedPatterns, nCategories, siteLogLikelihoods); - weighted_sum_kernel<<<1, 1024, 0, stream>>>(siteLogLikelihoods, patternWeights, nPatterns, sumLogLikelihood); + if (batch.count == 0) return; + bool rescale = false; + for (int k = 0; k < batch.count; k++) + rescale = rescale or batch.products[k].scalingFactors; + + if (not rescale) + { + const size_t n = (size_t)nRows * nPatterns; + const int blocks = (int)std::min((n + 255) / 256, 4096 / batch.count + 1); + partials_product_kernel<<>>(batch, n); + return; + } + + const RowChunks c = rowChunks(nRows, nPatterns); + const dim3 block(c.patternsPerBlock, c.rowsPerBlock); + const dim3 grid(c.patternBlocks, c.chunks, batch.count); + partials_product_max_kernel<<>>(batch, nRows, nPatterns, c.rowsPerChunk, scratch); + partials_product_scale_kernel<<>>(batch, nRows, nPatterns, c.rowsPerChunk, scratch, scalersLog); + if (cumulative) + accumulate_scaling_kernel<<<(nPatterns + 255) / 256, 256, 0, stream>>>(batch, nPatterns, cumulative, scalersLog); } -void cuda_root_log_likelihoods(const float* root, const float* categoryWeights, const float* frequencies, const float* cumulativeScaling, - const float* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, - float* siteLogLikelihoods, float* sumLogLikelihood, cudaStream_t stream) +void cuda_partials_products(const PartialsProductBatch& batch, int nRows, int nPatterns, float* cumulative, bool scalersLog, + float* scratch, cudaStream_t stream) { - cuda_root_log_likelihoods_impl(root, categoryWeights, frequencies, cumulativeScaling, patternWeights, - nStates, nPaddedPatterns, nPatterns, nCategories, siteLogLikelihoods, sumLogLikelihood, stream); + cuda_partials_products_impl(batch, nRows, nPatterns, cumulative, scalersLog, scratch, stream); } -void cuda_root_log_likelihoods(const double* root, const double* categoryWeights, const double* frequencies, const double* cumulativeScaling, - const double* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, - double* siteLogLikelihoods, double* sumLogLikelihood, cudaStream_t stream) +void cuda_partials_products(const PartialsProductBatch& batch, int nRows, int nPatterns, double* cumulative, bool scalersLog, + double* scratch, cudaStream_t stream) { - cuda_root_log_likelihoods_impl(root, categoryWeights, frequencies, cumulativeScaling, patternWeights, - nStates, nPaddedPatterns, nPatterns, nCategories, siteLogLikelihoods, sumLogLikelihood, stream); + cuda_partials_products_impl(batch, nRows, nPatterns, cumulative, scalersLog, scratch, stream); } +template +__global__ void accumulate_scale_factors_kernel(const ScaleFactorBatch batch, int nPatterns, T* cumulative, bool scalersLog, bool subtract) +{ + const int p = blockIdx.x * blockDim.x + threadIdx.x; + if (p >= nPatterns) return; + T total = 0; + for (int k = 0; k < batch.count; k++) + { + const T factor = batch.factors[k][p]; + if (scalersLog) + total += factor; + else if (factor != 1) + total += log(factor); + } + cumulative[p] += subtract ? -total : total; +} -// One Taylor term on column-major (nStates, nColumns) matrices, in a single pass: -// y = alpha * A * x; F = scale * (F + y) -// One thread per (row, column), consecutive threads on consecutive rows. Used when the work is small: -// it replaces cuSPARSE's partition, beta=0 fill and SpMM kernels plus the axpy. template -__global__ void csr_taylor_term_kernel(int nStates, int nColumns, const int* __restrict__ rowOffsets, - const int* __restrict__ columns, const T* __restrict__ values, T alpha, - const T* __restrict__ x, T* __restrict__ y, T* __restrict__ F, T scale) +void cuda_accumulate_scale_factors_impl(const ScaleFactorBatch& batch, int nPatterns, T* cumulative, bool scalersLog, + bool subtract, cudaStream_t stream) { - const size_t index = blockIdx.x * (size_t)blockDim.x + threadIdx.x; - if (index >= (size_t)nStates * nColumns) return; - const int row = index % nStates; - const T* xColumn = x + (index - row); + if (batch.count == 0) return; + accumulate_scale_factors_kernel<<<(nPatterns + 255) / 256, 256, 0, stream>>>(batch, nPatterns, cumulative, scalersLog, subtract); +} + +void cuda_accumulate_scale_factors(const ScaleFactorBatch& batch, int nPatterns, float* cumulative, bool scalersLog, + bool subtract, cudaStream_t stream) +{ + cuda_accumulate_scale_factors_impl(batch, nPatterns, cumulative, scalersLog, subtract, stream); +} + +void cuda_accumulate_scale_factors(const ScaleFactorBatch& batch, int nPatterns, double* cumulative, bool scalersLog, + bool subtract, cudaStream_t stream) +{ + cuda_accumulate_scale_factors_impl(batch, nPatterns, cumulative, scalersLog, subtract, stream); +} +// chunkSum[chunk, p] = sum over the chunk's rows r = (c, s) of root[r, p] * categoryWeights[c] * frequencies[s] +template +__global__ void root_chunk_sums_kernel(const T* root, const T* categoryWeights, const T* frequencies, int nStates, int nRows, + int nPatterns, int rowsPerChunk, T* chunkSum) +{ + const int p = blockIdx.x * blockDim.x + threadIdx.x; + const int rowEnd = min(nRows, (blockIdx.y + 1) * rowsPerChunk); T sum = 0; - for (int k = rowOffsets[row]; k < rowOffsets[row + 1]; k++) - sum += values[k] * __ldg(xColumn + columns[k]); - sum *= alpha; + if (p < nPatterns) + for (int r = blockIdx.y * rowsPerChunk + threadIdx.y; r < rowEnd; r += blockDim.y) + sum += root[(size_t)r * nPatterns + p] * categoryWeights[r / nStates] * frequencies[r % nStates]; + sum = column_reduce(sum, SumOp()); + if (threadIdx.y == 0 and p < nPatterns) + chunkSum[blockIdx.y * nPatterns + p] = sum; +} - y[index] = sum; - F[index] = scale * (F[index] + sum); +// siteLogLikelihoods[p] = log(sum of the chunk sums) + cumulativeScaling[p]; out[0] = sum_p patternWeights[p] * siteLogLikelihoods[p]. +// A single block, so the sum is deterministic. +template +__global__ void root_finish_kernel(const T* chunkSum, int nChunks, int nPaddedPatterns, int nPatterns, const T* cumulativeScaling, + const T* patternWeights, T* siteLogLikelihoods, T* out) +{ + T total = 0; + for (int p = threadIdx.x; p < nPatterns; p += blockDim.x) + { + T sum = 0; + for (int c = 0; c < nChunks; c++) + sum += chunkSum[c * nPaddedPatterns + p]; + const T site = log(sum) + (cumulativeScaling ? cumulativeScaling[p] : T(0)); + siteLogLikelihoods[p] = site; + total += site * patternWeights[p]; + } + total = block_reduce(total, T(0), SumOp()); + if (threadIdx.x == 0) out[0] = total; } template -void cuda_csr_taylor_term_impl(int nStates, int nColumns, const int* rowOffsets, const int* columns, const T* values, T alpha, - const T* x, T* y, T* F, T scale, cudaStream_t stream) +void cuda_root_log_likelihoods_impl(const T* root, const T* categoryWeights, const T* frequencies, const T* cumulativeScaling, + const T* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, + T* scratch, T* siteLogLikelihoods, T* sumLogLikelihood, cudaStream_t stream) { - const size_t n = (size_t)nStates * nColumns; - const int blockSize = 256; - csr_taylor_term_kernel<<<(n + blockSize - 1) / blockSize, blockSize, 0, stream>>>(nStates, nColumns, rowOffsets, columns, values, - alpha, x, y, F, scale); + const int nRows = nStates * nCategories; + const RowChunks c = rowChunks(nRows, nPaddedPatterns); + const dim3 block(c.patternsPerBlock, c.rowsPerBlock); + const dim3 grid(c.patternBlocks, c.chunks); + root_chunk_sums_kernel<<>>(root, categoryWeights, frequencies, nStates, nRows, nPaddedPatterns, + c.rowsPerChunk, scratch); + root_finish_kernel<<<1, 1024, 0, stream>>>(scratch, c.chunks, nPaddedPatterns, nPatterns, cumulativeScaling, patternWeights, + siteLogLikelihoods, sumLogLikelihood); } -void cuda_csr_taylor_term(int nStates, int nColumns, const int* rowOffsets, const int* columns, const float* values, float alpha, - const float* x, float* y, float* F, float scale, cudaStream_t stream) +void cuda_root_log_likelihoods(const float* root, const float* categoryWeights, const float* frequencies, const float* cumulativeScaling, + const float* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, + float* scratch, float* siteLogLikelihoods, float* sumLogLikelihood, cudaStream_t stream) { - cuda_csr_taylor_term_impl(nStates, nColumns, rowOffsets, columns, values, alpha, x, y, F, scale, stream); + cuda_root_log_likelihoods_impl(root, categoryWeights, frequencies, cumulativeScaling, patternWeights, + nStates, nPaddedPatterns, nPatterns, nCategories, scratch, siteLogLikelihoods, sumLogLikelihood, stream); } -void cuda_csr_taylor_term(int nStates, int nColumns, const int* rowOffsets, const int* columns, const double* values, double alpha, - const double* x, double* y, double* F, double scale, cudaStream_t stream) +void cuda_root_log_likelihoods(const double* root, const double* categoryWeights, const double* frequencies, const double* cumulativeScaling, + const double* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, + double* scratch, double* siteLogLikelihoods, double* sumLogLikelihood, cudaStream_t stream) { - cuda_csr_taylor_term_impl(nStates, nColumns, rowOffsets, columns, values, alpha, x, y, F, scale, stream); + cuda_root_log_likelihoods_impl(root, categoryWeights, frequencies, cumulativeScaling, patternWeights, + nStates, nPaddedPatterns, nPatterns, nCategories, scratch, siteLogLikelihoods, sumLogLikelihood, stream); } diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 4e23d81a..a5a7d8a9 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -6,7 +6,7 @@ #include // for std::runtime_error -inline void* cudaMallocWrapped(int n) +inline void* cudaMallocWrapped(size_t n) { void* result; auto status = cudaMalloc((void**)&result, n); @@ -16,7 +16,7 @@ inline void* cudaMallocWrapped(int n) } template -T* cudaDeviceNew(int n) +T* cudaDeviceNew(size_t n) { return (T*)cudaMallocWrapped(n*sizeof(T)); } @@ -103,26 +103,90 @@ void cuda_select_unit_columns(const double* Z, int n, int t, int nBlocks, int* u void cuda_combine_norm_estimates(const float* sums1, const float* sums2, int t, int pTop, float* norms, cudaStream_t stream); void cuda_combine_norm_estimates(const double* sums1, const double* sums2, int t, int pTop, double* norms, cudaStream_t stream); -// parent = left * right for (nStates, nPatterns, nCategories) partials. If scalingFactors is non-null, also -// rescale each pattern by its largest entry m, write m (log m if scalersLog) to scalingFactors[pattern], -// and add log m to cumulative[pattern] if cumulative is non-null. -void cuda_partials_product(const float* left, const float* right, float* parent, int nStates, int nPatterns, int nCategories, - float* scalingFactors, float* cumulative, bool scalersLog, cudaStream_t stream); -void cuda_partials_product(const double* left, const double* right, double* parent, int nStates, int nPatterns, int nCategories, - double* scalingFactors, double* cumulative, bool scalersLog, cudaStream_t stream); +// Action partials are stored pattern-fastest, [category][state][pattern]: a (nRows, nPatterns) row-major +// matrix with nRows = nCategories * nStates. -// siteLogLikelihoods[p] = log(sum_{c,s} root * categoryWeights[c] * frequencies[s]) + cumulativeScaling[p] (if non-null), +// One Horner step of a Taylor polynomial for several independent (nRows, nCols) row-major matrices that share A: +// out = scale * (b + alpha * A * x) +// Each matrix has its own row stride (ld*, in elements), so the groups' matrices can be stored side by side in +// one wider matrix. out must not alias x or b. +template +struct TaylorTermGroup +{ + const T* x; + const T* b; + T* out; + T alpha; + T scale; + int ldx; + int ldb; + int ldOut; +}; + +// 64 groups keep the kernel parameters under 4 KB. +constexpr int kMaxTaylorGroups = 64; + +template +struct TaylorTermBatch +{ + int count = 0; + TaylorTermGroup groups[kMaxTaylorGroups]; +}; + +void cuda_taylor_horner_terms(int nRows, int nCols, const int* rowOffsets, const int* columns, const float* values, + const TaylorTermBatch& batch, cudaStream_t stream); +void cuda_taylor_horner_terms(int nRows, int nCols, const int* rowOffsets, const int* columns, const double* values, + const TaylorTermBatch& batch, cudaStream_t stream); + +// The product and root kernels need a scratch buffer of cuda_row_chunk_count(nRows, nPatterns) * nPatterns values +// (per product, for the products). +int cuda_row_chunk_count(int nRows, int nPatterns); + +// parent = left * right for (nRows, nPatterns) partials. If scalingFactors is non-null, also rescale each +// pattern by its largest entry m and write m (log m if scalersLog) to scalingFactors[pattern]. +template +struct PartialsProduct +{ + const T* left; + const T* right; + T* parent; + T* scalingFactors; +}; + +constexpr int kMaxPartialsProducts = 16; + +template +struct PartialsProductBatch +{ + int count = 0; + PartialsProduct products[kMaxPartialsProducts]; +}; + +// All the products of the batch, then (if cumulative is non-null) cumulative[pattern] += log m of each in turn. +void cuda_partials_products(const PartialsProductBatch& batch, int nRows, int nPatterns, float* cumulative, bool scalersLog, + float* scratch, cudaStream_t stream); +void cuda_partials_products(const PartialsProductBatch& batch, int nRows, int nPatterns, double* cumulative, bool scalersLog, + double* scratch, cudaStream_t stream); + +// cumulative[p] += (or -= if subtract) the sum over the buffers of factors[p] (log factors[p] unless scalersLog). +template +struct ScaleFactorBatch +{ + int count = 0; + const T* factors[64]; +}; + +void cuda_accumulate_scale_factors(const ScaleFactorBatch& batch, int nPatterns, float* cumulative, bool scalersLog, + bool subtract, cudaStream_t stream); +void cuda_accumulate_scale_factors(const ScaleFactorBatch& batch, int nPatterns, double* cumulative, bool scalersLog, + bool subtract, cudaStream_t stream); + +// siteLogLikelihoods[p] = log(sum_{c,s} root(c,s,p) * categoryWeights[c] * frequencies[s]) + cumulativeScaling[p] (if non-null), // sumLogLikelihood[0] = sum_p patternWeights[p] * siteLogLikelihoods[p]. void cuda_root_log_likelihoods(const float* root, const float* categoryWeights, const float* frequencies, const float* cumulativeScaling, const float* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, - float* siteLogLikelihoods, float* sumLogLikelihood, cudaStream_t stream); + float* scratch, float* siteLogLikelihoods, float* sumLogLikelihood, cudaStream_t stream); void cuda_root_log_likelihoods(const double* root, const double* categoryWeights, const double* frequencies, const double* cumulativeScaling, const double* patternWeights, int nStates, int nPaddedPatterns, int nPatterns, int nCategories, - double* siteLogLikelihoods, double* sumLogLikelihood, cudaStream_t stream); - -// One Taylor term on column-major (nStates, nColumns) matrices: y = alpha * A * x; F = scale * (F + y), A in CSR. -void cuda_csr_taylor_term(int nStates, int nColumns, const int* rowOffsets, const int* columns, const float* values, float alpha, - const float* x, float* y, float* F, float scale, cudaStream_t stream); -void cuda_csr_taylor_term(int nStates, int nColumns, const int* rowOffsets, const int* columns, const double* values, double alpha, - const double* x, double* y, double* F, double scale, cudaStream_t stream); + double* scratch, double* siteLogLikelihoods, double* sumLogLikelihood, cudaStream_t stream); #endif From 93fb0017c7ccf1617572fa13d4e5de70075971f2 Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Sun, 25 Oct 2026 00:11:26 -0500 Subject: [PATCH 416/421] GPU action: cheaper setSparseMatrix (cached CSR structure, row-major norm estimator) - ShiftedCsrBuilder remembers where each triplet of Q lands in B = Q - mu*I, and the structure of B^T. When Q has the same triplet pattern as last time (only the rates change), it only scatters the values, adding them in the same order as a full build (bit-identical). Only B's values go to the device; B^T's values come from one gather instead of csr2csc. - The block 1-norm estimator works on row-major matrices with lanes over the columns of a row, so the gathers are coalesced. Column sums use one block per column, and the unit-vector selection computes each row's maximum once. X0 has the same entries as before; the d_p are unchanged. A100, setQ per call: IGC 1.57 -> 0.62 ms, MM M=32 1.68 -> 0.57 ms, MM M=8 0.44 -> 0.26 ms. IGC at 1 pattern: 2.36 -> 1.41 ms per evaluation. logL unchanged; compute-sanitizer memcheck/racecheck clean. Co-Authored-By: Claude Opus 5.5 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 170 +++++++++++++--------- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 62 +++++--- libhmsbeagle/GPU/cuda_ops.cu | 175 +++++++++++++++-------- libhmsbeagle/GPU/cuda_ops.h | 38 ++--- 4 files changed, 288 insertions(+), 157 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 022ebc61..83c2dfd4 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -138,8 +138,10 @@ struct HostCsr // triplets are summed, as Eigen's setFromTriplets( ) does, and every row gets a diagonal entry. // Two stable counting sorts (by column, then by row) leave each row sorted, in O(nnz + n) for any input // order. (Going through Eigen took ~2 ms for 75K non-zeros.) +// If slot is non-null, it receives the position in B.values of each triplet, then of each row's -mu. template -void buildShiftedCsr(int n, const int* rows, const int* cols, const Real* values, int nnz, Real mu, HostCsr& B) +void buildShiftedCsr(int n, const int* rows, const int* cols, const Real* values, int nnz, Real mu, HostCsr& B, + std::vector* slot = nullptr) { const int m = nnz + n; // the triplets, plus (i, i, -mu) for each row auto rowOf = [&](int k) {return (k < nnz) ? rows[k] : k - nnz;}; @@ -165,14 +167,17 @@ void buildShiftedCsr(int n, const int* rows, const int* cols, const Real* values std::vector next(B.offsets.begin(), B.offsets.end() - 1); B.columns.resize(m); B.values.resize(m); + if (slot) slot->resize(m); for (int k: byColumn) { const int a = next[rowOf(k)]++; B.columns[a] = colOf(k); B.values[a] = valueOf(k); + if (slot) (*slot)[k] = a; } // 3. Sum duplicates, compacting in place. + std::vector compacted(slot ? m : 0); int out = 0; for (int i = 0; i < n; i++) { @@ -188,13 +193,69 @@ void buildShiftedCsr(int n, const int* rows, const int* cols, const Real* values B.columns[out] = B.columns[a]; B.values[out++] = B.values[a]; } + if (slot) compacted[a] = out - 1; } } B.offsets[n] = out; B.columns.resize(out); B.values.resize(out); + if (slot) + for (int& a: *slot) + a = compacted[a]; } +// Builds B = Q - mu*I like buildShiftedCsr( ), and the structure of B^T. It remembers where each triplet +// went, so when Q has the same triplet pattern as last time (the usual case: only the rates change), it only +// scatters the values, in O(nnz), adding them in the same order as a full build. +template +struct ShiftedCsrBuilder +{ + std::vector rows; // the triplet pattern of the last full build + std::vector cols; + std::vector slot; // see buildShiftedCsr( ) + std::vector transposeOffsets; // B^T in CSR: offsets and columns; + std::vector transposeColumns; + std::vector transposeSlot; // its values are B.values[transposeSlot[j]] + + // Returns true if the structure of B (and so of B^T) changed. + bool build(int n, const int* rowIndices, const int* colIndices, const Real* values, int nnz, Real mu, HostCsr& B) + { + const bool samePattern = (int) B.offsets.size() == n + 1 and (int) rows.size() == nnz and + std::equal(rows.begin(), rows.end(), rowIndices) and std::equal(cols.begin(), cols.end(), colIndices); + if (samePattern) + { + std::fill(B.values.begin(), B.values.end(), Real(0)); + for (int k = 0; k < nnz; k++) + B.values[slot[k]] += values[k]; + for (int i = 0; i < n; i++) + B.values[slot[nnz + i]] += -mu; + return false; + } + + rows.assign(rowIndices, rowIndices + nnz); + cols.assign(colIndices, colIndices + nnz); + buildShiftedCsr(n, rowIndices, colIndices, values, nnz, mu, B, &slot); + + // B^T by a counting sort on the columns; rows are visited in order, so each row of B^T is sorted. + transposeOffsets.assign(n + 1, 0); + for (int c: B.columns) + transposeOffsets[c + 1]++; + for (int j = 0; j < n; j++) + transposeOffsets[j + 1] += transposeOffsets[j]; + std::vector next(transposeOffsets.begin(), transposeOffsets.end() - 1); + transposeColumns.resize(B.nonZeros()); + transposeSlot.resize(B.nonZeros()); + for (int i = 0; i < n; i++) + for (int a = B.offsets[i]; a < B.offsets[i + 1]; a++) + { + const int b = next[B.columns[a]]++; + transposeColumns[b] = i; + transposeSlot[b] = a; + } + return true; + } +}; + // max_j sum_i |B(i,j)| template double normP1(int n, const HostCsr& B) @@ -702,7 +763,8 @@ std::ostream& operator<<(std::ostream& o, const std::vector& H) // 3. X_p = the t unit vectors e_i with the largest max_j |Z_p(i,j)|. // 4. B^p X_p, by the same appending chain with B. // The estimate for p is the largest column 1-norm of Y_p and of B^p X_p. Everything stays on the -// device until one copy of the pTop+1 results. +// device until one copy of the pTop+1 results. The matrices are row-major (n rows), so that the gathers of a +// product read contiguous rows. template struct BlockNormEstimator { @@ -711,104 +773,84 @@ struct BlockNormEstimator int pTop = 0; Real* X0 = nullptr; // (n, t) - Real* Y = nullptr; // (n, pTop*t): block p-1 is B^p X0 - Real* chain[2] = {nullptr, nullptr}; // (n, (pTop-1)*t) each + Real* Y = nullptr; // (n, pTop*t): columns (p-1)*t .. p*t-1 are B^p X0 + Real* chain[2] = {nullptr, nullptr}; // up to (n, (pTop-1)*t) each Real* sums1 = nullptr; // pTop*t column sums of Y Real* sums2 = nullptr; // (pTop-1)*t column sums of step 4 int* unitRows = nullptr; // (pTop-1)*t + Real* rowMax = nullptr; // (pTop-1)*n, for step 3 Real* norms = nullptr; // pTop+1 - // B^T in CSR (that is, B in CSC) - int* btOffsets = nullptr; - int* btColumns = nullptr; - Real* btValues = nullptr; - int btCapacity = 0; - void* transposeBuffer = nullptr; - size_t transposeBufferSize = 0; - void allocate(int n_, int t_, int pTop_) { n = n_; t = std::min(t_, n); pTop = pTop_; assert(t <= 32); // cuda_select_unit_columns( ) - X0 = cudaDeviceNew(n * t); - Y = cudaDeviceNew(n * t * pTop); - chain[0] = cudaDeviceNew(n * t * (pTop - 1)); - chain[1] = cudaDeviceNew(n * t * (pTop - 1)); + X0 = cudaDeviceNew((size_t) n * t); + Y = cudaDeviceNew((size_t) n * t * pTop); + chain[0] = cudaDeviceNew((size_t) n * t * (pTop - 1)); + chain[1] = cudaDeviceNew((size_t) n * t * (pTop - 1)); sums1 = cudaDeviceNew(t * pTop); sums2 = cudaDeviceNew(t * (pTop - 1)); unitRows = cudaDeviceNew(t * (pTop - 1)); + rowMax = cudaDeviceNew((size_t) n * (pTop - 1)); norms = cudaDeviceNew(pTop + 1); - btOffsets = cudaDeviceNew(n + 1); initialize_norm_x_matrix(X0, n, t); } // Returns ||B^p||_1 estimates for p = 2..pTop at index p. (Indices 0 and 1 are not set.) - std::vector estimate(const SpMatrixDevice& B, cudaStream_t stream) + // BT is B^T in CSR. + std::vector estimate(const SpMatrixDevice& B, const int* btOffsets, const int* btColumns, const Real* btValues, + cudaStream_t stream) { assert(B.rows() == n and B.cols() == n); - const int nnz = B.num_non_zeros; const int nBlocks = pTop - 1; - const size_t blockSize = (size_t) n * t; - - // B^T - if (nnz > btCapacity) - { - cudaDeviceDelete(btColumns); - cudaDeviceDelete(btValues); - btColumns = cudaDeviceNew(nnz); - btValues = cudaDeviceNew(nnz); - btCapacity = nnz; - } - size_t size = 0; - checkCusparse(cusparseCsr2cscEx2_bufferSize(B.cusparseHandle, n, n, nnz, B.values, B.offsets, B.inner, - btValues, btOffsets, btColumns, DataType, CUSPARSE_ACTION_NUMERIC, - CUSPARSE_INDEX_BASE_ZERO, CUSPARSE_CSR2CSC_ALG1, &size)); - if (size > transposeBufferSize or transposeBuffer == nullptr) - { - cudaDeviceDelete(transposeBuffer); - transposeBufferSize = std::max(size, 1); - transposeBuffer = cudaMallocWrapped(transposeBufferSize); - } - checkCusparse(cusparseCsr2cscEx2(B.cusparseHandle, n, n, nnz, B.values, B.offsets, B.inner, - btValues, btOffsets, btColumns, DataType, CUSPARSE_ACTION_NUMERIC, - CUSPARSE_INDEX_BASE_ZERO, CUSPARSE_CSR2CSC_ALG1, transposeBuffer)); + const int ldY = pTop * t; // 1. Y_p = B^p X0 for (int p = 1; p <= pTop; p++) { - const Real* x = (p == 1) ? X0 : Y + (p - 2) * blockSize; - cuda_norm_chain_step(n, B.offsets, B.inner, B.values, x, Y + (p - 1) * blockSize, t, 0, (Real*)nullptr, nullptr, stream); + const Real* x = (p == 1) ? X0 : Y + (p - 2) * t; + cuda_norm_chain_step(n, B.offsets, B.inner, B.values, x, (p == 1) ? t : ldY, Y + (p - 1) * t, ldY, + t, 0, (Real*)nullptr, 0, nullptr, stream); } - cuda_column_abs_sums(Y, n, pTop * t, sums1, stream); + cuda_column_abs_sums(Y, n, pTop * t, ldY, sums1, stream); - // 2. Z_p = (B^T)^p sign(Y_p), in block pTop - p. + // 2. Z_p = (B^T)^p sign(Y_p), in block pTop - p. Each step's output has `columns` columns. Real* in = chain[0]; Real* out = chain[1]; - cuda_norm_chain_step(n, btOffsets, btColumns, btValues, (Real*)nullptr, in, 0, t, Y + (pTop - 1) * blockSize, nullptr, stream); + int columns = t; + cuda_norm_chain_step(n, btOffsets, btColumns, btValues, (Real*)nullptr, 0, in, columns, 0, t, Y + (pTop - 1) * t, ldY, nullptr, stream); for (int k = 1; k <= pTop; k++) { const bool append = (k < nBlocks); - const Real* sign = append ? Y + (pTop - k - 1) * blockSize : nullptr; - cuda_norm_chain_step(n, btOffsets, btColumns, btValues, in, out, std::min(k, nBlocks) * t, append ? t : 0, sign, nullptr, stream); + const Real* sign = append ? Y + (pTop - k - 1) * t : nullptr; + const int outColumns = columns + (append ? t : 0); + cuda_norm_chain_step(n, btOffsets, btColumns, btValues, in, columns, out, outColumns, columns, append ? t : 0, + sign, ldY, nullptr, stream); std::swap(in, out); + columns = outColumns; } + assert(columns == nBlocks * t); // 3. Unit vectors for the second iteration. - cuda_select_unit_columns(in, n, t, nBlocks, unitRows, stream); + cuda_select_unit_columns(in, n, t, nBlocks, columns, rowMax, unitRows, stream); // 4. B^p X_p, in block pTop - p. - cuda_norm_chain_step(n, B.offsets, B.inner, B.values, (Real*)nullptr, out, 0, t, (Real*)nullptr, unitRows, stream); + columns = t; + cuda_norm_chain_step(n, B.offsets, B.inner, B.values, (Real*)nullptr, 0, out, columns, 0, t, (Real*)nullptr, 0, unitRows, stream); std::swap(in, out); for (int k = 1; k <= pTop; k++) { const bool append = (k < nBlocks); - cuda_norm_chain_step(n, B.offsets, B.inner, B.values, in, out, std::min(k, nBlocks) * t, append ? t : 0, - (Real*)nullptr, append ? unitRows + k * t : nullptr, stream); + const int outColumns = columns + (append ? t : 0); + cuda_norm_chain_step(n, B.offsets, B.inner, B.values, in, columns, out, outColumns, columns, append ? t : 0, + (Real*)nullptr, 0, append ? unitRows + k * t : nullptr, stream); std::swap(in, out); + columns = outColumns; } - cuda_column_abs_sums(in, n, nBlocks * t, sums2, stream); + cuda_column_abs_sums(in, n, nBlocks * t, columns, sums2, stream); cuda_combine_norm_estimates(sums1, sums2, t, pTop, norms, stream); @@ -821,12 +863,6 @@ struct BlockNormEstimator return result; } - static void checkCusparse(cusparseStatus_t status) - { - if (status != CUSPARSE_STATUS_SUCCESS) - throw std::runtime_error(std::string("BlockNormEstimator: ") + cusparseGetErrorString(status)); - } - BlockNormEstimator() = default; BlockNormEstimator(const BlockNormEstimator&) = delete; BlockNormEstimator& operator=(const BlockNormEstimator&) = delete; @@ -840,11 +876,8 @@ struct BlockNormEstimator cudaDeviceDelete(sums1); cudaDeviceDelete(sums2); cudaDeviceDelete(unitRows); + cudaDeviceDelete(rowMax); cudaDeviceDelete(norms); - cudaDeviceDelete(btOffsets); - cudaDeviceDelete(btColumns); - cudaDeviceDelete(btValues); - cudaDeviceDelete(transposeBuffer); } }; @@ -936,6 +969,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // Per eigen index: B = Q - mu*I. std::vector> hBs; + std::vector> csrBuilders; std::vector hMuBs; std::vector hB1Norms; const int mMax = 55; @@ -1003,6 +1037,12 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // copies and kernels (two different blocking streams would not be). See setCudaDevice( ). cudaStream_t stream = nullptr; + // Per eigen index: B^T in CSR, and the positions in B's values of its values. + std::vector dBtOffsets; + std::vector dBtColumns; + std::vector dBtSlots; + std::vector dBtValues; + std::vector dBsCsrOffsetsCache; std::vector dBsCsrColumnsCache; std::vector dBsCsrValuesCache; diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 5c26fd1e..c899bbcf 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -292,6 +292,7 @@ int BeagleGPUActionImpl::createInstance(int tipCount, setCudaDevice(); hBs.resize(kEigenDecompCount); + csrBuilders.resize(kEigenDecompCount); hMuBs.resize(kEigenDecompCount); hB1Norms.resize(kEigenDecompCount); hds.resize(kEigenDecompCount); @@ -308,8 +309,13 @@ int BeagleGPUActionImpl::createInstance(int tipCount, normEstimator.allocate(kPaddedStateCount, 5, getPMax() + 1); currentCacheNNZs = std::vector(kEigenDecompCount, kPaddedStateCount); + dBtOffsets.assign(kEigenDecompCount, nullptr); + dBtColumns.assign(kEigenDecompCount, nullptr); + dBtSlots.assign(kEigenDecompCount, nullptr); + dBtValues.assign(kEigenDecompCount, nullptr); for (int i = 0; i < kEigenDecompCount; i++) { dBsCsrOffsetsCache[i] = cudaDeviceNew(kPaddedStateCount + 1); + dBtOffsets[i] = cudaDeviceNew(kPaddedStateCount + 1); dBsCsrColumnsCache[i] = cudaDeviceNew(currentCacheNNZs[i]); dBsCsrValuesCache[i] = cudaDeviceNew(currentCacheNNZs[i]); } @@ -343,6 +349,10 @@ BeagleGPUActionImpl::~BeagleGPUActionImpl() for (int i = 0; i < kEigenDecompCount; i++) { cudaFree(dBsCsrOffsetsCache[i]); + cudaFree(dBtOffsets[i]); + cudaFree(dBtColumns[i]); + cudaFree(dBtSlots[i]); + cudaFree(dBtValues[i]); cudaFree(dBsCsrColumnsCache[i]); cudaFree(dBsCsrValuesCache[i]); } @@ -1216,7 +1226,8 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, mu_B /= (Real) kStateCount; hMuBs[matrixIndex] = mu_B; - buildShiftedCsr(kPaddedStateCount, rowIndices, colIndices, values, numNonZeros, mu_B, hBs[matrixIndex]); + auto& builder = csrBuilders[matrixIndex]; + const bool structureChanged = builder.build(kPaddedStateCount, rowIndices, colIndices, values, numNonZeros, mu_B, hBs[matrixIndex]); hB1Norms[matrixIndex] = normP1(kPaddedStateCount, hBs[matrixIndex]); #ifdef BEAGLE_DEBUG_FLOW @@ -1229,18 +1240,39 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, std::cerr<<"\n"; #endif + // With an unchanged structure only the values of B go to the device; B^T's values are gathered from them. const int currentNNZ = hBs[matrixIndex].nonZeros(); - if (currentCacheNNZs[matrixIndex] != currentNNZ) { - currentCacheNNZs[matrixIndex] = currentNNZ; - cudaDeviceDelete(dBsCsrColumnsCache[matrixIndex]); - cudaDeviceDelete(dBsCsrValuesCache[matrixIndex]); - dBsCsrColumnsCache[matrixIndex] = cudaDeviceNew(currentNNZ); - dBsCsrValuesCache[matrixIndex] = cudaDeviceNew(currentNNZ); + if (structureChanged) { + if (currentCacheNNZs[matrixIndex] != currentNNZ) { + currentCacheNNZs[matrixIndex] = currentNNZ; + cudaDeviceDelete(dBsCsrColumnsCache[matrixIndex]); + cudaDeviceDelete(dBsCsrValuesCache[matrixIndex]); + dBsCsrColumnsCache[matrixIndex] = cudaDeviceNew(currentNNZ); + dBsCsrValuesCache[matrixIndex] = cudaDeviceNew(currentNNZ); + } + cudaDeviceDelete(dBtColumns[matrixIndex]); + cudaDeviceDelete(dBtSlots[matrixIndex]); + cudaDeviceDelete(dBtValues[matrixIndex]); + dBtColumns[matrixIndex] = cudaDeviceNew(currentNNZ); + dBtSlots[matrixIndex] = cudaDeviceNew(currentNNZ); + dBtValues[matrixIndex] = cudaDeviceNew(currentNNZ); + + MemcpyHostToDevice(dBsCsrOffsetsCache[matrixIndex], hBs[matrixIndex].offsets.data(), kPaddedStateCount + 1); + MemcpyHostToDevice(dBsCsrColumnsCache[matrixIndex], hBs[matrixIndex].columns.data(), currentNNZ); + MemcpyHostToDevice(dBtOffsets[matrixIndex], builder.transposeOffsets.data(), kPaddedStateCount + 1); + MemcpyHostToDevice(dBtColumns[matrixIndex], builder.transposeColumns.data(), currentNNZ); + MemcpyHostToDevice(dBtSlots[matrixIndex], builder.transposeSlot.data(), currentNNZ); + + dBs[matrixIndex] = SpMatrixDevice(cublasHandle, cusparseHandle, + kPaddedStateCount, kPaddedStateCount, + currentNNZ, + dBsCsrValuesCache[matrixIndex], + dBsCsrColumnsCache[matrixIndex], + dBsCsrOffsetsCache[matrixIndex], + sparseFormat::csr); } - - MemcpyHostToDevice(dBsCsrOffsetsCache[matrixIndex], hBs[matrixIndex].offsets.data(), kPaddedStateCount + 1); - MemcpyHostToDevice(dBsCsrColumnsCache[matrixIndex], hBs[matrixIndex].columns.data(), currentNNZ); MemcpyHostToDevice(dBsCsrValuesCache[matrixIndex], hBs[matrixIndex].values.data(), currentNNZ); + cuda_gather(dBsCsrValuesCache[matrixIndex], dBtSlots[matrixIndex], currentNNZ, dBtValues[matrixIndex], stream); #ifdef BEAGLE_DEBUG_FLOW @@ -1256,19 +1288,11 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, PrintfDeviceVector(dBsCsrValuesCache[matrixIndex], currentNNZ, -1, 0, 0); #endif - - dBs[matrixIndex] = SpMatrixDevice(cublasHandle, cusparseHandle, - kPaddedStateCount, kPaddedStateCount, - currentNNZ, - dBsCsrValuesCache[matrixIndex], - dBsCsrColumnsCache[matrixIndex], - dBsCsrOffsetsCache[matrixIndex], - sparseFormat::csr); auto& dB = dBs[matrixIndex]; // equation 3.7 in Al-Mohy and Higham: d_p = ||B^p||_1^(1/p). d_1 is known exactly; d_0 is not used. const int pTop = getPMax() + 1; - auto norms = normEstimator.estimate(dB, stream); + auto norms = normEstimator.estimate(dB, dBtOffsets[matrixIndex], dBtColumns[matrixIndex], dBtValues[matrixIndex], stream); hds[matrixIndex].assign(pTop + 1, 1.0); hds[matrixIndex][1] = hB1Norms[matrixIndex]; for (int p = 2; p <= pTop; p++) diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 297cfe10..0b21ec34 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -39,18 +39,23 @@ double cuda_max_abs(double* values, int length) template void initialize_norm_x_matrix(device_ptr out_ptr, int n, int m) { + // Element (row, column) is stored at row * m + column. Its sign comes from its column-major index + // i = column * n + row. auto indices = thrust::counting_iterator(0); - auto initialize = [n,m] __host__ __device__ (int i) + auto initialize = [n,m] __host__ __device__ (int index) { + const int row = index / m; + const int column = index % m; + const int i = column * n + row; thrust::default_random_engine rng; thrust::uniform_real_distribution dist(0, 1); - if (i 0.5) - return 1.0/n; + return T(1.0/n); else - return -1.0/n; + return T(-1.0/n); }; thrust::transform(indices, indices + n*m, // in @@ -547,99 +552,157 @@ void cuda_root_log_likelihoods(const double* root, const double* categoryWeights // --------------------------------------------------------------------------------------------------- // Block 1-norm estimation of ||B^p||_1 for all p at once (Higham & Tisseur Algorithm 2.4, 2 iterations). -// All matrices are column-major with n rows. See BlockNormEstimator in BeagleGPUActionImpl.h. +// All matrices are row-major with n rows and an explicit row stride. See BlockNormEstimator in +// BeagleGPUActionImpl.h. // y(:, 0:nApplied) = A * x(:, 0:nApplied), then nAppend more columns of y are set: // from signSource (n, nAppend): y(i, c) = +1 if signSource(i, c) >= 0, else -1 // from unitRows (nAppend): y(i, c) = 1 if i == unitRows[c], else 0 -template +// LANES lanes share a row and cover its columns, so the gathers of x are coalesced. +template __global__ void norm_chain_step_kernel(int n, const int* __restrict__ rowOffsets, const int* __restrict__ columns, - const T* __restrict__ values, const T* __restrict__ x, T* __restrict__ y, - int nApplied, int nAppend, const T* __restrict__ signSource, const int* __restrict__ unitRows) + const T* __restrict__ values, const T* __restrict__ x, int ldx, T* __restrict__ y, int ldy, + int nApplied, int nAppend, const T* __restrict__ signSource, int ldSign, + const int* __restrict__ unitRows) { - const size_t index = blockIdx.x * (size_t)blockDim.x + threadIdx.x; - if (index >= (size_t)n * (nApplied + nAppend)) return; - const int row = index % n; - const int column = index / n; + const int lane = threadIdx.x % 32; + const int row = (blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32) * (32 / LANES) + lane / LANES; + if (row >= n) return; + T* yRow = y + (size_t)row * ldy; - if (column < nApplied) + const int kBegin = rowOffsets[row]; + const int kEnd = rowOffsets[row + 1]; + for (int c = lane % LANES; c < nApplied; c += LANES) { - const T* xColumn = x + (size_t)column * n; T sum = 0; - for (int k = rowOffsets[row]; k < rowOffsets[row + 1]; k++) - sum += values[k] * xColumn[columns[k]]; - y[index] = sum; + for (int k = kBegin; k < kEnd; k++) + sum += values[k] * x[(size_t)columns[k] * ldx + c]; + yRow[c] = sum; } - else + for (int c = lane % LANES; c < nAppend; c += LANES) { - const int c = column - nApplied; if (signSource) - y[index] = (signSource[(size_t)c * n + row] >= 0) ? T(1) : T(-1); + yRow[nApplied + c] = (signSource[(size_t)row * ldSign + c] >= 0) ? T(1) : T(-1); else - y[index] = (row == unitRows[c]) ? T(1) : T(0); + yRow[nApplied + c] = (row == unitRows[c]) ? T(1) : T(0); } } +template +void launch_norm_chain_step(int n, const int* rowOffsets, const int* columns, const T* values, const T* x, int ldx, T* y, int ldy, + int nApplied, int nAppend, const T* signSource, int ldSign, const int* unitRows, cudaStream_t stream) +{ + const int threads = 256; + const int rowsPerBlock = (threads / 32) * (32 / LANES); + norm_chain_step_kernel<<<(n + rowsPerBlock - 1) / rowsPerBlock, threads, 0, stream>>>( + n, rowOffsets, columns, values, x, ldx, y, ldy, nApplied, nAppend, signSource, ldSign, unitRows); +} + template -void cuda_norm_chain_step_impl(int n, const int* rowOffsets, const int* columns, const T* values, const T* x, T* y, - int nApplied, int nAppend, const T* signSource, const int* unitRows, cudaStream_t stream) +void cuda_norm_chain_step_impl(int n, const int* rowOffsets, const int* columns, const T* values, const T* x, int ldx, T* y, int ldy, + int nApplied, int nAppend, const T* signSource, int ldSign, const int* unitRows, cudaStream_t stream) { - const size_t total = (size_t)n * (nApplied + nAppend); - const int blockSize = 256; - norm_chain_step_kernel<<<(total + blockSize - 1) / blockSize, blockSize, 0, stream>>>(n, rowOffsets, columns, values, x, y, - nApplied, nAppend, signSource, unitRows); + switch (nextPowerOfTwo(std::max(nApplied, nAppend), 32)) + { + case 1: + case 2: + case 4: launch_norm_chain_step(n, rowOffsets, columns, values, x, ldx, y, ldy, nApplied, nAppend, signSource, ldSign, unitRows, stream); break; + case 8: launch_norm_chain_step(n, rowOffsets, columns, values, x, ldx, y, ldy, nApplied, nAppend, signSource, ldSign, unitRows, stream); break; + case 16: launch_norm_chain_step(n, rowOffsets, columns, values, x, ldx, y, ldy, nApplied, nAppend, signSource, ldSign, unitRows, stream); break; + default: launch_norm_chain_step(n, rowOffsets, columns, values, x, ldx, y, ldy, nApplied, nAppend, signSource, ldSign, unitRows, stream); break; + } } -void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const float* values, const float* x, float* y, - int nApplied, int nAppend, const float* signSource, const int* unitRows, cudaStream_t stream) +void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const float* values, const float* x, int ldx, float* y, int ldy, + int nApplied, int nAppend, const float* signSource, int ldSign, const int* unitRows, cudaStream_t stream) { - cuda_norm_chain_step_impl(n, rowOffsets, columns, values, x, y, nApplied, nAppend, signSource, unitRows, stream); + cuda_norm_chain_step_impl(n, rowOffsets, columns, values, x, ldx, y, ldy, nApplied, nAppend, signSource, ldSign, unitRows, stream); } -void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const double* values, const double* x, double* y, - int nApplied, int nAppend, const double* signSource, const int* unitRows, cudaStream_t stream) +void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const double* values, const double* x, int ldx, double* y, int ldy, + int nApplied, int nAppend, const double* signSource, int ldSign, const int* unitRows, cudaStream_t stream) { - cuda_norm_chain_step_impl(n, rowOffsets, columns, values, x, y, nApplied, nAppend, signSource, unitRows, stream); + cuda_norm_chain_step_impl(n, rowOffsets, columns, values, x, ldx, y, ldy, nApplied, nAppend, signSource, ldSign, unitRows, stream); } // sums[c] = sum_i |x(i, c)|, one block per column. template -__global__ void column_abs_sums_kernel(const T* x, int n, T* sums) +__global__ void column_abs_sums_kernel(const T* x, int n, int ld, T* sums) { - const T* column = x + (size_t)blockIdx.x * n; + const int c = blockIdx.x; T sum = 0; for (int i = threadIdx.x; i < n; i += blockDim.x) - sum += abs(column[i]); + sum += abs(x[(size_t)i * ld + c]); sum = block_reduce(sum, T(0), SumOp()); - if (threadIdx.x == 0) sums[blockIdx.x] = sum; + if (threadIdx.x == 0) sums[c] = sum; +} + +template +void cuda_column_abs_sums_impl(const T* x, int n, int nColumns, int ld, T* sums, cudaStream_t stream) +{ + column_abs_sums_kernel<<>>(x, n, ld, sums); +} + +void cuda_column_abs_sums(const float* x, int n, int nColumns, int ld, float* sums, cudaStream_t stream) +{ + cuda_column_abs_sums_impl(x, n, nColumns, ld, sums, stream); +} + +void cuda_column_abs_sums(const double* x, int n, int nColumns, int ld, double* sums, cudaStream_t stream) +{ + cuda_column_abs_sums_impl(x, n, nColumns, ld, sums, stream); } +// dst[i] = src[index[i]] template -void cuda_column_abs_sums_impl(const T* x, int n, int nColumns, T* sums, cudaStream_t stream) +__global__ void gather_kernel(const T* src, const int* index, int n, T* dst) { - column_abs_sums_kernel<<>>(x, n, sums); + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) dst[i] = src[index[i]]; } -void cuda_column_abs_sums(const float* x, int n, int nColumns, float* sums, cudaStream_t stream) +template +void cuda_gather_impl(const T* src, const int* index, int n, T* dst, cudaStream_t stream) +{ + if (n > 0) + gather_kernel<<<(n + 255) / 256, 256, 0, stream>>>(src, index, n, dst); +} + +void cuda_gather(const float* src, const int* index, int n, float* dst, cudaStream_t stream) { - cuda_column_abs_sums_impl(x, n, nColumns, sums, stream); + cuda_gather_impl(src, index, n, dst, stream); } -void cuda_column_abs_sums(const double* x, int n, int nColumns, double* sums, cudaStream_t stream) +void cuda_gather(const double* src, const int* index, int n, double* dst, cudaStream_t stream) { - cuda_column_abs_sums_impl(x, n, nColumns, sums, stream); + cuda_gather_impl(src, index, n, dst, stream); +} + +// rowMax[b * n + i] = max_j |Z(i, b*t + j)| for each block b of t columns of Z (row stride ld). +template +__global__ void block_row_max_kernel(const T* Z, int n, int t, int nBlocks, int ld, T* rowMax) +{ + const int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= n * nBlocks) return; + const int b = index / n; + const int i = index % n; + const T* z = Z + (size_t)i * ld + b * t; + T h = 0; + for (int j = 0; j < t; j++) + h = max(h, abs(z[j])); + rowMax[index] = h; } -// For each block b of t columns of Z: h(i) = max_j |Z_b(i, j)|; unitRows[b*t + r] = the row with the -// r-th largest h (ties go to the lower row). One thread block per column block. +// unitRows[b*t + r] = the row with the r-th largest rowMax[b * n + i] (ties go to the lower row). +// One thread block per column block. template -__global__ void select_unit_columns_kernel(const T* Z, int n, int t, int* unitRows) +__global__ void select_unit_columns_kernel(const T* rowMax, int n, int t, int* unitRows) { __shared__ T bestValue[1024]; __shared__ int bestRow[1024]; __shared__ int chosen[32]; - const T* block = Z + (size_t)blockIdx.x * n * t; + const T* h = rowMax + (size_t)blockIdx.x * n; for (int r = 0; r < t; r++) { @@ -652,10 +715,7 @@ __global__ void select_unit_columns_kernel(const T* Z, int n, int t, int* unitRo used = used or (chosen[q] == i); if (used) continue; - T h = 0; - for (int j = 0; j < t; j++) - h = max(h, abs(block[(size_t)j * n + i])); - if (h > value) { value = h; row = i; } + if (h[i] > value) { value = h[i]; row = i; } } bestValue[threadIdx.x] = value; bestRow[threadIdx.x] = row; @@ -686,19 +746,20 @@ __global__ void select_unit_columns_kernel(const T* Z, int n, int t, int* unitRo } template -void cuda_select_unit_columns_impl(const T* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream) +void cuda_select_unit_columns_impl(const T* Z, int n, int t, int nBlocks, int ld, T* rowMax, int* unitRows, cudaStream_t stream) { - select_unit_columns_kernel<<>>(Z, n, t, unitRows); + block_row_max_kernel<<<(n * nBlocks + 255) / 256, 256, 0, stream>>>(Z, n, t, nBlocks, ld, rowMax); + select_unit_columns_kernel<<>>(rowMax, n, t, unitRows); } -void cuda_select_unit_columns(const float* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream) +void cuda_select_unit_columns(const float* Z, int n, int t, int nBlocks, int ld, float* rowMax, int* unitRows, cudaStream_t stream) { - cuda_select_unit_columns_impl(Z, n, t, nBlocks, unitRows, stream); + cuda_select_unit_columns_impl(Z, n, t, nBlocks, ld, rowMax, unitRows, stream); } -void cuda_select_unit_columns(const double* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream) +void cuda_select_unit_columns(const double* Z, int n, int t, int nBlocks, int ld, double* rowMax, int* unitRows, cudaStream_t stream) { - cuda_select_unit_columns_impl(Z, n, t, nBlocks, unitRows, stream); + cuda_select_unit_columns_impl(Z, n, t, nBlocks, ld, rowMax, unitRows, stream); } // norms[p] = max over the t columns for p in both iterations, for p = 2 .. pTop: diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index a5a7d8a9..7468258f 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -79,25 +79,31 @@ std::vector MemcpyDeviceToHostVector(const T* dptr, int n) double cuda_max_abs(double* v, int length); float cuda_max_abs(float* v, int length); -// The starting matrix X (n, t) of the block 1-norm estimator: first column 1/n, others random +-1/n. +// The starting matrix X (n, t), row-major, of the block 1-norm estimator: first column 1/n, others random +-1/n. void initialize_norm_x_matrix(float* data, int n, int m); void initialize_norm_x_matrix(double* data, int n, int m); -// Block 1-norm estimation, see BlockNormEstimator in BeagleGPUActionImpl.h. -// y(:, 0:nApplied) = A x(:, 0:nApplied) for column-major y, x with n rows and A in CSR; then nAppend more -// columns of y are set to sign(signSource) if signSource is non-null, otherwise to unit vectors e(unitRows[c]). -void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const float* values, const float* x, float* y, - int nApplied, int nAppend, const float* signSource, const int* unitRows, cudaStream_t stream); -void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const double* values, const double* x, double* y, - int nApplied, int nAppend, const double* signSource, const int* unitRows, cudaStream_t stream); - -// sums[c] = sum_i |x(i, c)| for the nColumns columns of x (n rows). -void cuda_column_abs_sums(const float* x, int n, int nColumns, float* sums, cudaStream_t stream); -void cuda_column_abs_sums(const double* x, int n, int nColumns, double* sums, cudaStream_t stream); - -// For each of nBlocks blocks of t columns of Z (n rows): the t rows with the largest max_j |Z(i,j)| -> unitRows[b*t ...]. -void cuda_select_unit_columns(const float* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream); -void cuda_select_unit_columns(const double* Z, int n, int t, int nBlocks, int* unitRows, cudaStream_t stream); +// Block 1-norm estimation, see BlockNormEstimator in BeagleGPUActionImpl.h. Matrices are row-major with n rows +// and row stride ld*. +// y(:, 0:nApplied) = A x(:, 0:nApplied) with A in CSR; then nAppend more columns of y are set to sign(signSource) +// if signSource is non-null, otherwise to unit vectors e(unitRows[c]). +void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const float* values, const float* x, int ldx, float* y, int ldy, + int nApplied, int nAppend, const float* signSource, int ldSign, const int* unitRows, cudaStream_t stream); +void cuda_norm_chain_step(int n, const int* rowOffsets, const int* columns, const double* values, const double* x, int ldx, double* y, int ldy, + int nApplied, int nAppend, const double* signSource, int ldSign, const int* unitRows, cudaStream_t stream); + +// sums[c] = sum_i |x(i, c)| for the first nColumns columns of x (n rows, row stride ld). +void cuda_column_abs_sums(const float* x, int n, int nColumns, int ld, float* sums, cudaStream_t stream); +void cuda_column_abs_sums(const double* x, int n, int nColumns, int ld, double* sums, cudaStream_t stream); + +// For each of nBlocks blocks of t columns of Z (n rows, row stride ld): the t rows with the largest +// max_j |Z(i,j)| -> unitRows[b*t ...]. rowMax is scratch for n * nBlocks values. +void cuda_select_unit_columns(const float* Z, int n, int t, int nBlocks, int ld, float* rowMax, int* unitRows, cudaStream_t stream); +void cuda_select_unit_columns(const double* Z, int n, int t, int nBlocks, int ld, double* rowMax, int* unitRows, cudaStream_t stream); + +// dst[i] = src[index[i]] for i < n. +void cuda_gather(const float* src, const int* index, int n, float* dst, cudaStream_t stream); +void cuda_gather(const double* src, const int* index, int n, double* dst, cudaStream_t stream); // norms[p] for p = 2..pTop: the largest column sum for p in either iteration (see the kernel). void cuda_combine_norm_estimates(const float* sums1, const float* sums2, int t, int pTop, float* norms, cudaStream_t stream); From c7542de197138f64068c3d2458e5e8ae0949558e Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Wed, 28 Oct 2026 15:06:25 -0500 Subject: [PATCH 417/421] GPU action: pre-order partials, edge derivatives and cross products - updatePrePartials: dest = exp(t Q^T) (exp(t_sib Q) sibling o parent), scheduled in dependency levels and batched in the Horner kernels like post-order. Q^T actions use the device B^T with its own norm estimates. Only a write scaling index rescales, as in the CPU plugin. - Sibling actions: once pre-order is used, post-order keeps each child's exp(t Q) partials in a per-buffer cache. A cached action is used only if the buffer, its matrix (edge length, eigen index) and Q are unchanged, tracked by generation counters, so a rejected proposal undone by index flips or re-sent edge lengths cannot leave a stale action (the CPU plugin's reject/restore bug). Otherwise it is recomputed. - setRootPrePartials, setSparseDifferentialMatrix. - calculateEdgeDerivatives: fused D*post, pre and per-pattern reduction, with D = Q (B + mu I) or a differential matrix; ratio and weighted sums on the device, one copy back. - calculateCrossProducts: SDDMM restricted to Q's sparsity pattern, accumulated into the caller's dense S x S array (the base-class version would read the new partials layout wrongly). - Product kernels accept right == nullptr (rescale only). Matches the CPU action plugin to <= 2e-13 (pre-order partials, dlogL/dt, differential-matrix derivatives, cross products on Q's pattern); 0 wrong gradients in the reject/restore test; compute-sanitizer clean. Gradient (pre-order + derivatives) is 6-33x faster than the CPU plugin on 10 threads for MM M=8..32 and IGC. Co-Authored-By: Claude Opus 5.5 --- libhmsbeagle/GPU/BeagleGPUActionImpl.h | 122 ++++- libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 624 ++++++++++++++++++++--- libhmsbeagle/GPU/cuda_ops.cu | 219 +++++++- libhmsbeagle/GPU/cuda_ops.h | 52 +- 4 files changed, 926 insertions(+), 91 deletions(-) diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 83c2dfd4..16e195d6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -35,6 +35,8 @@ #include "libhmsbeagle/GPU/BeagleGPUImpl.h" #include +#include +#include #include #include #include @@ -256,6 +258,20 @@ struct ShiftedCsrBuilder } }; +// max_i sum_j |B(i,j)| = ||B^T||_1 +template +double normPInf(int n, const HostCsr& B) +{ + double result = 0; + for (int i = 0; i < n; i++) { + double sum = 0; + for (int k = B.offsets[i]; k < B.offsets[i + 1]; k++) + sum += std::abs(B.values[k]); + result = std::max(result, sum); + } + return result; +} + // max_j sum_i |B(i,j)| template double normP1(int n, const HostCsr& B) @@ -800,11 +816,10 @@ struct BlockNormEstimator } // Returns ||B^p||_1 estimates for p = 2..pTop at index p. (Indices 0 and 1 are not set.) - // BT is B^T in CSR. - std::vector estimate(const SpMatrixDevice& B, const int* btOffsets, const int* btColumns, const Real* btValues, - cudaStream_t stream) + // B and B^T are (n, n) in CSR. + std::vector estimate(const int* bOffsets, const int* bColumns, const Real* bValues, + const int* btOffsets, const int* btColumns, const Real* btValues, cudaStream_t stream) { - assert(B.rows() == n and B.cols() == n); const int nBlocks = pTop - 1; const int ldY = pTop * t; @@ -812,7 +827,7 @@ struct BlockNormEstimator for (int p = 1; p <= pTop; p++) { const Real* x = (p == 1) ? X0 : Y + (p - 2) * t; - cuda_norm_chain_step(n, B.offsets, B.inner, B.values, x, (p == 1) ? t : ldY, Y + (p - 1) * t, ldY, + cuda_norm_chain_step(n, bOffsets, bColumns, bValues, x, (p == 1) ? t : ldY, Y + (p - 1) * t, ldY, t, 0, (Real*)nullptr, 0, nullptr, stream); } cuda_column_abs_sums(Y, n, pTop * t, ldY, sums1, stream); @@ -839,13 +854,13 @@ struct BlockNormEstimator // 4. B^p X_p, in block pTop - p. columns = t; - cuda_norm_chain_step(n, B.offsets, B.inner, B.values, (Real*)nullptr, 0, out, columns, 0, t, (Real*)nullptr, 0, unitRows, stream); + cuda_norm_chain_step(n, bOffsets, bColumns, bValues, (Real*)nullptr, 0, out, columns, 0, t, (Real*)nullptr, 0, unitRows, stream); std::swap(in, out); for (int k = 1; k <= pTop; k++) { const bool append = (k < nBlocks); const int outColumns = columns + (append ? t : 0); - cuda_norm_chain_step(n, B.offsets, B.inner, B.values, in, columns, out, outColumns, columns, append ? t : 0, + cuda_norm_chain_step(n, bOffsets, bColumns, bValues, in, columns, out, outColumns, columns, append ? t : 0, (Real*)nullptr, 0, append ? unitRows + k * t : nullptr, stream); std::swap(in, out); columns = outColumns; @@ -930,6 +945,13 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int scaleIndex, double* outPartials) override; + // Pre-order partials are the stationary frequencies. (Only count == 1, as in the CPU implementation.) + int setRootPrePartials(const int* bufferIndices, const int* stateFrequenciesIndices, int count) override; + + // A derivative matrix for calculateEdgeDerivatives( ), at an index of its own or replacing setSparseMatrix( )'s Q there. + int setSparseDifferentialMatrix(int matrixIndex, const int* rowIndices, const int* colIndices, const double* values, + int numNonZeros) override; + int setSparseMatrix(int matrixIndex, const int* rowIndices, const int* colIndices, @@ -947,7 +969,8 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // Action instances have no dense transition matrices (see createInstance), so these are not available. // (The base versions of the first three call the dense, non-virtual upPartials/upPrePartials.) - int updatePrePartials(const int*, int, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + // Pre-order: dest = exp(t Q^T) (exp(t_sibling Q) sibling o parent), see calcPrePartials( ). + int updatePrePartials(const int* operations, int operationCount, int cumulativeScalingIndex) override; int updatePartialsByPartition(const int*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } int updatePrePartialsByPartition(const int*, int) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } int setEigenDecomposition(int, const double*, const double*, const double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } @@ -962,13 +985,29 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int, double*, double*, double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } int calculateEdgeLogLikelihoodsByPartition(const int*, const int*, const int*, const int*, const int*, const int*, const int*, const int*, const int*, int, int, double*, double*, double*, double*, double*, double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } - int calculateEdgeDerivatives(const int*, const int*, const int*, const int*, const int*, const int*, - int, double*, double*, double*) override { return BEAGLE_ERROR_NO_IMPLEMENTATION; } + + // Per pattern: sum_c w_c r_c pre^T D post / sum_c w_c pre^T post, with D the derivative matrix (Q itself for + // an index set by setSparseMatrix( )). Categories use rate set 0, as in the CPU implementation. + int calculateEdgeDerivatives(const int* postBufferIndices, const int* preBufferIndices, const int* derivativeMatrixIndices, + const int* categoryWeightsIndices, const int* categoryRatesIndices, const int* cumulativeScaleIndices, + int count, double* outDerivatives, double* outSumDerivatives, double* outSumSquaredDerivatives) override; + + // Adds sum_p (patternWeight_p / pre^T post) sum_c w_c r_c t pre(c, i, p) post(c, j, p) to outSumDerivatives[i * S + j], + // but only for the (i, j) in the sparsity pattern of Q (the union over the eigen indices that are set), which is + // where substitution-model derivatives are non-zero. (The CPU implementation computes all S^2 entries.) + // outSumSquaredDerivatives is not used, as in the CPU implementation. + int calculateCrossProducts(const int* postBufferIndices, const int* preBufferIndices, const int* categoryRatesIndices, + const int* categoryWeightsIndices, const double* edgeLengths, int count, + double* outSumDerivatives, double* outSumSquaredDerivatives) override; protected: // Per eigen index: B = Q - mu*I. std::vector> hBs; + // ||B^T||_1 and the d_p of B^T, for the actions of pre-order (estimated on first use after setSparseMatrix( )). + std::vector hBt1Norms; + std::vector> hdsT; + std::vector hdsTCurrent; std::vector> csrBuilders; std::vector hMuBs; std::vector hB1Norms; @@ -1033,6 +1072,45 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // Per-pattern partial results of the product and root kernels. Real* dReductionScratch = nullptr; + // Post-order caches each child's action exp(t Q) partials, which pre-order needs for the sibling. A cached action + // is used only if the buffer, the matrix (edge length, Q) and Q are all unchanged since it was computed; generation + // counters track that, so a rejected proposal that BEAST undoes by flipping indices or re-sending edge lengths + // cannot leave a stale action behind. + std::vector partialsGenerations; // per partials buffer: bumped whenever it is written + std::vector matrixGenerations; // per matrix index: bumped by updateTransitionMatrices( ) + std::vector qGenerations; // per eigen index: bumped by setSparseMatrix( ) + struct ActionCacheEntry + { + Real* data = nullptr; // kCategoryCount matrices + bool valid = false; + int matrixIndex = -1; + uint64_t partialsGeneration = 0; + uint64_t matrixGeneration = 0; + uint64_t qGeneration = 0; + }; + std::vector actionCache; // per partials buffer + bool cacheActions = false; // turned on by the first pre-order update + + // Derivative matrices, indexed like setSparseMatrix( ) and setSparseDifferentialMatrix( ). eigenIndex >= 0 means Q + // of that eigen index (B + mu I); otherwise the matrix is stored here. + struct DerivativeMatrix + { + int eigenIndex = -1; + int* offsets = nullptr; + int* columns = nullptr; + Real* values = nullptr; + }; + std::vector derivativeMatrices; + Real* dDerivativeScratch = nullptr; + Real* dDerivativeOutput = nullptr; + size_t derivativeOutputSize = 0; + std::vector hDerivativeOutput; + // Cross products: the pattern (when it is not one eigen index's) and the values. + int* dCrossOffsets = nullptr; + int* dCrossColumns = nullptr; + Real* dCrossValues = nullptr; + size_t crossCapacity = 0; + // All work is issued on the GPUInterface's stream 0, so it is ordered with the base class's // copies and kernels (two different blocking streams would not be). See setCudaDevice( ). cudaStream_t stream = nullptr; @@ -1131,6 +1209,9 @@ class BeagleGPUActionImpl : public BeagleGPUImpl // Copy BEAGLE-layout partials (for one category if tipPartials, else for all) into dPartials[bufferIndex]. void uploadPartials(int bufferIndex, const Real* inPartials, bool tipPartials); + // Post-order: destination = action(children[0], edges[0]) o action(children[1], edges[1]). + // Pre-order: children[0] is the parent's pre-order buffer and children[1] the sibling's post-order buffer; + // edges[0] is the destination's edge and edges[1] the sibling's. struct PartialsOperation { int destination; @@ -1139,8 +1220,21 @@ class BeagleGPUActionImpl : public BeagleGPUImpl Real* scalingFactors; // nullptr means no rescaling }; + // Groups operations into batches of independent ones (one tree level, up to maxBatchOperations each), in an + // order that respects their dependencies through the partials and scaling buffers. + std::vector> scheduleBatches(const std::vector& operations) const; + // The operations must be independent: no operation reads or writes another one's destination. void calcPartialsPartials(const std::vector& operations, Real* cumulativeScalingBuffer); + void calcPrePartials(const std::vector& operations, Real* cumulativeScalingBuffer); + + // Whether actionCache[buffer] holds exp(t Q) partials for the buffer's current contents and matrixIndex. + bool actionCached(int buffer, int matrixIndex) const; + // Where post-order may put a child's action so that it is cached, or nullptr. + Real* actionCacheSlot(int buffer, int matrixIndex); + + // Makes hdsT[eigenIndex] current. + void estimateTransposedNorms(int eigenIndex); // out = exp(multiplier * Q_eigenIndex) * in for one (kPaddedStateCount, kPaddedPatternCount) matrix. // in is not modified; scratch is 2 or 3 matrices of work space with row stride scratchStride. @@ -1153,8 +1247,12 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int scratchStride; int eigenIndex; double multiplier; + bool transpose; // use Q^T }; + // Points group.scratch at the work blocks of group number g (< 2 * kCategoryCount * maxBatchOperations). + void setActionScratch(ActionGroup& group, size_t g); + // With fewer than 32 patterns, the Horner work blocks are interleaved (see calcPartialsPartials( )). bool interleavedScratch() const { return kPaddedPatternCount < 32; } @@ -1162,9 +1260,9 @@ class BeagleGPUActionImpl : public BeagleGPUImpl void computeActions(const std::vector& groups); std::tuple getStatistics2(double t, int nCol, double edgeMultiplier, - int eigenIndex) const; + int eigenIndex, bool transpose = false) const; - double getDValue(int p, int eigenIndex) const; + double getDValue(int p, int eigenIndex, bool transpose = false) const; int PrintfDeviceVector(Real* dPtr, int length, double checkValue, int *signal, Real r); int PrintfDeviceVector(int* dPtr, int length, double checkValue, int *signal, Real r); diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index c899bbcf..97bd18f6 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -293,6 +293,14 @@ int BeagleGPUActionImpl::createInstance(int tipCount, hBs.resize(kEigenDecompCount); csrBuilders.resize(kEigenDecompCount); + hBt1Norms.resize(kEigenDecompCount); + hdsT.resize(kEigenDecompCount); + hdsTCurrent.assign(kEigenDecompCount, false); + partialsGenerations.assign(kPartialsBufferCount, 0); + matrixGenerations.assign(kMatrixCount, 0); + qGenerations.assign(kEigenDecompCount, 0); + actionCache.resize(kPartialsBufferCount); + derivativeMatrices.resize(kEigenDecompCount + kMatrixCount); hMuBs.resize(kEigenDecompCount); hB1Norms.resize(kEigenDecompCount); hds.resize(kEigenDecompCount); @@ -346,6 +354,18 @@ BeagleGPUActionImpl::~BeagleGPUActionImpl() cudaFree(dActionResults); cudaFree(dActionScratch); cudaFree(dReductionScratch); + for (auto& entry: actionCache) + cudaFree(entry.data); + for (auto& matrix: derivativeMatrices) { + cudaFree(matrix.offsets); + cudaFree(matrix.columns); + cudaFree(matrix.values); + } + cudaFree(dDerivativeScratch); + cudaFree(dDerivativeOutput); + cudaFree(dCrossOffsets); + cudaFree(dCrossColumns); + cudaFree(dCrossValues); for (int i = 0; i < kEigenDecompCount; i++) { cudaFree(dBsCsrOffsetsCache[i]); @@ -418,6 +438,7 @@ void BeagleGPUActionImpl::uploadPartials(int bufferIndex, co out[(size_t) state * kPaddedPatternCount + pattern] = in[(size_t) pattern * kStateCount + state]; } gpu->MemcpyHostToDevice(dPartials[bufferIndex], hPartialsTransposed.data(), sizeof(Real) * kPartialsSize); + partialsGenerations[bufferIndex]++; } BEAGLE_GPU_TEMPLATE @@ -573,16 +594,7 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, if (cumulativeScalingIndex != BEAGLE_OP_NONE) cumulativeScalingBuffer = (Real*)dScalingFactors[cumulativeScalingIndex]; - // Operations are grouped into levels: each goes to the earliest level after every earlier operation that - // writes a buffer it reads, or reads or writes a buffer (partials or scaling) it writes. The operations of - // a level are independent, so they can be computed together. std::vector parsed(operationCount); - std::vector levels(operationCount); - std::vector writeLevel(kPartialsBufferCount, -1); - std::vector readLevel(kPartialsBufferCount, -1); - std::map scalingWriteLevel; - int levelCount = 0; - for (int op = 0; op < operationCount; op++) { const int numOps = BEAGLE_OP_COUNT; @@ -620,27 +632,9 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, operation.edges[0] = firstChildSubstitutionMatrixIndex; operation.edges[1] = secondChildSubstitutionMatrixIndex; operation.scalingFactors = (rescale == 1) ? scalingFactors : nullptr; - - int level = std::max(writeLevel[operation.destination], readLevel[operation.destination]) + 1; - for (int child: operation.children) - level = std::max(level, writeLevel[child] + 1); - if (operation.scalingFactors) { - auto it = scalingWriteLevel.find(operation.scalingFactors); - if (it != scalingWriteLevel.end()) - level = std::max(level, it->second + 1); - scalingWriteLevel[operation.scalingFactors] = level; - } - writeLevel[operation.destination] = level; - for (int child: operation.children) - readLevel[child] = std::max(readLevel[child], level); - levels[op] = level; - levelCount = std::max(levelCount, level + 1); } - std::vector batch; - auto flush = [&]() { - if (batch.empty()) - return; + for (const auto& batch: scheduleBatches(parsed)) { calcPartialsPartials(batch, cumulativeScalingBuffer); if (kFlags & BEAGLE_FLAG_SCALING_ALWAYS) { for (const auto& operation: batch) { @@ -659,28 +653,408 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, } } } - batch.clear(); - }; + } + +#ifdef BEAGLE_DEBUG_SYNCH + gpu->SynchronizeHost(); +#endif + +#ifdef BEAGLE_DEBUG_FLOW + fprintf(stderr, "\tLeaving BeagleGPUActionImpl::upPartials\n"); +#endif + return BEAGLE_SUCCESS; +} + +BEAGLE_GPU_TEMPLATE +std::vector::PartialsOperation>> +BeagleGPUActionImpl::scheduleBatches(const std::vector& operations) const +{ + // Each operation goes to the earliest level after every earlier operation that writes a buffer it reads, or + // reads or writes a buffer (partials or scaling) it writes. The operations of a level are independent. + const int operationCount = (int) operations.size(); + std::vector levels(operationCount); + std::vector writeLevel(kPartialsBufferCount, -1); + std::vector readLevel(kPartialsBufferCount, -1); + std::map scalingWriteLevel; + int levelCount = 0; + + for (int op = 0; op < operationCount; op++) { + const PartialsOperation& operation = operations[op]; + int level = std::max(writeLevel[operation.destination], readLevel[operation.destination]) + 1; + for (int child: operation.children) + level = std::max(level, writeLevel[child] + 1); + if (operation.scalingFactors) { + auto it = scalingWriteLevel.find(operation.scalingFactors); + if (it != scalingWriteLevel.end()) + level = std::max(level, it->second + 1); + scalingWriteLevel[operation.scalingFactors] = level; + } + writeLevel[operation.destination] = level; + for (int child: operation.children) + readLevel[child] = std::max(readLevel[child], level); + levels[op] = level; + levelCount = std::max(levelCount, level + 1); + } + + std::vector> batches; for (int level = 0; level < levelCount; level++) { + std::vector batch; for (int op = 0; op < operationCount; op++) { if (levels[op] != level) continue; - if ((int) batch.size() == maxBatchOperations) - flush(); - batch.push_back(parsed[op]); + if ((int) batch.size() == maxBatchOperations) { + batches.push_back(batch); + batch.clear(); + } + batch.push_back(operations[op]); } - flush(); + if (not batch.empty()) + batches.push_back(batch); } + return batches; +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::updatePrePartials(const int* operations, + int operationCount, + int cumulativeScalingIndex) +{ + setCudaDevice(); + return upPrePartials(false, operations, operationCount, cumulativeScalingIndex); +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::upPrePartials(bool byPartition, + const int *operations, + int operationCount, + int cumulativeScalingIndex) +{ + // From now on post-order keeps the children's actions for the siblings of later pre-order updates. + cacheActions = true; + + Real* cumulativeScalingBuffer = nullptr; + if (cumulativeScalingIndex != BEAGLE_OP_NONE) + cumulativeScalingBuffer = (Real*)dScalingFactors[cumulativeScalingIndex]; + + // [dest, writeScaling, readScaling, parent (pre-order), dest's matrix, sibling (post-order), sibling's matrix]. + // As in the CPU implementation, only a write scaling index rescales. + std::vector parsed(operationCount); + for (int op = 0; op < operationCount; op++) { + const int* x = operations + op * BEAGLE_OP_COUNT; + PartialsOperation& operation = parsed[op]; + operation.destination = x[0]; + operation.children[0] = x[3]; + operation.children[1] = x[5]; + operation.edges[0] = x[4]; + operation.edges[1] = x[6]; + operation.scalingFactors = (x[1] >= 0) ? (Real*)dScalingFactors[x[1]] : nullptr; + for (int buffer: {x[0], x[3], x[5]}) + if (buffer < 0 or buffer >= kPartialsBufferCount or dPartials[buffer] == 0) + return BEAGLE_ERROR_OUT_OF_RANGE; + } + + for (const auto& batch: scheduleBatches(parsed)) + calcPrePartials(batch, cumulativeScalingBuffer); #ifdef BEAGLE_DEBUG_SYNCH gpu->SynchronizeHost(); #endif + return BEAGLE_SUCCESS; +} -#ifdef BEAGLE_DEBUG_FLOW - fprintf(stderr, "\tLeaving BeagleGPUActionImpl::upPartials\n"); -#endif +BEAGLE_GPU_TEMPLATE +bool BeagleGPUActionImpl::actionCached(int buffer, int matrixIndex) const +{ + const ActionCacheEntry& entry = actionCache[buffer]; + return entry.valid and entry.data and entry.matrixIndex == matrixIndex and + entry.partialsGeneration == partialsGenerations[buffer] and + entry.matrixGeneration == matrixGenerations[matrixIndex] and + entry.qGeneration == qGenerations[hEigenMaps[matrixIndex]]; +} + +BEAGLE_GPU_TEMPLATE +Real* BeagleGPUActionImpl::actionCacheSlot(int buffer, int matrixIndex) +{ + if (not cacheActions) + return nullptr; + ActionCacheEntry& entry = actionCache[buffer]; + if (not entry.data) { + const size_t size = (size_t) kCategoryCount * kPaddedStateCount * kPaddedPatternCount; + if (cudaMalloc((void**) &entry.data, size * sizeof(Real)) != cudaSuccess) { + // Out of memory: pre-order recomputes the siblings' actions instead. + cudaGetLastError(); + entry.data = nullptr; + return nullptr; + } + } + entry.valid = true; + entry.matrixIndex = matrixIndex; + entry.partialsGeneration = partialsGenerations[buffer]; + entry.matrixGeneration = matrixGenerations[matrixIndex]; + entry.qGeneration = qGenerations[hEigenMaps[matrixIndex]]; + return entry.data; +} + +BEAGLE_GPU_TEMPLATE +void BeagleGPUActionImpl::estimateTransposedNorms(int eigenIndex) +{ + if (hdsTCurrent[eigenIndex]) + return; + // The estimator for B^T: its transpose is B. + const auto& dB = dBs[eigenIndex]; + auto norms = normEstimator.estimate(dBtOffsets[eigenIndex], dBtColumns[eigenIndex], dBtValues[eigenIndex], + dB.offsets, dB.inner, dB.values, stream); + const int pTop = getPMax() + 1; + hdsT[eigenIndex].assign(pTop + 1, 1.0); + hdsT[eigenIndex][1] = hBt1Norms[eigenIndex]; + for (int p = 2; p <= pTop; p++) + hdsT[eigenIndex][p] = pow(norms[p], 1.0 / Real(p)); + hdsTCurrent[eigenIndex] = true; +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setRootPrePartials(const int* bufferIndices, + const int* stateFrequenciesIndices, + int count) +{ + if (count != 1) + return BEAGLE_ERROR_NO_IMPLEMENTATION; + const int buffer = bufferIndices[0]; + if (buffer < 0 or buffer >= kPartialsBufferCount or dPartials[buffer] == 0) + return BEAGLE_ERROR_OUT_OF_RANGE; + setCudaDevice(); + cuda_root_pre_partials((Real*) dPartials[buffer], (const Real*) dFrequencies[stateFrequenciesIndices[0]], + kPaddedStateCount, kCategoryCount, kPaddedPatternCount, kPatternCount, stream); + partialsGenerations[buffer]++; + return BEAGLE_SUCCESS; +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::setSparseDifferentialMatrix(int matrixIndex, + const int* rowIndices, + const int* colIndices, + const double* values, + int numNonZeros) +{ + if (matrixIndex < 0 or matrixIndex >= (int) derivativeMatrices.size()) + return BEAGLE_ERROR_OUT_OF_RANGE; + setCudaDevice(); + + // Duplicates are summed; every row gets a (possibly zero) diagonal entry, which is harmless. + std::vector realValues(values, values + numNonZeros); + HostCsr D; + buildShiftedCsr(kPaddedStateCount, rowIndices, colIndices, realValues.data(), numNonZeros, Real(0), D); + + DerivativeMatrix& matrix = derivativeMatrices[matrixIndex]; + cudaDeviceDelete(matrix.offsets); + cudaDeviceDelete(matrix.columns); + cudaDeviceDelete(matrix.values); + matrix.offsets = cudaDeviceNew(kPaddedStateCount + 1); + matrix.columns = cudaDeviceNew(D.nonZeros()); + matrix.values = cudaDeviceNew(D.nonZeros()); + MemcpyHostToDevice(matrix.offsets, D.offsets.data(), kPaddedStateCount + 1); + MemcpyHostToDevice(matrix.columns, D.columns.data(), D.nonZeros()); + MemcpyHostToDevice(matrix.values, D.values.data(), D.nonZeros()); + matrix.eigenIndex = -1; + return BEAGLE_SUCCESS; +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::calculateEdgeDerivatives(const int* postBufferIndices, + const int* preBufferIndices, + const int* derivativeMatrixIndices, + const int* categoryWeightsIndices, + const int* categoryRatesIndices, + const int* cumulativeScaleIndices, + int count, + double* outDerivatives, + double* outSumDerivatives, + double* outSumSquaredDerivatives) +{ + if (count <= 0) + return BEAGLE_SUCCESS; + if (kCategoryCount > kMaxDerivativeCategories) + return BEAGLE_ERROR_NO_IMPLEMENTATION; + for (int node = 0; node < count; node++) { + const int d = derivativeMatrixIndices[node]; + if (d < 0 or d >= (int) derivativeMatrices.size() or + (derivativeMatrices[d].eigenIndex < 0 and derivativeMatrices[d].offsets == nullptr)) + return BEAGLE_ERROR_OUT_OF_RANGE; + for (int buffer: {postBufferIndices[node], preBufferIndices[node]}) + if (buffer < 0 or buffer >= kPartialsBufferCount or dPartials[buffer] == 0) + return BEAGLE_ERROR_OUT_OF_RANGE; + } + setCudaDevice(); + if (not dDerivativeScratch) + dDerivativeScratch = cudaDeviceNew(cuda_edge_derivative_scratch_size(kPaddedStateCount, kCategoryCount, kPaddedPatternCount)); + // derivatives (count * kPatternCount), then the sums (count), then the sums of squares (count) + const size_t outputSize = (size_t) count * (kPatternCount + 2); + if (outputSize > derivativeOutputSize) { + cudaDeviceDelete(dDerivativeOutput); + dDerivativeOutput = cudaDeviceNew(outputSize); + derivativeOutputSize = outputSize; + } + Real* derivatives = dDerivativeOutput; + Real* sums = derivatives + (size_t) count * kPatternCount; + Real* sumSquares = sums + count; + + // As in the CPU implementation, the categories use rate set 0. + EdgeDerivativeBatch batch; + for (int c = 0; c < kCategoryCount; c++) + batch.categoryMultipliers[c] = (Real) hCategoryRates[0][c]; + const Real* categoryWeights = (const Real*) dWeights[categoryWeightsIndices[0]]; + + // Consecutive nodes with the same derivative matrix go in one batch. + for (int first = 0; first < count;) { + const int d = derivativeMatrixIndices[first]; + batch.count = 0; + while (first + batch.count < count and batch.count < kMaxEdgeDerivativeNodes and + derivativeMatrixIndices[first + batch.count] == d) { + batch.post[batch.count] = (const Real*) dPartials[postBufferIndices[first + batch.count]]; + batch.pre[batch.count] = (const Real*) dPartials[preBufferIndices[first + batch.count]]; + batch.count++; + } + const DerivativeMatrix& matrix = derivativeMatrices[d]; + const int e = matrix.eigenIndex; + // Q = B + mu I + const int* offsets = (e >= 0) ? dBs[e].offsets : matrix.offsets; + const int* columns = (e >= 0) ? dBs[e].inner : matrix.columns; + const Real* values = (e >= 0) ? dBs[e].values : matrix.values; + const Real shift = (e >= 0) ? hMuBs[e] : Real(0); + cuda_edge_derivatives(batch, kPaddedStateCount, kCategoryCount, kPaddedPatternCount, kPatternCount, + offsets, columns, values, shift, categoryWeights, (const Real*) dPatternWeights, + dDerivativeScratch, derivatives + (size_t) first * kPatternCount, sums + first, sumSquares + first, stream); + first += batch.count; + } + + hDerivativeOutput.resize(outputSize); + auto status = cudaMemcpyAsync(hDerivativeOutput.data(), dDerivativeOutput, outputSize * sizeof(Real), cudaMemcpyDeviceToHost, stream); + if (status == cudaSuccess) + status = cudaStreamSynchronize(stream); + if (status != cudaSuccess) + return BEAGLE_ERROR_GENERAL; + + const Real* h = hDerivativeOutput.data(); + if (outDerivatives) + std::copy(h, h + (size_t) count * kPatternCount, outDerivatives); + if (outSumDerivatives) + std::copy(h + (size_t) count * kPatternCount, h + (size_t) count * kPatternCount + count, outSumDerivatives); + if (outSumSquaredDerivatives) + std::copy(h + (size_t) count * (kPatternCount + 1), h + outputSize, outSumSquaredDerivatives); + return BEAGLE_SUCCESS; +} + +BEAGLE_GPU_TEMPLATE +int BeagleGPUActionImpl::calculateCrossProducts(const int* postBufferIndices, + const int* preBufferIndices, + const int* categoryRatesIndices, + const int* categoryWeightsIndices, + const double* edgeLengths, + int count, + double* outSumDerivatives, + double* outSumSquaredDerivatives) +{ + if (count <= 0) + return BEAGLE_SUCCESS; + if (kCategoryCount > kMaxDerivativeCategories) + return BEAGLE_ERROR_NO_IMPLEMENTATION; + for (int node = 0; node < count; node++) + for (int buffer: {postBufferIndices[node], preBufferIndices[node]}) + if (buffer < 0 or buffer >= kPartialsBufferCount or dPartials[buffer] == 0) + return BEAGLE_ERROR_OUT_OF_RANGE; + setCudaDevice(); + + // The pattern: the union of the patterns of the eigen indices that are set (normally they are all the same). + std::vector set; + for (int e = 0; e < kEigenDecompCount; e++) + if (not hBs[e].offsets.empty()) + set.push_back(e); + if (set.empty()) + return BEAGLE_ERROR_GENERAL; + const int n = kPaddedStateCount; + HostCsr pattern = hBs[set[0]]; + bool shared = true; + for (int e: set) + shared = shared and hBs[e].offsets == pattern.offsets and hBs[e].columns == pattern.columns; + if (not shared) { + std::vector> rows(n); + for (int e: set) + for (int i = 0; i < n; i++) + for (int k = hBs[e].offsets[i]; k < hBs[e].offsets[i + 1]; k++) + rows[i].insert(hBs[e].columns[k]); + pattern.offsets.assign(n + 1, 0); + pattern.columns.clear(); + for (int i = 0; i < n; i++) { + pattern.columns.insert(pattern.columns.end(), rows[i].begin(), rows[i].end()); + pattern.offsets[i + 1] = (int) pattern.columns.size(); + } + } + const int nnz = (int) pattern.columns.size(); + if ((size_t) nnz > crossCapacity) { + cudaDeviceDelete(dCrossColumns); + cudaDeviceDelete(dCrossValues); + dCrossColumns = cudaDeviceNew(nnz); + dCrossValues = cudaDeviceNew(nnz); + crossCapacity = nnz; + } + if (not dCrossOffsets) + dCrossOffsets = cudaDeviceNew(n + 1); + const int* offsets = shared ? dBs[set[0]].offsets : dCrossOffsets; + const int* columns = shared ? dBs[set[0]].inner : dCrossColumns; + if (not shared) { + MemcpyHostToDevice(dCrossOffsets, pattern.offsets.data(), n + 1); + MemcpyHostToDevice(dCrossColumns, pattern.columns.data(), nnz); + } + CHECK_CUDA(cudaMemsetAsync(dCrossValues, 0, nnz * sizeof(Real), stream)); + + // Per node: patternScale[p] = patternWeight[p] / sum_c w_c pre^T post, then the products scaled by w_c r_c t. + if (not dDerivativeScratch) + dDerivativeScratch = cudaDeviceNew(cuda_edge_derivative_scratch_size(kPaddedStateCount, kCategoryCount, kPaddedPatternCount)); + const size_t scaleSize = (size_t) kMaxEdgeDerivativeNodes * kPaddedPatternCount; + if (scaleSize > derivativeOutputSize) { + cudaDeviceDelete(dDerivativeOutput); + dDerivativeOutput = cudaDeviceNew(scaleSize); + derivativeOutputSize = scaleSize; + } + CHECK_CUDA(cudaMemsetAsync(dDerivativeOutput, 0, scaleSize * sizeof(Real), stream)); + + int ratesIndex = categoryRatesIndices ? categoryRatesIndices[0] : 0; + if (ratesIndex < 0 or ratesIndex >= kEigenDecompCount or hCategoryRates[ratesIndex] == nullptr) + ratesIndex = 0; + std::vector weights(kCategoryCount); + CHECK_CUDA(cudaMemcpyAsync(weights.data(), (const Real*) dWeights[categoryWeightsIndices[0]], kCategoryCount * sizeof(Real), + cudaMemcpyDeviceToHost, stream)); + CHECK_CUDA(cudaStreamSynchronize(stream)); + + EdgeDerivativeBatch batch; + for (int first = 0; first < count; first += batch.count) { + batch.count = std::min(count - first, kMaxEdgeDerivativeNodes); + for (int z = 0; z < batch.count; z++) { + batch.post[z] = (const Real*) dPartials[postBufferIndices[first + z]]; + batch.pre[z] = (const Real*) dPartials[preBufferIndices[first + z]]; + } + // Scales are written with stride kPatternCount; kPatternCount == kPaddedPatternCount for this plugin. + cuda_edge_derivatives(batch, kPaddedStateCount, kCategoryCount, kPaddedPatternCount, kPatternCount, + nullptr, nullptr, nullptr, Real(0), (const Real*) dWeights[categoryWeightsIndices[0]], + (const Real*) dPatternWeights, dDerivativeScratch, dDerivativeOutput, nullptr, nullptr, stream); + for (int z = 0; z < batch.count; z++) { + CategoryMultipliers multipliers; + for (int c = 0; c < kCategoryCount; c++) + multipliers.values[c] = weights[c] * (Real) (hCategoryRates[ratesIndex][c] * edgeLengths[first + z]); + cuda_cross_products(batch.pre[z], batch.post[z], kPaddedStateCount, kCategoryCount, kPaddedPatternCount, offsets, columns, + multipliers, dDerivativeOutput + (size_t) z * kPatternCount, dCrossValues, stream); + } + } + + std::vector values(nnz); + CHECK_CUDA(cudaMemcpyAsync(values.data(), dCrossValues, nnz * sizeof(Real), cudaMemcpyDeviceToHost, stream)); + CHECK_CUDA(cudaStreamSynchronize(stream)); + for (int i = 0; i < kStateCount; i++) + for (int k = pattern.offsets[i]; k < pattern.offsets[i + 1]; k++) + if (pattern.columns[k] < kStateCount) + outSumDerivatives[(size_t) i * kStateCount + pattern.columns[k]] += values[k]; return BEAGLE_SUCCESS; } @@ -799,30 +1173,47 @@ int BeagleGPUActionImpl::PrintfDeviceVector(cusparseDnMatDes } BEAGLE_GPU_TEMPLATE -void BeagleGPUActionImpl::calcPartialsPartials(const std::vector& operations, - Real* cumulativeScalingBuffer) { - const size_t matrixSize = (size_t) kPaddedStateCount * kPaddedPatternCount; +void BeagleGPUActionImpl::setActionScratch(ActionGroup& group, size_t g) +{ // With few patterns, the groups' blocks of the 3 scratch matrices are side by side (coalescing the gathers // across groups); otherwise each group uses 2 contiguous blocks. See computeActions( ). + const size_t matrixSize = (size_t) kPaddedStateCount * kPaddedPatternCount; const size_t scratchMatrixSize = (size_t) 2 * kCategoryCount * maxBatchOperations * matrixSize; const bool interleaved = interleavedScratch(); - const size_t scratchStride = interleaved ? (size_t) 2 * kCategoryCount * maxBatchOperations * kPaddedPatternCount : kPaddedPatternCount; + group.scratchStride = interleaved ? (int) (2 * kCategoryCount * maxBatchOperations * kPaddedPatternCount) : kPaddedPatternCount; + group.scratchCount = interleaved ? 3 : 2; + for (int k = 0; k < group.scratchCount; k++) + group.scratch[k] = dActionScratch + k * scratchMatrixSize + g * (interleaved ? kPaddedPatternCount : matrixSize); +} + +BEAGLE_GPU_TEMPLATE +void BeagleGPUActionImpl::calcPartialsPartials(const std::vector& operations, + Real* cumulativeScalingBuffer) { + const size_t matrixSize = (size_t) kPaddedStateCount * kPaddedPatternCount; + // Each child's action goes to the action cache once pre-order is in use (unless the same child appears twice in + // this batch), otherwise to dActionResults. std::vector groups; + std::vector actions(2 * operations.size()); + std::set cachedChildren; for (size_t op = 0; op < operations.size(); op++) { for (int child = 0; child < 2; child++) { + const int buffer = operations[op].children[child]; const int edgeIndex = operations[op].edges[child]; + Real* out = nullptr; + if (cachedChildren.insert(buffer).second) + out = actionCacheSlot(buffer, edgeIndex); + if (not out) + out = dActionResults + (op * 2 + child) * kCategoryCount * matrixSize; + actions[op * 2 + child] = out; for (int category = 0; category < kCategoryCount; category++) { - const size_t g = (op * 2 + child) * kCategoryCount + category; ActionGroup group; - group.in = (const Real*) dPartials[operations[op].children[child]] + category * matrixSize; - group.out = dActionResults + g * matrixSize; - group.scratchCount = interleaved ? 3 : 2; - for (int k = 0; k < group.scratchCount; k++) - group.scratch[k] = dActionScratch + k * scratchMatrixSize + g * (interleaved ? kPaddedPatternCount : matrixSize); - group.scratchStride = scratchStride; + group.in = (const Real*) dPartials[buffer] + category * matrixSize; + group.out = out + category * matrixSize; + setActionScratch(group, (op * 2 + child) * kCategoryCount + category); group.eigenIndex = hEigenMaps[edgeIndex]; group.multiplier = hEdgeMultipliers[edgeIndex * kCategoryCount + category]; + group.transpose = false; groups.push_back(group); } } @@ -833,10 +1224,91 @@ void BeagleGPUActionImpl::calcPartialsPartials(const std::ve bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; PartialsProductBatch products; for (size_t op = 0; op < operations.size(); op++) { - const Real* left = dActionResults + op * 2 * kCategoryCount * matrixSize; auto& product = products.products[products.count++]; - product.left = left; - product.right = left + kCategoryCount * matrixSize; + product.left = actions[op * 2]; + product.right = actions[op * 2 + 1]; + product.parent = (Real*)dPartials[operations[op].destination]; + product.scalingFactors = operations[op].scalingFactors; + partialsGenerations[operations[op].destination]++; + } + cuda_partials_products(products, kCategoryCount * kPaddedStateCount, kPaddedPatternCount, + cumulativeScalingBuffer, scalers_log, dReductionScratch, stream); +} + +BEAGLE_GPU_TEMPLATE +void BeagleGPUActionImpl::calcPrePartials(const std::vector& operations, + Real* cumulativeScalingBuffer) { + const size_t matrixSize = (size_t) kPaddedStateCount * kPaddedPatternCount; + const size_t partialsSize = kCategoryCount * matrixSize; + // Per operation, two blocks of dActionResults: block 0 = the sibling's action o the parent, block 1 = the + // sibling's action when it is not cached, then the result before rescaling. + auto block = [&](size_t op, int k) { return dActionResults + (op * 2 + k) * partialsSize; }; + + // 1. The siblings' actions exp(t_sibling Q) sibling, from the cache when it is current. + std::vector groups; + std::vector siblingActions(operations.size()); + for (size_t op = 0; op < operations.size(); op++) { + const int sibling = operations[op].children[1]; + const int edgeIndex = operations[op].edges[1]; + if (actionCached(sibling, edgeIndex)) { + siblingActions[op] = actionCache[sibling].data; + continue; + } + siblingActions[op] = block(op, 1); + for (int category = 0; category < kCategoryCount; category++) { + ActionGroup group; + group.in = (const Real*) dPartials[sibling] + category * matrixSize; + group.out = block(op, 1) + category * matrixSize; + setActionScratch(group, (op * 2 + 1) * kCategoryCount + category); + group.eigenIndex = hEigenMaps[edgeIndex]; + group.multiplier = hEdgeMultipliers[edgeIndex * kCategoryCount + category]; + group.transpose = false; + groups.push_back(group); + } + } + computeActions(groups); + + // 2. sibling's action o parent's pre-order partials. + PartialsProductBatch products; + for (size_t op = 0; op < operations.size(); op++) { + auto& product = products.products[products.count++]; + product.left = siblingActions[op]; + product.right = (const Real*) dPartials[operations[op].children[0]]; + product.parent = block(op, 0); + product.scalingFactors = nullptr; + } + cuda_partials_products(products, kCategoryCount * kPaddedStateCount, kPaddedPatternCount, nullptr, false, + dReductionScratch, stream); + + // 3. exp(t Q^T) along the destination's edge; straight into the destination unless it is rescaled. + groups.clear(); + for (size_t op = 0; op < operations.size(); op++) { + const int edgeIndex = operations[op].edges[0]; + estimateTransposedNorms(hEigenMaps[edgeIndex]); + Real* out = operations[op].scalingFactors ? block(op, 1) : (Real*) dPartials[operations[op].destination]; + for (int category = 0; category < kCategoryCount; category++) { + ActionGroup group; + group.in = block(op, 0) + category * matrixSize; + group.out = out + category * matrixSize; + setActionScratch(group, (op * 2) * kCategoryCount + category); + group.eigenIndex = hEigenMaps[edgeIndex]; + group.multiplier = hEdgeMultipliers[edgeIndex * kCategoryCount + category]; + group.transpose = true; + groups.push_back(group); + } + } + computeActions(groups); + + // 4. Rescale where requested. + bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; + products.count = 0; + for (size_t op = 0; op < operations.size(); op++) { + partialsGenerations[operations[op].destination]++; + if (not operations[op].scalingFactors) + continue; + auto& product = products.products[products.count++]; + product.left = block(op, 1); + product.right = nullptr; product.parent = (Real*)dPartials[operations[op].destination]; product.scalingFactors = operations[op].scalingFactors; } @@ -914,16 +1386,20 @@ void BeagleGPUActionImpl::computeActions(const std::vector schedules; for (size_t i = first; i < groups.size(); i++) { - if (done[i] or groups[i].eigenIndex != eigenIndex) + if (done[i] or groups[i].eigenIndex != eigenIndex or groups[i].transpose != transpose) continue; done[i] = true; const ActionGroup& group = groups[i]; - auto [m, s] = getStatistics2(1.0, kPaddedPatternCount, group.multiplier, eigenIndex); + auto [m, s] = getStatistics2(1.0, kPaddedPatternCount, group.multiplier, eigenIndex, transpose); if (m == 0) { auto status = cudaMemcpyAsync(group.out, group.in, matrixSize * sizeof(Real), cudaMemcpyDeviceToDevice, stream); if (status != cudaSuccess) @@ -967,7 +1443,7 @@ void BeagleGPUActionImpl::computeActions(const std::vector::computeActions(const std::vector::getPMax() const } BEAGLE_GPU_TEMPLATE -double BeagleGPUActionImpl::getDValue(int p, int eigenIndex) const +double BeagleGPUActionImpl::getDValue(int p, int eigenIndex, bool transpose) const { - assert(p >= 0 and p < hds[eigenIndex].size()); + const auto& d = transpose ? hdsT[eigenIndex] : hds[eigenIndex]; + assert(p >= 0 and p < d.size()); - return hds[eigenIndex][p]; + return d[p]; } std::ostream& showScalingInfo(std::ostream& o, std::uint64_t kFlags, const int* cumulativeScaleIndices, int kScaleBufferSize) @@ -1141,13 +1618,15 @@ int BeagleGPUActionImpl::calculateRootLogLikelihoods(const i BEAGLE_GPU_TEMPLATE std::tuple BeagleGPUActionImpl::getStatistics2(double t, int nCol, double edgeMultiplier, - int eigenIndex) const { + int eigenIndex, bool transpose) const { assert( t >= 0 ); assert( nCol >= 0); assert( edgeMultiplier >= 0 ); assert( eigenIndex >= 0); - if (t * hB1Norms[eigenIndex] == 0.0) + const double norm1 = transpose ? hBt1Norms[eigenIndex] : hB1Norms[eigenIndex]; + + if (t * norm1 == 0.0) return {0, 1}; int bestM = INT_MAX; @@ -1158,7 +1637,7 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub // 1. The best (m,s) based only on ||B||_1 (in some sense the result for p=1). for (auto& [thisM, thetaM]: thetaConstants) { - const double thisS = ceil(hB1Norms[eigenIndex] * edgeMultiplier / thetaM); + const double thisS = ceil(norm1 * edgeMultiplier / thetaM); if (bestM == INT_MAX || ((double) thisM) * thisS < bestM * bestS) { bestS = thisS; bestM = thisM; @@ -1173,8 +1652,8 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub auto it = thetaConstants.find(thisM); if (it != thetaConstants.end()) { // equation 3.7 in Al-Mohy and Higham - const double dValueP = getDValue(p, eigenIndex); - const double dValuePPlusOne = getDValue(p + 1, eigenIndex); + const double dValueP = getDValue(p, eigenIndex, transpose); + const double dValuePPlusOne = getDValue(p + 1, eigenIndex, transpose); const double alpha = std::max(dValueP, dValuePPlusOne) * edgeMultiplier; // part of equation 3.10 const double thisS = ceil(alpha / thetaConstants.at(thisM)); @@ -1198,15 +1677,6 @@ std::tuple BeagleGPUActionImpl::getStatistics2(doub return {m,s}; } -BEAGLE_GPU_TEMPLATE -int BeagleGPUActionImpl::upPrePartials(bool byPartition, - const int *operations, - int operationCount, - int cumulativeScalingIndex) -{ - // Not yet implemented for action; the dense fallback would read transition matrices that are never formed. - return BEAGLE_ERROR_NO_IMPLEMENTATION; -} BEAGLE_GPU_TEMPLATE int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, @@ -1229,6 +1699,10 @@ int BeagleGPUActionImpl::setSparseMatrix(int matrixIndex, auto& builder = csrBuilders[matrixIndex]; const bool structureChanged = builder.build(kPaddedStateCount, rowIndices, colIndices, values, numNonZeros, mu_B, hBs[matrixIndex]); hB1Norms[matrixIndex] = normP1(kPaddedStateCount, hBs[matrixIndex]); + hBt1Norms[matrixIndex] = normPInf(kPaddedStateCount, hBs[matrixIndex]); + hdsTCurrent[matrixIndex] = false; + qGenerations[matrixIndex]++; + derivativeMatrices[matrixIndex].eigenIndex = matrixIndex; #ifdef BEAGLE_DEBUG_FLOW std::cerr<<"B "<::setSparseMatrix(int matrixIndex, // equation 3.7 in Al-Mohy and Higham: d_p = ||B^p||_1^(1/p). d_1 is known exactly; d_0 is not used. const int pTop = getPMax() + 1; - auto norms = normEstimator.estimate(dB, dBtOffsets[matrixIndex], dBtColumns[matrixIndex], dBtValues[matrixIndex], stream); + auto norms = normEstimator.estimate(dB.offsets, dB.inner, dB.values, + dBtOffsets[matrixIndex], dBtColumns[matrixIndex], dBtValues[matrixIndex], stream); hds[matrixIndex].assign(pTop + 1, 1.0); hds[matrixIndex][1] = hB1Norms[matrixIndex]; for (int p = 2; p <= pTop; p++) @@ -1338,6 +1813,7 @@ int BeagleGPUActionImpl::updateTransitionMatrices(int eigenI for (int i = 0; i < count; i++) { const int nodeIndex = probabilityIndices[i]; hEigenMaps[nodeIndex] = eigenIndex; + matrixGenerations[nodeIndex]++; for (int category = 0; category < kCategoryCount; category++) { const double categoryRate = hCategoryRates[0][category]; // XJ: because rate categories are only set for first eigen index diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 0b21ec34..7fd60a74 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -335,7 +335,7 @@ __global__ void partials_product_kernel(const PartialsProductBatch batch, siz { const PartialsProduct& product = batch.products[blockIdx.y]; for (size_t i = blockIdx.x * (size_t)blockDim.x + threadIdx.x; i < n; i += (size_t)gridDim.x * blockDim.x) - product.parent[i] = product.left[i] * product.right[i]; + product.parent[i] = product.right ? product.left[i] * product.right[i] : product.left[i]; } // chunkMax[product, chunk, p] = max over the chunk's rows r of left[r, p] * right[r, p] (product = blockIdx.z) @@ -350,7 +350,7 @@ __global__ void partials_product_max_kernel(const PartialsProductBatch batch, for (int r = blockIdx.y * rowsPerChunk + threadIdx.y; r < rowEnd; r += blockDim.y) { const size_t i = (size_t)r * nPatterns + p; - m = max(m, product.left[i] * product.right[i]); + m = max(m, product.right ? product.left[i] * product.right[i] : product.left[i]); } m = column_reduce(m, MaxOp()); if (threadIdx.y == 0 and p < nPatterns) @@ -384,7 +384,7 @@ __global__ void partials_product_scale_kernel(const PartialsProductBatch batc for (int r = blockIdx.y * rowsPerChunk + threadIdx.y; r < rowEnd; r += blockDim.y) { const size_t i = (size_t)r * nPatterns + p; - product.parent[i] = product.left[i] * product.right[i] * inverse; + product.parent[i] = (product.right ? product.left[i] * product.right[i] : product.left[i]) * inverse; } if (product.scalingFactors and blockIdx.y == 0 and threadIdx.y == 0) @@ -482,6 +482,219 @@ void cuda_accumulate_scale_factors(const ScaleFactorBatch& batch, int nP cuda_accumulate_scale_factors_impl(batch, nPatterns, cumulative, scalersLog, subtract, stream); } +// out(c, s, p) = frequencies[s] for p < nPatterns, 0 for the padded patterns. +template +__global__ void root_pre_partials_kernel(T* out, const T* frequencies, int nStates, int nRows, int nPaddedPatterns, int nPatterns) +{ + const size_t n = (size_t)nRows * nPaddedPatterns; + for (size_t i = blockIdx.x * (size_t)blockDim.x + threadIdx.x; i < n; i += (size_t)gridDim.x * blockDim.x) + { + const int r = i / nPaddedPatterns; + const int p = i % nPaddedPatterns; + out[i] = (p < nPatterns) ? frequencies[r % nStates] : T(0); + } +} + +template +void cuda_root_pre_partials_impl(T* out, const T* frequencies, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + cudaStream_t stream) +{ + const size_t n = (size_t)nStates * nCategories * nPaddedPatterns; + const int blocks = (int)std::min((n + 255) / 256, 4096); + root_pre_partials_kernel<<>>(out, frequencies, nStates, nStates * nCategories, nPaddedPatterns, nPatterns); +} + +void cuda_root_pre_partials(float* out, const float* frequencies, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + cudaStream_t stream) +{ + cuda_root_pre_partials_impl(out, frequencies, nStates, nCategories, nPaddedPatterns, nPatterns, stream); +} + +void cuda_root_pre_partials(double* out, const double* frequencies, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + cudaStream_t stream) +{ + cuda_root_pre_partials_impl(out, frequencies, nStates, nCategories, nPaddedPatterns, nPatterns, stream); +} + +// For node z = blockIdx.z, over the chunk's rows r = (c, s): +// chunkNumerator[z, chunk, p] = sum w_c m_c pre(r, p) (D post)(r, p), D = A + shift I in CSR (if rowOffsets) +// chunkDenominator[z, chunk, p] = sum w_c pre(r, p) post(r, p) +// D acts within each category's (nStates, nPatterns) block. +template +__global__ void edge_derivative_sums_kernel(const EdgeDerivativeBatch batch, int nStates, int nRows, int nPatterns, int rowsPerChunk, + const int* __restrict__ rowOffsets, const int* __restrict__ columns, + const T* __restrict__ values, T shift, const T* __restrict__ categoryWeights, + T* chunkNumerator, T* chunkDenominator) +{ + const T* post = batch.post[blockIdx.z]; + const T* pre = batch.pre[blockIdx.z]; + const int p = blockIdx.x * blockDim.x + threadIdx.x; + const int rowEnd = min(nRows, (blockIdx.y + 1) * rowsPerChunk); + T numerator = 0; + T denominator = 0; + if (p < nPatterns) + for (int r = blockIdx.y * rowsPerChunk + threadIdx.y; r < rowEnd; r += blockDim.y) + { + const int c = r / nStates; + const int s = r % nStates; + const size_t i = (size_t)r * nPatterns + p; + const T w = categoryWeights[c]; + const T own = post[i]; + denominator += w * pre[i] * own; + if (rowOffsets) + { + const T* block = post + (size_t)c * nStates * nPatterns + p; + T sum = shift * own; + for (int k = rowOffsets[s]; k < rowOffsets[s + 1]; k++) + sum += values[k] * block[(size_t)columns[k] * nPatterns]; + numerator += w * batch.categoryMultipliers[c] * pre[i] * sum; + } + } + numerator = column_reduce(numerator, SumOp()); + __syncthreads(); + denominator = column_reduce(denominator, SumOp()); + if (threadIdx.y == 0 and p < nPatterns) + { + const size_t o = ((size_t)blockIdx.z * gridDim.y + blockIdx.y) * nPatterns + p; + chunkNumerator[o] = numerator; + chunkDenominator[o] = denominator; + } +} + +// For node z = blockIdx.x: derivatives[z, p] = numerator / denominator (each summed over the chunks), and +// sums[z] = sum_p patternWeights[p] * derivative, sumSquares[z] = sum_p patternWeights[p] * derivative^2. +// With numerators == nullptr, derivatives[z, p] = patternWeights[p] / denominator instead (no sums). +template +__global__ void edge_derivative_finish_kernel(const T* chunkNumerator, const T* chunkDenominator, int nChunks, int nPaddedPatterns, + int nPatterns, const T* patternWeights, T* derivatives, T* sums, T* sumSquares) +{ + const size_t base = (size_t)blockIdx.x * nChunks * nPaddedPatterns; + T sum = 0; + T sumSquare = 0; + for (int p = threadIdx.x; p < nPatterns; p += blockDim.x) + { + T numerator = 0; + T denominator = 0; + for (int c = 0; c < nChunks; c++) + { + denominator += chunkDenominator[base + (size_t)c * nPaddedPatterns + p]; + if (chunkNumerator) + numerator += chunkNumerator[base + (size_t)c * nPaddedPatterns + p]; + } + if (not chunkNumerator) + { + derivatives[(size_t)blockIdx.x * nPatterns + p] = patternWeights[p] / denominator; + continue; + } + const T derivative = numerator / denominator; + derivatives[(size_t)blockIdx.x * nPatterns + p] = derivative; + sum += derivative * patternWeights[p]; + sumSquare += derivative * derivative * patternWeights[p]; + } + if (not chunkNumerator) return; + sum = block_reduce(sum, T(0), SumOp()); + __syncthreads(); + sumSquare = block_reduce(sumSquare, T(0), SumOp()); + if (threadIdx.x == 0) + { + sums[blockIdx.x] = sum; + sumSquares[blockIdx.x] = sumSquare; + } +} + +template +void cuda_edge_derivatives_impl(const EdgeDerivativeBatch& batch, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + const int* rowOffsets, const int* columns, const T* values, T shift, const T* categoryWeights, + const T* patternWeights, T* scratch, T* derivatives, T* sums, T* sumSquares, cudaStream_t stream) +{ + if (batch.count == 0) return; + const int nRows = nStates * nCategories; + const RowChunks c = rowChunks(nRows, nPaddedPatterns); + const dim3 block(c.patternsPerBlock, c.rowsPerBlock); + const dim3 grid(c.patternBlocks, c.chunks, batch.count); + T* chunkNumerator = rowOffsets ? scratch : nullptr; + T* chunkDenominator = scratch + (size_t)batch.count * c.chunks * nPaddedPatterns; + edge_derivative_sums_kernel<<>>(batch, nStates, nRows, nPaddedPatterns, c.rowsPerChunk, + rowOffsets, columns, values, shift, categoryWeights, + scratch, chunkDenominator); + edge_derivative_finish_kernel<<>>(chunkNumerator, chunkDenominator, c.chunks, nPaddedPatterns, nPatterns, + patternWeights, derivatives, sums, sumSquares); +} + +size_t cuda_edge_derivative_scratch_size(int nStates, int nCategories, int nPaddedPatterns) +{ + return (size_t)2 * kMaxEdgeDerivativeNodes * rowChunks(nStates * nCategories, nPaddedPatterns).chunks * nPaddedPatterns; +} + +void cuda_edge_derivatives(const EdgeDerivativeBatch& batch, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + const int* rowOffsets, const int* columns, const float* values, float shift, const float* categoryWeights, + const float* patternWeights, float* scratch, float* derivatives, float* sums, float* sumSquares, cudaStream_t stream) +{ + cuda_edge_derivatives_impl(batch, nStates, nCategories, nPaddedPatterns, nPatterns, rowOffsets, columns, values, shift, categoryWeights, + patternWeights, scratch, derivatives, sums, sumSquares, stream); +} + +void cuda_edge_derivatives(const EdgeDerivativeBatch& batch, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + const int* rowOffsets, const int* columns, const double* values, double shift, const double* categoryWeights, + const double* patternWeights, double* scratch, double* derivatives, double* sums, double* sumSquares, cudaStream_t stream) +{ + cuda_edge_derivatives_impl(batch, nStates, nCategories, nPaddedPatterns, nPatterns, rowOffsets, columns, values, shift, categoryWeights, + patternWeights, scratch, derivatives, sums, sumSquares, stream); +} + +// crossProducts[k] += sum_{c,p} multipliers[c] * patternScale[p] * pre(c, i, p) * post(c, j, p) for each non-zero k = (i, j) +// of the CSR pattern. One warp per row i; the lanes cover (category, pattern) pairs. +template +__global__ void cross_products_kernel(const T* pre, const T* post, int nStates, int nCategories, int nPatterns, + const int* __restrict__ rowOffsets, const int* __restrict__ columns, + const CategoryMultipliers multipliers, const T* __restrict__ patternScale, T* crossProducts) +{ + const int lane = threadIdx.x % 32; + const int i = blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32; + if (i >= nStates) return; + const size_t categoryStride = (size_t)nStates * nPatterns; + const int n = nCategories * nPatterns; + for (int k = rowOffsets[i]; k < rowOffsets[i + 1]; k++) + { + const int j = columns[k]; + T sum = 0; + for (int q = lane; q < n; q += 32) + { + const int c = q / nPatterns; + const int p = q % nPatterns; + sum += multipliers.values[c] * patternScale[p] * pre[c * categoryStride + (size_t)i * nPatterns + p] * + post[c * categoryStride + (size_t)j * nPatterns + p]; + } + for (int offset = 16; offset > 0; offset /= 2) + sum += __shfl_down_sync(0xffffffff, sum, offset); + if (lane == 0) + crossProducts[k] += sum; + } +} + +template +void cuda_cross_products_impl(const T* pre, const T* post, int nStates, int nCategories, int nPatterns, const int* rowOffsets, + const int* columns, const CategoryMultipliers& multipliers, const T* patternScale, T* crossProducts, + cudaStream_t stream) +{ + cross_products_kernel<<<(nStates + 7) / 8, 256, 0, stream>>>(pre, post, nStates, nCategories, nPatterns, rowOffsets, columns, + multipliers, patternScale, crossProducts); +} + +void cuda_cross_products(const float* pre, const float* post, int nStates, int nCategories, int nPatterns, const int* rowOffsets, + const int* columns, const CategoryMultipliers& multipliers, const float* patternScale, float* crossProducts, + cudaStream_t stream) +{ + cuda_cross_products_impl(pre, post, nStates, nCategories, nPatterns, rowOffsets, columns, multipliers, patternScale, crossProducts, stream); +} + +void cuda_cross_products(const double* pre, const double* post, int nStates, int nCategories, int nPatterns, const int* rowOffsets, + const int* columns, const CategoryMultipliers& multipliers, const double* patternScale, double* crossProducts, + cudaStream_t stream) +{ + cuda_cross_products_impl(pre, post, nStates, nCategories, nPatterns, rowOffsets, columns, multipliers, patternScale, crossProducts, stream); +} + // chunkSum[chunk, p] = sum over the chunk's rows r = (c, s) of root[r, p] * categoryWeights[c] * frequencies[s] template __global__ void root_chunk_sums_kernel(const T* root, const T* categoryWeights, const T* frequencies, int nStates, int nRows, diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 7468258f..491ca9d6 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -148,8 +148,8 @@ void cuda_taylor_horner_terms(int nRows, int nCols, const int* rowOffsets, const // (per product, for the products). int cuda_row_chunk_count(int nRows, int nPatterns); -// parent = left * right for (nRows, nPatterns) partials. If scalingFactors is non-null, also rescale each -// pattern by its largest entry m and write m (log m if scalersLog) to scalingFactors[pattern]. +// parent = left * right for (nRows, nPatterns) partials (right == nullptr means 1). If scalingFactors is non-null, +// also rescale each pattern by its largest entry m and write m (log m if scalersLog) to scalingFactors[pattern]. template struct PartialsProduct { @@ -187,6 +187,54 @@ void cuda_accumulate_scale_factors(const ScaleFactorBatch& batch, int nPa void cuda_accumulate_scale_factors(const ScaleFactorBatch& batch, int nPatterns, double* cumulative, bool scalersLog, bool subtract, cudaStream_t stream); +// out(c, s, p) = frequencies[s] for p < nPatterns, 0 for the padded patterns: root pre-order partials. +void cuda_root_pre_partials(float* out, const float* frequencies, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + cudaStream_t stream); +void cuda_root_pre_partials(double* out, const double* frequencies, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + cudaStream_t stream); + +constexpr int kMaxEdgeDerivativeNodes = 64; +constexpr int kMaxDerivativeCategories = 64; + +// Up to kMaxEdgeDerivativeNodes (post-order, pre-order) pairs, with per-category multipliers m_c (the rates). +template +struct EdgeDerivativeBatch +{ + int count = 0; + const T* post[kMaxEdgeDerivativeNodes]; + const T* pre[kMaxEdgeDerivativeNodes]; + T categoryMultipliers[kMaxDerivativeCategories]; +}; + +// For each node z of the batch and pattern p < nPatterns (partials are (nCategories * nStates, nPaddedPatterns)): +// derivatives[z * nPatterns + p] = (sum_c w_c m_c pre^T D post) / (sum_c w_c pre^T post), D = A + shift I, +// sums[z] = sum_p patternWeights[p] * derivative, sumSquares[z] = sum_p patternWeights[p] * derivative^2, +// with A in CSR (nStates, nStates). With rowOffsets == nullptr only the denominators are formed, and +// derivatives[z * nPatterns + p] = patternWeights[p] / denominator (sums are not set). +// scratch holds cuda_edge_derivative_scratch_size( ) values; all outputs are on the device. +size_t cuda_edge_derivative_scratch_size(int nStates, int nCategories, int nPaddedPatterns); +void cuda_edge_derivatives(const EdgeDerivativeBatch& batch, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + const int* rowOffsets, const int* columns, const float* values, float shift, const float* categoryWeights, + const float* patternWeights, float* scratch, float* derivatives, float* sums, float* sumSquares, cudaStream_t stream); +void cuda_edge_derivatives(const EdgeDerivativeBatch& batch, int nStates, int nCategories, int nPaddedPatterns, int nPatterns, + const int* rowOffsets, const int* columns, const double* values, double shift, const double* categoryWeights, + const double* patternWeights, double* scratch, double* derivatives, double* sums, double* sumSquares, cudaStream_t stream); + +template +struct CategoryMultipliers +{ + T values[kMaxDerivativeCategories]; +}; + +// crossProducts[k] += sum_{c,p} multipliers[c] * patternScale[p] * pre(c, i, p) * post(c, j, p) for each non-zero k = (i, j) +// of the CSR pattern (rowOffsets, columns) of an (nStates, nStates) matrix. pre and post are (nCategories * nStates, nPatterns). +void cuda_cross_products(const float* pre, const float* post, int nStates, int nCategories, int nPatterns, const int* rowOffsets, + const int* columns, const CategoryMultipliers& multipliers, const float* patternScale, float* crossProducts, + cudaStream_t stream); +void cuda_cross_products(const double* pre, const double* post, int nStates, int nCategories, int nPatterns, const int* rowOffsets, + const int* columns, const CategoryMultipliers& multipliers, const double* patternScale, double* crossProducts, + cudaStream_t stream); + // siteLogLikelihoods[p] = log(sum_{c,s} root(c,s,p) * categoryWeights[c] * frequencies[s]) + cumulativeScaling[p] (if non-null), // sumLogLikelihood[0] = sum_p patternWeights[p] * siteLogLikelihoods[p]. void cuda_root_log_likelihoods(const float* root, const float* categoryWeights, const float* frequencies, const float* cumulativeScaling, From bd767d001c879cd35176a599ca54de5bfc57d214 Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Wed, 28 Oct 2026 20:06:43 -0500 Subject: [PATCH 418/421] Action plugins: apply existing scale factors for a read scaling index With a read scaling index (BEAST's default dynamic rescaling between recomputations), the standard plugins divide each pattern's new partials by the stored scale factor. The CPU and GPU action plugins skipped that while the root still added the factors back, so every evaluation after the first was wrong (BEAST: two clock moves gave -2491.08 instead of -1905.15). - CPU: applyFixedScaling() after post-order and pre-order operations with a read index, over the operation's pattern range. - GPU: the product kernels take optional fixed scale factors (exp'd for log scalers); post-order and pre-order pass the read index through, and the scheduler orders a read after any write of the same scaling buffer. Checked in BEAST on the CPU plugin (dynamic = always = none = from scratch) and with preorder_check on CPU and GPU: reusing the stored factors matches recomputing them to <= 2e-16. compute-sanitizer clean; post-order validation unchanged. Co-Authored-By: Claude Opus 5.5 --- libhmsbeagle/CPU/BeagleCPUActionImpl.h | 3 +++ libhmsbeagle/CPU/BeagleCPUActionImpl.hpp | 22 +++++++++++++++++++++ libhmsbeagle/GPU/BeagleGPUActionImpl.h | 1 + libhmsbeagle/GPU/BeagleGPUActionImpl.hpp | 18 +++++++++++++---- libhmsbeagle/GPU/cuda_ops.cu | 25 +++++++++++++++++++----- libhmsbeagle/GPU/cuda_ops.h | 2 ++ 6 files changed, 62 insertions(+), 9 deletions(-) diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.h b/libhmsbeagle/CPU/BeagleCPUActionImpl.h index ac0101ce..2958f2c0 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.h +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.h @@ -211,6 +211,9 @@ namespace beagle { // virtual int getPaddedPatternsModulus(); private: + // Divide the patterns [startPattern, endPattern) of a partials buffer by existing scale factors. + void applyFixedScaling(int destinationPartialIndex, const double* scalingFactors, int startPattern, int endPattern); + void calcEdgeLogDerivativesPartials(const int postOrderPartialIndex, const int preOrderPartialIndex, const int firstDerivativeIndex, diff --git a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp index 2bc2db4e..80eb25d4 100644 --- a/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp +++ b/libhmsbeagle/CPU/BeagleCPUActionImpl.hpp @@ -620,6 +620,9 @@ namespace beagle { } else { this->rescalePartials(destPartials, scalingFactors, cumulativeScaleBuffer, 0); } + } else if (scalingFactors != NULL) { + // Reuse the existing scale factors (e.g. dynamic rescaling between recomputations). + applyFixedScaling(destinationPartialIndex, scalingFactors, startPattern, endPattern); } } @@ -713,6 +716,9 @@ namespace beagle { } else { this->rescalePartials(destPartials, scalingFactors, cumulativeScaleBuffer, 0); } + } else if (scalingFactors != NULL) { + // Reuse the existing scale factors (e.g. dynamic rescaling between recomputations). + applyFixedScaling(destinationPartialIndex, scalingFactors, startPattern, endPattern); } if (DEBUGGING_OUTPUT) { @@ -729,6 +735,22 @@ namespace beagle { return BEAGLE_SUCCESS; } + BEAGLE_CPU_ACTION_TEMPLATE + void BeagleCPUActionImpl::applyFixedScaling(int destinationPartialIndex, + const double* scalingFactors, + int startPattern, + int endPattern) { + // As BeagleCPUImpl's *FixedScaling functions: divide each pattern by its stored scale factor. + const bool scalersLog = (this->kFlags & BEAGLE_FLAG_SCALERS_LOG) != 0; + for (int category = 0; category < kCategoryCount; category++) { + auto destP = partialsMap(destinationPartialIndex, category, startPattern, endPattern); + for (int j = 0; j < endPattern - startPattern; j++) { + const double factor = scalingFactors[startPattern + j]; + destP.col(j) *= 1.0 / (scalersLog ? exp(factor) : factor); + } + } + } + BEAGLE_CPU_ACTION_TEMPLATE void BeagleCPUActionImpl::calcEdgeLogDerivativesPartials(const int postOrderPartialIndex, const int preOrderPartialIndex, diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.h b/libhmsbeagle/GPU/BeagleGPUActionImpl.h index 16e195d6..799edbb0 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.h +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.h @@ -1218,6 +1218,7 @@ class BeagleGPUActionImpl : public BeagleGPUImpl int children[2]; int edges[2]; Real* scalingFactors; // nullptr means no rescaling + Real* fixedScalingFactors = nullptr; // otherwise, divide by these existing factors (a read scaling index) }; // Groups operations into batches of independent ones (one tree level, up to maxBatchOperations each), in an diff --git a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp index 97bd18f6..e3daebb2 100644 --- a/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp +++ b/libhmsbeagle/GPU/BeagleGPUActionImpl.hpp @@ -632,6 +632,7 @@ int BeagleGPUActionImpl::upPartials(bool byPartition, operation.edges[0] = firstChildSubstitutionMatrixIndex; operation.edges[1] = secondChildSubstitutionMatrixIndex; operation.scalingFactors = (rescale == 1) ? scalingFactors : nullptr; + operation.fixedScalingFactors = (rescale == 0) ? scalingFactors : nullptr; } for (const auto& batch: scheduleBatches(parsed)) { @@ -684,6 +685,11 @@ BeagleGPUActionImpl::scheduleBatches(const std::vectorsecond + 1); + } if (operation.scalingFactors) { auto it = scalingWriteLevel.find(operation.scalingFactors); if (it != scalingWriteLevel.end()) @@ -738,7 +744,7 @@ int BeagleGPUActionImpl::upPrePartials(bool byPartition, cumulativeScalingBuffer = (Real*)dScalingFactors[cumulativeScalingIndex]; // [dest, writeScaling, readScaling, parent (pre-order), dest's matrix, sibling (post-order), sibling's matrix]. - // As in the CPU implementation, only a write scaling index rescales. + // As in the CPU implementation, only manual scaling indices are used (not automatic or always-rescaling). std::vector parsed(operationCount); for (int op = 0; op < operationCount; op++) { const int* x = operations + op * BEAGLE_OP_COUNT; @@ -749,6 +755,7 @@ int BeagleGPUActionImpl::upPrePartials(bool byPartition, operation.edges[0] = x[4]; operation.edges[1] = x[6]; operation.scalingFactors = (x[1] >= 0) ? (Real*)dScalingFactors[x[1]] : nullptr; + operation.fixedScalingFactors = (x[1] < 0 and x[2] >= 0) ? (Real*)dScalingFactors[x[2]] : nullptr; for (int buffer: {x[0], x[3], x[5]}) if (buffer < 0 or buffer >= kPartialsBufferCount or dPartials[buffer] == 0) return BEAGLE_ERROR_OUT_OF_RANGE; @@ -1229,6 +1236,7 @@ void BeagleGPUActionImpl::calcPartialsPartials(const std::ve product.right = actions[op * 2 + 1]; product.parent = (Real*)dPartials[operations[op].destination]; product.scalingFactors = operations[op].scalingFactors; + product.fixedScalingFactors = operations[op].fixedScalingFactors; partialsGenerations[operations[op].destination]++; } cuda_partials_products(products, kCategoryCount * kPaddedStateCount, kPaddedPatternCount, @@ -1285,7 +1293,8 @@ void BeagleGPUActionImpl::calcPrePartials(const std::vector< for (size_t op = 0; op < operations.size(); op++) { const int edgeIndex = operations[op].edges[0]; estimateTransposedNorms(hEigenMaps[edgeIndex]); - Real* out = operations[op].scalingFactors ? block(op, 1) : (Real*) dPartials[operations[op].destination]; + const bool scaled = operations[op].scalingFactors or operations[op].fixedScalingFactors; + Real* out = scaled ? block(op, 1) : (Real*) dPartials[operations[op].destination]; for (int category = 0; category < kCategoryCount; category++) { ActionGroup group; group.in = block(op, 0) + category * matrixSize; @@ -1299,18 +1308,19 @@ void BeagleGPUActionImpl::calcPrePartials(const std::vector< } computeActions(groups); - // 4. Rescale where requested. + // 4. Rescale where requested, with new or existing scale factors. bool scalers_log = (kFlags & BEAGLE_FLAG_SCALERS_LOG)?true:false; products.count = 0; for (size_t op = 0; op < operations.size(); op++) { partialsGenerations[operations[op].destination]++; - if (not operations[op].scalingFactors) + if (not operations[op].scalingFactors and not operations[op].fixedScalingFactors) continue; auto& product = products.products[products.count++]; product.left = block(op, 1); product.right = nullptr; product.parent = (Real*)dPartials[operations[op].destination]; product.scalingFactors = operations[op].scalingFactors; + product.fixedScalingFactors = operations[op].fixedScalingFactors; } cuda_partials_products(products, kCategoryCount * kPaddedStateCount, kPaddedPatternCount, cumulativeScalingBuffer, scalers_log, dReductionScratch, stream); diff --git a/libhmsbeagle/GPU/cuda_ops.cu b/libhmsbeagle/GPU/cuda_ops.cu index 7fd60a74..93e1ff7e 100644 --- a/libhmsbeagle/GPU/cuda_ops.cu +++ b/libhmsbeagle/GPU/cuda_ops.cu @@ -329,13 +329,28 @@ __device__ T column_reduce(T v, Op op) return shared[threadIdx.x]; } -// parent = left * right for each product of the batch (blockIdx.y) +// 1 / the existing scale factor of pattern p, or 1 without fixed scaling. template -__global__ void partials_product_kernel(const PartialsProductBatch batch, size_t n) +__device__ T fixed_inverse(const PartialsProduct& product, int p, bool scalersLog) +{ + if (not product.fixedScalingFactors) + return T(1); + const T factor = product.fixedScalingFactors[p]; + return T(1) / (scalersLog ? exp(factor) : factor); +} + +// parent = left * right for each product of the batch (blockIdx.y), divided by any fixed scale factors. +template +__global__ void partials_product_kernel(const PartialsProductBatch batch, size_t n, int nPatterns, bool scalersLog) { const PartialsProduct& product = batch.products[blockIdx.y]; for (size_t i = blockIdx.x * (size_t)blockDim.x + threadIdx.x; i < n; i += (size_t)gridDim.x * blockDim.x) - product.parent[i] = product.right ? product.left[i] * product.right[i] : product.left[i]; + { + T value = product.right ? product.left[i] * product.right[i] : product.left[i]; + if (product.fixedScalingFactors) + value *= fixed_inverse(product, (int)(i % nPatterns), scalersLog); + product.parent[i] = value; + } } // chunkMax[product, chunk, p] = max over the chunk's rows r of left[r, p] * right[r, p] (product = blockIdx.z) @@ -378,7 +393,7 @@ __global__ void partials_product_scale_kernel(const PartialsProductBatch batc __syncthreads(); if (p >= nPatterns) return; const T m = patternMax[threadIdx.x]; - const T inverse = 1 / m; + const T inverse = product.scalingFactors ? 1 / m : fixed_inverse(product, p, scalersLog); const int rowEnd = min(nRows, (blockIdx.y + 1) * rowsPerChunk); for (int r = blockIdx.y * rowsPerChunk + threadIdx.y; r < rowEnd; r += blockDim.y) @@ -420,7 +435,7 @@ void cuda_partials_products_impl(const PartialsProductBatch& batch, int nRows { const size_t n = (size_t)nRows * nPatterns; const int blocks = (int)std::min((n + 255) / 256, 4096 / batch.count + 1); - partials_product_kernel<<>>(batch, n); + partials_product_kernel<<>>(batch, n, nPatterns, scalersLog); return; } diff --git a/libhmsbeagle/GPU/cuda_ops.h b/libhmsbeagle/GPU/cuda_ops.h index 491ca9d6..b1d3d2d5 100644 --- a/libhmsbeagle/GPU/cuda_ops.h +++ b/libhmsbeagle/GPU/cuda_ops.h @@ -150,6 +150,7 @@ int cuda_row_chunk_count(int nRows, int nPatterns); // parent = left * right for (nRows, nPatterns) partials (right == nullptr means 1). If scalingFactors is non-null, // also rescale each pattern by its largest entry m and write m (log m if scalersLog) to scalingFactors[pattern]. +// Otherwise, if fixedScalingFactors is non-null, divide each pattern by its existing factor (exp of it if scalersLog). template struct PartialsProduct { @@ -157,6 +158,7 @@ struct PartialsProduct const T* right; T* parent; T* scalingFactors; + const T* fixedScalingFactors = nullptr; }; constexpr int kMaxPartialsProducts = 16; From 4fbd5ff0a28612507f0f31ffed1d0997d3059da2 Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Wed, 28 Oct 2026 20:14:08 -0500 Subject: [PATCH 419/421] CI: include the CPU's instruction-set flags in the ccache key Runners with the same CPU model name can expose different instruction-set extensions, so -march=native objects cached on one could still be reused on another (Illegal instruction in synthetictest on ubuntu-gcc-12). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 96dcd109..03f6c95f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -183,10 +183,12 @@ jobs: - name: Identify CPU (for the ccache key) shell: bash run: | + # -march=native output depends on the instruction-set extensions, which differ between runners + # with the same CPU model, so the key includes them. if [ "$RUNNER_OS" = "macOS" ]; then - cpu=$(sysctl -n machdep.cpu.brand_string) + cpu="$(sysctl -n machdep.cpu.brand_string) $(sysctl -n machdep.cpu.features machdep.cpu.leaf7_features 2>/dev/null)" else - cpu=$(grep -m1 "model name" /proc/cpuinfo || uname -m) + cpu="$(grep -m1 "model name" /proc/cpuinfo) $(grep -m1 "^flags" /proc/cpuinfo || uname -m)" fi echo "CPU_KEY=$(echo "$cpu" | cksum | cut -d' ' -f1)" >> $GITHUB_ENV From 3d68dde8b0c0bbaa9b4dcadcbfd6cb443c0569c0 Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Wed, 28 Oct 2026 20:18:09 -0500 Subject: [PATCH 420/421] CI: the CPU feature lists exist only on Intel Macs Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03f6c95f..0adf0980 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -186,7 +186,8 @@ jobs: # -march=native output depends on the instruction-set extensions, which differ between runners # with the same CPU model, so the key includes them. if [ "$RUNNER_OS" = "macOS" ]; then - cpu="$(sysctl -n machdep.cpu.brand_string) $(sysctl -n machdep.cpu.features machdep.cpu.leaf7_features 2>/dev/null)" + # (the feature lists exist only on Intel Macs) + cpu="$(sysctl -n machdep.cpu.brand_string) $(sysctl -n machdep.cpu.features machdep.cpu.leaf7_features 2>/dev/null || true)" else cpu="$(grep -m1 "model name" /proc/cpuinfo) $(grep -m1 "^flags" /proc/cpuinfo || uname -m)" fi From 2fdc3176b72d2ca65015999df14cf6b56601125f Mon Sep 17 00:00:00 2001 From: Xiang Ji Date: Wed, 28 Oct 2026 20:32:39 -0500 Subject: [PATCH 421/421] Build option for portable binaries; SSE plugin needs only SSE4.1 -march=native was hard-coded, so binaries could crash with "Illegal instruction" on any CPU other than the build machine: GitHub runners differ, and ccache (keyed on the literal flag) reused objects across them. The same happens to BEAST CI, which caches one BEAGLE build for all its runners. - BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH is now an option (default ON, as before). - The SSE plugin's x86 code uses SSE4.1 intrinsics (_mm_dp_pd), which it got from -march=native; it now gets -msse4.1 itself (only the x86_64 slice of macOS universal builds; arm64 uses sse2neon.h). Its load-time check now requires SSE4.1 as well as SSE2. - CI builds with the option OFF, so the per-CPU ccache key is gone. Checked on hmm (x86-64 Linux) and an Apple-silicon Mac (universal, both slices): portable and native builds pass synthetictest, hmctest and actiontest, the SSE plugin loads and gives the same logL as the non-vector plugin, and the portable CPU plugin contains no AVX instructions. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 24 +++++------------------- CMakeLists.txt | 4 +++- libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp | 19 ++++++++++++------- libhmsbeagle/CPU/CMakeLists.txt | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0adf0980..24e303cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,7 +121,9 @@ jobs: - name: Set CMake build type run: | - CMAKE_ARGS="" + # Portable binaries: with -march=native, ccache would reuse objects built on one runner's CPU on + # another runner's ("Illegal instruction"). + CMAKE_ARGS="-DBEAGLE_OPTIMIZE_FOR_NATIVE_ARCH=OFF" if [ -n "${{ matrix.buildtype }}" ] ; then CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${{ matrix.buildtype }}" fi @@ -177,28 +179,12 @@ jobs: echo "CXX=ccache clang++" >> $GITHUB_ENV echo "CMAKE_ARGS=${CMAKE_ARGS} -DBUILD_OPENMP=OFF" >> $GITHUB_ENV - # The build uses -march=native, but ccache keys on the literal flag, not on the CPU. Without the - # CPU in the cache key, objects built on one runner get reused on a runner with an older CPU - # ("Illegal instruction"). - - name: Identify CPU (for the ccache key) - shell: bash - run: | - # -march=native output depends on the instruction-set extensions, which differ between runners - # with the same CPU model, so the key includes them. - if [ "$RUNNER_OS" = "macOS" ]; then - # (the feature lists exist only on Intel Macs) - cpu="$(sysctl -n machdep.cpu.brand_string) $(sysctl -n machdep.cpu.features machdep.cpu.leaf7_features 2>/dev/null || true)" - else - cpu="$(grep -m1 "model name" /proc/cpuinfo) $(grep -m1 "^flags" /proc/cpuinfo || uname -m)" - fi - echo "CPU_KEY=$(echo "$cpu" | cksum | cut -d' ' -f1)" >> $GITHUB_ENV - - name: Set up ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ matrix.name }}-${{ env.CPU_KEY }} + key: ${{ matrix.name }}-portable restore-keys: | - ${{ matrix.name }}-${{ env.CPU_KEY }} + ${{ matrix.name }}-portable - name: Configure run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 546dbef9..cf1efd46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,9 @@ set(BEAGLE_PLUGIN_VERSION "${BEAGLE_VERSION_MAJOR}${BEAGLE_VERSION_MINOR}") message(STATUS "BEAGLE_VERSION = ${BEAGLE_VERSION}") message(STATUS "BEAGLE_PLUGIN_VERSION = ${BEAGLE_PLUGIN_VERSION}") -set(BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH true) +# -march=native makes the fastest code for the build machine, but the binaries may not run on other CPUs +# (Illegal instruction). Turn it off for binaries that are distributed or cached across machines. +option(BEAGLE_OPTIMIZE_FOR_NATIVE_ARCH "Compile with -march=native (not portable to other CPUs)" ON) option(BUILD_OPENCL "Build beagle with OpenCL library" ON) option(BUILD_CUDA "Build beagle with CUDA library" ON) diff --git a/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp b/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp index 6c5390d6..69cc96bf 100644 --- a/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp +++ b/libhmsbeagle/CPU/BeagleCPUSSEPlugin.cpp @@ -60,6 +60,9 @@ Plugin("CPU-SSE", "CPU-SSE") extern "C" { +// check_sse2( ) also requires SSE4.1: the 4-state kernels use _mm_dp_pd. (CPUID leaf 1: SSE2 is EDX bit 26, +// SSE4.1 is ECX bit 19.) + #ifdef _WIN32 bool check_sse2(){ #ifdef _WIN64 @@ -67,6 +70,7 @@ bool check_sse2(){ return 1; #else unsigned int features; + unsigned int features2; __asm { @@ -76,10 +80,11 @@ bool check_sse2(){ push ecx push edx - // Get the feature flags (eax=1) from edx + // Get the feature flags (eax=1) from edx and ecx mov eax, 1 cpuid mov features, edx + mov features2, ecx // Restore registers pop edx @@ -88,8 +93,8 @@ bool check_sse2(){ pop eax } -// Bit 26 for SSE2 support - return features & 0x04000000; +// Bit 26 for SSE2 support, ECX bit 19 for SSE4.1 + return (features & 0x04000000) && (features2 & 0x00080000); #endif } @@ -112,8 +117,8 @@ bool check_sse2() return false; } - /* Run SSE2 test only if host has SSE2 support. */ - if (edx & bit_SSE2) + /* Run SSE2 test only if host has SSE2 and SSE4.1 support. */ + if ((edx & bit_SSE2) && (ecx & bit_SSE4_1)) return true; return false; #elif defined(__aarch64__) @@ -157,7 +162,7 @@ bool check_sse2() "=d" (result[3]) // EDX register -> result[3] : "0" (opcode) : "cc"); - return result[3] & 0x04000000; + return (result[3] & 0x04000000) && (result[2] & 0x00080000); #endif // HAVE_CPUID.H } #else @@ -169,7 +174,7 @@ bool check_sse2(){ __asm__("cpuid" : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) : "a" (op)); - return edx & 0x04000000; + return (edx & 0x04000000) && (ecx & 0x00080000); } #endif #endif diff --git a/libhmsbeagle/CPU/CMakeLists.txt b/libhmsbeagle/CPU/CMakeLists.txt index 9203dc94..a64f3b99 100644 --- a/libhmsbeagle/CPU/CMakeLists.txt +++ b/libhmsbeagle/CPU/CMakeLists.txt @@ -57,6 +57,20 @@ SET_TARGET_PROPERTIES(hmsbeagle-cpu-sse SUFFIX "${BEAGLE_PLUGIN_SUFFIX}" ) +# The plugin uses SSE4.1 intrinsics on x86 (on arm64, sse2neon.h provides them), which need -msse4.1 unless +# -march=native already enables them. The plugin checks for SSE4.1 when it is loaded. +include(CheckCXXCompilerFlag) +if(APPLE AND CMAKE_OSX_ARCHITECTURES MATCHES "x86_64" AND CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + # universal build: only the x86_64 slice + target_compile_options(hmsbeagle-cpu-sse PRIVATE "SHELL:-Xarch_x86_64 -msse4.1") +elseif(NOT MSVC) + # (compilers for arm64 reject the flag) + CHECK_CXX_COMPILER_FLAG("-msse4.1" COMPILER_SUPPORTS_SSE41) + if(COMPILER_SUPPORTS_SSE41) + target_compile_options(hmsbeagle-cpu-sse PRIVATE -msse4.1) + endif() +endif() + endif(BUILD_SSE) if(BUILD_ACTION)