From 835f086d43162b9e8946419f96fd7f3fd7b0f538 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 16:23:07 +0000 Subject: [PATCH 1/4] ci: build, test and release the framework from GitHub Actions Add the Build and release workflow: every pull request and push to main builds the dacpac on Windows and runs the integration tests on Linux against that build; a push to main publishes the GitHub release v when the version in .direct-version has none yet, with the notes from the matching section of ReleaseNotes.md and the dacpacs, the fabric folder and the Snowflake script as assets. The version files are checked for agreement before the build. Also lowercase the current and next folder names in the migration tests so they run on Linux, add the Snowflake edition to the 2.2.0 release notes, and describe the release flow in the README. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01P8xKsmMZRuuoGUZqqpAmVK --- .github/workflows/release.yml | 226 ++++++++++++++++++ .../Tests/MigrationTests/MigrationTests.cs | 4 +- Documentation/ReleaseNotes.md | 1 + README.md | 6 + 4 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..1659623 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,226 @@ +# Builds the DIRECT Framework dacpac, runs the integration tests and publishes a GitHub release. +# +# Every push to main and every pull request builds the dacpac on Windows (the SQL project runs +# cmd pre- and post-build scripts) and runs the integration tests on Linux (they need a Linux +# SQL Server container). A push to main also publishes the release v when the version +# in .direct-version has no release yet, so a release is cut by merging the version bump. +# +# A manual run (workflow_dispatch) builds and tests any branch; tick 'publish' to also publish +# the release from that branch. + +name: Build and release + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + inputs: + publish: + description: Publish the GitHub release when the version in .direct-version has none yet + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +env: + DOTNET_NOLOGO: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + +jobs: + build: + name: Build the dacpac + runs-on: windows-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + + - name: Check that the version files agree + id: version + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $version = (Get-Content .direct-version -Raw).Trim() + if ($version -notmatch '^\d+\.\d+\.\d+$') { throw ".direct-version holds '$version', expected major.minor.patch." } + $problems = @() + + $props = Get-Content Directory.Build.props -Raw + if ($props -notmatch "$([regex]::Escape($version))(-[0-9A-Za-z.]+)?") { $problems += "Directory.Build.props does not start with $version." } + if ($props -notmatch "$([regex]::Escape($version))\.0") { $problems += "Directory.Build.props is not $version.0." } + + $next = Get-Content Releases.Direct_Framework/next/version.json -Raw | ConvertFrom-Json + if ($next.directFrameworkVersion -ne $version) { $problems += "Releases.Direct_Framework/next/version.json has directFrameworkVersion $($next.directFrameworkVersion)." } + + $metadata = Get-Content Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.FRAMEWORK_METADATA.sql -Raw + if ($metadata -notmatch "N'DIRECT_VERSION',\s*N'$([regex]::Escape($version))'") { $problems += "omd_metadata.FRAMEWORK_METADATA.sql does not seed DIRECT_VERSION $version." } + + $notes = Get-Content Documentation/ReleaseNotes.md -Raw + if ($notes -notmatch "(?m)^## Release $([regex]::Escape($version))\s*$") { $problems += "Documentation/ReleaseNotes.md has no section '## Release $version'." } + + if ($problems.Count -gt 0) { + $problems | ForEach-Object { Write-Host "::error::$_" } + throw "The version files do not agree on $version." + } + Write-Host "DIRECT version $version" + "version=$version" >> $env:GITHUB_OUTPUT + + - name: Build Direct_Framework (Release) + shell: pwsh + run: dotnet build Direct_Framework/Direct_Framework.sqlproj --configuration Release + + - name: Check the build output + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + # The post-build script copies the output to the next release folder, which is what the tests deploy. + $db = 'Releases.Direct_Framework/next/db' + foreach ($file in 'Direct_Framework.dacpac', 'master.dacpac') { + if (-not (Test-Path (Join-Path $db $file))) { throw "$file is missing from $db." } + } + Get-ChildItem $db -Recurse -File | Select-Object FullName, Length | Format-Table -AutoSize + + - name: Upload the build output + uses: actions/upload-artifact@v4 + with: + name: direct-framework-db + path: Releases.Direct_Framework/next/db + if-no-files-found: error + retention-days: 30 + + test: + name: Integration tests + runs-on: ubuntu-latest + needs: build + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + + - name: Replace the next release folder with the build output + uses: actions/download-artifact@v4 + with: + name: direct-framework-db + path: Releases.Direct_Framework/next/db + + - name: Run the integration tests + run: > + dotnet run --project Direct_Framework.Integration.Tests --configuration Release -- + --report-trx --report-trx-filename TestResults.trx --results-directory TestResults + + - name: Upload the test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-results + path: Direct_Framework.Integration.Tests/TestResults + if-no-files-found: ignore + + release: + name: Publish the release + runs-on: ubuntu-latest + needs: [ build, test ] + if: > + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + (github.event_name == 'workflow_dispatch' && inputs.publish) + permissions: + contents: write + env: + VERSION: ${{ needs.build.outputs.version }} + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Skip when v${{ needs.build.outputs.version }} is already released + id: check + run: | + if gh release view "v$VERSION" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then + echo "::notice::Release v$VERSION exists already; nothing to publish. Bump .direct-version and the other version files to release again." + echo "publish=false" >> "$GITHUB_OUTPUT" + elif git rev-parse -q --verify "refs/tags/v$VERSION" > /dev/null; then + echo "::error::Tag v$VERSION exists without a release. Delete the tag or publish the release by hand." + exit 1 + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + + - name: Download the build output + if: steps.check.outputs.publish == 'true' + uses: actions/download-artifact@v4 + with: + name: direct-framework-db + path: dist/db + + - name: Assemble the release assets + if: steps.check.outputs.publish == 'true' + run: | + set -euo pipefail + mkdir -p dist/assets + cp dist/db/Direct_Framework.dacpac dist/db/master.dacpac dist/assets/ + (cd fabric && zip -q -r ../dist/assets/Direct_Fabric.zip .) + cp snowflake/Direct_Snowflake.sql dist/assets/ + ls -l dist/assets + + - name: Write the release notes + if: steps.check.outputs.publish == 'true' + run: | + set -euo pipefail + # The section of Documentation/ReleaseNotes.md for this version, without its heading. + awk -v v="$VERSION" ' + $0 ~ "^## Release " v "[[:space:]]*$" { on = 1; next } + on && /^## / { exit } + on { print } + ' Documentation/ReleaseNotes.md | sed -e :a -e '/^\n*$/{$d;N;ba' -e '}' > dist/notes.md + if [ ! -s dist/notes.md ]; then + echo "::error::Documentation/ReleaseNotes.md has no content under '## Release $VERSION'." + exit 1 + fi + previous=$(git tag --list 'v*' --sort=-v:refname | head -n 1) + { + echo + echo "### Assets" + echo + echo "- \`Direct_Framework.dacpac\` and \`master.dacpac\`: the SQL Server edition, deployed with SqlPackage or DacFx." + echo "- \`Direct_Fabric.zip\`: the Microsoft Fabric edition (standard pipelines, Variable Library and scripts of the \`fabric\` folder)." + echo "- \`Direct_Snowflake.sql\`: the Snowflake edition, one deployment script." + if [ -n "$previous" ]; then + echo + echo "Full Changelog: https://github.com/$GITHUB_REPOSITORY/compare/$previous...v$VERSION" + fi + } >> dist/notes.md + cat dist/notes.md + + - name: Publish v${{ needs.build.outputs.version }} + if: steps.check.outputs.publish == 'true' + run: | + set -euo pipefail + gh release create "v$VERSION" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "DIRECT v$VERSION" \ + --notes-file dist/notes.md \ + dist/assets/Direct_Framework.dacpac \ + dist/assets/master.dacpac \ + dist/assets/Direct_Fabric.zip \ + dist/assets/Direct_Snowflake.sql + echo "::notice::Published https://github.com/$GITHUB_REPOSITORY/releases/tag/v$VERSION" diff --git a/Direct_Framework.Integration.Tests/Tests/MigrationTests/MigrationTests.cs b/Direct_Framework.Integration.Tests/Tests/MigrationTests/MigrationTests.cs index 71063b8..3336362 100644 --- a/Direct_Framework.Integration.Tests/Tests/MigrationTests/MigrationTests.cs +++ b/Direct_Framework.Integration.Tests/Tests/MigrationTests/MigrationTests.cs @@ -33,7 +33,7 @@ public async Task Can_Migrate_From_Current_To_Next() // 1. Deploy current version DACPAC // A dacpac has already been deployed by the init, so make sure this drops and redeploys a clean current - var currentDacpacPath = Path.Combine(projectDir, "Releases.Direct_Framework", "Current", "db", "Direct_Framework.dacpac"); + var currentDacpacPath = Path.Combine(projectDir, "Releases.Direct_Framework", "current", "db", "Direct_Framework.dacpac"); var dacServices = new DacServices(ContainerManager.MasterConnectionString); var currentDacpac = DacPackage.Load(currentDacpacPath); @@ -113,7 +113,7 @@ IF EXISTS (SELECT * FROM sys.databases WHERE name = 'Direct_Framework') //} // 4. Deploy next version DACPAC (built from the database project) - var nextDacpacPath = Path.Combine(projectDir, "Releases.Direct_Framework", "Next", "db", "Direct_Framework.dacpac"); + var nextDacpacPath = Path.Combine(projectDir, "Releases.Direct_Framework", "next", "db", "Direct_Framework.dacpac"); var nextDacpac = DacPackage.Load(nextDacpacPath); try { diff --git a/Documentation/ReleaseNotes.md b/Documentation/ReleaseNotes.md index 12d803c..991f95f 100644 --- a/Documentation/ReleaseNotes.md +++ b/Documentation/ReleaseNotes.md @@ -12,6 +12,7 @@ Changes: * `omd.ModuleEvaluation` has a new parameter `@ExecuteRollbackIndicator` (default `Y`). With `N` the rollback statement for the target data object is not executed but returned in the new output parameter `@RollbackStatement`, for execution on the engine that holds the data. The Source Control rollback still happens in the repository. * `omd.GetConsistencyTimestamp` has a new parameter `@SourceDataLookupIndicator` (default `Y`). With `N` the procedure uses repository information only and does not query the source tables, which allows it to run when the data lives in another database or engine. * New `fabric` folder with the standard Microsoft Fabric pipelines `DIRECT Run Batch` (a Batch with all its child Batches, in three activity runs) and `DIRECT Run Module`, a Variable Library definition, deployment scripts and a README. The pipelines use the procedures above against a DIRECT repository in a SQL database in Fabric and run the Module executables on a Fabric Warehouse. +* New `snowflake` folder with the Snowflake edition of the framework: the repository schemas, tables, views and procedures in Snowflake Scripting (`Direct_Snowflake.sql`, built from the files in `snowflake/sql`), Batch and Module registration, instance creation and evaluation, `RUN_MODULE` and `RUN_BATCH` for running registered executables in sequence with the child Batches, a serverless Task per Batch as queue, deployment and run scripts, a regression test suite and a README. * Fixed: `omd.BatchEvaluation` built an invalid `IN ((1,2))` list in the partial rollback branch, which failed every Batch run after two failed Batch Instances. * Fixed: the post-deployment seed of `omd.MODULE_INSTANCE` merged on `BATCH_INSTANCE_ID` instead of `MODULE_INSTANCE_ID`, which overwrote every Module Instance that ran outside a Batch on each deployment. * Fixed: `omd.ModuleEvaluation` quoted an already quoted `DATA_OBJECT_TARGET` in the rollback statement. diff --git a/README.md b/README.md index d06c8ce..0dba431 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,9 @@ winget install RedHat.Podman-Desktop ``` The `RunTests.ps1` PowerShell script can be run to run the DIRECT unit and regression tests in the container. + +### Releases + +The `Build and release` GitHub Actions workflow (`.github/workflows/release.yml`) builds the dacpac and runs the integration tests for every pull request and every push to `main`. A push to `main` also publishes the GitHub release `v` when the version in `.direct-version` has no release yet, with the release notes taken from the matching `## Release ` section of [ReleaseNotes.md](./Documentation/ReleaseNotes.md). + +To cut a release, bump the version in `.direct-version`, `Directory.Build.props`, `Releases.Direct_Framework/next/version.json` and the `DIRECT_VERSION` seed in `omd_metadata.FRAMEWORK_METADATA.sql`, add the release notes section, and merge. The workflow checks that these agree before it builds. Each release carries `Direct_Framework.dacpac` and `master.dacpac` (SQL Server), `Direct_Fabric.zip` (the `fabric` folder for Microsoft Fabric) and `Direct_Snowflake.sql` (the Snowflake edition). From aa77a52100e28984952b5fc6c4784164ce1d95f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 16:28:31 +0000 Subject: [PATCH 2/4] ci: run the integration tests in the Debug configuration The Release configuration treats warnings as errors, and the NuGet audit of the test project's transitive packages (SSH.NET via Testcontainers, System.Security.Cryptography.Pkcs via DacFx) reports vulnerabilities, so the restore failed on the Linux runner. The repository's own test runner uses Debug. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01P8xKsmMZRuuoGUZqqpAmVK --- .github/workflows/release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1659623..faf3707 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,8 +121,11 @@ jobs: path: Releases.Direct_Framework/next/db - name: Run the integration tests + # Debug, as RunTestsSimple.ps1 does: the Release configuration treats warnings as errors, and the + # NuGet audit of the test project's transitive packages (SSH.NET through Testcontainers, + # System.Security.Cryptography.Pkcs through DacFx) raises warnings that would fail the restore. run: > - dotnet run --project Direct_Framework.Integration.Tests --configuration Release -- + dotnet run --project Direct_Framework.Integration.Tests --configuration Debug -- --report-trx --report-trx-filename TestResults.trx --results-directory TestResults - name: Upload the test results From b1d56b1696026bc16c5124da6e03007ea50a5e0f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 16:36:36 +0000 Subject: [PATCH 3/4] ci: stage the fresh dacpac for the tests and check its version The project's post-build script did not see the Release configuration on the runner and left the tracked, older dacpac in the next release folder, so the tests ran against DIRECT 2.1.0 and 15 of them failed on the missing procedures. The workflow now copies bin/Release to the next release folder itself and fails when the dacpac's DacMetadata version is not the version under test. The test results folder is addressed from the workspace root so the trx is uploaded. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01P8xKsmMZRuuoGUZqqpAmVK --- .github/workflows/release.yml | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index faf3707..344beea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,15 +81,37 @@ jobs: shell: pwsh run: dotnet build Direct_Framework/Direct_Framework.sqlproj --configuration Release - - name: Check the build output + - name: Stage the build output as the next release shell: pwsh run: | $ErrorActionPreference = 'Stop' - # The post-build script copies the output to the next release folder, which is what the tests deploy. + # The project's post-build script copies bin\ to the next release folder, but it does not + # see the configuration of a dotnet build and looks in bin\Debug, so the copy is done here. The folder + # is tracked in git and holds the previous build until then. + $source = 'Direct_Framework/bin/Release' $db = 'Releases.Direct_Framework/next/db' + if (-not (Test-Path (Join-Path $source 'Direct_Framework.dacpac'))) { throw "The build left no Direct_Framework.dacpac in $source." } + Get-ChildItem $db -Force | Remove-Item -Recurse -Force + Copy-Item -Path (Join-Path $source '*') -Destination $db -Recurse -Force foreach ($file in 'Direct_Framework.dacpac', 'master.dacpac') { if (-not (Test-Path (Join-Path $db $file))) { throw "$file is missing from $db." } } + + # The dacpac must carry the version under test, not a stale one. + Add-Type -AssemblyName System.IO.Compression.FileSystem + $zip = [System.IO.Compression.ZipFile]::OpenRead((Resolve-Path (Join-Path $db 'Direct_Framework.dacpac'))) + try { + $entry = $zip.GetEntry('DacMetadata.xml') + $reader = [System.IO.StreamReader]::new($entry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + } + finally { $zip.Dispose() } + $expected = "${{ steps.version.outputs.version }}.0" + if ($metadata -notmatch "$([regex]::Escape($expected))") { + throw "Direct_Framework.dacpac does not carry version $expected. DacMetadata.xml: $metadata" + } + Write-Host "Direct_Framework.dacpac version $expected" Get-ChildItem $db -Recurse -File | Select-Object FullName, Length | Format-Table -AutoSize - name: Upload the build output @@ -126,14 +148,14 @@ jobs: # System.Security.Cryptography.Pkcs through DacFx) raises warnings that would fail the restore. run: > dotnet run --project Direct_Framework.Integration.Tests --configuration Debug -- - --report-trx --report-trx-filename TestResults.trx --results-directory TestResults + --report-trx --report-trx-filename TestResults.trx --results-directory "${{ github.workspace }}/TestResults" - name: Upload the test results if: always() uses: actions/upload-artifact@v4 with: name: integration-test-results - path: Direct_Framework.Integration.Tests/TestResults + path: TestResults if-no-files-found: ignore release: From c1bc26f19f1152d1defa1b4be2e3da8686c2960d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 16:45:16 +0000 Subject: [PATCH 4/4] ci: install sqlcmd on the test runner The migration test runs the pre-dacpac deployment script through sqlcmd, which the Linux runner does not have; 22 of 23 tests passed without it. Install go-sqlcmd 1.10.0 before the tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01P8xKsmMZRuuoGUZqqpAmVK --- .github/workflows/release.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 344beea..f54b8f9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -136,6 +136,15 @@ jobs: with: dotnet-version: 10.0.x + - name: Install sqlcmd + # The migration test runs the pre-dacpac deployment script through sqlcmd (the Go edition, as on a + # developer machine), which the Linux runner does not carry. + run: | + set -euo pipefail + curl -sSL -o /tmp/sqlcmd.tar.bz2 https://github.com/microsoft/go-sqlcmd/releases/download/v1.10.0/sqlcmd-linux-amd64.tar.bz2 + sudo tar -xjf /tmp/sqlcmd.tar.bz2 -C /usr/local/bin sqlcmd + sqlcmd --version + - name: Replace the next release folder with the build output uses: actions/download-artifact@v4 with: