diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f54b8f9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,260 @@ +# Builds the DIRECT Framework dacpac, runs the integration tests and publishes a GitHub release. +# +# Every push to main and every pull request builds the dacpac on Windows (the SQL project runs +# cmd pre- and post-build scripts) and runs the integration tests on Linux (they need a Linux +# SQL Server container). A push to main also publishes the release v when the version +# in .direct-version has no release yet, so a release is cut by merging the version bump. +# +# A manual run (workflow_dispatch) builds and tests any branch; tick 'publish' to also publish +# the release from that branch. + +name: Build and release + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + inputs: + publish: + description: Publish the GitHub release when the version in .direct-version has none yet + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +env: + DOTNET_NOLOGO: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + +jobs: + build: + name: Build the dacpac + runs-on: windows-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + + - name: Check that the version files agree + id: version + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $version = (Get-Content .direct-version -Raw).Trim() + if ($version -notmatch '^\d+\.\d+\.\d+$') { throw ".direct-version holds '$version', expected major.minor.patch." } + $problems = @() + + $props = Get-Content Directory.Build.props -Raw + if ($props -notmatch "$([regex]::Escape($version))(-[0-9A-Za-z.]+)?") { $problems += "Directory.Build.props does not start with $version." } + if ($props -notmatch "$([regex]::Escape($version))\.0") { $problems += "Directory.Build.props is not $version.0." } + + $next = Get-Content Releases.Direct_Framework/next/version.json -Raw | ConvertFrom-Json + if ($next.directFrameworkVersion -ne $version) { $problems += "Releases.Direct_Framework/next/version.json has directFrameworkVersion $($next.directFrameworkVersion)." } + + $metadata = Get-Content Direct_Framework/DeploymentScripts/3-PostDeployment/omd_metadata.FRAMEWORK_METADATA.sql -Raw + if ($metadata -notmatch "N'DIRECT_VERSION',\s*N'$([regex]::Escape($version))'") { $problems += "omd_metadata.FRAMEWORK_METADATA.sql does not seed DIRECT_VERSION $version." } + + $notes = Get-Content Documentation/ReleaseNotes.md -Raw + if ($notes -notmatch "(?m)^## Release $([regex]::Escape($version))\s*$") { $problems += "Documentation/ReleaseNotes.md has no section '## Release $version'." } + + if ($problems.Count -gt 0) { + $problems | ForEach-Object { Write-Host "::error::$_" } + throw "The version files do not agree on $version." + } + Write-Host "DIRECT version $version" + "version=$version" >> $env:GITHUB_OUTPUT + + - name: Build Direct_Framework (Release) + shell: pwsh + run: dotnet build Direct_Framework/Direct_Framework.sqlproj --configuration Release + + - name: Stage the build output as the next release + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + # The project's post-build script copies bin\ to the next release folder, but it does not + # see the configuration of a dotnet build and looks in bin\Debug, so the copy is done here. The folder + # is tracked in git and holds the previous build until then. + $source = 'Direct_Framework/bin/Release' + $db = 'Releases.Direct_Framework/next/db' + if (-not (Test-Path (Join-Path $source 'Direct_Framework.dacpac'))) { throw "The build left no Direct_Framework.dacpac in $source." } + Get-ChildItem $db -Force | Remove-Item -Recurse -Force + Copy-Item -Path (Join-Path $source '*') -Destination $db -Recurse -Force + foreach ($file in 'Direct_Framework.dacpac', 'master.dacpac') { + if (-not (Test-Path (Join-Path $db $file))) { throw "$file is missing from $db." } + } + + # The dacpac must carry the version under test, not a stale one. + Add-Type -AssemblyName System.IO.Compression.FileSystem + $zip = [System.IO.Compression.ZipFile]::OpenRead((Resolve-Path (Join-Path $db 'Direct_Framework.dacpac'))) + try { + $entry = $zip.GetEntry('DacMetadata.xml') + $reader = [System.IO.StreamReader]::new($entry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + } + finally { $zip.Dispose() } + $expected = "${{ steps.version.outputs.version }}.0" + if ($metadata -notmatch "$([regex]::Escape($expected))") { + throw "Direct_Framework.dacpac does not carry version $expected. DacMetadata.xml: $metadata" + } + Write-Host "Direct_Framework.dacpac version $expected" + Get-ChildItem $db -Recurse -File | Select-Object FullName, Length | Format-Table -AutoSize + + - name: Upload the build output + uses: actions/upload-artifact@v4 + with: + name: direct-framework-db + path: Releases.Direct_Framework/next/db + if-no-files-found: error + retention-days: 30 + + test: + name: Integration tests + runs-on: ubuntu-latest + needs: build + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + + - name: Install sqlcmd + # The migration test runs the pre-dacpac deployment script through sqlcmd (the Go edition, as on a + # developer machine), which the Linux runner does not carry. + run: | + set -euo pipefail + curl -sSL -o /tmp/sqlcmd.tar.bz2 https://github.com/microsoft/go-sqlcmd/releases/download/v1.10.0/sqlcmd-linux-amd64.tar.bz2 + sudo tar -xjf /tmp/sqlcmd.tar.bz2 -C /usr/local/bin sqlcmd + sqlcmd --version + + - name: Replace the next release folder with the build output + uses: actions/download-artifact@v4 + with: + name: direct-framework-db + path: Releases.Direct_Framework/next/db + + - name: Run the integration tests + # Debug, as RunTestsSimple.ps1 does: the Release configuration treats warnings as errors, and the + # NuGet audit of the test project's transitive packages (SSH.NET through Testcontainers, + # System.Security.Cryptography.Pkcs through DacFx) raises warnings that would fail the restore. + run: > + dotnet run --project Direct_Framework.Integration.Tests --configuration Debug -- + --report-trx --report-trx-filename TestResults.trx --results-directory "${{ github.workspace }}/TestResults" + + - name: Upload the test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-test-results + path: TestResults + if-no-files-found: ignore + + release: + name: Publish the release + runs-on: ubuntu-latest + needs: [ build, test ] + if: > + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + (github.event_name == 'workflow_dispatch' && inputs.publish) + permissions: + contents: write + env: + VERSION: ${{ needs.build.outputs.version }} + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Skip when v${{ needs.build.outputs.version }} is already released + id: check + run: | + if gh release view "v$VERSION" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then + echo "::notice::Release v$VERSION exists already; nothing to publish. Bump .direct-version and the other version files to release again." + echo "publish=false" >> "$GITHUB_OUTPUT" + elif git rev-parse -q --verify "refs/tags/v$VERSION" > /dev/null; then + echo "::error::Tag v$VERSION exists without a release. Delete the tag or publish the release by hand." + exit 1 + else + echo "publish=true" >> "$GITHUB_OUTPUT" + fi + + - name: Download the build output + if: steps.check.outputs.publish == 'true' + uses: actions/download-artifact@v4 + with: + name: direct-framework-db + path: dist/db + + - name: Assemble the release assets + if: steps.check.outputs.publish == 'true' + run: | + set -euo pipefail + mkdir -p dist/assets + cp dist/db/Direct_Framework.dacpac dist/db/master.dacpac dist/assets/ + (cd fabric && zip -q -r ../dist/assets/Direct_Fabric.zip .) + cp snowflake/Direct_Snowflake.sql dist/assets/ + ls -l dist/assets + + - name: Write the release notes + if: steps.check.outputs.publish == 'true' + run: | + set -euo pipefail + # The section of Documentation/ReleaseNotes.md for this version, without its heading. + awk -v v="$VERSION" ' + $0 ~ "^## Release " v "[[:space:]]*$" { on = 1; next } + on && /^## / { exit } + on { print } + ' Documentation/ReleaseNotes.md | sed -e :a -e '/^\n*$/{$d;N;ba' -e '}' > dist/notes.md + if [ ! -s dist/notes.md ]; then + echo "::error::Documentation/ReleaseNotes.md has no content under '## Release $VERSION'." + exit 1 + fi + previous=$(git tag --list 'v*' --sort=-v:refname | head -n 1) + { + echo + echo "### Assets" + echo + echo "- \`Direct_Framework.dacpac\` and \`master.dacpac\`: the SQL Server edition, deployed with SqlPackage or DacFx." + echo "- \`Direct_Fabric.zip\`: the Microsoft Fabric edition (standard pipelines, Variable Library and scripts of the \`fabric\` folder)." + echo "- \`Direct_Snowflake.sql\`: the Snowflake edition, one deployment script." + if [ -n "$previous" ]; then + echo + echo "Full Changelog: https://github.com/$GITHUB_REPOSITORY/compare/$previous...v$VERSION" + fi + } >> dist/notes.md + cat dist/notes.md + + - name: Publish v${{ needs.build.outputs.version }} + if: steps.check.outputs.publish == 'true' + run: | + set -euo pipefail + gh release create "v$VERSION" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "DIRECT v$VERSION" \ + --notes-file dist/notes.md \ + dist/assets/Direct_Framework.dacpac \ + dist/assets/master.dacpac \ + dist/assets/Direct_Fabric.zip \ + dist/assets/Direct_Snowflake.sql + echo "::notice::Published https://github.com/$GITHUB_REPOSITORY/releases/tag/v$VERSION" diff --git a/Direct_Framework.Integration.Tests/Tests/MigrationTests/MigrationTests.cs b/Direct_Framework.Integration.Tests/Tests/MigrationTests/MigrationTests.cs index 71063b8..3336362 100644 --- a/Direct_Framework.Integration.Tests/Tests/MigrationTests/MigrationTests.cs +++ b/Direct_Framework.Integration.Tests/Tests/MigrationTests/MigrationTests.cs @@ -33,7 +33,7 @@ public async Task Can_Migrate_From_Current_To_Next() // 1. Deploy current version DACPAC // A dacpac has already been deployed by the init, so make sure this drops and redeploys a clean current - var currentDacpacPath = Path.Combine(projectDir, "Releases.Direct_Framework", "Current", "db", "Direct_Framework.dacpac"); + var currentDacpacPath = Path.Combine(projectDir, "Releases.Direct_Framework", "current", "db", "Direct_Framework.dacpac"); var dacServices = new DacServices(ContainerManager.MasterConnectionString); var currentDacpac = DacPackage.Load(currentDacpacPath); @@ -113,7 +113,7 @@ IF EXISTS (SELECT * FROM sys.databases WHERE name = 'Direct_Framework') //} // 4. Deploy next version DACPAC (built from the database project) - var nextDacpacPath = Path.Combine(projectDir, "Releases.Direct_Framework", "Next", "db", "Direct_Framework.dacpac"); + var nextDacpacPath = Path.Combine(projectDir, "Releases.Direct_Framework", "next", "db", "Direct_Framework.dacpac"); var nextDacpac = DacPackage.Load(nextDacpacPath); try { diff --git a/Documentation/ReleaseNotes.md b/Documentation/ReleaseNotes.md index 12d803c..991f95f 100644 --- a/Documentation/ReleaseNotes.md +++ b/Documentation/ReleaseNotes.md @@ -12,6 +12,7 @@ Changes: * `omd.ModuleEvaluation` has a new parameter `@ExecuteRollbackIndicator` (default `Y`). With `N` the rollback statement for the target data object is not executed but returned in the new output parameter `@RollbackStatement`, for execution on the engine that holds the data. The Source Control rollback still happens in the repository. * `omd.GetConsistencyTimestamp` has a new parameter `@SourceDataLookupIndicator` (default `Y`). With `N` the procedure uses repository information only and does not query the source tables, which allows it to run when the data lives in another database or engine. * New `fabric` folder with the standard Microsoft Fabric pipelines `DIRECT Run Batch` (a Batch with all its child Batches, in three activity runs) and `DIRECT Run Module`, a Variable Library definition, deployment scripts and a README. The pipelines use the procedures above against a DIRECT repository in a SQL database in Fabric and run the Module executables on a Fabric Warehouse. +* New `snowflake` folder with the Snowflake edition of the framework: the repository schemas, tables, views and procedures in Snowflake Scripting (`Direct_Snowflake.sql`, built from the files in `snowflake/sql`), Batch and Module registration, instance creation and evaluation, `RUN_MODULE` and `RUN_BATCH` for running registered executables in sequence with the child Batches, a serverless Task per Batch as queue, deployment and run scripts, a regression test suite and a README. * Fixed: `omd.BatchEvaluation` built an invalid `IN ((1,2))` list in the partial rollback branch, which failed every Batch run after two failed Batch Instances. * Fixed: the post-deployment seed of `omd.MODULE_INSTANCE` merged on `BATCH_INSTANCE_ID` instead of `MODULE_INSTANCE_ID`, which overwrote every Module Instance that ran outside a Batch on each deployment. * Fixed: `omd.ModuleEvaluation` quoted an already quoted `DATA_OBJECT_TARGET` in the rollback statement. diff --git a/README.md b/README.md index d06c8ce..0dba431 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,9 @@ winget install RedHat.Podman-Desktop ``` The `RunTests.ps1` PowerShell script can be run to run the DIRECT unit and regression tests in the container. + +### Releases + +The `Build and release` GitHub Actions workflow (`.github/workflows/release.yml`) builds the dacpac and runs the integration tests for every pull request and every push to `main`. A push to `main` also publishes the GitHub release `v` when the version in `.direct-version` has no release yet, with the release notes taken from the matching `## Release ` section of [ReleaseNotes.md](./Documentation/ReleaseNotes.md). + +To cut a release, bump the version in `.direct-version`, `Directory.Build.props`, `Releases.Direct_Framework/next/version.json` and the `DIRECT_VERSION` seed in `omd_metadata.FRAMEWORK_METADATA.sql`, add the release notes section, and merge. The workflow checks that these agree before it builds. Each release carries `Direct_Framework.dacpac` and `master.dacpac` (SQL Server), `Direct_Fabric.zip` (the `fabric` folder for Microsoft Fabric) and `Direct_Snowflake.sql` (the Snowflake edition).