diff --git a/.azure-pipelines/hidi-release.yml b/.azure-pipelines/hidi-release.yml index 67fde2b0..216eae56 100644 --- a/.azure-pipelines/hidi-release.yml +++ b/.azure-pipelines/hidi-release.yml @@ -17,6 +17,8 @@ pr: variables: buildConfiguration: Release + NuGetOrganizationName: 'openapinet' + privateFeedBaseUrl: 'https://microsoftgraph.pkgs.visualstudio.com/0985d294-5762-4bc2-a565-161ef349ca3e/_packaging/GraphDeveloperExperiences_Public' # Keep OFF until the destination PR lands and source publishing is cut over. hidiPublishingEnabled: 'false' @@ -162,7 +164,7 @@ extends: Copy-Item artifacts\hidi\signing\Microsoft.OpenApi.Hidi.dll src\Microsoft.OpenApi.Hidi\bin\$(buildConfiguration)\net8.0\Microsoft.OpenApi.Hidi.dll Copy-Item artifacts\hidi\signing\Microsoft.OpenApi.Hidi.exe artifacts\hidi\win-x64\Microsoft.OpenApi.Hidi.exe New-Item -ItemType Directory -Force '$(Build.ArtifactStagingDirectory)\hidi' | Out-Null - dotnet pack src\Microsoft.OpenApi.Hidi\Microsoft.OpenApi.Hidi.csproj -c $(buildConfiguration) --no-build -o '$(Build.ArtifactStagingDirectory)\hidi' + dotnet pack src\Microsoft.OpenApi.Hidi\Microsoft.OpenApi.Hidi.csproj -c $(buildConfiguration) --no-build --include-symbols --include-source /p:SymbolPackageFormat=snupkg -o '$(Build.ArtifactStagingDirectory)\hidi' if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Compress-Archive -Path artifacts\hidi\win-x64\* -DestinationPath '$(Build.ArtifactStagingDirectory)\hidi\hidi-win-x64-$(HidiVersion).zip' Copy-Item artifacts\hidi\win-x64\Microsoft.OpenApi.Hidi.exe '$(Build.ArtifactStagingDirectory)\hidi\Microsoft.OpenApi.Hidi.exe' @@ -200,6 +202,12 @@ extends: MaxConcurrency: '50' MaxRetryAttempts: '5' PendingAnalysisWaitTimeoutMinutes: '5' + - task: CopyFiles@2 + displayName: Include private-feed version check in Hidi artifact + inputs: + SourceFolder: '$(Build.SourcesDirectory)\scripts' + Contents: 'check-nuget-package-published.ps1' + TargetFolder: '$(Build.ArtifactStagingDirectory)\hidi\scripts' - stage: publish dependsOn: build condition: and(succeeded(), eq(variables.hidiPublishingEnabled, 'true'), startsWith(variables['Build.SourceBranch'], 'refs/tags/hidi-v2.')) @@ -222,15 +230,50 @@ extends: if ([version]($tag -split '-')[0] -le [version]'2.12.2') { throw "Never republish the source migration baseline or an older hidi version." } $package = "$(Pipeline.Workspace)\hidi\Microsoft.OpenApi.Hidi.$tag.nupkg" if (-not (Test-Path $package)) { throw "Missing exact hidi package: $package" } + $symbols = "$(Pipeline.Workspace)\hidi\Microsoft.OpenApi.Hidi.$tag.snupkg" + if (-not (Test-Path $symbols)) { throw "Missing exact hidi symbols: $symbols" } Write-Host "##vso[task.setvariable variable=HidiReleaseVersion]$tag" displayName: Verify new hidi release version and exact package - - task: 1ES.PublishNuget@1 - displayName: Publish hidi NuGet tool + - task: PowerShell@2 + displayName: Check whether Hidi NuGet version is already published + inputs: + targetType: filePath + filePath: '$(Pipeline.Workspace)\hidi\scripts\check-nuget-package-published.ps1' + arguments: '-PackageId "Microsoft.OpenApi.Hidi" -PackageDirectory "$(Pipeline.Workspace)\hidi" -NuGetServiceIndexUrl "$(privateFeedBaseUrl)/nuget/v3/index.json"' + pwsh: true + env: + FEED_ACCESS_TOKEN: $(System.AccessToken) + - task: CopyFiles@2 + displayName: Stage exact Hidi NuGet packages for ESRP release + condition: and(succeeded(), ne(variables['nugetAlreadyPublished'], 'true')) + inputs: + SourceFolder: '$(Pipeline.Workspace)\hidi' + Contents: | + Microsoft.OpenApi.Hidi.$(HidiReleaseVersion).nupkg + Microsoft.OpenApi.Hidi.$(HidiReleaseVersion).snupkg + TargetFolder: '$(Pipeline.Workspace)\nuget-packages\$(NuGetOrganizationName)\hidi' + CleanTargetFolder: true + - task: EsrpRelease@14 + displayName: ESRP Release - Hidi NuGet + condition: and(succeeded(), ne(variables['nugetAlreadyPublished'], 'true')) inputs: - packagesToPush: '$(Pipeline.Workspace)\hidi\Microsoft.OpenApi.Hidi.$(HidiReleaseVersion).nupkg' - nuGetFeedType: external - publishFeedCredentials: OpenAPI Nuget Connection - packageParentPath: '$(Pipeline.Workspace)\hidi' + connectedservicename: 'Federated DevX ESRP Managed Identity Connection' + usemanagedidentity: false + keyvaultname: 'akv-prod-eastus' + authcertname: 'ReferenceLibraryPrivateCert' + signcertname: 'ReferencePackagePublisherCertificate' + clientid: '65035b7f-7357-4f29-bf25-c5ee5c3949f8' + intent: 'packagedistribution' + contenttype: 'NuGet' + organizationname: '$(NuGetOrganizationName)' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)\nuget-packages\$(NuGetOrganizationName)\hidi' + waitforreleasecompletion: true + owners: 'christiano@microsoft.com,ramsess@microsoft.com,gavinbarron@microsoft.com,jingjingjia@microsoft.com,peombwa@microsoft.com,treicys@microsoft.com' + approvers: 'christiano@microsoft.com,ramsess@microsoft.com,gavinbarron@microsoft.com,jingjingjia@microsoft.com,peombwa@microsoft.com,treicys@microsoft.com' + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMAN' + domaintenantid: 'cdc5aeea-15c5-4db6-b079-fcadd2505dc2' - task: GitHubRelease@1 displayName: Attach signed hidi release artifacts inputs: diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 3b5522a6..ca6bd5d3 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -57,6 +57,62 @@ jobs: restore-keys: ${{ runner.os }}-sonar - name: Install SonarCloud scanner run: dotnet tool install dotnet-sonarscanner --create-manifest-if-needed + - name: Install PowerShell test dependency + shell: pwsh + run: Install-Module Pester -RequiredVersion 5.7.1 -Scope CurrentUser -Force + - name: Test Hidi NuGet helper with measured coverage + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + Import-Module Pester -RequiredVersion 5.7.1 -ErrorAction Stop + $helperPath = 'scripts/check-nuget-package-published.ps1' + $helper = (Resolve-Path $helperPath).Path + $outputRoot = Join-Path $PWD 'artifacts\hidi\powershell-coverage' + New-Item -ItemType Directory -Force -Path $outputRoot | Out-Null + $configuration = New-PesterConfiguration + $configuration.Run.Path = 'test\scripts\check-nuget-package-published.Tests.ps1' + $configuration.Run.PassThru = $true + $configuration.CodeCoverage.Enabled = $true + $configuration.CodeCoverage.Path = $helper + $configuration.CodeCoverage.OutputFormat = 'JaCoCo' + $configuration.CodeCoverage.OutputPath = Join-Path $outputRoot 'pester-coverage.xml' + $configuration.CodeCoverage.CoveragePercentTarget = 80 + $result = Invoke-Pester -Configuration $configuration + if ($result.FailedCount -gt 0 -or $result.PassedCount -lt 24) { + throw "Hidi NuGet helper tests failed or did not execute all 24 cases." + } + $report = [xml](Get-Content $configuration.CodeCoverage.OutputPath.Value -Raw) + $sourceFiles = @($report.SelectNodes('//sourcefile')) + if ($sourceFiles.Count -ne 1 -or $sourceFiles[0].name -ne [IO.Path]::GetFileName($helper)) { + throw 'Expected measured coverage of only the Hidi NuGet helper.' + } + $lines = @($sourceFiles[0].SelectNodes('line')) + $sourceLineCount = @(Get-Content $helper).Count + $lineNumbers = @($lines | ForEach-Object { [int]$_.nr }) + if ($lines.Count -eq 0 -or @($lineNumbers | Select-Object -Unique).Count -ne $lines.Count -or + @($lines | Where-Object { + [int]$_.nr -le 0 -or [int]$_.nr -gt $sourceLineCount -or + -not $_.HasAttribute('ci') -or -not $_.HasAttribute('mi') -or + [int]$_.ci -lt 0 -or [int]$_.mi -lt 0 -or ([int]$_.ci + [int]$_.mi) -le 0 + }).Count -gt 0) { + throw 'Missing or invalid measured helper coverage lines.' + } + $covered = @($lines | Where-Object { [int]$_.ci -gt 0 }).Count + if ($covered / $lines.Count -lt 0.8) { + throw "Insufficient measured Hidi helper coverage: $covered/$($lines.Count) lines." + } + $coverage = [xml]'' + $file = $coverage.CreateElement('file') + $file.SetAttribute('path', $helperPath) + [void]$coverage.DocumentElement.AppendChild($file) + foreach ($line in $lines) { + $entry = $coverage.CreateElement('lineToCover') + $entry.SetAttribute('lineNumber', $line.nr) + $entry.SetAttribute('covered', ([int]$line.ci -gt 0).ToString().ToLowerInvariant()) + [void]$file.AppendChild($entry) + } + $coverage.Save((Join-Path $outputRoot 'sonar-coverage.xml')) + Write-Host "Measured Hidi NuGet helper coverage: $covered/$($lines.Count) lines; Sonar generic report generated." - name: Build and analyze env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any @@ -64,7 +120,7 @@ jobs: CoverletOutputFormat: 'opencover' # https://github.com/microsoft/vstest/issues/4014#issuecomment-1307913682 shell: pwsh run: | - dotnet tool run dotnet-sonarscanner begin /k:"microsoft_OpenAPI.NET.OData" /o:"microsoft" /d:sonar.token="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.opencover.reportsPaths="test/**/coverage.net8.0.opencover.xml,artifacts/hidi/coverage/*opencover*.xml" + dotnet tool run dotnet-sonarscanner begin /k:"microsoft_OpenAPI.NET.OData" /o:"microsoft" /d:sonar.token="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.opencover.reportsPaths="test/**/coverage.net8.0.opencover.xml,artifacts/hidi/coverage/*opencover*.xml" /d:sonar.coverageReportPaths="artifacts/hidi/powershell-coverage/sonar-coverage.xml" dotnet workload restore dotnet build dotnet test test\Microsoft.OpenAPI.OData.Reader.Tests\Microsoft.OpenAPI.OData.Reader.Tests.csproj --no-build --verbosity normal /p:CollectCoverage=true /p:CoverletOutputFormat=opencover diff --git a/README.md b/README.md index 49bca1a8..a1e490db 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,20 @@ imported commits can also be retained by the later main-branch migration. Destination hidi release and production publishing are disabled until source cutover; OpenAPI.NET remains the publisher in the meantime. +The gated hidi NuGet release uses `EsrpRelease@14`, staging only the exact +`Microsoft.OpenApi.Hidi` package and its `.snupkg` symbols from the Hidi build +artifact. The authenticated private-feed version check is ported from +[microsoft/OpenAPI.NET#3107](https://github.com/microsoft/OpenAPI.NET/pull/3107). +An existing version skips ESRP on a re-run; authentication, network, and malformed +feed responses fail closed. Releases must use an exact `hidi-v2.` tag +matching the project version and be newer than the `2.12.2` migration baseline. +This pipeline implementation does not enable publishing or authorize resources. + +The SonarCloud workflow runs the private-feed helper tests with Pester 5.7.1 and +converts measured JaCoCo line hits to Sonar generic coverage, alongside the +existing C# OpenCover reports. Run the helper tests locally with +`Import-Module Pester -RequiredVersion 5.7.1; Invoke-Pester .\test\scripts\check-nuget-package-published.Tests.ps1`. + --- # Contributing diff --git a/scripts/check-nuget-package-published.ps1 b/scripts/check-nuget-package-published.ps1 new file mode 100644 index 00000000..fea22f1c --- /dev/null +++ b/scripts/check-nuget-package-published.ps1 @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.SYNOPSIS +Checks whether a NuGet artifact's version exists in an authenticated Azure Artifacts feed. +.DESCRIPTION +Resolves the package content endpoint from the private feed's NuGet v3 service index. +Sets nugetAlreadyPublished for the ESRP release steps; only a missing package or version +permits publishing. Feed authentication and other lookup failures fail the step. +.NOTES +Ported from microsoft/OpenAPI.NET scripts/check-nuget-package-published.ps1 +at e1a75437b76ebfc7c9eb446e9fd0b59a21afb14b (#3107), with fail-closed validation +of malformed feed version lists. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$PackageDirectory, + [Parameter(Mandatory = $true)] + [string]$PackageId, + [Parameter(Mandatory = $true)] + [string]$NuGetServiceIndexUrl, + [string]$FeedAccessToken = $env:FEED_ACCESS_TOKEN +) + +$ErrorActionPreference = 'Stop' + +function Assert-PrivateFeedUrl { + param([string]$Url) + + $uri = [uri]$Url + if (-not $uri.IsAbsoluteUri -or $uri.Scheme -ne 'https' -or + ($uri.Host -ne 'pkgs.dev.azure.com' -and -not $uri.Host.EndsWith('.pkgs.visualstudio.com'))) { + throw "NuGet lookups must use an HTTPS Azure Artifacts feed: $Url" + } +} + +Assert-PrivateFeedUrl -Url $NuGetServiceIndexUrl +if ([string]::IsNullOrWhiteSpace($FeedAccessToken)) { + throw 'FEED_ACCESS_TOKEN is required to query the private NuGet feed.' +} + +$packagePattern = '^' + [regex]::Escape($PackageId) + '\.(\d[\w\.\-]*)\.nupkg$' +$packages = @(Get-ChildItem -Path $PackageDirectory -File -Filter "$PackageId.*.nupkg" | + Where-Object { $_.Name -match $packagePattern }) +if ($packages.Count -ne 1) { + throw "Expected exactly one $PackageId nupkg to publish; found $($packages.Count)." +} +$version = [regex]::Match($packages[0].Name, $packagePattern, 'IgnoreCase').Groups[1].Value +$id = $PackageId.ToLowerInvariant() +$credentials = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes("AzureDevOps:$FeedAccessToken")) +$headers = @{ + 'Authorization' = "Basic $credentials" + 'User-Agent' = 'openapi-azdo-pipeline' +} + +$index = Invoke-RestMethod -Uri $NuGetServiceIndexUrl -Headers $headers -MaximumRedirection 0 +$resource = $index.resources | Where-Object { $_.'@type' -eq 'PackageBaseAddress/3.0.0' } | Select-Object -First 1 +if ([string]::IsNullOrWhiteSpace($resource.'@id')) { + throw "No PackageBaseAddress resource found in the NuGet service index at $NuGetServiceIndexUrl" +} +$uri = "$($resource.'@id'.TrimEnd('/'))/$id/index.json" +Assert-PrivateFeedUrl -Url $uri + +try { + $response = Invoke-RestMethod -Uri $uri -Headers $headers -MaximumRedirection 0 + if ($null -eq $response.versions) { + throw "No versions returned for NuGet $id by the private feed." + } + if ($response.versions -isnot [array] -or @($response.versions | Where-Object { + $_ -isnot [string] -or $_ -notmatch '^\d[\w\.\-]*$' + }).Count -gt 0) { + throw "Invalid version list returned for NuGet $id by the private feed." + } + $alreadyPublished = $response.versions -contains $version +} +catch { + if ([int]$_.Exception.Response.StatusCode -eq 404) { + $alreadyPublished = $false + } + else { + throw + } +} + +if ($alreadyPublished) { + Write-Host "NuGet $id $version already present in the private feed; skipping ESRP release (idempotent re-run)." +} +else { + Write-Host "NuGet $id $version not found in the private feed; will publish via ESRP." +} +Write-Host "##vso[task.setvariable variable=nugetAlreadyPublished]$($alreadyPublished.ToString().ToLowerInvariant())" diff --git a/test/scripts/check-nuget-package-published.Tests.ps1 b/test/scripts/check-nuget-package-published.Tests.ps1 new file mode 100644 index 00000000..0862f728 --- /dev/null +++ b/test/scripts/check-nuget-package-published.Tests.ps1 @@ -0,0 +1,96 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +BeforeDiscovery { + $cases = @( + @{ Name = 'published version'; Versions = @('2.12.2', '2.12.3'); Expected = 'true' } + @{ Name = 'case-insensitive prerelease'; Versions = @('2.12.3-PREVIEW.1'); PackageVersion = '2.12.3-preview.1'; Expected = 'true' } + @{ Name = 'missing version'; Versions = @('2.12.2'); Expected = 'false' } + @{ Name = 'empty version list'; Versions = @(); Expected = 'false' } + @{ Name = 'package 404'; PackageStatus = 404; Expected = 'false' } + @{ Name = 'package 401'; PackageStatus = 401; Fail = $true } + @{ Name = 'package 403'; PackageStatus = 403; Fail = $true } + @{ Name = 'package 500'; PackageStatus = 500; Fail = $true } + @{ Name = 'service index 404'; IndexStatus = 404; Fail = $true } + @{ Name = 'service index 401'; IndexStatus = 401; Fail = $true } + @{ Name = 'network failure'; NetworkFailure = $true; Fail = $true } + @{ Name = 'missing versions'; MissingVersions = $true; Fail = $true } + @{ Name = 'string instead of version array'; Versions = '2.12.2'; Fail = $true } + @{ Name = 'object instead of version array'; Versions = @{ error = 'unauthorized' }; Fail = $true } + @{ Name = 'invalid version'; Versions = @('not-a-version'); Fail = $true } + @{ Name = 'null version'; Versions = @($null); Fail = $true } + @{ Name = 'missing content resource'; MissingResource = $true; Fail = $true } + @{ Name = 'unsafe content resource'; UnsafeResource = $true; Fail = $true } + @{ Name = 'missing package'; Files = @(); Fail = $true; Requests = 0 } + @{ Name = 'wrong package'; Files = @('Microsoft.OpenApi.OData.2.12.3.nupkg'); Fail = $true; Requests = 0 } + @{ Name = 'ambiguous Hidi versions'; Files = @('Microsoft.OpenApi.Hidi.2.12.3.nupkg', 'Microsoft.OpenApi.Hidi.2.12.4.nupkg'); Fail = $true; Requests = 0 } + @{ Name = 'missing token'; Token = ''; Fail = $true; Requests = 0 } + @{ Name = 'public service index'; Url = 'https://api.nuget.org/v3/index.json'; Fail = $true; Requests = 0 } + @{ Name = 'mixed artifacts'; Files = @('Microsoft.OpenApi.Hidi.2.12.3.nupkg', 'Microsoft.OpenApi.Hidi.2.12.3.snupkg', 'Microsoft.OpenApi.OData.2.12.3.nupkg', 'Microsoft.OpenApi.2.12.3.nupkg'); Versions = @('2.12.3'); Expected = 'true' } + ) +} + +Describe 'Authenticated Hidi NuGet idempotency' { + BeforeAll { + $helper = Join-Path $PSScriptRoot '..\..\scripts\check-nuget-package-published.ps1' + $serviceIndex = 'https://microsoftgraph.pkgs.visualstudio.com/project/_packaging/feed/nuget/v3/index.json' + $baseAddress = 'https://microsoftgraph.pkgs.visualstudio.com/project/_packaging/feed/nuget/v3/flat2' + } + + It '' -ForEach $cases { + $testCase = $_ + $directory = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Path $directory | Out-Null + $version = if ($PackageVersion) { $PackageVersion } else { '2.12.3' } + $packageFiles = if ($testCase.ContainsKey('Files')) { $Files } else { @("Microsoft.OpenApi.Hidi.$version.nupkg") } + foreach ($file in $packageFiles) { New-Item -ItemType File -Path (Join-Path $directory $file) | Out-Null } + $token = if ($testCase.ContainsKey('Token')) { $Token } else { 'mock-test-token' } + $url = if ($Url) { $Url } else { $serviceIndex } + + Mock Invoke-RestMethod { + param($Uri, $Headers, $MaximumRedirection) + $expectedAuth = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes('AzureDevOps:mock-test-token')) + $Headers.Authorization | Should -BeExactly $expectedAuth + $MaximumRedirection | Should -Be 0 + if ($Uri -eq $serviceIndex) { + if ($IndexStatus) { + $response = [System.Net.Http.HttpResponseMessage]::new([System.Net.HttpStatusCode]$IndexStatus) + throw [Microsoft.PowerShell.Commands.HttpResponseException]::new('Mock service-index failure', $response) + } + if ($MissingResource) { return @{ resources = @() } } + $address = if ($UnsafeResource) { 'https://example.invalid/flat2' } else { $baseAddress } + return @{ resources = @(@{ '@type' = 'PackageBaseAddress/3.0.0'; '@id' = $address }) } + } + $Uri | Should -BeExactly "$baseAddress/microsoft.openapi.hidi/index.json" + if ($NetworkFailure) { throw [System.Net.Http.HttpRequestException]::new('Mock network failure') } + if ($PackageStatus) { + $response = [System.Net.Http.HttpResponseMessage]::new([System.Net.HttpStatusCode]$PackageStatus) + throw [Microsoft.PowerShell.Commands.HttpResponseException]::new('Mock package failure', $response) + } + if ($MissingVersions) { return @{} } + return @{ versions = $Versions } + } + + $output = [Collections.Generic.List[string]]::new() + $failure = $null + try { + & $helper -PackageDirectory $directory -PackageId 'Microsoft.OpenApi.Hidi' -NuGetServiceIndexUrl $url -FeedAccessToken $token 6>&1 | + ForEach-Object { $output.Add($_.ToString()) } + } + catch { $failure = $_ } + $text = $output -join "`n" + if ($Fail) { + $failure | Should -Not -BeNullOrEmpty + $text | Should -Not -Match '##vso\[task.setvariable' + } + else { + $failure | Should -BeNullOrEmpty + $text | Should -Match "##vso\[task.setvariable variable=nugetAlreadyPublished\]$Expected" + } + $text | Should -Not -Match 'mock-test-token|AzureDevOps:' + $expectedRequests = if ($testCase.ContainsKey('Requests')) { $Requests } + elseif ($IndexStatus -or $MissingResource -or $UnsafeResource) { 1 } + else { 2 } + Should -Invoke Invoke-RestMethod -Times $expectedRequests -Exactly + } +}