diff --git a/.azure-pipelines/ci-build.yml b/.azure-pipelines/ci-build.yml index 2c8cb8735..5e8a40e98 100644 --- a/.azure-pipelines/ci-build.yml +++ b/.azure-pipelines/ci-build.yml @@ -21,6 +21,8 @@ pr: variables: buildPlatform: 'Any CPU' buildConfiguration: 'Release' + NuGetOrganizationName: 'openapinet' + privateFeedBaseUrl: 'https://microsoftgraph.pkgs.visualstudio.com/0985d294-5762-4bc2-a565-161ef349ca3e/_packaging/GraphDeveloperExperiences_Public' ProductBinPath: '$(Build.SourcesDirectory)\src\Microsoft.OpenApi\bin\$(BuildConfiguration)' REGISTRY: 'msgraphprodregistry.azurecr.io' IMAGE_NAME: 'public/openapi/hidi' @@ -88,7 +90,7 @@ extends: - + "@ | Set-Content -Path "$(Build.SourcesDirectory)/nuget.config" -Encoding UTF8 @@ -226,7 +228,16 @@ extends: inputs: targetFolder: $(Build.ArtifactStagingDirectory)/Nugets sourceFolder: $(Build.ArtifactStagingDirectory) - content: '*.nupkg' + Contents: | + *.nupkg + *.snupkg + + - task: CopyFiles@2 + displayName: 'Include version-check script in Nugets artifact' + inputs: + SourceFolder: '$(Build.SourcesDirectory)/scripts' + Contents: 'check-nuget-package-published.ps1' + TargetFolder: '$(Build.ArtifactStagingDirectory)/Nugets/scripts' # Copy repository files to be used in the deploy stage - task: CopyFiles@2 @@ -265,13 +276,46 @@ extends: pool: vmImage: ubuntu-latest steps: - - task: 1ES.PublishNuget@1 - displayName: 'NuGet push' + - task: PowerShell@2 + displayName: 'Check whether NuGet package version already published (idempotent)' + inputs: + targetType: filePath + filePath: '$(Pipeline.Workspace)/scripts/check-nuget-package-published.ps1' + arguments: '-PackageId "Microsoft.OpenApi.Hidi" -PackageDirectory "$(Pipeline.Workspace)" -NuGetServiceIndexUrl "$(privateFeedBaseUrl)/nuget/v3/index.json"' + pwsh: true + env: + FEED_ACCESS_TOKEN: $(System.AccessToken) + - task: CopyFiles@2 + displayName: 'Stage Hidi NuGet packages for ESRP release' + condition: and(succeeded(), ne(variables['nugetAlreadyPublished'], 'true')) + inputs: + SourceFolder: '$(Pipeline.Workspace)' + Contents: | + Microsoft.OpenApi.Hidi.*.nupkg + Microsoft.OpenApi.Hidi.*.snupkg + TargetFolder: '$(Pipeline.Workspace)/nuget-packages/$(NuGetOrganizationName)/hidi' + CleanTargetFolder: true + - task: EsrpRelease@14 + displayName: 'ESRP Release - Hidi NuGet' + condition: and(succeeded(), ne(variables['nugetAlreadyPublished'], 'true')) inputs: - packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.Hidi.*.nupkg' - packageParentPath: '$(Pipeline.Workspace)' - nuGetFeedType: external - publishFeedCredentials: 'OpenAPI Nuget Connection' + connectedservicename: 'Federated DevX ESRP Managed Identity Connection' + usemanagedidentity: false + keyvaultname: 'akv-prod-eastus' + authcertname: 'ReferenceLibraryPrivateCert' + signcertname: 'ReferencePackagePublisherCertificate' + clientid: '65035b7f-7357-4f29-bf25-c5ee5c3949f8' + intent: 'packagedistribution' + contenttype: 'NuGet' + organizationname: '$(NuGetOrganizationName)' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/nuget-packages/$(NuGetOrganizationName)/hidi' + waitforreleasecompletion: true + owners: 'christiano@microsoft.com,ramsess@microsoft.com,gavinbarron@microsoft.com,jingjingjia@microsoft.com,peombwa@microsoft.com,treicys@microsoft.com' + approvers: 'christiano@microsoft.com,ramsess@microsoft.com,gavinbarron@microsoft.com,jingjingjia@microsoft.com,peombwa@microsoft.com,treicys@microsoft.com' + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMAN' + domaintenantid: 'cdc5aeea-15c5-4db6-b079-fcadd2505dc2' - deployment: deploy_lib condition: and(contains(variables['build.SourceBranch'], 'refs/tags/v'), succeeded()) @@ -290,21 +334,48 @@ extends: pool: vmImage: ubuntu-latest steps: - - pwsh: | - $fileNames = "$(Pipeline.Workspace)/Microsoft.OpenApi.Hidi.*.nupkg", "$(Pipeline.Workspace)/Microsoft.OpenApi.YamlReader.*.nupkg" - foreach($fileName in $fileNames) { - if(Test-Path $fileName) { - Remove-Item $fileName -Verbose - } - } - displayName: remove other nupkgs to avoid duplication - - task: 1ES.PublishNuget@1 - displayName: 'NuGet push' + - task: PowerShell@2 + displayName: 'Check whether NuGet package version already published (idempotent)' + inputs: + targetType: filePath + filePath: '$(Pipeline.Workspace)/scripts/check-nuget-package-published.ps1' + arguments: '-PackageId "Microsoft.OpenApi" -PackageDirectory "$(Pipeline.Workspace)" -NuGetServiceIndexUrl "$(privateFeedBaseUrl)/nuget/v3/index.json"' + pwsh: true + env: + FEED_ACCESS_TOKEN: $(System.AccessToken) + - task: CopyFiles@2 + displayName: 'Stage OpenAPI NuGet packages for ESRP release' + condition: and(succeeded(), ne(variables['nugetAlreadyPublished'], 'true')) + inputs: + SourceFolder: '$(Pipeline.Workspace)' + Contents: | + Microsoft.OpenApi.*.nupkg + Microsoft.OpenApi.*.snupkg + !Microsoft.OpenApi.Hidi.* + !Microsoft.OpenApi.YamlReader.* + TargetFolder: '$(Pipeline.Workspace)/nuget-packages/$(NuGetOrganizationName)/openapi' + CleanTargetFolder: true + - task: EsrpRelease@14 + displayName: 'ESRP Release - OpenAPI NuGet' + condition: and(succeeded(), ne(variables['nugetAlreadyPublished'], 'true')) inputs: - packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.*.nupkg' - packageParentPath: '$(Pipeline.Workspace)' - nuGetFeedType: external - publishFeedCredentials: 'OpenAPI Nuget Connection' + connectedservicename: 'Federated DevX ESRP Managed Identity Connection' + usemanagedidentity: false + keyvaultname: 'akv-prod-eastus' + authcertname: 'ReferenceLibraryPrivateCert' + signcertname: 'ReferencePackagePublisherCertificate' + clientid: '65035b7f-7357-4f29-bf25-c5ee5c3949f8' + intent: 'packagedistribution' + contenttype: 'NuGet' + organizationname: '$(NuGetOrganizationName)' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/nuget-packages/$(NuGetOrganizationName)/openapi' + waitforreleasecompletion: true + owners: 'christiano@microsoft.com,ramsess@microsoft.com,gavinbarron@microsoft.com,jingjingjia@microsoft.com,peombwa@microsoft.com,treicys@microsoft.com' + approvers: 'christiano@microsoft.com,ramsess@microsoft.com,gavinbarron@microsoft.com,jingjingjia@microsoft.com,peombwa@microsoft.com,treicys@microsoft.com' + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMAN' + domaintenantid: 'cdc5aeea-15c5-4db6-b079-fcadd2505dc2' - deployment: deploy_yaml_reader condition: and(contains(variables['build.SourceBranch'], 'refs/tags/v'), succeeded()) @@ -323,13 +394,46 @@ extends: pool: vmImage: ubuntu-latest steps: - - task: 1ES.PublishNuget@1 - displayName: 'NuGet push' + - task: PowerShell@2 + displayName: 'Check whether NuGet package version already published (idempotent)' inputs: - packagesToPush: '$(Pipeline.Workspace)/Microsoft.OpenApi.YamlReader.*.nupkg' - packageParentPath: '$(Pipeline.Workspace)' - nuGetFeedType: external - publishFeedCredentials: 'OpenAPI Nuget Connection' + targetType: filePath + filePath: '$(Pipeline.Workspace)/scripts/check-nuget-package-published.ps1' + arguments: '-PackageId "Microsoft.OpenApi.YamlReader" -PackageDirectory "$(Pipeline.Workspace)" -NuGetServiceIndexUrl "$(privateFeedBaseUrl)/nuget/v3/index.json"' + pwsh: true + env: + FEED_ACCESS_TOKEN: $(System.AccessToken) + - task: CopyFiles@2 + displayName: 'Stage YAML reader NuGet packages for ESRP release' + condition: and(succeeded(), ne(variables['nugetAlreadyPublished'], 'true')) + inputs: + SourceFolder: '$(Pipeline.Workspace)' + Contents: | + Microsoft.OpenApi.YamlReader.*.nupkg + Microsoft.OpenApi.YamlReader.*.snupkg + TargetFolder: '$(Pipeline.Workspace)/nuget-packages/$(NuGetOrganizationName)/yaml-reader' + CleanTargetFolder: true + - task: EsrpRelease@14 + displayName: 'ESRP Release - YAML reader NuGet' + condition: and(succeeded(), ne(variables['nugetAlreadyPublished'], 'true')) + inputs: + connectedservicename: 'Federated DevX ESRP Managed Identity Connection' + usemanagedidentity: false + keyvaultname: 'akv-prod-eastus' + authcertname: 'ReferenceLibraryPrivateCert' + signcertname: 'ReferencePackagePublisherCertificate' + clientid: '65035b7f-7357-4f29-bf25-c5ee5c3949f8' + intent: 'packagedistribution' + contenttype: 'NuGet' + organizationname: '$(NuGetOrganizationName)' + contentsource: 'Folder' + folderlocation: '$(Pipeline.Workspace)/nuget-packages/$(NuGetOrganizationName)/yaml-reader' + waitforreleasecompletion: true + owners: 'christiano@microsoft.com,ramsess@microsoft.com,gavinbarron@microsoft.com,jingjingjia@microsoft.com,peombwa@microsoft.com,treicys@microsoft.com' + approvers: 'christiano@microsoft.com,ramsess@microsoft.com,gavinbarron@microsoft.com,jingjingjia@microsoft.com,peombwa@microsoft.com,treicys@microsoft.com' + serviceendpointurl: 'https://api.esrp.microsoft.com/' + mainpublisher: 'ESRPRELPACMAN' + domaintenantid: 'cdc5aeea-15c5-4db6-b079-fcadd2505dc2' - deployment: create_github_release condition: and(contains(variables['build.SourceBranch'], 'refs/tags/v'), succeeded()) @@ -459,7 +563,33 @@ extends: displayName: 'Get current date' name: setdate condition: eq(variables['Build.SourceBranch'], variables['PREVIEW_BRANCH']) - + + # Keep feed credentials out of the Docker build context and image layers. + - pwsh: | + if ([string]::IsNullOrWhiteSpace($env:FEED_ACCESS_TOKEN)) { + throw "No Azure Artifacts access token available for the Docker build." + } + $feedAccessToken = [System.Security.SecurityElement]::Escape($env:FEED_ACCESS_TOKEN) + @" + + + + + + + + + + + + + + + "@ | Set-Content -Path "$(Agent.TempDirectory)/hidi-docker.nuget.config" -Encoding UTF8 + displayName: 'Create Docker NuGet config (central feed)' + env: + FEED_ACCESS_TOKEN: $(System.AccessToken) + - script: | docker run --privileged --rm msgraphprodregistry.azurecr.io/tonistiigi/binfmt --install all displayName: "Enable multi-platform builds" @@ -478,6 +608,7 @@ extends: # Using quotes around tags to prevent flag interpretation docker buildx build \ --platform linux/amd64,linux/arm64/v8 \ + --secret id=nuget_config,src="$(Agent.TempDirectory)/hidi-docker.nuget.config" \ --push \ -t "$(REGISTRY)/$(IMAGE_NAME):nightly" \ -t "$(REGISTRY)/$(IMAGE_NAME):${VERSION}.${BUILDDATE}${RUNNUMBER}" \ @@ -490,6 +621,7 @@ extends: echo "Building Docker image for release..." docker buildx build\ --platform linux/amd64,linux/arm64/v8 \ + --secret id=nuget_config,src="$(Agent.TempDirectory)/hidi-docker.nuget.config" \ --push \ -t "$(REGISTRY)/$(IMAGE_NAME):latest" \ -t "$(REGISTRY)/$(IMAGE_NAME):${VERSION}" \ @@ -497,6 +629,14 @@ extends: displayName: 'Build and Push Release Image' condition: contains(variables['Build.SourceBranch'], 'refs/tags/v') + - pwsh: | + $configPath = "$(Agent.TempDirectory)/hidi-docker.nuget.config" + if (Test-Path $configPath) { + Remove-Item $configPath -Force + } + displayName: 'Remove Docker NuGet config' + condition: always() + # once the nuget has been released, fill this form to get the public documentation updated. # https://dev.azure.com/msft-skilling/Content/_workitems/create/User%20Story?templateId=39fb91e3-64a2-4c8a-83db-b2bdf3603dd3&ownerId=c4a28f90-17ae-4384-b514-7273392b082b # https://learn.microsoft.com/en-us/dotnet/api/microsoft.openapi diff --git a/Dockerfile b/Dockerfile index 46cb00637..4e932b975 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,9 @@ COPY ./src ./hidi/src COPY ./Directory.Build.props ./hidi/Directory.Build.props COPY ./README.md ./hidi/README.md WORKDIR /app/hidi -RUN dotnet publish ./src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj -c Release +# CI supplies the private feed config as a secret; local builds use default NuGet sources. +RUN --mount=type=secret,id=nuget_config,target=/app/hidi/NuGet.Config \ + dotnet publish ./src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj -c Release FROM mcr.microsoft.com/dotnet/runtime:8.0-jammy-chiseled AS runtime WORKDIR /app diff --git a/scripts/check-nuget-package-published.ps1 b/scripts/check-nuget-package-published.ps1 new file mode 100644 index 000000000..84f45419d --- /dev/null +++ b/scripts/check-nuget-package-published.ps1 @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.SYNOPSIS +Checks whether a NuGet artifact's version exists in an authenticated Azure Artifacts feed. +.DESCRIPTION +Resolves the package content endpoint from the private feed's NuGet v3 service index. +Sets nugetAlreadyPublished for the ESRP release steps; only a missing package or version +permits publishing. Feed authentication and other lookup failures fail the step. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$PackageDirectory, + [Parameter(Mandatory = $true)] + [string]$PackageId, + [Parameter(Mandatory = $true)] + [string]$NuGetServiceIndexUrl, + [string]$FeedAccessToken = $env:FEED_ACCESS_TOKEN +) + +$ErrorActionPreference = 'Stop' + +function Assert-PrivateFeedUrl { + param([string]$Url) + + $uri = [uri]$Url + if (-not $uri.IsAbsoluteUri -or $uri.Scheme -ne 'https' -or + ($uri.Host -ne 'pkgs.dev.azure.com' -and -not $uri.Host.EndsWith('.pkgs.visualstudio.com'))) { + throw "NuGet lookups must use an HTTPS Azure Artifacts feed: $Url" + } +} + +Assert-PrivateFeedUrl -Url $NuGetServiceIndexUrl +if ([string]::IsNullOrWhiteSpace($FeedAccessToken)) { + throw 'FEED_ACCESS_TOKEN is required to query the private NuGet feed.' +} + +$packagePattern = '^' + [regex]::Escape($PackageId) + '\.(\d[\w\.\-]*)\.nupkg$' +$packages = @(Get-ChildItem -Path $PackageDirectory -File -Filter "$PackageId.*.nupkg" | + Where-Object { $_.Name -match $packagePattern }) +if ($packages.Count -ne 1) { + throw "Expected exactly one $PackageId nupkg to publish; found $($packages.Count)." +} +$version = [regex]::Match($packages[0].Name, $packagePattern, 'IgnoreCase').Groups[1].Value +$id = $PackageId.ToLowerInvariant() +$credentials = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes("AzureDevOps:$FeedAccessToken")) +$headers = @{ + 'Authorization' = "Basic $credentials" + 'User-Agent' = 'openapi-azdo-pipeline' +} + +$index = Invoke-RestMethod -Uri $NuGetServiceIndexUrl -Headers $headers -MaximumRedirection 0 +$resource = $index.resources | Where-Object { $_.'@type' -eq 'PackageBaseAddress/3.0.0' } | Select-Object -First 1 +if ([string]::IsNullOrWhiteSpace($resource.'@id')) { + throw "No PackageBaseAddress resource found in the NuGet service index at $NuGetServiceIndexUrl" +} +$uri = "$($resource.'@id'.TrimEnd('/'))/$id/index.json" +Assert-PrivateFeedUrl -Url $uri + +try { + $response = Invoke-RestMethod -Uri $uri -Headers $headers -MaximumRedirection 0 + if ($null -eq $response.versions) { + throw "No versions returned for NuGet $id by the private feed." + } + $alreadyPublished = $response.versions -contains $version +} +catch { + if ([int]$_.Exception.Response.StatusCode -eq 404) { + $alreadyPublished = $false + } + else { + throw + } +} + +if ($alreadyPublished) { + Write-Host "NuGet $id $version already present in the private feed; skipping ESRP release (idempotent re-run)." +} +else { + Write-Host "NuGet $id $version not found in the private feed; will publish via ESRP." +} +Write-Host "##vso[task.setvariable variable=nugetAlreadyPublished]$($alreadyPublished.ToString().ToLowerInvariant())"