mirrors/PowerShell
0
0
Fork 0
mirror of https://github.com/PowerShell/PowerShell.git synced 2024-11-23 09:43:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixes to Azure Public feed usage (#24149)

Browse Source 
* Fix code PATs

* Fix feed PATs

* remove gallery commit

* update feed url

* Update to new variable groups

* Fix Variable name

* Fix credential template

* Disable Signing setup in prep stage

* Capture nuget source list

* lock down the firewall

* Add creds to feed switch to allow single switch location

* Use switch from build.psm1

* Use switch template instead of commands

* update to test feed

* disable codeql in jobs where we don't compile

* disable code sign validation for prep

* move capture steps to restore phase to see if it speeds things up

* remove duplicate capture of nuget config

* update test service

* Only build windows test service on windows

* warn when no config is generated

* try to fix test service

* fix web listener refs

* try removing dotnet tool

* update feedname with user info

* update package version that is not found

* try moving failing jobs to restore phase

* allow nuget inset in either phase

* update package ref

* use the right reporoot

* Move everything to restore

* Try adding build phase

* put nuget files in the right place

* move bootstrap into yaml

* remove onebranch agent items from macos build

* switch to environment variable

* bump a couple of packages

* fix formatting

* Fix static analysis issue

* update feed url to test restoring everything

* install the AzFeed cred provider

* fix binlog issues
This commit is contained in:
Travis Plunk 2024-08-15 09:06:29 -07:00 committed by GitHub
parent 8c98b5cf60
commit 5919a249a1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 237 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
.pipelines/PowerShell-Coordinated_Packages-Official.yml
View File

@ -70,8 +70,8 @@ variables:
value: ${{ parameters.ReleaseTagVar }}
- name: SKIP_SIGNING
value: ${{ parameters.SKIP_SIGNING }}
- group: 'AzDevOpsArtifacts'
- group: 'mscodehub-feed-read-akv'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- name: ENABLE_MSBUILD_BINLOGS
value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }}
@ -79,6 +79,11 @@ extends:
template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates
parameters:
customTags: 'ES365AIMigrationTooling'
featureFlags:
LinuxHostVersion:
Network: KS3
WindowsHostVersion:
Network: KS3
globalSdl:
disableLegacyManifest: true
# disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates.
@ -121,15 +126,18 @@ extends:
type: windows
variables:
- name: ob_sdl_tsa_configFile
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson'
- name: ob_sdl_codeSignValidation_enabled
value: false
- name: ob_sdl_codeql_compiled_enabled
value: false
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- name: ob_sdl_tsa_configFile
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_signing_setup_enabled
value: false
steps:
- checkout: self
@ -140,6 +148,8 @@ extends:
- pwsh: |
Get-ChildItem Env:
displayName: Capture environment variables
env:
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
- template: /.pipelines/templates/SetVersionVariables.yml@self
parameters:

1
.pipelines/PowerShell-Packages-Official.yml
View File

@ -52,6 +52,7 @@ variables:
value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project
- name: LinuxContainerImage
value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- name: branchCounterKey
value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])]

5
.pipelines/templates/checkAzureContainer.yml
View File

@ -13,9 +13,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false
displayName: Delete blob is exists
pool:

64
.pipelines/templates/insert-nuget-config-azfeed.yml
View File

@ -1,39 +1,53 @@
parameters:
- name: "repoRoot"
default: $(REPOROOT)
- name: "ob_restore_phase"
type: boolean
default: true
steps:
- task: NuGetAuthenticate@1
displayName: Install Azure Artifacts Credential Provider
inputs:
forceReinstallCredentialProvider: true
- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
try {
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
$powerShellPublicPackages = New-NugetPackageSource -Url '$(PowerShellCore_PublicPackages)' -Name 'AzDevOpsFeed'
Write-Verbose -Verbose "Running: Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)'"
Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)'
New-NugetConfigFile -NugetPackageSource $powerShellPublicPackages -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -Destination "${env:NugetConfigDir}"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules'
condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], ''))
catch {
Get-Error
throw
}
displayName: 'Switch to production Azure DevOps feed for all nuget.configs'
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
ob_restore_phase: ${{ parameters.ob_restore_phase }}
- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
$powerShellPublicPackages = New-NugetPackageSource -Url '$(PowerShellCore_PublicPackages)' -Name 'AzDevOpsFeed'
New-NugetConfigFile -NugetPackageSource $powerShellPublicPackages -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -Destination "${env:NugetConfigDir}"
if (-not (Test-Path $configPath))
{
throw "nuget.config is not created"
Get-ChildItem ${{ parameters.repoRoot }}/nuget.config -Recurse | Foreach-Object {
Write-Verbose -Verbose "--- START $($_.fullname) ---"
get-content $_.fullname | Out-String -width 9999 -Stream | write-Verbose -Verbose
Write-Verbose -Verbose "--- END $($_.fullname) ---"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
displayName: 'Capture all nuget.config files'
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
ob_restore_phase: ${{ parameters.ob_restore_phase }}
- pwsh: |
Get-ChildItem -Path env:VSS* | Out-String -width 9999 -Stream | write-Verbose -Verbose
displayName: Capture VSS* Environment
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
ob_restore_phase: ${{ parameters.ob_restore_phase }}

18
.pipelines/templates/mac.yml
View File

@ -20,10 +20,9 @@ jobs:
- group: DotNetPrivateBuildAccess
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
- name: ob_sdl_binskim_enabled
value: true
- name: ob_sdl_credscan_suppressionsfileforartifacts
value: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json
- name: PowerShellRoot
value: $(Build.SourcesDirectory)
steps:
- checkout: self
clean: true
@ -39,19 +38,19 @@ jobs:
# make the current user the owner
sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell"
displayName: 'Create $(Agent.TempDirectory)/PowerShell'
- template: /.pipelines/templates/cloneToOfficialPath.yml@self
parameters:
nativePathRoot: '$(Agent.TempDirectory)'
- pwsh: |
tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap
Import-Module $(PowerShellRoot)/build.psm1 -Force
Start-PSBootstrap -Package
displayName: 'Bootstrap VM'
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(PowerShellRoot)
- pwsh: |
$env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)'
$env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)'
# Add -SkipReleaseChecks as a mitigation to unblock release.
# macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
@ -76,6 +75,7 @@ jobs:
displayName: 'Build'
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
- template: /.pipelines/templates/step/finalize.yml@self
- job: sign_${{ parameters.buildArchitecture }}

23
.pipelines/templates/nupkg.yml
View File

@ -22,7 +22,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- group: 'AzDevOpsArtifacts'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
steps:
@ -89,23 +90,9 @@ jobs:
env:
ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue
- pwsh: |
$repoRoot = "$(PowerShellRoot)"
Write-Verbose -Verbose "repoRoot: $repoRoot"
$configPath = "$repoRoot/nuget.config"
Import-Module "$repoRoot/build.psm1" -Force
New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PowerShellCore_PublicPackages'], ''))
env:
ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(PowerShellRoot)
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'

5
.pipelines/templates/release-MakeBlobPublic.yml
View File

@ -30,9 +30,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false
steps:
- checkout: self

3
.pipelines/templates/release-validate-sdk.yml
View File

@ -16,7 +16,8 @@ jobs:
type: ${{ parameters.jobtype }}
variables:
- group: AzDevOpsArtifacts
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'

31
.pipelines/templates/testartifacts.yml
View File

@ -22,15 +22,22 @@ jobs:
steps:
- checkout: self
clean: true
env:
ob_restore_phase: true
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(Build.SourcesDirectory)
repoRoot: $(Build.SourcesDirectory)/PowerShell
ob_restore_phase: true
- pwsh: |
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Start-PSBootstrap
displayName: Bootstrap
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
ob_restore_phase: true
- pwsh: |
New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
@ -58,6 +65,13 @@ jobs:
BuildTestPackage -runtime win-arm64
displayName: Build test package and upload
retryCountOnTaskFailure: 1
env:
ob_restore_phase: true
- pwsh: |
Write-Host "This doesn't do anything but make the build phase run."
displayName: Dummy build task
- job: build_testartifacts_nonwin
variables:
@ -75,15 +89,22 @@ jobs:
steps:
- checkout: self
clean: true
env:
ob_restore_phase: true
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(Build.SourcesDirectory)
repoRoot: $(Build.SourcesDirectory)/PowerShell
ob_restore_phase: true
- pwsh: |
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Start-PSBootstrap
displayName: Bootstrap
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
ob_restore_phase: true
- pwsh: |
New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
@ -113,3 +134,9 @@ jobs:
BuildTestPackage -runtime linux-musl-x64
displayName: Build test package and upload
retryCountOnTaskFailure: 1
env:
ob_restore_phase: true
- pwsh: |
Write-Host "This doesn't do anything but make the build phase run."
displayName: Dummy build task

5
.pipelines/templates/uploadToAzure.yml
View File

@ -21,9 +21,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false
steps:
- checkout: self

2
.vsts-ci/templates/ci-build.yml
View File

@ -54,7 +54,7 @@ jobs:
displayName: Set Build Name for Non-PR
condition: ne(variables['Build.Reason'], 'PullRequest')
- ${{ if ne(variables['AzDevOpsFeed'], '') }}:
- ${{ if ne(variables['UseAzDevOpsFeed'], '') }}:
- template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
- pwsh: |

107
build.psm1
View File

@ -708,6 +708,8 @@ Fix steps:
if ($config.Count -gt 0) {
$configPublishPath = Join-Path -Path $publishPath -ChildPath "powershell.config.json"
Set-Content -Path $configPublishPath -Value ($config | ConvertTo-Json) -Force -ErrorAction Stop
} else {
Write-Warning "No powershell.config.json generated for $publishPath"
}
# Restore the Pester module
@ -729,25 +731,47 @@ Fix steps:
function Switch-PSNugetConfig {
param(
[Parameter(Mandatory = $true, ParameterSetName = 'user')]
[Parameter(Mandatory = $true, ParameterSetName = 'nouser')]
[ValidateSet('Public', 'Private')]
[string] $Source = 'Public'
[string] $Source,
[Parameter(Mandatory = $true, ParameterSetName = 'user')]
[string] $UserName,
[Parameter(Mandatory = $true, ParameterSetName = 'user')]
[string] $ClearTextPAT
)
Clear-PipelineNugetAuthentication
$extraParams = @()
if ($UserName) {
$extraParams = @{
UserName = $UserName
ClearTextPAT = $ClearTextPAT
}
}
if ( $Source -eq 'Public') {
$dotnetSdk = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet9/nuget/v2'; Name = 'dotnet' }
$gallery = [NugetPackageSource] @{Url = 'https://www.powershellgallery.com/api/v2/'; Name = 'psgallery' }
$nugetorg = [NugetPackageSource] @{Url = 'https://api.nuget.org/v3/index.json'; Name = 'nuget.org' }
New-NugetConfigFile -NugetPackageSource $nugetorg, $dotnetSdk -Destination "$PSScriptRoot/"
New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/src/Modules/"
New-NugetConfigFile -NugetPackageSource $nugetorg, $dotnetSdk -Destination "$PSScriptRoot/" @extraParams
New-NugetConfigFile -NugetPackageSource $gallery -Destination "$PSScriptRoot/src/Modules/" @extraParams
} elseif ( $Source -eq 'Private') {
$powerShellPackages = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/nuget/v3/index.json'; Name = 'powershell' }
$powerShellPackages = [NugetPackageSource] @{Url = 'https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/PowerShell-7-5-preview-test-2/nuget/v3/index.json'; Name = 'powershell' }
New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/"
New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/src/Modules/"
New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/" @extraParams
New-NugetConfigFile -NugetPackageSource $powerShellPackages -Destination "$PSScriptRoot/src/Modules/" @extraParams
} else {
throw "Unknown source: $Source"
}
if ($UserName -or $ClearTextPAT) {
Set-PipelineNugetAuthentication
}
}
function Test-ShouldGenerateExperimentalFeatures
@ -853,7 +877,7 @@ function Restore-PSPackage
$RestoreArguments += "--interactive"
}
if ($env:ENABLE_MSBUILD_BINLOGS) {
if ($env:ENABLE_MSBUILD_BINLOGS -eq 'true') {
$RestoreArguments += '-bl'
}
@ -874,7 +898,7 @@ function Restore-PSPackage
$retryCount++
if($retryCount -ge $maxTries)
{
if ($env:ENABLE_MSBUILD_BINLOGS) {
if ($env:ENABLE_MSBUILD_BINLOGS -eq 'true') {
if ( Test-Path ./msbuild.binlog ) {
if (!(Test-Path $env:OB_OUTPUTDIRECTORY -PathType Container)) {
$null = New-Item -path $env:OB_OUTPUTDIRECTORY -ItemType Directory -Force -Verbose
@ -1231,11 +1255,15 @@ function Publish-PSTestTools {
$tools = @(
@{ Path="${PSScriptRoot}/test/tools/TestAlc"; Output="library" }
@{ Path="${PSScriptRoot}/test/tools/TestExe"; Output="exe" }
@{ Path="${PSScriptRoot}/test/tools/TestService"; Output="exe" }
@{ Path="${PSScriptRoot}/test/tools/UnixSocket"; Output="exe" }
@{ Path="${PSScriptRoot}/test/tools/WebListener"; Output="exe" }
)
# This is a windows service, so it only works on windows
if ($environment.IsWindows) {
$tools += @{ Path = "${PSScriptRoot}/test/tools/TestService"; Output = "exe" }
}
$Options = Get-PSOptions -DefaultToNew
# Publish tools so it can be run by tests
@ -2359,6 +2387,12 @@ function Start-PSBootstrap {
Install-Wix -arm64:$isArm64
}
}
if ($env:TF_BUILD) {
Write-Verbose -Verbose "--- Start - Capturing nuget sources"
dotnet nuget list source --format detailed
Write-Verbose -Verbose "--- End - Capturing nuget sources"
}
} finally {
Pop-Location
}
@ -3466,6 +3500,7 @@ function New-NugetPackageSource {
return [NugetPackageSource] @{Url = $Url; Name = $Name }
}
$script:NuGetEndpointCredentials = [System.Collections.Generic.Dictionary[String,System.Object]]::new()
function New-NugetConfigFile {
param(
[Parameter(Mandatory = $true, ParameterSetName ='user')]
@ -3509,23 +3544,53 @@ function New-NugetConfigFile {
</configuration>
'@
$content = $nugetConfigHeaderTemplate
$feedNamePostfix = ''
if ($UserName) {
$feedNamePostfix += '-' + $UserName.Replace('@', '-').Replace('.', '-')
}
[NugetPackageSource]$source = $null
$newLine = [Environment]::NewLine
foreach ($source in $NugetPackageSource) {
$content += $nugetPackageSourceTemplate.Replace('[FEED]', $source.Url).Replace('[FEEDNAME]', $source.Name)
$content += $newLine + $nugetPackageSourceTemplate.Replace('[FEED]', $source.Url).Replace('[FEEDNAME]', $source.Name + $feedNamePostfix)
}
$content += $nugetPackageSourceFooterTemplate
$content += $newLine + $nugetPackageSourceFooterTemplate
if ($UserName -or $ClearTextPAT) {
$content += $nugetCredentialsTemplate.Replace('[USERNAME]', $UserName).Replace('[PASSWORD]', $ClearTextPAT)
foreach ($source in $NugetPackageSource) {
if (!$script:NuGetEndpointCredentials.ContainsKey($source.Url)) {
$script:NuGetEndpointCredentials.Add($source.Url, @{
endpoint = $source.Url
username = $UserName
password = $ClearTextPAT
})
}
}
}
$content += $nugetConfigFooterTemplate
$content += $newLine + $nugetConfigFooterTemplate
Set-Content -Path (Join-Path $Destination 'nuget.config') -Value $content -Force
}
function Clear-PipelineNugetAuthentication {
$script:NuGetEndpointCredentials.Clear()
}
function Set-PipelineNugetAuthentication {
$endpointcredentials = @()
foreach ($key in $script:NuGetEndpointCredentials.Keys) {
$endpointcredentials += $script:NuGetEndpointCredentials[$key]
}
$json = @{
endpointCredentials = $endpointcredentials
} | convertto-json -Compress
Set-PipelineVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $json
}
function Set-CorrectLocale
{
if (-not $IsLinux)
@ -3659,3 +3724,19 @@ function Update-DotNetSdkVersion {
$dotnetRuntimeMeta.sdk.sdkImageVersion = $version
$dotnetRuntimeMeta | ConvertTo-Json | Out-File $dotnetRuntimeMetaPath
}
function Set-PipelineVariable {
param(
[parameter(Mandatory)]
[string] $Name,
[parameter(Mandatory)]
[string] $Value
)
$vstsCommandString = "vso[task.setvariable variable=$Name]$Value"
Write-Verbose -Verbose -Message ("sending " + $vstsCommandString)
Write-Host "##$vstsCommandString"
# also set in the current session
Set-Item -Path "env:$Name" -Value $Value
}

2
nuget.config
View File

@ -2,7 +2,7 @@
<configuration>
<packageSources>
<clear />
<add key="powershell" value="https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/nuget/v3/index.json" />
<add key="powershell" value="https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell-dotnet-9/nuget/v3/index.json" />
</packageSources>
<disabledPackageSources>
<clear />

2
src/Microsoft.PowerShell.Commands.Utility/Microsoft.PowerShell.Commands.Utility.csproj
View File

@ -35,7 +35,7 @@
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.9.2" />
<PackageReference Include="System.Threading.AccessControl" Version="9.0.0-preview.6.24327.7" />
<PackageReference Include="System.Drawing.Common" Version="9.0.0-preview.6.24327.6" />
<PackageReference Include="JsonSchema.Net" Version="7.0.1" />
<PackageReference Include="JsonSchema.Net" Version="7.0.4" />
</ItemGroup>
</Project>

2
src/Microsoft.PowerShell.SDK/Microsoft.PowerShell.SDK.csproj
View File

@ -17,7 +17,7 @@
<ItemGroup>
<!-- This section is to force the version of non-direct dependencies -->
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.ObjectPool" Version="8.0.4" />
<PackageReference Include="Microsoft.Extensions.ObjectPool" Version="8.0.8" />
<!-- the following package(s) are from https://github.com/dotnet/fxdac -->
<PackageReference Include="System.Data.SqlClient" Version="4.8.6" />
<!-- the following package(s) are from https://github.com/dotnet/corefx -->

2
src/Modules/nuget.config
View File

@ -2,7 +2,7 @@
<configuration>
<packageSources>
<clear />
<add key="powershell" value="https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell/nuget/v3/index.json" />
<add key="powershell" value="https://pkgs.dev.azure.com/powershell/PowerShell/_packaging/powershell-dotnet-9/nuget/v3/index.json" />
</packageSources>
<disabledPackageSources>
<clear />

4
test/tools/TestService/TestService.csproj
View File

@ -10,10 +10,12 @@
<TieredCompilation>true</TieredCompilation>
<TieredCompilationQuickJit>true</TieredCompilationQuickJit>
<RuntimeIdentifiers>win-x86;win-x64</RuntimeIdentifiers>
<TargetPlatformIdentifier>Windows</TargetPlatformIdentifier>
<TargetPlatformVersion>8.0</TargetPlatformVersion>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Windows.Compatibility" Version="8.0.0" />
<PackageReference Include="Microsoft.Windows.Compatibility" Version="9.0.0-preview.6.24327.6" />
<PackageReference Include="System.Formats.Asn1" Version="9.0.0-preview.6.24327.7" />
<PackageReference Include="System.Data.SqlClient" Version="4.8.6" />
</ItemGroup>

9
test/tools/WebListener/WebListener.csproj
View File

@ -7,12 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.0" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="9.0.0-preview.6.24328.4" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="9.0.0-preview.6.24327.7" />
</ItemGroup>
<ItemGroup>
<DotNetCliToolReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Tools" Version="2.1.0-*" />
</ItemGroup>
</Project>

6
tools/releaseBuild/azureDevOps/AzArtifactFeed/PSGalleryToAzArtifacts.yml
View File

@ -19,12 +19,6 @@ steps:
displayName: Update PSGet and PackageManagement
condition: succeededOrFailed()
- pwsh: |
Import-Module -Force "$(Build.SourcesDirectory)/tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1"
SyncGalleryToAzArtifacts -AzDevOpsFeedUserName $(AzDevOpsFeedUserName) -AzDevOpsPAT $(AzDevOpsFeedPAT2) -Destination $(Build.ArtifactStagingDirectory)
displayName: Download packages from PSGallery that need to be updated
condition: succeededOrFailed()
- pwsh: |
Write-Verbose -Verbose "Packages to upload"
if(Test-Path $(Build.ArtifactStagingDirectory)) { Get-ChildItem "$(Build.ArtifactStagingDirectory)/*.nupkg" | ForEach-Object { $_.FullName }}

186
tools/releaseBuild/azureDevOps/AzArtifactFeed/SyncGalleryToAzArtifacts.psm1
View File

@ -1,186 +0,0 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
<#
.SYNOPSIS
Downloads to packages from PowerShell Gallery which are missing from the Azure DevOps Artifacts feed.
.PARAMETER AzureDevOpsPAT
PAT for the username used for authenticating to the Azure DevOps Artifacts feed.
.PARAMETER Destination
Path to the folder where the packages should be stored for uploading to Azure DevOps Artifacts feed.
#>
function SyncGalleryToAzArtifacts {
param(
[Parameter(Mandatory = $true)] [string] $AzDevOpsFeedUserName,
[Parameter(Mandatory = $true)] [string] $AzDevOpsPAT,
[Parameter(Mandatory = $true)] [string] $Destination
)
$csproj = [xml] (Get-Content 'src/Modules/PSGalleryModules.csproj')
$packages = @($csproj.Project.ItemGroup.PackageReference | ForEach-Object { [ordered] @{Name = $_.Include; Version = $_.Version }})
$galleryPackages = @()
$azArtifactsPackages = @()
$modulesToUpdate = @()
$galleryUrl = 'https://www.powershellgallery.com/api/v2/'
$azArtifactsUrl = 'https://mscodehub.pkgs.visualstudio.com/_packaging/pscore-release/nuget/v2'
$azDevOpsCreds = [pscredential]::new($AzDevOpsFeedUserName, (ConvertTo-SecureString -String $AzDevOpsPAT -AsPlainText -Force))
foreach ($package in $packages) {
try {
# Get module from gallery
$foundPackageOnGallery = Find-Package -ProviderName NuGet -Source $galleryUrl -AllVersions -Name $package.Name -Force -AllowPreReleaseVersion | SortPackage | Select-Object -First 1
Write-Verbose -Verbose "Found module $($package.Name) - $($foundPackageOnGallery.Version) in gallery"
$galleryPackages += $foundPackageOnGallery
} catch {
if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') {
# Log and ignore failure is required version is not found on gallery.
Write-Warning "Module not found on gallery $($package.Name) - $($package.Version)"
}
else {
Write-Error $_
}
}
try {
# Get module from Az Artifacts
# There seems to be a bug in the feed with RequiredVersion matching. Adding workaround with post filtering.
# Issue: https://github.com/OneGet/oneget/issues/397
$foundPackageOnAz = Find-Package -ProviderName NuGet -Source $azArtifactsUrl -AllVersions -Name $package.Name -Force -Credential $azDevOpsCreds -AllowPreReleaseVersion | SortPackage | Select-Object -First 1
Write-Verbose -Verbose "Found module $($package.Name) - $($foundPackageOnAz.Version) in azArtifacts"
$azArtifactsPackages += $foundPackageOnAz
} catch {
if ($_.FullyQualifiedErrorId -eq 'NoMatchFoundForCriteria,Microsoft.PowerShell.PackageManagement.Cmdlets.FindPackage') {
# Log and add the module to update list.
Write-Verbose -Verbose "Az Artifacts Module needs update to - $($package.Name) - $($package.Version)"
$modulesToUpdate += $package
}
else {
Write-Error $_
}
}
# Check if Az package version is less that gallery version
$pkgOnAzVersion = [semver]::new($foundPackageOnAz.Version)
$pkgOnGalleryVersion = [semver]::new($foundPackageOnGallery.Version)
if ($pkgOnAzVersion -lt $pkgOnGalleryVersion) {
Write-Verbose -Verbose "Module needs to be updated $($package.Name) - $($foundPackageOnGallery.Version)"
$modulesToUpdate += $foundPackageOnGallery
} elseif ($pkgOnGalleryVersion -lt $pkgOnAzVersion) {
Write-Warning "Newer version found on Az Artifacts - $($foundPackageOnAz.Name) - $($foundPackageOnAz.Version)"
} else {
Write-Verbose -Verbose "Module is in sync - $($package.Name)"
}
}
"`nGallery Packages:"
$galleryPackages
"`nAz Artifacts Packages:`n"
$azArtifactsPackages
"`nModules to update:`n"
$modulesToUpdate
foreach ($package in $modulesToUpdate) {
Write-Verbose -Verbose "Saving package $($package.Name) - $($package.Version)"
Save-Package -Provider NuGet -Source $galleryUrl -Name $package.Name -RequiredVersion $package.Version -Path $Destination
}
if ($modulesToUpdate.Length -gt 0)
{
# Remove dependent packages downloaded by Save-Package if there are already present in AzArtifacts feed.
try {
$null = Register-PackageSource -Name local -Location $Destination -ProviderName NuGet -Force
$packageNamesToKeep = @()
$savedPackages = Find-Package -Source local -AllVersions -AllowPreReleaseVersion
Write-Verbose -Verbose "Saved packages:"
$savedPackages | Out-String | Write-Verbose -Verbose
foreach($package in $savedPackages) {
$pkgVersion = NormalizeVersion -version $package.Version
$foundMatch = $azArtifactsPackages | Where-Object { $_.Name -eq $package.Name -and (NormalizeVersion -version $_.Version) -eq $pkgVersion }
if(-not $foundMatch) {
Write-Verbose "Keeping package $($package.PackageFileName)" -Verbose
$packageNamesToKeep += "{0}*.nupkg" -f $package.Name
}
}
if ($packageNamesToKeep.Length -gt 0) {
## Removing only if we do have some packages to keep,
## otherwise the '$Destination' folder will be removed.
Remove-Item -Path $Destination -Exclude $packageNamesToKeep -Recurse -Force -Verbose
}
Write-Verbose -Verbose "Packages kept for upload"
Get-ChildItem $Destination | Out-String | Write-Verbose -Verbose
}
finally {
Unregister-PackageSource -Name local -Force -ErrorAction SilentlyContinue
}
}
}
Function SortPackage {
param(
[Parameter(ValueFromPipeline = $true)]
[Microsoft.PackageManagement.Packaging.SoftwareIdentity[]]
$packages
)
Begin {
$allPackages = @()
}
Process {
$allPackages += $packages
}
End {
$versions = $allPackages.Version |
ForEach-Object { ($_ -split '-')[0] } |
Select-Object -Unique |
Sort-Object -Descending -Property Version
foreach ($version in $versions) {
$exactMatch = $allPackages | Where-Object {
Write-Verbose "testing $($_.version) -eq $version"
$_.version -eq $version
}
if ($exactMatch) {
Write-Output $exactMatch
}
$allPackages | Where-Object {
$_.version -like "${version}-*"
} | Sort-Object -Descending -Property Version | Write-Output
}
}
}
function NormalizeVersion {
param ([string] $version)
$sVer = if ($version -match "(\d+.\d+.\d+).0") {
$Matches[1]
} elseif ($version -match "^\d+.\d+$") {
# Two digit versions are stored as three digit versions
"$version.0"
} else {
$version
}
$sVer
}
Export-ModuleMember -Function 'SyncGalleryToAzArtifacts', 'SortPackage'

8
tools/releaseBuild/azureDevOps/releasePipeline.yml
View File

@ -356,7 +356,8 @@ stages:
variables:
- group: 'Azure Blob variable group'
- group: 'AzDevOpsArtifacts'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: ReleasePipelineSecrets
steps:
- template: templates/release-CreateGitHubDraft.yml
@ -398,9 +399,10 @@ stages:
- ImageOverride -equals PSMMSUbuntu20.04-Secure
variables:
- group: 'AzDevOpsArtifacts'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: 'packages.microsoft.com'
- group: 'mscodehub-feed-read-akv'
- group: 'mscodehub-code-read-akv'
steps:
- template: templates/release-PublishPackageMsftCom.yml
parameters:

3
tools/releaseBuild/azureDevOps/templates/compliance/apiscan.yml
View File

@ -19,7 +19,8 @@ jobs:
- group: DotNetPrivateBuildAccess
- group: Azure Blob variable group
- group: ReleasePipelineSecrets
- group: AzDevOpsArtifacts
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
pool:
name: PowerShell1ES

40
tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml
View File

@ -1,38 +1,8 @@
parameters:
- name: "repoRoot"
default: $(REPOROOT)
- name: "repoRoot"
default: $(REPOROOT)
steps:
- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
New-NugetConfigFile -NugetFeedUrl $(AzDevOpsFeed) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}"
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(REPOROOT)
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules'
condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules
- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination "${env:NugetConfigDir}"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}
- task: nuget-security-analysis@0
displayName: 'Run Secure Supply Chain analysis'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''), ne(variables['AzDevOpsFeed'], ''))

2
tools/releaseBuild/azureDevOps/templates/mac.yml
View File

@ -50,7 +50,7 @@ jobs:
repoRoot: $(PowerShellRoot)
- pwsh: |
$env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)'
$env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)'
# Add -SkipReleaseChecks as a mitigation to unblock release.
# macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
$(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -Symbols -location $(PowerShellRoot) -Build -ArtifactName macosBinResults -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks

15
tools/releaseBuild/azureDevOps/templates/nuget-pkg-sbom.yml
View File

@ -20,18 +20,9 @@ parameters:
steps:
- pwsh: |
$configPath = "$(REPOROOT)/nuget.config"
Import-Module '$(REPOROOT)/build.psm1' -Force
New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination "$(REPOROOT)"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(REPOROOT)
- pwsh: |
Import-Module "$env:REPOROOT/build.psm1" -Force

2
tools/releaseBuild/azureDevOps/templates/release-PublishPackageMsftCom.yml
View File

@ -19,7 +19,7 @@ steps:
"--verbose",
"--branch",
"$branch",
"https://$(mscodehubPackageReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools",
"https://$(mscodehubCodeReadPat)@mscodehub.visualstudio.com/PowerShellCore/_git/Internal-PowerShellTeam-Tools",
'$(Pipeline.Workspace)/tools'
$gitArgs | Write-Verbose -Verbose
git $gitArgs

8
tools/releaseBuild/azureDevOps/templates/release-SDKTests.yml
View File

@ -10,7 +10,8 @@ jobs:
# testing
vmImage: ${{ parameters.imageName }}
variables:
- group: AzDevOpsArtifacts
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
steps:
- checkout: self
@ -39,9 +40,12 @@ jobs:
artifact: metadata
path: '$(Pipeline.Workspace)/releasePipeline/metadata'
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(Build.SourcesDirectory)
- pwsh: |
Import-Module "$(Build.SourcesDirectory)/build.psm1" -Force
New-NugetConfigFile -NugetFeedUrl $(PSInternalNugetFeed) -UserName $(PSInternalNugetFeedUserName) -ClearTextPAT $(PSInternalNugetFeedPAT) -FeedName AzDevOpsFeed -Destination '$(Build.SourcesDirectory)/test/hosting'
Write-Verbose -Verbose "Capture hosting folder files"
Get-ChildItem '$(Build.SourcesDirectory)/test/hosting'