Start rotating through all images (#17315)

* Use day of week for fallback seed

* Use new properties to select CI image

* move container selection code into

* use parameter

* fix azDO variable refernce

* move container name left in non-PR build name

* Use get-command to detect package manager

* update dependencies in build.psm1

* Add RHEL to WSMan support function

* Update build.psm1

* Update tools/ci.psm1

Co-authored-by: James Truher [MSFT] <jimtru@microsoft.com>

Co-authored-by: James Truher [MSFT] <jimtru@microsoft.com>

.vsts-ci/linux.yml

@@ -1,3 +1,12 @@
+parameters:
+  - name: ContainerPattern
+    displayName: |
+      Pattern to match JobName of the container.
+      Update this to force a container.
+      `.` will match everything
+    type: string
+    default: .
+
 name: PR-$(System.PullRequest.PullRequestNumber)-$(Date:yyyyMMdd)$(Rev:.rr)
 trigger:
   # Batch merge builds together while a merge build is running
@@ -116,54 +125,11 @@ stages:
       clean: true
 
     - pwsh: |
-        $buildId = $env:BUILD_BUILDID
-        Write-Verbose "Fall back seed: $buildId" -Verbose
-        # For PRs set the seed to the PR number so that the image is always the same
-        $seed = $env:SYSTEM_PULLREQUEST_PULLREQUESTID
-        if(!$seed) {
-          # for non-PRs use the integer identifier of the build as the seed.
-          $seed = $buildId
-        }
-        Write-Verbose "Seed: $seed" -Verbose
-
-        # Get the latest image matrix JSON for preview
-        $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json
-
-        # Filter out containers that won't run on AzDevOps Linux host as an agent
-        # Windows, nano server, alpine, and any ARM image
-        $linuxImages = $matrix.preview |
-          Where-Object {$_.osversion -notmatch 'windows|nano|alpine|arm'} |
-          Select-Object JobName, Taglist |
-          Sort-Object -property JobName
-
-        # Force mariner for now
-        $marinerImage = $linuxImages | where-object {$_.JobName -like 'mariner*'}
-
-        # Use the selected seed to pick a container
-        $index = Get-Random -Minimum 0 -Maximum $marinerImage.Count -SetSeed $seed
-        $selectedImage = $marinerImage[$index]
-
-        # Filter to the first test-deps compatible tag
-        $tag = $selectedImage.Taglist -split ';' | Where-Object {$_ -match 'preview-\D+'} | Select-Object -First 1
-
-        # Calculate the container name
-        $containerName = "mcr.microsoft.com/powershell/test-deps:$tag"
-
-        # Set variables for later jobs to use
+        # Initialize container test stage
         Import-Module ./PowerShell/tools/ci.psm1
-        Set-BuildVariable -Name containerName -Value $containerName -IsOutput
-        Set-BuildVariable -Name containerBuildName -Value $selectedImage.JobName -IsOutput
-
-        if($env:BUILD_REASON -eq 'PullRequest') {
-          Write-Host "##vso[build.updatebuildnumber]PR-$(System.PullRequest.PullRequestNumber)-$($selectedImage.JobName)-$((get-date).ToString("yyyyMMddhhmmss"))"
-        } else {
-          Write-Host "##vso[build.updatebuildnumber]${env:BUILD_SOURCEBRANCHNAME}-${env:BUILD_SOURCEVERSION}-$($selectedImage.JobName)-$((get-date).ToString("yyyyMMddhhmmss"))"
-
-          # Cannot do this for a PR
-          Write-Host "##vso[build.addbuildtag]$($selectedImage.JobName)"
-        }
+        Invoke-InitializeContainerStage -ContainerPattern '${{ parameters.ContainerPattern }}'
       name: getContainerTask
-      displayName: Get Container
+      displayName: Initialize Container Stage
       continueOnError: true
 
   - template: templates/test/nix-container-test.yml

build.psm1

@@ -1959,9 +1959,9 @@ function Install-Dotnet {
 }
 
 function Get-RedHatPackageManager {
-    if ($environment.IsCentOS) {
+    if ($environment.IsCentOS -or (Get-Command -Name yum -CommandType Application -ErrorAction SilentlyContinue)) {
         "yum install -y -q"
-    } elseif ($environment.IsFedora) {
+    } elseif ($environment.IsFedora -or (Get-Command -Name dnf -CommandType Application -ErrorAction SilentlyContinue)) {
         "dnf install -y -q"
     } else {
         throw "Error determining package manager for this distribution."
@@ -2044,7 +2044,6 @@ function Start-PSBootstrap {
                 $Deps += "curl", "wget"
 
                 # .NET Core required runtime libraries
-                $Deps += "libunwind8"
                 if ($environment.IsUbuntu16) { $Deps += "libicu55" }
                 elseif ($environment.IsUbuntu18) { $Deps += "libicu60"}
 
@@ -2070,7 +2069,7 @@ function Start-PSBootstrap {
                 $Deps += "which", "curl", "wget"
 
                 # .NET Core required runtime libraries
-                $Deps += "libicu", "libunwind"
+                $Deps += "libicu", "openssl-libs"
 
                 # Packaging tools
                 if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel', "gcc-c++" }

test/tools/Modules/HelpersCommon/HelpersCommon.psm1

@@ -391,10 +391,11 @@ function Get-WsManSupport {
     $platformInfo = Get-PlatformInfo
     if (
         ($platformInfo.Platform -match "alpine|raspbian") -or
-        ($platformInfo.Platform -eq "debian" -and $platformInfo.Version -ne '9') -or 
+        ($platformInfo.Platform -eq "debian" -and $platformInfo.Version -ne '9') -or
         ($platformInfo.Platform -eq 'centos' -and $platformInfo.Version -eq '8') -or
         ($platformInfo.Platform -eq 'ubuntu' -and $platformInfo.Version -eq '20.04') -or
         ($platformInfo.Platform -eq 'mariner') -or
+        ($platformInfo.Platform -eq 'rhel') -or
         ($IsMacOS)
     ) {
         return $false

tools/ci.psm1

@@ -811,3 +811,55 @@ function New-LinuxPackage
         Copy-Item $armPackage -Destination "${env:BUILD_ARTIFACTSTAGINGDIRECTORY}" -Force
     }
 }
+
+function Invoke-InitializeContainerStage {
+    param(
+        [string]
+        $ContainerPattern = '.'
+    )
+
+    Write-Verbose "Invoking InitializeContainerStage with ContainerPattern: ${ContainerPattern}" -Verbose
+
+    $fallbackSeed = (get-date).DayOfYear
+    Write-Verbose "Fall back seed: $fallbackSeed" -Verbose
+
+    # For PRs set the seed to the PR number so that the image is always the same
+    $seed = $env:SYSTEM_PULLREQUEST_PULLREQUESTID
+    if(!$seed) {
+      # for non-PRs use the integer identifier of the build as the seed.
+      $seed = $fallbackSeed
+    }
+
+    Write-Verbose "Seed: $seed" -Verbose
+
+    # Get the latest image matrix JSON for preview
+    $matrix = ./PowerShell-Docker/build.ps1 -GenerateMatrixJson -FullJson -Channel preview | ConvertFrom-Json
+
+    # Chose images that are validated or validating, Linux and can be used in CI.
+    $linuxImages = $matrix.preview |
+      Where-Object {$_.IsLinux -and $_.UseInCi -and $_.DistributionState -match 'Validat.*' -and $_.JobName -match $ContainerPattern} |
+      Select-Object JobName, Taglist |
+      Sort-Object -property JobName
+
+    # Use the selected seed to pick a container
+    $index = Get-Random -Minimum 0 -Maximum $linuxImages.Count -SetSeed $seed
+    $selectedImage = $linuxImages[$index]
+
+    # Filter to the first test-deps compatible tag
+    $tag = $selectedImage.Taglist -split ';' | Where-Object {$_ -match 'preview-\D+'} | Select-Object -First 1
+
+    # Calculate the container name
+    $containerName = "mcr.microsoft.com/powershell/test-deps:$tag"
+
+    Set-BuildVariable -Name containerName -Value $containerName -IsOutput
+    Set-BuildVariable -Name containerBuildName -Value $selectedImage.JobName -IsOutput
+
+    if($env:BUILD_REASON -eq 'PullRequest') {
+      Write-Host "##vso[build.updatebuildnumber]PR-${env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER}-$($selectedImage.JobName)-$((get-date).ToString("yyyyMMddhhmmss"))"
+    } else {
+      Write-Host "##vso[build.updatebuildnumber]${env:BUILD_SOURCEBRANCHNAME}-$($selectedImage.JobName)-${env:BUILD_SOURCEVERSION}-$((get-date).ToString("yyyyMMddhhmmss"))"
+
+      # Cannot do this for a PR
+      Write-Host "##vso[build.addbuildtag]$($selectedImage.JobName)"
+    }
+}