Base work for adding mariner amd64 package (#17417)

Backport #17057
2024-11-23 17:53:58 +08:00 · 2022-05-24 13:17:15 -07:00 · 2022-05-24 13:17:15 -07:00 · 3cfa8eb258
commit 3cfa8eb258
parent 8af2f5c611
7 changed files with 196 additions and 85 deletions
--- a/tools/packaging/packaging.psm1
+++ b/tools/packaging/packaging.psm1
@ -7,7 +7,11 @@ $RepoRoot = (Resolve-Path -Path "$PSScriptRoot/../..").Path
 $packagingStrings = Import-PowerShellDataFile "$PSScriptRoot\packaging.strings.psd1"
 Import-Module "$PSScriptRoot\..\Xml" -ErrorAction Stop -Force
 $DebianDistributions = @("deb")
-$RedhatDistributions = @("rh")
+$RedhatDistributions = @("rh","cm1")
+$AllDistributions = @()
+$AllDistributions += $DebianDistributions
+$AllDistributions += $RedhatDistributions
+$AllDistributions += 'macOs'
 $script:netCoreRuntime = 'net6.0'
 $script:iconFileName = "Powershell_black_64.png"
 $script:iconPath = Join-Path -path $PSScriptRoot -ChildPath "../../assets/$iconFileName" -Resolve
@ -500,6 +504,7 @@ function Start-PSPackage {
                foreach ($Distro in $Script:RedhatDistributions) {
                    $Arguments["Distribution"] = $Distro
                    if ($PSCmdlet.ShouldProcess("Create RPM Package for $Distro")) {
+                        Write-Verbose -Verbose "Creating RPM Package for $Distro"
                        New-UnixPackage @Arguments
                    }
                }
@ -981,7 +986,7 @@ function New-UnixPackage {
        # Build package
        try {
            if ($PSCmdlet.ShouldProcess("Create $type package")) {
-                Write-Log "Creating package with fpm..."
+                Write-Log "Creating package with fpm $Arguments..."
                try {
                    $Output = Start-NativeExecution { fpm $Arguments }
                }
@ -1237,7 +1242,8 @@ function Get-FpmArguments
        "-t", $Type,
        "-s", "dir"
    )
-    if ($Distribution -eq 'rh') {
+
+    if ($Distribution -in $script:RedHatDistributions) {
        $Arguments += @("--rpm-digest", "sha256")
        $Arguments += @("--rpm-dist", $Distribution)
        $Arguments += @("--rpm-os", "linux")
@ -1283,13 +1289,29 @@ function Get-FpmArguments

 function Get-PackageDependencies
 {
-    param(
-        [String]
-        [ValidateSet('rh','deb','macOS')]
-        $Distribution
-    )
+    [CmdletBinding()]
+    param()
+    DynamicParam {
+        # Add a dynamic parameter '-Distribution' when the specified package type is 'deb'.
+        # The '-Distribution' parameter can be used to indicate which Debian distro this pacakge is targeting.
+        $ParameterAttr = New-Object "System.Management.Automation.ParameterAttribute"
+        $ParameterAttr.Mandatory = $true
+        $ValidateSetAttr = New-Object "System.Management.Automation.ValidateSetAttribute" -ArgumentList $Script:AllDistributions
+        $Attributes = New-Object "System.Collections.ObjectModel.Collection``1[System.Attribute]"
+        $Attributes.Add($ParameterAttr) > $null
+        $Attributes.Add($ValidateSetAttr) > $null
+
+        $Parameter = New-Object "System.Management.Automation.RuntimeDefinedParameter" -ArgumentList ("Distribution", [string], $Attributes)
+        $Dict = New-Object "System.Management.Automation.RuntimeDefinedParameterDictionary"
+        $Dict.Add("Distribution", $Parameter) > $null
+        return $Dict
+    }

    End {
+        if ($PSBoundParameters.ContainsKey('Distribution')) {
+            $Distribution = $PSBoundParameters['Distribution']
+        }
+
        # These should match those in the Dockerfiles, but exclude tools like Git, which, and curl
        $Dependencies = @()
        if ($Distribution -eq 'deb') {
@ -1308,6 +1330,22 @@ function Get-PackageDependencies
                "openssl-libs",
                "libicu"
            )
+        } elseif ($Distribution -eq 'cm1') {
+            # Taken from the list here:
+            # https://github.com/dotnet/dotnet-docker/blob/d451d6e9427f58c8508f1297c862663a27eb609f/src/runtime-deps/6.0/cbl-mariner1.0/amd64/Dockerfile#L6
+            $Dependencies = @(
+                "glibc"
+                "libgcc"
+                "krb5"
+                "libstdc++"
+                "zlib"
+                "icu"
+                "openssl-libs"
+            )
+        } elseif ($Distribution -eq 'macOS') {
+            # do nothing
+        } else {
+            throw "Unknown distribution $Distribution"
        }

        return $Dependencies
@ -1358,7 +1396,7 @@ function New-AfterScripts

    Write-Verbose -Message "AfterScript Distribution: $Distribution" -Verbose

-    if ($Distribution -eq 'rh') {
+    if ($Distribution -in $script:RedHatDistributions) {
        $AfterInstallScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName()))
        $AfterRemoveScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName()))
        $packagingStrings.RedHatAfterInstallScript -f "$Link", $Destination  | Out-File -FilePath $AfterInstallScript -Encoding ascii
@ -4197,7 +4235,7 @@ function Invoke-AzDevOpsLinuxPackageCreation {
    }

    try {
-        Write-Verbose "Packaging '$BuildType'-LTS:$LTS for $ReleaseTag ..." -Verbose
+        Write-Verbose "Packaging '$BuildType'; LTS:$LTS for $ReleaseTag ..." -Verbose

        Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}-meta\psoptions.json"

--- a/tools/packaging/releaseTests/sbom.tests.ps1
+++ b/tools/packaging/releaseTests/sbom.tests.ps1
@ -8,57 +8,102 @@ Describe "Verify SBOMs" {
        $missingFromPackageCases = @()
        $missingFromManifestCases = @()
        Write-Verbose "${env:PACKAGE_FOLDER}" -Verbose
-        Get-ChildItem $env:PACKAGE_FOLDER -Filter *.zip | ForEach-Object {
-            Write-Verbose "Found $($_.Name)..." -Verbose
-            $testCases += @{
-                FilePath = $_.FullName
-                Name = $_.Name
+        Get-ChildItem $env:PACKAGE_FOLDER -Filter *.zip |
+            Where-Object { $_.Name -notlike 'powershell-symbols*' } |
+            ForEach-Object {
+                Write-Verbose "Found $($_.Name)..." -Verbose
+                $testCases += @{
+                    FilePath = $_.FullName
+                    Name = $_.Name
+                    Extension = $_.Extension
+            }
+        }
+
+        if ($IsLinux) {
+            Get-ChildItem $env:PACKAGE_FOLDER -Filter *.rpm | ForEach-Object {
+                Write-Verbose "Found $($_.Name)..." -Verbose
+                $testCases += @{
+                    FilePath  = $_.FullName
+                    Name      = $_.Name
+                    Extension = $_.Extension
+                }
            }
        }

        foreach($case in $testCases) {
+            $skip = $null
            $name = $case.Name
            Write-Verbose "Testing $name..." -Verbose
            $extractedPath = Join-Path Testdrive:\ -ChildPath ([System.io.path]::GetRandomFileName())
            $null = New-Item -Path $extractedPath -ItemType Directory -Force
-            Expand-Archive -Path $case.FilePath -DestinationPath $extractedPath
-            $manifestPath = Join-Path $extractedPath -ChildPath '/_manifest/spdx_2.2/manifest.spdx.json'
+            $resolvedPath = (Resolve-Path -Path $extractedPath).ProviderPath
+            switch ($case.Extension) {
+                '.zip' {
+                    Expand-Archive -Path $case.FilePath -DestinationPath $extractedPath
+                    $manifestPath = Join-Path $extractedPath -ChildPath '/_manifest/spdx_2.2/manifest.spdx.json'
+                }
+                '.rpm' {
+                    Push-Location $resolvedPath
+                    try {
+                        if (Get-Command -Name rpm2cpio -ErrorAction SilentlyContinue) {
+                            rpm2cpio $case.FilePath | cpio -i --make-directories 2>cpio.log
+                            $manifestPath = Get-ChildItem -Path manifest.spdx.json -Recurse | Select-Object -First 1 -ExpandProperty FullName
+                            $extractedPath = Get-ChildItem -Path system.management.automation.dll -Recurse  | Select-Object -First 1 -ExpandProperty DirectoryName
+                        } else {
+                            $skip = "rpm2cpio not found"
+                        }
+                    } finally {
+                        Pop-Location
+                    }
+                }
+                Default {
+                    throw "Unkown extension $($case.Extension)"
+                }
+            }
+
            It "$name has a BOM" {
+                if ($skip) {
+                    Set-ItResult -Pending -Because $skip
+                }
                $manifestPath | Should -Exist
            }
-            Test-PackageManifest -PackagePath $extractedPath | ForEach-Object {
-                $status = $_.Status
-                $expectedHash = $_.ExpectedHash
-                $actual = $_.ActualHash
-                $file = $_.File

-                switch($status) {
-                    # cover match and mismatch
-                    default {
-                        $matchCases += @{
-                            Name = $name
-                            File = $file
-                            ActualHash = $actual
-                            ExpectedHash = $ExpectedHash
-                            Status = $status
+            # RPM hashes are broken, skip that
+            if ($case.Extension -in '.zip') {
+                Test-PackageManifest -PackagePath $extractedPath | ForEach-Object {
+                    $status = $_.Status
+                    $expectedHash = $_.ExpectedHash
+                    $actual = $_.ActualHash
+                    $file = $_.File
+
+                    switch ($status) {
+                        # cover match and mismatch
+                        default {
+                            $matchCases += @{
+                                Name         = $name
+                                File         = $file
+                                ActualHash   = $actual
+                                ExpectedHash = $ExpectedHash
+                                Status       = $status
+                            }
                        }
-                    }
-                    "MissingFromPackage" {
-                        $missingFromPackageCases = @{
-                            Name = $name
-                            File = $file
-                            ActualHash = $actual
-                            ExpectedHash = $ExpectedHash
-                            Status = $status
+                        "MissingFromPackage" {
+                            $missingFromPackageCases = @{
+                                Name         = $name
+                                File         = $file
+                                ActualHash   = $actual
+                                ExpectedHash = $ExpectedHash
+                                Status       = $status
+                            }
                        }
-                    }
-                    "MissingFromManifest" {
-                        $missingFromManifestCases = @{
-                            Name = $name
-                            File = $file
-                            ActualHash = $actual
-                            ExpectedHash = $ExpectedHash
-                            Status = $status
+                        "MissingFromManifest" {
+                            $missingFromManifestCases = @{
+                                Name         = $name
+                                File         = $file
+                                ActualHash   = $actual
+                                ExpectedHash = $ExpectedHash
+                                Status       = $status
+                            }
                        }
                    }
                }
@ -66,11 +111,7 @@ Describe "Verify SBOMs" {
        }
    }

-    Context "Zip files" {
-        BeforeAll {
-            Write-Verbose "In Context BeforeAll" -Verbose
-        }
-
+    Context "Package files" {
        It "<name> should have <file> with matching hash" -TestCases $matchCases {
            param(
                $Name,
--- a/tools/releaseBuild/azureDevOps/releasePipeline.yml
+++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml
@ -122,9 +122,7 @@ stages:
    displayName: Validate Package BOM
    pool:
      # testing
-      vmImage: windows-latest
-    variables:
-    - group: 'Azure Blob variable group'
+      vmImage: ubuntu-latest
    steps:
      - template: templates/release-ValidatePackageBOM.yml

--- a/tools/releaseBuild/azureDevOps/templates/linux.yml
+++ b/tools/releaseBuild/azureDevOps/templates/linux.yml
@ -61,12 +61,16 @@ jobs:
    condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
    workingDirectory: $(PowerShellRoot)

-  - powershell: |
+  - pwsh: |
+      try {
+        Import-Module "$env:POWERSHELLROOT/build.psm1"
+        Import-Module "$env:POWERSHELLROOT/tools/packaging"

-      Import-Module "$env:POWERSHELLROOT/build.psm1"
-      Import-Module "$env:POWERSHELLROOT/tools/packaging"
-
-      Invoke-AzDevOpsLinuxPackageBuild -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
+        Invoke-AzDevOpsLinuxPackageBuild -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
+      } catch {
+        Get-Error
+        throw
+      }
    displayName: 'Build'
    condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
    workingDirectory: $(PowerShellRoot)
@ -111,20 +115,25 @@ jobs:
        sourceScanPath: '$(PowerShellRoot)/tools'

  - powershell: |
-      Import-Module "$env:POWERSHELLROOT/build.psm1"
-      Import-Module "$env:POWERSHELLROOT/tools/packaging"
+      try {
+        Import-Module "$env:POWERSHELLROOT/build.psm1"
+        Import-Module "$env:POWERSHELLROOT/tools/packaging"

-      $metadata = Get-Content "$env:POWERSHELLROOT/tools/metadata.json" -Raw | ConvertFrom-Json
+        $metadata = Get-Content "$env:POWERSHELLROOT/tools/metadata.json" -Raw | ConvertFrom-Json

-      # LTSRelease.Package indicates that the release should be packaged as an LTS
-      $LTS = $metadata.LTSRelease.Package
-      Write-Verbose -Verbose -Message "LTS is set to: $LTS"
+        # LTSRelease.Package indicates that the release should be packaged as an LTS
+        $LTS = $metadata.LTSRelease.Package
+        Write-Verbose -Verbose -Message "LTS is set to: $LTS"

-      Invoke-AzDevOpsLinuxPackageCreation -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
+        Invoke-AzDevOpsLinuxPackageCreation -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'

-      if ($LTS) {
-          Write-Verbose -Verbose "Packaging LTS"
-          Invoke-AzDevOpsLinuxPackageCreation -LTS  -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
+        if ($LTS) {
+            Write-Verbose -Verbose "Packaging LTS"
+            Invoke-AzDevOpsLinuxPackageCreation -LTS  -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
+        }
+      } catch {
+        Get-Error
+        throw
      }
    displayName: 'Package'
    condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
@ -244,10 +253,25 @@ jobs:
          signOutputPath: $(Build.StagingDirectory)\signedPackages
          certificateId: "CP-450779-Pgp"
          pattern: |
-            **\*.rpm
+            **\*.rh.*.rpm
          useMinimatch: true
          shouldSign: $(SHOULD_SIGN)
-          displayName: Sign RPM
+          displayName: Sign RedHat RPM
+          OutputMode: AlwaysCopy
+
+  - ${{ if eq(variables['buildName'], 'RPM') }}:
+    - template: EsrpSign.yml@ComplianceRepo
+      parameters:
+          # Sign in-place, previous task copied the files to this folder
+          buildOutputPath: $(Build.StagingDirectory)\signedPackages
+          signOutputPath: $(Build.StagingDirectory)\signedPackages
+          certificateId: "CP-459159-Pgp"
+          pattern: |
+            **\*.cm1.*.rpm
+          useMinimatch: true
+          shouldSign: $(SHOULD_SIGN)
+          displayName: Sign Mariner RPM
+          OutputMode: NeverCopy

  # requires windows
  - ${{ if ne(variables['buildName'], 'RPM') }}:
--- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml
+++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml
@ -109,13 +109,23 @@ jobs:
    displayName: Compress signed files

  - pwsh: |
-      tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap
+      try {
+        tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap
+      } catch {
+        Get-Error
+        throw
+      }
    displayName: 'Bootstrap VM'

  - pwsh: |
      # Add -SkipReleaseChecks as a mitigation to unblock release.
      # macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
-      $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks
+      try {
+        $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks
+      } catch {
+        Get-Error
+        throw
+      }
    displayName: 'Package'

  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
--- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml
+++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml
@ -14,17 +14,17 @@ steps:
    Write-Host "##vso[build.updatebuildnumber]$name"
  displayName: Set Release Name

- pwsh: |
-    Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1'
-    Install-AzCopy
-  displayName: Install AzCopy
+- task: DownloadPipelineArtifact@2
+  inputs:
+    source: specific
+    project: PowerShellCore
+    pipeline: '696'
+    preferTriggeringPipeline: true
+    runVersion: latestFromBranch
+    runBranch: '$(Build.SourceBranch)'
+    artifact: finalResults
+    path: $(System.ArtifactsDirectory)

- pwsh: |
-    Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1'
-    $azcopy = Find-AzCopy
-
-    & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey)
-  displayName: Download Azure Artifacts

 - pwsh: |
    Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name
--- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml
+++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml
@ -30,7 +30,7 @@ steps:
 - pwsh: |
    $message = @()
    Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.rpm | ForEach-Object {
-        if($_.Name -notmatch 'powershell\-(preview-|lts-)?\d\.\d\.\d(_[a-z]*\.\d+)?-1.rh.x86_64\.rpm')
+        if($_.Name -notmatch 'powershell\-(preview-|lts-)?\d\.\d\.\d(_[a-z]*\.\d+)?-1.(rh|cm1).x86_64\.rpm')
        {
            $messageInstance = "$($_.Name) is not a valid package name"
            $message += $messageInstance