Base work for adding mariner amd64 package (#17417)

Backport #17057
2024-11-24 02:04:07 +08:00 · 2022-05-24 13:17:15 -07:00 · 2022-05-24 13:17:15 -07:00 · 3cfa8eb258
commit 3cfa8eb258
parent 8af2f5c611
7 changed files with 196 additions and 85 deletions
--- a/tools/packaging/packaging.psm1
+++ b/tools/packaging/packaging.psm1
@ -7,7 +7,11 @@ $RepoRoot = (Resolve-Path -Path "$PSScriptRoot/../..").Path
 $packagingStrings = Import-PowerShellDataFile "$PSScriptRoot\packaging.strings.psd1"
 Import-Module "$PSScriptRoot\..\Xml" -ErrorAction Stop -Force
 $DebianDistributions = @("deb")
-$RedhatDistributions = @("rh")
+$RedhatDistributions = @("rh","cm1")
+$AllDistributions = @()
+$AllDistributions += $DebianDistributions
+$AllDistributions += $RedhatDistributions
+$AllDistributions += 'macOs'
 $script:netCoreRuntime = 'net6.0'
 $script:iconFileName = "Powershell_black_64.png"
 $script:iconPath = Join-Path -path $PSScriptRoot -ChildPath "../../assets/$iconFileName" -Resolve
@ -500,6 +504,7 @@ function Start-PSPackage {
                foreach ($Distro in $Script:RedhatDistributions) {
                    $Arguments["Distribution"] = $Distro
                    if ($PSCmdlet.ShouldProcess("Create RPM Package for $Distro")) {
+                        Write-Verbose -Verbose "Creating RPM Package for $Distro"
                        New-UnixPackage @Arguments
                    }
                }
@ -981,7 +986,7 @@ function New-UnixPackage {
        # Build package
        try {
            if ($PSCmdlet.ShouldProcess("Create $type package")) {
-                Write-Log "Creating package with fpm..."
+                Write-Log "Creating package with fpm $Arguments..."
                try {
                    $Output = Start-NativeExecution { fpm $Arguments }
                }
@ -1237,7 +1242,8 @@ function Get-FpmArguments
        "-t", $Type,
        "-s", "dir"
    )
-    if ($Distribution -eq 'rh') {
+
+    if ($Distribution -in $script:RedHatDistributions) {
        $Arguments += @("--rpm-digest", "sha256")
        $Arguments += @("--rpm-dist", $Distribution)
        $Arguments += @("--rpm-os", "linux")
@ -1283,13 +1289,29 @@ function Get-FpmArguments

 function Get-PackageDependencies
 {
-    param(
-        [String]
-        [ValidateSet('rh','deb','macOS')]
-        $Distribution
-    )
+    [CmdletBinding()]
+    param()
+    DynamicParam {
+        # Add a dynamic parameter '-Distribution' when the specified package type is 'deb'.
+        # The '-Distribution' parameter can be used to indicate which Debian distro this pacakge is targeting.
+        $ParameterAttr = New-Object "System.Management.Automation.ParameterAttribute"
+        $ParameterAttr.Mandatory = $true
+        $ValidateSetAttr = New-Object "System.Management.Automation.ValidateSetAttribute" -ArgumentList $Script:AllDistributions
+        $Attributes = New-Object "System.Collections.ObjectModel.Collection``1[System.Attribute]"
+        $Attributes.Add($ParameterAttr) > $null
+        $Attributes.Add($ValidateSetAttr) > $null
+
+        $Parameter = New-Object "System.Management.Automation.RuntimeDefinedParameter" -ArgumentList ("Distribution", [string], $Attributes)
+        $Dict = New-Object "System.Management.Automation.RuntimeDefinedParameterDictionary"
+        $Dict.Add("Distribution", $Parameter) > $null
+        return $Dict
+    }

    End {
+        if ($PSBoundParameters.ContainsKey('Distribution')) {
+            $Distribution = $PSBoundParameters['Distribution']
+        }
+
        # These should match those in the Dockerfiles, but exclude tools like Git, which, and curl
        $Dependencies = @()
        if ($Distribution -eq 'deb') {
@ -1308,6 +1330,22 @@ function Get-PackageDependencies
                "openssl-libs",
                "libicu"
            )
+        } elseif ($Distribution -eq 'cm1') {
+            # Taken from the list here:
+            # https://github.com/dotnet/dotnet-docker/blob/d451d6e9427f58c8508f1297c862663a27eb609f/src/runtime-deps/6.0/cbl-mariner1.0/amd64/Dockerfile#L6
+            $Dependencies = @(
+                "glibc"
+                "libgcc"
+                "krb5"
+                "libstdc++"
+                "zlib"
+                "icu"
+                "openssl-libs"
+            )
+        } elseif ($Distribution -eq 'macOS') {
+            # do nothing
+        } else {
+            throw "Unknown distribution $Distribution"
        }

        return $Dependencies
@ -1358,7 +1396,7 @@ function New-AfterScripts

    Write-Verbose -Message "AfterScript Distribution: $Distribution" -Verbose

-    if ($Distribution -eq 'rh') {
+    if ($Distribution -in $script:RedHatDistributions) {
        $AfterInstallScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName()))
        $AfterRemoveScript = (Join-Path $env:HOME $([System.IO.Path]::GetRandomFileName()))
        $packagingStrings.RedHatAfterInstallScript -f "$Link", $Destination  | Out-File -FilePath $AfterInstallScript -Encoding ascii
@ -4197,7 +4235,7 @@ function Invoke-AzDevOpsLinuxPackageCreation {
    }

    try {
-        Write-Verbose "Packaging '$BuildType'-LTS:$LTS for $ReleaseTag ..." -Verbose
+        Write-Verbose "Packaging '$BuildType'; LTS:$LTS for $ReleaseTag ..." -Verbose

        Restore-PSOptions -PSOptionsPath "${env:SYSTEM_ARTIFACTSDIRECTORY}\${mainLinuxBuildFolder}-meta\psoptions.json"

--- a/tools/packaging/releaseTests/sbom.tests.ps1
+++ b/tools/packaging/releaseTests/sbom.tests.ps1
@ -8,24 +8,68 @@ Describe "Verify SBOMs" {
        $missingFromPackageCases = @()
        $missingFromManifestCases = @()
        Write-Verbose "${env:PACKAGE_FOLDER}" -Verbose
-        Get-ChildItem $env:PACKAGE_FOLDER -Filter *.zip | ForEach-Object {
+        Get-ChildItem $env:PACKAGE_FOLDER -Filter *.zip |
+            Where-Object { $_.Name -notlike 'powershell-symbols*' } |
+            ForEach-Object {
                Write-Verbose "Found $($_.Name)..." -Verbose
                $testCases += @{
                    FilePath = $_.FullName
                    Name = $_.Name
+                    Extension = $_.Extension
+            }
+        }
+
+        if ($IsLinux) {
+            Get-ChildItem $env:PACKAGE_FOLDER -Filter *.rpm | ForEach-Object {
+                Write-Verbose "Found $($_.Name)..." -Verbose
+                $testCases += @{
+                    FilePath  = $_.FullName
+                    Name      = $_.Name
+                    Extension = $_.Extension
+                }
            }
        }

        foreach($case in $testCases) {
+            $skip = $null
            $name = $case.Name
            Write-Verbose "Testing $name..." -Verbose
            $extractedPath = Join-Path Testdrive:\ -ChildPath ([System.io.path]::GetRandomFileName())
            $null = New-Item -Path $extractedPath -ItemType Directory -Force
+            $resolvedPath = (Resolve-Path -Path $extractedPath).ProviderPath
+            switch ($case.Extension) {
+                '.zip' {
                    Expand-Archive -Path $case.FilePath -DestinationPath $extractedPath
                    $manifestPath = Join-Path $extractedPath -ChildPath '/_manifest/spdx_2.2/manifest.spdx.json'
+                }
+                '.rpm' {
+                    Push-Location $resolvedPath
+                    try {
+                        if (Get-Command -Name rpm2cpio -ErrorAction SilentlyContinue) {
+                            rpm2cpio $case.FilePath | cpio -i --make-directories 2>cpio.log
+                            $manifestPath = Get-ChildItem -Path manifest.spdx.json -Recurse | Select-Object -First 1 -ExpandProperty FullName
+                            $extractedPath = Get-ChildItem -Path system.management.automation.dll -Recurse  | Select-Object -First 1 -ExpandProperty DirectoryName
+                        } else {
+                            $skip = "rpm2cpio not found"
+                        }
+                    } finally {
+                        Pop-Location
+                    }
+                }
+                Default {
+                    throw "Unkown extension $($case.Extension)"
+                }
+            }
+
            It "$name has a BOM" {
+                if ($skip) {
+                    Set-ItResult -Pending -Because $skip
+                }
                $manifestPath | Should -Exist
            }
+
+            # RPM hashes are broken, skip that
+            if ($case.Extension -in '.zip') {
                Test-PackageManifest -PackagePath $extractedPath | ForEach-Object {
                    $status = $_.Status
                    $expectedHash = $_.ExpectedHash
@ -65,12 +109,9 @@ Describe "Verify SBOMs" {
                }
            }
        }
-
-    Context "Zip files" {
-        BeforeAll {
-            Write-Verbose "In Context BeforeAll" -Verbose
    }

+    Context "Package files" {
        It "<name> should have <file> with matching hash" -TestCases $matchCases {
            param(
                $Name,
--- a/tools/releaseBuild/azureDevOps/releasePipeline.yml
+++ b/tools/releaseBuild/azureDevOps/releasePipeline.yml
@ -122,9 +122,7 @@ stages:
    displayName: Validate Package BOM
    pool:
      # testing
-      vmImage: windows-latest
-    variables:
-    - group: 'Azure Blob variable group'
+      vmImage: ubuntu-latest
    steps:
      - template: templates/release-ValidatePackageBOM.yml

--- a/tools/releaseBuild/azureDevOps/templates/linux.yml
+++ b/tools/releaseBuild/azureDevOps/templates/linux.yml
@ -61,12 +61,16 @@ jobs:
    condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
    workingDirectory: $(PowerShellRoot)

-  - powershell: |
-
+  - pwsh: |
+      try {
        Import-Module "$env:POWERSHELLROOT/build.psm1"
        Import-Module "$env:POWERSHELLROOT/tools/packaging"

        Invoke-AzDevOpsLinuxPackageBuild -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
+      } catch {
+        Get-Error
+        throw
+      }
    displayName: 'Build'
    condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
    workingDirectory: $(PowerShellRoot)
@ -111,6 +115,7 @@ jobs:
        sourceScanPath: '$(PowerShellRoot)/tools'

  - powershell: |
+      try {
        Import-Module "$env:POWERSHELLROOT/build.psm1"
        Import-Module "$env:POWERSHELLROOT/tools/packaging"

@ -126,6 +131,10 @@ jobs:
            Write-Verbose -Verbose "Packaging LTS"
            Invoke-AzDevOpsLinuxPackageCreation -LTS  -ReleaseTag '$(ReleaseTagVar)' -BuildType '$(build)'
        }
+      } catch {
+        Get-Error
+        throw
+      }
    displayName: 'Package'
    condition: and(succeeded(), ne(variables['SkipBuild'], 'true'))
    workingDirectory: $(PowerShellRoot)
@ -244,10 +253,25 @@ jobs:
          signOutputPath: $(Build.StagingDirectory)\signedPackages
          certificateId: "CP-450779-Pgp"
          pattern: |
-            **\*.rpm
+            **\*.rh.*.rpm
          useMinimatch: true
          shouldSign: $(SHOULD_SIGN)
-          displayName: Sign RPM
+          displayName: Sign RedHat RPM
+          OutputMode: AlwaysCopy
+
+  - ${{ if eq(variables['buildName'], 'RPM') }}:
+    - template: EsrpSign.yml@ComplianceRepo
+      parameters:
+          # Sign in-place, previous task copied the files to this folder
+          buildOutputPath: $(Build.StagingDirectory)\signedPackages
+          signOutputPath: $(Build.StagingDirectory)\signedPackages
+          certificateId: "CP-459159-Pgp"
+          pattern: |
+            **\*.cm1.*.rpm
+          useMinimatch: true
+          shouldSign: $(SHOULD_SIGN)
+          displayName: Sign Mariner RPM
+          OutputMode: NeverCopy

  # requires windows
  - ${{ if ne(variables['buildName'], 'RPM') }}:
--- a/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml
+++ b/tools/releaseBuild/azureDevOps/templates/mac-package-build.yml
@ -109,13 +109,23 @@ jobs:
    displayName: Compress signed files

  - pwsh: |
+      try {
        tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap
+      } catch {
+        Get-Error
+        throw
+      }
    displayName: 'Bootstrap VM'

  - pwsh: |
      # Add -SkipReleaseChecks as a mitigation to unblock release.
      # macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
+      try {
        $(Build.SourcesDirectory)/tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -ReleaseTag $(ReleaseTagVar) -Destination $(System.ArtifactsDirectory) -location $(PowerShellRoot) -ArtifactName macosPkgResults -BuildZip $(BuildPackagePath) -ExtraPackage "tar" -Runtime 'osx-${{ parameters.buildArchitecture }}' -SkipReleaseChecks
+      } catch {
+        Get-Error
+        throw
+      }
    displayName: 'Package'

  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
--- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml
+++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageBOM.yml
@ -14,17 +14,17 @@ steps:
    Write-Host "##vso[build.updatebuildnumber]$name"
  displayName: Set Release Name

- pwsh: |
-    Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1'
-    Install-AzCopy
-  displayName: Install AzCopy
+- task: DownloadPipelineArtifact@2
+  inputs:
+    source: specific
+    project: PowerShellCore
+    pipeline: '696'
+    preferTriggeringPipeline: true
+    runVersion: latestFromBranch
+    runBranch: '$(Build.SourceBranch)'
+    artifact: finalResults
+    path: $(System.ArtifactsDirectory)

- pwsh: |
-    Import-module '$(BUILD.SOURCESDIRECTORY)/build.psm1'
-    $azcopy = Find-AzCopy
-
-    & $azcopy /Source:https://$(StorageAccount).blob.core.windows.net/$(AzureVersion) /Dest:$(System.ArtifactsDirectory) /S /SourceKey:$(StorageAccountKey)
-  displayName: Download Azure Artifacts

 - pwsh: |
    Get-ChildItem $(System.ArtifactsDirectory)\* -recurse | Select-Object -ExpandProperty Name
--- a/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml
+++ b/tools/releaseBuild/azureDevOps/templates/release-ValidatePackageNames.yml
@ -30,7 +30,7 @@ steps:
 - pwsh: |
    $message = @()
    Get-ChildItem $(System.ArtifactsDirectory)\* -recurse -filter *.rpm | ForEach-Object {
-        if($_.Name -notmatch 'powershell\-(preview-|lts-)?\d\.\d\.\d(_[a-z]*\.\d+)?-1.rh.x86_64\.rpm')
+        if($_.Name -notmatch 'powershell\-(preview-|lts-)?\d\.\d\.\d(_[a-z]*\.\d+)?-1.(rh|cm1).x86_64\.rpm')
        {
            $messageInstance = "$($_.Name) is not a valid package name"
            $message += $messageInstance