Add GitHub Workflow to keep notices up to date (#16284)

Co-authored-by: Robert Holt <rjmholt_msft@outlook.com>
2024-11-23 01:34:19 +08:00 · 2021-10-28 11:44:06 -07:00 · 2021-10-28 11:44:06 -07:00 · f41b6b415e
commit f41b6b415e
parent 52b6d89407
4 changed files with 242 additions and 92 deletions
--- a/.github/workflows/update-cgmanifest.yml
+++ b/.github/workflows/update-cgmanifest.yml
@ -0,0 +1,64 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+name: Update cgmanifest
+on:
+    workflow_dispatch:
+    schedule:
+    # At 13:00 UTC every day.
+    - cron:  '0 13 * * *'
+
+defaults:
+  run:
+    shell: pwsh
+
+env:
+  DOTNET_CLI_TELEMETRY_OPTOUT: 1
+  POWERSHELL_TELEMETRY_OPTOUT: 1
+
+jobs:
+  update-cgmanifest:
+    name: Update cgmanifest
+    timeout-minutes: 15
+    runs-on: windows-latest
+    if: github.repository == 'PowerShell/PowerShell'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Sync tags
+        run: |
+          git fetch --prune --unshallow --tags
+      - name: Install Ships provider to deal with project.assets.json
+        run: |
+          Install-Module -Name dotnet.project.assets -force
+      - name: Bootstrap
+        run: |
+          Import-Module ./build.psm1
+          Start-PSBootStrap
+      - name: Update Notices file
+        run: |
+          Invoke-WebRequest -Uri https://aka.ms/pwsh-daily-tpn -OutFile ./ThirdPartyNotices.txt
+      - name: Execute script to update cgmanifest
+        run: |
+          Import-Module ./build.psm1
+          Find-Dotnet
+          ./tools/findMissingNotices.ps1
+      - name: Microsoft Teams Notifier
+        uses: skitionek/notify-microsoft-teams@master
+        if: failure()
+        with:
+          webhook_url: ${{ secrets.PS_BUILD_TEAMS_CHANNEL }}
+          overwrite: "{title: `Failure in updating cgmanifest. Look at ${workflow_link}`}"
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v3
+        id: cpr
+        if: env.CREATE_PR == 'true'
+        with:
+          commit-message: "Update the cgmanifest with missing or updated components"
+          committer: GitHub <noreply@github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          title: "Update ${{ env.FORMULA_NAME }} formula to version ${{ env.NEW_FORMULA_VERSION }}"
+          reviewers: travisez13
+          base: master
+          draft: false
+          branch: update-cgmanifest
--- a/build.psm1
+++ b/build.psm1
@ -6,6 +6,8 @@ param(
    [parameter(Mandatory = $false)][switch]$SkipLinuxDistroCheck = $false
 )

+. "$PSScriptRoot\tools\buildCommon\startNativeExecution.ps1"
+
 Set-StrictMode -Version 3.0

 # On Unix paths is separated by colon
@ -2357,53 +2359,6 @@ function script:precheck([string]$command, [string]$missedMessage) {
    }
 }

-# this function wraps native command Execution
-# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
-function script:Start-NativeExecution
-{
-    param(
-        [scriptblock]$sb,
-        [switch]$IgnoreExitcode,
-        [switch]$VerboseOutputOnError
-    )
-    $backupEAP = $ErrorActionPreference
-    $ErrorActionPreference = "Continue"
-    try {
-        if($VerboseOutputOnError.IsPresent)
-        {
-            $output = & $sb 2>&1
-        }
-        else
-        {
-            & $sb
-        }
-
-        # note, if $sb doesn't have a native invocation, $LASTEXITCODE will
-        # point to the obsolete value
-        if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
-            if($VerboseOutputOnError.IsPresent -and $output)
-            {
-                $output | Out-String | Write-Verbose -Verbose
-            }
-
-            # Get caller location for easier debugging
-            $caller = Get-PSCallStack -ErrorAction SilentlyContinue
-            if($caller)
-            {
-                $callerLocationParts = $caller[1].Location -split ":\s*line\s*"
-                $callerFile = $callerLocationParts[0]
-                $callerLine = $callerLocationParts[1]
-
-                $errorMessage = "Execution of {$sb} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
-                throw $errorMessage
-            }
-            throw "Execution of {$sb} failed with exit code $LASTEXITCODE"
-        }
-    } finally {
-        $ErrorActionPreference = $backupEAP
-    }
-}
-
 # Cleans the PowerShell repo - everything but the root folder
 function Clear-PSRepo
 {
--- a/tools/buildCommon/startNativeExecution.ps1
+++ b/tools/buildCommon/startNativeExecution.ps1
@ -0,0 +1,47 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# this function wraps native command Execution
+# for more information, read https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
+function script:Start-NativeExecution {
+    param(
+        [Alias('sb')]
+        [Parameter(Mandatory=$true)]
+        [scriptblock]$ScriptBlock,
+        [switch]$IgnoreExitcode,
+        [switch]$VerboseOutputOnError
+    )
+
+    $backupEAP = $ErrorActionPreference
+    $ErrorActionPreference = "Continue"
+    Write-Verbose "Executing: $ScriptBlock"
+    try {
+        if ($VerboseOutputOnError.IsPresent) {
+            $output = & $ScriptBlock 2>&1
+        } else {
+            & $ScriptBlock
+        }
+
+        # note, if $ScriptBlock doesn't have a native invocation, $LASTEXITCODE will
+        # point to the obsolete value
+        if ($LASTEXITCODE -ne 0 -and -not $IgnoreExitcode) {
+            if ($VerboseOutputOnError.IsPresent -and $output) {
+                $output | Out-String | Write-Verbose -Verbose
+            }
+
+            # Get caller location for easier debugging
+            $caller = Get-PSCallStack -ErrorAction SilentlyContinue
+            if ($caller) {
+                $callerLocationParts = $caller[1].Location -split ":\s*line\s*"
+                $callerFile = $callerLocationParts[0]
+                $callerLine = $callerLocationParts[1]
+
+                $errorMessage = "Execution of {$ScriptBlock} by ${callerFile}: line $callerLine failed with exit code $LASTEXITCODE"
+                throw $errorMessage
+            }
+            throw "Execution of {$ScriptBlock} failed with exit code $LASTEXITCODE"
+        }
+    } finally {
+        $ErrorActionPreference = $backupEAP
+    }
+}
--- a/tools/findMissingNotices.ps1
+++ b/tools/findMissingNotices.ps1
@ -1,9 +1,23 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.

+# This script is used to completely rebuild the cgmanifgest.json file,
+# which is used to generate the notice file.
 # Requires the module dotnet.project.assets from the PowerShell Gallery authored by @TravisEz13

-import-module dotnet.project.assets
+Import-Module dotnet.project.assets
+. "$PSScriptRoot\..\tools\buildCommon\startNativeExecution.ps1"
+
+$existingRegistrationTable = @{}
+$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable
+$existingRegistrationsJson.Registrations | ForEach-Object {
+    $registration = [Registration]$_
+    if($registration.Component) {
+        $name = $registration.Component.Name()
+        $existingRegistrationTable.Add($name, $registration)
+    }
+}
+
 Class Registration {
    [Component]$Component
    [bool]$DevelopmentDependency
@ -79,58 +93,128 @@ function New-NugetComponent {
    return $registration
 }

-$existingRegistrationTable = @{}
-$newRegistrations = @()
-$existingRegistrationsJson = Get-Content $PSScriptRoot\..\cgmanifest.json | ConvertFrom-Json -AsHashtable
-$existingRegistrationsJson.Registrations | ForEach-Object {
-    $registration = [Registration]$_
-    $existingRegistrationTable.Add($registration.Component.Name(), $registration)
-    $newRegistrations += $registration
+$winDesktopSdk = 'Microsoft.NET.Sdk.WindowsDesktop'
+if (!$IsWindows) {
+    $winDesktopSdk = 'Microsoft.NET.Sdk'
+    Write-Warning "Always using $winDesktopSdk since this is not windows!!!"
 }

-Get-PSDrive -Name pwsh-win-core -ErrorAction Ignore | Remove-PSDrive
-Push-Location $PSScriptRoot\..\src\powershell-win-core
-$null = dotnet restore
-$null = New-PADrive -Path $PSScriptRoot\..\src\powershell-win-core\obj\project.assets.json -Name pwsh-win-core
-$targets = Get-ChildItem -Path 'pwsh-win-core:/targets/net6.0-windows7.0|win7-x64' | Where-Object {
-    $_.Type -eq 'package' -and
-    $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and
-    $_.Name -notlike 'StyleCop*' -and
-    $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and
-    $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*'
-}  | select-object -ExpandProperty name
-Pop-Location
-Get-PSDrive -Name pwsh-win-core | Remove-PSDrive
+Function Get-CGRegistrations {
+    param(
+        [Parameter(Mandatory)]
+        [ValidateSet(
+            "alpine-x64",
+            "linux-arm",
+            "linux-arm64",
+            "linux-x64",
+            "osx-arm64",
+            "osx-x64",
+            "win-arm",
+            "win-arm64",
+            "win7-x64",
+            "win7-x86",
+            "modules")]
+        [string]$Runtime,

-$updateRegistrations = @()
-$targets | ForEach-Object {
-    $target = $_
-    $parts = ($target -split '\|')
-    $name = $parts[0]
-    $targetVersion = $parts[1]
-    $pattern = [regex]::Escape($name) + " "
-    $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
-    if (!$tpnMatch) {
-        if ($existingRegistrationTable.ContainsKey($name)) {
-            $registrationVersion = $existingRegistrationTable.$name.Component.Version()
-            if ($registrationVersion -ne $targetVersion) {
-                $registration = New-NugetComponent -Name $name -Version $targetVersion
-                $updateRegistrations += $registration
-            } else {
-                Write-Verbose "$target already registered: $registrationVersion" -Verbose
-            }
-        } else {
+        [Parameter(Mandatory)]
+        [System.Collections.Generic.Dictionary[string, Registration]] $RegistrationTable
+    )
+
+    $newRegistrations = $Registrations
+
+    $dotnetTargetName = 'net6.0'
+    $dotnetTargetNameWin7 = 'net6.0-windows7.0'
+    $unixProjectName = 'powershell-unix'
+    $windowsProjectName = 'powershell-win-core'
+    $actualRuntime = $Runtime
+
+    switch -regex ($Runtime) {
+        "alpine-.*" {
+            $folder = $unixProjectName
+            $target = "$dotnetTargetName|$Runtime"
+        }
+        "linux-.*" {
+            $folder = $unixProjectName
+            $target = "$dotnetTargetName|$Runtime"
+        }
+        "osx-.*" {
+            $folder = $unixProjectName
+            $target = "$dotnetTargetName|$Runtime"
+        }
+        "win7-.*" {
+            $sdkToUse = $winDesktopSdk
+            $folder = $windowsProjectName
+            $target = "$dotnetTargetNameWin7|$Runtime"
+        }
+        "win-.*" {
+            $folder = $windowsProjectName
+            $target = "$dotnetTargetNameWin7|$Runtime"
+        }
+        "modules" {
+            $folder = "modules"
+            $actualRuntime = 'linux-x64'
+            $target = "$dotnetTargetName|$actualRuntime"
+        }
+        Default {
+            throw "Invalid runtime name: $Runtime"
+        }
+    }
+
+    Write-Verbose "Getting registrations for $folder - $actualRuntime ..." -Verbose
+    Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive
+    Push-Location $PSScriptRoot\..\src\$folder
+    try {
+        Start-NativeExecution -VerboseOutputOnError -sb {
+            dotnet restore --runtime $actualRuntime  "/property:SDKToUse=$sdkToUse"
+        }
+        $null = New-PADrive -Path $PSScriptRoot\..\src\$folder\obj\project.assets.json -Name $folder
+        try {
+            $targets = Get-ChildItem -Path "${folder}:/targets/$target" -ErrorAction Stop | Where-Object {
+                $_.Type -eq 'package' -and
+                $_.Name -notlike 'DotNetAnalyzers.DocumentationAnalyzers*' -and
+                $_.Name -notlike 'StyleCop*' -and
+                $_.Name -notlike 'Microsoft.CodeAnalysis.Analyzers*' -and
+                $_.Name -notlike 'Microsoft.CodeAnalysis.NetAnalyzers*'
+            }  | select-object -ExpandProperty name
+        } catch {
+            Get-ChildItem -Path "${folder}:/targets" | Out-String | Write-Verbose -Verbose
+            throw
+        }
+    } finally {
+        Pop-Location
+        Get-PSDrive -Name $folder -ErrorAction Ignore | Remove-PSDrive
+    }
+
+    $targets | ForEach-Object {
+        $target = $_
+        $parts = ($target -split '\|')
+        $name = $parts[0]
+        $targetVersion = $parts[1]
+        $pattern = [regex]::Escape($name) + " "
+        $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
+
+        # Add the registration to the cgmanifest if the TPN does not contain the name of the target OR
+        # the exisitng CG contains the registration, because if the existing CG contains the registration,
+        # that might be the only reason it is in the TPN.
+        if ((!$tpnMatch -or $existingRegistrationTable.ContainsKey($name)) -and !$RegistrationTable.ContainsKey($target)) {
            $registration = New-NugetComponent -Name $name -Version $targetVersion
-            $newRegistrations += $registration
+            $RegistrationTable.Add($target, $registration)
        }
    }
 }

-if ($updateRegistrations.count -gt 0) {
-    #TODO delete existing and add new registration
-    throw "updating registrations is not implemented"
+$registrations = [System.Collections.Generic.Dictionary[string, Registration]]::new()
+$lastCount = 0
+foreach ($runtime in "win7-x64", "linux-x64", "osx-x64", "alpine-x64", "win-arm", "linux-arm", "linux-arm64", "osx-arm64", "win-arm64", "win7-x86") {
+    Get-CGRegistrations -Runtime $runtime -RegistrationTable $registrations
+    $count = $registrations.Count
+    $newCount = $count - $lastCount
+    $lastCount = $count
+    Write-Verbose "$newCount new registrations, $count total..." -Verbose
 }

-$newCount = $newRegistrations.count - $existingRegistrationTable.count
+$newRegistrations = $registrations.Keys | Sort-Object | ForEach-Object { $registrations[$_] }
+
+$count = $newRegistrations.Count
@{Registrations = $newRegistrations } | ConvertTo-Json -depth 99 | Set-Content $PSScriptRoot\..\cgmanifest.json
-Write-Verbose "$newCount registrations added" -Verbose
+Write-Verbose "$count registrations created!" -Verbose