Create compliance build (#16286)

Co-authored-by: Robert Holt <rjmholt@gmail.com>

assets/additionalAttributions.txt

@@ -1,6 +1,9 @@
-## Used to generate a new TPN
-## Copy this into the additional attributions fields
-## Copy everything below here, but do not include this line
+
+-------------------------------------------------------------------
+
+-------------------------------------------------------------------
+
+Additional  -
 
 -------------------------------------------------
 Microsoft.PowerShell.Archive

tools/ComponentGovernance/ComponentGovernance.psm1

@@ -0,0 +1,135 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+Class CgData {
+    [pscredential]$Pat
+    [string]$Organization
+    [string]$Project
+
+    CgData ([pscredential]$Pat, [string]$Organization, [string]$Project) {
+        $this.Pat = $Pat
+        $this.Organization = $Organization
+        $this.Project = $Project
+    }
+}
+
+# Get the Component Governance data needed to connect to the API
+function Get-CgData {
+    if($Script:cgData){return $Script:cgData}
+    throw "First call Set-CgCredentials"
+}
+
+# Get a Component Governance API URI
+function Get-Uri {
+    param(
+        [Parameter(Mandatory=$true)]
+        [string]$PathPortion
+    )
+    $cgData = Get-CgData
+    $baseUri = "https://governance.dev.azure.com/$($cgData.Organization)/$($cgData.Project)/_apis/componentgovernance/$PathPortion"
+    Write-Verbose "uri: $baseUri" -Verbose
+    return $baseUri
+}
+
+# A class representing a Component Governance repository
+class CgRepository {
+    #Json formatted summary information about this repository. Currently contains number of active nuget.config alerts.
+    [Object] $additionalInformation
+
+    #The associations for this governed repository. For example, all service tree related entries.
+    [Object] $associations
+
+    #Creator of the governed repository.
+    [Object] $createdBy
+
+    [string] $createdDate
+
+    [int] $id
+
+    [string] $modifiedBy
+
+    [string] $modifiedDate
+
+    [string] $name
+
+    #The policies that are configured in the governed repository.
+    [object[]] $policies
+
+    [object] $projectReference
+
+    [string] $repositoryMoniker
+
+    [object] $repositoryOptions
+
+    [object] $type
+
+    [string] $url
+
+    [object] $userRole
+}
+
+# Gets a list of all repositories governed in the current project
+function Get-CgRepositories {
+    $uri = Get-Uri -PathPortion "governedrepositories?api-version=6.1-preview.1"
+    $cgData = Get-CgData
+    [CgRepository[]] (Invoke-RestMethod -Uri $uri -Authentication Basic -Credential $cgData.Pat).value
+}
+
+# Gets this PowerShell master repository
+Function Get-CgPsRepository {
+    Get-CgRepositories | Where-Object {$_.name -eq 'PowerShell' -and $_.repositoryMoniker -notlike '*/*'}
+}
+
+# Gets the Component Governance Snapshot Type (unique to each Pipeline and Job) in the repository
+function Get-CgSnapshotType {
+    param(
+        [CgRepository]
+        $CgRepository
+    )
+
+    $id = $CgRepository.Id
+    $uri = Get-Uri -PathPortion "GovernedRepositories/$id/snapshottypes?api-version=6.1-preview.1"
+    $cgData = Get-CgData
+    (Invoke-RestMethod -Authentication Basic -Credential $cgData.Pat -Uri $uri).Value
+}
+
+# Gets a Component Governance Notice for a given snapshot type
+function Get-CgNotice {
+    param(
+        [CgRepository]
+        $CgRepository,
+        [int]
+        $SnapshotTypeId
+    )
+
+    $id = $CgRepository.Id
+    $uri = Get-Uri -PathPortion "GovernedRepositories/${id}/notice?snapshotTypeId=${SnapshotTypeId}&api-version=6.1-preview.1"
+    $cgData = Get-CgData
+    (Invoke-RestMethod -Authentication Basic -Credential $cgData.Pat -Uri $uri).content
+}
+
+# Sets the Component Governance credentials used by other functions to connect to the API
+function Set-CgCredentials {
+    param(
+        [Parameter(Mandatory=$true)]
+        [securestring] $Pat,
+
+        [Parameter(Mandatory=$true)]
+        [string] $Organization,
+
+        [Parameter(Mandatory=$true)]
+        [string] $Project
+    )
+
+    $pscred = [PSCredential]::new("PAT",$Pat)
+    $script:cgData = [CgData]::new($pscred, $Organization, $Project)
+}
+
+Export-ModuleMember -Function @(
+    'Get-CgRepositories'
+    'Set-CgCredentials'
+    'Get-CgRepository'
+    'Get-CgPsRepository'
+    'Get-CgSnapshotType'
+    'Get-CgNotice'
+)

tools/findMissingNotices.ps1

@@ -109,7 +109,7 @@ $targets | ForEach-Object {
     $name = $parts[0]
     $targetVersion = $parts[1]
     $pattern = [regex]::Escape($name) + " "
-    $tpnMatch = select-string -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
+    $tpnMatch = Select-String -Path $PSScriptRoot\..\ThirdPartyNotices.txt -Pattern $pattern
     if (!$tpnMatch) {
         if ($existingRegistrationTable.ContainsKey($name)) {
             $registrationVersion = $existingRegistrationTable.$name.Component.Version()

tools/releaseBuild/azureDevOps/compliance.yml

@@ -0,0 +1,44 @@
+name: Compliance-$(Build.BuildId)
+
+trigger: none
+pr: none
+
+schedules:
+  # Chrontab format, see https://en.wikipedia.org/wiki/Cron
+  # this is in UTC
+  - cron: '0 13 * * *'
+    branches:
+      include:
+      - master
+
+resources:
+  repositories:
+  - repository: ComplianceRepo
+    type: github
+    endpoint: ComplianceGHRepo
+    name: PowerShell/compliance
+    ref: master
+
+variables:
+  - name: DOTNET_CLI_TELEMETRY_OPTOUT
+    value: 1
+  - name: POWERSHELL_TELEMETRY_OPTOUT
+    value: 1
+  - name: NugetSecurityAnalysisWarningLevel
+    value: none
+  # Defines the variables AzureFileCopySubscription, StorageAccount, StorageAccountKey, StorageResourceGroup, StorageSubscriptionName
+  - group: 'Azure Blob variable group'
+  # Defines the variables CgPat, CgOrganization, and CgProject
+  - group: 'ComponentGovernance'
+
+stages:
+  - stage: compliance
+    dependsOn: []
+    jobs:
+      - template: templates/compliance/compliance.yml
+        parameters:
+          parentJobs: []
+      - template: templates/compliance/generateNotice.yml
+        parameters:
+          parentJobs: []
+

tools/releaseBuild/azureDevOps/templates/compliance.yml

@@ -62,13 +62,6 @@ jobs:
   - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3
     displayName: 'Run Defender Scan'
 
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2
-    displayName: 'Run CredScan'
-    inputs:
-      suppressionsFile: tools/credScan/suppress.json
-      debugMode: false
-    continueOnError: true
-
   - task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3
     displayName: 'Run BinSkim '
     inputs:
@@ -80,18 +73,6 @@ jobs:
       AnalyzeStatistics: true
     continueOnError: true
 
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1
-    displayName: 'Run PoliCheck'
-    inputs:
-      targetType: F
-      optionsFC: 0
-      optionsXS: 1
-      optionsPE: '1|2|3|4'
-      optionsHMENABLE: 0
-      optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb'
-      optionsUEPath: $(Build.SourcesDirectory)\tools\terms\TermsExclusion.xml
-    continueOnError: true
-
   # add RoslynAnalyzers
 
   - task: securedevelopmentteam.vss-secure-development-tools.build-task-autoapplicability.AutoApplicability@1
@@ -142,7 +123,10 @@ jobs:
       uploadPoliCheck: false
       uploadPREfast: false
       uploadRoslyn: false
+      uploadBinSkim: false
       uploadTSLint: false
+      uploadCredScan: false
+      uploadPoliCheck: false
 
   - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
     displayName: 'Create Security Analysis Report'

tools/releaseBuild/azureDevOps/templates/compliance/compliance.yml

@@ -0,0 +1,90 @@
+parameters:
+  - name: parentJobs
+    type: jobList
+
+jobs:
+- job: compliance
+  variables:
+  - name: runCodesignValidationInjection
+    value : false
+  - name: NugetSecurityAnalysisWarningLevel
+    value: none
+
+  # Defines the variables APIScanClient, APIScanTenant and APIScanSecret
+  - group: PS-PS-APIScan
+
+  displayName: Compliance
+  dependsOn:
+    ${{ parameters.parentJobs }}
+  pool:
+    name: PowerShell1ES
+    demands:
+    - ImageOverride -equals MMS2019
+
+  # APIScan can take a long time
+  timeoutInMinutes: 180
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
+    displayName: 'Run CredScan'
+    inputs:
+      suppressionsFile: tools/credScan/suppress.json
+      debugMode: false
+    continueOnError: true
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1
+    displayName: 'Run PoliCheck'
+    inputs:
+      # targetType F means file or folder and is the only applicable value and the default
+      targetType: F
+      # 1 to enable source code comment scanning, which is what we should do for open source
+      optionsFC: 1
+      # recurse
+      optionsXS: 1
+      # run for severity 1, 2, 3 and 4 issues
+      optionsPE: '1|2|3|4'
+      # disable history management
+      optionsHMENABLE: 0
+      # Excluclusion access database
+      optionsRulesDBPath: '$(Build.SourcesDirectory)\tools\terms\PowerShell-Terms-Rules.mdb'
+      # Terms Exclusion xml file
+      optionsUEPath: $(Build.SourcesDirectory)\tools\terms\TermsExclusion.xml
+    continueOnError: true
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
+    displayName: 'Publish Security Analysis Logs to Build Artifacts'
+    continueOnError: true
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1
+    displayName: 'TSA upload to Codebase: PowerShellCore_201906'
+    inputs:
+      tsaVersion: TsaV2
+      codeBaseName: 'PowerShellCore_201906'
+      uploadFortifySCA: false
+      uploadFxCop: false
+      uploadModernCop: false
+      uploadPREfast: false
+      uploadRoslyn: false
+      uploadTSLint: false
+      uploadCredScan: true
+      uploadPoliCheck: true
+      uploadBinSkim: false
+
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
+    displayName: 'Create Security Analysis Report'
+    inputs:
+      TsvFile: false
+      APIScan: false
+      BinSkim: false
+      CredScan: true
+      PoliCheck: true
+      PoliCheckBreakOn: Severity2Above
+
+  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+    displayName: 'Component Detection'
+    inputs:
+      sourceScanPath: '$(Build.SourcesDirectory)'
+      snapshotForceEnabled: true

tools/releaseBuild/azureDevOps/templates/compliance/generateNotice.yml

@@ -0,0 +1,64 @@
+parameters:
+  - name: parentJobs
+    type: jobList
+
+jobs:
+- job: generateNotice
+  variables:
+  - name: runCodesignValidationInjection
+    value : false
+  - name: NugetSecurityAnalysisWarningLevel
+    value: none
+
+  displayName: Generate Notice
+  dependsOn:
+    ${{ parameters.parentJobs }}
+  pool:
+    name: PowerShell1ES
+    demands:
+    - ImageOverride -equals MMS2019
+
+  # APIScan can take a long time
+  timeoutInMinutes: 15
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - pwsh: |
+      Get-Content ./assets/additionalAttributions.txt | Out-File '$(System.ArtifactsDirectory)\additionalAttributions.txt' -Encoding utf8NoBOM -Force
+      Get-Content -Raw -Path  '$(System.ArtifactsDirectory)\additionalAttributions.txt'
+    displayName: Get Additional Attributions
+
+  - pwsh: |
+      Import-Module ./tools/ComponentGovernance
+      $pat = ConvertTo-SecureString -String "$(CgPat)" -AsPlainText -Force
+      Set-CgCredentials -Pat $pat -Organization $(CgOrganization) -Project $(CgProject)
+      $repo = Get-CgPsRepository
+      $typeId = (Get-CgSnapshotType -CgRepository $repo | where-object {$_.isTracked -eq 'True' -and $_.buildDisplayType -eq 'Coordinated Packages'}).typeId
+      $notice = Get-CgNotice -CgRepository $repo -SnapshotTypeId $typeId
+      $notice | Out-File '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' -Encoding utf8NoBOM -Force
+      Get-Content '$(System.ArtifactsDirectory)\additionalAttributions.txt' | Out-File '$(System.ArtifactsDirectory)\ThirdPartyNotices.txt' -Encoding utf8NoBOM -Force -Append
+    displayName: Get Notice
+
+  - task: AzureFileCopy@4
+    displayName: 'upload Notice'
+    inputs:
+      SourcePath: $(System.ArtifactsDirectory)\ThirdPartyNotices.txt
+      azureSubscription: '$(AzureFileCopySubscription)'
+      Destination: AzureBlob
+      storage: '$(StorageAccount)'
+      ContainerName: 'tpn'
+      resourceGroup: '$(StorageResourceGroup)'
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: $(System.ArtifactsDirectory)
+      artifactName: notice
+    displayName: Publish notice artifacts
+
+  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+    displayName: 'Component Detection'
+    inputs:
+      sourceScanPath: '$(Build.SourcesDirectory)'
+      snapshotForceEnabled: true