Add tool to trigger license information gathering for NuGet modules (#18827)

.spelling

@@ -1671,3 +1671,5 @@ centos-7
 Security.types.ps1xml
  - ADOPTERS.md
 MicrosoftPowerBIMgmt
+ - tools/clearlyDefined/readme.md
+ClearlyDefined

tools/clearlyDefined/ClearlyDefined.ps1

@@ -0,0 +1,47 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+param(
+    [parameter(Mandatory = $true, ParameterSetName='Harvest')]
+    [switch]
+    $Harvest,
+    [parameter(Mandatory = $true, ParameterSetName='Test')]
+    [switch]
+    $Test,
+    [switch]
+    $ForceModuleReload
+)
+
+$extraParams = @{}
+if ($ForceModuleReload) {
+    $extraParams['Force'] = $true
+}
+
+Import-Module -Name "$PSScriptRoot/src/ClearlyDefined" @extraParams
+
+$cgManfest = Get-Content "$PSScriptRoot/../cgmanifest.json" | ConvertFrom-Json
+$fullCgList = $cgManfest.Registrations.Component |
+    ForEach-Object {
+        [Pscustomobject]@{
+            type = $_.Type
+            Name = $_.Nuget.Name
+            PackageVersion = $_.Nuget.Version
+        }
+    }
+
+$fullList = $fullCgList | Get-ClearlyDefinedData
+
+$needHarvest = $fullList | Where-Object { !$_.harvested }
+
+Write-Verbose "Full List count: $($fullList.Count)" -Verbose
+Write-Verbose "Need harvest: $($needHarvest.Count)" -Verbose
+
+if ($Harvest) {
+    $needHarvest | select-object -ExpandProperty coordinates | Start-ClearlyDefinedHarvest
+} elseif ($Test) {
+    if($needHarvest.Count -gt 0) {
+        $needHarvest | Format-List | Out-String -Width 9999 | Write-Verbose -Verbose
+        throw "There are $($needHarvest.Count) packages that need to be harvested"
+    } else {
+        Write-Verbose "All packages have been harvested" -Verbose
+    }
+}

tools/clearlyDefined/readme.md

@@ -0,0 +1,30 @@
+# ClearlyDefined
+
+## Purpose
+
+This tool is intended to test if all the license data in [ClearlyDefined](https://clearlydefined.io) is present to generate the PowerShell license.
+If the data is not present, it can request that ClearlyDefined gather (called Harvest in their terminology) the data.
+
+## Use
+
+### Testing
+
+Run `./ClearlyDefined.ps1 -test`.
+
+If there is any missing data, the script should write verbose messages about the missing data and throw.
+If there is no missing data, the script should not throw.
+
+### Harvesting
+
+Run `./ClearlyDefined.ps1 -Harvest`.
+The script will trigger the harvest and output the result from ClearlyDefined.
+**Give ClearlyDefined 24 hours to harvest the data.**
+You can use the `-Test` switch without the `-Harvest` switch to test if Harvesting is done.
+
+## Caching
+
+If you run in the same PowerShell session, the script will be faster due to caching.
+
+The module will cache any results from ClearlyDefined that indicate the package is Harvested for 60 minutes.
+No caching is done for packages that are not yet harvested.
+To clear the cache, run with the `-ForceModuleReload` switch.

tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1

@@ -0,0 +1,137 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# Start the collection (known as harvest) of ClearlyDefined data for a package
+function Start-ClearlyDefinedHarvest {
+    [CmdletBinding()]
+    param(
+        [Parameter(ValueFromPipelineByPropertyName=$true)]
+        [Alias('Type')]
+        [validateset('nuget')]
+        [string]
+        $PackageType = 'nuget',
+
+        [parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)]
+        [Alias('Name')]
+        [string]
+        $PackageName,
+
+        [parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)]
+        [Alias('Version')]
+        [Alias('Revision')]
+        [string]
+        $PackageVersion
+    )
+
+    Process {
+        $coordinates = Get-ClearlyDefinedCoordinates @PSBoundParameters
+        $body = @{tool='package';coordinates=$coordinates} | convertto-json
+        Write-Verbose $body -Verbose
+        (Invoke-WebRequest -Method Post  -Uri 'https://api.clearlydefined.io/harvest' -Body $body -ContentType 'application/json').Content
+    }
+}
+
+function ConvertFrom-ClearlyDefinedCoordinates {
+    [CmdletBinding()]
+    param(
+        [parameter(mandatory = $true, ValueFromPipeline = $true)]
+        [string]
+        $Coordinates
+    )
+
+    Begin {}
+    Process {
+        $parts = $Coordinates.Split('/')
+        [PSCustomObject]@{
+            type = $parts[0]
+            provider = $parts[1]
+            namespace = $parts[2]
+            name = $parts[3]
+            revision = $parts[4]
+        }
+    }
+    End {}
+}
+
+# Get the coordinate string for a package
+Function Get-ClearlyDefinedCoordinates {
+    [CmdletBinding()]
+    param(
+        [validateset('nuget')]
+        [string]
+        $PackageType = 'nuget',
+        [parameter(mandatory = $true)]
+        [string]
+        $PackageName,
+        [parameter(mandatory = $true)]
+        [string]
+        $PackageVersion
+    )
+
+    return "$PackageType/$PackageType/-/$PackageName/$PackageVersion"
+}
+
+# Cache of ClearlyDefined data
+$cdCache = @{}
+
+# Get the ClearlyDefined data for a package
+Function Get-ClearlyDefinedData {
+    [CmdletBinding()]
+    param(
+        [Parameter(ValueFromPipelineByPropertyName=$true)]
+        [Alias('Type')]
+        [validateset('nuget')]
+        [string]
+        $PackageType = 'nuget',
+
+        [parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)]
+        [Alias('Name')]
+        [string]
+        $PackageName,
+
+        [parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)]
+        [Alias('Revision')]
+        [string]
+        $PackageVersion
+    )
+
+    Begin {
+        $cacheMinutes = 60
+        $cacheCutoff = (get-date).AddMinutes(-$cacheMinutes)
+        $coordinateList = @()
+    }
+
+    Process {
+        $coordinateList += Get-ClearlyDefinedCoordinates @PSBoundParameters
+    }
+
+    end {
+        $total = $coordinateList.Count
+        $completed = 0
+        foreach($coordinates in $coordinateList) {
+            Write-Progress -Activity "Getting ClearlyDefined data" -Status "Getting data for $coordinates" -PercentComplete (($completed / $total) * 100)
+            $containsKey = $cdCache.ContainsKey($coordinates)
+            if ($containsKey -and $cdCache[$coordinates].cachedTime -gt $cacheCutoff) {
+                Write-Verbose "Returning cached data for $coordinates"
+                Write-Output $cdCache[$coordinates]
+                continue
+            }
+
+            Invoke-RestMethod  -Uri "https://api.clearlydefined.io/definitions/$coordinates" | ForEach-Object {
+                [bool] $harvested = if ($_.licensed.declared) { $true } else { $false }
+                Add-Member -NotePropertyName cachedTime -NotePropertyValue (get-date) -InputObject $_ -PassThru | Add-Member -NotePropertyName harvested -NotePropertyValue $harvested -PassThru
+                if ($_.harvested) {
+                    Write-Verbose "Caching data for $coordinates"
+                    $cdCache[$coordinates] = $_
+                }
+            }
+            $completed++
+        }
+    }
+}
+
+Export-ModuleMember -Function @(
+    'Start-ClearlyDefinedHarvest'
+    'Get-ClearlyDefinedData'
+    'ConvertFrom-ClearlyDefinedCoordinates'
+)