From 049a1f29d599d2b95469a6139a4b3d0a2fcec10c Mon Sep 17 00:00:00 2001 From: Travis Plunk Date: Tue, 31 Jan 2023 12:58:15 -0800 Subject: [PATCH] Add tool to trigger license information gathering for NuGet modules (#18827) --- .spelling | 2 + tools/clearlyDefined/ClearlyDefined.ps1 | 47 ++++++ tools/clearlyDefined/readme.md | 30 ++++ .../src/ClearlyDefined/ClearlyDefined.psm1 | 137 ++++++++++++++++++ 4 files changed, 216 insertions(+) create mode 100644 tools/clearlyDefined/ClearlyDefined.ps1 create mode 100644 tools/clearlyDefined/readme.md create mode 100644 tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1 diff --git a/.spelling b/.spelling index 38ca53d607..958114e035 100644 --- a/.spelling +++ b/.spelling @@ -1671,3 +1671,5 @@ centos-7 Security.types.ps1xml - ADOPTERS.md MicrosoftPowerBIMgmt + - tools/clearlyDefined/readme.md +ClearlyDefined diff --git a/tools/clearlyDefined/ClearlyDefined.ps1 b/tools/clearlyDefined/ClearlyDefined.ps1 new file mode 100644 index 0000000000..ded0b22993 --- /dev/null +++ b/tools/clearlyDefined/ClearlyDefined.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +param( + [parameter(Mandatory = $true, ParameterSetName='Harvest')] + [switch] + $Harvest, + [parameter(Mandatory = $true, ParameterSetName='Test')] + [switch] + $Test, + [switch] + $ForceModuleReload +) + +$extraParams = @{} +if ($ForceModuleReload) { + $extraParams['Force'] = $true +} + +Import-Module -Name "$PSScriptRoot/src/ClearlyDefined" @extraParams + +$cgManfest = Get-Content "$PSScriptRoot/../cgmanifest.json" | ConvertFrom-Json +$fullCgList = $cgManfest.Registrations.Component | + ForEach-Object { + [Pscustomobject]@{ + type = $_.Type + Name = $_.Nuget.Name + PackageVersion = $_.Nuget.Version + } + } + +$fullList = $fullCgList | Get-ClearlyDefinedData + +$needHarvest = $fullList | Where-Object { !$_.harvested } + +Write-Verbose "Full List count: $($fullList.Count)" -Verbose +Write-Verbose "Need harvest: $($needHarvest.Count)" -Verbose + +if ($Harvest) { + $needHarvest | select-object -ExpandProperty coordinates | Start-ClearlyDefinedHarvest +} elseif ($Test) { + if($needHarvest.Count -gt 0) { + $needHarvest | Format-List | Out-String -Width 9999 | Write-Verbose -Verbose + throw "There are $($needHarvest.Count) packages that need to be harvested" + } else { + Write-Verbose "All packages have been harvested" -Verbose + } +} diff --git a/tools/clearlyDefined/readme.md b/tools/clearlyDefined/readme.md new file mode 100644 index 0000000000..839fc7019a --- /dev/null +++ b/tools/clearlyDefined/readme.md @@ -0,0 +1,30 @@ +# ClearlyDefined + +## Purpose + +This tool is intended to test if all the license data in [ClearlyDefined](https://clearlydefined.io) is present to generate the PowerShell license. +If the data is not present, it can request that ClearlyDefined gather (called Harvest in their terminology) the data. + +## Use + +### Testing + +Run `./ClearlyDefined.ps1 -test`. + +If there is any missing data, the script should write verbose messages about the missing data and throw. +If there is no missing data, the script should not throw. + +### Harvesting + +Run `./ClearlyDefined.ps1 -Harvest`. +The script will trigger the harvest and output the result from ClearlyDefined. +**Give ClearlyDefined 24 hours to harvest the data.** +You can use the `-Test` switch without the `-Harvest` switch to test if Harvesting is done. + +## Caching + +If you run in the same PowerShell session, the script will be faster due to caching. + +The module will cache any results from ClearlyDefined that indicate the package is Harvested for 60 minutes. +No caching is done for packages that are not yet harvested. +To clear the cache, run with the `-ForceModuleReload` switch. diff --git a/tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1 b/tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1 new file mode 100644 index 0000000000..2a9434c9cb --- /dev/null +++ b/tools/clearlyDefined/src/ClearlyDefined/ClearlyDefined.psm1 @@ -0,0 +1,137 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Start the collection (known as harvest) of ClearlyDefined data for a package +function Start-ClearlyDefinedHarvest { + [CmdletBinding()] + param( + [Parameter(ValueFromPipelineByPropertyName=$true)] + [Alias('Type')] + [validateset('nuget')] + [string] + $PackageType = 'nuget', + + [parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)] + [Alias('Name')] + [string] + $PackageName, + + [parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)] + [Alias('Version')] + [Alias('Revision')] + [string] + $PackageVersion + ) + + Process { + $coordinates = Get-ClearlyDefinedCoordinates @PSBoundParameters + $body = @{tool='package';coordinates=$coordinates} | convertto-json + Write-Verbose $body -Verbose + (Invoke-WebRequest -Method Post -Uri 'https://api.clearlydefined.io/harvest' -Body $body -ContentType 'application/json').Content + } +} + +function ConvertFrom-ClearlyDefinedCoordinates { + [CmdletBinding()] + param( + [parameter(mandatory = $true, ValueFromPipeline = $true)] + [string] + $Coordinates + ) + + Begin {} + Process { + $parts = $Coordinates.Split('/') + [PSCustomObject]@{ + type = $parts[0] + provider = $parts[1] + namespace = $parts[2] + name = $parts[3] + revision = $parts[4] + } + } + End {} +} + +# Get the coordinate string for a package +Function Get-ClearlyDefinedCoordinates { + [CmdletBinding()] + param( + [validateset('nuget')] + [string] + $PackageType = 'nuget', + [parameter(mandatory = $true)] + [string] + $PackageName, + [parameter(mandatory = $true)] + [string] + $PackageVersion + ) + + return "$PackageType/$PackageType/-/$PackageName/$PackageVersion" +} + +# Cache of ClearlyDefined data +$cdCache = @{} + +# Get the ClearlyDefined data for a package +Function Get-ClearlyDefinedData { + [CmdletBinding()] + param( + [Parameter(ValueFromPipelineByPropertyName=$true)] + [Alias('Type')] + [validateset('nuget')] + [string] + $PackageType = 'nuget', + + [parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)] + [Alias('Name')] + [string] + $PackageName, + + [parameter(mandatory = $true, ValueFromPipelineByPropertyName=$true)] + [Alias('Revision')] + [string] + $PackageVersion + ) + + Begin { + $cacheMinutes = 60 + $cacheCutoff = (get-date).AddMinutes(-$cacheMinutes) + $coordinateList = @() + } + + Process { + $coordinateList += Get-ClearlyDefinedCoordinates @PSBoundParameters + } + + end { + $total = $coordinateList.Count + $completed = 0 + foreach($coordinates in $coordinateList) { + Write-Progress -Activity "Getting ClearlyDefined data" -Status "Getting data for $coordinates" -PercentComplete (($completed / $total) * 100) + $containsKey = $cdCache.ContainsKey($coordinates) + if ($containsKey -and $cdCache[$coordinates].cachedTime -gt $cacheCutoff) { + Write-Verbose "Returning cached data for $coordinates" + Write-Output $cdCache[$coordinates] + continue + } + + Invoke-RestMethod -Uri "https://api.clearlydefined.io/definitions/$coordinates" | ForEach-Object { + [bool] $harvested = if ($_.licensed.declared) { $true } else { $false } + Add-Member -NotePropertyName cachedTime -NotePropertyValue (get-date) -InputObject $_ -PassThru | Add-Member -NotePropertyName harvested -NotePropertyValue $harvested -PassThru + if ($_.harvested) { + Write-Verbose "Caching data for $coordinates" + $cdCache[$coordinates] = $_ + } + } + $completed++ + } + } +} + +Export-ModuleMember -Function @( + 'Start-ClearlyDefinedHarvest' + 'Get-ClearlyDefinedData' + 'ConvertFrom-ClearlyDefinedCoordinates' +)