Copy to static site instead of making blob public (#24269)

2024-11-23 01:34:19 +08:00 · 2024-09-25 12:49:26 -04:00 · 2024-09-25 12:49:26 -04:00 · ba493b6f1a
commit ba493b6f1a
parent b9c0127e83
3 changed files with 82 additions and 58 deletions
--- a/.pipelines/templates/release-MakeBlobPublic.yml
+++ b/.pipelines/templates/release-MakeBlobPublic.yml
@ -1,31 +1,22 @@
 jobs:
 - template: /.pipelines/templates/approvalJob.yml@self
  parameters:
-    displayName: Approve Blob Public
-    jobName: ApproveBlobPublic
+    displayName: Approve Copy release packages to PSInfra storage
+    jobName: CopyReleaseBlobApproval
    instructions: |
-      Are you sure you want to make the blob public?
+      Approval for Copy release packages to PSInfra storage

- job: blobPublic
-  displayName: Make Azure Blob Public
-  dependsOn: ApproveBlobPublic
-  condition: succeeded()
+- job: PSInfraReleaseBlobPublic
+  displayName: Copy release to PSInfra storage
+  dependsOn: CopyReleaseBlobApproval
  pool:
    type: windows
+
  variables:
-  - name: runCodesignValidationInjection
-    value: false
-  - name: NugetSecurityAnalysisWarningLevel
-    value: none
-  - name: DOTNET_SKIP_FIRST_TIME_EXPERIENCE
-    value: 1
+  - group: 'PSInfraStorage'
  - group: 'Azure Blob variable group'
  - name: ob_outputDirectory
    value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
-  - name: ob_sdl_codeSignValidation_enabled
-    value: false
-  - name: ob_sdl_binskim_enabled
-    value: false
  - name: ob_sdl_tsa_configFile
    value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
  - name: ob_sdl_credscan_suppressionsFile
@ -34,47 +25,81 @@ jobs:
    value: false

  steps:
-  - checkout: self
-    clean: true
-    env:
-      ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
+    - checkout: self
+      clean: true
+      env:
+        ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase

-  - template: /.pipelines/templates/SetVersionVariables.yml@self
-    parameters:
-      ReleaseTagVar: $(ReleaseTagVar)
-      CreateJson: yes
-      UseJson: no
+    - template: /.pipelines/templates/SetVersionVariables.yml@self
+      parameters:
+        ReleaseTagVar: $(ReleaseTagVar)
+        CreateJson: yes
+        UseJson: no

  - pwsh: |
      Get-ChildItem Env: | Out-String -width 9999 -Stream | write-Verbose -Verbose
    displayName: 'Capture Environment Variables'

-  - pwsh: |
-      $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose
-      if ($azureRmModule) {
-        Write-Host 'AzureRM module exists. Removing it'
-        Uninstall-AzureRm
-        Write-Host 'AzureRM module removed'
-      }
+    - pwsh: |
+        $azureRmModule = Get-InstalledModule AzureRM -ErrorAction SilentlyContinue -Verbose
+        if ($azureRmModule) {
+          Write-Host 'AzureRM module exists. Removing it'
+          Uninstall-AzureRm
+          Write-Host 'AzureRM module removed'
+        }

-      Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose
-    displayName: Remove AzRM modules
+        Install-Module -Name Az.Storage -Force -AllowClobber -Scope CurrentUser -Verbose
+      displayName: Remove AzRM modules
+
+    - task: AzurePowerShell@5
+      displayName: Copy blobs to PSInfra storage
+      inputs:
+        azureSubscription: az-blob-cicd-infra
+        scriptType: inlineScript
+        azurePowerShellVersion: LatestVersion
+        pwsh: true
+        inline: |
+          $sourceStorageAccountName = '$(StorageAccount)'
+          $destinationStorageAccountName = '$(PSInfraStorageAccount)'
+          $destinationContainerName = '$web'
+          $destinationPrefix = 'install/$(ReleaseTagVar)'
+
+          $sourceContext = New-AzStorageContext -StorageAccountName $sourceStorageAccountName
+          Write-Verbose -Verbose "Source context: $($sourceContext.BlobEndPoint)"
+
+          $destinationContext = New-AzStorageContext -StorageAccountName $destinationStorageAccountName
+          Write-Verbose -Verbose "Destination context: $($destinationContext.BlobEndPoint)"
+
+          foreach ($sourceContainerName in '$(AzureVersion)', '$(AzureVersion)-gc') {
+            $blobs = Get-AzStorageBlob -Context $sourceContext -Container $sourceContainerName
+
+            Write-Verbose -Verbose "Blobs found in $sourceContainerName"
+            $blobs.Name | Write-Verbose -Verbose
+
+            Write-Verbose -Verbose "Copying blobs from $sourceContainerName to $destinationContainerName/$destinationPrefix"
+
+            foreach ($blob in $blobs) {
+              $sourceBlobName = $blob.Name
+              Write-Verbose -Verbose "sourceBlobName = $sourceBlobName"
+
+              $destinationBlobName = "$destinationPrefix/$sourceBlobName"
+              Write-Verbose -Verbose "destinationBlobName = $destinationBlobName"
+              $existingBlob = Get-AzStorageBlob -Blob $destinationBlobName -Container $destinationContainerName -Context $destinationContext -ErrorAction Ignore
+              if ($existingBlob) {
+                Write-Verbose -Verbose "Blob $destinationBlobName already exists in '$destinationStorageAccountName/$destinationContainerName', removing before copy."
+                $existingBlob | Remove-AzStorageBlob -ErrorAction Stop -Verbose
+              }
+
+              Copy-AzStorageBlob -SourceContext $sourceContext -DestinationContext $destinationContext -SrcContainer $sourceContainerName -SrcBlob $sourceBlobName -DestContainer $destinationContainerName -DestBlob $destinationBlobName -Force -Verbose -Confirm:$false
+            }
+          }

-  - task: AzureCLI@2
-    displayName: 'Set blob permissions'
-    inputs:
-      azureSubscription: az-blob-cicd-infra
-      scriptType: 'pscore'
-      scriptLocation: 'inlineScript'
-      inlineScript: |
-        az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion) --public-access blob
-        az storage container set-permission --account-name $(StorageAccount) --name $(azureVersion)-gc --public-access blob

 - template: /.pipelines/templates/approvalJob.yml@self
  parameters:
    displayName: Approve Copy Global tool packages to PSInfra storage
    jobName: CopyBlobApproval
-    dependsOnJob: blobPublic
+    dependsOnJob: PSInfraReleaseBlobPublic
    instructions: |
      Approval for Copy global tool packages to PSInfra storage

--- a/.pipelines/templates/release-upload-buildinfo.yml
+++ b/.pipelines/templates/release-upload-buildinfo.yml
@ -45,14 +45,14 @@ jobs:
    displayName: Download build info artifact

  - pwsh: |
-      Import-Module '$(Build.SourcesDirectory)/tools/ci.psm1'
+      Import-Module '$(Build.SourcesDirectory)/PowerShell/tools/ci.psm1'
      $jsonFile = Get-Item "$ENV:PIPELINE_WORKSPACE/PSPackagesOfficial/BuildInfoJson/*.json"
      $fileName = Split-Path $jsonFile -Leaf

      $dateTime = [datetime]::UtcNow
      $dateTime = [datetime]::new($dateTime.Ticks - ($dateTime.Ticks % [timespan]::TicksPerSecond), $dateTime.Kind)

-      $metadata = Get-Content ./tools/metadata.json | ConvertFrom-Json
+      $metadata = Get-Content -LiteralPath '$(Build.SourcesDirectory)/PowerShell/tools/metadata.json' -ErrorAction Stop | ConvertFrom-Json
      $stableRelease = $metadata.StableRelease.Latest
      $ltsRelease = $metadata.LTSRelease.Latest

@ -118,29 +118,30 @@ jobs:
      azurePowerShellVersion: LatestVersion
      pwsh: true
      inline: |
-        $containerName = "buildinfo"
-        $storageAccount = '$(StorageAccount)'
+        $containerName = '$web'
+        $storageAccount = '$(PSInfraStorageAccount)'
+        $prefix = "buildinfo"

        $storageContext = New-AzStorageContext -StorageAccountName $storageAccount -UseConnectedAccount

        if ($env:CopyMainBuildInfo -eq 'YES') {
          $jsonFile = "$env:BuildInfoJsonFile"
          $blobName = Get-Item $jsonFile | Split-Path -Leaf
-          Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName"
-          Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force
+          Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$prefix/$blobName"
+          Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob "$prefix/$blobName" -Context $storageContext -Force
        }

        if ($env:CopyLTSBuildInfo -eq 'YES') {
          $jsonFile = "$env:LtsBuildInfoJsonFile"
          $blobName = Get-Item $jsonFile | Split-Path -Leaf
-          Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName"
-          Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force
+          Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$prefix/$blobName"
+          Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob "$prefix/$blobName" -Context $storageContext -Force
        }

        if ($env:CopyVersionBuildInfo -eq 'YES') {
          $jsonFile = "$env:VersionBuildInfoJsonFile"
          $blobName = Get-Item $jsonFile | Split-Path -Leaf
-          Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$blobName"
-          Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob $blobName -Context $storageContext -Force
+          Write-Verbose -Verbose "Uploading $jsonFile to $containerName/$prefix/$blobName"
+          Set-AzStorageBlobContent -File $jsonFile -Container $containerName -Blob "$prefix/$blobName" -Context $storageContext -Force
        }
    condition: and(succeeded(), eq(variables['CopyMainBuildInfo'], 'YES'))
--- a/tools/install-powershell.ps1
+++ b/tools/install-powershell.ps1
@ -268,7 +268,6 @@ try {
    if ($Daily) {
        $metadata = Invoke-RestMethod 'https://aka.ms/pwsh-buildinfo-daily'
        $release = $metadata.ReleaseTag -replace '^v'
-        $blobName = $metadata.BlobName

        # Get version from currently installed PowerShell Daily if available.
        $pwshPath = if ($IsWinEnv) {Join-Path $Destination "pwsh.exe"} else {Join-Path $Destination "pwsh"}
@ -297,8 +296,7 @@ try {
            throw "The OS architecture is '$architecture'. However, we currently only support daily package for x64."
        }

-
-        $downloadURL = "https://pscoretestdata.blob.core.windows.net/${blobName}/${packageName}"
+        $downloadURL = "https://powershellinfraartifacts-gkhedzdeaghdezhr.z01.azurefd.net/install/$($metadata.ReleaseTag)/$packageName"
        Write-Verbose "About to download package from '$downloadURL'" -Verbose

        $packagePath = Join-Path -Path $tempDir -ChildPath $packageName