Implement Windows release builds in Azure Pipelines (GH-14065)

.azure-pipelines/windows-release.yml

@@ -0,0 +1,96 @@
+name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+# QUEUE TIME VARIABLES
+# variables:
+#   GitRemote: python
+#   SourceTag:
+#   DoPGO: true
+#   SigningCertificate: 'Python Software Foundation'
+#   SigningDescription: 'Built: $(Build.BuildNumber)'
+#   DoLayout: true
+#   DoMSIX: true
+#   DoNuget: true
+#   DoEmbed: true
+#   DoMSI: true
+#   DoPublish: false
+
+trigger: none
+pr: none
+
+stages:
+- stage: Build
+  displayName: Build binaries
+  jobs:
+  - template: windows-release/stage-build.yml
+
+- stage: Sign
+  displayName: Sign binaries
+  dependsOn: Build
+  jobs:
+  - template: windows-release/stage-sign.yml
+
+- stage: Layout
+  displayName: Generate layouts
+  dependsOn: Sign
+  jobs:
+  - template: windows-release/stage-layout-full.yml
+  - template: windows-release/stage-layout-embed.yml
+  - template: windows-release/stage-layout-nuget.yml
+
+- stage: Pack
+  dependsOn: Layout
+  jobs:
+  - template: windows-release/stage-pack-nuget.yml
+
+- stage: Test
+  dependsOn: Pack
+  jobs:
+  - template: windows-release/stage-test-embed.yml
+  - template: windows-release/stage-test-nuget.yml
+
+- stage: Layout_MSIX
+  displayName: Generate MSIX layouts
+  dependsOn: Sign
+  condition: and(succeeded(), eq(variables['DoMSIX'], 'true'))
+  jobs:
+  - template: windows-release/stage-layout-msix.yml
+
+- stage: Pack_MSIX
+  displayName: Package MSIX
+  dependsOn: Layout_MSIX
+  jobs:
+  - template: windows-release/stage-pack-msix.yml
+
+- stage: Build_MSI
+  displayName: Build MSI installer
+  dependsOn: Sign
+  condition: and(succeeded(), eq(variables['DoMSI'], 'true'))
+  jobs:
+  - template: windows-release/stage-msi.yml
+
+- stage: Test_MSI
+  displayName: Test MSI installer
+  dependsOn: Build_MSI
+  jobs:
+  - template: windows-release/stage-test-msi.yml
+
+- stage: PublishPyDotOrg
+  displayName: Publish to python.org
+  dependsOn: ['Test_MSI', 'Test']
+  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  jobs:
+  - template: windows-release/stage-publish-pythonorg.yml
+
+- stage: PublishNuget
+  displayName: Publish to nuget.org
+  dependsOn: Test
+  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  jobs:
+  - template: windows-release/stage-publish-nugetorg.yml
+
+- stage: PublishStore
+  displayName: Publish to Store
+  dependsOn: Pack_MSIX
+  condition: and(succeeded(), eq(variables['DoPublish'], 'true'))
+  jobs:
+  - template: windows-release/stage-publish-store.yml

.azure-pipelines/windows-release/build-steps.yml

@@ -0,0 +1,83 @@
+parameters:
+  ShouldPGO: false
+
+steps:
+- template: ./checkout.yml
+
+- powershell: |
+    $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+    Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)"
+    Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)"
+    Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)"
+    Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)"
+    Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)"
+    Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)"
+  displayName: 'Extract version numbers'
+
+- ${{ if eq(parameters.ShouldPGO, 'false') }}:
+  - powershell: |
+      $env:SigningCertificate = $null
+      .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration)
+    displayName: 'Run build'
+    env:
+      IncludeUwp: true
+      Py_OutDir: '$(Build.BinariesDirectory)\bin'
+
+- ${{ if eq(parameters.ShouldPGO, 'true') }}:
+  - powershell: |
+      $env:SigningCertificate = $null
+      .\PCbuild\build.bat -v -p $(Platform) --pgo
+    displayName: 'Run build with PGO'
+    env:
+      IncludeUwp: true
+      Py_OutDir: '$(Build.BinariesDirectory)\bin'
+
+- powershell: |
+    $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
+    $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1)
+    if (-not $tool) {
+      throw "SDK is not available"
+    }
+    Write-Host "##vso[task.prependpath]$($tool.Directory)"
+  displayName: 'Add WinSDK tools to path'
+
+- powershell: |
+    $env:SigningCertificate = $null
+    .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default
+    makecat "${env:CAT}.cdf"
+    del "${env:CAT}.cdf"
+    if (-not (Test-Path "${env:CAT}.cat")) {
+      throw "Failed to build catalog file"
+    }
+  displayName: 'Generate catalog'
+  env:
+    CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish binaries'
+  condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])))
+  inputs:
+    PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)'
+    ArtifactName: bin_$(Name)
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish binaries for signing'
+  condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))
+  inputs:
+    PathtoPublish: '$(Build.BinariesDirectory)\bin\$(Arch)'
+    ArtifactName: unsigned_bin_$(Name)
+
+- task: CopyFiles@2
+  displayName: 'Layout Artifact: symbols'
+  inputs:
+    sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch)
+    targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name)
+    flatten: true
+    contents: |
+      **\*.pdb
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish Artifact: symbols'
+  inputs:
+    PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols'
+    ArtifactName: symbols

.azure-pipelines/windows-release/checkout.yml

@@ -0,0 +1,21 @@
+parameters:
+  depth: 3
+
+steps:
+- checkout: none
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git .
+  displayName: 'git clone ($(GitRemote)/$(SourceTag))'
+  condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag']))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) .
+  displayName: 'git clone (<default>/$(SourceTag))'
+  condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag']))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git .
+  displayName: 'git clone ($(GitRemote)/<default>)'
+  condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag'])))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) .
+  displayName: 'git clone'
+  condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag'])))

.azure-pipelines/windows-release/find-sdk.yml

@@ -0,0 +1,17 @@
+# Locate the Windows SDK and add its binaries directory to PATH
+#
+# `toolname` can be overridden to use a different marker file.
+
+parameters:
+  toolname: signtool.exe
+
+steps:
+  - powershell: |
+      $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
+      $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1)
+      if (-not $tool) {
+          throw "SDK is not available"
+      }
+      Write-Host "##vso[task.prependpath]$($tool.Directory)"
+      Write-Host "Adding $($tool.Directory) to PATH"
+    displayName: 'Add WinSDK tools to path'

.azure-pipelines/windows-release/layout-command.yml

@@ -0,0 +1,20 @@
+steps:
+- powershell: >
+    Write-Host (
+    '##vso[task.setvariable variable=LayoutCmd]&
+    "{0}"
+    "{1}\PC\layout"
+    -vv
+    --source "{1}"
+    --build "{2}"
+    --temp "{3}"
+    --include-cat "{2}\python.cat"
+    --doc-build "{4}"'
+    -f (
+    "$(PYTHON)",
+    "$(Build.SourcesDirectory)",
+    (Split-Path -Parent "$(PYTHON)"),
+    "$(Build.BinariesDirectory)\layout-temp",
+    "$(Build.BinariesDirectory)\doc"
+    ))
+  displayName: 'Set LayoutCmd'

.azure-pipelines/windows-release/mingw-lib.yml

@@ -0,0 +1,13 @@
+parameters:
+  DllToolOpt: -m i386:x86-64
+  #DllToolOpt: -m i386 --as-flags=--32
+
+steps:
+- powershell: |
+    git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils"
+    gci "bin\$(Arch)\python*.dll" | %{
+      & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def
+      & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }}
+    }
+  displayName: 'Generate MinGW import library'
+  workingDirectory: $(Build.BinariesDirectory)

.azure-pipelines/windows-release/msi-steps.yml

@@ -0,0 +1,142 @@
+steps:
+  - template: ./checkout.yml
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: doc'
+    inputs:
+      artifactName: doc
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: CopyFiles@2
+    displayName: 'Merge documentation files'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\doc
+      targetFolder: $(Build.SourcesDirectory)\Doc\build
+      contents: |
+        htmlhelp\*.chm
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_win32'
+    inputs:
+      artifactName: bin_win32
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_win32_d'
+    inputs:
+      artifactName: bin_win32_d
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: CopyFiles@2
+    displayName: 'Merge win32 debug files'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\bin_win32_d
+      targetFolder: $(Build.BinariesDirectory)\bin_win32
+      contents: |
+        **\*_d.*
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_amd64'
+    inputs:
+      artifactName: bin_amd64
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_amd64_d'
+    inputs:
+      artifactName: bin_amd64_d
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: CopyFiles@2
+    displayName: 'Merge amd64 debug files'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\bin_amd64_d
+      targetFolder: $(Build.BinariesDirectory)\bin_amd64
+      contents: |
+        **\*_d.*
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: tcltk_lib_win32'
+    inputs:
+      artifactName: tcltk_lib_win32
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: tcltk_lib_amd64'
+    inputs:
+      artifactName: tcltk_lib_amd64
+      downloadPath: $(Build.BinariesDirectory)
+
+  - script: |
+      ren bin_win32 win32
+      ren bin_amd64 amd64
+    displayName: 'Correct artifact directory names'
+    workingDirectory: $(Build.BinariesDirectory)
+
+  - script: |
+      call Tools\msi\get_externals.bat
+      call PCbuild\find_python.bat
+      echo ##vso[task.setvariable variable=PYTHON]%PYTHON%
+      call PCbuild/find_msbuild.bat
+      echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD%
+    displayName: 'Get external dependencies'
+
+  - script: |
+      %PYTHON% -m pip install blurb
+      %PYTHON% -m blurb merge -f Misc\NEWS
+    displayName: 'Merge NEWS file'
+
+  - script: |
+      %MSBUILD% Tools\msi\launcher\launcher.wixproj
+    displayName: 'Build launcher installer'
+    env:
+      Platform: x86
+      Py_OutDir: $(Build.BinariesDirectory)
+
+  - script: |
+      %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true /p:BuildForRelease=true
+      %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false /p:BuildForRelease=true
+    displayName: 'Build win32 installer'
+    env:
+      Platform: x86
+      Py_OutDir: $(Build.BinariesDirectory)
+      PYTHON: $(Build.BinariesDirectory)\win32\python.exe
+      PYTHONHOME: $(Build.SourcesDirectory)
+      TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32
+
+  - script: |
+      %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true /p:BuildForRelease=true
+      %MSBUILD% Tools\msi\bundle\releaseweb.wixproj /t:Rebuild /p:RebuildAll=false /p:BuildForRelease=true
+    displayName: 'Build amd64 installer'
+    env:
+      Platform: x64
+      Py_OutDir: $(Build.BinariesDirectory)
+      PYTHON: $(Build.BinariesDirectory)\amd64\python.exe
+      PYTHONHOME: $(Build.SourcesDirectory)
+      TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64
+
+  - task: CopyFiles@2
+    displayName: 'Assemble artifact: msi (1/2)'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\win32\en-us
+      targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32
+      contents: |
+        *.msi
+        *.cab
+        *.exe
+
+  - task: CopyFiles@2
+    displayName: 'Assemble artifact: msi (2/2)'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\amd64\en-us
+      targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64
+      contents: |
+        *.msi
+        *.cab
+        *.exe
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish MSI'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\msi'
+      ArtifactName: msi

.azure-pipelines/windows-release/stage-build.yml

@@ -0,0 +1,157 @@
+jobs:
+- job: Build_Docs
+  displayName: Docs build
+  pool:
+    name: 'Windows Release'
+    #vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  steps:
+  - template: ./checkout.yml
+
+  - script: Doc\make.bat html
+    displayName: 'Build HTML docs'
+    env:
+      BUILDDIR: $(Build.BinariesDirectory)\Doc
+
+  #- powershell: iwr "https://www.python.org/ftp/python/3.7.3/python373.chm" -OutFile "$(Build.BinariesDirectory)\python390a0.chm"
+  #  displayName: 'Cheat at building CHM docs'
+
+  - script: Doc\make.bat htmlhelp
+    displayName: 'Build CHM docs'
+    env:
+      BUILDDIR: $(Build.BinariesDirectory)\Doc
+
+  - task: CopyFiles@2
+    displayName: 'Assemble artifact: Doc'
+    inputs:
+      sourceFolder: $(Build.BinariesDirectory)\Doc
+      targetFolder: $(Build.ArtifactStagingDirectory)\Doc
+      contents: |
+        html\**\*
+        htmlhelp\*.chm
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish artifact: doc'
+    inputs:
+      PathtoPublish: $(Build.ArtifactStagingDirectory)\Doc
+      ArtifactName: doc
+
+- job: Build_Python
+  displayName: Python build
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      win32:
+        Name: win32
+        Arch: win32
+        Platform: x86
+        Configuration: Release
+      win32_d:
+        Name: win32_d
+        Arch: win32
+        Platform: x86
+        Configuration: Debug
+      amd64_d:
+        Name: amd64_d
+        Arch: amd64
+        Platform: x64
+        Configuration: Debug
+
+  steps:
+    - template: ./build-steps.yml
+
+- job: Build_Python_NonPGO
+  displayName: Python non-PGO build
+  condition: and(succeeded(), ne(variables['DoPGO'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      amd64:
+        Name: amd64
+        Arch: amd64
+        Platform: x64
+        Configuration: Release
+
+  steps:
+    - template: ./build-steps.yml
+
+
+- job: Build_Python_PGO
+  displayName: Python PGO build
+  condition: and(succeeded(), eq(variables['DoPGO'], 'true'))
+
+  pool:
+    name: 'Windows Release'
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      amd64:
+        Name: amd64
+        Arch: amd64
+        Platform: x64
+        Configuration: Release
+
+  steps:
+    - template: ./build-steps.yml
+      parameters:
+        ShouldPGO: true
+
+
+- job: TclTk_Lib
+  displayName: Publish Tcl/Tk Library
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  steps:
+  - template: ./checkout.yml
+
+  - script: PCbuild\get_externals.bat --no-openssl --no-libffi
+    displayName: 'Get external dependencies'
+
+  - task: MSBuild@1
+    displayName: 'Copy Tcl/Tk lib for publish'
+    inputs:
+      solution: PCbuild\tcltk.props
+      platform: x86
+      msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_win32"
+
+  - task: MSBuild@1
+    displayName: 'Copy Tcl/Tk lib for publish'
+    inputs:
+      solution: PCbuild\tcltk.props
+      platform: x64
+      msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64"
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish artifact: tcltk_lib_win32'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\tcl_win32'
+      ArtifactName: tcltk_lib_win32
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish artifact: tcltk_lib_amd64'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\tcl_amd64'
+      ArtifactName: tcltk_lib_amd64

.azure-pipelines/windows-release/stage-layout-embed.yml

@@ -0,0 +1,56 @@
+jobs:
+- job: Make_Embed_Layout
+  displayName: Make embeddable layout
+  condition: and(succeeded(), eq(variables['DoEmbed'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      win32:
+        Name: win32
+        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+      amd64:
+        Name: amd64
+        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+
+  steps:
+  - template: ./checkout.yml
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_$(Name)'
+    inputs:
+      artifactName: bin_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - template: ./layout-command.yml
+
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)"
+    displayName: 'Extract version numbers'
+
+  - powershell: >
+      $(LayoutCmd)
+      --copy "$(Build.ArtifactStagingDirectory)\layout"
+      --zip "$(Build.ArtifactStagingDirectory)\embed\$(VersionText)-embed-$(Name).zip"
+      --preset-embed
+    displayName: 'Generate embeddable layout'
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: layout_embed_$(Name)'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\layout'
+      ArtifactName: layout_embed_$(Name)
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: embed'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed'
+      ArtifactName: embed

.azure-pipelines/windows-release/stage-layout-full.yml

@@ -0,0 +1,62 @@
+jobs:
+- job: Make_Layouts
+  displayName: Make layouts
+  condition: and(succeeded(), eq(variables['DoLayout'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      win32:
+        Name: win32
+        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+      amd64:
+        Name: amd64
+        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+
+  steps:
+  - template: ./checkout.yml
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_$(Name)'
+    inputs:
+      artifactName: bin_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_$(Name)_d'
+    inputs:
+      artifactName: bin_$(Name)_d
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: doc'
+    inputs:
+      artifactName: doc
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: tcltk_lib_$(Name)'
+    inputs:
+      artifactName: tcltk_lib_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - template: ./layout-command.yml
+
+  - powershell: |
+      $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default
+    displayName: 'Generate full layout'
+    env:
+      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: layout_full_$(Name)'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\layout'
+      ArtifactName: layout_full_$(Name)

.azure-pipelines/windows-release/stage-layout-msix.yml

@@ -0,0 +1,86 @@
+jobs:
+- job: Make_MSIX_Layout
+  displayName: Make MSIX layout
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      #win32:
+      #  Name: win32
+      #  Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+      #  PYTHONHOME: $(Build.SourcesDirectory)
+      amd64:
+        Name: amd64
+        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+
+  steps:
+  - template: ./checkout.yml
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_$(Name)'
+    inputs:
+      artifactName: bin_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_$(Name)_d'
+    inputs:
+      artifactName: bin_$(Name)_d
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: tcltk_lib_$(Name)'
+    inputs:
+      artifactName: tcltk_lib_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - template: ./layout-command.yml
+
+  - powershell: |
+     Remove-Item "$(Build.ArtifactStagingDirectory)\appx-store" -Recurse -Force -EA 0
+      $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile
+    displayName: 'Generate store APPX layout'
+    env:
+      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: layout_appxstore_$(Name)'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\appx-store'
+      ArtifactName: layout_appxstore_$(Name)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: cert'
+    condition: and(succeeded(), variables['SigningCertificate'])
+    inputs:
+      artifactName: cert
+      downloadPath: $(Build.BinariesDirectory)
+
+  - powershell: |
+      $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON)
+      Write-Host "Side-loadable APPX must be signed with '$($info.Subject)'"
+      Write-Host "##vso[task.setvariable variable=APPX_DATA_PUBLISHER]$($info.Subject)"
+      Write-Host "##vso[task.setvariable variable=APPX_DATA_SHA256]$($info.SHA256)"
+    displayName: 'Override signing parameters'
+    condition: and(succeeded(), variables['SigningCertificate'])
+
+  - powershell: |
+      Remove-Item "$(Build.ArtifactStagingDirectory)\appx" -Recurse -Force -EA 0
+      $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests
+    displayName: 'Generate sideloading APPX layout'
+    env:
+      TCL_LIBRARY: $(Build.BinariesDirectory)\tcltk_lib_$(Name)\tcl8
+      APPX_DATA_PUBLISHER: $(APPX_DATA_PUBLISHER)
+      APPX_DATA_SHA256: $(APPX_DATA_SHA256)
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: layout_appx_$(Name)'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\appx'
+      ArtifactName: layout_appx_$(Name)

.azure-pipelines/windows-release/stage-layout-nuget.yml

@@ -0,0 +1,44 @@
+jobs:
+- job: Make_Nuget_Layout
+  displayName: Make Nuget layout
+  condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      win32:
+        Name: win32
+        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+      amd64:
+        Name: amd64
+        Python: $(Build.BinariesDirectory)\bin_$(Name)\python.exe
+        PYTHONHOME: $(Build.SourcesDirectory)
+
+  steps:
+  - template: ./checkout.yml
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: bin_$(Name)'
+    inputs:
+      artifactName: bin_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - template: ./layout-command.yml
+
+  - powershell: |
+      $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget
+    displayName: 'Generate nuget layout'
+    env:
+      TCL_LIBRARY: $(Build.BinariesDirectory)\bin_$(Name)\tcl\tcl8
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: layout_nuget_$(Name)'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\nuget'
+      ArtifactName: layout_nuget_$(Name)

.azure-pipelines/windows-release/stage-msi.yml

@@ -0,0 +1,36 @@
+jobs:
+- job: Make_MSI
+  displayName: Make MSI
+  condition: and(succeeded(), not(variables['SigningCertificate']))
+
+  pool:
+    vmName: win2016-vs2017
+
+  variables:
+    ReleaseUri: http://www.python.org/{arch}
+    DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi}
+    Py_OutDir: $(Build.BinariesDirectory)
+
+  workspace:
+    clean: all
+
+  steps:
+  - template: msi-steps.yml
+
+- job: Make_Signed_MSI
+  displayName: Make signed MSI
+  condition: and(succeeded(), variables['SigningCertificate'])
+
+  pool:
+    name: 'Windows Release'
+
+  variables:
+    ReleaseUri: http://www.python.org/{arch}
+    DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi}
+    Py_OutDir: $(Build.BinariesDirectory)
+
+  workspace:
+    clean: all
+
+  steps:
+  - template: msi-steps.yml

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -0,0 +1,127 @@
+jobs:
+- job: Pack_MSIX
+  displayName: Pack MSIX bundles
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      amd64:
+        Name: amd64
+        Artifact: appx
+        Suffix:
+        ShouldSign: true
+      amd64_store:
+        Name: amd64
+        Artifact: appxstore
+        Suffix: -store
+        Upload: true
+
+  steps:
+  - template: ./checkout.yml
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: layout_$(Artifact)_$(Name)'
+    inputs:
+      artifactName: layout_$(Artifact)_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: symbols'
+    inputs:
+      artifactName: symbols
+      downloadPath: $(Build.BinariesDirectory)
+
+  - powershell: |
+      $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+      Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)"
+      Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)"
+      Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)"
+      Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)"
+      Write-Host "##vso[task.setvariable variable=Filename]python-$($d.PythonVersion)-$(Name)$(Suffix)"
+    displayName: 'Extract version numbers'
+
+  - powershell: |
+      ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout_$(Artifact)_$(Name)" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix"
+    displayName: 'Build msix'
+
+  - powershell: |
+      7z a -tzip "$(Build.ArtifactStagingDirectory)\msix\$(Filename).appxsym" *.pdb
+    displayName: 'Build appxsym'
+    workingDirectory: $(Build.BinariesDirectory)\symbols\$(Name)
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: MSIX'
+    condition: and(succeeded(), or(ne(variables['ShouldSign'], 'true'), not(variables['SigningCertificate'])))
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix'
+      ArtifactName: msix
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: MSIX'
+    condition: and(succeeded(), and(eq(variables['ShouldSign'], 'true'), variables['SigningCertificate']))
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix'
+      ArtifactName: unsigned_msix
+
+  - powershell: |
+      7z a -tzip "$(Build.ArtifactStagingDirectory)\msixupload\$(Filename).msixupload" *
+    displayName: 'Build msixupload'
+    condition: and(succeeded(), eq(variables['Upload'], 'true'))
+    workingDirectory: $(Build.ArtifactStagingDirectory)\msix
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: MSIXUpload'
+    condition: and(succeeded(), eq(variables['Upload'], 'true'))
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\msixupload'
+      ArtifactName: msixupload
+
+
+- job: Sign_MSIX
+  displayName: Sign side-loadable MSIX bundles
+  dependsOn:
+  - Pack_MSIX
+  condition: and(succeeded(), variables['SigningCertificate'])
+
+  pool:
+    name: 'Windows Release'
+
+  workspace:
+    clean: all
+
+  steps:
+  - checkout: none
+  - template: ./find-sdk.yml
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download Artifact: unsigned_msix'
+    inputs:
+      artifactName: unsigned_msix
+      downloadPath: $(Build.BinariesDirectory)
+
+  - powershell: |
+      $failed = $true
+      foreach ($retry in 1..3) {
+          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix)
+          if ($?) {
+              $failed = $false
+              break
+          }
+          sleep 1
+      }
+      if ($failed) {
+          throw "Failed to sign MSIX"
+      }
+    displayName: 'Sign MSIX'
+    workingDirectory: $(Build.BinariesDirectory)\unsigned_msix
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: MSIX'
+    inputs:
+      PathtoPublish: '$(Build.BinariesDirectory)\unsigned_msix'
+      ArtifactName: msix

.azure-pipelines/windows-release/stage-pack-nuget.yml

@@ -0,0 +1,41 @@
+jobs:
+- job: Pack_Nuget
+  displayName: Pack Nuget bundles
+  condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      amd64:
+        Name: amd64
+      win32:
+        Name: win32
+
+  steps:
+  - checkout: none
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: layout_nuget_$(Name)'
+    inputs:
+      artifactName: layout_nuget_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: NugetToolInstaller@0
+    displayName: 'Install Nuget'
+    inputs:
+      versionSpec: '>=5.0'
+
+  - powershell: |
+      nuget pack "$(Build.BinariesDirectory)\layout_nuget_$(Name)\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive
+    displayName: 'Create nuget package'
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: nuget'
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)'
+      ArtifactName: nuget

.azure-pipelines/windows-release/stage-publish-nugetorg.yml

@@ -0,0 +1,28 @@
+jobs:
+- job: Publish_Nuget
+  displayName: Publish Nuget packages
+  condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  steps:
+  - checkout: none
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: nuget'
+    inputs:
+      artifactName: nuget
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: NuGetCommand@2
+    displayName: Push packages
+    condition: and(succeeded(), eq(variables['SigningCertificate'], 'Python Software Foundation'))
+    inputs:
+      command: push
+      packagesToPush: $(Build.BinariesDirectory)\nuget\*.nupkg'
+      nuGetFeedType: external
+      publishFeedCredentials: 'Python on Nuget'

.azure-pipelines/windows-release/stage-publish-pythonorg.yml

@@ -0,0 +1,34 @@
+jobs:
+- job: Publish_Python
+  displayName: Publish python.org packages
+  condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true')))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  steps:
+  - checkout: none
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: Doc'
+    inputs:
+      artifactName: Doc
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: msi'
+    inputs:
+      artifactName: msi
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: embed'
+    inputs:
+      artifactName: embed
+      downloadPath: $(Build.BinariesDirectory)
+
+  # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation')
+  # If we are not real-signed, DO NOT PUBLISH

.azure-pipelines/windows-release/stage-publish-store.yml

@@ -0,0 +1,22 @@
+jobs:
+- job: Publish_Store
+  displayName: Publish Store packages
+  condition: and(succeeded(), eq(variables['DoMSIX'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  steps:
+  - checkout: none
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: msixupload'
+    inputs:
+      artifactName: msixupload
+      downloadPath: $(Build.BinariesDirectory)
+
+  # TODO: eq(variables['SigningCertificate'], 'Python Software Foundation')
+  # If we are not real-signed, DO NOT PUBLISH

.azure-pipelines/windows-release/stage-sign.yml

@@ -0,0 +1,113 @@
+jobs:
+- job: Sign_Python
+  displayName: Sign Python binaries
+  condition: and(succeeded(), variables['SigningCertificate'])
+
+  pool:
+    name: 'Windows Release'
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      win32:
+        Name: win32
+      amd64:
+        Name: amd64
+
+  steps:
+  - checkout: none
+  - template: ./find-sdk.yml
+
+  - powershell: |
+      Write-Host "##vso[build.addbuildtag]signed"
+    displayName: 'Add build tags'
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: unsigned_bin_$(Name)'
+    inputs:
+      artifactName: unsigned_bin_$(Name)
+      downloadPath: $(Build.BinariesDirectory)
+
+  - powershell: |
+      $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*)
+      signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files
+    displayName: 'Sign binaries'
+    workingDirectory: $(Build.BinariesDirectory)\unsigned_bin_$(Name)
+
+  - powershell: |
+      $files = (gi *.exe, *.dll, *.pyd, *.cat -Exclude vcruntime*, libffi*, libcrypto*, libssl*)
+      $failed = $true
+      foreach ($retry in 1..10) {
+          signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files
+          if ($?) {
+              $failed = $false
+              break
+          }
+          sleep 5
+      }
+      if ($failed) {
+          Write-Host "##vso[task.logissue type=error]Failed to timestamp files"
+      }
+    displayName: 'Timestamp binaries'
+    workingDirectory: $(Build.BinariesDirectory)\unsigned_bin_$(Name)
+    continueOnError: true
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish artifact: bin_$(Name)'
+    inputs:
+      PathtoPublish: '$(Build.BinariesDirectory)\unsigned_bin_$(Name)'
+      ArtifactName: bin_$(Name)
+
+
+- job: Dump_CertInfo
+  displayName: Capture certificate info
+  condition: and(succeeded(), variables['SigningCertificate'])
+
+  pool:
+    name: 'Windows Release'
+
+  steps:
+  - checkout: none
+
+  - powershell: |
+      $m = 'CN=$(SigningCertificate)'
+      $c = ((gci Cert:\CurrentUser\My), (gci Cert:\LocalMachine\My)) | %{ $_ } | `
+         ?{ $_.Subject -match $m } | `
+         select -First 1
+      if (-not $c) {
+          Write-Host "Failed to find certificate for $(SigningCertificate)"
+          exit
+      }
+      $d = mkdir "$(Build.BinariesDirectory)\tmp" -Force
+      $cf = "$d\cert.cer"
+      [IO.File]::WriteAllBytes($cf, $c.Export("Cer"))
+      $csha = (certutil -dump $cf | sls "Cert Hash\(sha256\): (.+)").Matches.Groups[1].Value
+
+      $info = @{ Subject=$c.Subject; SHA256=$csha; }
+
+      $d = mkdir "$(Build.BinariesDirectory)\cert" -Force
+      $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json"
+    displayName: "Extract certificate info"
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish artifact: cert'
+    inputs:
+      PathtoPublish: '$(Build.BinariesDirectory)\cert'
+      ArtifactName: cert
+
+
+- job: Mark_Unsigned
+  displayName: Tag unsigned build
+  condition: and(succeeded(), not(variables['SigningCertificate']))
+
+  pool:
+    vmName: win2016-vs2017
+
+  steps:
+  - checkout: none
+
+  - powershell: |
+      Write-Host "##vso[build.addbuildtag]unsigned"
+    displayName: 'Add build tag'

.azure-pipelines/windows-release/stage-test-embed.yml

@@ -0,0 +1,40 @@
+jobs:
+- job: Test_Embed
+  displayName: Test Embed
+  condition: and(succeeded(), eq(variables['DoEmbed'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      win32:
+        Name: win32
+      amd64:
+        Name: amd64
+
+  steps:
+  - checkout: none
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: embed'
+    inputs:
+      artifactName: embed
+      downloadPath: $(Build.BinariesDirectory)
+
+  - powershell: |
+      Expand-Archive -Path "$(Build.BinariesDirectory)\embed\embed-$(Name).zip" -DestinationPath "$(Build.BinariesDirectory)\Python"
+      $p = gi "$(Build.BinariesDirectory)\Python\python.exe"
+      Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)"
+    displayName: 'Install Python and add to PATH'
+
+  - script: |
+      python -c "import sys; print(sys.version)"
+    displayName: 'Collect version number'
+
+  - script: |
+      python -m site
+    displayName: 'Collect site'

.azure-pipelines/windows-release/stage-test-msi.yml

@@ -0,0 +1,108 @@
+jobs:
+- job: Test_MSI
+  displayName: Test MSI
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      win32_User:
+        ExeMatch: 'python-[\dabrc.]+-webinstall\.exe'
+        Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User
+        InstallAllUsers: 0
+      win32_Machine:
+        ExeMatch: 'python-[\dabrc.]+-webinstall\.exe'
+        Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine
+        InstallAllUsers: 1
+      amd64_User:
+        ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe'
+        Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User
+        InstallAllUsers: 0
+      amd64_Machine:
+        ExeMatch: 'python-[\dabrc.]+-amd64-webinstall\.exe'
+        Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine
+        InstallAllUsers: 1
+
+  steps:
+  - checkout: none
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: msi'
+    inputs:
+      artifactName: msi
+      downloadPath: $(Build.BinariesDirectory)
+
+  - powershell: |
+      $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1)
+      Write-Host "##vso[task.setvariable variable=SetupExe]$($p.FullName)"
+      Write-Host "##vso[task.setvariable variable=SetupExeName]$($p.Name)"
+    displayName: 'Find installer executable'
+    workingDirectory: $(Build.BinariesDirectory)\msi
+
+  - script: >
+      "$(SetupExe)"
+      /passive
+      /log "$(Logs)\install\log.txt"
+      TargetDir="$(Build.BinariesDirectory)\Python"
+      Include_debug=1
+      Include_symbols=1
+      InstallAllUsers=$(InstallAllUsers)
+    displayName: 'Install Python'
+
+  - powershell: |
+      $p = gi "$(Build.BinariesDirectory)\Python\python.exe"
+      Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)"
+    displayName: 'Add test Python to PATH'
+
+  - script: |
+      python -c "import sys; print(sys.version)"
+    displayName: 'Collect version number'
+
+  - script: |
+      python -m site
+    displayName: 'Collect site'
+
+  - powershell: |
+      gci -r "${env:PROGRAMDATA}\Microsoft\Windows\Start Menu\Programs\Python*"
+    displayName: 'Capture per-machine Start Menu items'
+  - powershell: |
+      gci -r "${env:APPDATA}\Microsoft\Windows\Start Menu\Programs\Python*"
+    displayName: 'Capture per-user Start Menu items'
+
+  - powershell: |
+      gci -r "HKLM:\Software\WOW6432Node\Python"
+    displayName: 'Capture per-machine 32-bit registry'
+  - powershell: |
+      gci -r "HKLM:\Software\Python"
+    displayName: 'Capture per-machine native registry'
+  - powershell: |
+      gci -r "HKCU:\Software\Python"
+    displayName: 'Capture current-user registry'
+
+  - script: |
+      python -m pip install "azure<0.10"
+      python -m pip uninstall -y azure python-dateutil six
+    displayName: 'Test (un)install package'
+
+  - script: |
+      python -m test -uall -v test_ttk_guionly test_tk test_idle
+    displayName: 'Test Tkinter and Idle'
+
+  - script: >
+      "$(SetupExe)"
+      /passive
+      /uninstall
+      /log "$(Logs)\uninstall\log.txt"
+    displayName: 'Uninstall Python'
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: logs'
+    condition: true
+    continueOnError: true
+    inputs:
+      PathtoPublish: '$(Build.ArtifactStagingDirectory)\logs'
+      ArtifactName: msi_testlogs

.azure-pipelines/windows-release/stage-test-nuget.yml

@@ -0,0 +1,58 @@
+jobs:
+- job: Test_Nuget
+  displayName: Test Nuget
+  condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
+
+  pool:
+    vmName: win2016-vs2017
+
+  workspace:
+    clean: all
+
+  strategy:
+    matrix:
+      win32:
+        Package: pythonx86
+      amd64:
+        Package: python
+
+  steps:
+  - checkout: none
+
+  - task: DownloadBuildArtifacts@0
+    displayName: 'Download artifact: nuget'
+    inputs:
+      artifactName: nuget
+      downloadPath: $(Build.BinariesDirectory)
+
+  - task: NugetToolInstaller@0
+    inputs:
+      versionSpec: '>= 5'
+
+  - powershell: >
+      nuget install
+      $(Package)
+      -Source "$(Build.BinariesDirectory)\nuget"
+      -OutputDirectory "$(Build.BinariesDirectory)\install"
+      -Prerelease
+      -ExcludeVersion
+      -NonInteractive
+    displayName: 'Install Python'
+
+  - powershell: |
+      $p = gi "$(Build.BinariesDirectory)\install\$(Package)\tools\python.exe"
+      Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)"
+    displayName: 'Add test Python to PATH'
+
+  - script: |
+      python -c "import sys; print(sys.version)"
+    displayName: 'Collect version number'
+
+  - script: |
+      python -m site
+    displayName: 'Collect site'
+
+  - script: |
+      python -m pip install "azure<0.10"
+      python -m pip uninstall -y azure python-dateutil six
+    displayName: 'Test (un)install package'

Doc/make.bat

@@ -117,13 +117,13 @@ if not exist "%BUILDDIR%" mkdir "%BUILDDIR%"
 
 rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py
 if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1
+if not exist "%PY_MISC_NEWS_DIR%" mkdir "%PY_MISC_NEWS_DIR%"
 if exist ..\Misc\NEWS (
     echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS
     copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul
 ) else if exist ..\Misc\NEWS.D (
     if defined BLURB (
         echo.Merging Misc/NEWS with %BLURB%
-        if not exist build mkdir build
         %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS"
     ) else (
         echo.No Misc/NEWS file and Blurb is not available.

PC/layout/main.py

@@ -31,6 +31,7 @@ from .support.logging import *
 from .support.options import *
 from .support.pip import *
 from .support.props import *
+from .support.nuspec import *
 
 BDIST_WININST_FILES_ONLY = FileNameSet("wininst-*", "bdist_wininst.py")
 BDIST_WININST_STUB = "PC/layout/support/distutils.command.bdist_wininst.py"
@@ -66,6 +67,7 @@ DATA_DIRS = FileNameSet("data")
 TOOLS_DIRS = FileNameSet("scripts", "i18n", "pynche", "demo", "parser")
 TOOLS_FILES = FileSuffixSet(".py", ".pyw", ".txt")
 
+
 def copy_if_modified(src, dest):
     try:
         dest_stat = os.stat(dest)
@@ -73,12 +75,15 @@ def copy_if_modified(src, dest):
         do_copy = True
     else:
         src_stat = os.stat(src)
-        do_copy = (src_stat.st_mtime != dest_stat.st_mtime or
-                   src_stat.st_size != dest_stat.st_size)
+        do_copy = (
+            src_stat.st_mtime != dest_stat.st_mtime
+            or src_stat.st_size != dest_stat.st_size
+        )
 
     if do_copy:
         shutil.copy2(src, dest)
 
+
 def get_lib_layout(ns):
     def _c(f):
         if f in EXCLUDE_FROM_LIB:
@@ -119,7 +124,7 @@ def get_tcltk_lib(ns):
         except FileNotFoundError:
             pass
         if not tcl_lib or not os.path.isdir(tcl_lib):
-            warn("Failed to find TCL_LIBRARY")
+            log_warning("Failed to find TCL_LIBRARY")
             return
 
     for dest, src in rglob(Path(tcl_lib).parent, "**/*"):
@@ -168,7 +173,7 @@ def get_layout(ns):
     for dest, src in rglob(ns.build, "vcruntime*.dll"):
         yield dest, src
 
-    yield "LICENSE.txt", ns.source / "LICENSE"
+    yield "LICENSE.txt", ns.build / "LICENSE.txt"
 
     for dest, src in rglob(ns.build, ("*.pyd", "*.dll")):
         if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS:
@@ -222,15 +227,12 @@ def get_layout(ns):
         yield dest, src
 
     if ns.include_pip:
-        pip_dir = get_pip_dir(ns)
-        if not pip_dir.is_dir():
-            log_warning("Failed to find {} - pip will not be included", pip_dir)
-        else:
-            pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}"
-            for dest, src in rglob(pip_dir, "**/*"):
-                if src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB:
-                    continue
-                yield pkg_root.format(dest), src
+        for dest, src in get_pip_layout(ns):
+            if isinstance(src, tuple) or not (
+                src in EXCLUDE_FROM_LIB or src in EXCLUDE_FROM_PACKAGED_LIB
+            ):
+                continue
+            yield dest, src
 
     if ns.include_chm:
         for dest, src in rglob(ns.doc_build / "htmlhelp", PYTHON_CHM_NAME):
@@ -244,6 +246,10 @@ def get_layout(ns):
         for dest, src in get_props_layout(ns):
             yield dest, src
 
+    if ns.include_nuspec:
+        for dest, src in get_nuspec_layout(ns):
+            yield dest, src
+
     for dest, src in get_appx_layout(ns):
         yield dest, src
 
@@ -287,7 +293,9 @@ def _py_temp_compile(src, ns, dest_dir=None, checked=True):
         return None
 
     dest = (dest_dir or ns.temp) / (src.stem + ".py")
-    return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked)
+    return _compile_one_py(
+        src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked
+    )
 
 
 def _write_to_zip(zf, dest, src, ns, checked=True):
@@ -361,28 +369,9 @@ def generate_source_files(ns):
             print("# Uncomment to run site.main() automatically", file=f)
             print("#import site", file=f)
 
-    if ns.include_appxmanifest:
-        log_info("Generating AppxManifest.xml in {}", ns.temp)
-        ns.temp.mkdir(parents=True, exist_ok=True)
-
-        with open(ns.temp / "AppxManifest.xml", "wb") as f:
-            f.write(get_appxmanifest(ns))
-
-        with open(ns.temp / "_resources.xml", "wb") as f:
-            f.write(get_resources_xml(ns))
-
     if ns.include_pip:
-        pip_dir = get_pip_dir(ns)
-        if not (pip_dir / "pip").is_dir():
-            log_info("Extracting pip to {}", pip_dir)
-            pip_dir.mkdir(parents=True, exist_ok=True)
-            extract_pip_files(ns)
-
-    if ns.include_props:
-        log_info("Generating {} in {}", PYTHON_PROPS_NAME, ns.temp)
-        ns.temp.mkdir(parents=True, exist_ok=True)
-        with open(ns.temp / PYTHON_PROPS_NAME, "wb") as f:
-            f.write(get_props(ns))
+        log_info("Extracting pip")
+        extract_pip_files(ns)
 
 
 def _create_zip_file(ns):
@@ -427,6 +416,18 @@ def copy_files(files, ns):
                     log_info("Processed {} files", count)
             log_debug("Processing {!s}", src)
 
+            if isinstance(src, tuple):
+                src, content = src
+                if ns.copy:
+                    log_debug("Copy {} -> {}", src, ns.copy / dest)
+                    (ns.copy / dest).parent.mkdir(parents=True, exist_ok=True)
+                    with open(ns.copy / dest, "wb") as f:
+                        f.write(content)
+                if ns.zip:
+                    log_debug("Zip {} into {}", src, ns.zip)
+                    zip_file.writestr(str(dest), content)
+                continue
+
             if (
                 ns.precompile
                 and src in PY_FILES

PC/layout/support/appxmanifest.py

@@ -17,12 +17,7 @@ from xml.etree import ElementTree as ET
 
 from .constants import *
 
-__all__ = []
-
-
-def public(f):
-    __all__.append(f.__name__)
-    return f
+__all__ = ["get_appx_layout"]
 
 
 APPX_DATA = dict(
@@ -166,9 +161,7 @@ REGISTRY = {
             "Help": {
                 "Main Python Documentation": {
                     "_condition": lambda ns: ns.include_chm,
-                    "": "[{{AppVPackageRoot}}]\\Doc\\{}".format(
-                        PYTHON_CHM_NAME
-                    ),
+                    "": "[{{AppVPackageRoot}}]\\Doc\\{}".format(PYTHON_CHM_NAME),
                 },
                 "Local Python Documentation": {
                     "_condition": lambda ns: ns.include_html_doc,
@@ -239,31 +232,6 @@ def _fixup_sccd(ns, sccd, new_hash=None):
     return sccd
 
 
-@public
-def get_appx_layout(ns):
-    if not ns.include_appxmanifest:
-        return
-
-    yield "AppxManifest.xml", ns.temp / "AppxManifest.xml"
-    yield "_resources.xml", ns.temp / "_resources.xml"
-    icons = ns.source / "PC" / "icons"
-    yield "_resources/pythonx44.png", icons / "pythonx44.png"
-    yield "_resources/pythonx44$targetsize-44_altform-unplated.png", icons / "pythonx44.png"
-    yield "_resources/pythonx50.png", icons / "pythonx50.png"
-    yield "_resources/pythonx50$targetsize-50_altform-unplated.png", icons / "pythonx50.png"
-    yield "_resources/pythonx150.png", icons / "pythonx150.png"
-    yield "_resources/pythonx150$targetsize-150_altform-unplated.png", icons / "pythonx150.png"
-    yield "_resources/pythonwx44.png", icons / "pythonwx44.png"
-    yield "_resources/pythonwx44$targetsize-44_altform-unplated.png", icons / "pythonwx44.png"
-    yield "_resources/pythonwx150.png", icons / "pythonwx150.png"
-    yield "_resources/pythonwx150$targetsize-150_altform-unplated.png", icons / "pythonwx150.png"
-    sccd = ns.source / SCCD_FILENAME
-    if sccd.is_file():
-        # This should only be set for side-loading purposes.
-        sccd = _fixup_sccd(ns, sccd, os.getenv("APPX_DATA_SHA256"))
-        yield sccd.name, sccd
-
-
 def find_or_add(xml, element, attr=None, always_add=False):
     if always_add:
         e = None
@@ -393,7 +361,6 @@ def disable_registry_virtualization(xml):
     e = find_or_add(e, "rescap:Capability", ("Name", "unvirtualizedResources"))
 
 
-@public
 def get_appxmanifest(ns):
     for k, v in APPXMANIFEST_NS.items():
         ET.register_namespace(k, v)
@@ -481,6 +448,29 @@ def get_appxmanifest(ns):
     return buffer.getbuffer()
 
 
-@public
 def get_resources_xml(ns):
     return RESOURCES_XML_TEMPLATE.encode("utf-8")
+
+
+def get_appx_layout(ns):
+    if not ns.include_appxmanifest:
+        return
+
+    yield "AppxManifest.xml", ("AppxManifest.xml", get_appxmanifest(ns))
+    yield "_resources.xml", ("_resources.xml", get_resources_xml(ns))
+    icons = ns.source / "PC" / "icons"
+    yield "_resources/pythonx44.png", icons / "pythonx44.png"
+    yield "_resources/pythonx44$targetsize-44_altform-unplated.png", icons / "pythonx44.png"
+    yield "_resources/pythonx50.png", icons / "pythonx50.png"
+    yield "_resources/pythonx50$targetsize-50_altform-unplated.png", icons / "pythonx50.png"
+    yield "_resources/pythonx150.png", icons / "pythonx150.png"
+    yield "_resources/pythonx150$targetsize-150_altform-unplated.png", icons / "pythonx150.png"
+    yield "_resources/pythonwx44.png", icons / "pythonwx44.png"
+    yield "_resources/pythonwx44$targetsize-44_altform-unplated.png", icons / "pythonwx44.png"
+    yield "_resources/pythonwx150.png", icons / "pythonwx150.png"
+    yield "_resources/pythonwx150$targetsize-150_altform-unplated.png", icons / "pythonwx150.png"
+    sccd = ns.source / SCCD_FILENAME
+    if sccd.is_file():
+        # This should only be set for side-loading purposes.
+        sccd = _fixup_sccd(ns, sccd, os.getenv("APPX_DATA_SHA256"))
+        yield sccd.name, sccd

PC/layout/support/nuspec.py

@@ -0,0 +1,66 @@
+"""
+Provides .props file.
+"""
+
+import os
+
+from .constants import *
+
+__all__ = ["get_nuspec_layout"]
+
+PYTHON_NUSPEC_NAME = "python.nuspec"
+
+NUSPEC_DATA = {
+    "PYTHON_TAG": VER_DOT,
+    "PYTHON_VERSION": os.getenv("PYTHON_NUSPEC_VERSION"),
+    "PYTHON_BITNESS": "64-bit" if IS_X64 else "32-bit",
+    "PACKAGENAME": os.getenv("PYTHON_NUSPEC_PACKAGENAME"),
+    "PACKAGETITLE": os.getenv("PYTHON_NUSPEC_PACKAGETITLE"),
+    "FILELIST": r'    <file src="**\*" target="tools" />',
+}
+
+if not NUSPEC_DATA["PYTHON_VERSION"]:
+    if VER_NAME:
+        NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}-{}{}".format(
+            VER_DOT, VER_MICRO, VER_NAME, VER_SERIAL
+        )
+    else:
+        NUSPEC_DATA["PYTHON_VERSION"] = "{}.{}".format(VER_DOT, VER_MICRO)
+
+if not NUSPEC_DATA["PACKAGETITLE"]:
+    NUSPEC_DATA["PACKAGETITLE"] = "Python" if IS_X64 else "Python (32-bit)"
+
+if not NUSPEC_DATA["PACKAGENAME"]:
+    NUSPEC_DATA["PACKAGENAME"] = "python" if IS_X64 else "pythonx86"
+
+FILELIST_WITH_PROPS = r"""    <file src="**\*" exclude="python.props" target="tools" />
+    <file src="python.props" target="build\native" />"""
+
+NUSPEC_TEMPLATE = r"""<?xml version="1.0"?>
+<package>
+  <metadata>
+    <id>{PACKAGENAME}</id>
+    <title>{PACKAGETITLE}</title>
+    <version>{PYTHON_VERSION}</version>
+    <authors>Python Software Foundation</authors>
+    <license type="file">tools\LICENSE.txt</license>
+    <projectUrl>https://www.python.org/</projectUrl>
+    <description>Installs {PYTHON_BITNESS} Python for use in build scenarios.</description>
+    <iconUrl>https://www.python.org/static/favicon.ico</iconUrl>
+    <tags>python</tags>
+  </metadata>
+  <files>
+{FILELIST}
+  </files>
+</package>
+"""
+
+
+def get_nuspec_layout(ns):
+    if ns.include_all or ns.include_nuspec:
+        data = NUSPEC_DATA
+        if ns.include_all or ns.include_props:
+            data = dict(data)
+            data["FILELIST"] = FILELIST_WITH_PROPS
+        nuspec = NUSPEC_TEMPLATE.format_map(data)
+        yield "python.nuspec", ("python.nuspec", nuspec.encode("utf-8"))

PC/layout/support/options.py

@@ -30,6 +30,7 @@ OPTIONS = {
     "launchers": {"help": "specific launchers"},
     "appxmanifest": {"help": "an appxmanifest"},
     "props": {"help": "a python.props file"},
+    "nuspec": {"help": "a python.nuspec file"},
     "chm": {"help": "the CHM documentation"},
     "html-doc": {"help": "the HTML documentation"},
 }
@@ -60,13 +61,11 @@ PRESETS = {
             "stable",
             "distutils",
             "venv",
-            "props"
+            "props",
+            "nuspec",
         ],
     },
-    "iot": {
-        "help": "Windows IoT Core",
-        "options": ["stable", "pip"],
-    },
+    "iot": {"help": "Windows IoT Core", "options": ["stable", "pip"]},
     "default": {
         "help": "development kit package",
         "options": [

PC/layout/support/pip.py

@@ -11,15 +11,11 @@ import shutil
 import subprocess
 import sys
 
-__all__ = []
+from .filesets import *
+
+__all__ = ["extract_pip_files", "get_pip_layout"]
 
 
-def public(f):
-    __all__.append(f.__name__)
-    return f
-
-
-@public
 def get_pip_dir(ns):
     if ns.copy:
         if ns.zip_lib:
@@ -29,10 +25,23 @@ def get_pip_dir(ns):
         return ns.temp / "packages"
 
 
-@public
+def get_pip_layout(ns):
+    pip_dir = get_pip_dir(ns)
+    if not pip_dir.is_dir():
+        log_warning("Failed to find {} - pip will not be included", pip_dir)
+    else:
+        pkg_root = "packages/{}" if ns.zip_lib else "Lib/site-packages/{}"
+        for dest, src in rglob(pip_dir, "**/*"):
+            yield pkg_root.format(dest), src
+        yield "pip.ini", ("pip.ini", b"[global]\nuser=yes")
+
+
 def extract_pip_files(ns):
     dest = get_pip_dir(ns)
-    dest.mkdir(parents=True, exist_ok=True)
+    try:
+        dest.mkdir(parents=True, exist_ok=False)
+    except IOError:
+        return
 
     src = ns.source / "Lib" / "ensurepip" / "_bundled"
 
@@ -58,6 +67,7 @@ def extract_pip_files(ns):
             "--target",
             str(dest),
             "--no-index",
+            "--no-compile",
             "--no-cache-dir",
             "-f",
             str(src),

PC/layout/support/props.py

@@ -6,13 +6,7 @@ import os
 
 from .constants import *
 
-__all__ = ["PYTHON_PROPS_NAME"]
-
-
-def public(f):
-    __all__.append(f.__name__)
-    return f
-
+__all__ = ["get_props_layout"]
 
 PYTHON_PROPS_NAME = "python.props"
 
@@ -97,14 +91,8 @@ PROPS_TEMPLATE = r"""<?xml version="1.0" encoding="utf-8"?>
 """
 
 
-@public
 def get_props_layout(ns):
     if ns.include_all or ns.include_props:
-        yield "python.props", ns.temp / "python.props"
-
-
-@public
-def get_props(ns):
-    # TODO: Filter contents of props file according to included/excluded items
-    props = PROPS_TEMPLATE.format_map(PROPS_DATA)
-    return props.encode("utf-8")
+        # TODO: Filter contents of props file according to included/excluded items
+        props = PROPS_TEMPLATE.format_map(PROPS_DATA)
+        yield "python.props", ("python.props", props.encode("utf-8"))

PC/python_uwp.cpp

@@ -182,9 +182,9 @@ wmain(int argc, wchar_t **argv)
             if (*p++ == L'\\') {
                 if (wcsnicmp(p, L"pip", 3) == 0) {
                     moduleName = L"pip";
+                    /* No longer required when pip 19.1 is added */
                     _wputenv_s(L"PIP_USER", L"true");
-                }
-                else if (wcsnicmp(p, L"idle", 4) == 0) {
+                } else if (wcsnicmp(p, L"idle", 4) == 0) {
                     moduleName = L"idlelib";
                 }
             }

PCbuild/_tkinter.vcxproj

@@ -122,7 +122,7 @@
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
   <Target Name="_CopyTclTkDLL" Inputs="@(_TclTkDLL)" Outputs="@(_TclTkDLL->'$(OutDir)%(Filename)%(Extension)')" AfterTargets="Build">
-    <Copy SourceFiles="@(_TclTkDLL)" DestinationFolder="$(OutDir)" />
+    <Copy SourceFiles="@(_TclTkDLL)" DestinationFolder="$(OutDir)" UseHardlinksIfPossible="true" />
   </Target>
   <Target Name="_CleanTclTkDLL" BeforeTargets="Clean">
     <Delete Files="@(_TclTkDLL->'$(OutDir)%(Filename)%(Extension)')" />

PCbuild/build.bat

@@ -76,7 +76,7 @@ if "%~1"=="-k" (set kill=true) & shift & goto CheckOpts
 if "%~1"=="--pgo" (set do_pgo=true) & shift & goto CheckOpts
 if "%~1"=="--pgo-job" (set do_pgo=true) & (set pgo_job=%~2) & shift & shift & goto CheckOpts
 if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts
-if "%~1"=="-V" shift & goto Version
+if "%~1"=="-V" shift & goto :Version
 rem These use the actual property names used by MSBuild.  We could just let
 rem them in through the environment, but we specify them on the command line
 rem anyway for visibility so set defaults after this
@@ -111,10 +111,16 @@ call "%dir%find_msbuild.bat" %MSBUILD%
 if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
 if "%kill%"=="true" call :Kill
+if ERRORLEVEL 1 exit /B 3
 
 if "%do_pgo%"=="true" (
     set conf=PGInstrument
     call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
+)
+rem %VARS% are evaluated eagerly, which would lose the ERRORLEVEL
+rem value if we didn't split it out here.
+if "%do_pgo%"=="true" if ERRORLEVEL 1 exit /B %ERRORLEVEL%
+if "%do_pgo%"=="true" (
     del /s "%dir%\*.pgc"
     del /s "%dir%\..\Lib\*.pyc"
     echo on
@@ -124,7 +130,8 @@ if "%do_pgo%"=="true" (
     set conf=PGUpdate
     set target=Build
 )
-goto Build
+goto :Build
+
 :Kill
 echo on
 %MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
@@ -132,7 +139,7 @@ echo on
  /p:KillPython=true
 
 @echo off
-goto :eof
+exit /B %ERRORLEVEL%
 
 :Build
 rem Call on MSBuild to do the work, echo the command.
@@ -148,9 +155,11 @@ echo on
  %1 %2 %3 %4 %5 %6 %7 %8 %9
 
 @echo off
-goto :eof
+exit /b %ERRORLEVEL%
 
 :Version
 rem Display the current build version information
 call "%dir%find_msbuild.bat" %MSBUILD%
-if not ERRORLEVEL 1 %MSBUILD% "%dir%pythoncore.vcxproj" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
+%MSBUILD% "%dir%pythoncore.vcxproj" /t:ShowVersionInfo /v:m /nologo %1 %2 %3 %4 %5 %6 %7 %8 %9
+if ERRORLEVEL 1 exit /b 3

PCbuild/pyproject.props

@@ -1,6 +1,8 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="Py_IntDir">
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="Py_IntDir">
+  <Import Project="python.props" Condition="$(__Python_Props_Imported) != 'true'" />
   <PropertyGroup Label="Globals">
+    <__PyProject_Props_Imported>true</__PyProject_Props_Imported>
     <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>
     <OutDir>$(BuildPath)</OutDir>
@@ -29,7 +31,7 @@
     <ClCompile>
       <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)PC;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      
+
       <Optimization>MaxSpeed</Optimization>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <StringPooling>true</StringPooling>
@@ -147,15 +149,15 @@ public override bool Execute() {
       </Code>
     </Task>
   </UsingTask>
-  
+
   <Target Name="KillPython" BeforeTargets="PrepareForBuild" Condition="'$(KillPython)' == 'true'">
     <Message Text="Killing any running python$(PyDebugExt)$(PyTestExt).exe instances..." Importance="high" />
     <KillPython FileName="$(OutDir)python$(PyDebugExt)$(PyTestExt).exe" />
   </Target>
-  
+
   <!--
   A default target to handle msbuild pcbuild.proj /t:CleanAll.
-  
+
   Some externals projects don't respond to /t:Clean, so we invoke
   CleanAll on them when we really want to clean up.
   -->
@@ -189,8 +191,8 @@ public override bool Execute() {
     <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86</SdkBinPath>
     <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86</SdkBinPath>
     <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\</SdkBinPath>
-    <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand>
-    <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificateSha1)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /sha1 "$(SigningCertificateSha1)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand>
+    <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand>
+    <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificateSha1)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /a /sha1 "$(SigningCertificateSha1)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand>
     <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe"</_MakeCatCommand>
   </PropertyGroup>
 

PCbuild/python.props

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
+    <__Python_Props_Imported>true</__Python_Props_Imported>
     <Platform Condition="'$(Platform)' == ''">Win32</Platform>
     <Configuration Condition="'$(Configuration)' == ''">Release</Configuration>
     <!--
@@ -215,6 +216,7 @@
     <Message Importance="high" Text="PythonVersionNumber: $(PythonVersionNumber)" />
     <Message Importance="high" Text="PythonVersion:       $(PythonVersion)" />
     <Message Importance="high" Text="PythonVersionHex:    0x$([System.UInt32]::Parse($(PythonVersionHex)).ToString(`X08`))" />
+    <Message Importance="high" Text="PythonVersionUnique: $(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value)" />
     <Message Importance="high" Text="Field3Value:         $(Field3Value)" />
     <Message Importance="high" Text="SysWinVer:           $(SysWinVer)" />
     <Message Importance="high" Text="PyDllName:           $(PyDllName)" />

PCbuild/python.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|ARM">
@@ -82,6 +82,7 @@
   <ImportGroup Label="PropertySheets">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
     <Import Project="pyproject.props" />
+    <Import Project="tcltk.props" />
   </ImportGroup>
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup>
@@ -144,4 +145,22 @@ $(_PGOPath)
     </PropertyGroup>
     <WriteLinesToFile File="$(PySourcePath)python.bat" Lines="$(_Content)" Overwrite="true" Condition="'$(_Content)' != '$(_ExistingContent)'" />
   </Target>
+  <Target Name="GenerateLicense" AfterTargets="AfterBuild">
+    <ItemGroup>
+      <LicenseFiles Include="$(PySourcePath)LICENSE;
+                             $(PySourcePath)PC\crtlicense.txt;
+                             $(bz2Dir)LICENSE;
+                             $(opensslOutDir)LICENSE;
+                             $(tcltkDir)tcllicense.terms;
+                             $(tcltkDir)tklicense.terms;
+                             $(tcltkDir)tixlicense.terms" />
+      <_LicenseFiles Include="@(LicenseFiles)">
+        <Content>$([System.IO.File]::ReadAllText(%(FullPath)))</Content>
+      </_LicenseFiles>
+    </ItemGroup>
+
+    <WriteLinesToFile File="$(OutDir)LICENSE.txt"
+                      Overwrite="true"
+                      Lines="@(_LicenseFiles->'%(Content)')" />
+  </Target>
 </Project>

PCbuild/tcltk.props

@@ -1,6 +1,6 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <Import Project="pyproject.props" />
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="pyproject.props" Condition="$(__PyProject_Props_Imported) != 'true'" />
   <PropertyGroup>
     <TclMajorVersion>8</TclMajorVersion>
     <TclMinorVersion>6</TclMinorVersion>
@@ -42,4 +42,19 @@
     <BuildDirTop Condition="$(PlatformToolset) == 'v110'">$(BuildDirTop)_VC11</BuildDirTop>
     <BuildDirTop Condition="$(PlatformToolset) == 'v100'">$(BuildDirTop)_VC10</BuildDirTop>
   </PropertyGroup>
+
+  <!--
+  Helper target for copying the lib to a specific directory.
+
+  Using "msbuild tcltk.props /t:CopyTclTkLib /p:OutDir=..." is generally
+  easier than trying to extract the value of $(tcltkdir).
+   -->
+  <Target Name="CopyTclTkLib">
+    <ItemGroup>
+      <_TclTkLib Include="$(tcltkdir)\lib\**\*" />
+    </ItemGroup>
+    <Copy SourceFiles="@(_TclTkLib)"
+          DestinationFiles="$(OutDir)\%(RecursiveDir)\%(Filename)%(Extension)"
+          UseHardlinksIfPossible="true" />
+  </Target>
 </Project>

Tools/msi/buildrelease.bat

@@ -29,7 +29,7 @@ set DOWNLOAD_URL=https://www.python.org/ftp/python/{version}/{arch}{releasename}
 
 set D=%~dp0
 set PCBUILD=%D%..\..\PCbuild\
-if "%Py_OutDir%"=="" set Py_OutDir=%PCBUILD%
+if NOT DEFINED Py_OutDir set Py_OutDir=%PCBUILD%
 set EXTERNALS=%D%..\..\externals\windows-installer\
 
 set BUILDX86=

Tools/msi/exe/exe.wixproj

@@ -21,25 +21,6 @@
         <WxlTemplate Include="*.wxl_template" />
     </ItemGroup>
     
-    <Target Name="_GenerateLicense" AfterTargets="PrepareForBuild">
-        <ItemGroup>
-            <LicenseFiles Include="$(PySourcePath)LICENSE;
-                                   crtlicense.txt;
-                                   $(bz2Dir)LICENSE;
-                                   $(opensslOutDir)LICENSE;
-                                   $(tcltkDir)tcllicense.terms;
-                                   $(tcltkDir)tklicense.terms;
-                                   $(tcltkDir)tixlicense.terms" />
-            <_LicenseFiles Include="@(LicenseFiles)">
-                <Content>$([System.IO.File]::ReadAllText(%(FullPath)))</Content>
-            </_LicenseFiles>
-        </ItemGroup>
-        
-        <WriteLinesToFile File="$(BuildPath)LICENSE"
-                          Overwrite="true"
-                          Lines="@(_LicenseFiles->'%(Content)')" />
-    </Target>
-    
     <Target Name="_CopyMiscNews" AfterTargets="PrepareForBuild" Condition="Exists('$(PySourcePath)Misc\NEWS')">
         <Copy SourceFiles="$(PySourcePath)Misc\NEWS" DestinationFiles="$(BuildPath)NEWS.txt" />
     </Target>

Tools/msi/exe/exe_files.wxs

@@ -3,7 +3,7 @@
     <Fragment>
         <ComponentGroup Id="exe_txt">
             <Component Id="LICENSE.txt" Directory="InstallDirectory" Guid="*">
-                <File Name="LICENSE.txt" Source="LICENSE" KeyPath="yes" />
+                <File Name="LICENSE.txt" Source="LICENSE.txt" KeyPath="yes" />
             </Component>
             <Component Id="NEWS.txt" Directory="InstallDirectory" Guid="*">
                 <File Name="NEWS.txt" KeyPath="yes" />

Tools/msi/make_cat.ps1

@@ -7,6 +7,8 @@
     The path to the catalog definition file to compile and
     sign. It is assumed that the .cat file will be the same
     name with a new extension.
+.Parameter outfile
+    The path to move the built .cat file to (optional).
 .Parameter description
     The description to add to the signature (optional).
 .Parameter certname
@@ -16,6 +18,7 @@
 #>
 param(
     [Parameter(Mandatory=$true)][string]$catalog,
+    [string]$outfile,
     [switch]$sign,
     [string]$description,
     [string]$certname,
@@ -35,3 +38,8 @@ if (-not $?) {
 if ($sign) {
     Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files @($catalog -replace 'cdf$', 'cat')
 }
+
+if ($outfile) {
+    Split-Path -Parent $outfile | ?{ $_ } | %{ mkdir -Force $_; }
+    Move-Item ($catalog -replace 'cdf$', 'cat') $outfile
+}

Tools/msi/msi.props

@@ -56,6 +56,7 @@
         <ReuseCabinetCache>true</ReuseCabinetCache>
         <CRTRedist Condition="'$(CRTRedist)' == ''">$(ExternalsDir)\windows-installer\redist-1\$(Platform)</CRTRedist>
         <CRTRedist>$([System.IO.Path]::GetFullPath($(CRTRedist)))</CRTRedist>
+        <TclTkLibraryDir Condition="$(TclTkLibraryDir) == ''">$(tcltkDir)lib</TclTkLibraryDir>
         <DocFilename>python$(MajorVersionNumber)$(MinorVersionNumber)$(MicroVersionNumber)$(ReleaseLevelName).chm</DocFilename>
 
         <InstallerVersion>$(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value).0</InstallerVersion>
@@ -121,7 +122,7 @@
         <LinkerBindInputPaths Include="$(PySourcePath)">
             <BindName>src</BindName>
         </LinkerBindInputPaths>
-        <LinkerBindInputPaths Include="$(tcltkDir)">
+        <LinkerBindInputPaths Include="$(TclTkLibraryDir)">
             <BindName>tcltk</BindName>
         </LinkerBindInputPaths>
         <LinkerBindInputPaths Include="$(CRTRedist)">

Tools/msi/msi.targets

@@ -47,7 +47,7 @@ EncodingType=
 
         <WriteLinesToFile File="$(_CatFileSourceTarget)" Lines="$(_CatFile)" Overwrite="true" />
         <Exec Command='$(_MakeCatCommand) "$(_CatFileSourceTarget)"' WorkingDirectory="$(MSBuildThisFileDirectory)" />
-        <Exec Command='$(_SignCommand) "$(_CatFileTarget)"' WorkingDirectory="$(MSBuildThisFileDirectory)"
+        <Exec Command='$(_SignCommand) "$(_CatFileTarget)" || $(_SignCommand) "$(_CatFileTarget)" || $(_SignCommand) "$(_CatFileTarget)"' WorkingDirectory="$(MSBuildThisFileDirectory)"
               Condition="Exists($(_CatFileTarget)) and '$(_SignCommand)' != ''" />
 
         <ItemGroup>
@@ -76,18 +76,18 @@ EncodingType=
 
     <Target Name="SignCabs">
         <Error Text="Unable to locate signtool.exe. Set /p:SignToolPath and rebuild" Condition="'$(_SignCommand)' == ''" />
-        <Exec Command="$(_SignCommand) @(SignCabs->'&quot;%(FullPath)&quot;',' ')" ContinueOnError="false" />
+        <Exec Command="$(_SignCommand) @(SignCabs->'&quot;%(FullPath)&quot;',' ') || $(_SignCommand) @(SignCabs->'&quot;%(FullPath)&quot;',' ') || $(_SignCommand) @(SignCabs->'&quot;%(FullPath)&quot;',' ')" ContinueOnError="false" />
     </Target>
     <Target Name="SignMsi">
         <Error Text="Unable to locate signtool.exe. Set /p:SignToolPath and rebuild" Condition="'$(_SignCommand)' == ''" />
-        <Exec Command="$(_SignCommand) @(SignMsi->'&quot;%(FullPath)&quot;',' ')" ContinueOnError="false" />
+        <Exec Command="$(_SignCommand) @(SignMsi->'&quot;%(FullPath)&quot;',' ') || $(_SignCommand) @(SignMsi->'&quot;%(FullPath)&quot;',' ') || $(_SignCommand) @(SignMsi->'&quot;%(FullPath)&quot;',' ')" ContinueOnError="false" />
     </Target>
     <Target Name="SignBundleEngine">
         <Error Text="Unable to locate signtool.exe. Set /p:SignToolPath and rebuild" Condition="'$(_SignCommand)' == ''" />
-        <Exec Command="$(_SignCommand) @(SignBundleEngine->'&quot;%(FullPath)&quot;',' ')" ContinueOnError="false" />
+        <Exec Command="$(_SignCommand) @(SignBundleEngine->'&quot;%(FullPath)&quot;',' ') || $(_SignCommand) @(SignBundleEngine->'&quot;%(FullPath)&quot;',' ') || $(_SignCommand) @(SignBundleEngine->'&quot;%(FullPath)&quot;',' ')" ContinueOnError="false" />
     </Target>
     <Target Name="SignBundle">
         <Error Text="Unable to locate signtool.exe. Set /p:SignToolPath and rebuild" Condition="'$(_SignCommand)' == ''" />
-        <Exec Command="$(_SignCommand) @(SignBundle->'&quot;%(FullPath)&quot;',' ')" ContinueOnError="false" />
+        <Exec Command="$(_SignCommand) @(SignBundle->'&quot;%(FullPath)&quot;',' ') || $(_SignCommand) @(SignBundle->'&quot;%(FullPath)&quot;',' ') || $(_SignCommand) @(SignBundle->'&quot;%(FullPath)&quot;',' ')" ContinueOnError="false" />
     </Target>
 </Project>

Tools/msi/sign_build.ps1

@@ -16,7 +16,7 @@
 #>
 param(
     [Parameter(Mandatory=$true)][string]$root,
-    [string[]]$patterns=@("*.exe", "*.dll", "*.pyd"),
+    [string[]]$patterns=@("*.exe", "*.dll", "*.pyd", "*.cat"),
     [string]$description,
     [string]$certname,
     [string]$certsha1,

Tools/msi/tcltk/tcltk.wixproj

@@ -20,10 +20,10 @@
         <WxlTemplate Include="*.wxl_template" />
     </ItemGroup>
     <ItemGroup>
-        <InstallFiles Include="$(tcltkDir)lib\**\*">
-            <SourceBase>$(tcltkDir)</SourceBase>
+        <InstallFiles Include="$(TclTkLibraryDir)\**\*">
+            <SourceBase>$(TclTkLibraryDir)</SourceBase>
             <Source>!(bindpath.tcltk)</Source>
-            <TargetBase>$(tcltkDir)lib</TargetBase>
+            <TargetBase>$(TclTkLibraryDir)</TargetBase>
             <Target_>tcl\</Target_>
             <Group>tcltk_lib</Group>
         </InstallFiles>

Tools/msi/uploadrelease.ps1

@@ -15,6 +15,10 @@
     The subdirectory on the host to copy files to.
 .Parameter tests
     The path to run download tests in.
+.Parameter doc_htmlhelp
+    Optional path besides -build to locate CHM files.
+.Parameter embed
+    Optional path besides -build to locate ZIP files.
 .Parameter skipupload
     Skip uploading
 .Parameter skippurge
@@ -30,6 +34,8 @@ param(
     [string]$server="python-downloads",
     [string]$target="/srv/www.python.org/ftp/python",
     [string]$tests=${env:TEMP},
+    [string]$doc_htmlhelp=$null,
+    [string]$embed=$null,
     [switch]$skipupload,
     [switch]$skippurge,
     [switch]$skiptest,
@@ -73,32 +79,45 @@ if (-not $skipupload) {
     "Upload using $pscp and $plink"
     ""
 
-    pushd $build
-    $doc = gci python*.chm, python*.chm.asc
+    if ($doc_htmlhelp) {
+        pushd $doc_htmlhelp
+    } else {
+        pushd $build
+    }
+    $chm = gci python*.chm, python*.chm.asc
     popd
 
     $d = "$target/$($p[0])/"
     & $plink -batch $user@$server mkdir $d
     & $plink -batch $user@$server chgrp downloads $d
     & $plink -batch $user@$server chmod g-x,o+rx $d
-    & $pscp -batch $doc.FullName "$user@${server}:$d"
+    & $pscp -batch $chm.FullName "$user@${server}:$d"
 
-    foreach ($a in gci "$build" -Directory) {
+    $dirs = gci "$build" -Directory
+    if ($embed) {
+        $dirs = ($dirs, (gi $embed)) | %{ $_ }
+    }
+
+    foreach ($a in $dirs) {
         "Uploading files from $($a.FullName)"
         pushd "$($a.FullName)"
         $exe = gci *.exe, *.exe.asc, *.zip, *.zip.asc
         $msi = gci *.msi, *.msi.asc, *.msu, *.msu.asc
         popd
 
-        & $pscp -batch $exe.FullName "$user@${server}:$d"
+        if ($exe) {
+            & $pscp -batch $exe.FullName "$user@${server}:$d"
+        }
 
-        $sd = "$d$($a.Name)$($p[1])/"
-        & $plink -batch $user@$server mkdir $sd
-        & $plink -batch $user@$server chgrp downloads $sd
-        & $plink -batch $user@$server chmod g-x,o+rx $sd
-        & $pscp -batch $msi.FullName "$user@${server}:$sd"
-        & $plink -batch $user@$server chgrp downloads $sd*
-        & $plink -batch $user@$server chmod g-x,o+r $sd*
+        if ($msi) {
+            $sd = "$d$($a.Name)$($p[1])/"
+            & $plink -batch $user@$server mkdir $sd
+            & $plink -batch $user@$server chgrp downloads $sd
+            & $plink -batch $user@$server chmod g-x,o+rx $sd
+            & $pscp -batch $msi.FullName "$user@${server}:$sd"
+            & $plink -batch $user@$server chgrp downloads $sd*
+            & $plink -batch $user@$server chmod g-x,o+r $sd*
+        }
     }
 
     & $plink -batch $user@$server chgrp downloads $d*
@@ -128,7 +147,18 @@ if (-not $skiptest) {
 if (-not $skiphash) {
     # Display MD5 hash and size of each downloadable file
     pushd $build
-    $hashes = gci python*.chm, *\*.exe, *\*.zip | `
+    $files = gci python*.chm, *\*.exe, *\*.zip
+    if ($doc_htmlhelp) {
+        cd $doc_htmlhelp
+        $files = ($files, (gci python*.chm)) | %{ $_ }
+    }
+    if ($embed) {
+        cd $embed
+        $files = ($files, (gci *.zip)) | %{ $_ }
+    }
+    popd
+
+    $hashes = $files | `
         Sort-Object Name | `
         Format-Table Name, @{Label="MD5"; Expression={(Get-FileHash $_ -Algorithm MD5).Hash}}, Length -AutoSize | `
         Out-String -Width 4096