Move xUnit tests in new folder (#8356)

## Motivation

I have a PR where there are many new xUnit tests.
It would also be useful to create new xUnit tests for public APIs.
The number of xUnit tests will increase and their ordering is required.

## PR Summary

- Move C# xUnit tests in new folder. This allows to put new xUnit tests in directory structure in accordance with directory structure where cs files are.
- Use an xUnit TestCaseOrderer attribute to sequentially process tests for `powershell.config.json`.
- Update README.md
- A race condition was fixed which allowed  to run all XUnit tests in single batch job.

build.psm1

@@ -1369,7 +1369,6 @@ function Test-PSPesterResults
 
 function Start-PSxUnit {
     [CmdletBinding()]param(
-        [string] $SequentialTestResultsFile = "SequentialXUnitResults.xml",
         [string] $ParallelTestResultsFile = "ParallelXUnitResults.xml"
     )
 
@@ -1382,10 +1381,9 @@ function Start-PSxUnit {
     }
 
     try {
-        Push-Location $PSScriptRoot/test/csharp
+        Push-Location $PSScriptRoot/test/xUnit
 
         # Path manipulation to obtain test project output directory
-        dotnet restore
 
         if(-not $Environment.IsWindows)
         {
@@ -1417,30 +1415,22 @@ function Start-PSxUnit {
             }
         }
 
-        # Run sequential tests first, and then run the tests that can execute in parallel
-        if (Test-Path $SequentialTestResultsFile) {
-            Remove-Item $SequentialTestResultsFile -Force -ErrorAction SilentlyContinue
-        }
-        dotnet test --configuration $Options.configuration --filter FullyQualifiedName~PSTests.Sequential -p:ParallelizeTestCollections=false --test-adapter-path:. "--logger:xunit;LogFilePath=$SequentialTestResultsFile"
-        Publish-TestResults -Path $SequentialTestResultsFile -Type 'XUnit' -Title 'Xunit Sequential'
+        dotnet build --configuration $Options.configuration 
 
-        $extraParams = @()
+        if (Test-Path $ParallelTestResultsFile) {
+            Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue
+        }
 
         # we are having intermittent issues on macOS with these tests failing.
         # VSTS has suggested forcing them to be sequential
         if($env:TF_BUILD -and $IsMacOS)
         {
             Write-Log 'Forcing parallel xunit tests to run sequentially.'
-            $extraParams += @(
-                '-parallel'
-                'none'
-            )
+            dotnet test -p:ParallelizeTestCollections=false --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile"
+        } else {
+            dotnet test --configuration $Options.configuration --no-restore --no-build --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile"
         }
 
-        if (Test-Path $ParallelTestResultsFile) {
-            Remove-Item $ParallelTestResultsFile -Force -ErrorAction SilentlyContinue
-        }
-        dotnet test --configuration $Options.configuration --filter FullyQualifiedName~PSTests.Parallel --no-build  --test-adapter-path:. "--logger:xunit;LogFilePath=$ParallelTestResultsFile"
         Publish-TestResults -Path $ParallelTestResultsFile -Type 'XUnit' -Title 'Xunit Parallel'
     }
     finally {

test/README.md

@@ -4,8 +4,7 @@ Testing
 The tests are organized by testing language. Thus Pester tests, which
 are written in the PowerShell language, are in
 [./powershell](./powershell) and xUnit tests, written in C#, are in
-[./csharp](./csharp). The sanity tests for the Full .NET build of
-PowerShell are in [./fullclr](./fullclr), and the third-party
+[./xUnit](./xUnit). The third-party
 [shebang][] test is in [./shebang](./shebang).
 
 [shebang]: https://en.wikipedia.org/wiki/Shebang_(Unix)

test/csharp/README.md

@@ -1,20 +0,0 @@
-xUnit Tests
-===========
-
-These tests are completely Linux specific.
-
-Every test class *must* belong to
-`[Collection("AssemblyLoadContext")]`. This ensures that PowerShell's
-AssemblyLoadContext is initialized before any other code is executed.
-When this is not the case, late initialization fails with
-`System.InvalidOperationException : Binding model is already locked
-for the AppDomain and cannot be reset.`
-
-Having every class in the same collection is as close to an xUnit
-global init hook as can be done.
-
-Running xUnit Tests
--------------------
-
-Go to the top level of the PowerShell repository and run:
-`Start-PSxUnit` inside a self-hosted copy of PowerShell.

test/xUnit/Asserts/PriorityAttribute.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+public class TestPriorityAttribute : Attribute
+{
+    public TestPriorityAttribute(int priority)
+    {
+        Priority = priority;
+    }
+
+    public int Priority { get; private set; }
+}

test/xUnit/Asserts/PriorityOrderer.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestOrder.TestCaseOrdering
+{
+    public class PriorityOrderer : ITestCaseOrderer
+    {
+        public IEnumerable<TTestCase> OrderTestCases<TTestCase>(IEnumerable<TTestCase> testCases) where TTestCase : ITestCase
+        {
+            var sortedMethods = new SortedDictionary<int, List<TTestCase>>();
+
+            foreach (TTestCase testCase in testCases)
+            {
+                int priority = 0;
+
+                foreach (IAttributeInfo attr in testCase.TestMethod.Method.GetCustomAttributes((typeof(TestPriorityAttribute).AssemblyQualifiedName)))
+                    priority = attr.GetNamedArgument<int>("Priority");
+
+                GetOrCreate(sortedMethods, priority).Add(testCase);
+            }
+
+            foreach (var list in sortedMethods.Keys.Select(priority => sortedMethods[priority]))
+            {
+                list.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.TestMethod.Method.Name, y.TestMethod.Method.Name));
+                foreach (TTestCase testCase in list)
+                    yield return testCase;
+            }
+        }
+
+        static TValue GetOrCreate<TKey, TValue>(IDictionary<TKey, TValue> dictionary, TKey key) where TValue : new()
+        {
+            TValue result;
+
+            if (dictionary.TryGetValue(key, out result)) return result;
+
+            result = new TValue();
+            dictionary[key] = result;
+
+            return result;
+        }
+    }
+}

test/xUnit/README.md

@@ -0,0 +1,29 @@
+# xUnit Tests
+
+The folder contains xUnit tests for PowerShell Core project.
+
+## Running xUnit Tests
+
+Go to the top level of the PowerShell repository and run full set of tests:
+`Start-PSxUnit` inside a self-hosted copy of PowerShell.
+
+Go to the test project folder and run `dotnet test -c Release`.
+
+Use [`filter`](xunit-filter) parameter to run only needed tests:
+```powershell
+dotnet test -c Release --filter "FullyQualifiedName~UnitTest1   # Runs tests which have UnitTest1 in FullyQualifiedName
+dotnet test --filter Name~TestMethod1   # Runs tests whose name contains TestMethod1
+```
+
+## Creating xUnit Tests
+
+Keep the folder structure that is for Pester [../../test/powershell](../../test/powershell) and C# files [../../src](../../src).
+
+Use namespace names started with `PSTests`.
+```c#
+namespace PSTests.YourNameSpace
+{
+}
+```
+
+[xunit-filter]: https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests

test/xUnit/csharp/test_PSConfiguration.cs

@@ -12,6 +12,7 @@ using Newtonsoft.Json.Linq;
 
 namespace PSTests.Sequential
 {
+    [TestCaseOrderer("TestOrder.TestCaseOrdering.PriorityOrderer", "powershell-tests")]
     public class PowerShellPolicyFixture : IDisposable
     {
         private const string configFileName = "powershell.config.json";
@@ -344,7 +345,7 @@ namespace PSTests.Sequential
             this.fixture = fixture;
         }
 
-        [Fact]
+        [Fact, TestPriority(1)]
         public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotEmpty()
         {
             fixture.SetupConfigFile1();
@@ -358,7 +359,7 @@ namespace PSTests.Sequential
             fixture.CompareTwoPolicies(userPolicies, fixture.CurrentUserPolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(2)]
         public void PowerShellConfig_GetPowerShellPolicies_EmptyUserConfig()
         {
             fixture.SetupConfigFile2();
@@ -371,7 +372,7 @@ namespace PSTests.Sequential
             fixture.CompareTwoPolicies(sysPolicies, fixture.SystemWidePolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(3)]
         public void PowerShellConfig_GetPowerShellPolicies_EmptySystemConfig()
         {
             fixture.SetupConfigFile3();
@@ -384,7 +385,7 @@ namespace PSTests.Sequential
             fixture.CompareTwoPolicies(userPolicies, fixture.CurrentUserPolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(4)]
         public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesEmpty()
         {
             fixture.SetupConfigFile4();
@@ -395,7 +396,7 @@ namespace PSTests.Sequential
             Assert.Null(userPolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(5)]
         public void PowerShellConfig_GetPowerShellPolicies_BothConfigFilesNotExist()
         {
             fixture.CleanupConfigFiles();
@@ -406,7 +407,7 @@ namespace PSTests.Sequential
             Assert.Null(userPolicies);
         }
 
-        [Fact]
+        [Fact, TestPriority(6)]
         public void Utils_GetPolicySetting_BothConfigFilesNotEmpty()
         {
             fixture.SetupConfigFile1();
@@ -504,7 +505,7 @@ namespace PSTests.Sequential
             fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, fixture.SystemWidePolicies.ConsoleSessionConfiguration);
         }
 
-        [Fact]
+        [Fact, TestPriority(7)]
         public void Utils_GetPolicySetting_EmptyUserConfig()
         {
             fixture.SetupConfigFile2();
@@ -602,7 +603,7 @@ namespace PSTests.Sequential
             fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, fixture.SystemWidePolicies.ConsoleSessionConfiguration);
         }
 
-        [Fact]
+        [Fact, TestPriority(8)]
         public void Utils_GetPolicySetting_EmptySystemConfig()
         {
             fixture.SetupConfigFile3();
@@ -701,7 +702,7 @@ namespace PSTests.Sequential
             fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, null);
         }
 
-        [Fact]
+        [Fact, TestPriority(9)]
         public void Utils_GetPolicySetting_BothConfigFilesEmpty()
         {
             fixture.SetupConfigFile4();
@@ -800,7 +801,7 @@ namespace PSTests.Sequential
             fixture.CompareConsoleSessionConfiguration(consoleSessionConfiguration, null);
         }
 
-        [Fact]
+        [Fact, TestPriority(10)]
         public void Utils_GetPolicySetting_BothConfigFilesNotExist()
         {
             fixture.CleanupConfigFiles();

test/xUnit/csharp/test_Runspace.cs

@@ -66,7 +66,8 @@ namespace PSTests.Parallel
         [Fact]
         public void TestRunspaceWithPowerShellAndInitialSessionState()
         {
-            InitialSessionState iss = InitialSessionState.CreateDefault2();
+            // CreateDefault2 is intentional.
+            InitialSessionState iss = InitialSessionState.CreateDefault();
 
             // NOTE: instantiate custom host myHost for the next line to capture stdout and stderr output
             //       in addition to just the PSObjects
@@ -90,9 +91,11 @@ namespace PSTests.Parallel
                         ++objCount;
                         Assert.NotNull(result);
                     }
+
                     Assert.Equal(count, objCount);
-                    powerShell.Dispose();
                 }
+
+                runspace.Close();
             }
         }
     }

test/xUnit/xUnit.tests.csproj

@@ -5,6 +5,7 @@
   <PropertyGroup>
     <Description>PowerShell xUnit Tests</Description>
     <AssemblyName>powershell-tests</AssemblyName>
+    <GenerateProgramFile>true</GenerateProgramFile>
     <RuntimeIdentifiers>win7-x86;win7-x64;osx-x64;linux-x64</RuntimeIdentifiers>
   </PropertyGroup>
 
@@ -26,5 +27,4 @@
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
     <PackageReference Include="XunitXml.TestLogger" Version="2.0.0" />
   </ItemGroup>
-
 </Project>

tools/appveyor.psm1

@@ -369,7 +369,6 @@ function Invoke-AppVeyorTest
     Write-Host -Foreground Green 'Run CoreCLR tests'
     $testResultsNonAdminFile = "$pwd\TestsResultsNonAdmin.xml"
     $testResultsAdminFile = "$pwd\TestsResultsAdmin.xml"
-    $SequentialXUnitTestResultsFile = "$pwd\SequentialXUnitTestResults.xml"
     $ParallelXUnitTestResultsFile = "$pwd\ParallelXUnitTestResults.xml"
     if(!(Test-Path "$env:CoreOutput\pwsh.exe"))
     {
@@ -443,19 +442,13 @@ function Invoke-AppVeyorTest
         Write-Host -Foreground Green 'Upload CoreCLR Admin test results'
         Update-AppVeyorTestResults -resultsFile $testResultsAdminFile
 
-        Start-PSxUnit -SequentialTestResultsFile $SequentialXUnitTestResultsFile -ParallelTestResultsFile $ParallelXUnitTestResultsFile
+        Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile
         Write-Host -ForegroundColor Green 'Uploading PSxUnit test results'
-        Update-AppVeyorTestResults -resultsFile $SequentialXUnitTestResultsFile
         Update-AppVeyorTestResults -resultsFile $ParallelXUnitTestResultsFile
 
         # Fail the build, if tests failed
         Test-PSPesterResults -TestResultsFile $testResultsAdminFile
-        @(
-            $SequentialXUnitTestResultsFile,
-            $ParallelXUnitTestResultsFile
-        ) | ForEach-Object {
-            Test-XUnitTestResults -TestResultsFile $_
-        }
+        Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile
 
         # Run tests with specified experimental features enabled
         foreach ($entry in $ExperimentalFeatureTests.GetEnumerator()) {

tools/travis.ps1

@@ -330,12 +330,11 @@ elseif($Stage -eq 'Build')
     }
 
     try {
-        $SequentialXUnitTestResultsFile = "$pwd/SequentialXUnitTestResults.xml"
         $ParallelXUnitTestResultsFile = "$pwd/ParallelXUnitTestResults.xml"
 
-        Start-PSxUnit -SequentialTestResultsFile $SequentialXUnitTestResultsFile -ParallelTestResultsFile $ParallelXUnitTestResultsFile
+        Start-PSxUnit -ParallelTestResultsFile $ParallelXUnitTestResultsFile
         # If there are failures, Test-XUnitTestResults throws
-        $SequentialXUnitTestResultsFile, $ParallelXUnitTestResultsFile | ForEach-Object { Test-XUnitTestResults -TestResultsFile $_ }
+        Test-XUnitTestResults -TestResultsFile $ParallelXUnitTestResultsFile
     }
     catch {
         $result = "FAIL"