Helix proof of concept for unit tests

Winforms.sln

@@ -19,7 +19,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Windows.Forms.Design
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Windows.Forms.Design.Editors.Tests", "src\System.Windows.Forms.Design.Editors\tests\UnitTests\System.Windows.Forms.Design.Editors.Tests.csproj", "{27F24D0C-5F0A-472E-AE12-98B386A13D50}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Windows.Forms.Design.Tests", "src\System.Windows.Forms.Design\tests\System.Windows.Forms.Design.Tests.csproj", "{F977CA8C-FBF9-4D82-80BA-FE5B2F33486E}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Windows.Forms.Design.Tests", "src\System.Windows.Forms.Design\tests\UnitTests\System.Windows.Forms.Design.Tests.csproj", "{F977CA8C-FBF9-4D82-80BA-FE5B2F33486E}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Windows.Forms.Design", "src\System.Windows.Forms.Design\src\System.Windows.Forms.Design.csproj", "{61D06BBD-B0CF-4CE1-9139-1CC4B82F0F9B}"
 EndProject

eng/Version.Details.xml

@@ -57,5 +57,9 @@ Note: if the Uri is a new place, you will need to add a subscription from that p
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>1d2edb248bc2dc37fa0669053b0c88f3bffffb26</Sha>
     </Dependency>
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19156.20">
+      <Uri>https://github.com/dotnet/arcade</Uri>
+      <Sha>b94a591377451bda3ac80e597d7faff03a06c29e</Sha>
+    </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -24,10 +24,18 @@
     <MicrosoftDotNetGenFacadesPackageVersion>1.0.0-beta.19156.20</MicrosoftDotNetGenFacadesPackageVersion>
     <MicrosoftDotNetXUnitExtensionsPackageVersion>2.4.0-beta.19156.20</MicrosoftDotNetXUnitExtensionsPackageVersion>
   </PropertyGroup>
-  <!-- Additional dependencies -->
+  <!-- XUnit-related unchanging dependencies -->
   <PropertyGroup>
-    <AccessibilityPackageVersion>4.6.0-alpha-27122-5</AccessibilityPackageVersion>
+    <XUnitVersion>2.4.1-pre.build.4059</XUnitVersion>
+    <XUnitAssertVersion>$(XUnitVersion)</XUnitAssertVersion>
+    <XUnitRunnerConsoleVersion>$(XUnitVersion)</XUnitRunnerConsoleVersion>
+    <XUnitRunnerVisualStudioVersion>$(XUnitVersion)</XUnitRunnerVisualStudioVersion>
+    <XUnitExtensibilityExecutionVersion>$(XUnitVersion)</XUnitExtensibilityExecutionVersion>
     <XUnitStaFactPackageVersion>0.3.5</XUnitStaFactPackageVersion>
+  </PropertyGroup>
+  <!-- Additional unchanging dependencies -->
+  <PropertyGroup>
+    <AccessibilityPackageVersion>4.6.0-alpha-27122-5</AccessibilityPackageVersion>
     <MoqPackageVersion>4.10.0</MoqPackageVersion>
     <MicrosoftTargetingPackNETFrameworkv472PackageVersion>1.0.0</MicrosoftTargetingPackNETFrameworkv472PackageVersion>
   </PropertyGroup>

eng/ci-helix-test.yml

@@ -0,0 +1,29 @@
+parameters:
+  # All parameters are required for our purposes; see send-to-helix.yml for details
+  RepoName: ''            
+  HelixTargetQueues: ''    
+  HelixAccessToken: ''          
+  XUnitProjects: ''
+  DisplayNamePrefix: ''        
+
+steps:
+- template: /eng/common/templates/steps/send-to-helix.yml
+  parameters:
+    HelixSource: ${{ parameters.RepoName }}
+    HelixType: 'tests/default'      
+    HelixBuild: $(Build.BuildNumber)
+    HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+    HelixAccessToken: ${{ parameters.HelixAccessToken }}            # only defined for internal CI
+    Creator: ${{ parameters.RepoName }}         # required for public / external (when there is no access token)
+    XUnitProjects: ${{ parameters.XUnitProjects }} 
+    XUnitPublishTargetFramework: netcoreapp3.0  # Whatever tfm will work for our test projects for `dotnet publish`
+    XUnitRuntimeTargetFramework: netcoreapp2.0  # Whatever tfm to pick from the xunit package, it must exist in that package or the helix job will fail
+    XUnitRunnerVersion: '2.4.1'                 # Should match XUnitRunnerConsoleVersion in eng/Versions.props
+    IncludeDotNetCli: true
+    DotNetCliPackageType: 'sdk'
+    DotNetCliVersion: '3.0.100-preview-010184'  # MUST be official release: https://dotnet.microsoft.com/download/dotnet-core/3.0 ; does not need to match sdk we build against
+    EnableXUnitReporter: true
+    WaitForWorkItemCompletion: true
+    DisplayNamePrefix: ${{ parameters.DisplayNamePrefix }}
+    condition: succeeded()
+    continueOnError: false

eng/ci.yml

@@ -1,10 +1,8 @@
 parameters:
-
-  # Needed because agent pool can't be read from a user-defined variable (Azure DevOps limitation)
-  agentPool: dotnet-external-temp-vs2019
-
-  # Needed because runAsPublic is used in template expressions, which can't read from user-defined variables
-  runAsPublic: true
+  # This section is required because agent pool can't be read from a user-defined variable (Azure DevOps limitation)
+  agentPool: dotnet-external-temp-vs2019 
+  runAsPublic: true                      
+  repoName: dotnet/winforms              
 
 jobs:
 - template: /eng/common/templates/jobs/jobs.yml
@@ -15,7 +13,7 @@ jobs:
     enablePublishTestResults: true
     enablePublishBuildAssets: true
     enableTelemetry: true
-    helixRepo: dotnet/winforms
+    helixRepo: $(repoName)
 
     jobs:
     - job: Windows_NT
@@ -44,16 +42,19 @@ jobs:
           value: ''
 
         # needed for helix jobs
-        - name: _HelixAgentPool
+        - name: _UnitTestHelixAgentPool
           value: 'Windows.10.Amd64.Open'
         - name: _WinformsControlsTestBinDir
           value: $(BUILD.SOURCESDIRECTORY)\artifacts\bin\WinformsControlsTest\$(_BuildConfig)\netcoreapp3.0
         - name: _WinformsFuncTestBinDir
           value: $(BUILD.SOURCESDIRECTORY)\artifacts\bin\System.Windows.Forms.Func.Tests\$(_BuildConfig)\netcoreapp3.0
         - name: _HelixStagingDir
           value: $(BUILD.STAGINGDIRECTORY)\helix\functests
+        - name: _HelixToken
+          value: ''
 
-        # Override some values if we're building internally
+
+        # Override some values if we're building internally (not public)
         - ${{ if eq(parameters.runAsPublic, 'false') }}:
 
           # note: You have to use list syntax here (-name: value) or you will get errors about declaring the same variable multiple times
@@ -68,9 +69,8 @@ jobs:
           - name: _PublishBlobFeedUrl
             value: https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json
 
-          # note: Even though they are referenced here, user defined variables (like $(_SignType)) are not resolved 
-          # until the agent is running on the machine. They can be overridden any time before they are resolved,
-          # like in the job matrix below (see Build_Debug)
+          # note: Even though they are referenced here, user defined variables (like $(_SignType)) are not resolved until the agent
+          # is running on the machine. They can be overridden any time before they are resolved, like in the job matrix below (see Build_Debug)
           - name: _SignArgs
             value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:Sign=$(_Sign)
           - name: _PublishArgs
@@ -82,8 +82,11 @@ jobs:
           - name: _OfficialBuildIdArgs
             value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
 
-          - name: _HelixAgentPool
+          # Windows.81.Amd64 will fail on Debug.Assert in TryGetSystemMetricsForDpi in UnsafeNativeMethods
+          - name: _UnitTestHelixAgentPool
             value: 'Windows.10.Amd64'
+          - name: _HelixToken
+            value: '$(helix-token)'
 
       strategy:
         matrix:
@@ -109,15 +112,20 @@ jobs:
           /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Build.binlog          
         displayName: Build
 
-      # Run unit tests and rename binlog
-      - script: eng\cibuild.cmd
-          -configuration $(_BuildConfig)
-          $(_OfficialBuildIdArgs)
-          -test
-          /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Test.binlog
-        displayName: Run Unit Tests     
-
+      # Run unit tests on Helix
+      - template: /eng/ci-helix-test.yml
+        parameters:
+          RepoName: ${{ parameters.repoName }}
+          HelixTargetQueues: $(_UnitTestHelixAgentPool)
+          HelixAccessToken: $(_HelixToken)            # only defined for internal CI
+          # Should be $(BUILD.SOURCESDIRECTOR)System.Windows.Forms/src/**/*.Tests.csproj, but cannot until https://github.com/dotnet/arcade/issues/2156 is fixed
+          XUnitProjects: '$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms\tests\UnitTests\System.Windows.Forms.Tests.csproj;$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms.Design\tests\UnitTests\System.Windows.Forms.Design.Tests.csproj;$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms.Design.Editors\tests\UnitTests\System.Windows.Forms.Design.Editors.Tests.csproj'
+          DisplayNamePrefix: 'Send Unit Tests (Windows.10) to Helix'
+
       # Run integration tests and rename binlog
+      # These are not ready to run on Helix because:
+        # 1. Helix doesn't build projects (it only uses publish); we need .dotnet and also bin from build
+        # 2. we need an interactive pool for SendKeys to work
       - script: eng\cibuild.cmd
           -configuration $(_BuildConfig)
           $(_OfficialBuildIdArgs)
@@ -150,4 +158,4 @@ jobs:
         # Run component governance detection (only for release; only for internal)
         - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
           displayName: Component Governance Detection
-          condition: and(succeeded(), eq(variables['_BuildConfig'], 'Release'))
+          condition: and(succeeded(), eq(variables['_BuildConfig'], 'Release'))

eng/common/templates/steps/send-to-helix.yml

@@ -1,32 +1,33 @@
 parameters:
-  HelixSource: 'pr/default'          # required -- sources must start with pr/, official/, prodcon/, or agent/
-  HelixType: 'tests/default/'        # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
-  HelixBuild: $(Build.BuildNumber)   # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
-  HelixTargetQueues: ''              # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
-  HelixAccessToken: ''               # required -- access token to make Helix API requests; should be provided by the appropriate variable group
-  HelixPreCommands: ''               # optional -- commands to run before Helix work item execution
-  HelixPostCommands: ''              # optional -- commands to run after Helix work item execution
-  WorkItemDirectory: ''              # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
-  WorkItemCommand: ''                # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
-  WorkItemTimeout: ''                # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
-  CorrelationPayloadDirectory: ''    # optional -- a directory to zip up and send to Helix as a correlation payload
-  XUnitProjects: ''                  # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
-  XUnitPublishTargetFramework: ''    # optional -- framework to use to publish your xUnit projects
-  XUnitRuntimeTargetFramework: ''    # optional -- framework to use for the xUnit console runner
-  XUnitRunnerVersion: ''             # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
-  IncludeDotNetCli: false            # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
-  DotNetCliPackageType: ''           # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
-  DotNetCliVersion: ''               # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
-  EnableXUnitReporter: false         # optional -- true enables XUnit result reporting to Mission Control
-  WaitForWorkItemCompletion: true    # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
-  IsExternal: false                  # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
-  Creator: ''                        # optional -- if the build is external, use this to specify who is sending the job
-  condition: succeeded()             # optional -- condition for step to execute; defaults to succeeded()
-  continueOnError: false             # optional -- determines whether to continue the build if the step errors; defaults to false
+  HelixSource: 'pr/default'              # required -- sources must start with pr/, official/, prodcon/, or agent/
+  HelixType: 'tests/default/'            # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+  HelixBuild: $(Build.BuildNumber)       # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+  HelixTargetQueues: ''                  # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
+  HelixAccessToken: ''                   # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+  HelixPreCommands: ''                   # optional -- commands to run before Helix work item execution
+  HelixPostCommands: ''                  # optional -- commands to run after Helix work item execution
+  WorkItemDirectory: ''                  # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+  WorkItemCommand: ''                    # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+  WorkItemTimeout: ''                    # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
+  CorrelationPayloadDirectory: ''        # optional -- a directory to zip up and send to Helix as a correlation payload
+  XUnitProjects: ''                      # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+  XUnitPublishTargetFramework: ''        # optional -- framework to use to publish your xUnit projects
+  XUnitRuntimeTargetFramework: ''        # optional -- framework to use for the xUnit console runner
+  XUnitRunnerVersion: ''                 # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+  IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+  DotNetCliPackageType: ''               # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  EnableXUnitReporter: false             # optional -- true enables XUnit result reporting to Mission Control
+  WaitForWorkItemCompletion: true        # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+  IsExternal: false                      # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+  Creator: ''                            # optional -- if the build is external, use this to specify who is sending the job
+  DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO 
+  condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
+  continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
 
 steps:
   - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
-    displayName: Send job to Helix (Windows)
+    displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
     env:
       BuildConfig: $(_BuildConfig)
       HelixSource: ${{ parameters.HelixSource }}
@@ -54,7 +55,7 @@ steps:
     condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
     continueOnError: ${{ parameters.continueOnError }}
   - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
-    displayName: Send job to Helix (Unix)
+    displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
     env:
       BuildConfig: $(_BuildConfig)
       HelixSource: ${{ parameters.HelixSource }}

global.json

@@ -9,6 +9,7 @@
     "version": "3.0.100-preview-010024"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19156.20"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19156.20",
+    "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19156.20"
   }
 }

src/Common/tests/InternalUtilitiesForTests/InternalUtilitiesForTests.csproj

@@ -10,7 +10,7 @@
   <ItemGroup>
     <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
     <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
-    <PackageReference Include= "Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsPackageVersion)" />
+    <PackageReference Include="Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsPackageVersion)" />
     <PackageReference Include="Microsoft.Win32.Registry" Version="$(MicrosoftWin32RegistryPackageVersion)" />
   </ItemGroup>
 </Project>

src/System.Windows.Forms.Design.Editors/tests/UnitTests/System.Windows.Forms.Design.Editors.Tests.csproj

@@ -6,6 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>    
+    <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
     <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
   </ItemGroup>