Start automating benchmarks in pipeline (#1114)

* Run benchmarks in pipeline * fix msbuild not found * Move benchmarks to separate stage. * Enable cloning testwinrt * Attempt fix * Restore packages * Add results comparison * Fix yml file * Fix YML file * Adding baseline benchmarks * Add result comparison * Escape % * Increase threshold for now due to CPU differences and re-enable pipeline * Move comparison to benchmark.cmd to enable local comparisons. * Rearrange pipeline * Fix yml * Revert last 2 commits * Try adding depends * Undo * Try removing dependency for benchmark * Move benchmark stage * Rename * Fix template * Fix template * Move around templates * Fix template * Remove depends * Rename * Rename * Run all benchmarks * Add component detection task
microsoft · Feb 25, 2022 · 330a511 · 330a511
1 parent 765ae46
commit 330a511
Show file tree

Hide file tree

Showing 19 changed files with 56,284 additions and 2 deletions.
diff --git a/build/AzurePipelineTemplates/CsWinRT-Benchmarks-Steps.yml b/build/AzurePipelineTemplates/CsWinRT-Benchmarks-Steps.yml
@@ -0,0 +1,91 @@
+steps:
+- checkout: self
+  clean: true
+  persistCredentials: true
+
+# Clone TestWinRT
+- task: CmdLine@2
+  displayName: Clone TestWinRT
+  inputs: 
+    script: get_testwinrt.cmd
+    workingDirectory: $(Build.SourcesDirectory)\src
+
+# Use .NET Core SDK 2.1
+- task: UseDotNet@2
+  displayName: Use .NET Core SDK 2.1
+  inputs:
+    version: 2.1.x
+    installationPath: C:\Users\VssAdministrator\AppData\Local\Microsoft\dotnet\
+    performMultiLevelLookup: true
+
+# Install .NET 5 SDK
+- task: PowerShell@2 
+  displayName: Install .NET 5 SDK
+  inputs:
+    targetType: inline
+    failOnStderr: true
+    script: |
+      Write-Host ##vso[task.setvariable variable=PATH;]${env:LocalAppData}\Microsoft\dotnet;${env:PATH};
+        
+      [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
+                
+      &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version "$($env:NET5_SDK_VERSION)" -Architecture "x64" -AzureFeed "$($env:NET5_SDK_FEED)"
+
+# Install .NET 6 SDK 
+- task: PowerShell@2
+  displayName: Install .NET 6 SDK
+  inputs:
+    targetType: inline
+    failOnStderr: true
+    script: |
+      Write-Host ##vso[task.setvariable variable=PATH;]${env:LocalAppData}\Microsoft\dotnet;${env:PATH}; 
+        
+      [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; 
+         
+      dotnet new globaljson --sdk-version "$($env:NET6_SDK_VERSION)" 
+         
+      &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version "$($env:NET6_SDK_VERSION)" -Architecture "x64" -AzureFeed "$($env:NET5_SDK_FEED)"
+
+# Verify .NET SDK
+- task: CmdLine@2
+  displayName: Verify .NET SDK
+  inputs:
+    workingDirectory: $(Build.SourcesDirectory)
+    script: |
+      where dotnet
+      dotnet --info
+
+# Run Benchmarks
+- task: CmdLine@2
+  displayName: Run Benchmarks for x64 Release
+  inputs:
+    workingDirectory: $(Build.SourcesDirectory)\src
+    script: |
+      if "%VSCMD_VER%"=="" (
+        pushd c:
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" >nul 2>&1
+        popd
+      )
+
+      benchmark.cmd
+
+# Component Detection
+- task: ComponentGovernanceComponentDetection@0
+  displayName: Component Detection
+
+# Stage Benchmark Results
+- task: CopyFiles@2
+  displayName: Stage Benchmark Results
+  condition: always()
+  inputs:
+    SourceFolder: $(Build.SourcesDirectory)\src\BenchmarkDotNet.Artifacts\results\
+    Contents: '*'
+    TargetFolder: $(Build.ArtifactStagingDirectory)\benchmarks
+
+# Publish Results
+- task: PublishBuildArtifacts@1
+  displayName: Publish Results
+  condition: always()
+  inputs:
+    PathtoPublish: $(Build.ArtifactStagingDirectory)\benchmarks
+    ArtifactName: benchmarks
diff --git a/build/AzurePipelineTemplates/CsWinRT-BuildAndTest-Stage.yml b/build/AzurePipelineTemplates/CsWinRT-BuildAndTest-Stage.yml
@@ -425,3 +425,12 @@ stages:
           dir _build\$(BuildPlatform)\$(BuildConfiguration)\AuthoringConsumptionTest\bin
           _build\$(BuildPlatform)\$(BuildConfiguration)\AuthoringConsumptionTest\bin\AuthoringConsumptionTest.exe --gtest_output=xml:AUTHORINGTEST-$(Build.BuildNumber).xml 
           exit /b 0 
+
+  - job: Benchmarks
+    displayName: Run Benchmarks
+    dependsOn: []
+    pool:
+      vmImage: windows-2019
+    timeoutInMinutes: 120
+    steps:
+    - template: CsWinRT-Benchmarks-Steps.yml
diff --git a/src/Benchmarks/Program.cs b/src/Benchmarks/Program.cs
@@ -1,6 +1,7 @@
 using BenchmarkDotNet.Running;
 using System;
 using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Exporters.Json;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Toolchains.CsProj;
 using BenchmarkDotNet.Toolchains.DotNetCli;
@@ -33,7 +34,9 @@ public CustomConfig()
                             new MsBuildArgument("/p:platform=x64"),
                             new MsBuildArgument("/p:IsDotnetBuild=true")
                         }
-                    ).AsDefault();
+                    ).AsDefault();
+
+                Config = Config.AddExporter(JsonExporter.Full);
 
                 // Test WinMD support
 #if NETCOREAPP3_1