Enable publishing in VMR (#18917)

Co-authored-by: Přemek Vysoký <[email protected]> Co-authored-by: Viktor Hofer <[email protected]>
dotnet · Mar 15, 2024 · 3384a8d · 3384a8d
1 parent 3ee5304
commit 3384a8d
Show file tree

Hide file tree

Showing 19 changed files with 2,130 additions and 11 deletions.
diff --git a/eng/Publishing.props b/eng/Publishing.props
@@ -135,7 +135,12 @@
       <IsStableBuild Condition="'$(DotNetFinalVersionKind)' == 'release'">true</IsStableBuild>
     </PropertyGroup>
 
+    <!--
+      Delete PushToAzureDevOpsArtifacts task and remove condition from PushToBuildStorage task
+      after this repo picks up arcade with the following change: https://github.com/dotnet/arcade/pull/14559
+    -->
     <PushToAzureDevOpsArtifacts
+      Condition="'$(DotNetBuildOrchestrator)' != 'true'"
       AzureDevOpsCollectionUri="$(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)"
       AzureDevOpsProject="$(SYSTEM_TEAMPROJECT)"
       AzureDevOpsBuildId="$(BUILD_BUILDID)"
@@ -154,5 +159,31 @@
       PublishFlatContainer="true"
       IsStableBuild="$(IsStableBuild)"
       PublishingVersion="3" />
+
+    <PushToBuildStorage
+      Condition="'$(DotNetBuildOrchestrator)' == 'true'"
+      AzureDevOpsCollectionUri="$(SYSTEM_TEAMFOUNDATIONCOLLECTIONURI)"
+      AzureDevOpsProject="$(SYSTEM_TEAMPROJECT)"
+      AzureDevOpsBuildId="$(BUILD_BUILDID)"
+      ItemsToPush="@(SdkAssetsToPushToBlobFeed);@(ChecksumsToPushToBlobFeed)"
+      ItemsToSign="@(ItemsToSignPostBuild)"
+      CertificatesSignInfo="@(CertificatesSignInfo)"
+      StrongNameSignInfo="@(StrongNameSignInfo)"
+      FileSignInfo="@(FileSignInfo)"
+      FileExtensionSignInfo="@(FileExtensionSignInfo)"
+      ManifestBuildData="@(ManifestBuildData)"
+      ManifestRepoName="$(BUILD_REPOSITORY_NAME)"
+      ManifestBranch="$(BUILD_SOURCEBRANCH)"
+      ManifestBuildId="$(BUILD_BUILDNUMBER)"
+      ManifestCommit="$(BUILD_SOURCEVERSION)"
+      AssetManifestPath="$(InstallersAssetManifestFilePath)"
+      PublishFlatContainer="true"
+      IsStableBuild="$(IsStableBuild)"
+      PublishingVersion="3"
+      PushToLocalStorage="$(PushToLocalStorage)"
+      AssetsLocalStorageDir="$(SourceBuiltAssetsDir)"
+      ShippingPackagesLocalStorageDir="$(SourceBuiltShippingPackagesDir)"
+      NonShippingPackagesLocalStorageDir="$(SourceBuiltNonShippingPackagesDir)"
+      AssetManifestsLocalStorageDir="$(SourceBuiltAssetManifestsDir)" />
   </Target>
 </Project>
diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props
@@ -173,7 +173,9 @@
 
     <VSMSBuildSdkResolversDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers'))</VSMSBuildSdkResolversDir>
     <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
+    <AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'AssetManifests'))</AssetManifestsIntermediateDir>
     <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
+    <ArtifactsAssetsSymbolsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsAssetsDir)', 'Symbols'))</ArtifactsAssetsSymbolsDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
     <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>

diff --git a/src/SourceBuild/content/Directory.Build.targets b/src/SourceBuild/content/Directory.Build.targets
@@ -7,7 +7,7 @@
       <SdkFilenamePrefix>dotnet-sdk-</SdkFilenamePrefix>
     </PropertyGroup>
     <ItemGroup>
-      <SdkTarballItem Include="$(ArtifactsAssetsDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsAssetsDir)Sdk/**/$(SdkFilenamePrefix)*$(ArchiveExtension)" />
     </ItemGroup>
 
     <!--

diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props
@@ -38,12 +38,14 @@
     <!-- Update nuget.config property to point to modified file that will be created/updated during build. -->
     <NuGetConfigFile Condition="'$(OriginalNuGetConfigFile)' != ''">$(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)'))</NuGetConfigFile>
 
+    <RepoAssetManifestsDir>$([MSBuild]::NormalizeDirectory('$(AssetManifestsIntermediateDir)', '$(RepositoryName)'))</RepoAssetManifestsDir>
+
     <SourceBuiltSdksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks'))</SourceBuiltSdksDir>
+
     <RepoArtifactsShippingPackagesDir Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$([MSBuild]::NormalizeDirectory('$(ArtifactsShippingPackagesDir)', '$(RepositoryName)'))</RepoArtifactsShippingPackagesDir>
     <RepoArtifactsShippingPackagesDir Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</RepoArtifactsShippingPackagesDir>
     <RepoArtifactsNonShippingPackagesDir Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$([MSBuild]::NormalizeDirectory('$(ArtifactsNonShippingPackagesDir)', '$(RepositoryName)'))</RepoArtifactsNonShippingPackagesDir>
     <RepoArtifactsNonShippingPackagesDir Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</RepoArtifactsNonShippingPackagesDir>
-
 
     <!-- Set the bootstrap version to the VMR's version if empty. (no bootstrap set). -->
     <ArcadeBootstrapVersion>$([MSBuild]::ValueOrDefault('$(ARCADE_BOOTSTRAP_VERSION)', '$(ArcadeSdkVersion)'))</ArcadeBootstrapVersion>
@@ -86,6 +88,11 @@
     <BuildArgs Condition="'$(NuGetConfigFile)' != ''">$(BuildArgs) /p:RestoreConfigFile=$(NuGetConfigFile)</BuildArgs>
     <!-- TODO rename this property https://github.com/dotnet/source-build/issues/4165 -->
     <BuildArgs Condition="'$(SourceBuildUseMonoRuntime)' == 'true'">$(BuildArgs) /p:SourceBuildUseMonoRuntime=$(SourceBuildUseMonoRuntime)</BuildArgs>
+    <!-- Pass locations for assets and packages -->
+    <BuildArgs>$(BuildArgs) /p:SourceBuiltAssetsDir=$(ArtifactsAssetsDir)</BuildArgs>
+    <BuildArgs>$(BuildArgs) /p:SourceBuiltShippingPackagesDir=$(RepoArtifactsShippingPackagesDir)</BuildArgs>
+    <BuildArgs>$(BuildArgs) /p:SourceBuiltNonShippingPackagesDir=$(RepoArtifactsNonShippingPackagesDir)</BuildArgs>
+    <BuildArgs>$(BuildArgs) /p:SourceBuiltAssetManifestsDir=$(RepoAssetManifestsDir)</BuildArgs>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">

diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets
@@ -24,6 +24,7 @@
     <ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
     <ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
     <RepoManifestFile>$([MSBuild]::NormalizePath('$(ProjectDirectory)', 'artifacts', 'RepoManifest.xml'))</RepoManifestFile>
+    <RepoAssetsSymbolsArchive>$(ArtifactsAssetsSymbolsDir)Symbols.$(RepositoryName)$(ArchiveExtension)</RepoAssetsSymbolsArchive>
 
     <IntermediateSymbolsRepoDir>$(IntermediateSymbolsRootDir)$(RepositoryName)</IntermediateSymbolsRepoDir>
   </PropertyGroup>
@@ -408,9 +409,13 @@
     <Message Importance="High" Text="See '$(RepoConsoleLogFile)' for more information." Condition="Exists('$(RepoConsoleLogFile)') and '$(MinimalConsoleLogOutput)' == 'true'" />
   </Target>
 
+  <!--
+    Remove this target after arcade changes for publishing have flown to installer repo.
+    https://github.com/dotnet/source-build/issues/4207
+  -->
   <Target Name="CopyRepoArtifacts"
           DependsOnTargets="RepoBuild"
-          Condition="'$(IsUtilityProject)' != 'true'"
+          Condition="'$(IsUtilityProject)' != 'true' and Exists('$(RepoManifestFile)')"
           Inputs="$(RepoManifestFile)"
           Outputs="$(BaseIntermediateOutputPath)CopyRepoArtifacts.complete">
     <Error Text="Repo manifest file doesn't exist." Condition="!Exists('$(RepoManifestFile)')" />
@@ -601,8 +606,18 @@
     </Touch>
   </Target>
 
+  <Target Name="DiscoverSymbolsArchiveFile"
+          Condition="'$(DotNetBuildSourceOnly)' == 'true' and
+                     '$(IsUtilityProject)' != 'true' and
+                     Exists('$(RepoAssetsSymbolsArchive)')">
+
+    <ItemGroup>
+      <RepoSymbolsArchive Include="$(RepoAssetsSymbolsArchive)" />
+    </ItemGroup>
+  </Target>
+
   <Target Name="ExtractSymbolsArchiveFile"
-          DependsOnTargets="CopyRepoArtifacts"
+          DependsOnTargets="CopyRepoArtifacts;DiscoverSymbolsArchiveFile"
           Condition="'$(DotNetBuildSourceOnly)' == 'true' and
                      '$(IsUtilityProject)' != 'true' and
                      '@(RepoSymbolsArchive)' != ''"
@@ -624,6 +639,7 @@
     CopyRepoArtifacts;
     CopyInnerBuildRestoredPackages;
     ExtractToolPackage;
+    DiscoverSymbolsArchiveFile;
     ExtractSymbolsArchiveFile;
     CleanupRepo" />
 

diff --git a/src/SourceBuild/content/repo-projects/aspnetcore.proj b/src/SourceBuild/content/repo-projects/aspnetcore.proj
@@ -4,8 +4,6 @@
     <!-- AspNetCore doesn't have a root build script but one under the eng folder. -->
     <BuildScript>$(ProjectDirectory)eng\build$(ShellExtension)</BuildScript>
 
-    <!-- BuildActions includes -publish which is not supported by the aspnetcore build script. -->
-    <BuildActions>$(FlagParameterPrefix)restore $(FlagParameterPrefix)build $(FlagParameterPrefix)pack</BuildActions>
     <!-- On Windows, build all for the VB PoC -->
     <BuildActions Condition="'$(DotNetBuildSourceOnly)' != 'true' and '$(BuildOS)' == 'windows'">$(FlagParameterPrefix)restore $(FlagParameterPrefix)all $(FlagParameterPrefix)pack</BuildActions>
 

diff --git a/src/SourceBuild/content/repo-projects/installer.proj b/src/SourceBuild/content/repo-projects/installer.proj
@@ -137,4 +137,15 @@
     <Message Importance="High" Text="Packaged source-built artifacts to $(SourceBuiltTarballName)" />
   </Target>
 
+  <!-- Copy SDK archive to assets root. -->
+  <Target Name="CopySdkArchive"
+          AfterTargets="Build"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion">
+
+    <Copy SourceFiles="@(SdkTarballItem)"
+          DestinationFolder="$(ArtifactsAssetsDir)"
+          SkipUnchangedFiles="true"
+          UseHardlinksIfPossible="true" />
+  </Target>
+
 </Project>
diff --git a/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj b/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj
@@ -31,7 +31,14 @@
       SourcePath="$(LocalNuGetPackageCacheDirectory)" />
   </Target>
 
+  <!--
+    Remove this target after VMR picks up new arcade publishing changes: https://github.com/dotnet/arcade/pull/14559
+    Tracking issue: https://github.com/dotnet/source-build/issues/4210
+
+    We only run this target if we're publishing using repo manifest.
+  -->
   <Target Name="ClearPreviousSBRP"
+          Condition="Exists('$(RepoManifestFile)')"
           BeforeTargets="CopyRepoArtifacts"
           DependsOnTargets="RepoBuild">
     <!-- Building SBRP: At this point the References directory contains the previously-source-built SBRPs,