Add support to produce WebAssembly runtime assets (#41777)

* Add support to produce WebAssembly runtime assets

* Add support to build in OSX and address PR feedback

* Change AND -> and in Directory.Build.props change region

* PR Feedback: run build in rolling CI builds as well

* Add TODO comment for WebAssembly OSGroup

.azure-ci.yml

@@ -55,6 +55,9 @@ resources:
   - container: ubuntu_1604_arm_cross_container
     image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548
 
+  - container: ubuntu_1604
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-bd0fe7c-20190923200211
+
 variables:
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}:
     - group: DotNet-Blob-Feed

Directory.Build.props

@@ -57,15 +57,20 @@
     <OSGroup Condition="'$(OSGroup)' == ''">$(DefaultOSGroup)</OSGroup>
     <ConfigurationGroup Condition="'$(ConfigurationGroup)' == ''">Debug</ConfigurationGroup>
     <HostArch>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant)</HostArch>
-    <ArchGroup Condition="'$(ArchGroup)' == '' AND '$(HostArch)' == 'arm'">arm</ArchGroup>
-    <ArchGroup Condition="'$(ArchGroup)' == '' AND '$(HostArch)' == 'arm64'">arm64</ArchGroup>
+    <ArchGroup Condition="'$(ArchGroup)' == '' and '$(HostArch)' == 'arm'">arm</ArchGroup>
+    <ArchGroup Condition="'$(ArchGroup)' == '' and '$(HostArch)' == 'arm64'">arm64</ArchGroup>
+    <ArchGroup Condition="'$(ArchGroup)' == '' and '$(OSGroup)' == 'WebAssembly'">wasm</ArchGroup>
     <ArchGroup Condition="'$(ArchGroup)' == ''">x64</ArchGroup>
 
+    <!-- RuntimeOS is calculated based on the build system OS, however if building for WebAssembly we need to let
+         the build system to use webassembly as the RuntimeOS for produced package RIDs. -->
+    <RuntimeOS Condition="'$(OSGroup)' == 'WebAssembly'">$(OSGroup.ToLowerInvariant())</RuntimeOS>
+
     <!-- Initialize BuildConfiguration from the individual properties if it wasn't already explicitly set -->
     <BuildConfiguration Condition="'$(BuildConfiguration)' == ''">$(TargetGroup)-$(OSGroup)-$(ConfigurationGroup)-$(ArchGroup)</BuildConfiguration>
 
     <!-- if PKGPROJ doesn't set BuildConfigurations, make sure it only builds for TargetGroup=package or BuildAllConfigurations -->
-    <BuildConfigurations Condition="'$(MSBuildProjectExtension)' == '.pkgproj' AND '$(BuildConfigurations)' == ''">package</BuildConfigurations>
+    <BuildConfigurations Condition="'$(MSBuildProjectExtension)' == '.pkgproj' and '$(BuildConfigurations)' == ''">package</BuildConfigurations>
   </PropertyGroup>
 
   <!-- Informs build tools to apply .NET Framework metadata if not a test project -->
@@ -130,13 +135,16 @@
     <ToolRuntimeRID Condition="'$(BuildingInsideVisualStudio)' == 'true'">$(_runtimeOS)-x64</ToolRuntimeRID>
     <ToolRuntimeRID Condition="'$(ToolRuntimeID)' == ''">$(_runtimeOS)-$(HostArch)</ToolRuntimeRID>
     <!-- We build linux-musl-arm on a ubuntu container, so we can't use the toolset build for alpine runtime. We need to use portable linux RID for our toolset in order to be able to use it. -->
-    <ToolRuntimeRID Condition="'$(_runtimeOS)' == 'linux-musl' AND $(ArchGroup.StartsWith('arm')) AND !$(HostArch.StartsWith('arm'))">linux-x64</ToolRuntimeRID>
+    <ToolRuntimeRID Condition="'$(_runtimeOS)' == 'linux-musl' and $(ArchGroup.StartsWith('arm')) and !$(HostArch.StartsWith('arm'))">linux-x64</ToolRuntimeRID>
 
-    <!-- There are no WebAssembly tools, so treat them as Windows -->
-    <ToolRuntimeRID Condition="'$(RuntimeOS)' == 'WebAssembly'">win-x64</ToolRuntimeRID>
+    <!-- There are no WebAssembly tools, so use the default ones -->
+    <_buildingInOSX>$([MSBuild]::IsOSPlatform('OSX'))</_buildingInOSX>
+    <ToolRuntimeRID Condition="'$(RuntimeOS)' == 'webassembly' and '$(OS)' == 'Windows_NT'">win-x64</ToolRuntimeRID>
+    <ToolRuntimeRID Condition="'$(RuntimeOS)' == 'webassembly' and '$(OS)' != 'Windows_NT' and $(_buildingInOSX)">osx-x64</ToolRuntimeRID>
+    <ToolRuntimeRID Condition="'$(RuntimeOS)' == 'webassembly' and '$(OS)' != 'Windows_NT' and !$(_buildingInOSX)">linux-x64</ToolRuntimeRID>
 
     <!-- support cross-targeting by choosing a RID to restore when running on a different machine that what we're build for -->
-    <_portableOS Condition="'$(OSGroup)' == 'Unix' AND '$(_runtimeOSFamily)' != 'osx' AND '$(_runtimeOSFamily)' != 'FreeBSD' AND '$(_runtimeOS)' != 'linux-musl'">linux</_portableOS>
+    <_portableOS Condition="'$(OSGroup)' == 'Unix' and '$(_runtimeOSFamily)' != 'osx' and '$(_runtimeOSFamily)' != 'FreeBSD' AND '$(_runtimeOS)' != 'linux-musl'">linux</_portableOS>
 
     <_packageRID />
     <_packageRID Condition="'$(PortableBuild)' == 'true'">$(_portableOS)-$(ArchGroup)</_packageRID>

eng/configurations/osgroups.props

@@ -35,8 +35,10 @@
       <PackageTargetRuntime>netbsd</PackageTargetRuntime>
     </OSGroups>
     <OSGroups Include="WebAssembly">
-      <Imports>Unix</Imports>
-      <TargetsUnix>true</TargetsUnix>
+      <!-- TODO: we need to change this to import Unix instead whenever
+      we want to start using managed implementation for WebAssembly -->
+      <Imports>Linux</Imports>
+      <TargetsLinux>true</TargetsLinux>
       <TargetsWebAssembly>true</TargetsWebAssembly>
     </OSGroups>
     <OSGroups Include="AnyOS" />

eng/pipelines/linux.yml

@@ -134,35 +134,57 @@ stages:
             - linuxArm64Queues: \(Ubuntu.1604.Arm64.Open\)[email protected]/dotnet-buildtools/prereqs:ubuntu-16.04-helix-arm64v8-a45aeeb-20190620160300
 
       # Legs without helix testing
-      # Only run this leg in PRs.
-      - ${{ if and(eq(parameters.fullMatrix, 'false'), and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all'))) }}:
+      # Don't run these legs in outerloop builds
+      - ${{ if and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all')) }}:
         - job: LinuxNoTest
           displayName: Build
           strategy:
             matrix:
-              arm_Debug:
-                _BuildConfig: Debug
-                _architecture: arm
-                _framework: netcoreapp
-                _buildExtraArguments: /p:RuntimeOS=ubuntu.16.04 -warnAsError false
-                _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm '
-                _dockerContainer: ubuntu_1604_arm_cross_container
-
-              musl_arm64_Debug:
-                _BuildConfig: Debug
-                _architecture: arm64
-                _framework: netcoreapp
-                _dockerContainer: alpine_37_arm64_container
-                _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
-                _buildExtraArguments: -warnAsError false /p:BuildNativeCompiler=--clang5.0 /p:RuntimeOS=linux-musl
-
-              arm64_Debug:
-                _BuildConfig: Debug
-                _architecture: arm64
+              wasm_Release:
+                _BuildConfig: Release
+                _architecture: wasm
                 _framework: netcoreapp
-                _dockerContainer: ubuntu_1604_arm64_cross_container
-                _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
-                _buildExtraArguments: --warnAsError false
+                _dockerContainer: ubuntu_1604
+                _emsdkPath: $(Build.BinariesDirectory)/emsdk
+                _buildScriptPrefix: 'EMSDK_PATH=$(_emsdkPath) '
+                _buildExtraArguments: -os WebAssembly -warnAsError false
+                _installEmscripten: true
+
+              # Only run these legs in PRs.
+              ${{ if eq(parameters.fullMatrix, 'false') }}:
+                arm_Debug:
+                  _BuildConfig: Debug
+                  _architecture: arm
+                  _framework: netcoreapp
+                  _buildExtraArguments: /p:RuntimeOS=ubuntu.16.04 -warnAsError false
+                  _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm '
+                  _dockerContainer: ubuntu_1604_arm_cross_container
+
+                musl_arm64_Debug:
+                  _BuildConfig: Debug
+                  _architecture: arm64
+                  _framework: netcoreapp
+                  _dockerContainer: alpine_37_arm64_container
+                  _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
+                  _buildExtraArguments: -warnAsError false /p:BuildNativeCompiler=--clang5.0 /p:RuntimeOS=linux-musl
+
+                arm64_Debug:
+                  _BuildConfig: Debug
+                  _architecture: arm64
+                  _framework: netcoreapp
+                  _dockerContainer: ubuntu_1604_arm64_cross_container
+                  _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 '
+                  _buildExtraArguments: --warnAsError false
+
+          preBuildSteps:
+            - script: |
+                EMSCRIPTEN_VERSION=1.38.47
+                git clone https://github.com/emscripten-core/emsdk.git $(_emsdkPath)
+                cd $(_emsdkPath)
+                ./emsdk install ${EMSCRIPTEN_VERSION}-upstream
+                ./emsdk activate --embedded ${EMSCRIPTEN_VERSION}-upstream
+              displayName: Install Emscripten
+              condition: and(succeeded(), eq(variables['_installEmscripten'], 'true'))
 
           pool:
             name: Hosted Ubuntu 1604

eng/restore/runtime/runtime.depproj

@@ -1,6 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <RuntimeIdentifier>$(PackageRID)</RuntimeIdentifier>
+    <!-- We're using ToolRuntimeRID as a placeholder for the real corelib/runtime components until we have an actual set of runtime bits to consume for webassembly. -->
+    <RuntimeIdentifier Condition="'$(RuntimeOS)' == 'webassembly'">$(ToolRuntimeRID)</RuntimeIdentifier>
     <NoWarn>$(NoWarn);NU1603;NU1605</NoWarn>
     <SwapNativeForIL Condition="'$(SwapNativeForIL)' == '' AND ('$(ConfigurationGroup)' == 'Debug' OR '$(Coverage)' == 'true')">true</SwapNativeForIL>
   </PropertyGroup>

src/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj

@@ -3,7 +3,7 @@
     <DefineConstants>$(DefineConstants);XMLSERIALIZERGENERATORTESTS</DefineConstants>
     <Configurations>netcoreapp-Debug;netcoreapp-Release</Configurations>
     <CoverageSupported>false</CoverageSupported>
-    <SkipTestsOnPlatform Condition="'$(ArchGroup)' == 'arm' OR '$(ArchGroup)' == 'arm64' OR '$(ArchGroup)' == 'armel'">true</SkipTestsOnPlatform>
+    <SkipTestsOnPlatform Condition="'$(ArchGroup)' == 'arm' or '$(ArchGroup)' == 'arm64' or '$(ArchGroup)' == 'armel' or '$(ArchGroup)' == 'wasm'">true</SkipTestsOnPlatform>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Reuse the same runtimeconfig used by MSBuild. -->

src/Native/Unix/CMakeLists.txt

@@ -18,7 +18,6 @@ add_compile_options(-Wno-empty-translation-unit)
 add_compile_options(-Wno-cast-align)
 add_compile_options(-Wno-typedef-redefinition)
 add_compile_options(-Wno-c11-extensions)
-add_compile_options(-fPIC)
 add_compile_options(-I${CMAKE_CURRENT_SOURCE_DIR}/Common)
 add_compile_options(-I${CMAKE_CURRENT_BINARY_DIR}/Common)
 add_compile_options(-g)
@@ -29,9 +28,19 @@ endif()
 add_compile_options(-Werror)
 
 if(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+    # Build a static library so no -fPIC
     set(CLR_CMAKE_PLATFORM_WASM 1)
     add_definitions(-D_WASM_)
+    # The emscripten build has additional warnings so -Werror breaks
+	add_compile_options(-Wno-unused-parameter)
+    add_compile_options(-Wno-unused-function)
+	add_compile_options(-Wno-alloca)
+    add_compile_options(-Wno-implicit-int-float-conversion)
+else()
+  add_compile_options(-fPIC)
+  set(GEN_SHARED_LIB 1)
 endif(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+
 if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64)
     add_definitions(-DBIT64=1)
     add_definitions(-D_AMD64_)
@@ -95,7 +104,8 @@ else ()
     message(FATAL_ERROR "Unknown build type. Set CMAKE_BUILD_TYPE to DEBUG or RELEASE.")
 endif ()
 
-if (APPLE)
+if(CMAKE_SYSTEM_NAME STREQUAL Emscripten)
+elseif (APPLE)
     add_definitions(-D__APPLE_USE_RFC_3542)
 
    # We cannot enable "stack-protector-strong" on OS X due to a bug in clang compiler (current version 7.0.2)

src/Native/Unix/Common/pal_config.h.in

@@ -67,6 +67,7 @@
 #cmakedefine01 HAVE_SYS_SYSCTL_H
 #cmakedefine01 HAVE_NET_IFMEDIA_H
 #cmakedefine01 HAVE_LINUX_RTNETLINK_H
+#cmakedefine01 HAVE_LINUX_CAN_H
 #cmakedefine01 HAVE_GETDOMAINNAME_SIZET
 #cmakedefine01 HAVE_INOTIFY
 #cmakedefine01 HAVE_CLOCK_MONOTONIC

src/Native/Unix/System.Native/CMakeLists.txt

@@ -31,12 +31,25 @@ if (CMAKE_SYSTEM_NAME STREQUAL Linux)
     endif ()
 endif ()
 
-add_library(System.Native
-    SHARED
-    ${NATIVE_SOURCES}
-    ${VERSION_FILE_PATH}
-)
+if (GEN_SHARED_LIB)
+    add_library(System.Native
+        SHARED
+        ${NATIVE_SOURCES}
+        ${VERSION_FILE_PATH}
+    )
+    if (CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT CLR_CMAKE_PLATFORM_ANDROID)
+        target_link_libraries(System.Native rt)
+    endif ()
 
+    if (CMAKE_SYSTEM_NAME STREQUAL FreeBSD)
+        target_link_libraries(System.Native pthread)
+    if (HAVE_INOTIFY)
+        find_library(INOTIFY_LIBRARY inotify HINTS /usr/local/lib)
+        target_link_libraries(System.Native ${INOTIFY_LIBRARY})
+    endif ()
+    endif ()
+    install_library_and_symbols (System.Native)
+endif ()
 
 add_library(System.Native-Static
     STATIC
@@ -48,17 +61,4 @@ add_library(System.Native-Static
 set_target_properties(System.Native-Static PROPERTIES PREFIX "")
 set_target_properties(System.Native-Static PROPERTIES OUTPUT_NAME System.Native CLEAN_DIRECT_OUTPUT 1)
 
-if (CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT CLR_CMAKE_PLATFORM_ANDROID)
-    target_link_libraries(System.Native rt)
-endif ()
-
-if (CMAKE_SYSTEM_NAME STREQUAL FreeBSD)
-    target_link_libraries(System.Native pthread)
-if (HAVE_INOTIFY)
-    find_library(INOTIFY_LIBRARY inotify HINTS /usr/local/lib)
-    target_link_libraries(System.Native ${INOTIFY_LIBRARY})
-endif()
-endif ()
-
-install_library_and_symbols (System.Native)
 install (TARGETS System.Native-Static DESTINATION .)

src/Native/Unix/System.Native/pal_networking.c

@@ -55,7 +55,7 @@
 #include <ifaddrs.h>
 #endif
 #endif
-#ifdef AF_CAN
+#if HAVE_LINUX_CAN_H
 #include <linux/can.h>
 #endif
 #if HAVE_KQUEUE
@@ -2001,7 +2001,7 @@ static bool TryConvertProtocolTypePalToPlatform(int32_t palAddressFamily, int32_
             *platformProtocolType = palProtocolType;
             return true;
 #endif
-#ifdef AF_CAN
+#if HAVE_LINUX_CAN_H
         case AddressFamily_AF_CAN:
             switch (palProtocolType)
             {
@@ -2492,8 +2492,7 @@ static int32_t WaitForSocketEventsInner(int32_t port, SocketEvent* buffer, int32
 }
 
 #else
-#warning epoll/kqueue not detected; building with stub socket events support
-static const size_t SocketEventBufferElementSize = sizeof(struct pollfd);
+static const size_t SocketEventBufferElementSize = 0;
 
 static SocketEvents GetSocketEvents(int16_t filter, uint16_t flags)
 {

src/Native/Unix/System.Native/pal_process.c

@@ -154,7 +154,7 @@ static int compare_groups(const void * a, const void * b)
 
 static int SetGroups(uint32_t* userGroups, int32_t userGroupsLength, uint32_t* processGroups)
 {
-#ifdef __linux__
+#if defined(__linux__) || defined(_WASM_)
     size_t platformGroupsLength = Int32ToSizeT(userGroupsLength);
 #else // BSD
     int platformGroupsLength = userGroupsLength;

src/Native/Unix/configure.cmake

@@ -655,6 +655,10 @@ check_include_files(
     linux/rtnetlink.h
     HAVE_LINUX_RTNETLINK_H)
 
+check_include_files(
+    linux/can.h
+    HAVE_LINUX_CAN_H)
+
 check_symbol_exists(
     getpeereid
     unistd.h

src/Native/Unix/gen-buildsys-clang.sh

@@ -44,6 +44,7 @@ else
   buildtype="$5"
 fi
 
+cmake_cmd=cmake
 cmake_extra_defines="-DCMAKE_BUILD_TYPE=$buildtype"
 if [[ -n "$CROSSCOMPILE" ]]; then
     if ! [[ -n "$ROOTFS_DIR" ]]; then
@@ -60,6 +61,13 @@ fi
 if [ "$build_arch" == "armel" ]; then
     cmake_extra_defines="$cmake_extra_defines -DARM_SOFTFP=1"
 fi
+if [ "$build_arch" == "wasm" ]; then
+   if [ "$EMSCRIPTEN_ROOT" == "" ]; then
+       EMSCRIPTEN_ROOT="$EMSDK_PATH/upstream/emscripten"
+   fi
+   cmake_cmd="emcmake cmake"
+   cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=$EMSCRIPTEN_ROOT/cmake/Modules/Platform/Emscripten.cmake -DEMSCRIPTEN_GENERATE_BITCODE_STATIC_LIBRARIES=1"
+fi
 
 __UnprocessedCMakeArgs=""
 if [ -z "$6" ]; then
@@ -68,6 +76,7 @@ else
     __UnprocessedCMakeArgs="$6"
 fi
 
-cmake $cmake_extra_defines \
+echo "Invoking \"$cmake_cmd $cmake_extra_defines $__UnprocessedCMakeArgs $1\""
+$cmake_cmd $cmake_extra_defines \
     $__UnprocessedCMakeArgs \
     $1

src/Native/Windows/gen-buildsys-win.bat

@@ -36,6 +36,14 @@ popd
 :DoGen
 
 if "%3" == "wasm" (
+    if "%EMSDK_PATH%" == "" (
+       echo "Error: Should set EMSDK_PATH environment variable pointing to emsdk root."
+       exit /B 1
+    )
+
+    if "%EMSCRIPTEN_ROOT%" == "" (
+      set EMSCRIPTEN_ROOT="%EMSDK_PATH/upstream/emscripten%"
+    )
     emcmake cmake "-DEMSCRIPTEN_GENERATE_BITCODE_STATIC_LIBRARIES=1" "-DCMAKE_TOOLCHAIN_FILE=%EMSCRIPTEN%/cmake/Modules/Platform/Emscripten.cmake" "-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%" -G "NMake Makefiles" %__sourceDir%
 ) else (
     "%CMakePath%" %__SDKVersion% "-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%" "-DCMAKE_INSTALL_PREFIX=%__CMakeBinDir%" -G "Visual Studio %__VSString%" -B. -H%1 %__ExtraCmakeParams%

src/Native/build-native.cmd

@@ -160,10 +160,6 @@ IF ERRORLEVEL 1 (
     goto :Failure
 )
 
-:: Copy results to native_aot since packaging expects a copy there too
-mkdir "%__artifactsDir%\bin\native\%__outConfig%-aot"
-copy "%__artifactsDir%\bin\native\%__outConfig%\*" "%__artifactsDir%\bin\native\%__outConfig%-aot"
-
 exit /B 0
 
 :Failure

src/Native/build-native.proj

@@ -20,7 +20,7 @@
         then we should consider calling Environment.ProcessorCount
       -->
       <_ProcessorCountArg> --numproc $(MSBuildNodeCount)</_ProcessorCountArg>
-      <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true'"> stripsymbols</_StripSymbolsArg>
+      <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true' and '$(OSGroup)' != 'WebAssembly'"> stripsymbols</_StripSymbolsArg>
       <_PortableBuildArg Condition="'$(PortableBuild)' == 'true'"> -portable</_PortableBuildArg>
 
       <!--