SVE: Add support for fma-like intrinsics (#1474)

* Add support for fma-like intrinsics

* Factorize handling of sve masking for ignore_none

* USe _x when using ignore_none

include/eve/arch/arm/sve/sve_true.hpp

@@ -16,13 +16,12 @@ namespace eve::detail
 // Calling svptrue_b8 is OK for most cases, where you OK with
 // 1s in insignificant bits but sometimes you are not.
 template<typename T>
-svbool_t
+EVE_FORCEINLINE svbool_t
 sve_true()
 {
   if constexpr( sizeof(T) == 1 ) return svptrue_b8();
   else if constexpr( sizeof(T) == 2 ) return svptrue_b16();
   else if constexpr( sizeof(T) == 4 ) return svptrue_b32();
   else if constexpr( sizeof(T) == 8 ) return svptrue_b64();
 }
-
 } // namespace eve::detail

include/eve/module/core/regular/fam.hpp

@@ -87,3 +87,7 @@ EVE_MAKE_CALLABLE(fam_, fam);
 #if defined(EVE_INCLUDE_X86_HEADER)
 #  include <eve/module/core/regular/impl/simd/x86/fam.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/fam.hpp>
+#endif

include/eve/module/core/regular/fanm.hpp

@@ -87,3 +87,7 @@ EVE_MAKE_CALLABLE(fanm_, fanm);
 #if defined(EVE_INCLUDE_X86_HEADER)
 #  include <eve/module/core/regular/impl/simd/x86/fanm.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/fanm.hpp>
+#endif

include/eve/module/core/regular/fma.hpp

@@ -95,3 +95,7 @@ EVE_MAKE_CALLABLE(fma_, fma);
 #if defined(EVE_INCLUDE_ARM_HEADER)
 #  include <eve/module/core/regular/impl/simd/arm/neon/fma.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/fma.hpp>
+#endif

include/eve/module/core/regular/fnma.hpp

@@ -91,3 +91,7 @@ EVE_MAKE_CALLABLE(fnma_, fnma);
 #if defined(EVE_INCLUDE_ARM_HEADER)
 #  include <eve/module/core/regular/impl/simd/arm/neon/fnma.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/fnma.hpp>
+#endif

include/eve/module/core/regular/fsm.hpp

@@ -87,3 +87,7 @@ EVE_MAKE_CALLABLE(fsm_, fsm);
 #if defined(EVE_INCLUDE_X86_HEADER)
 #  include <eve/module/core/regular/impl/simd/x86/fsm.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/fsm.hpp>
+#endif

include/eve/module/core/regular/fsnm.hpp

@@ -88,3 +88,7 @@ EVE_MAKE_CALLABLE(fsnm_, fsnm);
 #if defined(EVE_INCLUDE_X86_HEADER)
 #  include <eve/module/core/regular/impl/simd/x86/fsnm.hpp>
 #endif
+
+#if defined(EVE_INCLUDE_SVE_HEADER)
+#  include <eve/module/core/regular/impl/simd/arm/sve/fsnm.hpp>
+#endif

include/eve/module/core/regular/impl/simd/arm/sve/fam.hpp

@@ -0,0 +1,37 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/arch/arm/sve/sve_true.hpp>
+#include <eve/concept/value.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fam_(EVE_SUPPORTS(sve_), wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  return fam[ignore_none](v0, v1, v2);
+}
+
+template<conditional_expr C, arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fam_(EVE_SUPPORTS(sve_), C cond, wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  if constexpr( C::is_complete && C::is_inverted ) return svmla_x(sve_true<T>(), v0, v1, v2);
+  else
+  {
+    auto const alt = alternative(cond, v0, as(v0));
+    if constexpr( C::is_complete && !C::is_inverted ) return alt;
+    else return svmla_m(cond.mask(as<T>{}), alt, v1, v2);
+  }
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/fanm.hpp

@@ -0,0 +1,37 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/arch/arm/sve/sve_true.hpp>
+#include <eve/concept/value.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fanm_(EVE_SUPPORTS(sve_), wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  return fanm[ignore_none](v0, v1, v2);
+}
+
+template<conditional_expr C, arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fanm_(EVE_SUPPORTS(sve_), C cond, wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  if constexpr( C::is_complete && C::is_inverted ) return svmls_x(sve_true<T>(), v0, v1, v2);
+  else
+  {
+    auto const alt = alternative(cond, v0, as(v0));
+    if constexpr( C::is_complete && !C::is_inverted ) return alt;
+    else return svmls_m(cond.mask(as<T>{}), alt, v1, v2);
+  }
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/fma.hpp

@@ -0,0 +1,37 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/arch/arm/sve/sve_true.hpp>
+#include <eve/concept/value.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fma_(EVE_SUPPORTS(sve_), wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  return fma[ignore_none](v0, v1, v2);
+}
+
+template<conditional_expr C, arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fma_(EVE_SUPPORTS(sve_), C cond, wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  if constexpr( C::is_complete && C::is_inverted ) return svmad_x(sve_true<T>(), v0, v1, v2);
+  else
+  {
+    auto const alt = alternative(cond, v0, as(v0));
+    if constexpr( C::is_complete && !C::is_inverted ) return alt;
+    else return svmad_m(cond.mask(as<T>{}), alt, v1, v2);
+  }
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/fnma.hpp

@@ -0,0 +1,37 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/arch/arm/sve/sve_true.hpp>
+#include <eve/concept/value.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fnma_(EVE_SUPPORTS(sve_), wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  return fnma[ignore_none](v0, v1, v2);
+}
+
+template<conditional_expr C, arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fnma_(EVE_SUPPORTS(sve_), C cond, wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  if constexpr( C::is_complete && C::is_inverted ) return svmsb_x(sve_true<T>(), v0, v1, v2);
+  else
+  {
+    auto const alt = alternative(cond, v0, as(v0));
+    if constexpr( C::is_complete && !C::is_inverted ) return alt;
+    else return svmsb_m(cond.mask(as<T>{}), alt, v1, v2);
+  }
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/fsm.hpp

@@ -0,0 +1,22 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/concept/value.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fsm_(EVE_SUPPORTS(sve_), wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  return -fanm(v0, v1, v2);
+}
+}

include/eve/module/core/regular/impl/simd/arm/sve/fsnm.hpp

@@ -0,0 +1,22 @@
+//==================================================================================================
+/*
+  EVE - Expressive Vector Engine
+  Copyright : EVE Project Contributors
+  SPDX-License-Identifier: BSL-1.0
+*/
+//==================================================================================================
+#pragma once
+
+#include <eve/concept/value.hpp>
+#include <eve/detail/implementation.hpp>
+
+namespace eve::detail
+{
+template<arithmetic_scalar_value T, typename N>
+EVE_FORCEINLINE auto
+fsnm_(EVE_SUPPORTS(sve_), wide<T, N> v0, wide<T, N> v1, wide<T, N> v2) noexcept -> wide<T, N>
+requires sve_abi<abi_t<T, N>>
+{
+  return -fam(v0, v1, v2);
+}
+}

test/unit/module/core/fms.cpp

@@ -91,7 +91,7 @@ TTS_CASE_WITH("Check behavior of fms on all types full range",
   {
     TTS_ULP_EQUAL(fms((a0), (a1), (a2)),
                   map([&](auto e, auto f, auto g) -> v_t { return e * f - g; }, a0, a1, a2),
-                  2);
+                  5);
   }
   TTS_ULP_EQUAL(
       eve::pedantic(fms)((a0), (a1), (a2)),