From 991924b817e69b35ddddda78fb796f30a78b468b Mon Sep 17 00:00:00 2001
From: Marco Castelluccio <mcastelluccio@mozilla.com>
Date: Thu, 3 Oct 2019 03:45:51 +0000
Subject: [PATCH] Bug 1465709 - Hook rust OOM handler on rustc 1.28. r=froydnj

Bug 1458161 added a rust OOM handler based on an unstable API that was
removed in 1.27, replaced with something that didn't allow to get the
failed allocation size.

Latest 1.28 nightly (2018-06-13) has
https://github.com/rust-lang/rust/pull/50880,
https://github.com/rust-lang/rust/pull/51264 and
https://github.com/rust-lang/rust/pull/51241 merged, which allow to
hook the OOM handler and get the failed allocation size again.

Because this is still an unstable API, we explicitly depend on strict
versions of rustc. We also explicitly error out if automation builds
end up using a rustc version that doesn't allow us to get the allocation
size for rust OOM, because we don't want that to happen without knowing.

UltraBlame original commit: 5182bca90d0609f182d5a7b6b48ed2ffbbce32c2
---
 toolkit/crashreporter/nsExceptionHandler.cpp  |   9 ++
 toolkit/library/gtest/rust/Cargo.toml         |  11 ++
 toolkit/library/rust/Cargo.toml               |  11 ++
 toolkit/library/rust/gkrust-features.mozbuild |  85 ++++++++++++
 toolkit/library/rust/shared/Cargo.toml        |   4 +
 toolkit/library/rust/shared/build.rs          |   8 ++
 toolkit/library/rust/shared/lib.rs            | 127 ++++++++++++++++++
 7 files changed, 255 insertions(+)

diff --git a/toolkit/crashreporter/nsExceptionHandler.cpp b/toolkit/crashreporter/nsExceptionHandler.cpp
index 37639f286209f..fb30a1c808422 100644
--- a/toolkit/crashreporter/nsExceptionHandler.cpp
+++ b/toolkit/crashreporter/nsExceptionHandler.cpp
@@ -901,6 +901,11 @@ install_rust_panic_hook
 (
 )
 ;
+void
+install_rust_oom_hook
+(
+)
+;
 bool
 get_rust_panic_reason
 (
@@ -8983,6 +8988,10 @@ install_rust_panic_hook
 (
 )
 ;
+install_rust_oom_hook
+(
+)
+;
 InitThreadAnnotation
 (
 )
diff --git a/toolkit/library/gtest/rust/Cargo.toml b/toolkit/library/gtest/rust/Cargo.toml
index 29497b759292e..5a2a5d26373d9 100644
--- a/toolkit/library/gtest/rust/Cargo.toml
+++ b/toolkit/library/gtest/rust/Cargo.toml
@@ -143,6 +143,17 @@ shared
 oom_with_global_alloc
 "
 ]
+oom_with_hook
+=
+[
+"
+gkrust
+-
+shared
+/
+oom_with_hook
+"
+]
 moz_memory
 =
 [
diff --git a/toolkit/library/rust/Cargo.toml b/toolkit/library/rust/Cargo.toml
index fa4e4ef0962f7..14dbfd001409a 100644
--- a/toolkit/library/rust/Cargo.toml
+++ b/toolkit/library/rust/Cargo.toml
@@ -141,6 +141,17 @@ shared
 oom_with_global_alloc
 "
 ]
+oom_with_hook
+=
+[
+"
+gkrust
+-
+shared
+/
+oom_with_hook
+"
+]
 moz_memory
 =
 [
diff --git a/toolkit/library/rust/gkrust-features.mozbuild b/toolkit/library/rust/gkrust-features.mozbuild
index 2f4df8f2babab..eeec42589b4fb 100644
--- a/toolkit/library/rust/gkrust-features.mozbuild
+++ b/toolkit/library/rust/gkrust-features.mozbuild
@@ -310,3 +310,88 @@ gkrust_features
 oom_with_global_alloc
 '
 ]
+elif
+CONFIG
+[
+'
+RUSTC_VERSION
+'
+]
+>
+=
+"
+1
+.
+28
+"
+and
+CONFIG
+[
+'
+RUSTC_VERSION
+'
+]
+<
+"
+1
+.
+29
+"
+:
+gkrust_features
++
+=
+[
+'
+oom_with_hook
+'
+]
+elif
+not
+CONFIG
+[
+'
+MOZ_AUTOMATION
+'
+]
+:
+#
+We
+don
+'
+t
+want
+builds
+on
+automation
+to
+unwillingly
+stop
+annotating
+OOM
+#
+crash
+reports
+from
+rust
+.
+error
+(
+'
+Builds
+on
+automation
+must
+use
+a
+version
+of
+rust
+that
+supports
+OOM
+'
+'
+hooking
+'
+)
diff --git a/toolkit/library/rust/shared/Cargo.toml b/toolkit/library/rust/shared/Cargo.toml
index c844ec6ad96c4..c53979dad3522 100644
--- a/toolkit/library/rust/shared/Cargo.toml
+++ b/toolkit/library/rust/shared/Cargo.toml
@@ -679,6 +679,10 @@ oom_with_global_alloc
 =
 [
 ]
+oom_with_hook
+=
+[
+]
 moz_memory
 =
 [
diff --git a/toolkit/library/rust/shared/build.rs b/toolkit/library/rust/shared/build.rs
index 7c824129cdfb1..77a368ea4bbfa 100644
--- a/toolkit/library/rust/shared/build.rs
+++ b/toolkit/library/rust/shared/build.rs
@@ -73,11 +73,19 @@ release
 [
 cfg
 (
+any
+(
 feature
 =
 "
 oom_with_global_alloc
 "
+feature
+=
+"
+oom_with_hook
+"
+)
 )
 ]
 println
diff --git a/toolkit/library/rust/shared/lib.rs b/toolkit/library/rust/shared/lib.rs
index 487fe0053a955..8cb7891aa2bcd 100644
--- a/toolkit/library/rust/shared/lib.rs
+++ b/toolkit/library/rust/shared/lib.rs
@@ -76,6 +76,22 @@ allocator_api
 )
 ]
 #
+!
+[
+cfg_attr
+(
+feature
+=
+"
+oom_with_hook
+"
+feature
+(
+oom_hook
+)
+)
+]
+#
 [
 cfg
 (
@@ -1557,3 +1573,114 @@ global_alloc
 :
 GeckoHeap
 ;
+#
+[
+cfg
+(
+feature
+=
+"
+oom_with_hook
+"
+)
+]
+mod
+oom_hook
+{
+use
+std
+:
+:
+alloc
+:
+:
+{
+Layout
+set_oom_hook
+}
+;
+extern
+"
+C
+"
+{
+fn
+GeckoHandleOOM
+(
+size
+:
+usize
+)
+-
+>
+!
+;
+}
+pub
+fn
+hook
+(
+layout
+:
+Layout
+)
+{
+unsafe
+{
+GeckoHandleOOM
+(
+layout
+.
+size
+(
+)
+)
+;
+}
+}
+pub
+fn
+install
+(
+)
+{
+set_oom_hook
+(
+hook
+)
+;
+}
+}
+#
+[
+no_mangle
+]
+pub
+extern
+"
+C
+"
+fn
+install_rust_oom_hook
+(
+)
+{
+#
+[
+cfg
+(
+feature
+=
+"
+oom_with_hook
+"
+)
+]
+oom_hook
+:
+:
+install
+(
+)
+;
+}