Auto merge of #58486 - oli-obk:import_miri_from_future, r=<try>

always back `ty::Const` by an `Allocation`

This reduces the number of conversion between `Allocation` and `Value` by caching both.

r? @RalfJung

src/librustc/ich/impls_ty.rs

@@ -300,15 +300,18 @@ impl_stable_hash_for!(struct ty::FieldDef {
 });
 
 impl_stable_hash_for!(
-    impl<'tcx> for enum mir::interpret::ConstValue<'tcx> [ mir::interpret::ConstValue ] {
+    impl<'tcx> for enum mir::interpret::ConstValue [ mir::interpret::ConstValue ] {
         Scalar(val),
         Slice(a, b),
-        ByRef(id, alloc, offset),
+        ByRef,
     }
 );
+
 impl_stable_hash_for!(struct crate::mir::interpret::RawConst<'tcx> {
+    // FIXME(oli-obk): is ignoring the `alloc_id` for perf reasons ok?
     alloc_id,
     ty,
+    alloc,
 });
 
 impl_stable_hash_for! {
@@ -372,10 +375,30 @@ impl_stable_hash_for!(enum ::syntax::ast::Mutability {
     Mutable
 });
 
-impl_stable_hash_for!(struct ty::Const<'tcx> {
-    ty,
-    val
-});
+
+impl<'a, 'gcx> HashStable<StableHashingContext<'a>> for ty::Const<'gcx> {
+    fn hash_stable<W: StableHasherResult>(
+        &self,
+        hcx: &mut StableHashingContext<'a>,
+        hasher: &mut StableHasher<W>,
+    ) {
+        let ty::Const { ty, val, alloc } = self;
+        ty.hash_stable(hcx, hasher);
+        val.hash_stable(hcx, hasher);
+        // don't hash the memory for `Scalar` and `Slice`. There's nothing to be gained
+        // by it. All the relevant info is contained in the value.
+        if let mir::interpret::ConstValue::ByRef = val {
+            let (alloc, ptr) = alloc.unwrap();
+            // type check for future changes
+            let alloc: &'gcx mir::interpret::Allocation = alloc;
+            alloc.hash_stable(hcx, hasher);
+            ptr.offset.hash_stable(hcx, hasher);
+            // do not hash the alloc id in the pointer. It does not add anything new to the hash.
+            // If the hash of the alloc id is the same, then the hash of the allocation would also
+            // be the same.
+        }
+    }
+}
 
 impl_stable_hash_for!(impl<'tcx> for enum ty::LazyConst<'tcx> [ty::LazyConst] {
     Unevaluated(did, substs),

src/librustc/mir/interpret/value.rs

@@ -2,21 +2,27 @@ use std::fmt;
 
 use crate::ty::{Ty, layout::{HasDataLayout, Size}};
 
-use super::{EvalResult, Pointer, PointerArithmetic, Allocation, AllocId, sign_extend, truncate};
+use super::{EvalResult, Pointer, PointerArithmetic, AllocId, sign_extend, truncate, Allocation};
 
 /// Represents the result of a raw const operation, pre-validation.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, RustcEncodable, RustcDecodable, Hash)]
 pub struct RawConst<'tcx> {
-    // the value lives here, at offset 0, and that allocation definitely is a `AllocKind::Memory`
-    // (so you can use `AllocMap::unwrap_memory`).
+    /// the value lives here, at offset 0, and the allocation that it refers to is the one in the
+    /// `alloc` field
     pub alloc_id: AllocId,
     pub ty: Ty<'tcx>,
+    /// the allocation that would be returned by using
+    /// `tcx.alloc_map.lock().unwrap_memory(self.alloc_id)`
+    ///
+    /// This is an optimization so we don't actually have to go fetch said allocation from the
+    /// `alloc_map` at most use sites of the `const_eval_raw` query
+    pub alloc: &'tcx Allocation,
 }
 
 /// Represents a constant value in Rust. `Scalar` and `ScalarPair` are optimizations that
 /// match the `LocalState` optimizations for easy conversions between `Value` and `ConstValue`.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, RustcEncodable, RustcDecodable, Hash)]
-pub enum ConstValue<'tcx> {
+pub enum ConstValue {
     /// Used only for types with `layout::abi::Scalar` ABI and ZSTs.
     ///
     /// Not using the enum `Value` to encode that this must not be `Undef`.
@@ -31,19 +37,17 @@ pub enum ConstValue<'tcx> {
     /// it.
     Slice(Scalar, u64),
 
-    /// An allocation together with an offset into the allocation.
-    /// Invariant: the `AllocId` matches the allocation.
-    ByRef(AllocId, &'tcx Allocation, Size),
+    ByRef,
 }
 
 #[cfg(target_arch = "x86_64")]
-static_assert!(CONST_SIZE: ::std::mem::size_of::<ConstValue<'static>>() == 40);
+static_assert!(CONST_SIZE: ::std::mem::size_of::<ConstValue>() == 40);
 
-impl<'tcx> ConstValue<'tcx> {
+impl ConstValue {
     #[inline]
     pub fn try_to_scalar(&self) -> Option<Scalar> {
         match *self {
-            ConstValue::ByRef(..) |
+            ConstValue::ByRef |
             ConstValue::Slice(..) => None,
             ConstValue::Scalar(val) => Some(val),
         }

src/librustc/mir/mod.rs

@@ -1665,6 +1665,7 @@ impl<'tcx> TerminatorKind<'tcx> {
                                 }.into(),
                             ),
                             ty: switch_ty,
+                            alloc: None,
                         };
                         fmt_const_val(&mut s, c).unwrap();
                         s.into()

src/librustc/ty/structural_impls.rs

@@ -4,7 +4,6 @@
 //! `BraceStructLiftImpl!`) to help with the tedium.
 
 use crate::mir::ProjectionKind;
-use crate::mir::interpret::ConstValue;
 use crate::ty::{self, Lift, Ty, TyCtxt};
 use crate::ty::fold::{TypeFoldable, TypeFolder, TypeVisitor};
 use rustc_data_structures::indexed_vec::{IndexVec, Idx};
@@ -486,23 +485,20 @@ BraceStructLiftImpl! {
     }
 }
 
-BraceStructLiftImpl! {
-    impl<'a, 'tcx> Lift<'tcx> for ty::Const<'a> {
-        type Lifted = ty::Const<'tcx>;
-        val, ty
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ConstValue<'a> {
-    type Lifted = ConstValue<'tcx>;
+impl<'a, 'tcx> Lift<'tcx> for ty::Const<'a> {
+    type Lifted = ty::Const<'tcx>;
     fn lift_to_tcx<'b, 'gcx>(&self, tcx: TyCtxt<'b, 'gcx, 'tcx>) -> Option<Self::Lifted> {
-        match *self {
-            ConstValue::Scalar(x) => Some(ConstValue::Scalar(x)),
-            ConstValue::Slice(x, y) => Some(ConstValue::Slice(x, y)),
-            ConstValue::ByRef(x, alloc, z) => Some(ConstValue::ByRef(
-                x, alloc.lift_to_tcx(tcx)?, z,
-            )),
-        }
+        let ty = self.ty.lift_to_tcx(tcx)?;
+        let alloc = match self.alloc {
+            // can't use `and_then` or `map`, because the inner `lift_to_tcx` needs to early return
+            Some((a, p)) => Some((a.lift_to_tcx(tcx)?, p)),
+            None => None,
+        };
+        Some(ty::Const {
+            val: self.val,
+            ty,
+            alloc,
+        })
     }
 }
 
@@ -1064,24 +1060,14 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::LazyConst<'tcx> {
 impl<'tcx> TypeFoldable<'tcx> for ty::Const<'tcx> {
     fn super_fold_with<'gcx: 'tcx, F: TypeFolder<'gcx, 'tcx>>(&self, folder: &mut F) -> Self {
         let ty = self.ty.fold_with(folder);
-        let val = self.val.fold_with(folder);
         ty::Const {
             ty,
-            val
+            val: self.val,
+            alloc: self.alloc,
         }
     }
 
     fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
-        self.ty.visit_with(visitor) || self.val.visit_with(visitor)
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for ConstValue<'tcx> {
-    fn super_fold_with<'gcx: 'tcx, F: TypeFolder<'gcx, 'tcx>>(&self, _folder: &mut F) -> Self {
-        *self
-    }
-
-    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> bool {
-        false
+        self.ty.visit_with(visitor)
     }
 }

src/librustc/ty/sty.rs

@@ -11,8 +11,9 @@ use crate::ty::subst::{Substs, Subst, Kind, UnpackedKind};
 use crate::ty::{self, AdtDef, TypeFlags, Ty, TyCtxt, TypeFoldable};
 use crate::ty::{List, TyS, ParamEnvAnd, ParamEnv};
 use crate::util::captures::Captures;
-use crate::mir::interpret::{Scalar, Pointer};
+use crate::mir::interpret::{Scalar, Pointer, Allocation};
 
+use std::hash::{Hash, Hasher};
 use smallvec::SmallVec;
 use std::iter;
 use std::cmp::Ordering;
@@ -2065,7 +2066,7 @@ pub enum LazyConst<'tcx> {
 }
 
 #[cfg(target_arch = "x86_64")]
-static_assert!(LAZY_CONST_SIZE: ::std::mem::size_of::<LazyConst<'static>>() == 56);
+static_assert!(LAZY_CONST_SIZE: ::std::mem::size_of::<LazyConst<'static>>() == 80);
 
 impl<'tcx> LazyConst<'tcx> {
     pub fn map_evaluated<R>(self, f: impl FnOnce(Const<'tcx>) -> Option<R>) -> Option<R> {
@@ -2086,15 +2087,68 @@ impl<'tcx> LazyConst<'tcx> {
 }
 
 /// Typed constant value.
-#[derive(Copy, Clone, Debug, Hash, RustcEncodable, RustcDecodable, Eq, PartialEq, Ord, PartialOrd)]
+#[derive(Copy, Clone, Debug, RustcEncodable, RustcDecodable, Eq, Ord, PartialOrd)]
 pub struct Const<'tcx> {
     pub ty: Ty<'tcx>,
 
-    pub val: ConstValue<'tcx>,
+    /// This field is an optimization for caching commonly needed values of constants like `usize`
+    /// (or other integers for enum discriminants) and slices (e.g. from `b"foo"` and `"foo"`
+    /// literals)
+    pub val: ConstValue,
+
+    /// The actual backing storage of the constant and a pointer which can be resolved back to the
+    /// `allocation` field
+    ///
+    /// Can be `None` for trivial constants created from literals or directly. Is always `Some` for
+    /// aggregate constants or any named constant that you can actually end up taking a reference
+    /// to. This will get unwrapped in situations where we do know that it's a referencable
+    pub alloc: Option<(&'tcx Allocation, Pointer)>,
+}
+
+impl<'tcx> PartialEq for Const<'tcx> {
+    fn eq(&self, other: &Self) -> bool {
+
+        self.ty == other.ty && match (self.val, other.val) {
+            (ConstValue::ByRef, ConstValue::ByRef) => {
+                let (a, pa) = self.alloc.unwrap();
+                let (b, pb) = other.alloc.unwrap();
+                // only use the alloc ids to not have to compare the full allocations
+                // the ids may differ if the allocation is the same
+                (pa.offset == pb.offset) && (pa.alloc_id == pb.alloc_id || a == b)
+            },
+            // ignore the actual allocation, just compare the values
+            (ConstValue::Scalar(a), ConstValue::Scalar(b)) => a == b,
+            (ConstValue::Slice(a, an), ConstValue::Slice(b, bn)) => an == bn && a == b,
+            // if the values don't match, the consts can't be equal and the type equality should
+            // have already failed, because we make the decision for non-byref solely based on the
+            // type
+            _ => bug!("same type but different value kind in constant: {:#?} {:#?}", self, other),
+        }
+    }
+}
+
+impl<'tcx> Hash for Const<'tcx> {
+    fn hash<H: Hasher>(&self, hasher: &mut H) {
+        let Const { ty, val, alloc } = self;
+        ty.hash(hasher);
+        val.hash(hasher);
+        // don't hash the memory for `Scalar` and `Slice`. There's nothing to be gained
+        // by it. All the relevant info is contained in the value.
+        if let ConstValue::ByRef = val {
+            let (alloc, ptr) = alloc.unwrap();
+            // type check for future changes
+            let alloc: &'tcx Allocation = alloc;
+            alloc.hash(hasher);
+            ptr.offset.hash(hasher);
+            // do not hash the alloc id in the pointer. It does not add anything new to the hash.
+            // If the hash of the alloc id is the same, then the hash of the allocation would also
+            // be the same.
+        }
+    }
 }
 
 #[cfg(target_arch = "x86_64")]
-static_assert!(CONST_SIZE: ::std::mem::size_of::<Const<'static>>() == 48);
+static_assert!(CONST_SIZE: ::std::mem::size_of::<Const<'static>>() == 72);
 
 impl<'tcx> Const<'tcx> {
     #[inline]
@@ -2105,6 +2159,7 @@ impl<'tcx> Const<'tcx> {
         Self {
             val: ConstValue::Scalar(val),
             ty,
+            alloc: None,
         }
     }
 

src/librustc_codegen_llvm/consts.rs

@@ -1,7 +1,7 @@
 use libc::c_uint;
 use llvm::{self, SetUnnamedAddr, True};
 use rustc::hir::def_id::DefId;
-use rustc::mir::interpret::{ConstValue, Allocation, read_target_uint,
+use rustc::mir::interpret::{Allocation, read_target_uint,
     Pointer, ErrorHandled, GlobalId};
 use rustc::hir::Node;
 use debuginfo;
@@ -69,11 +69,11 @@ pub fn codegen_static_initializer(
     };
     let param_env = ty::ParamEnv::reveal_all();
     let static_ = cx.tcx.const_eval(param_env.and(cid))?;
-
-    let alloc = match static_.val {
-        ConstValue::ByRef(_, alloc, n) if n.bytes() == 0 => alloc,
+    let (alloc, ptr) = match static_.alloc {
+        Some(alloc) => alloc,
         _ => bug!("static const eval returned {:#?}", static_),
     };
+    assert_eq!(ptr.offset.bytes(), 0);
     Ok((const_alloc_to_llvm(cx, alloc), alloc))
 }
 

src/librustc_codegen_ssa/mir/operand.rs

@@ -101,8 +101,9 @@ impl<'a, 'tcx: 'a, V: CodegenObject> OperandRef<'tcx, V> {
                 let b_llval = bx.cx().const_usize(b);
                 OperandValue::Pair(a_llval, b_llval)
             },
-            ConstValue::ByRef(_, alloc, offset) => {
-                return Ok(bx.load_operand(bx.cx().from_const_alloc(layout, alloc, offset)));
+            ConstValue::ByRef => {
+                let (alloc, ptr) = val.alloc.unwrap();
+                return Ok(bx.load_operand(bx.cx().from_const_alloc(layout, alloc, ptr.offset)));
             },
         };
 

src/librustc_codegen_ssa/mir/place.rs

@@ -416,10 +416,8 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 };
                 let layout = cx.layout_of(self.monomorphize(&ty));
                 match bx.tcx().const_eval(param_env.and(cid)) {
-                    Ok(val) => match val.val {
-                        mir::interpret::ConstValue::ByRef(_, alloc, offset) => {
-                            bx.cx().from_const_alloc(layout, alloc, offset)
-                        }
+                    Ok(val) => match val.alloc {
+                        Some((alloc, ptr)) => bx.cx().from_const_alloc(layout, alloc, ptr.offset),
                         _ => bug!("promoteds should have an allocation: {:?}", val),
                     },
                     Err(_) => {