Skip to content

Commit

Permalink
Merge root and inlined item placement.
Browse files Browse the repository at this point in the history 
There's no longer any need for them to be separate, and putting them
together reduces the amount of code.
  • Loading branch information
@nnethercote
nnethercote committed Jun 21, 2023
1 parent 6f228e3 commit 105ac1c
Showing 1 changed file with 44 additions and 71 deletions.
115 changes: 44 additions & 71 deletions compiler/rustc_monomorphize/src/partitioning.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ struct PartitioningCx<'a, 'tcx> {
usage_map: &'a UsageMap<'tcx>,
}

struct PlacedRootMonoItems<'tcx> {
struct PlacedMonoItems<'tcx> {
/// The codegen units, sorted by name to make things deterministic.
codegen_units: Vec<CodegenUnit<'tcx>>,

Expand All @@ -150,36 +150,20 @@ where

let cx = &PartitioningCx { tcx, usage_map };

// In the first step, we place all regular monomorphizations into their
// respective 'home' codegen unit. Regular monomorphizations are all
// functions and statics defined in the local crate.
let PlacedRootMonoItems { mut codegen_units, internalization_candidates, unique_inlined_stats } = {
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_roots");
let mut placed = place_root_mono_items(cx, mono_items);
// Place all mono items into a codegen unit.
let PlacedMonoItems { mut codegen_units, internalization_candidates, unique_inlined_stats } = {
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_items");
let mut placed = place_mono_items(cx, mono_items);

for cgu in &mut placed.codegen_units {
cgu.create_size_estimate(tcx);
}

debug_dump(tcx, "ROOTS", &placed.codegen_units, placed.unique_inlined_stats);
debug_dump(tcx, "PLACE", &placed.codegen_units, placed.unique_inlined_stats);

placed
};

// Use the usage map to put additional mono items in each codegen unit:
// drop-glue, functions from external crates, and local functions the
// definition of which is marked with `#[inline]`.
{
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_inline_items");
place_inlined_mono_items(cx, &mut codegen_units);

for cgu in &mut codegen_units {
cgu.create_size_estimate(tcx);
}

debug_dump(tcx, "INLINE", &codegen_units, unique_inlined_stats);
}

// Merge until we have at most `max_cgu_count` codegen units.
// `merge_codegen_units` is responsible for updating the CGU size
// estimates.
Expand Down Expand Up @@ -211,10 +195,7 @@ where
codegen_units
}

fn place_root_mono_items<'tcx, I>(
cx: &PartitioningCx<'_, 'tcx>,
mono_items: I,
) -> PlacedRootMonoItems<'tcx>
fn place_mono_items<'tcx, I>(cx: &PartitioningCx<'_, 'tcx>, mono_items: I) -> PlacedMonoItems<'tcx>
where
I: Iterator<Item = MonoItem<'tcx>>,
{
Expand All @@ -235,6 +216,8 @@ where
let mut num_unique_inlined_items = 0;
let mut unique_inlined_items_size = 0;
for mono_item in mono_items {
// Handle only root items directly here. Inlined items are handled at
// the bottom of the loop based on reachability.
match mono_item.instantiation_mode(cx.tcx) {
InstantiationMode::GloballyShared { .. } => {}
InstantiationMode::LocalCopy => {
Expand All @@ -247,7 +230,7 @@ where
let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item);
let is_volatile = is_incremental_build && mono_item.is_generic_fn();

let codegen_unit_name = match characteristic_def_id {
let cgu_name = match characteristic_def_id {
Some(def_id) => compute_codegen_unit_name(
cx.tcx,
cgu_name_builder,
Expand All @@ -258,9 +241,7 @@ where
None => fallback_cgu_name(cgu_name_builder),
};

let codegen_unit = codegen_units
.entry(codegen_unit_name)
.or_insert_with(|| CodegenUnit::new(codegen_unit_name));
let cgu = codegen_units.entry(cgu_name).or_insert_with(|| CodegenUnit::new(cgu_name));

let mut can_be_internalized = true;
let (linkage, visibility) = mono_item_linkage_and_visibility(
Expand All @@ -273,23 +254,52 @@ where
internalization_candidates.insert(mono_item);
}

codegen_unit.items_mut().insert(mono_item, (linkage, visibility));
cgu.items_mut().insert(mono_item, (linkage, visibility));

// Get all inlined items that are reachable from `mono_item` without
// going via another root item. This includes drop-glue, functions from
// external crates, and local functions the definition of which is
// marked with `#[inline]`.
let mut reachable_inlined_items = FxHashSet::default();
get_reachable_inlined_items(cx.tcx, mono_item, cx.usage_map, &mut reachable_inlined_items);

// Add those inlined items. It's possible an inlined item is reachable
// from multiple root items within a CGU, which is fine, it just means
// the `insert` will be a no-op.
for inlined_item in reachable_inlined_items {
// This is a CGU-private copy.
cgu.items_mut().insert(inlined_item, (Linkage::Internal, Visibility::Default));
}
}

// Always ensure we have at least one CGU; otherwise, if we have a
// crate with just types (for example), we could wind up with no CGU.
if codegen_units.is_empty() {
let codegen_unit_name = fallback_cgu_name(cgu_name_builder);
codegen_units.insert(codegen_unit_name, CodegenUnit::new(codegen_unit_name));
let cgu_name = fallback_cgu_name(cgu_name_builder);
codegen_units.insert(cgu_name, CodegenUnit::new(cgu_name));
}

let mut codegen_units: Vec<_> = codegen_units.into_values().collect();
codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str()));

PlacedRootMonoItems {
return PlacedMonoItems {
codegen_units,
internalization_candidates,
unique_inlined_stats: (num_unique_inlined_items, unique_inlined_items_size),
};

fn get_reachable_inlined_items<'tcx>(
tcx: TyCtxt<'tcx>,
item: MonoItem<'tcx>,
usage_map: &UsageMap<'tcx>,
visited: &mut FxHashSet<MonoItem<'tcx>>,
) {
usage_map.for_each_inlined_used_item(tcx, item, |inlined_item| {
let is_new = visited.insert(inlined_item);
if is_new {
get_reachable_inlined_items(tcx, inlined_item, usage_map, visited);
}
});
}
}

Expand Down Expand Up @@ -407,43 +417,6 @@ fn merge_codegen_units<'tcx>(
codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str()));
}

fn place_inlined_mono_items<'tcx>(
cx: &PartitioningCx<'_, 'tcx>,
codegen_units: &mut [CodegenUnit<'tcx>],
) {
for cgu in codegen_units.iter_mut() {
// Collect all inlined items that need to be available in this codegen unit.
let mut reachable_inlined_items = FxHashSet::default();
for root in cgu.items().keys() {
// Get all inlined items that are reachable from it without going
// via another root item.
get_reachable_inlined_items(cx.tcx, *root, cx.usage_map, &mut reachable_inlined_items);
}

// Add all monomorphizations that are not already there.
for inlined_item in reachable_inlined_items {
assert!(!cgu.items().contains_key(&inlined_item));

// This is a CGU-private copy.
cgu.items_mut().insert(inlined_item, (Linkage::Internal, Visibility::Default));
}
}

fn get_reachable_inlined_items<'tcx>(
tcx: TyCtxt<'tcx>,
item: MonoItem<'tcx>,
usage_map: &UsageMap<'tcx>,
visited: &mut FxHashSet<MonoItem<'tcx>>,
) {
usage_map.for_each_inlined_used_item(tcx, item, |inlined_item| {
let is_new = visited.insert(inlined_item);
if is_new {
get_reachable_inlined_items(tcx, inlined_item, usage_map, visited);
}
});
}
}

fn internalize_symbols<'tcx>(
cx: &PartitioningCx<'_, 'tcx>,
codegen_units: &mut [CodegenUnit<'tcx>],
Expand Down

0 comments on commit 105ac1c

Please sign in to comment.