Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

Commit

Permalink
implement ancestors as rolling bit field
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeffwashington
jeffwashington committed May 26, 2021
1 parent a5c2067 commit cb1b479
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 138 deletions.
62 changes: 58 additions & 4 deletions runtime/src/accounts_index.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,30 @@ pub struct RollingBitField {
// We only expect these items in conditions where there is some other bug in the system.
excess: HashSet<u64>,
}

impl PartialEq<RollingBitField> for RollingBitField {
fn eq(&self, other: &Self) -> bool {
self.len() == other.len() && {
for item in self.get_all() {
if !other.contains(&item) {
return false;
}
}
true
}
}
}

impl Clone for RollingBitField {
fn clone(&self) -> Self {
let mut result = Self::new(self.max_width);
for item in self.get_all() {
result.insert(item);
}
result
}
}

// functionally similar to a hashset
// Relies on there being a sliding window of key values. The key values continue to increase.
// Old key values are removed from the lesser values and do not accumulate.
Expand Down Expand Up @@ -264,7 +288,7 @@ impl RollingBitField {
}

pub fn insert(&mut self, key: u64) {
let bits_empty = self.count == 0 || self.count == self.excess.len();
let mut bits_empty = self.count == 0 || self.count == self.excess.len();
let update_bits = if bits_empty {
true // nothing in bits, so in range
} else if key < self.min {
Expand Down Expand Up @@ -293,6 +317,12 @@ impl RollingBitField {
let address = self.get_address(&key);
self.bits.set(address, false);
self.purge(&key);

if self.count == self.excess.len() {
// if we moved the last existing item to excess, then we are ready to insert the new item in the bits
bits_empty = true;
break;
}
}

true // moved things to excess if necessary, so update bits with the new entry
Expand All @@ -309,6 +339,13 @@ impl RollingBitField {
} else {
self.min = std::cmp::min(self.min, key);
self.max = std::cmp::max(self.max, key + 1);
assert!(
self.min + self.max_width >= self.max,
"min: {}, max: {}, max_width: {}",
self.min,
self.max,
self.max_width
);
}
self.count += 1;
}
Expand Down Expand Up @@ -340,9 +377,13 @@ impl RollingBitField {
}
}

fn reset_min_max(&mut self) {
self.min = self.max
}

// after removing 'key' where 'key' = min, make min the correct new min value
fn purge(&mut self, key: &u64) {
if self.count > 0 {
if self.count > 0 && self.excess.len() != self.count {
if key == &self.min {
let start = self.min + 1; // min just got removed
for key in start..self.max {
Expand All @@ -353,8 +394,7 @@ impl RollingBitField {
}
}
} else {
self.min = Slot::default();
self.max = Slot::default();
self.reset_min_max();
}
}

Expand Down Expand Up @@ -1625,6 +1665,20 @@ pub mod tests {
)
}

#[test]
fn test_bitfield_insert_excess() {
solana_logger::setup();
let len = 16;
let mut bitfield = RollingBitField::new(len);

bitfield.insert(0);
let too_big = len + 1;
bitfield.insert(too_big);
assert!(bitfield.contains(&0));
assert!(bitfield.contains(&too_big));
assert_eq!(bitfield.len(), 2);
}

#[test]
fn test_bitfield_permutations() {
solana_logger::setup();
Expand Down
160 changes: 26 additions & 134 deletions runtime/src/ancestors.rs
View file
Original file line number Diff line number Diff line change
@@ -1,47 +1,32 @@
use crate::accounts_index::RollingBitField;
use solana_sdk::clock::Slot;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;

pub type AncestorsForSerialization = HashMap<Slot, usize>;

#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize, AbiExample)]
#[derive(Debug, Clone, PartialEq, AbiExample)]
pub struct Ancestors {
min: Slot,
slots: Vec<Option<usize>>,
count: usize,
max: Slot,
large_range_slots: HashSet<Slot>,
ancestors: RollingBitField,
}

// some tests produce ancestors ranges that are too large such
// that we prefer to implement them in a sparse HashMap
const ANCESTORS_HASH_MAP_SIZE: u64 = 10_000;
const ANCESTORS_HASH_MAP_SIZE: u64 = 8192;

impl Default for Ancestors {
fn default() -> Self {
Self {
ancestors: RollingBitField::new(ANCESTORS_HASH_MAP_SIZE),
}
}
}

impl From<Vec<Slot>> for Ancestors {
fn from(source: Vec<Slot>) -> Ancestors {
let mut result = Ancestors::default();
if !source.is_empty() {
result.min = Slot::MAX;
result.max = Slot::MIN;
source.iter().for_each(|slot| {
result.min = std::cmp::min(result.min, *slot);
result.max = std::cmp::max(result.max, *slot + 1);
});
let range = result.range();
if range > ANCESTORS_HASH_MAP_SIZE {
result.large_range_slots = source.into_iter().collect();
result.min = 0;
result.max = 0;
} else {
result.slots = vec![None; range as usize];
source.into_iter().for_each(|slot| {
let slot = result.slot_index(&slot);
if result.slots[slot].is_none() {
result.count += 1;
}
result.slots[slot] = Some(0);
});
}
}
source.into_iter().for_each(|slot| {
result.ancestors.insert(slot);
});

result
}
Expand All @@ -50,29 +35,9 @@ impl From<Vec<Slot>> for Ancestors {
impl From<&HashMap<Slot, usize>> for Ancestors {
fn from(source: &HashMap<Slot, usize>) -> Ancestors {
let mut result = Ancestors::default();
if !source.is_empty() {
result.min = Slot::MAX;
result.max = Slot::MIN;
source.iter().for_each(|(slot, _)| {
result.min = std::cmp::min(result.min, *slot);
result.max = std::cmp::max(result.max, *slot + 1);
});
let range = result.range();
if range > ANCESTORS_HASH_MAP_SIZE {
result.large_range_slots = source.iter().map(|(slot, _size)| *slot).collect();
result.min = 0;
result.max = 0;
} else {
result.slots = vec![None; range as usize];
source.iter().for_each(|(slot, size)| {
let slot = result.slot_index(&slot);
if result.slots[slot].is_none() {
result.count += 1;
}
result.slots[slot] = Some(*size);
});
}
}
source.iter().for_each(|(slot, _)| {
result.ancestors.insert(*slot);
});

result
}
Expand All @@ -90,75 +55,28 @@ impl From<&Ancestors> for HashMap<Slot, usize> {

impl Ancestors {
pub fn keys(&self) -> Vec<Slot> {
if self.large_range_slots.is_empty() {
self.slots
.iter()
.enumerate()
.filter_map(|(size, i)| i.map(|_| size as u64 + self.min))
.collect::<Vec<_>>()
} else {
self.large_range_slots.iter().copied().collect::<Vec<_>>()
}
self.ancestors.get_all()
}

pub fn get(&self, slot: &Slot) -> bool {
if self.large_range_slots.is_empty() {
if slot < &self.min || slot >= &self.max {
return false;
}
let slot = self.slot_index(slot);
self.slots[slot].is_some()
} else {
self.large_range_slots.get(slot).is_some()
}
self.ancestors.contains(slot)
}

pub fn remove(&mut self, slot: &Slot) {
if self.large_range_slots.is_empty() {
if slot < &self.min || slot >= &self.max {
return;
}
let slot = self.slot_index(slot);
if self.slots[slot].is_some() {
self.count -= 1;
self.slots[slot] = None;
}
} else {
self.large_range_slots.remove(slot);
}
self.ancestors.remove(slot);
}

pub fn contains_key(&self, slot: &Slot) -> bool {
if self.large_range_slots.is_empty() {
if slot < &self.min || slot >= &self.max {
return false;
}
let slot = self.slot_index(slot);
self.slots[slot].is_some()
} else {
self.large_range_slots.contains(slot)
}
self.ancestors.contains(slot)
}

pub fn len(&self) -> usize {
if self.large_range_slots.is_empty() {
self.count
} else {
self.large_range_slots.len()
}
self.ancestors.len()
}

pub fn is_empty(&self) -> bool {
self.len() == 0
}

fn slot_index(&self, slot: &Slot) -> usize {
(slot - self.min) as usize
}

fn range(&self) -> Slot {
self.max - self.min
}
}
#[cfg(test)]
pub mod tests {
Expand Down Expand Up @@ -187,34 +105,8 @@ pub mod tests {
}
}
impl Ancestors {
pub fn insert(&mut self, mut slot: Slot, size: usize) {
if self.large_range_slots.is_empty() {
if slot < self.min || slot >= self.max {
let new_min = std::cmp::min(self.min, slot);
let new_max = std::cmp::max(self.max, slot + 1);
let new_range = new_max - new_min;
if new_min == self.min {
self.max = slot + 1;
self.slots.resize(new_range as usize, None);
} else {
// min changed
let mut new_slots = vec![None; new_range as usize];
self.slots.iter().enumerate().for_each(|(i, size)| {
new_slots[i as usize + self.min as usize - slot as usize] = *size
});
self.slots = new_slots;
self.min = slot;
// fall through and set this value in
}
}
slot -= self.min;
if self.slots[slot as usize].is_none() {
self.count += 1;
}
self.slots[slot as usize] = Some(size);
} else {
self.large_range_slots.insert(slot);
}
pub fn insert(&mut self, slot: Slot, _size: usize) {
self.ancestors.insert(slot);
}
}

Expand Down

0 comments on commit cb1b479

Please sign in to comment.