Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Applying comments from the PR for the storage crate #1610

Merged
merged 19 commits into from
Jan 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
df6e1ae
Move storage traits implementation to the `fuel-core-storage` crate
xgreenx Dec 24, 2023
de82e25
Added comments to all newly added stuff. Made self-review and applied…
xgreenx Dec 26, 2023
7ccc722
Merge branch 'master' into feature/move-storage-implementation-to-own…
xgreenx Dec 26, 2023
33661bc
Updated CHANGELOG.md
xgreenx Dec 26, 2023
1b8295a
Merge remote-tracking branch 'origin/feature/move-storage-implementat…
xgreenx Dec 26, 2023
4c3f18c
Apply suggestions from the PR
xgreenx Jan 5, 2024
0eaab98
Merge branch 'master' into feature/move-storage-implementation-to-own…
xgreenx Jan 5, 2024
0cea5bb
Merge branch 'master' into feature/move-storage-implementation-to-own…
xgreenx Jan 6, 2024
c9977a4
Merge branch 'master' into feature/move-storage-implementation-to-own…
xgreenx Jan 6, 2024
b0ed3e9
Fixed compilation
xgreenx Jan 6, 2024
541527e
Merge branch 'master' into feature/move-storage-implementation-to-own…
xgreenx Jan 6, 2024
452418c
Merge remote-tracking branch 'origin/feature/move-storage-implementat…
xgreenx Jan 19, 2024
1fbc318
Merge branch 'master' into feature/move-storage-implementation-to-own…
xgreenx Jan 19, 2024
c5956a8
Use "blueprint" instead of "structure"
xgreenx Jan 19, 2024
cc9966c
Fix documents
xgreenx Jan 19, 2024
2261bda
Add one more test for `contains_key` function
xgreenx Jan 19, 2024
417ea9c
Merge branch 'master' into feature/follow-up-storage-move-comments
xgreenx Jan 19, 2024
78be6f1
Applied comments from the PR.
xgreenx Jan 19, 2024
4608e56
Merge branch 'master' into feature/follow-up-storage-move-comments
xgreenx Jan 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/fuel-core/src/database/transactions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ use fuel_core_types::{
services::txpool::TransactionStatus,
};

/// Teh tables allows to iterate over all transactions owned by an address.
/// These tables allow iteration over all transactions owned by an address.
pub struct OwnedTransactions;

impl Mappable for OwnedTransactions {
Expand Down
29 changes: 18 additions & 11 deletions crates/storage/src/blueprint.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,23 +110,30 @@ where
{
/// Initializes the storage with a bunch of key-value pairs.
/// In some cases, this method may be more performant than [`Self::insert`].
fn init(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = (&M::Key, &M::Value)>,
) -> StorageResult<()>;
fn init<'a, Iter>(storage: &mut S, column: S::Column, set: Iter) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = (&'a M::Key, &'a M::Value)>,
M::Key: 'a,
M::Value: 'a;

/// Inserts the batch of key-value pairs into the storage.
fn insert(
fn insert<'a, Iter>(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = (&M::Key, &M::Value)>,
) -> StorageResult<()>;
set: Iter,
) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = (&'a M::Key, &'a M::Value)>,
M::Key: 'a,
M::Value: 'a;

/// Removes the batch of key-value pairs from the storage.
fn remove(
fn remove<'a, Iter>(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = &M::Key>,
) -> StorageResult<()>;
set: Iter,
) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = &'a M::Key>,
M::Key: 'a;
}
32 changes: 21 additions & 11 deletions crates/storage/src/blueprint/plain.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,19 +98,25 @@ where
M: Mappable + TableWithBlueprint<Blueprint = Plain<KeyCodec, ValueCodec>>,
M::Blueprint: Blueprint<M, S>,
{
fn init(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = (&M::Key, &M::Value)>,
) -> StorageResult<()> {
fn init<'a, Iter>(storage: &mut S, column: S::Column, set: Iter) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = (&'a M::Key, &'a M::Value)>,
M::Key: 'a,
M::Value: 'a,
{
Self::insert(storage, column, set)
}

fn insert(
fn insert<'a, Iter>(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = (&M::Key, &M::Value)>,
) -> StorageResult<()> {
set: Iter,
) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = (&'a M::Key, &'a M::Value)>,
M::Key: 'a,
M::Value: 'a,
{
storage.batch_write(&mut set.map(|(key, value)| {
let key_encoder = <M::Blueprint as Blueprint<M, S>>::KeyCodec::encode(key);
let key_bytes = key_encoder.as_bytes().to_vec();
Expand All @@ -120,11 +126,15 @@ where
}))
}

fn remove(
fn remove<'a, Iter>(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = &M::Key>,
) -> StorageResult<()> {
set: Iter,
) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = &'a M::Key>,
M::Key: 'a,
{
storage.batch_write(&mut set.map(|key| {
let key_encoder = <M::Blueprint as Blueprint<M, S>>::KeyCodec::encode(key);
let key_bytes = key_encoder.as_bytes().to_vec();
Expand Down
32 changes: 21 additions & 11 deletions crates/storage/src/blueprint/sparse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -292,11 +292,12 @@ where
+ StorageMutate<Metadata, Error = StorageError>
+ StorageMutate<Nodes, Error = StorageError>,
{
fn init(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = (&M::Key, &M::Value)>,
) -> StorageResult<()> {
fn init<'a, Iter>(storage: &mut S, column: S::Column, set: Iter) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = (&'a M::Key, &'a M::Value)>,
M::Key: 'a,
M::Value: 'a,
{
let mut set = set.peekable();

let primary_key;
Expand Down Expand Up @@ -353,11 +354,16 @@ where
Ok(())
}

fn insert(
fn insert<'a, Iter>(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = (&M::Key, &M::Value)>,
) -> StorageResult<()> {
set: Iter,
) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = (&'a M::Key, &'a M::Value)>,
M::Key: 'a,
M::Value: 'a,
{
let mut set = set.peekable();

let primary_key;
Expand Down Expand Up @@ -406,11 +412,15 @@ where
Ok(())
}

fn remove(
fn remove<'a, Iter>(
storage: &mut S,
column: S::Column,
set: &mut dyn Iterator<Item = &M::Key>,
) -> StorageResult<()> {
set: Iter,
) -> StorageResult<()>
where
Iter: 'a + Iterator<Item = &'a M::Key>,
M::Key: 'a,
{
let mut set = set.peekable();

let primary_key;
Expand Down
6 changes: 4 additions & 2 deletions crates/storage/src/codec/manual.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ use crate::codec::{
Decode,
Encode,
};
use fuel_core_types::fuel_vm::ContractsAssetKey;
use fuel_vm_private::storage::ContractsStateKey;
use fuel_core_types::fuel_vm::{
ContractsAssetKey,
ContractsStateKey,
};
use std::borrow::Cow;

/// The codec allows the definition of manual implementation for specific type `T`.
Expand Down
14 changes: 7 additions & 7 deletions crates/storage/src/codec/postcard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,24 @@ use std::borrow::Cow;
/// The codec is used to serialized/deserialized types that supports `serde::Serialize` and `serde::Deserialize`.
pub struct Postcard;

impl<K> Encode<K> for Postcard
impl<T> Encode<T> for Postcard
where
K: ?Sized + serde::Serialize,
T: ?Sized + serde::Serialize,
{
type Encoder<'a> = Cow<'a, [u8]> where K: 'a;
type Encoder<'a> = Cow<'a, [u8]> where T: 'a;

fn encode(value: &K) -> Self::Encoder<'_> {
fn encode(value: &T) -> Self::Encoder<'_> {
Cow::Owned(postcard::to_allocvec(value).expect(
"It should be impossible to fail unless serialization is not implemented, which is not true for our types.",
))
}
}

impl<V> Decode<V> for Postcard
impl<T> Decode<T> for Postcard
where
V: serde::de::DeserializeOwned,
T: serde::de::DeserializeOwned,
{
fn decode(bytes: &[u8]) -> anyhow::Result<V> {
fn decode(bytes: &[u8]) -> anyhow::Result<T> {
Ok(postcard::from_bytes(bytes)?)
}
}
16 changes: 8 additions & 8 deletions crates/storage/src/codec/raw.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,22 @@ use std::borrow::Cow;
/// The codec is used for types that are already represented by bytes.
pub struct Raw;

impl<K> Encode<K> for Raw
impl<T> Encode<T> for Raw
where
K: ?Sized + AsRef<[u8]>,
T: ?Sized + AsRef<[u8]>,
{
type Encoder<'a> = Cow<'a, [u8]> where K: 'a;
type Encoder<'a> = Cow<'a, [u8]> where T: 'a;

fn encode(t: &K) -> Self::Encoder<'_> {
fn encode(t: &T) -> Self::Encoder<'_> {
Cow::Borrowed(t.as_ref())
}
}

impl<V> Decode<V> for Raw
impl<T> Decode<T> for Raw
where
for<'a> V: TryFrom<&'a [u8]>,
for<'a> T: TryFrom<&'a [u8]>,
{
fn decode(bytes: &[u8]) -> anyhow::Result<V> {
V::try_from(bytes).map_err(|_| anyhow::anyhow!("Unable to decode bytes"))
fn decode(bytes: &[u8]) -> anyhow::Result<T> {
T::try_from(bytes).map_err(|_| anyhow::anyhow!("Unable to decode bytes"))
}
}
1 change: 1 addition & 0 deletions crates/storage/src/kv_store.rs
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ pub enum WriteOperation {
#[impl_tools::autoimpl(for<T: trait> &T, &mut T, Box<T>, Arc<T>)]
pub trait BatchOperations: KeyValueStore {
/// Writes the batch of the entries into the storage.
// TODO: Replace `dyn Iterator` with a generic iterator when `Database` will not use `dyn BatchOperations`.
fn batch_write(
&self,
entries: &mut dyn Iterator<Item = (Vec<u8>, Self::Column, WriteOperation)>,
Expand Down
23 changes: 14 additions & 9 deletions crates/storage/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -125,19 +125,24 @@ pub trait StorageBatchMutate<Type: Mappable>: StorageMutate<Type> {
/// # Errors
///
/// Returns an error if the storage is already initialized.
fn init_storage(
&mut self,
set: &mut dyn Iterator<Item = (&Type::Key, &Type::Value)>,
) -> Result<()>;
fn init_storage<'a, Iter>(&mut self, set: Iter) -> Result<()>
where
Iter: 'a + Iterator<Item = (&'a Type::Key, &'a Type::Value)>,
Type::Key: 'a,
Type::Value: 'a;

/// Inserts the key-value pair into the storage in batch.
fn insert_batch(
&mut self,
set: &mut dyn Iterator<Item = (&Type::Key, &Type::Value)>,
) -> Result<()>;
fn insert_batch<'a, Iter>(&mut self, set: Iter) -> Result<()>
where
Iter: 'a + Iterator<Item = (&'a Type::Key, &'a Type::Value)>,
Type::Key: 'a,
Type::Value: 'a;

/// Removes the key-value pairs from the storage in batch.
fn remove_batch(&mut self, set: &mut dyn Iterator<Item = &Type::Key>) -> Result<()>;
fn remove_batch<'a, Iter>(&mut self, set: Iter) -> Result<()>
where
Iter: 'a + Iterator<Item = &'a Type::Key>,
Type::Key: 'a;
}

/// Creates `StorageError::NotFound` error with file and line information inside.
Expand Down
62 changes: 50 additions & 12 deletions crates/storage/src/structured_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -131,24 +131,29 @@ where
M: Mappable + TableWithBlueprint,
M::Blueprint: SupportsBatching<M, S>,
{
fn init_storage(
&mut self,
set: &mut dyn Iterator<Item = (&M::Key, &M::Value)>,
) -> Result<(), Self::Error> {
fn init_storage<'a, Iter>(&mut self, set: Iter) -> Result<(), Self::Error>
where
Iter: 'a + Iterator<Item = (&'a M::Key, &'a M::Value)>,
M::Key: 'a,
M::Value: 'a,
{
<M as TableWithBlueprint>::Blueprint::init(&mut self.storage, M::column(), set)
}

fn insert_batch(
&mut self,
set: &mut dyn Iterator<Item = (&M::Key, &M::Value)>,
) -> Result<(), Self::Error> {
fn insert_batch<'a, Iter>(&mut self, set: Iter) -> Result<(), Self::Error>
where
Iter: 'a + Iterator<Item = (&'a M::Key, &'a M::Value)>,
M::Key: 'a,
M::Value: 'a,
{
<M as TableWithBlueprint>::Blueprint::insert(&mut self.storage, M::column(), set)
}

fn remove_batch(
&mut self,
set: &mut dyn Iterator<Item = &M::Key>,
) -> Result<(), Self::Error> {
fn remove_batch<'a, Iter>(&mut self, set: Iter) -> Result<(), Self::Error>
where
Iter: 'a + Iterator<Item = &'a M::Key>,
M::Key: 'a,
{
<M as TableWithBlueprint>::Blueprint::remove(&mut self.storage, M::column(), set)
}
}
Expand Down Expand Up @@ -308,17 +313,50 @@ pub mod test {
let mut structured_storage = StructuredStorage::new(&mut storage);
let key = $key;

// Given
assert!(!structured_storage
.storage_as_mut::<$table>()
.contains_key(&key)
.unwrap());

// When
structured_storage
.storage_as_mut::<$table>()
.insert(&key, &$value_insert)
.unwrap();

// Then
assert!(structured_storage
.storage_as_mut::<$table>()
.contains_key(&key)
.unwrap());
}

#[test]
fn exists_false_after_removing() {
let mut storage = InMemoryStorage::default();
let mut structured_storage = StructuredStorage::new(&mut storage);
let key = $key;

// Given
structured_storage
.storage_as_mut::<$table>()
.insert(&key, &$value_insert)
.unwrap();

// When
structured_storage
.storage_as_mut::<$table>()
.remove(&key)
.unwrap();

// Then
assert!(!structured_storage
.storage_as_mut::<$table>()
.contains_key(&key)
.unwrap());
}

#[test]
fn batch_mutate_works() {
use $crate::rand::{
Expand Down
Loading