From bf11b947519c28c4d4667da40973e5da0cf5e21d Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 5 May 2022 10:49:00 -0700 Subject: [PATCH 1/2] sui-network: manually define rpc service Manually define our Validator rpc service instead of relying on a .proto file. This has the advantange that for now we can use types already defined in Rust for wire messages instead of requiring proto definitions or using the awkward double serialization we were currently using. --- Cargo.lock | 66 +-------- crates/sui-network/Cargo.toml | 4 +- crates/sui-network/README.md | 2 +- crates/sui-network/proto/common.proto | 12 -- crates/sui-network/proto/validator.proto | 19 --- crates/sui-network/src/api.rs | 24 +-- crates/sui-network/src/codec.rs | 65 +++++++++ .../sui-network/src/generated/sui.common.rs | 9 -- ...alidator.rs => sui.validator.Validator.rs} | 138 +++++++++--------- crates/sui-network/src/lib.rs | 2 +- crates/sui-network/tests/bootstrap.rs | 84 +++++++++-- sui_core/src/authority_client.rs | 99 +++++-------- sui_core/src/authority_server.rs | 104 ++++--------- 13 files changed, 280 insertions(+), 348 deletions(-) delete mode 100644 crates/sui-network/proto/common.proto delete mode 100644 crates/sui-network/proto/validator.proto create mode 100644 crates/sui-network/src/codec.rs delete mode 100644 crates/sui-network/src/generated/sui.common.rs rename crates/sui-network/src/generated/{sui.validator.rs => sui.validator.Validator.rs} (82%) diff --git a/Cargo.lock b/Cargo.lock index 00557b4721cbd..aaca27af01fe0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -826,15 +826,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "cmake" -version = "0.1.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" -dependencies = [ - "cc", -] - [[package]] name = "codespan" version = "0.11.1" @@ -3212,12 +3203,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "name_variant" version = "1.0.0" @@ -3999,28 +3984,6 @@ dependencies = [ "prost-derive", ] -[[package]] -name = "prost-build" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120fbe7988713f39d780a58cf1a7ef0d7ef66c6d87e5aa3438940c05357929f4" -dependencies = [ - "bytes", - "cfg-if 1.0.0", - "cmake", - "heck 0.4.0", - "itertools", - "lazy_static 1.4.0", - "log", - "multimap", - "petgraph 0.6.0", - "prost", - "prost-types", - "regex", - "tempfile", - "which", -] - [[package]] name = "prost-derive" version = "0.10.1" @@ -4034,16 +3997,6 @@ dependencies = [ "syn", ] -[[package]] -name = "prost-types" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" -dependencies = [ - "bytes", - "prost", -] - [[package]] name = "ptree" version = "0.4.0" @@ -5111,8 +5064,8 @@ dependencies = [ "bincode", "bytes", "prost", - "prost-build", "serde 1.0.137", + "sui-types", "tokio", "tonic", "tonic-build", @@ -5611,13 +5564,11 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03447cdc9eaf8feffb6412dcb27baf2db11669a6c4789f29da799aabfb99547" +version = "0.7.2" +source = "git+https://github.com/hyperium/tonic.git?rev=de2e4ac077c076736dc451f3415ea7da1a61a560#de2e4ac077c076736dc451f3415ea7da1a61a560" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", "quote", "syn", ] @@ -6148,17 +6099,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "which" -version = "4.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" -dependencies = [ - "either", - "lazy_static 1.4.0", - "libc", -] - [[package]] name = "widestring" version = "0.4.3" diff --git a/crates/sui-network/Cargo.toml b/crates/sui-network/Cargo.toml index 236efcc30cc35..4a7af56231e26 100644 --- a/crates/sui-network/Cargo.toml +++ b/crates/sui-network/Cargo.toml @@ -15,7 +15,7 @@ tonic = "0.7" prost = "0.10" bincode = "1.3.3" serde = "1.0.136" +sui-types = { path = "../../sui_types" } [dev-dependencies] -tonic-build = { version = "0.7", features = [ "prost", "transport" ] } -prost-build = "0.10.1" +tonic-build = { git = "https://github.com/hyperium/tonic.git", rev = "de2e4ac077c076736dc451f3415ea7da1a61a560", default-features = false, features = [ "transport" ] } diff --git a/crates/sui-network/README.md b/crates/sui-network/README.md index b01fa1074ea3c..4b467feddaeb4 100644 --- a/crates/sui-network/README.md +++ b/crates/sui-network/README.md @@ -3,7 +3,7 @@ ## Changing an RPC service The general process for changing an RPC service is as follows: -1. Change the corresponding `.proto` file in the `proto` directory +1. Change the service definition in the `tests/bootstrap.rs` file. 2. Run `cargo test --test bootstrap` to re-run the code generation. Generated rust files are in the `src/generated` directory. 3. Update any other corresponding logic that would have been affected by diff --git a/crates/sui-network/proto/common.proto b/crates/sui-network/proto/common.proto deleted file mode 100644 index 8d87a9efbc575..0000000000000 --- a/crates/sui-network/proto/common.proto +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2022, Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; - -package sui.common; - -// A bincode encoded payload. This is intended to be used in the short-term -// while we don't have good protobuf definitions for sui types -message BincodeEncodedPayload { - bytes payload = 1; -} diff --git a/crates/sui-network/proto/validator.proto b/crates/sui-network/proto/validator.proto deleted file mode 100644 index cda0588d9df55..0000000000000 --- a/crates/sui-network/proto/validator.proto +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) 2022, Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -syntax = "proto3"; - -package sui.validator; - -import "common.proto"; - -// The Validator interface -service Validator { - rpc Transaction (common.BincodeEncodedPayload) returns (common.BincodeEncodedPayload) {} - rpc ConfirmationTransaction (common.BincodeEncodedPayload) returns (common.BincodeEncodedPayload) {} - rpc ConsensusTransaction (common.BincodeEncodedPayload) returns (common.BincodeEncodedPayload) {} - rpc AccountInfo (common.BincodeEncodedPayload) returns (common.BincodeEncodedPayload) {} - rpc ObjectInfo (common.BincodeEncodedPayload) returns (common.BincodeEncodedPayload) {} - rpc TransactionInfo (common.BincodeEncodedPayload) returns (common.BincodeEncodedPayload) {} - rpc BatchInfo (common.BincodeEncodedPayload) returns (stream common.BincodeEncodedPayload) {} -} diff --git a/crates/sui-network/src/api.rs b/crates/sui-network/src/api.rs index ebae7a6f47c9a..d056437c23685 100644 --- a/crates/sui-network/src/api.rs +++ b/crates/sui-network/src/api.rs @@ -1,33 +1,11 @@ // Copyright (c) 2022, Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -#[path = "generated/sui.validator.rs"] +#[path = "generated/sui.validator.Validator.rs"] #[rustfmt::skip] mod validator; -#[path = "generated/sui.common.rs"] -#[rustfmt::skip] -mod common; - -pub use common::BincodeEncodedPayload; pub use validator::{ validator_client::ValidatorClient, validator_server::{Validator, ValidatorServer}, }; - -impl BincodeEncodedPayload { - pub fn deserialize(&self) -> Result { - bincode::deserialize(self.payload.as_ref()) - } - - pub fn try_from(value: &T) -> Result { - let payload = bincode::serialize(value)?.into(); - Ok(Self { payload }) - } -} - -impl From for BincodeEncodedPayload { - fn from(payload: bytes::Bytes) -> Self { - Self { payload } - } -} diff --git a/crates/sui-network/src/codec.rs b/crates/sui-network/src/codec.rs new file mode 100644 index 0000000000000..bc7802a2aca78 --- /dev/null +++ b/crates/sui-network/src/codec.rs @@ -0,0 +1,65 @@ +use bytes::{Buf, BufMut}; +use std::marker::PhantomData; +use tonic::{ + codec::{Codec, DecodeBuf, Decoder, EncodeBuf, Encoder}, + Status, +}; + +#[derive(Debug)] +pub struct BincodeEncoder(PhantomData); + +impl Encoder for BincodeEncoder { + type Item = T; + type Error = Status; + + fn encode(&mut self, item: Self::Item, buf: &mut EncodeBuf<'_>) -> Result<(), Self::Error> { + bincode::serialize_into(buf.writer(), &item).map_err(|e| Status::internal(e.to_string())) + } +} + +#[derive(Debug)] +pub struct BincodeDecoder(PhantomData); + +impl Decoder for BincodeDecoder { + type Item = U; + type Error = Status; + + fn decode(&mut self, buf: &mut DecodeBuf<'_>) -> Result, Self::Error> { + if !buf.has_remaining() { + return Ok(None); + } + + let item: Self::Item = + bincode::deserialize_from(buf.reader()).map_err(|e| Status::internal(e.to_string()))?; + Ok(Some(item)) + } +} + +/// A [`Codec`] that implements `application/grpc+bincode` via the serde library. +#[derive(Debug, Clone)] +pub struct BincodeCodec(PhantomData<(T, U)>); + +impl Default for BincodeCodec { + fn default() -> Self { + Self(PhantomData) + } +} + +impl Codec for BincodeCodec +where + T: serde::Serialize + Send + 'static, + U: serde::de::DeserializeOwned + Send + 'static, +{ + type Encode = T; + type Decode = U; + type Encoder = BincodeEncoder; + type Decoder = BincodeDecoder; + + fn encoder(&mut self) -> Self::Encoder { + BincodeEncoder(PhantomData) + } + + fn decoder(&mut self) -> Self::Decoder { + BincodeDecoder(PhantomData) + } +} diff --git a/crates/sui-network/src/generated/sui.common.rs b/crates/sui-network/src/generated/sui.common.rs deleted file mode 100644 index 988bfa3a17766..0000000000000 --- a/crates/sui-network/src/generated/sui.common.rs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2022, Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 -/// A bincode encoded payload. This is intended to be used in the short-term -/// while we don't have good protobuf definitions for sui types -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BincodeEncodedPayload { - #[prost(bytes="bytes", tag="1")] - pub payload: ::prost::bytes::Bytes, -} diff --git a/crates/sui-network/src/generated/sui.validator.rs b/crates/sui-network/src/generated/sui.validator.Validator.rs similarity index 82% rename from crates/sui-network/src/generated/sui.validator.rs rename to crates/sui-network/src/generated/sui.validator.Validator.rs index 37dff6bc11355..74e4eb1d7a011 100644 --- a/crates/sui-network/src/generated/sui.validator.rs +++ b/crates/sui-network/src/generated/sui.validator.Validator.rs @@ -4,7 +4,7 @@ pub mod validator_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] use tonic::codegen::*; - /// The Validator interface + ///The Validator interface #[derive(Debug, Clone)] pub struct ValidatorClient { inner: tonic::client::Grpc, @@ -67,9 +67,9 @@ pub mod validator_client { } pub async fn transaction( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -81,7 +81,7 @@ pub mod validator_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let path = http::uri::PathAndQuery::from_static( "/sui.validator.Validator/Transaction", ); @@ -89,9 +89,9 @@ pub mod validator_client { } pub async fn confirmation_transaction( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -103,7 +103,7 @@ pub mod validator_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let path = http::uri::PathAndQuery::from_static( "/sui.validator.Validator/ConfirmationTransaction", ); @@ -111,9 +111,9 @@ pub mod validator_client { } pub async fn consensus_transaction( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -125,7 +125,7 @@ pub mod validator_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let path = http::uri::PathAndQuery::from_static( "/sui.validator.Validator/ConsensusTransaction", ); @@ -133,9 +133,9 @@ pub mod validator_client { } pub async fn account_info( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -147,7 +147,7 @@ pub mod validator_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let path = http::uri::PathAndQuery::from_static( "/sui.validator.Validator/AccountInfo", ); @@ -155,9 +155,9 @@ pub mod validator_client { } pub async fn object_info( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -169,7 +169,7 @@ pub mod validator_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let path = http::uri::PathAndQuery::from_static( "/sui.validator.Validator/ObjectInfo", ); @@ -177,9 +177,9 @@ pub mod validator_client { } pub async fn transaction_info( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, > { self.inner @@ -191,7 +191,7 @@ pub mod validator_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let path = http::uri::PathAndQuery::from_static( "/sui.validator.Validator/TransactionInfo", ); @@ -199,10 +199,10 @@ pub mod validator_client { } pub async fn batch_info( &mut self, - request: impl tonic::IntoRequest, + request: impl tonic::IntoRequest, ) -> Result< tonic::Response< - tonic::codec::Streaming, + tonic::codec::Streaming, >, tonic::Status, > { @@ -215,7 +215,7 @@ pub mod validator_client { format!("Service was not ready: {}", e.into()), ) })?; - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let path = http::uri::PathAndQuery::from_static( "/sui.validator.Validator/BatchInfo", ); @@ -232,58 +232,58 @@ pub mod validator_server { pub trait Validator: Send + Sync + 'static { async fn transaction( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, >; async fn confirmation_transaction( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, >; async fn consensus_transaction( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, >; async fn account_info( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, >; async fn object_info( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, >; async fn transaction_info( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result< - tonic::Response, + tonic::Response, tonic::Status, >; ///Server streaming response type for the BatchInfo method. type BatchInfoStream: futures_core::Stream< - Item = Result, + Item = Result, > + Send + 'static; async fn batch_info( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result, tonic::Status>; } - /// The Validator interface + ///The Validator interface #[derive(Debug)] pub struct ValidatorServer { inner: _Inner, @@ -336,19 +336,16 @@ pub mod validator_server { struct TransactionSvc(pub Arc); impl< T: Validator, - > tonic::server::UnaryService< - super::super::common::BincodeEncodedPayload, - > for TransactionSvc { - type Response = super::super::common::BincodeEncodedPayload; + > tonic::server::UnaryService + for TransactionSvc { + type Response = sui_types::messages::TransactionInfoResponse; type Future = BoxFuture< tonic::Response, tonic::Status, >; fn call( &mut self, - request: tonic::Request< - super::super::common::BincodeEncodedPayload, - >, + request: tonic::Request, ) -> Self::Future { let inner = self.0.clone(); let fut = async move { (*inner).transaction(request).await }; @@ -361,7 +358,7 @@ pub mod validator_server { let fut = async move { let inner = inner.0; let method = TransactionSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -378,9 +375,9 @@ pub mod validator_server { impl< T: Validator, > tonic::server::UnaryService< - super::super::common::BincodeEncodedPayload, + sui_types::messages::CertifiedTransaction, > for ConfirmationTransactionSvc { - type Response = super::super::common::BincodeEncodedPayload; + type Response = sui_types::messages::TransactionInfoResponse; type Future = BoxFuture< tonic::Response, tonic::Status, @@ -388,7 +385,7 @@ pub mod validator_server { fn call( &mut self, request: tonic::Request< - super::super::common::BincodeEncodedPayload, + sui_types::messages::CertifiedTransaction, >, ) -> Self::Future { let inner = self.0.clone(); @@ -404,7 +401,7 @@ pub mod validator_server { let fut = async move { let inner = inner.0; let method = ConfirmationTransactionSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -421,9 +418,9 @@ pub mod validator_server { impl< T: Validator, > tonic::server::UnaryService< - super::super::common::BincodeEncodedPayload, + sui_types::messages::ConsensusTransaction, > for ConsensusTransactionSvc { - type Response = super::super::common::BincodeEncodedPayload; + type Response = sui_types::messages::TransactionInfoResponse; type Future = BoxFuture< tonic::Response, tonic::Status, @@ -431,7 +428,7 @@ pub mod validator_server { fn call( &mut self, request: tonic::Request< - super::super::common::BincodeEncodedPayload, + sui_types::messages::ConsensusTransaction, >, ) -> Self::Future { let inner = self.0.clone(); @@ -447,7 +444,7 @@ pub mod validator_server { let fut = async move { let inner = inner.0; let method = ConsensusTransactionSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -464,9 +461,9 @@ pub mod validator_server { impl< T: Validator, > tonic::server::UnaryService< - super::super::common::BincodeEncodedPayload, + sui_types::messages::AccountInfoRequest, > for AccountInfoSvc { - type Response = super::super::common::BincodeEncodedPayload; + type Response = sui_types::messages::AccountInfoResponse; type Future = BoxFuture< tonic::Response, tonic::Status, @@ -474,7 +471,7 @@ pub mod validator_server { fn call( &mut self, request: tonic::Request< - super::super::common::BincodeEncodedPayload, + sui_types::messages::AccountInfoRequest, >, ) -> Self::Future { let inner = self.0.clone(); @@ -490,7 +487,7 @@ pub mod validator_server { let fut = async move { let inner = inner.0; let method = AccountInfoSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -506,10 +503,9 @@ pub mod validator_server { struct ObjectInfoSvc(pub Arc); impl< T: Validator, - > tonic::server::UnaryService< - super::super::common::BincodeEncodedPayload, - > for ObjectInfoSvc { - type Response = super::super::common::BincodeEncodedPayload; + > tonic::server::UnaryService + for ObjectInfoSvc { + type Response = sui_types::messages::ObjectInfoResponse; type Future = BoxFuture< tonic::Response, tonic::Status, @@ -517,7 +513,7 @@ pub mod validator_server { fn call( &mut self, request: tonic::Request< - super::super::common::BincodeEncodedPayload, + sui_types::messages::ObjectInfoRequest, >, ) -> Self::Future { let inner = self.0.clone(); @@ -531,7 +527,7 @@ pub mod validator_server { let fut = async move { let inner = inner.0; let method = ObjectInfoSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -548,9 +544,9 @@ pub mod validator_server { impl< T: Validator, > tonic::server::UnaryService< - super::super::common::BincodeEncodedPayload, + sui_types::messages::TransactionInfoRequest, > for TransactionInfoSvc { - type Response = super::super::common::BincodeEncodedPayload; + type Response = sui_types::messages::TransactionInfoResponse; type Future = BoxFuture< tonic::Response, tonic::Status, @@ -558,7 +554,7 @@ pub mod validator_server { fn call( &mut self, request: tonic::Request< - super::super::common::BincodeEncodedPayload, + sui_types::messages::TransactionInfoRequest, >, ) -> Self::Future { let inner = self.0.clone(); @@ -574,7 +570,7 @@ pub mod validator_server { let fut = async move { let inner = inner.0; let method = TransactionInfoSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, @@ -591,9 +587,9 @@ pub mod validator_server { impl< T: Validator, > tonic::server::ServerStreamingService< - super::super::common::BincodeEncodedPayload, + sui_types::messages::BatchInfoRequest, > for BatchInfoSvc { - type Response = super::super::common::BincodeEncodedPayload; + type Response = sui_types::messages::BatchInfoResponseItem; type ResponseStream = T::BatchInfoStream; type Future = BoxFuture< tonic::Response, @@ -602,7 +598,7 @@ pub mod validator_server { fn call( &mut self, request: tonic::Request< - super::super::common::BincodeEncodedPayload, + sui_types::messages::BatchInfoRequest, >, ) -> Self::Future { let inner = self.0.clone(); @@ -616,7 +612,7 @@ pub mod validator_server { let fut = async move { let inner = inner.0; let method = BatchInfoSvc(inner); - let codec = tonic::codec::ProstCodec::default(); + let codec = crate::codec::BincodeCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) .apply_compression_config( accept_compression_encodings, diff --git a/crates/sui-network/src/lib.rs b/crates/sui-network/src/lib.rs index 915210676d05e..0d1ae8be02761 100644 --- a/crates/sui-network/src/lib.rs +++ b/crates/sui-network/src/lib.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 pub mod api; +pub mod codec; pub mod network; -pub use prost; pub use tonic; diff --git a/crates/sui-network/tests/bootstrap.rs b/crates/sui-network/tests/bootstrap.rs index 9f52a8c0cdfc5..e1bc649bac0dc 100644 --- a/crates/sui-network/tests/bootstrap.rs +++ b/crates/sui-network/tests/bootstrap.rs @@ -6,26 +6,90 @@ use std::{ path::{Path, PathBuf}, process::Command, }; +use tonic_build::manual::{Builder, Method, Service}; type Result = ::std::result::Result>; #[test] fn bootstrap() { - let proto_files = &["proto/validator.proto", "proto/common.proto"]; - let dirs = &["proto"]; - let out_dir = PathBuf::from(std::env!("CARGO_MANIFEST_DIR")) .join("src") .join("generated"); + let codec_path = "crate::codec::BincodeCodec"; - // Use `Bytes` instead of `Vec` for bytes fields - let mut config = prost_build::Config::new(); - config.bytes(&["."]); + let validator_service = Service::builder() + .name("Validator") + .package("sui.validator") + .comment("The Validator interface") + .method( + Method::builder() + .name("transaction") + .route_name("Transaction") + .input_type("sui_types::messages::Transaction") + .output_type("sui_types::messages::TransactionInfoResponse") + .codec_path(codec_path) + .build(), + ) + .method( + Method::builder() + .name("confirmation_transaction") + .route_name("ConfirmationTransaction") + .input_type("sui_types::messages::CertifiedTransaction") + .output_type("sui_types::messages::TransactionInfoResponse") + .codec_path(codec_path) + .build(), + ) + .method( + Method::builder() + .name("consensus_transaction") + .route_name("ConsensusTransaction") + .input_type("sui_types::messages::ConsensusTransaction") + .output_type("sui_types::messages::TransactionInfoResponse") + .codec_path(codec_path) + .build(), + ) + .method( + Method::builder() + .name("account_info") + .route_name("AccountInfo") + .input_type("sui_types::messages::AccountInfoRequest") + .output_type("sui_types::messages::AccountInfoResponse") + .codec_path(codec_path) + .build(), + ) + .method( + Method::builder() + .name("object_info") + .route_name("ObjectInfo") + .input_type("sui_types::messages::ObjectInfoRequest") + .output_type("sui_types::messages::ObjectInfoResponse") + .codec_path(codec_path) + .build(), + ) + .method( + Method::builder() + .name("transaction_info") + .route_name("TransactionInfo") + .input_type("sui_types::messages::TransactionInfoRequest") + .output_type("sui_types::messages::TransactionInfoResponse") + .codec_path(codec_path) + .build(), + ) + .method( + Method::builder() + .name("batch_info") + .route_name("BatchInfo") + .input_type("sui_types::messages::BatchInfoRequest") + .output_type("sui_types::messages::BatchInfoResponseItem") + .server_streaming() + .codec_path(codec_path) + .build(), + ) + .build(); - tonic_build::configure() - .out_dir(format!("{}", out_dir.display())) - .compile_with_config(config, proto_files, dirs) - .unwrap(); + Builder::new() + .out_dir(&out_dir) + .compile(&[validator_service]); prepend_license(&out_dir).unwrap(); diff --git a/sui_core/src/authority_client.rs b/sui_core/src/authority_client.rs index 99d86565cefef..d95cc5e721949 100644 --- a/sui_core/src/authority_client.rs +++ b/sui_core/src/authority_client.rs @@ -6,11 +6,7 @@ use crate::authority::AuthorityState; use async_trait::async_trait; use futures::{stream::BoxStream, TryStreamExt}; use std::sync::Arc; -use sui_network::{ - api::{BincodeEncodedPayload, ValidatorClient}, - network::NetworkClient, - tonic, -}; +use sui_network::{api::ValidatorClient, network::NetworkClient, tonic}; use sui_types::{error::SuiError, messages::*}; #[cfg(test)] @@ -112,12 +108,11 @@ impl AuthorityAPI for NetworkAuthorityClient { &self, transaction: Transaction, ) -> Result { - let request = BincodeEncodedPayload::try_from(&transaction).unwrap(); - let response = self.client().transaction(request).await?.into_inner(); - - response - .deserialize() - .map_err(|_| SuiError::UnexpectedMessage) + self.client() + .transaction(transaction) + .await + .map(tonic::Response::into_inner) + .map_err(Into::into) } /// Confirm a transfer to a Sui or Primary account. @@ -125,58 +120,44 @@ impl AuthorityAPI for NetworkAuthorityClient { &self, transaction: ConfirmationTransaction, ) -> Result { - let request = BincodeEncodedPayload::try_from(&transaction).unwrap(); - let response = self - .client() - .confirmation_transaction(request) - .await? - .into_inner(); - - response - .deserialize() - .map_err(|_| SuiError::UnexpectedMessage) + self.client() + .confirmation_transaction(transaction.certificate) + .await + .map(tonic::Response::into_inner) + .map_err(Into::into) } async fn handle_consensus_transaction( &self, transaction: ConsensusTransaction, ) -> Result { - let request = BincodeEncodedPayload::try_from(&transaction).unwrap(); - let response = self - .client() - .consensus_transaction(request) - .await? - .into_inner(); - - response - .deserialize() - .map_err(|e| SuiError::GenericAuthorityError { - error: e.to_string(), - }) + self.client() + .consensus_transaction(transaction) + .await + .map(tonic::Response::into_inner) + .map_err(Into::into) } async fn handle_account_info_request( &self, request: AccountInfoRequest, ) -> Result { - let request = BincodeEncodedPayload::try_from(&request).unwrap(); - let response = self.client().account_info(request).await?.into_inner(); - - response - .deserialize() - .map_err(|_| SuiError::UnexpectedMessage) + self.client() + .account_info(request) + .await + .map(tonic::Response::into_inner) + .map_err(Into::into) } async fn handle_object_info_request( &self, request: ObjectInfoRequest, ) -> Result { - let request = BincodeEncodedPayload::try_from(&request).unwrap(); - let response = self.client().object_info(request).await?.into_inner(); - - response - .deserialize() - .map_err(|_| SuiError::UnexpectedMessage) + self.client() + .object_info(request) + .await + .map(tonic::Response::into_inner) + .map_err(Into::into) } /// Handle Object information requests for this account. @@ -184,12 +165,11 @@ impl AuthorityAPI for NetworkAuthorityClient { &self, request: TransactionInfoRequest, ) -> Result { - let request = BincodeEncodedPayload::try_from(&request).unwrap(); - let response = self.client().transaction_info(request).await?.into_inner(); - - response - .deserialize() - .map_err(|_| SuiError::UnexpectedMessage) + self.client() + .transaction_info(request) + .await + .map(tonic::Response::into_inner) + .map_err(Into::into) } /// Handle Batch information requests for this authority. @@ -197,17 +177,12 @@ impl AuthorityAPI for NetworkAuthorityClient { &self, request: BatchInfoRequest, ) -> Result { - let request = BincodeEncodedPayload::try_from(&request).unwrap(); - let response_stream = self.client().batch_info(request).await?.into_inner(); - - let stream = response_stream - .map_err(|_| SuiError::UnexpectedMessage) - .and_then(|item| { - let response_item = item - .deserialize::() - .map_err(|_| SuiError::UnexpectedMessage); - futures::future::ready(response_item) - }); + let stream = self + .client() + .batch_info(request) + .await + .map(tonic::Response::into_inner)? + .map_err(Into::into); Ok(Box::pin(stream)) } diff --git a/sui_core/src/authority_server.rs b/sui_core/src/authority_server.rs index f16bcfab6f481..d60e8270ea770 100644 --- a/sui_core/src/authority_server.rs +++ b/sui_core/src/authority_server.rs @@ -10,7 +10,7 @@ use async_trait::async_trait; use futures::{stream::BoxStream, FutureExt, TryStreamExt}; use std::{io, net::SocketAddr, sync::Arc, time::Duration}; use sui_network::{ - api::{BincodeEncodedPayload, Validator, ValidatorServer}, + api::{Validator, ValidatorServer}, network::NetworkServer, tonic, }; @@ -145,12 +145,9 @@ impl AuthorityServer { impl Validator for AuthorityServer { async fn transaction( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let mut transaction: Transaction = request - .into_inner() - .deserialize() - .map_err(|e| tonic::Status::invalid_argument(e.to_string()))?; + request: tonic::Request, + ) -> Result, tonic::Status> { + let mut transaction = request.into_inner(); let mut obligation = VerificationObligation::default(); transaction @@ -178,20 +175,14 @@ impl Validator for AuthorityServer { .await .map_err(|e| tonic::Status::internal(e.to_string()))?; - let payload = BincodeEncodedPayload::try_from(&info) - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - Ok(tonic::Response::new(payload)) + Ok(tonic::Response::new(info)) } async fn confirmation_transaction( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let mut transaction: CertifiedTransaction = request - .into_inner() - .deserialize() - .map_err(|e| tonic::Status::invalid_argument(e.to_string()))?; + request: tonic::Request, + ) -> Result, tonic::Status> { + let mut transaction = request.into_inner(); let mut obligation = VerificationObligation::default(); transaction @@ -221,20 +212,14 @@ impl Validator for AuthorityServer { .await .map_err(|e| tonic::Status::internal(e.to_string()))?; - let payload = BincodeEncodedPayload::try_from(&info) - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - Ok(tonic::Response::new(payload)) + Ok(tonic::Response::new(info)) } async fn consensus_transaction( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let transaction: ConsensusTransaction = request - .into_inner() - .deserialize() - .map_err(|e| tonic::Status::invalid_argument(e.to_string()))?; + request: tonic::Request, + ) -> Result, tonic::Status> { + let transaction = request.into_inner(); let info = self .consensus_adapter @@ -242,20 +227,14 @@ impl Validator for AuthorityServer { .await .map_err(|e| tonic::Status::internal(e.to_string()))?; - let payload = BincodeEncodedPayload::try_from(&info) - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - Ok(tonic::Response::new(payload)) + Ok(tonic::Response::new(info)) } async fn account_info( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let request: AccountInfoRequest = request - .into_inner() - .deserialize() - .map_err(|e| tonic::Status::invalid_argument(e.to_string()))?; + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); let response = self .state @@ -263,20 +242,14 @@ impl Validator for AuthorityServer { .await .map_err(|e| tonic::Status::internal(e.to_string()))?; - let payload = BincodeEncodedPayload::try_from(&response) - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - Ok(tonic::Response::new(payload)) + Ok(tonic::Response::new(response)) } async fn object_info( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let request: ObjectInfoRequest = request - .into_inner() - .deserialize() - .map_err(|e| tonic::Status::invalid_argument(e.to_string()))?; + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); let response = self .state @@ -284,20 +257,14 @@ impl Validator for AuthorityServer { .await .map_err(|e| tonic::Status::internal(e.to_string()))?; - let payload = BincodeEncodedPayload::try_from(&response) - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - Ok(tonic::Response::new(payload)) + Ok(tonic::Response::new(response)) } async fn transaction_info( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let request: TransactionInfoRequest = request - .into_inner() - .deserialize() - .map_err(|e| tonic::Status::invalid_argument(e.to_string()))?; + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); let response = self .state @@ -305,22 +272,16 @@ impl Validator for AuthorityServer { .await .map_err(|e| tonic::Status::internal(e.to_string()))?; - let payload = BincodeEncodedPayload::try_from(&response) - .map_err(|e| tonic::Status::internal(e.to_string()))?; - - Ok(tonic::Response::new(payload)) + Ok(tonic::Response::new(response)) } - type BatchInfoStream = BoxStream<'static, Result>; + type BatchInfoStream = BoxStream<'static, Result>; async fn batch_info( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result, tonic::Status> { - let request: BatchInfoRequest = request - .into_inner() - .deserialize() - .map_err(|e| tonic::Status::invalid_argument(e.to_string()))?; + let request = request.into_inner(); let xstream = self .state @@ -328,14 +289,7 @@ impl Validator for AuthorityServer { .await .map_err(|e| tonic::Status::internal(e.to_string()))?; - let response = - // items - // .chain(subscriber) - xstream - .map_err(|e| tonic::Status::internal(e.to_string())) - .map_ok(|item| { - BincodeEncodedPayload::try_from(&item).expect("serialization should not fail") - }); + let response = xstream.map_err(|e| tonic::Status::internal(e.to_string())); Ok(tonic::Response::new(Box::pin(response))) } From 8f9fd212781b3affc6e17f0d63e09e6222216fb0 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 5 May 2022 15:11:21 -0700 Subject: [PATCH 2/2] fixup! sui-network: manually define rpc service --- crates/sui-network/src/codec.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/sui-network/src/codec.rs b/crates/sui-network/src/codec.rs index bc7802a2aca78..9cb2acdecf3d5 100644 --- a/crates/sui-network/src/codec.rs +++ b/crates/sui-network/src/codec.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2022, Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + use bytes::{Buf, BufMut}; use std::marker::PhantomData; use tonic::{