Skip to content

Commit

Permalink
*: fix builds (#351)
Browse files Browse the repository at this point in the history 
This PR breaks the circle dependencies between grpcio and grpcio-proto
by adding a new crate named tests and examples. We can bring things back
once rust-lang/cargo#6915 is fixed.

The new structure also fixes a problem that crates.io denies to accept
circle dependencies in dev-dependencies. More see rust-lang/cargo#4242.

This PR also updates protobuf-build to the latest version.
  • Loading branch information
@BusyJay
BusyJay authored Jul 24, 2019
1 parent fd9b6c6 commit f0a26b5
Show file tree
Hide file tree
Showing 24 changed files with 96 additions and 245 deletions.
30 changes: 1 addition & 29 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ bytes = { version = "0.4.11", optional = true }
log = "0.4"

[workspace]
members = ["proto", "benchmark", "compiler", "interop"]
members = ["proto", "benchmark", "compiler", "interop", "tests-and-examples"]

[features]
default = ["protobuf-codec", "secure"]
Expand All @@ -37,34 +37,6 @@ openssl = ["secure", "grpcio-sys/openssl"]
openssl-vendored = ["secure", "grpcio-sys/openssl-vendored"]
no-omit-frame-pointer = ["grpcio-sys/no-omit-frame-pointer"]

[[example]]
name = "route_guide_client"
path = "examples/route_guide/client.rs"

[[example]]
name = "route_guide_server"
path = "examples/route_guide/server.rs"

[[example]]
name = "greeter_client"
path = "examples/hello_world/client.rs"

[[example]]
name = "greeter_server"
path = "examples/hello_world/server.rs"

[dev-dependencies]
serde_json = "1.0"
serde = "1.0"
serde_derive = "1.0"
grpcio-proto = { path = "proto", version = "0.5.0-alpha.2", default-features = false }
rand = "0.4"
slog = "2.0"
slog-async = "2.1"
slog-stdlog = "3.0"
slog-scope = "4.0"
slog-term = "2.2"

[profile.release]
debug = true

Expand Down
2 changes: 1 addition & 1 deletion benchmark/src/util.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use std::time::{Duration, Instant};
use grpc_proto::testing::stats::HistogramData;
use grpc_sys;

#[path = "../../examples/log_util.rs"]
#[path = "../../tests-and-examples/examples/log_util.rs"]
pub mod log_util;

pub struct Sample {
Expand Down
11 changes: 4 additions & 7 deletions proto/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ build = "build.rs"

[features]
default = ["protobuf-codec"]
protobuf-codec = ["grpcio/protobuf-codec", "protobuf-codegen", "protobuf", "grpcio-compiler/protobuf-codec"]
prost-codec = ["prost-derive", "bytes", "lazy_static", "grpcio/prost-codec", "prost-types", "prost", "grpcio-compiler/prost-codec"]
protobuf-codec = ["grpcio/protobuf-codec", "grpcio-compiler/protobuf-codec", "protobuf-build/grpcio-protobuf-codec"]
prost-codec = ["prost-derive", "bytes", "lazy_static", "grpcio/prost-codec", "prost", "grpcio-compiler/prost-codec", "protobuf-build/grpcio-prost-codec"]

[dependencies]
futures = "0.1"
Expand All @@ -27,9 +27,6 @@ protobuf = "2"
lazy_static = { version = "1.3", optional = true }

[build-dependencies]
protobuf-build = "0.7"
protobuf-build = { version = "0.8", default-features = false }
grpcio-compiler = { path = "../compiler", version = "0.5.0-alpha.2", default-features = false }
prost = { version = "0.5", optional = true }
prost-types = { version = "0.5", optional = true }
protobuf = { version = "2", optional = true }
protobuf-codegen = { version = "2", optional = true }
walkdir = "2.2"
151 changes: 17 additions & 134 deletions proto/build.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,144 +11,27 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use std::path::Path;
use std::env;

fn main() {
for package in &["testing", "example", "health"] {
compile(format!("{}.desc", package), package);
}
}

#[cfg(feature = "prost-codec")]
fn compile<P: AsRef<Path>>(desc_path: P, _package: &str) {
prost::desc_to_module(desc_path.as_ref());
}

#[cfg(feature = "prost-codec")]
mod prost {
use std::env;
use std::fs;
use std::path::{Path, PathBuf};

use prost::Message;
use prost_types::FileDescriptorSet as ProstFileDescriptorSet;

/// Descriptor file to module file using Prost.
pub fn desc_to_module(descriptor: &Path) {
let buf = fs::read(descriptor).unwrap();
let proto_set: ProstFileDescriptorSet = Message::decode(buf).unwrap();
let files: Vec<_> = proto_set
.file
let out_dir = env::var("OUT_DIR").unwrap();
let modules = &[
("grpc/testing", "testing"),
("grpc/health/v1/", "health"),
("grpc/example", "example"),
];
for (dir, package) in modules {
let out_dir = format!("{}/{}", out_dir, package);
let files: Vec<_> = walkdir::WalkDir::new(format!("proto/{}", dir))
.into_iter()
.map(|f| {
let mut path = PathBuf::new();
path.push("proto");
path.push(f.name.unwrap());
path.to_str().unwrap().to_owned()
.filter_map(|p| {
let dent = p.expect("Error happened when search protos");
if !dent.file_type().is_file() {
return None;
}
Some(format!("{}", dent.path().display()))
})
.collect();

let packages = protobuf_build::generate_prost_files(&files, &env::var("OUT_DIR").unwrap());
assert!(!packages.is_empty());
protobuf_build::generate_wrappers(
&packages
.iter()
.map(|m| format!("{}/{}.rs", env::var("OUT_DIR").unwrap(), m))
.collect::<Vec<_>>(),
&env::var("OUT_DIR").unwrap(),
protobuf_build::GenOpt::all(),
);
}
}

#[cfg(feature = "protobuf-codec")]
fn compile<P: AsRef<Path>>(desc_path: P, package: &str) {
protobuf::compile(desc_path, package);
}

#[cfg(feature = "protobuf-codec")]
mod protobuf {
use std::env;
use std::fs::{self, File};
use std::io::Write;
use std::path::Path;

use grpcio_compiler::codegen as grpc_gen;
use protobuf::compiler_plugin::GenResult;
use protobuf::descriptor::{FileDescriptorProto, FileDescriptorSet};
use protobuf_codegen::{self as pb_gen, Customize};

/// Compile all related proto file to `FileDescriptorSet` and use it to generate
/// rust source using rust-protobuf.
///
/// Using `FileDescriptorSet` here so we don't need to compile the binaries like
/// protoc-gen-rust and grpc_rust_plugin.
pub fn compile<P: AsRef<Path>>(desc_path: P, package: &str) {
let out_dir = env::var("OUT_DIR").unwrap();
let module_path = Path::new(&out_dir).join(package);
if !module_path.exists() {
fs::create_dir_all(&module_path).unwrap();
}

let mod_rs = module_path.join("mod.rs");
let mut module_file = File::create(mod_rs).unwrap();

desc_to_module(
desc_path.as_ref(),
&module_path,
|a, b| {
let c = Customize::default();
pb_gen::gen(a, b, &c)
},
&mut module_file,
);
desc_to_module(
desc_path.as_ref(),
&module_path,
grpc_gen::gen,
&mut module_file,
);
}

/// Descriptor file to module file using rust-protobuf.
fn desc_to_module<G, W>(descriptor: &Path, output_dir: &Path, mut generate_files: G, out: W)
where
G: FnMut(&[FileDescriptorProto], &[String]) -> Vec<GenResult>,
W: Write,
{
let mut f = File::open(descriptor).unwrap();
let proto_set: FileDescriptorSet = protobuf::parse_from_reader(&mut f).unwrap();

let files: Vec<_> = proto_set
.get_file()
.into_iter()
.map(|f| f.get_name().to_owned())
.collect();

// All files need to be generated in our case.
let results = generate_files(proto_set.get_file(), &files);
write_files(
results.into_iter().map(|res| (res.name, res.content)),
output_dir,
out,
);
}

fn write_files<W: Write>(
results: impl Iterator<Item = (String, Vec<u8>)>,
output_dir: &Path,
mut out: W,
) {
if !output_dir.exists() {
fs::create_dir_all(output_dir).unwrap();
}

for (name, content) in results {
let out_file = output_dir.join(&name);
let mut f = File::create(&out_file).unwrap();
f.write_all(&content).unwrap();
let (module_name, _) = name.split_at(name.len() - 3); // ".rs".len() == 3
writeln!(out, "pub mod {};", module_name).unwrap();
}
protobuf_build::generate_files(&["proto".to_owned()], &files, &out_dir);
}
}
26 changes: 0 additions & 26 deletions proto/build.sh
View file

This file was deleted.

Binary file removed proto/example.desc
View file
Binary file not shown.
Binary file removed proto/health.desc
View file
Binary file not shown.
71 changes: 23 additions & 48 deletions proto/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,31 @@
// See the License for the specific language governing permissions and
// limitations under the License.

pub mod testing {
include!(concat!(env!("OUT_DIR"), "/testing/mod.rs"));

#[cfg(feature = "prost-codec")]
pub use self::grpc::testing::*;
}

pub mod example {
include!(concat!(env!("OUT_DIR"), "/example/mod.rs"));
}

pub mod health {
pub mod v1 {
include!(concat!(env!("OUT_DIR"), "/health/mod.rs"));

#[cfg(feature = "prost-codec")]
pub use self::grpc::health::v1::*;
}
}

#[cfg(feature = "prost-codec")]
#[allow(clippy::large_enum_variant)]
pub mod testing {
include!(concat!(env!("OUT_DIR"), "/grpc.testing.rs"));
include!(concat!(env!("OUT_DIR"), "/wrapper_grpc.testing.rs"));
pub mod help {

use super::testing::*;

// Wrapper functions for oneof fields.
impl ClientArgs {
Expand Down Expand Up @@ -114,49 +134,4 @@ pub mod testing {
}
}

#[cfg(feature = "prost-codec")]
pub mod example {
pub mod helloworld {
include!(concat!(env!("OUT_DIR"), "/helloworld.rs"));
include!(concat!(env!("OUT_DIR"), "/wrapper_helloworld.rs"));
}
pub mod route_guide {
include!(concat!(env!("OUT_DIR"), "/routeguide.rs"));
include!(concat!(env!("OUT_DIR"), "/wrapper_routeguide.rs"));
}
}

#[cfg(feature = "prost-codec")]
pub mod health {
pub mod v1 {
pub mod health {
include!(concat!(env!("OUT_DIR"), "/grpc.health.v1.rs"));
include!(concat!(env!("OUT_DIR"), "/wrapper_grpc.health.v1.rs"));
}
}
}

#[cfg(feature = "protobuf-codec")]
#[allow(bare_trait_objects)]
#[allow(renamed_and_removed_lints)]
pub mod testing {
include!(concat!(env!("OUT_DIR"), "/testing/mod.rs"));
}

#[cfg(feature = "protobuf-codec")]
#[allow(bare_trait_objects)]
#[allow(renamed_and_removed_lints)]
pub mod example {
include!(concat!(env!("OUT_DIR"), "/example/mod.rs"));
}

#[cfg(feature = "protobuf-codec")]
pub mod health {
#[allow(bare_trait_objects)]
#[allow(renamed_and_removed_lints)]
pub mod v1 {
include!(concat!(env!("OUT_DIR"), "/health/mod.rs"));
}
}

pub mod util;
Binary file removed proto/testing.desc
View file
Binary file not shown.
Loading

0 comments on commit f0a26b5

Please sign in to comment.