feat(risedev): support launching system rw in risedev-dev

Makefile.toml

@@ -369,13 +369,32 @@ ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "$
 [tasks.post-build-risingwave]
 category = "RiseDev - Build"
 description = "Copy RisingWave binaries to bin"
-condition = { env_set = ["ENABLE_BUILD_RUST"] }
+condition = { env_set = ["ENABLE_BUILD_RUST"], env_not_set = ["USE_SYSTEM_RISINGWAVE"] }
 dependencies = [
     "link-all-in-one-binaries",
     "link-user-bin",
     "codesign-binaries",
 ]
 
+[tasks.test-system-risingwave]
+category = "RiseDev - Start/Stop"
+description = "Test whether RisingWave binary is installed to PATH"
+condition = { env_set = ["USE_SYSTEM_RISINGWAVE"] }
+script = '''
+#!/usr/bin/env bash
+
+set -e
+
+rw_path=$(command -v risingwave || true)
+if [[ -z "${rw_path}" ]]; then
+  echo "$(tput setaf 1)$(tput bold)[ERROR]$(tput sgr0) env $(tput setaf 5)USE_SYSTEM_RISINGWAVE$(tput sgr0) is set, but $(tput setaf 5)risingwave$(tput sgr0) is not found in PATH."
+  exit 1
+fi
+
+version=$(risingwave --version)
+echo "$(tput setaf 5)$(tput bold)[Info]$(tput sgr0) env $(tput setaf 5)USE_SYSTEM_RISINGWAVE$(tput sgr0) is set, will use $(tput setaf 5)$version$(tput sgr0) found at $rw_path."
+'''
+
 [tasks.b]
 alias = "build-risingwave"
 
@@ -390,17 +409,26 @@ script = '''
 #!/usr/bin/env bash
 
 set -e
-echo "$(tput setaf 4)$(tput bold)[Reminder]$(tput sgr0) risedev will only build $(tput setaf 4)risingwave_cmd(_all) and risedev$(tput sgr0) crates."
+echo "$(tput setaf 4)$(tput bold)[Reminder]$(tput sgr0) risedev will only build $(tput setaf 4)risingwave_cmd_all and risedev$(tput sgr0) crates."
+
+if [[ -n "${USE_SYSTEM_RISINGWAVE}" ]]; then
+  echo "$(tput setaf 5)$(tput bold)[Info]$(tput sgr0) env $(tput setaf 5)USE_SYSTEM_RISINGWAVE$(tput sgr0) is set, will skip building risingwave_cmd_all."
+fi
 
 [[ -z "${RISEDEV_RUSTFLAGS}" ]] || export RUSTFLAGS="${RISEDEV_RUSTFLAGS}"
 echo + RUSTFLAGS="${RUSTFLAGS:-<not set>}"
-set -xe
+echo + cargo build -p risedev
 cargo build -p risedev
-cargo build -p risingwave_cmd_all ${BUILD_HDFS_BACKEND_CMD}\
-${BUILD_HUMMOCK_TRACE_CMD}\
-            --profile "${RISINGWAVE_BUILD_PROFILE}" \
-            ${RISINGWAVE_FEATURE_FLAGS} \
-            ${RISEDEV_CARGO_BUILD_EXTRA_ARGS}
+
+if [[ -z "${USE_SYSTEM_RISINGWAVE}" ]]; then
+  set -xe
+  cargo build -p risingwave_cmd_all \
+    ${BUILD_HDFS_BACKEND_CMD} \
+    ${BUILD_HUMMOCK_TRACE_CMD} \
+    --profile "${RISINGWAVE_BUILD_PROFILE}" \
+    ${RISINGWAVE_FEATURE_FLAGS} \
+    ${RISEDEV_CARGO_BUILD_EXTRA_ARGS}
+fi
 '''
 
 [tasks.build-risingwave-playground]
@@ -537,6 +565,7 @@ dependencies = [
     "build-risingwave",
     "build-connector-node",
     "post-build-risingwave",
+    "test-system-risingwave",
     "prepare-config",
 ]
 

src/risedevtool/config/src/main.rs

@@ -65,6 +65,7 @@ pub enum Components {
     Redis,
     Tracing,
     RustComponents,
+    UseSystem,
     BuildConnectorNode,
     Dashboard,
     Release,
@@ -89,6 +90,7 @@ impl Components {
             Self::Redis => "[Component] Redis",
             Self::BuildConnectorNode => "[Build] Build RisingWave Connector (Java)",
             Self::RustComponents => "[Build] Rust components",
+            Self::UseSystem => "[Build] Use system RisingWave",
             Self::Dashboard => "[Build] Dashboard",
             Self::Tracing => "[Component] Tracing: Grafana Tempo",
             Self::Release => "[Build] Enable release mode",
@@ -127,8 +129,14 @@ Required if you want to create source from Emulated Google Pub/sub.
             Self::RustComponents => {
                 "
 Required if you want to build compute-node and meta-node.
-Otherwise you will need to manually download and copy it
-to RiseDev directory."
+Otherwise you will need to enable `USE_SYSTEM_RISINGWAVE`, or
+manually download a binary and copy it to RiseDev directory."
+            }
+            Self::UseSystem => {
+                "
+Use the RisingWave installed in the PATH, instead of building it
+from source. This implies `ENABLE_BUILD_RUST` to be false.
+                "
             }
             Self::Dashboard => {
                 "
@@ -208,6 +216,7 @@ With this option enabled, RiseDev will not set `RUST_BACKTRACE` when launching n
             "ENABLE_PROMETHEUS_GRAFANA" => Some(Self::PrometheusAndGrafana),
             "ENABLE_PUBSUB" => Some(Self::Pubsub),
             "ENABLE_BUILD_RUST" => Some(Self::RustComponents),
+            "USE_SYSTEM_RISINGWAVE" => Some(Self::UseSystem),
             "ENABLE_BUILD_DASHBOARD" => Some(Self::Dashboard),
             "ENABLE_COMPUTE_TRACING" => Some(Self::Tracing),
             "ENABLE_RELEASE_PROFILE" => Some(Self::Release),
@@ -234,6 +243,7 @@ With this option enabled, RiseDev will not set `RUST_BACKTRACE` when launching n
             Self::Pubsub => "ENABLE_PUBSUB",
             Self::Redis => "ENABLE_REDIS",
             Self::RustComponents => "ENABLE_BUILD_RUST",
+            Self::UseSystem => "USE_SYSTEM_RISINGWAVE",
             Self::Dashboard => "ENABLE_BUILD_DASHBOARD",
             Self::Tracing => "ENABLE_COMPUTE_TRACING",
             Self::Release => "ENABLE_RELEASE_PROFILE",

src/risedevtool/src/task/compactor_service.rs

@@ -19,6 +19,7 @@ use std::process::Command;
 
 use anyhow::Result;
 
+use super::risingwave_cmd;
 use crate::util::{get_program_args, get_program_env_cmd, get_program_name};
 use crate::{add_meta_node, add_tempo_endpoint, CompactorConfig, ExecuteContext, Task};
 
@@ -31,14 +32,6 @@ impl CompactorService {
         Ok(Self { config })
     }
 
-    fn compactor(&self) -> Result<Command> {
-        let prefix_bin = env::var("PREFIX_BIN")?;
-
-        Ok(Command::new(
-            Path::new(&prefix_bin).join("risingwave").join("compactor"),
-        ))
-    }
-
     /// Apply command args according to config
     pub fn apply_command_args(cmd: &mut Command, config: &CompactorConfig) -> Result<()> {
         cmd.arg("--listen-addr")
@@ -74,7 +67,7 @@ impl Task for CompactorService {
 
         let prefix_config = env::var("PREFIX_CONFIG")?;
 
-        let mut cmd = self.compactor()?;
+        let mut cmd = risingwave_cmd("compactor")?;
 
         if crate::util::is_enable_backtrace() {
             cmd.env("RUST_BACKTRACE", "1");

src/risedevtool/src/task/compute_node_service.rs

@@ -18,7 +18,7 @@ use std::process::Command;
 
 use anyhow::Result;
 
-use super::{ExecuteContext, Task};
+use super::{risingwave_cmd, ExecuteContext, Task};
 use crate::util::{get_program_args, get_program_env_cmd, get_program_name};
 use crate::{add_meta_node, add_tempo_endpoint, ComputeNodeConfig};
 
@@ -31,16 +31,6 @@ impl ComputeNodeService {
         Ok(Self { config })
     }
 
-    fn compute_node(&self) -> Result<Command> {
-        let prefix_bin = env::var("PREFIX_BIN")?;
-
-        Ok(Command::new(
-            Path::new(&prefix_bin)
-                .join("risingwave")
-                .join("compute-node"),
-        ))
-    }
-
     /// Apply command args according to config
     pub fn apply_command_args(cmd: &mut Command, config: &ComputeNodeConfig) -> Result<()> {
         cmd.arg("--listen-addr")
@@ -78,7 +68,7 @@ impl Task for ComputeNodeService {
 
         let prefix_config = env::var("PREFIX_CONFIG")?;
 
-        let mut cmd = self.compute_node()?;
+        let mut cmd = risingwave_cmd("compute-node")?;
 
         cmd.env(
             "TOKIO_CONSOLE_BIND",

src/risedevtool/src/task/frontend_service.rs

@@ -19,7 +19,7 @@ use std::process::Command;
 use anyhow::{anyhow, Result};
 use itertools::Itertools;
 
-use super::{ExecuteContext, Task};
+use super::{risingwave_cmd, ExecuteContext, Task};
 use crate::util::{get_program_args, get_program_env_cmd, get_program_name};
 use crate::{add_tempo_endpoint, FrontendConfig};
 
@@ -32,16 +32,6 @@ impl FrontendService {
         Ok(Self { config })
     }
 
-    fn frontend(&self) -> Result<Command> {
-        let prefix_bin = env::var("PREFIX_BIN")?;
-
-        Ok(Command::new(
-            Path::new(&prefix_bin)
-                .join("risingwave")
-                .join("frontend-node"),
-        ))
-    }
-
     /// Apply command args according to config
     pub fn apply_command_args(cmd: &mut Command, config: &FrontendConfig) -> Result<()> {
         cmd.arg("--listen-addr")
@@ -85,7 +75,7 @@ impl Task for FrontendService {
         ctx.service(self);
         ctx.pb.set_message("starting...");
 
-        let mut cmd = self.frontend()?;
+        let mut cmd = risingwave_cmd("frontend-node")?;
 
         if crate::util::is_enable_backtrace() {
             cmd.env("RUST_BACKTRACE", "1");

src/risedevtool/src/task/meta_node_service.rs

@@ -19,7 +19,7 @@ use std::process::Command;
 use anyhow::{anyhow, Result};
 use itertools::Itertools;
 
-use super::{ExecuteContext, Task};
+use super::{risingwave_cmd, ExecuteContext, Task};
 use crate::util::{get_program_args, get_program_env_cmd, get_program_name};
 use crate::{
     add_hummock_backend, add_tempo_endpoint, Application, HummockInMemoryStrategy, MetaBackend,
@@ -35,14 +35,6 @@ impl MetaNodeService {
         Ok(Self { config })
     }
 
-    fn meta_node(&self) -> Result<Command> {
-        let prefix_bin = env::var("PREFIX_BIN")?;
-
-        Ok(Command::new(
-            Path::new(&prefix_bin).join("risingwave").join("meta-node"),
-        ))
-    }
-
     /// Apply command args according to config
     pub fn apply_command_args(
         cmd: &mut Command,
@@ -219,7 +211,7 @@ impl Task for MetaNodeService {
         ctx.service(self);
         ctx.pb.set_message("starting...");
 
-        let mut cmd = self.meta_node()?;
+        let mut cmd = risingwave_cmd("meta-node")?;
 
         if crate::util::is_enable_backtrace() {
             cmd.env("RUST_BACKTRACE", "1");

src/risedevtool/src/task/utils.rs

@@ -12,13 +12,30 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::path::Path;
 use std::process::Command;
 
 use anyhow::{anyhow, Result};
 use itertools::Itertools;
 
+use crate::util::is_env_set;
 use crate::{AwsS3Config, MetaNodeConfig, MinioConfig, OpendalConfig, TempoConfig};
 
+/// Get the command for starting the given component of RisingWave.
+pub fn risingwave_cmd(component: &str) -> Result<Command> {
+    let cmd = if is_env_set("USE_SYSTEM_RISINGWAVE") {
+        let mut cmd = Command::new("risingwave");
+        cmd.arg(component);
+        cmd
+    } else {
+        let prefix_bin = std::env::var("PREFIX_BIN")?;
+        let path = Path::new(&prefix_bin).join("risingwave").join(component);
+        Command::new(path)
+    };
+
+    Ok(cmd)
+}
+
 /// Add a meta node to the parameters.
 pub fn add_meta_node(provide_meta_node: &[MetaNodeConfig], cmd: &mut Command) -> Result<()> {
     match provide_meta_node {