From bfd6cd0291601d722698f9f6a31ee8ee7f1d3c3a Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Wed, 12 Feb 2025 21:18:02 -0500 Subject: [PATCH 01/24] feat: draft --watch option --- Cargo.lock | 75 ++++++++++++ crates/cli/Cargo.toml | 1 + crates/cli/src/cli/mod.rs | 3 + crates/cli/src/cli/simnet/mod.rs | 88 ++++++++++++-- crates/cli/src/runbook/mod.rs | 200 ++++++++++++++++++++++++++++++- crates/core/src/simnet/mod.rs | 2 +- 6 files changed, 360 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c2468b..2ffcf39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2369,6 +2369,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futures" version = "0.1.31" @@ -3337,6 +3346,26 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +[[package]] +name = "inotify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" +dependencies = [ + "bitflags 2.8.0", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.3" @@ -3648,6 +3677,26 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kqueue" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -4227,6 +4276,31 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "notify" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" +dependencies = [ + "bitflags 2.8.0", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log 0.4.25", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.59.0", +] + +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" + [[package]] name = "num" version = "0.2.1" @@ -9002,6 +9076,7 @@ dependencies = [ "hiro-system-kit", "mime_guess", "mustache", + "notify", "ratatui", "rust-embed", "serde", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 06287c2..6a58ed6 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -44,6 +44,7 @@ actix-web = "4" actix-cors = "0.7.0" rust-embed="8.2.0" mime_guess = "2.0.4" +notify = { version = "8.0.0" } [features] default = ["cli"] diff --git a/crates/cli/src/cli/mod.rs b/crates/cli/src/cli/mod.rs index bc500f4..572ab8b 100644 --- a/crates/cli/src/cli/mod.rs +++ b/crates/cli/src/cli/mod.rs @@ -106,6 +106,9 @@ pub struct StartSimnet { /// Disable explorer (default: false) #[clap(long = "no-explorer")] pub no_explorer: bool, + /// Watch programs (default: false) + #[clap(long = "watch", action=ArgAction::SetTrue)] + pub watch: bool, /// List of geyser plugins to load #[arg(long = "geyser-plugin-config", short = 'g')] pub plugin_config_path: Vec, diff --git a/crates/cli/src/cli/simnet/mod.rs b/crates/cli/src/cli/simnet/mod.rs index ff0ac40..d8b4a17 100644 --- a/crates/cli/src/cli/simnet/mod.rs +++ b/crates/cli/src/cli/simnet/mod.rs @@ -1,9 +1,9 @@ use std::{ - path::PathBuf, + path::{Path, PathBuf}, str::FromStr, sync::{ atomic::{AtomicBool, Ordering}, - Arc, + mpsc, Arc, }, thread::sleep, time::Duration, @@ -18,6 +18,10 @@ use crate::{ use super::{Context, StartSimnet, DEFAULT_EXPLORER_PORT}; use crossbeam::channel::Select; +use notify::{ + event::{CreateKind, DataChange, ModifyKind}, + Config, Event, EventKind, RecursiveMode, Result as NotifyResult, Watcher, +}; use surfpool_core::{ simnet::SimnetEvent, solana_sdk::{ @@ -135,6 +139,9 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re Ok(deployment) => deployment, }; + let (progress_tx, progress_rx) = crossbeam::channel::unbounded(); + deploy_progress_rx.push(progress_rx); + if let Some((_framework, programs)) = deployment { // Is infrastructure-as-code (IaC) already setup? let base_location = @@ -147,14 +154,13 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re } let mut futures = vec![]; - for runbook_id in cmd.runbooks.iter() { - let (progress_tx, progress_rx) = crossbeam::channel::unbounded(); + let runbooks_ids_to_execute = cmd.runbooks.clone(); + for runbook_id in runbooks_ids_to_execute.iter() { futures.push(execute_runbook( runbook_id.clone(), - progress_tx, + progress_tx.clone(), txtx_manifest_location.clone(), )); - deploy_progress_rx.push(progress_rx); } let _handle = hiro_system_kit::thread_named("simnet") @@ -163,7 +169,72 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re Ok::<(), String>(()) }) .map_err(|e| format!("{}", e))?; + + if cmd.watch { + let _handle = hiro_system_kit::thread_named("watch filesystem") + .spawn(move || { + let mut target_path = base_location.clone(); + let _ = target_path.append_path("target"); + let _ = target_path.append_path("deploy"); + let (tx, rx) = mpsc::channel::>(); + let mut watcher = + notify::recommended_watcher(tx).map_err(|e| e.to_string())?; + watcher + .watch( + Path::new(&target_path.to_string()), + RecursiveMode::NonRecursive, + ) + .map_err(|e| e.to_string())?; + let _ = watcher.configure( + Config::default() + .with_poll_interval(Duration::from_secs(1)) + .with_compare_contents(true), + ); + for res in rx { + // Disregard any event that would not create or modify a .so file + let mut found_candidates = false; + match res { + Ok(Event { + kind: EventKind::Modify(ModifyKind::Data(DataChange::Content)), + paths, + attrs: _, + }) + | Ok(Event { + kind: EventKind::Create(CreateKind::File), + paths, + attrs: _, + }) => { + for path in paths.iter() { + if path.to_string_lossy().ends_with(".so") { + found_candidates = true; + } + } + } + _ => continue, + } + + if !found_candidates { + continue; + } + + let mut futures = vec![]; + for runbook_id in runbooks_ids_to_execute.iter() { + futures.push(execute_runbook( + runbook_id.clone(), + progress_tx.clone(), + txtx_manifest_location.clone(), + )); + } + let _ = hiro_system_kit::nestable_block_on(join_all(futures)); + } + Ok::<(), String>(()) + }) + .map_err(|e| format!("{}", e)) + .unwrap(); + } } + + // clean-up state }; // Start frontend - kept on main thread @@ -282,7 +353,7 @@ fn log_events( i => match oper.recv(&deploy_progress_rx[i - 1]) { Ok(event) => match event { BlockEvent::UpdateProgressBarStatus(update) => { - info!( + debug!( ctx.expect_logger(), "{}", format!( @@ -291,6 +362,9 @@ fn log_events( ) ); } + BlockEvent::RunbookCompleted => { + info!(ctx.expect_logger(), "{}", format!("Deployment executed",)); + } _ => {} }, Err(_e) => { diff --git a/crates/cli/src/runbook/mod.rs b/crates/cli/src/runbook/mod.rs index 5ed4b22..c275e39 100644 --- a/crates/cli/src/runbook/mod.rs +++ b/crates/cli/src/runbook/mod.rs @@ -1,3 +1,4 @@ +use dialoguer::{console::Style, theme::ColorfulTheme, Confirm}; use txtx_addon_network_svm::SvmNetworkAddon; use txtx_core::{ kit::{ @@ -7,8 +8,10 @@ use txtx_core::{ Addon, }, manifest::{file::read_runbooks_from_manifest, WorkspaceManifest}, + runbook::{ConsolidatedChanges, SynthesizedChange}, start_unsupervised_runbook_runloop, std::StdAddon, + types::RunbookSnapshotContext, }; pub fn get_addon_by_namespace(namespace: &str) -> Option> { @@ -34,7 +37,8 @@ pub async fn execute_runbook( let top_level_inputs_map = manifest.get_runbook_inputs(&Some("localnet".into()), &vec![], None)?; - let Some((mut runbook, runbook_sources, _state, _smt)) = runbooks.swap_remove(&runbook_id) + let Some((mut runbook, runbook_sources, _, runbook_state_location)) = + runbooks.swap_remove(&runbook_id) else { return Err(format!("Deployment {} not found", runbook_id)); }; @@ -54,6 +58,109 @@ pub async fn execute_runbook( runbook.enable_full_execution_mode(); + let previous_state_opt = if let Some(state_file_location) = runbook_state_location.clone() { + match state_file_location.load_execution_snapshot( + true, + &runbook.runbook_id.name, + &runbook.top_level_inputs_map.current_top_level_input_name(), + ) { + Ok(snapshot) => Some(snapshot), + Err(e) => { + println!("{} {}", red!("x"), e); + None + } + } + } else { + None + }; + + if let Some(old) = previous_state_opt { + let ctx = RunbookSnapshotContext::new(); + + let execution_context_backups = runbook.backup_execution_contexts(); + let new = runbook.simulate_and_snapshot_flows(&old).await?; + + for flow_context in runbook.flow_contexts.iter() { + if old.flows.get(&flow_context.name).is_none() { + println!( + "{} Previous snapshot not found for flow {}", + yellow!("!"), + flow_context.name + ); + }; + } + + let consolidated_changes = ctx.diff(old, new); + + let Some(consolidated_changes) = display_snapshot_diffing(consolidated_changes) else { + return Ok(()); + }; + + runbook.prepare_flows_for_new_plans( + &consolidated_changes.new_plans_to_add, + execution_context_backups, + ); + + let (actions_to_re_execute, actions_to_execute) = + runbook.prepared_flows_for_updated_plans(&consolidated_changes.plans_to_update); + + let has_actions = actions_to_re_execute + .iter() + .filter(|(_, actions)| !actions.is_empty()) + .count(); + if has_actions > 0 { + println!("The following actions will be re-executed:"); + for (context, actions) in actions_to_re_execute.iter() { + let documentation_missing = black!(""); + println!("\n{}", yellow!(format!("{}", context))); + for (action_name, documentation) in actions.into_iter() { + println!( + "- {}: {}", + action_name, + documentation.as_ref().unwrap_or(&documentation_missing) + ); + } + } + println!("\n"); + } + + let has_actions = actions_to_execute + .iter() + .filter(|(_, actions)| !actions.is_empty()) + .count(); + if has_actions > 0 { + println!( + "The following actions have been added and will be executed for the first time:" + ); + for (context, actions) in actions_to_execute.iter() { + let documentation_missing = black!(""); + println!("\n{}", green!(format!("{}", context))); + for (action_name, documentation) in actions.into_iter() { + println!( + "- {}: {}", + action_name, + documentation.as_ref().unwrap_or(&documentation_missing) + ); + } + } + println!("\n"); + } + + let theme = ColorfulTheme { + values_style: Style::new().green(), + ..ColorfulTheme::default() + }; + + let confirm = Confirm::with_theme(&theme) + .with_prompt("Do you want to continue?") + .interact() + .unwrap(); + + if !confirm { + return Ok(()); + } + } + let res = start_unsupervised_runbook_runloop(&mut runbook, &progress_tx).await; if let Err(diags) = res { println!("{} Execution aborted", red!("x")); @@ -71,5 +178,96 @@ pub async fn execute_runbook( std::process::exit(1); } + match runbook.write_runbook_state(runbook_state_location) { + Ok(Some(location)) => { + println!("\n{} Saved execution state to {}", green!("✓"), location); + } + Ok(None) => {} + Err(e) => { + println!("{} Failed to write runbook state: {}", red!("x"), e); + } + }; + Ok(()) } + +pub fn display_snapshot_diffing( + consolidated_changes: ConsolidatedChanges, +) -> Option { + let synthesized_changes = consolidated_changes.get_synthesized_changes(); + + if synthesized_changes.is_empty() && consolidated_changes.new_plans_to_add.is_empty() { + println!( + "{} Latest snapshot in sync with latest runbook updates\n", + green!("✓") + ); + return None; + } + + if !consolidated_changes.new_plans_to_add.is_empty() { + println!("\n{}", yellow!("New chain to synchronize:")); + println!("{}\n", consolidated_changes.new_plans_to_add.join(", ")); + } + + let has_critical_changes = synthesized_changes + .iter() + .filter(|(c, _)| match c { + SynthesizedChange::Edition(_, _) => true, + SynthesizedChange::FormerFailure(_, _) => false, + SynthesizedChange::Addition(_) => false, + }) + .count(); + if has_critical_changes > 0 { + println!("\n{}\n", yellow!("Changes detected:")); + for (i, (change, _impacted)) in synthesized_changes.iter().enumerate() { + match change { + SynthesizedChange::Edition(change, _) => { + let formatted_change = change + .iter() + .map(|c| { + if c.starts_with("-") { + red!(c) + } else { + green!(c) + } + }) + .collect::>() + .join(""); + println!("{}. The following edits:\n-------------------------\n{}\n-------------------------", i + 1, formatted_change); + println!("will introduce breaking changes.\n\n"); + } + SynthesizedChange::FormerFailure(_construct_to_run, command_name) => { + println!("{}. The action error:\n-------------------------\n{}\n-------------------------", i + 1, command_name); + println!("will be re-executed.\n\n"); + } + SynthesizedChange::Addition(_new_construct_did) => {} + } + } + } + + let unexecuted = synthesized_changes + .iter() + .filter(|(c, _)| match c { + SynthesizedChange::Edition(_, _) => false, + SynthesizedChange::FormerFailure(_, _) => true, + SynthesizedChange::Addition(_) => false, + }) + .count(); + if unexecuted > 0 { + println!("\n{}", yellow!("Runbook Recovery Plan")); + println!("The previous runbook execution was interrupted before completion, causing the following actions to be aborted:"); + + for (_i, (change, _impacted)) in synthesized_changes.iter().enumerate() { + match change { + SynthesizedChange::Edition(_, _) => {} + SynthesizedChange::FormerFailure(_construct_to_run, command_name) => { + println!("- {}", command_name); + } + SynthesizedChange::Addition(_new_construct_did) => {} + } + } + println!("These actions will be re-executed in the next run.\n"); + } + + Some(consolidated_changes) +} diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index 09f3912..2698941 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -329,7 +329,7 @@ pub async fn start( let plugin_name = result["name"].as_str().map(|s| s.to_owned()); - let config_file = geyser_plugin_config_file + let _config_file = geyser_plugin_config_file .as_os_str() .to_str() .ok_or(GeyserPluginManagerError::InvalidPluginPath)?; From 4fb2f830cc01877a6920331cb674cf87e9717e07 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Thu, 13 Feb 2025 13:46:51 -0500 Subject: [PATCH 02/24] feat: import admin RPC stubs --- crates/core/src/rpc/admin.rs | 231 +++++++++++++++++++++++++++++++++++ crates/core/src/rpc/mod.rs | 1 + 2 files changed, 232 insertions(+) create mode 100644 crates/core/src/rpc/admin.rs diff --git a/crates/core/src/rpc/admin.rs b/crates/core/src/rpc/admin.rs new file mode 100644 index 0000000..9f00650 --- /dev/null +++ b/crates/core/src/rpc/admin.rs @@ -0,0 +1,231 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::time::SystemTime; + +use jsonrpc_core::BoxFuture; +use jsonrpc_core::Result; +use jsonrpc_derive::rpc; +use solana_client::rpc_config::RpcAccountIndex; +use solana_sdk::pubkey::Pubkey; + +use super::RunloopContext; + +#[rpc] +pub trait AdminRpc { + type Metadata; + + #[rpc(meta, name = "exit")] + fn exit(&self, meta: Self::Metadata) -> Result<()>; + + #[rpc(meta, name = "reloadPlugin")] + fn reload_plugin( + &self, + meta: Self::Metadata, + name: String, + config_file: String, + ) -> BoxFuture>; + + #[rpc(meta, name = "unloadPlugin")] + fn unload_plugin(&self, meta: Self::Metadata, name: String) -> BoxFuture>; + + #[rpc(meta, name = "loadPlugin")] + fn load_plugin(&self, meta: Self::Metadata, config_file: String) -> BoxFuture>; + + #[rpc(meta, name = "listPlugins")] + fn list_plugins(&self, meta: Self::Metadata) -> BoxFuture>>; + + #[rpc(meta, name = "rpcAddress")] + fn rpc_addr(&self, meta: Self::Metadata) -> Result>; + + #[rpc(name = "setLogFilter")] + fn set_log_filter(&self, filter: String) -> Result<()>; + + #[rpc(meta, name = "startTime")] + fn start_time(&self, meta: Self::Metadata) -> Result; + + // #[rpc(meta, name = "startProgress")] + // fn start_progress(&self, meta: Self::Metadata) -> Result; + + #[rpc(meta, name = "addAuthorizedVoter")] + fn add_authorized_voter(&self, meta: Self::Metadata, keypair_file: String) -> Result<()>; + + #[rpc(meta, name = "addAuthorizedVoterFromBytes")] + fn add_authorized_voter_from_bytes(&self, meta: Self::Metadata, keypair: Vec) + -> Result<()>; + + #[rpc(meta, name = "removeAllAuthorizedVoters")] + fn remove_all_authorized_voters(&self, meta: Self::Metadata) -> Result<()>; + + #[rpc(meta, name = "setIdentity")] + fn set_identity( + &self, + meta: Self::Metadata, + keypair_file: String, + require_tower: bool, + ) -> Result<()>; + + #[rpc(meta, name = "setIdentityFromBytes")] + fn set_identity_from_bytes( + &self, + meta: Self::Metadata, + identity_keypair: Vec, + require_tower: bool, + ) -> Result<()>; + + #[rpc(meta, name = "setStakedNodesOverrides")] + fn set_staked_nodes_overrides(&self, meta: Self::Metadata, path: String) -> Result<()>; + + // #[rpc(meta, name = "contactInfo")] + // fn contact_info(&self, meta: Self::Metadata) -> Result; + + #[rpc(meta, name = "repairShredFromPeer")] + fn repair_shred_from_peer( + &self, + meta: Self::Metadata, + pubkey: Option, + slot: u64, + shred_index: u64, + ) -> Result<()>; + + // #[rpc(meta, name = "repairWhitelist")] + // fn repair_whitelist(&self, meta: Self::Metadata) -> Result; + + #[rpc(meta, name = "setRepairWhitelist")] + fn set_repair_whitelist(&self, meta: Self::Metadata, whitelist: Vec) -> Result<()>; + + #[rpc(meta, name = "getSecondaryIndexKeySize")] + fn get_secondary_index_key_size( + &self, + meta: Self::Metadata, + pubkey_str: String, + ) -> Result>; + + #[rpc(meta, name = "setPublicTpuAddress")] + fn set_public_tpu_address( + &self, + meta: Self::Metadata, + public_tpu_addr: SocketAddr, + ) -> Result<()>; + + #[rpc(meta, name = "setPublicTpuForwardsAddress")] + fn set_public_tpu_forwards_address( + &self, + meta: Self::Metadata, + public_tpu_forwards_addr: SocketAddr, + ) -> Result<()>; +} + +pub struct SurfpoolAdminRpc; +impl AdminRpc for SurfpoolAdminRpc { + type Metadata = Option; + + fn exit(&self, _meta: Self::Metadata) -> Result<()> { + unimplemented!() + } + + fn reload_plugin( + &self, + _meta: Self::Metadata, + _name: String, + _config_file: String, + ) -> BoxFuture> { + unimplemented!() + } + + fn unload_plugin(&self, _meta: Self::Metadata, _name: String) -> BoxFuture> { + unimplemented!() + } + + fn load_plugin(&self, _meta: Self::Metadata, _config_file: String) -> BoxFuture> { + unimplemented!() + } + + fn list_plugins(&self, _meta: Self::Metadata) -> BoxFuture>> { + unimplemented!() + } + + fn rpc_addr(&self, _meta: Self::Metadata) -> Result> { + unimplemented!() + } + + fn set_log_filter(&self, _filter: String) -> Result<()> { + unimplemented!() + } + + fn start_time(&self, _meta: Self::Metadata) -> Result { + unimplemented!() + } + + fn add_authorized_voter(&self, _meta: Self::Metadata, _keypair_file: String) -> Result<()> { + unimplemented!() + } + + fn add_authorized_voter_from_bytes(&self, _meta: Self::Metadata, _keypair: Vec) + -> Result<()> { + unimplemented!() + } + + fn remove_all_authorized_voters(&self, _meta: Self::Metadata) -> Result<()> { + unimplemented!() + } + + fn set_identity( + &self, + _meta: Self::Metadata, + _keypair_file: String, + _require_tower: bool, + ) -> Result<()> { + unimplemented!() + } + + fn set_identity_from_bytes( + &self, + _meta: Self::Metadata, + _identity_keypair: Vec, + _require_tower: bool, + ) -> Result<()> { + unimplemented!() + } + + fn set_staked_nodes_overrides(&self, _meta: Self::Metadata, _path: String) -> Result<()> { + unimplemented!() + } + + fn repair_shred_from_peer( + &self, + _meta: Self::Metadata, + _pubkey: Option, + _slot: u64, + _shred_index: u64, + ) -> Result<()> { + unimplemented!() + } + + fn set_repair_whitelist(&self, _meta: Self::Metadata, _whitelist: Vec) -> Result<()> { + unimplemented!() + } + + fn get_secondary_index_key_size( + &self, + _meta: Self::Metadata, + _pubkey_str: String, + ) -> Result> { + unimplemented!() + } + + fn set_public_tpu_address( + &self, + _meta: Self::Metadata, + _public_tpu_addr: SocketAddr, + ) -> Result<()> { + unimplemented!() + } + + fn set_public_tpu_forwards_address( + &self, + _meta: Self::Metadata, + _public_tpu_forwards_addr: SocketAddr, + ) -> Result<()> { + unimplemented!() + } +} \ No newline at end of file diff --git a/crates/core/src/rpc/mod.rs b/crates/core/src/rpc/mod.rs index c675bdd..5042aeb 100644 --- a/crates/core/src/rpc/mod.rs +++ b/crates/core/src/rpc/mod.rs @@ -10,6 +10,7 @@ use solana_transaction_status::TransactionConfirmationStatus; pub mod accounts_data; pub mod accounts_scan; +pub mod admin; pub mod bank_data; pub mod full; pub mod minimal; From 516e81a1812d664eb96de85ddd8aa59e0aa05d6a Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Thu, 13 Feb 2025 13:48:23 -0500 Subject: [PATCH 03/24] fix: iterate on plugin loading --- crates/cli/src/cli/simnet/mod.rs | 6 ++++++ crates/cli/src/tui/simnet.rs | 7 +++++++ crates/core/src/simnet/mod.rs | 23 +++++++++++++++-------- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/crates/cli/src/cli/simnet/mod.rs b/crates/cli/src/cli/simnet/mod.rs index d8b4a17..dd48bf3 100644 --- a/crates/cli/src/cli/simnet/mod.rs +++ b/crates/cli/src/cli/simnet/mod.rs @@ -298,6 +298,12 @@ fn log_events( "Account {} retrieved from Mainnet", account ); } + SimnetEvent::PluginLoaded(plugin_name) => { + info!( + ctx.expect_logger(), + "Plugin {} successfully loaded", plugin_name + ); + } SimnetEvent::EpochInfoUpdate(epoch_info) => { info!( ctx.expect_logger(), diff --git a/crates/cli/src/tui/simnet.rs b/crates/cli/src/tui/simnet.rs index 20c3828..905cd33 100644 --- a/crates/cli/src/tui/simnet.rs +++ b/crates/cli/src/tui/simnet.rs @@ -234,6 +234,13 @@ fn run_app(terminal: &mut Terminal, mut app: App) -> io::Result<( format!("Account {} retrieved from Mainnet", account), )); } + SimnetEvent::PluginLoaded(plugin_name) => { + app.events.push_front(( + EventType::Success, + Local::now(), + format!("Plugin {} successfully loaded", plugin_name), + )); + } SimnetEvent::EpochInfoUpdate(epoch_info) => { app.epoch_info = epoch_info; app.events.push_front(( diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index 2698941..bbe3fca 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -192,6 +192,7 @@ pub enum SimnetEvent { ErrorLog(DateTime, String), WarnLog(DateTime, String), DebugLog(DateTime, String), + PluginLoaded(String), TransactionReceived(DateTime, VersionedTransaction), AccountUpdate(DateTime, Pubkey), } @@ -283,12 +284,13 @@ pub async fn start( }); let simnet_config = config.simnet.clone(); + let simnet_events_tx_copy = simnet_events_tx.clone(); let (plugins_data_tx, plugins_data_rx) = unbounded::<(Transaction, TransactionMetadata)>(); if !config.plugin_config_path.is_empty() { let _handle = hiro_system_kit::thread_named("geyser plugins handler").spawn(move || { let mut plugin_manager = GeyserPluginManager::new(); - for geyser_plugin_config_file in config.plugin_config_path.iter() { + for (i, geyser_plugin_config_file) in config.plugin_config_path.iter().enumerate() { let mut file = match File::open(geyser_plugin_config_file) { Ok(file) => file, Err(err) => { @@ -297,7 +299,6 @@ pub async fn start( ))); } }; - let mut contents = String::new(); if let Err(err) = file.read_to_string(&mut contents) { return Err(GeyserPluginManagerError::CannotReadConfigFile(format!( @@ -327,25 +328,31 @@ pub async fn start( libpath = config_dir.join(libpath); } - let plugin_name = result["name"].as_str().map(|s| s.to_owned()); + let plugin_name = result["name"].as_str().map(|s| s.to_owned()).unwrap_or(format!("plugin-{}", i)); let _config_file = geyser_plugin_config_file .as_os_str() .to_str() .ok_or(GeyserPluginManagerError::InvalidPluginPath)?; - let (plugin, lib) = unsafe { - let lib = Library::new(libpath) - .map_err(|e| GeyserPluginManagerError::PluginLoadError(e.to_string()))?; + let (mut plugin, lib) = unsafe { + let lib = match Library::new(libpath) { + Ok(lib) => lib, + Err(e) => { + let _ = simnet_events_tx_copy.send(SimnetEvent::ErrorLog(Local::now(), format!("Unable to load plugin {}: {}", plugin_name, e.to_string()))); + continue; + } + }; let constructor: Symbol = lib .get(b"_create_plugin") .map_err(|e| GeyserPluginManagerError::PluginLoadError(e.to_string()))?; let plugin_raw = constructor(); (Box::from_raw(plugin_raw), lib) }; - plugin_manager.plugins.push(LoadedGeyserPlugin::new(lib, plugin, plugin_name)); + let _res = plugin.on_load("", false); + plugin_manager.plugins.push(LoadedGeyserPlugin::new(lib, plugin, Some(plugin_name.clone()))); + let _ = simnet_events_tx_copy.send(SimnetEvent::PluginLoaded(plugin_name)); } - while let Ok((transaction, transaction_metadata)) = plugins_data_rx.recv() { let transaction_status_meta = TransactionStatusMeta { status: Ok(()), From 07e1692cd59068254080659f1fecdd087168fe9a Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Thu, 13 Feb 2025 13:49:35 -0500 Subject: [PATCH 04/24] feat: subgraph plugin boilerplate --- Cargo.lock | 8 +++ Cargo.toml | 35 ++++++++++- crates/core/Cargo.toml | 62 +++++++++---------- crates/subgraph/Cargo.toml | 16 +++++ crates/subgraph/src/lib.rs | 112 ++++++++++++++++++++++++++++++++++ surfpool_subgraph_plugin.json | 4 ++ 6 files changed, 205 insertions(+), 32 deletions(-) create mode 100644 crates/subgraph/Cargo.toml create mode 100644 crates/subgraph/src/lib.rs create mode 100644 surfpool_subgraph_plugin.json diff --git a/Cargo.lock b/Cargo.lock index 2ffcf39..3a39303 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9154,6 +9154,14 @@ dependencies = [ "zstd", ] +[[package]] +name = "surfpool-subgraph" +version = "0.1.7" +dependencies = [ + "agave-geyser-plugin-interface", + "solana-program", +] + [[package]] name = "symlink" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 32bc079..f0f4c8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,10 +10,43 @@ categories = ["cryptography"] [workspace] members = [ "crates/cli", - "crates/core" + "crates/core", + "crates/subgraph" ] default-members = ["crates/cli"] resolver = "2" [workspace.dependencies] surfpool-core = { path = "crates/core", default-features = false } +agave-geyser-plugin-interface = "2.1.10" +solana-sdk = "=2.1.10" +solana-program = "2.1.10" +solana-program-test = "2.1.10" +solana-rpc-client = "2.1.10" +solana-account = "2.1.10" +solana-account-decoder = "2.1.10" +solana-accounts-db = "2.1.10" +solana-client = "2.1.10" +solana-entry = "2.1.10" +solana-faucet = "2.1.10" +solana-feature-set = "2.1.10" +solana-gossip = "2.1.10" +solana-inline-spl = "2.1.10" +solana-ledger = "2.1.10" +solana-metrics = "2.1.10" +solana-perf = "2.1.10" +solana-rpc-client-api = "2.1.10" +solana-rpc = "2.1.10" +solana-runtime = "2.1.10" +solana-runtime-transaction = "2.1.10" +solana-send-transaction-service = "2.1.10" +solana-stake-program = "2.1.10" +solana-storage-bigtable = "2.1.10" +solana-transaction-status = "2.1.10" +solana-vote-program = "2.1.10" +solana-version = "2.1.10" +solana-poh = "2.1.10" +solana-svm = "2.1.10" +solana-program-runtime = "2.1.10" +solana-geyser-plugin-manager = "2.1.10" +solana-streamer = "2.1.10" \ No newline at end of file diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 2a8f3a4..a2a9573 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -38,39 +38,39 @@ regex = "1.11.1" litesvm = { version = "0.5.0", features = ["nodejs-internal"] } # litesvm = { path = "../../../litesvm/crates/litesvm" } crossbeam = "0.8.4" -solana-sdk = "=2.1.10" -solana-program-test = "2.1.10" -solana-rpc-client = "2.1.10" -solana-account = "2.1.10" -solana-account-decoder = "2.1.10" -solana-accounts-db = "2.1.10" -solana-client = "2.1.10" -solana-entry = "2.1.10" -solana-faucet = "2.1.10" -solana-feature-set = "2.1.10" -solana-gossip = "2.1.10" -solana-inline-spl = "2.1.10" -solana-ledger = "2.1.10" -solana-metrics = "2.1.10" -solana-perf = "2.1.10" -solana-rpc-client-api = "2.1.10" -solana-rpc = "2.1.10" -solana-runtime = "2.1.10" -solana-runtime-transaction = "2.1.10" -solana-send-transaction-service = "2.1.10" -solana-stake-program = "2.1.10" -solana-storage-bigtable = "2.1.10" -solana-transaction-status = "2.1.10" -solana-vote-program = "2.1.10" -solana-version = "2.1.10" -solana-poh = "2.1.10" -solana-svm = "2.1.10" -solana-program-runtime = "2.1.10" +agave-geyser-plugin-interface = { workspace = true } +solana-sdk = { workspace = true } +solana-program-test = { workspace = true } +solana-rpc-client = { workspace = true } +solana-account = { workspace = true } +solana-account-decoder = { workspace = true } +solana-accounts-db = { workspace = true } +solana-client = { workspace = true } +solana-entry = { workspace = true } +solana-faucet = { workspace = true } +solana-feature-set = { workspace = true } +solana-gossip = { workspace = true } +solana-inline-spl = { workspace = true } +solana-ledger = { workspace = true } +solana-metrics = { workspace = true } +solana-perf = { workspace = true } +solana-rpc-client-api = { workspace = true } +solana-rpc = { workspace = true } +solana-runtime = { workspace = true } +solana-runtime-transaction = { workspace = true } +solana-send-transaction-service = { workspace = true } +solana-stake-program = { workspace = true } +solana-storage-bigtable = { workspace = true } +solana-transaction-status = { workspace = true } +solana-vote-program = { workspace = true } +solana-version = { workspace = true } +solana-poh = { workspace = true } +solana-svm = { workspace = true } +solana-program-runtime = { workspace = true } +solana-geyser-plugin-manager = { workspace = true } +solana-streamer = { workspace = true } spl-token-2022 = "6.0.0" spl-token = "7.0.0" -solana-streamer = "2.1.10" zstd = "0.13.2" -agave-geyser-plugin-interface = "2.1.10" -solana-geyser-plugin-manager = "2.1.10" libloading = "0.7.4" json5 = "0.4.1" \ No newline at end of file diff --git a/crates/subgraph/Cargo.toml b/crates/subgraph/Cargo.toml new file mode 100644 index 0000000..5acbaf6 --- /dev/null +++ b/crates/subgraph/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "surfpool-subgraph" +version = { workspace = true } +readme = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +keywords = { workspace = true } +categories = { workspace = true } + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +agave-geyser-plugin-interface = { workspace = true } +solana-program = { workspace = true } diff --git a/crates/subgraph/src/lib.rs b/crates/subgraph/src/lib.rs new file mode 100644 index 0000000..0bb63b3 --- /dev/null +++ b/crates/subgraph/src/lib.rs @@ -0,0 +1,112 @@ +use { + agave_geyser_plugin_interface::geyser_plugin_interface::{ + GeyserPlugin, GeyserPluginError, ReplicaAccountInfoVersions, ReplicaBlockInfoVersions, ReplicaEntryInfoVersions, ReplicaTransactionInfoVersions, Result as PluginResult, SlotStatus + }, + solana_program::clock::Slot, +}; + +#[derive(Default, Debug)] +pub struct SurfpoolSubgraph {} + +impl GeyserPlugin for SurfpoolSubgraph { + fn name(&self) -> &'static str { + "surfpool-subgraph" + } + + fn on_load(&mut self, _config_file: &str, _is_reload: bool) -> PluginResult<()> { + Ok(()) + } + + fn on_unload(&mut self) {} + + fn notify_end_of_startup(&self) -> PluginResult<()> { + Ok(()) + } + + fn update_account( + &self, + account: ReplicaAccountInfoVersions, + _slot: Slot, + _is_startup: bool, + ) -> PluginResult<()> { + let account_info = match account { + ReplicaAccountInfoVersions::V0_0_1(_info) => { + unreachable!("ReplicaAccountInfoVersions::V0_0_1 is not supported") + } + ReplicaAccountInfoVersions::V0_0_2(_info) => { + unreachable!("ReplicaAccountInfoVersions::V0_0_2 is not supported") + } + ReplicaAccountInfoVersions::V0_0_3(info) => info, + }; + + println!("lamports: {}", account_info.lamports); + + Ok(()) + } + + fn update_slot_status( + &self, + _slot: Slot, + _parent: Option, + _status: &SlotStatus, + ) -> PluginResult<()> { + Ok(()) + } + + fn notify_transaction( + &self, + transaction: ReplicaTransactionInfoVersions, + _slot: Slot, + ) -> PluginResult<()> { + match transaction { + ReplicaTransactionInfoVersions::V0_0_2(data) => { + if data.is_vote { + return Ok(()) + } + + let Some(inner_instructions) = data.transaction_status_meta.inner_instructions else { + return Ok(()) + }; + + for inner_instructions in inner_instructions.iter() { + for instruction in inner_instructions.instructions.iter() { + + } + } + } + ReplicaTransactionInfoVersions::V0_0_1(_) => {} + } + Ok(()) + } + + fn notify_entry(&self, _entry: ReplicaEntryInfoVersions) -> PluginResult<()> { + Ok(()) + } + + fn notify_block_metadata(&self, _blockinfo: ReplicaBlockInfoVersions) -> PluginResult<()> { + Ok(()) + } + + fn account_data_notifications_enabled(&self) -> bool { + true + } + + fn transaction_notifications_enabled(&self) -> bool { + false + } + + fn entry_notifications_enabled(&self) -> bool { + false + } +} + +#[no_mangle] +#[allow(improper_ctypes_definitions)] + +/// # Safety +/// +/// This function returns the Plugin pointer as trait GeyserPlugin. +pub unsafe extern "C" fn _create_plugin() -> *mut dyn GeyserPlugin { + let plugin: Box = Box::::default(); + Box::into_raw(plugin) +} diff --git a/surfpool_subgraph_plugin.json b/surfpool_subgraph_plugin.json new file mode 100644 index 0000000..d31dc3e --- /dev/null +++ b/surfpool_subgraph_plugin.json @@ -0,0 +1,4 @@ +{ + "name": "surfpool-subgraph", + "libpath": "target/release/libsurfpool_subgraph.dylib" +} \ No newline at end of file From 7e1a34b43427d75f0e6af5978190f98c9c0c2b37 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Thu, 13 Feb 2025 22:07:43 -0500 Subject: [PATCH 05/24] wip --- Cargo.lock | 180 +++++++++++++++++++++++++++-- Cargo.toml | 9 +- crates/cli/Cargo.toml | 9 +- crates/cli/src/cli/simnet/mod.rs | 89 ++++++++++---- crates/cli/src/http/mod.rs | 108 ++++++++++++++++- crates/cli/src/tui/simnet.rs | 5 +- crates/core/Cargo.toml | 4 +- crates/core/src/lib.rs | 14 ++- crates/core/src/rpc/admin.rs | 33 ++++-- crates/core/src/rpc/full.rs | 10 +- crates/core/src/rpc/mod.rs | 22 ++-- crates/core/src/simnet/mod.rs | 85 ++++---------- crates/core/src/types.rs | 83 ++++++++++++- crates/gql/Cargo.toml | 23 ++++ crates/gql/src/lib.rs | 37 ++++++ crates/gql/src/mutation.rs | 13 +++ crates/gql/src/query.rs | 61 ++++++++++ crates/gql/src/subscription.rs | 27 +++++ crates/gql/src/types/collection.rs | 36 ++++++ crates/gql/src/types/entry.rs | 27 +++++ crates/gql/src/types/mod.rs | 2 + crates/subgraph/Cargo.toml | 1 + crates/subgraph/src/lib.rs | 21 ++-- 23 files changed, 759 insertions(+), 140 deletions(-) create mode 100644 crates/gql/Cargo.toml create mode 100644 crates/gql/src/lib.rs create mode 100644 crates/gql/src/mutation.rs create mode 100644 crates/gql/src/query.rs create mode 100644 crates/gql/src/subscription.rs create mode 100644 crates/gql/src/types/collection.rs create mode 100644 crates/gql/src/types/entry.rs create mode 100644 crates/gql/src/types/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 3a39303..e1820ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,6 +210,19 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "actix-ws" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535aec173810be3ca6f25dd5b4d431ae7125d62000aa3cbae1ec739921b02cf3" +dependencies = [ + "actix-codec", + "actix-http", + "actix-web", + "futures-core", + "tokio", +] + [[package]] name = "addr2line" version = "0.24.2" @@ -721,6 +734,12 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + [[package]] name = "atty" version = "0.2.14" @@ -732,6 +751,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "auto_enums" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781" +dependencies = [ + "derive_utils 0.15.0", + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -1868,6 +1899,28 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "derive_utils" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "532b4c15dccee12c7044f1fcad956e98410860b22231e44a3b827464797ca7bf" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_utils" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "dialoguer" version = "0.10.4" @@ -2415,6 +2468,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-enum" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3422d14de7903a52e9dbc10ae05a7e14445ec61890100e098754e120b2bd7b1e" +dependencies = [ + "derive_utils 0.11.2", + "quote", + "syn 1.0.109", +] + [[package]] name = "futures-executor" version = "0.3.31" @@ -2677,7 +2741,7 @@ dependencies = [ "fnv", "hcl-primitives", "vecmap-rs", - "winnow 0.7.1", + "winnow 0.7.2", ] [[package]] @@ -3648,6 +3712,74 @@ dependencies = [ "unicase", ] +[[package]] +name = "juniper" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943306315b1a7a03d27af9dfb0c288d9f4da8830c17df4bceb7d50a47da0982c" +dependencies = [ + "async-trait", + "auto_enums", + "fnv", + "futures 0.3.31", + "indexmap 2.7.1", + "juniper_codegen", + "serde", + "smartstring", + "static_assertions", +] + +[[package]] +name = "juniper_actix" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eafd05a9b03395e4c032e5d903109a5fceaf89bb7ae6731e416bc2088f9e556" +dependencies = [ + "actix-http", + "actix-web", + "actix-ws", + "anyhow", + "futures 0.3.31", + "juniper", + "juniper_graphql_ws", + "serde", + "serde_json", +] + +[[package]] +name = "juniper_codegen" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760dbe46660494d469023d661e8d268f413b2cb68c999975dcc237407096a693" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", + "url 2.5.4", +] + +[[package]] +name = "juniper_graphql_ws" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709eb11c716072f5c9fcbfa705dd684bd3c070943102f9fc56ccb812a36ba017" +dependencies = [ + "juniper", + "juniper_subscriptions", + "serde", + "tokio", +] + +[[package]] +name = "juniper_subscriptions" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6208a839bd4ca2131924a238311d088d6604ea267c0917903392bad7b70a92c" +dependencies = [ + "futures 0.3.31", + "juniper", +] + [[package]] name = "keccak" version = "0.1.5" @@ -6105,6 +6237,17 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "smpl_jwt" version = "0.7.1" @@ -9074,6 +9217,8 @@ dependencies = [ "dialoguer 0.11.0", "dirs 6.0.0", "hiro-system-kit", + "juniper_actix", + "juniper_graphql_ws", "mime_guess", "mustache", "notify", @@ -9082,6 +9227,7 @@ dependencies = [ "serde", "serde_json", "surfpool-core", + "surfpool-gql", "tokio", "toml 0.8.19", "txtx-addon-network-svm", @@ -9151,15 +9297,32 @@ dependencies = [ "symlink", "tokio", "tokio-util 0.7.13", + "txtx-addon-network-svm", + "uuid", "zstd", ] +[[package]] +name = "surfpool-gql" +version = "0.1.7" +dependencies = [ + "async-stream", + "futures 0.3.31", + "futures-enum", + "juniper", + "juniper_codegen", + "surfpool-core", + "tokio", + "uuid", +] + [[package]] name = "surfpool-subgraph" version = "0.1.7" dependencies = [ "agave-geyser-plugin-interface", "solana-program", + "txtx-addon-network-svm", ] [[package]] @@ -9879,9 +10042,7 @@ dependencies = [ [[package]] name = "txtx-addon-kit" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349567e68d1724bec1eec4ccf766232be8cdfd257f5453063e766b5b978eb90e" +version = "0.2.3" dependencies = [ "crossbeam-channel", "dirs 5.0.1", @@ -9913,8 +10074,6 @@ dependencies = [ [[package]] name = "txtx-addon-network-svm" version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ce3a54125dcded77c3e27cc3891cffbdd12aa1f1340dfc289d6c9d31e1d5353" dependencies = [ "anchor-lang-idl", "async-recursion", @@ -9935,9 +10094,7 @@ dependencies = [ [[package]] name = "txtx-core" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db49fab87668ddb4b4af2e0661566c23bbba269d3b36f4cc067dfc2863bfe1a0" +version = "0.2.3" dependencies = [ "base64 0.22.1", "better-debug", @@ -10142,6 +10299,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ + "atomic", "getrandom 0.2.15", "serde", ] @@ -10590,9 +10748,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f" +checksum = "59690dea168f2198d1a3b0cac23b8063efcd11012f10ae4698f284808c8ef603" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index f0f4c8b..bfa6181 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace.package] version = "0.1.7" edition = "2021" +description = "Surfpool is the best place to train before surfing Solana." license = "Apache-2.0" readme = "README.md" repository = "https://github.com/txtx/surfpool" @@ -11,6 +12,7 @@ categories = ["cryptography"] members = [ "crates/cli", "crates/core", + "crates/gql", "crates/subgraph" ] default-members = ["crates/cli"] @@ -18,6 +20,7 @@ resolver = "2" [workspace.dependencies] surfpool-core = { path = "crates/core", default-features = false } +surfpool-gql = { path = "crates/gql", default-features = false } agave-geyser-plugin-interface = "2.1.10" solana-sdk = "=2.1.10" solana-program = "2.1.10" @@ -49,4 +52,8 @@ solana-poh = "2.1.10" solana-svm = "2.1.10" solana-program-runtime = "2.1.10" solana-geyser-plugin-manager = "2.1.10" -solana-streamer = "2.1.10" \ No newline at end of file +solana-streamer = "2.1.10" +txtx-core = { path = "../txtx/crates/txtx-core" } +txtx-addon-network-svm = { package = "txtx-addon-network-svm", path = "../txtx/addons/svm" } +# txtx-core = { version = "0.2.2" } +# txtx-addon-network-svm = { version = "0.1.3" } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 6a58ed6..d9b6305 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -15,11 +15,10 @@ path = "src/main.rs" [dependencies] surfpool-core = { workspace = true } +surfpool-gql = { workspace = true } # surfpool-core = { version = "0.1" } -# txtx-core = { path = "../../../txtx/crates/txtx-core" } -# txtx-addon-network-svm = { package = "txtx-addon-network-svm", path = "../../../txtx/addons/svm" } -txtx-core = { version = "0.2.2" } -txtx-addon-network-svm = { version = "0.1.3" } +txtx-core = { workspace = true } +txtx-addon-network-svm = { workspace = true } hiro-system-kit = "0.3.1" atty = "0.2.13" ansi_term = "0.12.1" @@ -45,6 +44,8 @@ actix-cors = "0.7.0" rust-embed="8.2.0" mime_guess = "2.0.4" notify = { version = "8.0.0" } +juniper_actix = {version = "0.5.0", features = ["subscriptions"] } +juniper_graphql_ws = { version = "0.4.0", features = ["graphql-transport-ws"] } [features] default = ["cli"] diff --git a/crates/cli/src/cli/simnet/mod.rs b/crates/cli/src/cli/simnet/mod.rs index dd48bf3..64bd8b0 100644 --- a/crates/cli/src/cli/simnet/mod.rs +++ b/crates/cli/src/cli/simnet/mod.rs @@ -23,14 +23,16 @@ use notify::{ Config, Event, EventKind, RecursiveMode, Result as NotifyResult, Watcher, }; use surfpool_core::{ - simnet::SimnetEvent, solana_sdk::{ pubkey::Pubkey, signature::Keypair, signer::{EncodableKey, Signer}, }, start_simnet, - types::{RpcConfig, RunloopTriggerMode, SimnetConfig, SurfpoolConfig}, + types::{ + RpcConfig, RunloopTriggerMode, SimnetConfig, SimnetEvent, SubgraphConfig, SubgraphEvent, + SurfpoolConfig, + }, }; use txtx_core::kit::{ channel::Receiver, futures::future::join_all, helpers::fs::FileLocation, @@ -83,6 +85,7 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re airdrop_addresses, airdrop_token_amount: cmd.airdrop_token_amount, }, + subgraph: SubgraphConfig {}, plugin_config_path: cmd .plugin_config_path .iter() @@ -95,12 +98,23 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re // We start the simnet as soon as possible, as it needs to be ready for deployments let (simnet_commands_tx, simnet_commands_rx) = crossbeam::channel::unbounded(); let (simnet_events_tx, simnet_events_rx) = crossbeam::channel::unbounded(); - let ctx_cloned = ctx.clone(); + let (subgraph_commands_tx, subgraph_commands_rx) = crossbeam::channel::unbounded(); + let (subgraph_events_tx, subgraph_events_rx) = crossbeam::channel::unbounded(); + + let ctx_copy = ctx.clone(); + let simnet_commands_tx_copy = simnet_commands_tx.clone(); + let config_copy = config.clone(); let _handle = hiro_system_kit::thread_named("simnet") .spawn(move || { - let future = start_simnet(config, simnet_events_tx, simnet_commands_rx); + let future = start_simnet( + config_copy, + subgraph_commands_tx, + simnet_events_tx, + simnet_commands_tx_copy, + simnet_commands_rx, + ); if let Err(e) = hiro_system_kit::nestable_block_on(future) { - error!(ctx_cloned.expect_logger(), "{e}"); + error!(ctx_copy.expect_logger(), "{e}"); sleep(Duration::from_millis(500)); std::process::exit(1); } @@ -108,6 +122,8 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re }) .map_err(|e| format!("{}", e))?; + let ctx_copy = ctx.clone(); + let subgraph_events_tx_copy = subgraph_events_tx.clone(); loop { match simnet_events_rx.recv() { Ok(SimnetEvent::Aborted(error)) => return Err(error), @@ -117,16 +133,16 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re } } - let explorer_handle = if !cmd.no_explorer { - let ctx_cloned = ctx.clone(); - let network_binding = format!("{}:{}", cmd.network_host, DEFAULT_EXPLORER_PORT); - let future = start_server(&network_binding, &ctx_cloned) - .await - .map_err(|e| format!("{}", e.to_string()))?; - Some(future) - } else { - None - }; + let network_binding = format!("{}:{}", cmd.network_host, DEFAULT_EXPLORER_PORT); + let explorer_handle = start_server( + network_binding, + config, + subgraph_events_tx_copy, + subgraph_commands_rx, + &ctx_copy, + ) + .await + .map_err(|e| format!("{}", e.to_string()))?; let mut deploy_progress_rx = vec![]; if !cmd.no_deploy { @@ -239,7 +255,13 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re // Start frontend - kept on main thread if cmd.no_tui { - log_events(simnet_events_rx, cmd.debug, deploy_progress_rx, ctx)?; + log_events( + simnet_events_rx, + subgraph_events_rx, + cmd.debug, + deploy_progress_rx, + ctx, + )?; } else { tui::simnet::start_app( simnet_events_rx, @@ -252,14 +274,13 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re ) .map_err(|e| format!("{}", e))?; } - if let Some(explorer_handle) = explorer_handle { - let _ = explorer_handle.stop(true); - } + let _ = explorer_handle.stop(true); Ok(()) } fn log_events( simnet_events_rx: Receiver, + subgraph_events_rx: Receiver, include_debug_logs: bool, deploy_progress_rx: Vec>, ctx: &Context, @@ -280,6 +301,7 @@ fn log_events( let mut handles = vec![]; selector.recv(&simnet_events_rx); + selector.recv(&subgraph_events_rx); if !deployment_completed { for rx in deploy_progress_rx.iter() { @@ -334,7 +356,7 @@ fn log_events( SimnetEvent::WarnLog(_dt, log) => { warn!(ctx.expect_logger(), "{}", log); } - SimnetEvent::TransactionReceived(_dt, transaction) => { + SimnetEvent::TransactionSimulated(_dt, transaction) => { if deployment_completed { info!( ctx.expect_logger(), @@ -356,7 +378,32 @@ fn log_events( break; } }, - i => match oper.recv(&deploy_progress_rx[i - 1]) { + 1 => match oper.recv(&subgraph_events_rx) { + Ok(event) => match event { + SubgraphEvent::ErrorLog(_dt, log) => { + error!(ctx.expect_logger(), "{}", log); + } + SubgraphEvent::InfoLog(_dt, log) => { + info!(ctx.expect_logger(), "{}", log); + } + SubgraphEvent::DebugLog(_dt, log) => { + if include_debug_logs { + info!(ctx.expect_logger(), "{}", log); + } + } + SubgraphEvent::WarnLog(_dt, log) => { + warn!(ctx.expect_logger(), "{}", log); + } + SubgraphEvent::EndpointReady => {} + SubgraphEvent::Shutdown => { + break; + } + }, + Err(_e) => { + break; + } + }, + i => match oper.recv(&deploy_progress_rx[i - 2]) { Ok(event) => match event { BlockEvent::UpdateProgressBarStatus(update) => { debug!( diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 288f6e2..4db6f53 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -2,10 +2,21 @@ use crate::cli::Context; use actix_cors::Cors; use actix_web::dev::ServerHandle; use actix_web::http::header::{self}; -use actix_web::web; -use actix_web::Responder; +use actix_web::web::{self, Data}; +use actix_web::Error; use actix_web::{middleware, App, HttpResponse, HttpServer}; +use actix_web::{HttpRequest, Responder}; +use crossbeam::channel::{Receiver, Sender}; +use juniper_actix::{graphiql_handler, graphql_handler, playground_handler, subscriptions}; +use juniper_graphql_ws::ConnectionConfig; use std::error::Error as StdError; +use std::sync::RwLock; +use std::time::Duration; +use surfpool_core::types::{Collection, SubgraphCommand, SubgraphEvent, SurfpoolConfig}; +use surfpool_gql::types::collection::CollectionData; +use surfpool_gql::{new_graphql_schema, Context as GqlContext, GqlSchema}; +use txtx_core::kit::types::frontend::{ClientType, DiscoveryResponse}; +use txtx_core::kit::uuid::Uuid; #[cfg(feature = "explorer")] use rust_embed::RustEmbed; @@ -16,11 +27,51 @@ use rust_embed::RustEmbed; pub struct Asset; pub async fn start_server( - network_binding: &str, + network_binding: String, + config: SurfpoolConfig, + subgraph_events_tx: Sender, + subgraph_commands_rx: Receiver, _ctx: &Context, ) -> Result> { + let gql_schema = Data::new(new_graphql_schema()); + let gql_context: Data> = Data::new(RwLock::new(GqlContext::new())); + + let gql_context_copy = gql_context.clone(); + let _handle = hiro_system_kit::thread_named("subgraph") + .spawn(move || { + while let Ok(command) = subgraph_commands_rx.recv() { + match command { + SubgraphCommand::CreateEndpoint(config, sender) => { + println!("{:?}", config); + let gql_context = gql_context_copy.write().unwrap(); + let mut collections = gql_context.collections_store.write().unwrap(); + let collection_uuid = Uuid::new_v4(); + collections.insert( + collection_uuid.clone(), + CollectionData { + collection: Collection { + uuid: collection_uuid, + name: config.subgraph_name.clone(), + entries: vec![], + }, + }, + ); + println!("{:?}", collections); + let _ = sender.send("http://127.0.0.1:8900/graphql".into()); + } + SubgraphCommand::Shutdown => { + let _ = subgraph_events_tx.send(SubgraphEvent::Shutdown); + } + } + } + Ok::<(), String>(()) + }) + .map_err(|e| format!("{}", e))?; + let server = HttpServer::new(move || { App::new() + .app_data(gql_schema.clone()) + .app_data(gql_context.clone()) .wrap( Cors::default() .allow_any_origin() @@ -32,6 +83,14 @@ pub async fn start_server( ) .wrap(middleware::Compress::default()) .wrap(middleware::Logger::default()) + .service( + web::scope("/gql/v1") + .route("/graphql?", web::get().to(get_graphql)) + .route("/graphql", web::post().to(post_graphql)) + .route("/subscriptions", web::get().to(subscriptions)), + ) + .service(web::resource("/playground").route(web::get().to(playground))) + .service(web::resource("/graphiql").route(web::get().to(graphiql))) .service(dist) }) .workers(5) @@ -74,3 +133,46 @@ async fn dist(path: web::Path) -> impl Responder { }; handle_embedded_file(path_str) } + +async fn post_graphql( + req: HttpRequest, + payload: web::Payload, + schema: Data, + context: Data, +) -> Result { + println!("POST /graphql"); + graphql_handler(&schema, &context, req, payload).await +} + +async fn get_graphql( + req: HttpRequest, + payload: web::Payload, + schema: Data, + context: Data, +) -> Result { + println!("GET /graphql"); + graphql_handler(&schema, &context, req, payload).await +} + +async fn subscriptions( + req: HttpRequest, + stream: web::Payload, + schema: Data, + context: Data, +) -> Result { + let ctx = GqlContext { + collections_store: context.collections_store.clone(), + entries_broadcaster: context.entries_broadcaster.clone(), + }; + let config = ConnectionConfig::new(ctx); + let config = config.with_keep_alive_interval(Duration::from_secs(15)); + subscriptions::ws_handler(req, stream, schema.into_inner(), config).await +} + +async fn playground() -> Result { + playground_handler("/graphql", Some("/subscriptions")).await +} + +async fn graphiql() -> Result { + graphiql_handler("/graphql", Some("/subscriptions")).await +} diff --git a/crates/cli/src/tui/simnet.rs b/crates/cli/src/tui/simnet.rs index 905cd33..4d68092 100644 --- a/crates/cli/src/tui/simnet.rs +++ b/crates/cli/src/tui/simnet.rs @@ -12,14 +12,13 @@ use ratatui::{ }; use std::{collections::VecDeque, error::Error, io, time::Duration}; use surfpool_core::{ - simnet::{ClockCommand, SimnetCommand, SimnetEvent}, solana_rpc_client::rpc_client::RpcClient, solana_sdk::{ clock::Clock, commitment_config::CommitmentConfig, epoch_info::EpochInfo, message::Message, pubkey::Pubkey, signature::Keypair, signer::Signer, system_instruction, transaction::Transaction, }, - types::RunloopTriggerMode, + types::{ClockCommand, RunloopTriggerMode, SimnetCommand, SimnetEvent}, }; use txtx_core::kit::types::frontend::BlockEvent; use txtx_core::kit::{channel::Receiver, types::frontend::ProgressBarStatusColor}; @@ -279,7 +278,7 @@ fn run_app(terminal: &mut Terminal, mut app: App) -> io::Result<( SimnetEvent::WarnLog(dt, log) => { app.events.push_front((EventType::Warning, dt, log)); } - SimnetEvent::TransactionReceived(_dt, _transaction) => { + SimnetEvent::TransactionSimulated(_dt, _transaction) => { app.successful_transactions += 1; } SimnetEvent::BlockHashExpired => {} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index a2a9573..93cc378 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -73,4 +73,6 @@ spl-token-2022 = "6.0.0" spl-token = "7.0.0" zstd = "0.13.2" libloading = "0.7.4" -json5 = "0.4.1" \ No newline at end of file +json5 = "0.4.1" +txtx-addon-network-svm = { workspace = true } +uuid = "1.7.0" \ No newline at end of file diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index f8f079e..c62abab 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -18,16 +18,24 @@ use crossbeam_channel::{Receiver, Sender}; pub use jsonrpc_core; pub use jsonrpc_http_server; pub use litesvm; -use simnet::{SimnetCommand, SimnetEvent}; pub use solana_rpc_client; pub use solana_sdk; -use types::SurfpoolConfig; +use types::{SimnetCommand, SimnetEvent, SubgraphCommand, SubgraphEvent, SurfpoolConfig}; pub async fn start_simnet( config: SurfpoolConfig, + subgraph_commands_tx: Sender, simnet_events_tx: Sender, + simnet_commands_tx: Sender, simnet_commands_rx: Receiver, ) -> Result<(), Box> { - simnet::start(config, simnet_events_tx, simnet_commands_rx).await?; + simnet::start( + config, + subgraph_commands_tx, + simnet_events_tx, + simnet_commands_tx, + simnet_commands_rx, + ) + .await?; Ok(()) } diff --git a/crates/core/src/rpc/admin.rs b/crates/core/src/rpc/admin.rs index 9f00650..8785b0b 100644 --- a/crates/core/src/rpc/admin.rs +++ b/crates/core/src/rpc/admin.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; use std::net::SocketAddr; +use std::time::Duration; use std::time::SystemTime; use jsonrpc_core::BoxFuture; @@ -7,6 +8,10 @@ use jsonrpc_core::Result; use jsonrpc_derive::rpc; use solana_client::rpc_config::RpcAccountIndex; use solana_sdk::pubkey::Pubkey; +use txtx_addon_network_svm::codec::subgraph::PluginConfig; +use txtx_addon_network_svm::codec::subgraph::SubgraphRequest; + +use crate::types::SubgraphCommand; use super::RunloopContext; @@ -136,8 +141,17 @@ impl AdminRpc for SurfpoolAdminRpc { unimplemented!() } - fn load_plugin(&self, _meta: Self::Metadata, _config_file: String) -> BoxFuture> { - unimplemented!() + fn load_plugin(&self, meta: Self::Metadata, config_file: String) -> BoxFuture> { + let config = serde_json::from_str::(&config_file).unwrap(); + let ctx = meta.unwrap(); + let (tx, rx) = crossbeam_channel::bounded(1); + let _ = ctx + .subgraph_commands_tx + .send(SubgraphCommand::CreateEndpoint(config.data.clone(), tx)); + let Ok(endpoint_url) = rx.recv_timeout(Duration::from_secs(10)) else { + unimplemented!() + }; + Box::pin(async move { Ok(endpoint_url) }) } fn list_plugins(&self, _meta: Self::Metadata) -> BoxFuture>> { @@ -160,10 +174,13 @@ impl AdminRpc for SurfpoolAdminRpc { unimplemented!() } - fn add_authorized_voter_from_bytes(&self, _meta: Self::Metadata, _keypair: Vec) - -> Result<()> { - unimplemented!() - } + fn add_authorized_voter_from_bytes( + &self, + _meta: Self::Metadata, + _keypair: Vec, + ) -> Result<()> { + unimplemented!() + } fn remove_all_authorized_voters(&self, _meta: Self::Metadata) -> Result<()> { unimplemented!() @@ -200,7 +217,7 @@ impl AdminRpc for SurfpoolAdminRpc { ) -> Result<()> { unimplemented!() } - + fn set_repair_whitelist(&self, _meta: Self::Metadata, _whitelist: Vec) -> Result<()> { unimplemented!() } @@ -228,4 +245,4 @@ impl AdminRpc for SurfpoolAdminRpc { ) -> Result<()> { unimplemented!() } -} \ No newline at end of file +} diff --git a/crates/core/src/rpc/full.rs b/crates/core/src/rpc/full.rs index 6a06c41..17b141a 100644 --- a/crates/core/src/rpc/full.rs +++ b/crates/core/src/rpc/full.rs @@ -6,8 +6,10 @@ use jsonrpc_core::BoxFuture; use jsonrpc_core::{Error, Result}; use jsonrpc_derive::rpc; use solana_account_decoder::{encode_ui_account, UiAccountEncoding}; +use solana_client::nonblocking::rpc_client::RpcClient; use solana_client::rpc_config::RpcContextConfig; use solana_client::rpc_custom_error::RpcCustomError; +use solana_client::rpc_request::RpcRequest; use solana_client::rpc_response::RpcApiVersion; use solana_client::rpc_response::RpcResponseContext; use solana_client::{ @@ -340,8 +342,12 @@ impl Full for SurfpoolFullRpc { let signature = signatures[0]; let (status_update_tx, status_uptate_rx) = crossbeam_channel::bounded(1); let _ = ctx - .mempool_tx - .send((ctx.id.clone(), unsanitized_tx, status_update_tx)); + .simnet_commands_tx + .send(SimnetCommand::TransactionReceived( + ctx.id.clone(), + unsanitized_tx, + status_update_tx, + )); loop { match (status_uptate_rx.recv(), config.preflight_commitment) { ( diff --git a/crates/core/src/rpc/mod.rs b/crates/core/src/rpc/mod.rs index 5042aeb..f735a41 100644 --- a/crates/core/src/rpc/mod.rs +++ b/crates/core/src/rpc/mod.rs @@ -29,11 +29,8 @@ pub struct SurfpoolRpc; pub struct RunloopContext { pub id: Hash, pub state: Arc>, - pub mempool_tx: Sender<( - Hash, - VersionedTransaction, - Sender, - )>, + pub simnet_commands_tx: Sender, + pub subgraph_commands_tx: Sender, } trait State { @@ -77,18 +74,18 @@ impl State for Option { impl Metadata for RunloopContext {} -use crate::{simnet::GlobalState, types::RpcConfig}; +use crate::{ + simnet::GlobalState, + types::{RpcConfig, SimnetCommand, SubgraphCommand}, +}; use jsonrpc_core::futures::FutureExt; use std::future::Future; #[derive(Clone)] pub struct SurfpoolMiddleware { pub context: Arc>, - pub mempool_tx: Sender<( - Hash, - VersionedTransaction, - Sender, - )>, + pub simnet_commands_tx: Sender, + pub subgraph_commands_tx: Sender, pub config: RpcConfig, } @@ -109,7 +106,8 @@ impl Middleware> for SurfpoolMiddleware { let meta = Some(RunloopContext { id: Hash::new_unique(), state: self.context.clone(), - mempool_tx: self.mempool_tx.clone(), + simnet_commands_tx: self.simnet_commands_tx.clone(), + subgraph_commands_tx: self.subgraph_commands_tx.clone(), }); // println!("Processing request {:?}", request); Either::Left(Box::pin(next(request, meta).map(move |res| { diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index bbe3fca..c5013cc 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -2,7 +2,7 @@ use agave_geyser_plugin_interface::geyser_plugin_interface::{ GeyserPlugin, ReplicaTransactionInfoV2, ReplicaTransactionInfoVersions, }; use base64::prelude::{Engine, BASE64_STANDARD}; -use chrono::{DateTime, Local, Utc}; +use chrono::{Local, Utc}; use crossbeam::select; use crossbeam_channel::{unbounded, Receiver, Sender}; use jsonrpc_core::MetaIoHandler; @@ -16,12 +16,8 @@ use solana_sdk::{ clock::Clock, epoch_info::EpochInfo, message::v0::LoadedAddresses, - pubkey::Pubkey, signature::Signature, - transaction::{ - SanitizedTransaction, Transaction, TransactionError, TransactionVersion, - VersionedTransaction, - }, + transaction::{SanitizedTransaction, Transaction, TransactionError, TransactionVersion}, }; use solana_transaction_status::{ option_serializer::OptionSerializer, EncodedConfirmedTransactionWithStatusMeta, @@ -40,10 +36,13 @@ use std::{ use crate::{ rpc::{ - self, accounts_data::AccountsData, accounts_scan::AccountsScan, bank_data::BankData, - full::Full, minimal::Minimal, SurfpoolMiddleware, + self, accounts_data::AccountsData, accounts_scan::AccountsScan, admin::AdminRpc, + bank_data::BankData, full::Full, minimal::Minimal, SurfpoolMiddleware, + }, + types::{ + ClockCommand, ClockEvent, RunloopTriggerMode, SimnetCommand, SimnetEvent, SubgraphCommand, + SubgraphEvent, SurfpoolConfig, }, - types::{RunloopTriggerMode, SurfpoolConfig}, }; const BLOCKHASH_SLOT_TTL: u64 = 75; @@ -180,49 +179,15 @@ impl EntryStatus { } } -#[derive(Debug)] -pub enum SimnetEvent { - Ready, - Aborted(String), - Shutdown, - ClockUpdate(Clock), - EpochInfoUpdate(EpochInfo), - BlockHashExpired, - InfoLog(DateTime, String), - ErrorLog(DateTime, String), - WarnLog(DateTime, String), - DebugLog(DateTime, String), - PluginLoaded(String), - TransactionReceived(DateTime, VersionedTransaction), - AccountUpdate(DateTime, Pubkey), -} - -pub enum SimnetCommand { - SlotForward, - SlotBackward, - UpdateClock(ClockCommand), - UpdateRunloopMode(RunloopTriggerMode), -} - -pub enum ClockCommand { - Pause, - Resume, - Toggle, - UpdateSlotInterval(u64), -} - -pub enum ClockEvent { - Tick, - ExpireBlockHash, -} - use std::{fs::File, io::Read, path::PathBuf}; type PluginConstructor = unsafe fn() -> *mut dyn GeyserPlugin; use libloading::{Library, Symbol}; pub async fn start( config: SurfpoolConfig, + subgraph_commands_tx: Sender, simnet_events_tx: Sender, + simnet_commands_tx: Sender, simnet_commands_rx: Receiver, ) -> Result<(), Box> { let mut svm = LiteSVM::new(); @@ -254,10 +219,10 @@ pub async fn start( }; let context = Arc::new(RwLock::new(context)); - let (mempool_tx, mempool_rx) = unbounded(); let middleware = SurfpoolMiddleware { context: context.clone(), - mempool_tx, + simnet_commands_tx: simnet_commands_tx.clone(), + subgraph_commands_tx: subgraph_commands_tx.clone(), config: config.rpc.clone(), }; @@ -270,6 +235,7 @@ pub async fn start( io.extend_with(rpc::accounts_data::SurfpoolAccountsDataRpc.to_delegate()); io.extend_with(rpc::accounts_scan::SurfpoolAccountsScanRpc.to_delegate()); io.extend_with(rpc::bank_data::SurfpoolBankDataRpc.to_delegate()); + io.extend_with(rpc::admin::SurfpoolAdminRpc.to_delegate()); let _handle = hiro_system_kit::thread_named("rpc handler").spawn(move || { let server = ServerBuilder::new(io) @@ -471,19 +437,16 @@ pub async fn start( runloop_trigger_mode = update; continue } - } - }, - Err(_) => {}, - }, - recv(mempool_rx) -> msg => match msg { - Ok((key, transaction, status_tx)) => { - let signature = transaction.signatures[0].clone(); - transactions_to_process.push((key, transaction, status_tx)); - if let Ok(mut ctx) = context.write() { - ctx.transactions.insert( - signature, - EntryStatus::Received, - ); + SimnetCommand::TransactionReceived(key, transaction, status_tx) => { + let signature = transaction.signatures[0].clone(); + transactions_to_process.push((key, transaction, status_tx)); + if let Ok(mut ctx) = context.write() { + ctx.transactions.insert( + signature, + EntryStatus::Received, + ); + } + } } }, Err(_) => {}, @@ -498,7 +461,7 @@ pub async fn start( // Handle the transactions accumulated for (key, transaction, status_tx) in transactions_to_process.drain(..) { - let _ = simnet_events_tx.try_send(SimnetEvent::TransactionReceived( + let _ = simnet_events_tx.try_send(SimnetEvent::TransactionSimulated( Local::now(), transaction.clone(), )); diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 639c062..989f123 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -1,5 +1,13 @@ -use solana_sdk::pubkey::Pubkey; +use chrono::{DateTime, Local}; +use crossbeam_channel::Sender; +use solana_sdk::{ + blake3::Hash, clock::Clock, epoch_info::EpochInfo, pubkey::Pubkey, + transaction::VersionedTransaction, +}; +use solana_transaction_status::TransactionConfirmationStatus; use std::path::PathBuf; +use txtx_addon_network_svm::codec::subgraph::SubgraphRequest; +use uuid::Uuid; #[derive(Clone, Debug, PartialEq, Eq)] pub enum RunloopTriggerMode { @@ -8,10 +16,80 @@ pub enum RunloopTriggerMode { Transaction, } +#[derive(Debug, Clone)] +pub struct Collection { + pub uuid: Uuid, + pub name: String, + pub entries: Vec, +} + +#[derive(Debug, Clone)] +pub struct Entry { + pub uuid: Uuid, + pub value: String, +} + +#[derive(Debug)] +pub enum SubgraphEvent { + EndpointReady, + InfoLog(DateTime, String), + ErrorLog(DateTime, String), + WarnLog(DateTime, String), + DebugLog(DateTime, String), + Shutdown, +} + +pub enum SubgraphCommand { + CreateEndpoint(SubgraphRequest, Sender), + Shutdown, +} + #[derive(Debug)] +pub enum SimnetEvent { + Ready, + Aborted(String), + Shutdown, + ClockUpdate(Clock), + EpochInfoUpdate(EpochInfo), + BlockHashExpired, + InfoLog(DateTime, String), + ErrorLog(DateTime, String), + WarnLog(DateTime, String), + DebugLog(DateTime, String), + PluginLoaded(String), + TransactionSimulated(DateTime, VersionedTransaction), + AccountUpdate(DateTime, Pubkey), +} + +pub enum SimnetCommand { + SlotForward, + SlotBackward, + UpdateClock(ClockCommand), + UpdateRunloopMode(RunloopTriggerMode), + TransactionReceived( + Hash, + VersionedTransaction, + Sender, + ), +} + +pub enum ClockCommand { + Pause, + Resume, + Toggle, + UpdateSlotInterval(u64), +} + +pub enum ClockEvent { + Tick, + ExpireBlockHash, +} + +#[derive(Debug, Clone)] pub struct SurfpoolConfig { pub simnet: SimnetConfig, pub rpc: RpcConfig, + pub subgraph: SubgraphConfig, pub plugin_config_path: Vec, } @@ -24,6 +102,9 @@ pub struct SimnetConfig { pub airdrop_token_amount: u64, } +#[derive(Clone, Debug)] +pub struct SubgraphConfig {} + #[derive(Clone, Debug)] pub struct RpcConfig { pub bind_host: String, diff --git a/crates/gql/Cargo.toml b/crates/gql/Cargo.toml new file mode 100644 index 0000000..4fe76c4 --- /dev/null +++ b/crates/gql/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "surfpool-gql" +description = { workspace = true } +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +keywords = { workspace = true } +categories = { workspace = true } + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +# txtx-addon-kit = { version = "0.2.1" } +futures = { version = "0.3.22", features = ["alloc"], default-features = false } +futures-enum = { version = "0.1.12", default-features = false } +uuid = { version = "1.6.1", features = ["serde", "v7"] } +juniper = { version = "0.16.1" } +juniper_codegen = { version = "0.16.0" } +# juniper_codegen = { git = "https://github.com/graphql-rust/juniper", rev = "c0e1b3e" } +async-stream = "0.3.5" +tokio = "1.37.0" +surfpool-core = { workspace = true } \ No newline at end of file diff --git a/crates/gql/src/lib.rs b/crates/gql/src/lib.rs new file mode 100644 index 0000000..b06e284 --- /dev/null +++ b/crates/gql/src/lib.rs @@ -0,0 +1,37 @@ +use juniper::RootNode; +use mutation::Mutation; +use query::Query; +use std::sync::RwLock; +use std::{collections::BTreeMap, sync::Arc}; +use subscription::Subscription; +use types::{collection::CollectionData, entry::EntryData}; +use uuid::Uuid; + +pub mod mutation; +pub mod query; +pub mod subscription; +pub mod types; + +#[derive(Clone, Debug)] +pub struct Context { + pub collections_store: Arc>>, + pub entries_broadcaster: tokio::sync::broadcast::Sender, +} + +impl Context { + pub fn new() -> Context { + let (entries_broadcaster, _) = tokio::sync::broadcast::channel(128); + Context { + collections_store: Arc::new(RwLock::new(BTreeMap::new())), + entries_broadcaster, + } + } +} + +impl juniper::Context for Context {} + +pub type GqlSchema = RootNode<'static, Query, Mutation, Subscription>; + +pub fn new_graphql_schema() -> GqlSchema { + GqlSchema::new(Query, Mutation, Subscription) +} diff --git a/crates/gql/src/mutation.rs b/crates/gql/src/mutation.rs new file mode 100644 index 0000000..6d20bbc --- /dev/null +++ b/crates/gql/src/mutation.rs @@ -0,0 +1,13 @@ +use crate::Context; +use juniper_codegen::graphql_object; + +pub struct Mutation; + +#[graphql_object( + context = Context, +)] +impl Mutation { + fn api_version() -> &'static str { + "1.0" + } +} diff --git a/crates/gql/src/query.rs b/crates/gql/src/query.rs new file mode 100644 index 0000000..29d869d --- /dev/null +++ b/crates/gql/src/query.rs @@ -0,0 +1,61 @@ +use crate::Context; +use juniper_codegen::graphql_object; + +pub struct Query; + +#[graphql_object( + context = Context, +)] +impl Query { + fn api_version() -> &'static str { + "1.0" + } + + // async fn collections(context: &Context) -> Vec { + // let collections_store = context.collections_store.read().await; + // collections_store + // .values() + // .cloned() + // .filter(|b| if let Panel::ActionPanel(_) = b.panel { true } else { false }) + // .map(GqlActionBlock::new) + // .collect() + // } + + // async fn modal_blocks(context: &Context) -> Vec { + // let block_store = context.block_store.read().await; + // block_store + // .values() + // .cloned() + // .filter(|b| if let Panel::ModalPanel(_) = b.panel { true } else { false }) + // .map(GqlModalBlock::new) + // .collect() + // } + + // async fn error_blocks(context: &Context) -> Vec { + // let block_store = context.block_store.read().await; + // block_store + // .values() + // .cloned() + // .filter(|b| if let Panel::ErrorPanel(_) = b.panel { true } else { false }) + // .map(GqlErrorBlock::new) + // .collect() + // } + + // async fn progress_blocks(context: &Context) -> Vec { + // let block_store = context.block_store.read().await; + // block_store + // .values() + // .cloned() + // .filter(|b| if let Panel::ProgressBar(_) = b.panel { true } else { false }) + // .map(GqlProgressBlock::new) + // .collect() + // } + + // fn runbook(context: &Context) -> RunbookMetadata { + // RunbookMetadata::new( + // &context.runbook_name, + // &context.registered_addons, + // &context.runbook_description, + // ) + // } +} diff --git a/crates/gql/src/subscription.rs b/crates/gql/src/subscription.rs new file mode 100644 index 0000000..a824e0e --- /dev/null +++ b/crates/gql/src/subscription.rs @@ -0,0 +1,27 @@ +use std::pin::Pin; + +use crate::{types::entry::EntryData, Context}; +use futures::Stream; +use juniper::{graphql_subscription, FieldError}; + +pub struct Subscription; + +type GqlEntriesStream = Pin> + Send>>; + +#[graphql_subscription( + context = Context, +)] +impl Subscription { + async fn entries_event(context: &Context) -> GqlEntriesStream { + let entries_tx = context.entries_broadcaster.clone(); + let mut entries_tx = entries_tx.subscribe(); + let stream = async_stream::stream! { + loop { + if let Ok(entry_event) = entries_tx.recv().await { + yield Ok(entry_event) + } + } + }; + Box::pin(stream) + } +} diff --git a/crates/gql/src/types/collection.rs b/crates/gql/src/types/collection.rs new file mode 100644 index 0000000..ddae437 --- /dev/null +++ b/crates/gql/src/types/collection.rs @@ -0,0 +1,36 @@ +use super::entry::EntryData; +use crate::Context; +use juniper_codegen::graphql_object; +use surfpool_core::types::Collection; + +#[derive(Debug, Clone)] +pub struct CollectionData { + pub collection: Collection, +} + +impl CollectionData { + pub fn new(collection: &Collection) -> Self { + Self { + collection: collection.clone(), + } + } +} + +#[graphql_object(context = Context)] +impl CollectionData { + pub fn uuid(&self) -> String { + self.collection.uuid.to_string() + } + + pub fn name(&self) -> String { + self.collection.name.clone() + } + + pub fn entries(&self) -> Vec { + self.collection + .entries + .iter() + .map(|e| EntryData::new(e)) + .collect() + } +} diff --git a/crates/gql/src/types/entry.rs b/crates/gql/src/types/entry.rs new file mode 100644 index 0000000..61a777e --- /dev/null +++ b/crates/gql/src/types/entry.rs @@ -0,0 +1,27 @@ +use crate::Context; +use juniper_codegen::graphql_object; +use surfpool_core::types::Entry; + +#[derive(Debug, Clone)] +pub struct EntryData { + pub entry: Entry, +} + +impl EntryData { + pub fn new(entry: &Entry) -> Self { + Self { + entry: entry.clone(), + } + } +} + +#[graphql_object(context = Context)] +impl EntryData { + pub fn uuid(&self) -> String { + self.entry.uuid.to_string() + } + + pub fn value(&self) -> String { + self.entry.value.clone() + } +} diff --git a/crates/gql/src/types/mod.rs b/crates/gql/src/types/mod.rs new file mode 100644 index 0000000..668b69b --- /dev/null +++ b/crates/gql/src/types/mod.rs @@ -0,0 +1,2 @@ +pub mod collection; +pub mod entry; diff --git a/crates/subgraph/Cargo.toml b/crates/subgraph/Cargo.toml index 5acbaf6..93c3133 100644 --- a/crates/subgraph/Cargo.toml +++ b/crates/subgraph/Cargo.toml @@ -14,3 +14,4 @@ crate-type = ["cdylib", "rlib"] [dependencies] agave-geyser-plugin-interface = { workspace = true } solana-program = { workspace = true } +txtx-addon-network-svm = { workspace = true } diff --git a/crates/subgraph/src/lib.rs b/crates/subgraph/src/lib.rs index 0bb63b3..191a7ab 100644 --- a/crates/subgraph/src/lib.rs +++ b/crates/subgraph/src/lib.rs @@ -1,19 +1,23 @@ use { agave_geyser_plugin_interface::geyser_plugin_interface::{ - GeyserPlugin, GeyserPluginError, ReplicaAccountInfoVersions, ReplicaBlockInfoVersions, ReplicaEntryInfoVersions, ReplicaTransactionInfoVersions, Result as PluginResult, SlotStatus + GeyserPlugin, GeyserPluginError, ReplicaAccountInfoVersions, ReplicaBlockInfoVersions, + ReplicaEntryInfoVersions, ReplicaTransactionInfoVersions, Result as PluginResult, + SlotStatus, }, solana_program::clock::Slot, }; #[derive(Default, Debug)] -pub struct SurfpoolSubgraph {} +pub struct SurfpoolSubgraph { + pub id: String, +} impl GeyserPlugin for SurfpoolSubgraph { fn name(&self) -> &'static str { "surfpool-subgraph" } - fn on_load(&mut self, _config_file: &str, _is_reload: bool) -> PluginResult<()> { + fn on_load(&mut self, config_file: &str, _is_reload: bool) -> PluginResult<()> { Ok(()) } @@ -61,16 +65,15 @@ impl GeyserPlugin for SurfpoolSubgraph { match transaction { ReplicaTransactionInfoVersions::V0_0_2(data) => { if data.is_vote { - return Ok(()) + return Ok(()); } - - let Some(inner_instructions) = data.transaction_status_meta.inner_instructions else { - return Ok(()) + let Some(ref inner_instructions) = data.transaction_status_meta.inner_instructions + else { + return Ok(()); }; - for inner_instructions in inner_instructions.iter() { for instruction in inner_instructions.instructions.iter() { - + println!("{:?}", instruction); } } } From 83ba148f1f555359b92bb22111fb9652c88c28b3 Mon Sep 17 00:00:00 2001 From: MicaiahReid Date: Fri, 14 Feb 2025 09:18:43 -0500 Subject: [PATCH 06/24] fix(cli): get gql server responding --- crates/cli/src/http/mod.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 4db6f53..e841c18 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -3,9 +3,9 @@ use actix_cors::Cors; use actix_web::dev::ServerHandle; use actix_web::http::header::{self}; use actix_web::web::{self, Data}; -use actix_web::Error; +use actix_web::HttpRequest; use actix_web::{middleware, App, HttpResponse, HttpServer}; -use actix_web::{HttpRequest, Responder}; +use actix_web::{Error, Responder}; use crossbeam::channel::{Receiver, Sender}; use juniper_actix::{graphiql_handler, graphql_handler, playground_handler, subscriptions}; use juniper_graphql_ws::ConnectionConfig; @@ -15,7 +15,6 @@ use std::time::Duration; use surfpool_core::types::{Collection, SubgraphCommand, SubgraphEvent, SurfpoolConfig}; use surfpool_gql::types::collection::CollectionData; use surfpool_gql::{new_graphql_schema, Context as GqlContext, GqlSchema}; -use txtx_core::kit::types::frontend::{ClientType, DiscoveryResponse}; use txtx_core::kit::uuid::Uuid; #[cfg(feature = "explorer")] @@ -138,9 +137,12 @@ async fn post_graphql( req: HttpRequest, payload: web::Payload, schema: Data, - context: Data, + context: Data>, ) -> Result { println!("POST /graphql"); + let context = context + .read() + .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; graphql_handler(&schema, &context, req, payload).await } @@ -148,9 +150,12 @@ async fn get_graphql( req: HttpRequest, payload: web::Payload, schema: Data, - context: Data, + context: Data>, ) -> Result { println!("GET /graphql"); + let context = context + .read() + .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; graphql_handler(&schema, &context, req, payload).await } @@ -158,8 +163,11 @@ async fn subscriptions( req: HttpRequest, stream: web::Payload, schema: Data, - context: Data, + context: Data>, ) -> Result { + let context = context + .read() + .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; let ctx = GqlContext { collections_store: context.collections_store.clone(), entries_broadcaster: context.entries_broadcaster.clone(), From 1dcaf6d1e7f078e92a110ffe83c82bc86af7f8a1 Mon Sep 17 00:00:00 2001 From: MicaiahReid Date: Fri, 14 Feb 2025 10:40:55 -0500 Subject: [PATCH 07/24] fix(core): simulate tx before execution (if !config.skip_preflight); return tx errors to user --- crates/core/src/rpc/full.rs | 35 ++++++++++++++++++++++---- crates/core/src/simnet/mod.rs | 46 ++++++++++++++++++++++++++++------- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/crates/core/src/rpc/full.rs b/crates/core/src/rpc/full.rs index 17b141a..ad9a194 100644 --- a/crates/core/src/rpc/full.rs +++ b/crates/core/src/rpc/full.rs @@ -1,15 +1,15 @@ use crate::simnet::EntryStatus; +use crate::types::TransactionStatusEvent; use super::utils::{decode_and_deserialize, transform_tx_metadata_to_ui_accounts}; +use itertools::Itertools; use jsonrpc_core::futures::future::{self, join_all}; use jsonrpc_core::BoxFuture; use jsonrpc_core::{Error, Result}; use jsonrpc_derive::rpc; use solana_account_decoder::{encode_ui_account, UiAccountEncoding}; -use solana_client::nonblocking::rpc_client::RpcClient; use solana_client::rpc_config::RpcContextConfig; use solana_client::rpc_custom_error::RpcCustomError; -use solana_client::rpc_request::RpcRequest; use solana_client::rpc_response::RpcApiVersion; use solana_client::rpc_response::RpcResponseContext; use solana_client::{ @@ -347,19 +347,44 @@ impl Full for SurfpoolFullRpc { ctx.id.clone(), unsanitized_tx, status_update_tx, + config, )); loop { match (status_uptate_rx.recv(), config.preflight_commitment) { + (Ok(TransactionStatusEvent::SimulationFailure(e)), _) => { + return Err(Error { + data: None, + message: format!( + "Transaction simulation failed: {}: {} log messages:\n{}", + e.0.to_string(), + e.1.logs.len(), + e.1.logs.iter().map(|l| l.to_string()).join("\n") + ), + code: jsonrpc_core::ErrorCode::ServerError(-32002), + }) + } + (Ok(TransactionStatusEvent::ExecutionFailure(e)), _) => { + return Err(Error { + data: None, + message: format!( + "Transaction execution failed: {}: {} log messages:\n{}", + e.0.to_string(), + e.1.logs.len(), + e.1.logs.iter().map(|l| l.to_string()).join("\n") + ), + code: jsonrpc_core::ErrorCode::ServerError(-32002), + }) + } ( - Ok(TransactionConfirmationStatus::Processed), + Ok(TransactionStatusEvent::Success(TransactionConfirmationStatus::Processed)), Some(CommitmentLevel::Processed), ) => break, ( - Ok(TransactionConfirmationStatus::Confirmed), + Ok(TransactionStatusEvent::Success(TransactionConfirmationStatus::Confirmed)), None | Some(CommitmentLevel::Confirmed), ) => break, ( - Ok(TransactionConfirmationStatus::Finalized), + Ok(TransactionStatusEvent::Success(TransactionConfirmationStatus::Finalized)), Some(CommitmentLevel::Finalized), ) => break, (Err(_), _) => break, diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index c5013cc..d46069c 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -41,7 +41,7 @@ use crate::{ }, types::{ ClockCommand, ClockEvent, RunloopTriggerMode, SimnetCommand, SimnetEvent, SubgraphCommand, - SubgraphEvent, SurfpoolConfig, + SurfpoolConfig, TransactionStatusEvent, }, }; @@ -437,9 +437,9 @@ pub async fn start( runloop_trigger_mode = update; continue } - SimnetCommand::TransactionReceived(key, transaction, status_tx) => { + SimnetCommand::TransactionReceived(key, transaction, status_tx, config) => { let signature = transaction.signatures[0].clone(); - transactions_to_process.push((key, transaction, status_tx)); + transactions_to_process.push((key, transaction, status_tx, config)); if let Ok(mut ctx) = context.write() { ctx.transactions.insert( signature, @@ -460,7 +460,7 @@ pub async fn start( }; // Handle the transactions accumulated - for (key, transaction, status_tx) in transactions_to_process.drain(..) { + for (key, transaction, status_tx, tx_config) in transactions_to_process.drain(..) { let _ = simnet_events_tx.try_send(SimnetEvent::TransactionSimulated( Local::now(), transaction.clone(), @@ -492,6 +492,25 @@ pub async fn start( } } + if !tx_config.skip_preflight { + let (meta, err) = match ctx.svm.simulate_transaction(transaction.clone()) { + Ok(res) => (res.meta, None), + Err(e) => { + let _ = simnet_events_tx.try_send(SimnetEvent::ErrorLog( + Local::now(), + format!("Transaction simulation failed: {}", e.err.to_string()), + )); + (e.meta, Some(e.err)) + } + }; + + if let Some(e) = &err { + let _ = status_tx + .try_send(TransactionStatusEvent::SimulationFailure((e.clone(), meta))); + continue; + } + } + let (meta, err) = match ctx.svm.send_transaction(transaction.clone()) { Ok(res) => { let _ = plugins_data_tx.try_send((transaction.clone(), res.clone())); @@ -500,7 +519,7 @@ pub async fn start( Err(e) => { let _ = simnet_events_tx.try_send(SimnetEvent::ErrorLog( Local::now(), - format!("Error processing transaction: {}", e.err.to_string()), + format!("Transaction execution failed: {}", e.err.to_string()), )); (e.meta, Some(e.err)) } @@ -512,11 +531,18 @@ pub async fn start( EntryStatus::Processed(TransactionWithStatusMeta( slot, transaction.clone(), - meta, - err, + meta.clone(), + err.clone(), )), ); - let _ = status_tx.try_send(TransactionConfirmationStatus::Processed); + if let Some(e) = &err { + let _ = + status_tx.try_send(TransactionStatusEvent::ExecutionFailure((e.clone(), meta))); + } else { + let _ = status_tx.try_send(TransactionStatusEvent::Success( + TransactionConfirmationStatus::Processed, + )); + } transactions_processed.push((key, transaction, status_tx)); num_transactions += 1; } @@ -526,7 +552,9 @@ pub async fn start( } for (_key, _transaction, status_tx) in transactions_processed.iter() { - let _ = status_tx.try_send(TransactionConfirmationStatus::Confirmed); + let _ = status_tx.try_send(TransactionStatusEvent::Success( + TransactionConfirmationStatus::Confirmed, + )); } ctx.epoch_info.slot_index += 1; From 21b60e81ecad03c359e0f9bef690291293881191 Mon Sep 17 00:00:00 2001 From: MicaiahReid Date: Fri, 14 Feb 2025 14:16:25 -0500 Subject: [PATCH 08/24] fix(svm): add missing types --- crates/core/src/types.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 989f123..44a2edf 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -1,8 +1,13 @@ use chrono::{DateTime, Local}; use crossbeam_channel::Sender; +use litesvm::types::TransactionMetadata; +use solana_client::rpc_config::RpcSendTransactionConfig; use solana_sdk::{ - blake3::Hash, clock::Clock, epoch_info::EpochInfo, pubkey::Pubkey, - transaction::VersionedTransaction, + blake3::Hash, + clock::Clock, + epoch_info::EpochInfo, + pubkey::Pubkey, + transaction::{TransactionError, VersionedTransaction}, }; use solana_transaction_status::TransactionConfirmationStatus; use std::path::PathBuf; @@ -61,6 +66,12 @@ pub enum SimnetEvent { AccountUpdate(DateTime, Pubkey), } +pub enum TransactionStatusEvent { + Success(TransactionConfirmationStatus), + SimulationFailure((TransactionError, TransactionMetadata)), + ExecutionFailure((TransactionError, TransactionMetadata)), +} + pub enum SimnetCommand { SlotForward, SlotBackward, @@ -69,7 +80,8 @@ pub enum SimnetCommand { TransactionReceived( Hash, VersionedTransaction, - Sender, + Sender, + RpcSendTransactionConfig, ), } From 6ba8547ebff88169c113eb596cdb150d8aa37a1c Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 14 Feb 2025 15:15:29 -0500 Subject: [PATCH 09/24] feat: subgraph runloop --- Cargo.lock | 74 +++++++++++++++++++++++++++++++- Cargo.toml | 6 +++ crates/cli/Cargo.toml | 2 + crates/cli/src/cli/simnet/mod.rs | 24 +++++------ crates/cli/src/http/mod.rs | 51 ++++++++++++---------- crates/core/Cargo.toml | 11 ++--- crates/core/src/lib.rs | 2 +- crates/core/src/rpc/admin.rs | 8 +++- crates/core/src/rpc/mod.rs | 11 ++--- crates/core/src/simnet/mod.rs | 65 ++++++++++++++++++++-------- crates/core/src/types.rs | 18 +++++++- crates/subgraph/Cargo.toml | 5 +++ crates/subgraph/src/lib.rs | 11 ++++- 13 files changed, 218 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1820ea..77c9a7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3147,7 +3147,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -3461,6 +3461,25 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "ipc-channel" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8251fb7bcd9ccd3725ed8deae9fe7db8e586495c9eb5b0c52e6233e5e75ea" +dependencies = [ + "bincode", + "crossbeam-channel", + "fnv", + "lazy_static", + "libc", + "mio", + "rand 0.8.5", + "serde", + "tempfile", + "uuid", + "windows", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -9207,6 +9226,7 @@ dependencies = [ "ansi_term", "anyhow", "atty", + "bincode", "chrono", "clap 4.5.27", "clap_complete 4.5.44", @@ -9217,6 +9237,7 @@ dependencies = [ "dialoguer 0.11.0", "dirs 6.0.0", "hiro-system-kit", + "ipc-channel", "juniper_actix", "juniper_graphql_ws", "mime_guess", @@ -9248,6 +9269,7 @@ dependencies = [ "crossbeam-channel", "hiro-system-kit", "indexmap 2.7.1", + "ipc-channel", "itertools 0.14.0", "json5", "jsonrpc-core", @@ -9321,7 +9343,12 @@ name = "surfpool-subgraph" version = "0.1.7" dependencies = [ "agave-geyser-plugin-interface", + "bincode", + "ipc-channel", + "serde", + "serde_json", "solana-program", + "surfpool-core", "txtx-addon-network-svm", ] @@ -10550,6 +10577,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -10559,6 +10596,41 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "windows-registry" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index bfa6181..50749f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,13 @@ solana-svm = "2.1.10" solana-program-runtime = "2.1.10" solana-geyser-plugin-manager = "2.1.10" solana-streamer = "2.1.10" +ipc-channel = "0.19.0" +serde = "1.0.217" +serde_derive = "1.0.217" # must match the serde version, see https://github.com/serde-rs/serde/issues/2584#issuecomment-1685252251 +serde_json = "1.0.135" txtx-core = { path = "../txtx/crates/txtx-core" } txtx-addon-network-svm = { package = "txtx-addon-network-svm", path = "../txtx/addons/svm" } +bincode = "1.3.3" + # txtx-core = { version = "0.2.2" } # txtx-addon-network-svm = { version = "0.1.3" } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d9b6305..5183720 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -46,6 +46,8 @@ mime_guess = "2.0.4" notify = { version = "8.0.0" } juniper_actix = {version = "0.5.0", features = ["subscriptions"] } juniper_graphql_ws = { version = "0.4.0", features = ["graphql-transport-ws"] } +ipc-channel = { workspace = true } +bincode = { workspace = true } [features] default = ["cli"] diff --git a/crates/cli/src/cli/simnet/mod.rs b/crates/cli/src/cli/simnet/mod.rs index 64bd8b0..8f60536 100644 --- a/crates/cli/src/cli/simnet/mod.rs +++ b/crates/cli/src/cli/simnet/mod.rs @@ -101,6 +101,17 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re let (subgraph_commands_tx, subgraph_commands_rx) = crossbeam::channel::unbounded(); let (subgraph_events_tx, subgraph_events_rx) = crossbeam::channel::unbounded(); + let network_binding = format!("{}:{}", cmd.network_host, DEFAULT_EXPLORER_PORT); + let explorer_handle = start_server( + network_binding, + config.clone(), + subgraph_events_tx.clone(), + subgraph_commands_rx, + &ctx.clone(), + ) + .await + .map_err(|e| format!("{}", e.to_string()))?; + let ctx_copy = ctx.clone(); let simnet_commands_tx_copy = simnet_commands_tx.clone(); let config_copy = config.clone(); @@ -122,8 +133,6 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re }) .map_err(|e| format!("{}", e))?; - let ctx_copy = ctx.clone(); - let subgraph_events_tx_copy = subgraph_events_tx.clone(); loop { match simnet_events_rx.recv() { Ok(SimnetEvent::Aborted(error)) => return Err(error), @@ -133,17 +142,6 @@ pub async fn handle_start_simnet_command(cmd: &StartSimnet, ctx: &Context) -> Re } } - let network_binding = format!("{}:{}", cmd.network_host, DEFAULT_EXPLORER_PORT); - let explorer_handle = start_server( - network_binding, - config, - subgraph_events_tx_copy, - subgraph_commands_rx, - &ctx_copy, - ) - .await - .map_err(|e| format!("{}", e.to_string()))?; - let mut deploy_progress_rx = vec![]; if !cmd.no_deploy { // Are we in a project directory? diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index e841c18..21f32ae 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -7,6 +7,7 @@ use actix_web::HttpRequest; use actix_web::{middleware, App, HttpResponse, HttpServer}; use actix_web::{Error, Responder}; use crossbeam::channel::{Receiver, Sender}; +use ipc_channel::ipc::IpcReceiver; use juniper_actix::{graphiql_handler, graphql_handler, playground_handler, subscriptions}; use juniper_graphql_ws::ConnectionConfig; use std::error::Error as StdError; @@ -38,29 +39,35 @@ pub async fn start_server( let gql_context_copy = gql_context.clone(); let _handle = hiro_system_kit::thread_named("subgraph") .spawn(move || { - while let Ok(command) = subgraph_commands_rx.recv() { - match command { - SubgraphCommand::CreateEndpoint(config, sender) => { - println!("{:?}", config); - let gql_context = gql_context_copy.write().unwrap(); - let mut collections = gql_context.collections_store.write().unwrap(); - let collection_uuid = Uuid::new_v4(); - collections.insert( - collection_uuid.clone(), - CollectionData { - collection: Collection { - uuid: collection_uuid, - name: config.subgraph_name.clone(), - entries: vec![], - }, - }, - ); - println!("{:?}", collections); - let _ = sender.send("http://127.0.0.1:8900/graphql".into()); - } - SubgraphCommand::Shutdown => { - let _ = subgraph_events_tx.send(SubgraphEvent::Shutdown); + loop { + match subgraph_commands_rx.recv() { + Err(_e) => { + // todo } + Ok(cmd) => match cmd { + SubgraphCommand::CreateEndpoint(config, sender) => { + println!("Here: {:?}", config); + let gql_context = gql_context_copy.write().unwrap(); + let mut collections = gql_context.collections_store.write().unwrap(); + let collection_uuid = Uuid::new_v4(); + + collections.insert( + collection_uuid.clone(), + CollectionData { + collection: Collection { + uuid: collection_uuid, + name: config.subgraph_name.clone(), + entries: vec![], + }, + }, + ); + println!("{:?}", collections); + let _ = sender.send("http://127.0.0.1:8900/graphql".into()); + } + SubgraphCommand::Shutdown => { + let _ = subgraph_events_tx.send(SubgraphEvent::Shutdown); + } + }, } } Ok::<(), String>(()) diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 93cc378..7ace7f2 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -15,12 +15,12 @@ path = "src/lib.rs" [dependencies] base64 = "0.22.1" -bincode = "1.3.3" +bincode = { workspace = true } crossbeam-channel = "0.5.14" log = "0.4.22" -serde = "1.0.217" -serde_derive = "1.0.217" # must match the serde version, see https://github.com/serde-rs/serde/issues/2584#issuecomment-1685252251 -serde_json = "1.0.135" +serde = { workspace = true } +serde_derive = { workspace = true } # must match the serde version, see https://github.com/serde-rs/serde/issues/2584#issuecomment-1685252251 +serde_json = { workspace = true } itertools = "0.14.0" symlink = "0.1.0" tokio = "1.43.0" @@ -75,4 +75,5 @@ zstd = "0.13.2" libloading = "0.7.4" json5 = "0.4.1" txtx-addon-network-svm = { workspace = true } -uuid = "1.7.0" \ No newline at end of file +uuid = "1.7.0" +ipc-channel = { workspace = true } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index c62abab..078141a 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -20,7 +20,7 @@ pub use jsonrpc_http_server; pub use litesvm; pub use solana_rpc_client; pub use solana_sdk; -use types::{SimnetCommand, SimnetEvent, SubgraphCommand, SubgraphEvent, SurfpoolConfig}; +use types::{SimnetCommand, SimnetEvent, SubgraphCommand, SurfpoolConfig}; pub async fn start_simnet( config: SurfpoolConfig, diff --git a/crates/core/src/rpc/admin.rs b/crates/core/src/rpc/admin.rs index 8785b0b..0387129 100644 --- a/crates/core/src/rpc/admin.rs +++ b/crates/core/src/rpc/admin.rs @@ -1,8 +1,10 @@ use std::collections::HashMap; use std::net::SocketAddr; +use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; +use crossbeam_channel::unbounded; use jsonrpc_core::BoxFuture; use jsonrpc_core::Result; use jsonrpc_derive::rpc; @@ -11,7 +13,9 @@ use solana_sdk::pubkey::Pubkey; use txtx_addon_network_svm::codec::subgraph::PluginConfig; use txtx_addon_network_svm::codec::subgraph::SubgraphRequest; +use crate::types::PluginManagerCommand; use crate::types::SubgraphCommand; +use crate::types::SubgraphPluginConfig; use super::RunloopContext; @@ -146,8 +150,8 @@ impl AdminRpc for SurfpoolAdminRpc { let ctx = meta.unwrap(); let (tx, rx) = crossbeam_channel::bounded(1); let _ = ctx - .subgraph_commands_tx - .send(SubgraphCommand::CreateEndpoint(config.data.clone(), tx)); + .plugin_manager_commands_tx + .send(PluginManagerCommand::LoadConfig(config, tx)); let Ok(endpoint_url) = rx.recv_timeout(Duration::from_secs(10)) else { unimplemented!() }; diff --git a/crates/core/src/rpc/mod.rs b/crates/core/src/rpc/mod.rs index f735a41..a2c12b3 100644 --- a/crates/core/src/rpc/mod.rs +++ b/crates/core/src/rpc/mod.rs @@ -1,6 +1,7 @@ -use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; use crossbeam_channel::Sender; +use ipc_channel::ipc::IpcSender; use jsonrpc_core::{ futures::future::Either, middleware, FutureResponse, Metadata, Middleware, Request, Response, }; @@ -30,7 +31,7 @@ pub struct RunloopContext { pub id: Hash, pub state: Arc>, pub simnet_commands_tx: Sender, - pub subgraph_commands_tx: Sender, + pub plugin_manager_commands_tx: Sender, } trait State { @@ -76,7 +77,7 @@ impl Metadata for RunloopContext {} use crate::{ simnet::GlobalState, - types::{RpcConfig, SimnetCommand, SubgraphCommand}, + types::{PluginManagerCommand, RpcConfig, SimnetCommand, SubgraphCommand}, }; use jsonrpc_core::futures::FutureExt; use std::future::Future; @@ -85,7 +86,7 @@ use std::future::Future; pub struct SurfpoolMiddleware { pub context: Arc>, pub simnet_commands_tx: Sender, - pub subgraph_commands_tx: Sender, + pub plugin_manager_commands_tx: Sender, pub config: RpcConfig, } @@ -107,7 +108,7 @@ impl Middleware> for SurfpoolMiddleware { id: Hash::new_unique(), state: self.context.clone(), simnet_commands_tx: self.simnet_commands_tx.clone(), - subgraph_commands_tx: self.subgraph_commands_tx.clone(), + plugin_manager_commands_tx: self.plugin_manager_commands_tx.clone(), }); // println!("Processing request {:?}", request); Either::Left(Box::pin(next(request, meta).map(move |res| { diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index c5013cc..94a6674 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -5,9 +5,14 @@ use base64::prelude::{Engine, BASE64_STANDARD}; use chrono::{Local, Utc}; use crossbeam::select; use crossbeam_channel::{unbounded, Receiver, Sender}; +use ipc_channel::{ + ipc::{IpcOneShotServer, IpcReceiver, IpcSender}, + router::RouterProxy, +}; use jsonrpc_core::MetaIoHandler; use jsonrpc_http_server::{DomainsValidation, ServerBuilder}; use litesvm::{types::TransactionMetadata, LiteSVM}; +use serde::Serialize; use solana_client::{nonblocking::rpc_client::RpcClient, rpc_response::RpcPerfSample}; use solana_geyser_plugin_manager::geyser_plugin_manager::{ GeyserPluginManager, GeyserPluginManagerError, LoadedGeyserPlugin, @@ -40,8 +45,8 @@ use crate::{ bank_data::BankData, full::Full, minimal::Minimal, SurfpoolMiddleware, }, types::{ - ClockCommand, ClockEvent, RunloopTriggerMode, SimnetCommand, SimnetEvent, SubgraphCommand, - SubgraphEvent, SurfpoolConfig, + ClockCommand, ClockEvent, PluginManagerCommand, RunloopTriggerMode, SimnetCommand, + SimnetEvent, SubgraphCommand, SubgraphEvent, SubgraphPluginConfig, SurfpoolConfig, }, }; @@ -218,11 +223,13 @@ pub async fn start( rpc_client: rpc_client.clone(), }; + let (plugin_manager_commands_tx, plugin_manager_commands_rx) = unbounded(); + let context = Arc::new(RwLock::new(context)); let middleware = SurfpoolMiddleware { context: context.clone(), simnet_commands_tx: simnet_commands_tx.clone(), - subgraph_commands_tx: subgraph_commands_tx.clone(), + plugin_manager_commands_tx, config: config.rpc.clone(), }; @@ -252,6 +259,7 @@ pub async fn start( let simnet_config = config.simnet.clone(); let simnet_events_tx_copy = simnet_events_tx.clone(); let (plugins_data_tx, plugins_data_rx) = unbounded::<(Transaction, TransactionMetadata)>(); + if !config.plugin_config_path.is_empty() { let _handle = hiro_system_kit::thread_named("geyser plugins handler").spawn(move || { let mut plugin_manager = GeyserPluginManager::new(); @@ -301,23 +309,42 @@ pub async fn start( .to_str() .ok_or(GeyserPluginManagerError::InvalidPluginPath)?; - let (mut plugin, lib) = unsafe { - let lib = match Library::new(libpath) { - Ok(lib) => lib, - Err(e) => { - let _ = simnet_events_tx_copy.send(SimnetEvent::ErrorLog(Local::now(), format!("Unable to load plugin {}: {}", plugin_name, e.to_string()))); - continue; + while let Ok(command) = plugin_manager_commands_rx.recv() { + match command { + PluginManagerCommand::LoadConfig(config, notifier) => { + let _ = subgraph_commands_tx.send(SubgraphCommand::CreateEndpoint(config.data.clone(), notifier)); + + let (mut plugin, lib) = unsafe { + let lib = match Library::new(&libpath) { + Ok(lib) => lib, + Err(e) => { + let _ = simnet_events_tx_copy.send(SimnetEvent::ErrorLog(Local::now(), format!("Unable to load plugin {}: {}", plugin_name, e.to_string()))); + continue; + } + }; + let constructor: Symbol = lib + .get(b"_create_plugin") + .map_err(|e| GeyserPluginManagerError::PluginLoadError(e.to_string()))?; + let plugin_raw = constructor(); + (Box::from_raw(plugin_raw), lib) + }; + + let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); + let subgraph_plugin_config = SubgraphPluginConfig { + ipc_token, + subgraph_request: Some(config.data.clone()) + }; + let config_file = serde_json::to_string(&subgraph_plugin_config).unwrap(); + let _res = plugin.on_load(&config_file, false); + if let Ok(rx) = server.accept() { + println!("Game on: {:?}", rx); + }; + + plugin_manager.plugins.push(LoadedGeyserPlugin::new(lib, plugin, Some(plugin_name.clone()))); + let _ = simnet_events_tx_copy.send(SimnetEvent::PluginLoaded("surfpool-subgraph".into())); } - }; - let constructor: Symbol = lib - .get(b"_create_plugin") - .map_err(|e| GeyserPluginManagerError::PluginLoadError(e.to_string()))?; - let plugin_raw = constructor(); - (Box::from_raw(plugin_raw), lib) - }; - let _res = plugin.on_load("", false); - plugin_manager.plugins.push(LoadedGeyserPlugin::new(lib, plugin, Some(plugin_name.clone()))); - let _ = simnet_events_tx_copy.send(SimnetEvent::PluginLoaded(plugin_name)); + } + } } while let Ok((transaction, transaction_metadata)) = plugins_data_rx.recv() { let transaction_status_meta = TransactionStatusMeta { diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 989f123..393a616 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -6,7 +6,7 @@ use solana_sdk::{ }; use solana_transaction_status::TransactionConfirmationStatus; use std::path::PathBuf; -use txtx_addon_network_svm::codec::subgraph::SubgraphRequest; +use txtx_addon_network_svm::codec::subgraph::{PluginConfig, SubgraphRequest}; use uuid::Uuid; #[derive(Clone, Debug, PartialEq, Eq)] @@ -39,6 +39,12 @@ pub enum SubgraphEvent { Shutdown, } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub enum SubgraphIndexingEvent { + Entry(String), +} + +#[derive(Debug, Clone)] pub enum SubgraphCommand { CreateEndpoint(SubgraphRequest, Sender), Shutdown, @@ -73,6 +79,10 @@ pub enum SimnetCommand { ), } +pub enum PluginManagerCommand { + LoadConfig(PluginConfig, Sender), +} + pub enum ClockCommand { Pause, Resume, @@ -117,3 +127,9 @@ impl RpcConfig { format!("{}:{}", self.bind_host, self.bind_port) } } + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct SubgraphPluginConfig { + pub ipc_token: String, + pub subgraph_request: Option, +} diff --git a/crates/subgraph/Cargo.toml b/crates/subgraph/Cargo.toml index 93c3133..fea9c42 100644 --- a/crates/subgraph/Cargo.toml +++ b/crates/subgraph/Cargo.toml @@ -15,3 +15,8 @@ crate-type = ["cdylib", "rlib"] agave-geyser-plugin-interface = { workspace = true } solana-program = { workspace = true } txtx-addon-network-svm = { workspace = true } +ipc-channel = { workspace = true } +surfpool-core = { workspace = true } +serde_json = { workspace = true } +serde = { workspace = true } +bincode = { workspace = true } diff --git a/crates/subgraph/src/lib.rs b/crates/subgraph/src/lib.rs index 191a7ab..5a8184a 100644 --- a/crates/subgraph/src/lib.rs +++ b/crates/subgraph/src/lib.rs @@ -1,15 +1,19 @@ use { agave_geyser_plugin_interface::geyser_plugin_interface::{ - GeyserPlugin, GeyserPluginError, ReplicaAccountInfoVersions, ReplicaBlockInfoVersions, + GeyserPlugin, ReplicaAccountInfoVersions, ReplicaBlockInfoVersions, ReplicaEntryInfoVersions, ReplicaTransactionInfoVersions, Result as PluginResult, SlotStatus, }, + ipc_channel::ipc::IpcSender, solana_program::clock::Slot, + std::sync::Mutex, + surfpool_core::types::{SubgraphIndexingEvent, SubgraphPluginConfig}, }; #[derive(Default, Debug)] pub struct SurfpoolSubgraph { pub id: String, + subgraph_indexing_event_tx: Mutex>>, } impl GeyserPlugin for SurfpoolSubgraph { @@ -18,6 +22,11 @@ impl GeyserPlugin for SurfpoolSubgraph { } fn on_load(&mut self, config_file: &str, _is_reload: bool) -> PluginResult<()> { + let config = serde_json::from_str::(&config_file).unwrap(); + let oneshot_tx = IpcSender::connect(config.ipc_token).unwrap(); + let (tx, rx) = ipc_channel::ipc::channel().unwrap(); + let _ = oneshot_tx.send(rx); + self.subgraph_indexing_event_tx = Mutex::new(Some(tx)); Ok(()) } From 11a5be983452a04e904a69e23e57857cc7b27538 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 14 Feb 2025 15:43:11 -0500 Subject: [PATCH 10/24] feat: ability to observe a subgraph --- crates/cli/src/http/mod.rs | 71 ++++++++++++++++++++++------------- crates/core/src/rpc/admin.rs | 17 +++------ crates/core/src/rpc/mod.rs | 8 ++-- crates/core/src/simnet/mod.rs | 15 +++++--- crates/core/src/types.rs | 5 ++- 5 files changed, 66 insertions(+), 50 deletions(-) diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 21f32ae..b43d444 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -6,8 +6,7 @@ use actix_web::web::{self, Data}; use actix_web::HttpRequest; use actix_web::{middleware, App, HttpResponse, HttpServer}; use actix_web::{Error, Responder}; -use crossbeam::channel::{Receiver, Sender}; -use ipc_channel::ipc::IpcReceiver; +use crossbeam::channel::{Receiver, Select, Sender}; use juniper_actix::{graphiql_handler, graphql_handler, playground_handler, subscriptions}; use juniper_graphql_ws::ConnectionConfig; use std::error::Error as StdError; @@ -39,34 +38,54 @@ pub async fn start_server( let gql_context_copy = gql_context.clone(); let _handle = hiro_system_kit::thread_named("subgraph") .spawn(move || { + let mut observers = vec![]; loop { - match subgraph_commands_rx.recv() { - Err(_e) => { - // todo - } - Ok(cmd) => match cmd { - SubgraphCommand::CreateEndpoint(config, sender) => { - println!("Here: {:?}", config); - let gql_context = gql_context_copy.write().unwrap(); - let mut collections = gql_context.collections_store.write().unwrap(); - let collection_uuid = Uuid::new_v4(); + let mut selector = Select::new(); + let mut handles = vec![]; + selector.recv(&subgraph_commands_rx); + for rx in observers.iter() { + handles.push(selector.recv(rx)); + } + let oper = selector.select(); + match oper.index() { + 0 => match oper.recv(&subgraph_commands_rx) { + Err(_e) => { + // todo + } + Ok(cmd) => match cmd { + SubgraphCommand::CreateSubgraph(config, sender) => { + let gql_context = gql_context_copy.write().unwrap(); + let mut collections = + gql_context.collections_store.write().unwrap(); + let collection_uuid = Uuid::new_v4(); - collections.insert( - collection_uuid.clone(), - CollectionData { - collection: Collection { - uuid: collection_uuid, - name: config.subgraph_name.clone(), - entries: vec![], + collections.insert( + collection_uuid.clone(), + CollectionData { + collection: Collection { + uuid: collection_uuid, + name: config.subgraph_name.clone(), + entries: vec![], + }, }, - }, - ); - println!("{:?}", collections); - let _ = sender.send("http://127.0.0.1:8900/graphql".into()); - } - SubgraphCommand::Shutdown => { - let _ = subgraph_events_tx.send(SubgraphEvent::Shutdown); + ); + println!("{:?}", collections); + let _ = sender.send("http://127.0.0.1:8900/graphql".into()); + } + SubgraphCommand::ObserveSubgraph(subgraph_observer_rx) => { + println!("Observing new graph"); + observers.push(subgraph_observer_rx); + } + SubgraphCommand::Shutdown => { + let _ = subgraph_events_tx.send(SubgraphEvent::Shutdown); + } + }, + }, + i => match oper.recv(&observers[i - 1]) { + Ok(value) => { + println!("Inject {} in store", value); } + Err(_e) => {} }, } } diff --git a/crates/core/src/rpc/admin.rs b/crates/core/src/rpc/admin.rs index 0387129..42efdcd 100644 --- a/crates/core/src/rpc/admin.rs +++ b/crates/core/src/rpc/admin.rs @@ -1,23 +1,16 @@ -use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; -use std::time::SystemTime; - -use crossbeam_channel::unbounded; use jsonrpc_core::BoxFuture; use jsonrpc_core::Result; use jsonrpc_derive::rpc; use solana_client::rpc_config::RpcAccountIndex; use solana_sdk::pubkey::Pubkey; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::time::Duration; +use std::time::SystemTime; use txtx_addon_network_svm::codec::subgraph::PluginConfig; -use txtx_addon_network_svm::codec::subgraph::SubgraphRequest; - -use crate::types::PluginManagerCommand; -use crate::types::SubgraphCommand; -use crate::types::SubgraphPluginConfig; use super::RunloopContext; +use crate::types::PluginManagerCommand; #[rpc] pub trait AdminRpc { diff --git a/crates/core/src/rpc/mod.rs b/crates/core/src/rpc/mod.rs index a2c12b3..97cc422 100644 --- a/crates/core/src/rpc/mod.rs +++ b/crates/core/src/rpc/mod.rs @@ -1,13 +1,11 @@ -use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; use crossbeam_channel::Sender; -use ipc_channel::ipc::IpcSender; use jsonrpc_core::{ futures::future::Either, middleware, FutureResponse, Metadata, Middleware, Request, Response, }; use solana_client::rpc_custom_error::RpcCustomError; -use solana_sdk::{blake3::Hash, clock::Slot, transaction::VersionedTransaction}; -use solana_transaction_status::TransactionConfirmationStatus; +use solana_sdk::{blake3::Hash, clock::Slot}; pub mod accounts_data; pub mod accounts_scan; @@ -77,7 +75,7 @@ impl Metadata for RunloopContext {} use crate::{ simnet::GlobalState, - types::{PluginManagerCommand, RpcConfig, SimnetCommand, SubgraphCommand}, + types::{PluginManagerCommand, RpcConfig, SimnetCommand}, }; use jsonrpc_core::futures::FutureExt; use std::future::Future; diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index c483ffd..f437342 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -5,7 +5,10 @@ use base64::prelude::{Engine, BASE64_STANDARD}; use chrono::{Local, Utc}; use crossbeam::select; use crossbeam_channel::{unbounded, Receiver, Sender}; -use ipc_channel::ipc::{IpcOneShotServer, IpcReceiver}; +use ipc_channel::{ + ipc::{IpcOneShotServer, IpcReceiver}, + router::RouterProxy, +}; use jsonrpc_core::MetaIoHandler; use jsonrpc_http_server::{DomainsValidation, ServerBuilder}; use litesvm::{types::TransactionMetadata, LiteSVM}; @@ -305,10 +308,12 @@ pub async fn start( .to_str() .ok_or(GeyserPluginManagerError::InvalidPluginPath)?; + let ipc_router = RouterProxy::new(); + while let Ok(command) = plugin_manager_commands_rx.recv() { match command { PluginManagerCommand::LoadConfig(config, notifier) => { - let _ = subgraph_commands_tx.send(SubgraphCommand::CreateEndpoint(config.data.clone(), notifier)); + let _ = subgraph_commands_tx.send(SubgraphCommand::CreateSubgraph(config.data.clone(), notifier)); let (mut plugin, lib) = unsafe { let lib = match Library::new(&libpath) { @@ -332,10 +337,10 @@ pub async fn start( }; let config_file = serde_json::to_string(&subgraph_plugin_config).unwrap(); let _res = plugin.on_load(&config_file, false); - if let Ok(rx) = server.accept() { - println!("Game on: {:?}", rx); + if let Ok((_, rx)) = server.accept() { + let subgraph_rx = ipc_router.route_ipc_receiver_to_new_crossbeam_receiver(rx); + let _ = subgraph_commands_tx.send(SubgraphCommand::ObserveSubgraph(subgraph_rx)); }; - plugin_manager.plugins.push(LoadedGeyserPlugin::new(lib, plugin, Some(plugin_name.clone()))); let _ = simnet_events_tx_copy.send(SimnetEvent::PluginLoaded("surfpool-subgraph".into())); } diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index cfc24cd..e168912 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -1,5 +1,5 @@ use chrono::{DateTime, Local}; -use crossbeam_channel::Sender; +use crossbeam_channel::{Receiver, Sender}; use litesvm::types::TransactionMetadata; use solana_client::rpc_config::RpcSendTransactionConfig; use solana_sdk::{ @@ -51,7 +51,8 @@ pub enum SubgraphIndexingEvent { #[derive(Debug, Clone)] pub enum SubgraphCommand { - CreateEndpoint(SubgraphRequest, Sender), + CreateSubgraph(SubgraphRequest, Sender), + ObserveSubgraph(Receiver), Shutdown, } From 6bf839382820f002f5b82abfac8a76d64c60ab9f Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 14 Feb 2025 16:30:01 -0500 Subject: [PATCH 11/24] fix: runloop --- crates/core/src/simnet/mod.rs | 211 ++++++++++++++++++---------------- crates/subgraph/src/lib.rs | 5 + 2 files changed, 118 insertions(+), 98 deletions(-) diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index f437342..5894c11 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -263,124 +263,139 @@ pub async fn start( let _handle = hiro_system_kit::thread_named("geyser plugins handler").spawn(move || { let mut plugin_manager = GeyserPluginManager::new(); - for (i, geyser_plugin_config_file) in config.plugin_config_path.iter().enumerate() { - let mut file = match File::open(geyser_plugin_config_file) { - Ok(file) => file, - Err(err) => { - return Err(GeyserPluginManagerError::CannotOpenConfigFile(format!( - "Failed to open the plugin config file {geyser_plugin_config_file:?}, error: {err:?}" - ))); - } - }; - let mut contents = String::new(); - if let Err(err) = file.read_to_string(&mut contents) { - return Err(GeyserPluginManagerError::CannotReadConfigFile(format!( - "Failed to read the plugin config file {geyser_plugin_config_file:?}, error: {err:?}" - ))); - } - - let result: serde_json::Value = match json5::from_str(&contents) { - Ok(value) => value, - Err(err) => { - return Err(GeyserPluginManagerError::InvalidConfigFileFormat(format!( - "The config file {geyser_plugin_config_file:?} is not in a valid Json5 format, error: {err:?}" - ))); - } - }; + // for (i, geyser_plugin_config_file) in config.plugin_config_path.iter().enumerate() { + // let mut file = match File::open(geyser_plugin_config_file) { + // Ok(file) => file, + // Err(err) => { + // return Err(GeyserPluginManagerError::CannotOpenConfigFile(format!( + // "Failed to open the plugin config file {geyser_plugin_config_file:?}, error: {err:?}" + // ))); + // } + // }; + // let mut contents = String::new(); + // if let Err(err) = file.read_to_string(&mut contents) { + // return Err(GeyserPluginManagerError::CannotReadConfigFile(format!( + // "Failed to read the plugin config file {geyser_plugin_config_file:?}, error: {err:?}" + // ))); + // } + // let result: serde_json::Value = match json5::from_str(&contents) { + // Ok(value) => value, + // Err(err) => { + // return Err(GeyserPluginManagerError::InvalidConfigFileFormat(format!( + // "The config file {geyser_plugin_config_file:?} is not in a valid Json5 format, error: {err:?}" + // ))); + // } + // }; + + // let _config_file = geyser_plugin_config_file + // .as_os_str() + // .to_str() + // .ok_or(GeyserPluginManagerError::InvalidPluginPath)?; + + + let geyser_plugin_config_file = PathBuf::from("../../surfpool_subgraph_plugin.json"); + + let contents = "{\"name\": \"surfpool-subgraph\", \"libpath\": \"target/release/libsurfpool_subgraph.dylib\"}"; + let result: serde_json::Value = json5::from_str(&contents).unwrap(); let libpath = result["libpath"] .as_str() - .ok_or(GeyserPluginManagerError::LibPathNotSet)?; + .unwrap(); let mut libpath = PathBuf::from(libpath); if libpath.is_relative() { let config_dir = geyser_plugin_config_file.parent().ok_or_else(|| { GeyserPluginManagerError::CannotOpenConfigFile(format!( "Failed to resolve parent of {geyser_plugin_config_file:?}", )) - })?; + }).unwrap(); libpath = config_dir.join(libpath); } - let plugin_name = result["name"].as_str().map(|s| s.to_owned()).unwrap_or(format!("plugin-{}", i)); - - let _config_file = geyser_plugin_config_file - .as_os_str() - .to_str() - .ok_or(GeyserPluginManagerError::InvalidPluginPath)?; + let plugin_name = result["name"].as_str().map(|s| s.to_owned()).unwrap_or(format!("surfpool-subgraph")); let ipc_router = RouterProxy::new(); - while let Ok(command) = plugin_manager_commands_rx.recv() { - match command { - PluginManagerCommand::LoadConfig(config, notifier) => { - let _ = subgraph_commands_tx.send(SubgraphCommand::CreateSubgraph(config.data.clone(), notifier)); - - let (mut plugin, lib) = unsafe { - let lib = match Library::new(&libpath) { - Ok(lib) => lib, - Err(e) => { - let _ = simnet_events_tx_copy.send(SimnetEvent::ErrorLog(Local::now(), format!("Unable to load plugin {}: {}", plugin_name, e.to_string()))); - continue; + loop { + select! { + recv(plugin_manager_commands_rx) -> msg => match msg { + Ok(event) => { + match event { + PluginManagerCommand::LoadConfig(config, notifier) => { + let _ = subgraph_commands_tx.send(SubgraphCommand::CreateSubgraph(config.data.clone(), notifier)); + + let (mut plugin, lib) = unsafe { + let lib = match Library::new(&libpath) { + Ok(lib) => lib, + Err(e) => { + let _ = simnet_events_tx_copy.send(SimnetEvent::ErrorLog(Local::now(), format!("Unable to load plugin {}: {}", plugin_name, e.to_string()))); + continue; + } + }; + let constructor: Symbol = lib + .get(b"_create_plugin") + .map_err(|e| format!("{}", e.to_string()))?; + let plugin_raw = constructor(); + (Box::from_raw(plugin_raw), lib) + }; + + let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); + let subgraph_plugin_config = SubgraphPluginConfig { + ipc_token, + subgraph_request: Some(config.data.clone()) + }; + let config_file = serde_json::to_string(&subgraph_plugin_config).unwrap(); + let _res = plugin.on_load(&config_file, false); + if let Ok((_, rx)) = server.accept() { + let subgraph_rx = ipc_router.route_ipc_receiver_to_new_crossbeam_receiver(rx); + let _ = subgraph_commands_tx.send(SubgraphCommand::ObserveSubgraph(subgraph_rx)); + }; + plugin_manager.plugins.push(LoadedGeyserPlugin::new(lib, plugin, Some(plugin_name.clone()))); + let _ = simnet_events_tx_copy.send(SimnetEvent::PluginLoaded("surfpool-subgraph".into())); } + } + }, + Err(_) => {}, + }, + recv(plugins_data_rx) -> msg => match msg { + Err(_) => unreachable!(), + Ok((transaction, transaction_metadata)) => { + let transaction_status_meta = TransactionStatusMeta { + status: Ok(()), + fee: 0, + pre_balances: vec![], + post_balances: vec![], + inner_instructions: None, + log_messages: Some(transaction_metadata.logs.clone()), + pre_token_balances: None, + post_token_balances: None, + rewards: None, + loaded_addresses: LoadedAddresses { + writable: vec![], + readonly: vec![], + }, + return_data: Some(transaction_metadata.return_data.clone()), + compute_units_consumed: Some(transaction_metadata.compute_units_consumed), + }; + + let transaction = SanitizedTransaction::try_from_legacy_transaction(transaction, &HashSet::new()) + .unwrap(); + + let transaction_replica = ReplicaTransactionInfoV2 { + signature: &transaction_metadata.signature, + is_vote: false, + transaction: &transaction, + transaction_status_meta: &transaction_status_meta, + index: 0 }; - let constructor: Symbol = lib - .get(b"_create_plugin") - .map_err(|e| GeyserPluginManagerError::PluginLoadError(e.to_string()))?; - let plugin_raw = constructor(); - (Box::from_raw(plugin_raw), lib) - }; - - let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); - let subgraph_plugin_config = SubgraphPluginConfig { - ipc_token, - subgraph_request: Some(config.data.clone()) - }; - let config_file = serde_json::to_string(&subgraph_plugin_config).unwrap(); - let _res = plugin.on_load(&config_file, false); - if let Ok((_, rx)) = server.accept() { - let subgraph_rx = ipc_router.route_ipc_receiver_to_new_crossbeam_receiver(rx); - let _ = subgraph_commands_tx.send(SubgraphCommand::ObserveSubgraph(subgraph_rx)); - }; - plugin_manager.plugins.push(LoadedGeyserPlugin::new(lib, plugin, Some(plugin_name.clone()))); - let _ = simnet_events_tx_copy.send(SimnetEvent::PluginLoaded("surfpool-subgraph".into())); + for plugin in plugin_manager.plugins.iter() { + plugin.notify_transaction(ReplicaTransactionInfoVersions::V0_0_2(&transaction_replica), 0).unwrap(); + } + } } - } - } - } - while let Ok((transaction, transaction_metadata)) = plugins_data_rx.recv() { - let transaction_status_meta = TransactionStatusMeta { - status: Ok(()), - fee: 0, - pre_balances: vec![], - post_balances: vec![], - inner_instructions: None, - log_messages: Some(transaction_metadata.logs.clone()), - pre_token_balances: None, - post_token_balances: None, - rewards: None, - loaded_addresses: LoadedAddresses { - writable: vec![], - readonly: vec![], - }, - return_data: Some(transaction_metadata.return_data.clone()), - compute_units_consumed: Some(transaction_metadata.compute_units_consumed), - }; - let transaction = SanitizedTransaction::try_from_legacy_transaction(transaction, &HashSet::new()) - .unwrap(); - - let transaction_replica = ReplicaTransactionInfoV2 { - signature: &transaction_metadata.signature, - is_vote: false, - transaction: &transaction, - transaction_status_meta: &transaction_status_meta, - index: 0 - }; - for plugin in plugin_manager.plugins.iter() { - plugin.notify_transaction(ReplicaTransactionInfoVersions::V0_0_2(&transaction_replica), 0).unwrap(); + } } - } - Ok(()) + Ok::<(), String>(()) }); } diff --git a/crates/subgraph/src/lib.rs b/crates/subgraph/src/lib.rs index 5a8184a..cdfe185 100644 --- a/crates/subgraph/src/lib.rs +++ b/crates/subgraph/src/lib.rs @@ -71,8 +71,13 @@ impl GeyserPlugin for SurfpoolSubgraph { transaction: ReplicaTransactionInfoVersions, _slot: Slot, ) -> PluginResult<()> { + let Ok(tx) = self.subgraph_indexing_event_tx.lock() else { + return Ok(()); + }; + let tx = tx.as_ref().unwrap(); match transaction { ReplicaTransactionInfoVersions::V0_0_2(data) => { + let _ = tx.send(SubgraphIndexingEvent::Entry(format!("{}", data.signature))); if data.is_vote { return Ok(()); } From 42ec0a5379677301343af879fe3d79698cbf110e Mon Sep 17 00:00:00 2001 From: MicaiahReid Date: Fri, 14 Feb 2025 16:55:17 -0500 Subject: [PATCH 12/24] fix(core): make subgraph plugin config's subgraph_request non-optional --- crates/core/src/simnet/mod.rs | 2 +- crates/core/src/types.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index 5894c11..53c5603 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -341,7 +341,7 @@ pub async fn start( let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); let subgraph_plugin_config = SubgraphPluginConfig { ipc_token, - subgraph_request: Some(config.data.clone()) + subgraph_request: config.data.clone() }; let config_file = serde_json::to_string(&subgraph_plugin_config).unwrap(); let _res = plugin.on_load(&config_file, false); diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index e168912..2e8f91f 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -141,8 +141,8 @@ impl RpcConfig { } } -#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct SubgraphPluginConfig { pub ipc_token: String, - pub subgraph_request: Option, + pub subgraph_request: SubgraphRequest, } From 5556935e9411ab8f9ed887b7e5075f8ade5b62dd Mon Sep 17 00:00:00 2001 From: MicaiahReid Date: Fri, 14 Feb 2025 16:55:50 -0500 Subject: [PATCH 13/24] feat(plugin): first attempt at filtering event data on `notify_transaction` --- crates/subgraph/src/lib.rs | 55 +++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/crates/subgraph/src/lib.rs b/crates/subgraph/src/lib.rs index cdfe185..2212740 100644 --- a/crates/subgraph/src/lib.rs +++ b/crates/subgraph/src/lib.rs @@ -1,19 +1,24 @@ use { agave_geyser_plugin_interface::geyser_plugin_interface::{ - GeyserPlugin, ReplicaAccountInfoVersions, ReplicaBlockInfoVersions, + GeyserPlugin, GeyserPluginError, ReplicaAccountInfoVersions, ReplicaBlockInfoVersions, ReplicaEntryInfoVersions, ReplicaTransactionInfoVersions, Result as PluginResult, SlotStatus, }, ipc_channel::ipc::IpcSender, solana_program::clock::Slot, std::sync::Mutex, - surfpool_core::types::{SubgraphIndexingEvent, SubgraphPluginConfig}, + surfpool_core::{ + solana_sdk::{bs58, inner_instruction}, + types::{SubgraphIndexingEvent, SubgraphPluginConfig}, + }, + txtx_addon_network_svm::codec::subgraph::{IndexedSubgraphSourceType, SubgraphRequest}, }; #[derive(Default, Debug)] pub struct SurfpoolSubgraph { pub id: String, subgraph_indexing_event_tx: Mutex>>, + subgraph_request: Option, } impl GeyserPlugin for SurfpoolSubgraph { @@ -27,6 +32,7 @@ impl GeyserPlugin for SurfpoolSubgraph { let (tx, rx) = ipc_channel::ipc::channel().unwrap(); let _ = oneshot_tx.send(rx); self.subgraph_indexing_event_tx = Mutex::new(Some(tx)); + self.subgraph_request = Some(config.subgraph_request); Ok(()) } @@ -75,6 +81,9 @@ impl GeyserPlugin for SurfpoolSubgraph { return Ok(()); }; let tx = tx.as_ref().unwrap(); + let Some(subgraph_request) = self.subgraph_request else { + return Ok(()); + }; match transaction { ReplicaTransactionInfoVersions::V0_0_2(data) => { let _ = tx.send(SubgraphIndexingEvent::Entry(format!("{}", data.signature))); @@ -87,7 +96,47 @@ impl GeyserPlugin for SurfpoolSubgraph { }; for inner_instructions in inner_instructions.iter() { for instruction in inner_instructions.instructions.iter() { - println!("{:?}", instruction); + let instruction = instruction.instruction; + let decoded_data = bs58::decode(instruction.data).into_vec().map_err( + GeyserPluginError::TransactionUpdateError { + msg: format!("failed to decode instruction data"), + }, + )?; + // it's not valid cpi event data if there isn't an 8-byte signature + if decoded_data.len() < 8 { + continue; + } + let eight_bytes = decoded_data[0..8].to_vec(); + let decoded_signature = bs58::decode(eight_bytes).into_vec().map_err( + GeyserPluginError::TransactionUpdateError { + msg: format!("failed to decode instruction data"), + }, + )?; + for field in subgraph_request.fields.iter() { + match field.data_source { + IndexedSubgraphSourceType::Instruction( + instruction_subgraph_source, + ) => { + continue; + } + IndexedSubgraphSourceType::Event(event_subgraph_source) => { + if event_subgraph_source + .event + .discriminator + .eq(decoded_signature.as_slice()) + { + println!( + "found event with match!!!: {:?}", + event_subgraph_source.event.name + ); + let _ = tx.send(SubgraphIndexingEvent::IndexSubgraph { + subgraph_request: subgraph_request.clone(), + transaction, + }); + } + } + } + } } } } From 6fc369a47c072c1080aa881a38e637af29a46a41 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 14 Feb 2025 17:02:46 -0500 Subject: [PATCH 14/24] fix: functional runloop --- crates/cli/src/http/mod.rs | 8 +++++--- crates/core/src/simnet/mod.rs | 8 +++----- crates/core/src/types.rs | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index b43d444..58d3c60 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -12,7 +12,7 @@ use juniper_graphql_ws::ConnectionConfig; use std::error::Error as StdError; use std::sync::RwLock; use std::time::Duration; -use surfpool_core::types::{Collection, SubgraphCommand, SubgraphEvent, SurfpoolConfig}; +use surfpool_core::types::{Collection, SubgraphCommand, SubgraphEvent, SubgraphIndexingEvent, SurfpoolConfig}; use surfpool_gql::types::collection::CollectionData; use surfpool_gql::{new_graphql_schema, Context as GqlContext, GqlSchema}; use txtx_core::kit::uuid::Uuid; @@ -82,8 +82,10 @@ pub async fn start_server( }, }, i => match oper.recv(&observers[i - 1]) { - Ok(value) => { - println!("Inject {} in store", value); + Ok(cmd) => match cmd { + SubgraphIndexingEvent::Entry(value) => { + println!("Inject {} in store", value); + } } Err(_e) => {} }, diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index 5894c11..b13c289 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -44,8 +44,7 @@ use crate::{ bank_data::BankData, full::Full, minimal::Minimal, SurfpoolMiddleware, }, types::{ - ClockCommand, ClockEvent, PluginManagerCommand, RunloopTriggerMode, SimnetCommand, - SimnetEvent, SubgraphCommand, SubgraphPluginConfig, SurfpoolConfig, TransactionStatusEvent, + ClockCommand, ClockEvent, PluginManagerCommand, RunloopTriggerMode, SimnetCommand, SimnetEvent, SubgraphCommand, SubgraphIndexingEvent, SubgraphPluginConfig, SurfpoolConfig, TransactionStatusEvent }, }; @@ -183,7 +182,7 @@ impl EntryStatus { } } -use std::{fs::File, io::Read, path::PathBuf}; +use std::path::PathBuf; type PluginConstructor = unsafe fn() -> *mut dyn GeyserPlugin; use libloading::{Library, Symbol}; @@ -313,7 +312,6 @@ pub async fn start( let plugin_name = result["name"].as_str().map(|s| s.to_owned()).unwrap_or(format!("surfpool-subgraph")); - let ipc_router = RouterProxy::new(); loop { select! { @@ -338,7 +336,7 @@ pub async fn start( (Box::from_raw(plugin_raw), lib) }; - let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); + let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); let subgraph_plugin_config = SubgraphPluginConfig { ipc_token, subgraph_request: Some(config.data.clone()) diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index e168912..b88f1c6 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -52,7 +52,7 @@ pub enum SubgraphIndexingEvent { #[derive(Debug, Clone)] pub enum SubgraphCommand { CreateSubgraph(SubgraphRequest, Sender), - ObserveSubgraph(Receiver), + ObserveSubgraph(Receiver), Shutdown, } From e408e2451aac9940530353b22c804d7226ca55dd Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 14 Feb 2025 17:10:13 -0500 Subject: [PATCH 15/24] fix: graphql endpoint, build error --- crates/cli/src/http/mod.rs | 6 ++---- crates/core/src/simnet/mod.rs | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 58d3c60..5b8ebda 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -167,7 +167,6 @@ async fn post_graphql( schema: Data, context: Data>, ) -> Result { - println!("POST /graphql"); let context = context .read() .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; @@ -180,7 +179,6 @@ async fn get_graphql( schema: Data, context: Data>, ) -> Result { - println!("GET /graphql"); let context = context .read() .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; @@ -206,9 +204,9 @@ async fn subscriptions( } async fn playground() -> Result { - playground_handler("/graphql", Some("/subscriptions")).await + playground_handler("/gql/v1/graphql", Some("/gql/v1/subscriptions")).await } async fn graphiql() -> Result { - graphiql_handler("/graphql", Some("/subscriptions")).await + graphiql_handler("/gql/v1/graphql", Some("/gql/v1/subscriptions")).await } diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index be191bf..fdc0ebe 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -312,7 +312,8 @@ pub async fn start( let plugin_name = result["name"].as_str().map(|s| s.to_owned()).unwrap_or(format!("surfpool-subgraph")); - + let ipc_router = RouterProxy::new(); + loop { select! { recv(plugin_manager_commands_rx) -> msg => match msg { From 019ec6f4feb60d0a821eb6d17dd093421faf66fe Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Fri, 14 Feb 2025 19:24:38 -0500 Subject: [PATCH 16/24] feat: graphql insertion --- Cargo.lock | 1 + Cargo.toml | 2 +- crates/cli/src/http/mod.rs | 24 +++++++++---- crates/core/src/rpc/admin.rs | 4 ++- crates/core/src/simnet/mod.rs | 17 ++++++---- crates/core/src/types.rs | 13 ++++--- crates/gql/src/query.rs | 54 ++++-------------------------- crates/gql/src/types/collection.rs | 2 +- crates/subgraph/Cargo.toml | 1 + crates/subgraph/src/lib.rs | 47 ++++++++++++++------------ 10 files changed, 75 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77c9a7d..9b89c03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9350,6 +9350,7 @@ dependencies = [ "solana-program", "surfpool-core", "txtx-addon-network-svm", + "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 50749f9..f7af5ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,6 @@ serde_json = "1.0.135" txtx-core = { path = "../txtx/crates/txtx-core" } txtx-addon-network-svm = { package = "txtx-addon-network-svm", path = "../txtx/addons/svm" } bincode = "1.3.3" - +uuid = "1.7.0" # txtx-core = { version = "0.2.2" } # txtx-addon-network-svm = { version = "0.1.3" } diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 5b8ebda..7bd7d3c 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -12,7 +12,9 @@ use juniper_graphql_ws::ConnectionConfig; use std::error::Error as StdError; use std::sync::RwLock; use std::time::Duration; -use surfpool_core::types::{Collection, SubgraphCommand, SubgraphEvent, SubgraphIndexingEvent, SurfpoolConfig}; +use surfpool_core::types::{ + Collection, Entry, SubgraphCommand, SubgraphEvent, SubgraphIndexingEvent, SurfpoolConfig +}; use surfpool_gql::types::collection::CollectionData; use surfpool_gql::{new_graphql_schema, Context as GqlContext, GqlSchema}; use txtx_core::kit::uuid::Uuid; @@ -53,11 +55,11 @@ pub async fn start_server( // todo } Ok(cmd) => match cmd { - SubgraphCommand::CreateSubgraph(config, sender) => { + SubgraphCommand::CreateSubgraph(uuid, config, sender) => { let gql_context = gql_context_copy.write().unwrap(); let mut collections = gql_context.collections_store.write().unwrap(); - let collection_uuid = Uuid::new_v4(); + let collection_uuid = uuid; collections.insert( collection_uuid.clone(), @@ -83,10 +85,20 @@ pub async fn start_server( }, i => match oper.recv(&observers[i - 1]) { Ok(cmd) => match cmd { - SubgraphIndexingEvent::Entry(value) => { - println!("Inject {} in store", value); + SubgraphIndexingEvent::ApplyEntry(uuid, value/* , request, slot*/) => { + let gql_context = gql_context_copy.write().unwrap(); + let mut collections = + gql_context.collections_store.write().unwrap(); + let collection_data = collections.get_mut(&uuid).unwrap(); + collection_data.collection.entries.push(Entry { + uuid: Uuid::new_v4(), + value + }); } - } + SubgraphIndexingEvent::Rountrip(uuid) => { + println!("Subgraph components initialized {}", uuid); + } + }, Err(_e) => {} }, } diff --git a/crates/core/src/rpc/admin.rs b/crates/core/src/rpc/admin.rs index 42efdcd..b3628ad 100644 --- a/crates/core/src/rpc/admin.rs +++ b/crates/core/src/rpc/admin.rs @@ -3,6 +3,7 @@ use jsonrpc_core::Result; use jsonrpc_derive::rpc; use solana_client::rpc_config::RpcAccountIndex; use solana_sdk::pubkey::Pubkey; +use uuid::Uuid; use std::collections::HashMap; use std::net::SocketAddr; use std::time::Duration; @@ -141,10 +142,11 @@ impl AdminRpc for SurfpoolAdminRpc { fn load_plugin(&self, meta: Self::Metadata, config_file: String) -> BoxFuture> { let config = serde_json::from_str::(&config_file).unwrap(); let ctx = meta.unwrap(); + let uuid = Uuid::new_v4(); let (tx, rx) = crossbeam_channel::bounded(1); let _ = ctx .plugin_manager_commands_tx - .send(PluginManagerCommand::LoadConfig(config, tx)); + .send(PluginManagerCommand::LoadConfig(uuid, config, tx)); let Ok(endpoint_url) = rx.recv_timeout(Duration::from_secs(10)) else { unimplemented!() }; diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index fdc0ebe..e90da20 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -44,7 +44,9 @@ use crate::{ bank_data::BankData, full::Full, minimal::Minimal, SurfpoolMiddleware, }, types::{ - ClockCommand, ClockEvent, PluginManagerCommand, RunloopTriggerMode, SimnetCommand, SimnetEvent, SubgraphCommand, SubgraphIndexingEvent, SubgraphPluginConfig, SurfpoolConfig, TransactionStatusEvent + ClockCommand, ClockEvent, PluginManagerCommand, RunloopTriggerMode, SimnetCommand, + SimnetEvent, SubgraphCommand, SubgraphIndexingEvent, SubgraphPluginConfig, SurfpoolConfig, + TransactionStatusEvent, }, }; @@ -258,6 +260,8 @@ pub async fn start( let simnet_events_tx_copy = simnet_events_tx.clone(); let (plugins_data_tx, plugins_data_rx) = unbounded::<(Transaction, TransactionMetadata)>(); + let ipc_router = RouterProxy::new(); + if !config.plugin_config_path.is_empty() { let _handle = hiro_system_kit::thread_named("geyser plugins handler").spawn(move || { let mut plugin_manager = GeyserPluginManager::new(); @@ -312,15 +316,13 @@ pub async fn start( let plugin_name = result["name"].as_str().map(|s| s.to_owned()).unwrap_or(format!("surfpool-subgraph")); - let ipc_router = RouterProxy::new(); - loop { select! { recv(plugin_manager_commands_rx) -> msg => match msg { Ok(event) => { match event { - PluginManagerCommand::LoadConfig(config, notifier) => { - let _ = subgraph_commands_tx.send(SubgraphCommand::CreateSubgraph(config.data.clone(), notifier)); + PluginManagerCommand::LoadConfig(uuid, config, notifier) => { + let _ = subgraph_commands_tx.send(SubgraphCommand::CreateSubgraph(uuid.clone(), config.data.clone(), notifier)); let (mut plugin, lib) = unsafe { let lib = match Library::new(&libpath) { @@ -339,6 +341,7 @@ pub async fn start( let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); let subgraph_plugin_config = SubgraphPluginConfig { + uuid, ipc_token, subgraph_request: config.data.clone() }; @@ -375,10 +378,10 @@ pub async fn start( return_data: Some(transaction_metadata.return_data.clone()), compute_units_consumed: Some(transaction_metadata.compute_units_consumed), }; - + let transaction = SanitizedTransaction::try_from_legacy_transaction(transaction, &HashSet::new()) .unwrap(); - + let transaction_replica = ReplicaTransactionInfoV2 { signature: &transaction_metadata.signature, is_vote: false, diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 105be54..54b70e6 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -1,13 +1,14 @@ +use agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaTransactionInfoVersions; use chrono::{DateTime, Local}; use crossbeam_channel::{Receiver, Sender}; use litesvm::types::TransactionMetadata; use solana_client::rpc_config::RpcSendTransactionConfig; use solana_sdk::{ blake3::Hash, - clock::Clock, + clock::{Clock, Slot}, epoch_info::EpochInfo, pubkey::Pubkey, - transaction::{TransactionError, VersionedTransaction}, + transaction::{SanitizedTransaction, TransactionError, VersionedTransaction}, }; use solana_transaction_status::TransactionConfirmationStatus; use std::path::PathBuf; @@ -46,12 +47,13 @@ pub enum SubgraphEvent { #[derive(Debug, Clone, Deserialize, Serialize)] pub enum SubgraphIndexingEvent { - Entry(String), + Rountrip(Uuid), + ApplyEntry(Uuid, String), //, SubgraphRequest, u64), } #[derive(Debug, Clone)] pub enum SubgraphCommand { - CreateSubgraph(SubgraphRequest, Sender), + CreateSubgraph(Uuid, SubgraphRequest, Sender), ObserveSubgraph(Receiver), Shutdown, } @@ -93,7 +95,7 @@ pub enum SimnetCommand { } pub enum PluginManagerCommand { - LoadConfig(PluginConfig, Sender), + LoadConfig(Uuid, PluginConfig, Sender), } pub enum ClockCommand { @@ -143,6 +145,7 @@ impl RpcConfig { #[derive(Debug, Clone, Deserialize, Serialize)] pub struct SubgraphPluginConfig { + pub uuid: Uuid, pub ipc_token: String, pub subgraph_request: SubgraphRequest, } diff --git a/crates/gql/src/query.rs b/crates/gql/src/query.rs index 29d869d..fbed58c 100644 --- a/crates/gql/src/query.rs +++ b/crates/gql/src/query.rs @@ -1,4 +1,4 @@ -use crate::Context; +use crate::{types::collection::CollectionData, Context}; use juniper_codegen::graphql_object; pub struct Query; @@ -11,51 +11,9 @@ impl Query { "1.0" } - // async fn collections(context: &Context) -> Vec { - // let collections_store = context.collections_store.read().await; - // collections_store - // .values() - // .cloned() - // .filter(|b| if let Panel::ActionPanel(_) = b.panel { true } else { false }) - // .map(GqlActionBlock::new) - // .collect() - // } - - // async fn modal_blocks(context: &Context) -> Vec { - // let block_store = context.block_store.read().await; - // block_store - // .values() - // .cloned() - // .filter(|b| if let Panel::ModalPanel(_) = b.panel { true } else { false }) - // .map(GqlModalBlock::new) - // .collect() - // } - - // async fn error_blocks(context: &Context) -> Vec { - // let block_store = context.block_store.read().await; - // block_store - // .values() - // .cloned() - // .filter(|b| if let Panel::ErrorPanel(_) = b.panel { true } else { false }) - // .map(GqlErrorBlock::new) - // .collect() - // } - - // async fn progress_blocks(context: &Context) -> Vec { - // let block_store = context.block_store.read().await; - // block_store - // .values() - // .cloned() - // .filter(|b| if let Panel::ProgressBar(_) = b.panel { true } else { false }) - // .map(GqlProgressBlock::new) - // .collect() - // } - - // fn runbook(context: &Context) -> RunbookMetadata { - // RunbookMetadata::new( - // &context.runbook_name, - // &context.registered_addons, - // &context.runbook_description, - // ) - // } + async fn collections(context: &Context) -> Vec { + let collections_store = context.collections_store.read().unwrap(); + collections_store + .values().into_iter().map(|c| c.clone()).collect() + } } diff --git a/crates/gql/src/types/collection.rs b/crates/gql/src/types/collection.rs index ddae437..2a2b121 100644 --- a/crates/gql/src/types/collection.rs +++ b/crates/gql/src/types/collection.rs @@ -1,7 +1,7 @@ use super::entry::EntryData; use crate::Context; use juniper_codegen::graphql_object; -use surfpool_core::types::Collection; +use surfpool_core::types::{Collection, Entry}; #[derive(Debug, Clone)] pub struct CollectionData { diff --git a/crates/subgraph/Cargo.toml b/crates/subgraph/Cargo.toml index fea9c42..70dd69d 100644 --- a/crates/subgraph/Cargo.toml +++ b/crates/subgraph/Cargo.toml @@ -20,3 +20,4 @@ surfpool-core = { workspace = true } serde_json = { workspace = true } serde = { workspace = true } bincode = { workspace = true } +uuid = { workspace = true } diff --git a/crates/subgraph/src/lib.rs b/crates/subgraph/src/lib.rs index 2212740..9d0bfa8 100644 --- a/crates/subgraph/src/lib.rs +++ b/crates/subgraph/src/lib.rs @@ -11,12 +11,12 @@ use { solana_sdk::{bs58, inner_instruction}, types::{SubgraphIndexingEvent, SubgraphPluginConfig}, }, - txtx_addon_network_svm::codec::subgraph::{IndexedSubgraphSourceType, SubgraphRequest}, + txtx_addon_network_svm::codec::subgraph::{IndexedSubgraphSourceType, SubgraphRequest}, uuid::Uuid, }; #[derive(Default, Debug)] pub struct SurfpoolSubgraph { - pub id: String, + pub uuid: Uuid, subgraph_indexing_event_tx: Mutex>>, subgraph_request: Option, } @@ -30,7 +30,9 @@ impl GeyserPlugin for SurfpoolSubgraph { let config = serde_json::from_str::(&config_file).unwrap(); let oneshot_tx = IpcSender::connect(config.ipc_token).unwrap(); let (tx, rx) = ipc_channel::ipc::channel().unwrap(); + let _ = tx.send(SubgraphIndexingEvent::Rountrip(config.uuid.clone())); let _ = oneshot_tx.send(rx); + self.uuid = config.uuid.clone(); self.subgraph_indexing_event_tx = Mutex::new(Some(tx)); self.subgraph_request = Some(config.subgraph_request); Ok(()) @@ -75,18 +77,23 @@ impl GeyserPlugin for SurfpoolSubgraph { fn notify_transaction( &self, transaction: ReplicaTransactionInfoVersions, - _slot: Slot, + slot: Slot, ) -> PluginResult<()> { let Ok(tx) = self.subgraph_indexing_event_tx.lock() else { return Ok(()); }; let tx = tx.as_ref().unwrap(); - let Some(subgraph_request) = self.subgraph_request else { + let Some(ref subgraph_request) = self.subgraph_request else { return Ok(()); }; match transaction { ReplicaTransactionInfoVersions::V0_0_2(data) => { - let _ = tx.send(SubgraphIndexingEvent::Entry(format!("{}", data.signature))); + let _ = tx.send(SubgraphIndexingEvent::ApplyEntry( + self.uuid, + data.signature.to_string(), + // subgraph_request.clone(), + // slot, + )); if data.is_vote { return Ok(()); } @@ -96,24 +103,20 @@ impl GeyserPlugin for SurfpoolSubgraph { }; for inner_instructions in inner_instructions.iter() { for instruction in inner_instructions.instructions.iter() { - let instruction = instruction.instruction; - let decoded_data = bs58::decode(instruction.data).into_vec().map_err( - GeyserPluginError::TransactionUpdateError { - msg: format!("failed to decode instruction data"), - }, - )?; + let instruction = &instruction.instruction; + let decoded_data = bs58::decode(&instruction.data) + .into_vec() + .map_err(|e| GeyserPluginError::Custom(Box::new(e)))?; // it's not valid cpi event data if there isn't an 8-byte signature if decoded_data.len() < 8 { continue; } let eight_bytes = decoded_data[0..8].to_vec(); - let decoded_signature = bs58::decode(eight_bytes).into_vec().map_err( - GeyserPluginError::TransactionUpdateError { - msg: format!("failed to decode instruction data"), - }, - )?; + let decoded_signature = bs58::decode(eight_bytes) + .into_vec() + .map_err(|e| GeyserPluginError::Custom(Box::new(e)))?; for field in subgraph_request.fields.iter() { - match field.data_source { + match &field.data_source { IndexedSubgraphSourceType::Instruction( instruction_subgraph_source, ) => { @@ -129,10 +132,12 @@ impl GeyserPlugin for SurfpoolSubgraph { "found event with match!!!: {:?}", event_subgraph_source.event.name ); - let _ = tx.send(SubgraphIndexingEvent::IndexSubgraph { - subgraph_request: subgraph_request.clone(), - transaction, - }); + let _ = tx.send(SubgraphIndexingEvent::ApplyEntry( + self.uuid, + data.signature.to_string(), + // subgraph_request.clone(), + // slot, + )); } } } From 406b37f6820acc5e57bbb131ce6f503d9472977b Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Sat, 15 Feb 2025 12:08:41 -0500 Subject: [PATCH 17/24] feat: implement gql schema updates --- Cargo.lock | 34 +++++- crates/cli/Cargo.toml | 3 +- crates/cli/src/http/mod.rs | 54 ++++++--- crates/core/src/rpc/admin.rs | 2 +- crates/core/src/simnet/mod.rs | 8 +- crates/core/src/types.rs | 4 +- crates/gql/Cargo.toml | 3 +- crates/gql/src/lib.rs | 24 +++- crates/gql/src/mutation.rs | 11 ++ crates/gql/src/query.rs | 202 ++++++++++++++++++++++++++++++- crates/gql/src/subscription.rs | 20 +++ crates/gql/src/types/mod.rs | 2 + crates/gql/src/types/subgraph.rs | 89 ++++++++++++++ crates/subgraph/src/lib.rs | 13 +- 14 files changed, 426 insertions(+), 43 deletions(-) create mode 100644 crates/gql/src/types/subgraph.rs diff --git a/Cargo.lock b/Cargo.lock index 9b89c03..a9085c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,13 +212,14 @@ dependencies = [ [[package]] name = "actix-ws" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535aec173810be3ca6f25dd5b4d431ae7125d62000aa3cbae1ec739921b02cf3" +checksum = "a3a1fb4f9f2794b0aadaf2ba5f14a6f034c7e86957b458c506a8cb75953f2d99" dependencies = [ "actix-codec", "actix-http", "actix-web", + "bytestring", "futures-core", "tokio", ] @@ -1545,6 +1546,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.16.2" @@ -1892,7 +1902,7 @@ version = "0.99.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", @@ -2665,6 +2675,16 @@ dependencies = [ "spinning_top", ] +[[package]] +name = "graphql-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a818c0d883d7c0801df27be910917750932be279c7bc82dc541b8769425f409" +dependencies = [ + "combine 4.6.7", + "thiserror 1.0.69", +] + [[package]] name = "h2" version = "0.3.26" @@ -3741,18 +3761,20 @@ dependencies = [ "auto_enums", "fnv", "futures 0.3.31", + "graphql-parser", "indexmap 2.7.1", "juniper_codegen", "serde", "smartstring", "static_assertions", + "void", ] [[package]] name = "juniper_actix" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eafd05a9b03395e4c032e5d903109a5fceaf89bb7ae6731e416bc2088f9e556" +checksum = "457190a70f2a8d7c6d9ae8a5cfc134909c498c335dbb751a147eec7a513faced" dependencies = [ "actix-http", "actix-web", @@ -9238,6 +9260,7 @@ dependencies = [ "dirs 6.0.0", "hiro-system-kit", "ipc-channel", + "juniper", "juniper_actix", "juniper_graphql_ws", "mime_guess", @@ -9329,6 +9352,7 @@ name = "surfpool-gql" version = "0.1.7" dependencies = [ "async-stream", + "convert_case 0.7.1", "futures 0.3.31", "futures-enum", "juniper", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 5183720..63df909 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -44,8 +44,9 @@ actix-cors = "0.7.0" rust-embed="8.2.0" mime_guess = "2.0.4" notify = { version = "8.0.0" } -juniper_actix = {version = "0.5.0", features = ["subscriptions"] } +juniper_actix = {version = "0.6.0", features = ["subscriptions"] } juniper_graphql_ws = { version = "0.4.0", features = ["graphql-transport-ws"] } +juniper = { version = "0.16.1", features = ["schema-language"] } ipc-channel = { workspace = true } bincode = { workspace = true } diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 7bd7d3c..3e1c10a 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -9,14 +9,16 @@ use actix_web::{Error, Responder}; use crossbeam::channel::{Receiver, Select, Sender}; use juniper_actix::{graphiql_handler, graphql_handler, playground_handler, subscriptions}; use juniper_graphql_ws::ConnectionConfig; +use std::collections::BTreeMap; use std::error::Error as StdError; use std::sync::RwLock; use std::time::Duration; use surfpool_core::types::{ - Collection, Entry, SubgraphCommand, SubgraphEvent, SubgraphIndexingEvent, SurfpoolConfig + Collection, Entry, SchemaDatasourceingEvent, SubgraphCommand, SubgraphEvent, SurfpoolConfig, }; +use surfpool_gql::query::{SchemaDatasource, SchemaDatasourceEntry}; use surfpool_gql::types::collection::CollectionData; -use surfpool_gql::{new_graphql_schema, Context as GqlContext, GqlSchema}; +use surfpool_gql::{new_dynamic_schema, Context as GqlContext, GqlDynamicSchema as GqlSchema}; use txtx_core::kit::uuid::Uuid; #[cfg(feature = "explorer")] @@ -34,10 +36,15 @@ pub async fn start_server( subgraph_commands_rx: Receiver, _ctx: &Context, ) -> Result> { - let gql_schema = Data::new(new_graphql_schema()); - let gql_context: Data> = Data::new(RwLock::new(GqlContext::new())); + let context = GqlContext::new(); + let mut schema_datasource = SchemaDatasource::new(); + let schema = RwLock::new(Some(new_dynamic_schema(schema_datasource.clone()))); + let schema_wrapped = Data::new(schema); + let context_wrapped = Data::new(RwLock::new(context)); + + let gql_context_copy = context_wrapped.clone(); + let gql_schema_copy = schema_wrapped.clone(); - let gql_context_copy = gql_context.clone(); let _handle = hiro_system_kit::thread_named("subgraph") .spawn(move || { let mut observers = vec![]; @@ -56,6 +63,15 @@ pub async fn start_server( } Ok(cmd) => match cmd { SubgraphCommand::CreateSubgraph(uuid, config, sender) => { + let mut gql_schema = gql_schema_copy.write().unwrap(); + let subgraph_name = config.subgraph_name.clone(); + let mut schema = SchemaDatasourceEntry::new(&subgraph_name); + for fields in config.fields.iter() { + schema.fields.push(fields.display_name.clone()); + } + schema_datasource.add_entry(schema); + gql_schema.replace(new_dynamic_schema(schema_datasource.clone())); + let gql_context = gql_context_copy.write().unwrap(); let mut collections = gql_context.collections_store.write().unwrap(); @@ -71,7 +87,6 @@ pub async fn start_server( }, }, ); - println!("{:?}", collections); let _ = sender.send("http://127.0.0.1:8900/graphql".into()); } SubgraphCommand::ObserveSubgraph(subgraph_observer_rx) => { @@ -85,17 +100,20 @@ pub async fn start_server( }, i => match oper.recv(&observers[i - 1]) { Ok(cmd) => match cmd { - SubgraphIndexingEvent::ApplyEntry(uuid, value/* , request, slot*/) => { + SchemaDatasourceingEvent::ApplyEntry( + uuid, + value, /* , request, slot*/ + ) => { let gql_context = gql_context_copy.write().unwrap(); let mut collections = gql_context.collections_store.write().unwrap(); let collection_data = collections.get_mut(&uuid).unwrap(); collection_data.collection.entries.push(Entry { uuid: Uuid::new_v4(), - value + value, }); } - SubgraphIndexingEvent::Rountrip(uuid) => { + SchemaDatasourceingEvent::Rountrip(uuid) => { println!("Subgraph components initialized {}", uuid); } }, @@ -109,8 +127,8 @@ pub async fn start_server( let server = HttpServer::new(move || { App::new() - .app_data(gql_schema.clone()) - .app_data(gql_context.clone()) + .app_data(schema_wrapped.clone()) + .app_data(context_wrapped.clone()) .wrap( Cors::default() .allow_any_origin() @@ -176,25 +194,31 @@ async fn dist(path: web::Path) -> impl Responder { async fn post_graphql( req: HttpRequest, payload: web::Payload, - schema: Data, + schema: Data>>, context: Data>, ) -> Result { let context = context .read() .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; - graphql_handler(&schema, &context, req, payload).await + let schema = schema + .read() + .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; + graphql_handler(&(schema.as_ref().unwrap()), &context, req, payload).await } async fn get_graphql( req: HttpRequest, payload: web::Payload, - schema: Data, + schema: Data>>, context: Data>, ) -> Result { let context = context .read() .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; - graphql_handler(&schema, &context, req, payload).await + let schema = schema + .read() + .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; + graphql_handler(&(schema.as_ref().unwrap()), &context, req, payload).await } async fn subscriptions( diff --git a/crates/core/src/rpc/admin.rs b/crates/core/src/rpc/admin.rs index b3628ad..ca664b0 100644 --- a/crates/core/src/rpc/admin.rs +++ b/crates/core/src/rpc/admin.rs @@ -3,12 +3,12 @@ use jsonrpc_core::Result; use jsonrpc_derive::rpc; use solana_client::rpc_config::RpcAccountIndex; use solana_sdk::pubkey::Pubkey; -use uuid::Uuid; use std::collections::HashMap; use std::net::SocketAddr; use std::time::Duration; use std::time::SystemTime; use txtx_addon_network_svm::codec::subgraph::PluginConfig; +use uuid::Uuid; use super::RunloopContext; use crate::types::PluginManagerCommand; diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index e90da20..60ee45e 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -44,9 +44,9 @@ use crate::{ bank_data::BankData, full::Full, minimal::Minimal, SurfpoolMiddleware, }, types::{ - ClockCommand, ClockEvent, PluginManagerCommand, RunloopTriggerMode, SimnetCommand, - SimnetEvent, SubgraphCommand, SubgraphIndexingEvent, SubgraphPluginConfig, SurfpoolConfig, - TransactionStatusEvent, + ClockCommand, ClockEvent, PluginManagerCommand, RunloopTriggerMode, + SchemaDatasourceingEvent, SimnetCommand, SimnetEvent, SubgraphCommand, + SubgraphPluginConfig, SurfpoolConfig, TransactionStatusEvent, }, }; @@ -339,7 +339,7 @@ pub async fn start( (Box::from_raw(plugin_raw), lib) }; - let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); + let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); let subgraph_plugin_config = SubgraphPluginConfig { uuid, ipc_token, diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 54b70e6..9c2bbe7 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -46,7 +46,7 @@ pub enum SubgraphEvent { } #[derive(Debug, Clone, Deserialize, Serialize)] -pub enum SubgraphIndexingEvent { +pub enum SchemaDatasourceingEvent { Rountrip(Uuid), ApplyEntry(Uuid, String), //, SubgraphRequest, u64), } @@ -54,7 +54,7 @@ pub enum SubgraphIndexingEvent { #[derive(Debug, Clone)] pub enum SubgraphCommand { CreateSubgraph(Uuid, SubgraphRequest, Sender), - ObserveSubgraph(Receiver), + ObserveSubgraph(Receiver), Shutdown, } diff --git a/crates/gql/Cargo.toml b/crates/gql/Cargo.toml index 4fe76c4..52a3ce1 100644 --- a/crates/gql/Cargo.toml +++ b/crates/gql/Cargo.toml @@ -20,4 +20,5 @@ juniper_codegen = { version = "0.16.0" } # juniper_codegen = { git = "https://github.com/graphql-rust/juniper", rev = "c0e1b3e" } async-stream = "0.3.5" tokio = "1.37.0" -surfpool-core = { workspace = true } \ No newline at end of file +surfpool-core = { workspace = true } +convert_case = "0.7.1" \ No newline at end of file diff --git a/crates/gql/src/lib.rs b/crates/gql/src/lib.rs index b06e284..9b4a3c0 100644 --- a/crates/gql/src/lib.rs +++ b/crates/gql/src/lib.rs @@ -1,9 +1,9 @@ use juniper::RootNode; -use mutation::Mutation; -use query::Query; +use mutation::{DynamicMutation, Mutation}; +use query::{DynamicQuery, Query, SchemaDatasource}; use std::sync::RwLock; use std::{collections::BTreeMap, sync::Arc}; -use subscription::Subscription; +use subscription::{DynamicSubscription, Subscription}; use types::{collection::CollectionData, entry::EntryData}; use uuid::Uuid; @@ -30,8 +30,20 @@ impl Context { impl juniper::Context for Context {} -pub type GqlSchema = RootNode<'static, Query, Mutation, Subscription>; +pub type GqlStaticSchema = RootNode<'static, Query, Mutation, Subscription>; +pub type GqlDynamicSchema = RootNode<'static, DynamicQuery, Mutation, DynamicSubscription>; -pub fn new_graphql_schema() -> GqlSchema { - GqlSchema::new(Query, Mutation, Subscription) +pub fn new_static_schema() -> GqlStaticSchema { + GqlStaticSchema::new(Query, Mutation, Subscription) +} + +pub fn new_dynamic_schema(subgraph_index: SchemaDatasource) -> GqlDynamicSchema { + GqlDynamicSchema::new_with_info( + DynamicQuery::new(), + Mutation, + DynamicSubscription, + subgraph_index, + (), + (), + ) } diff --git a/crates/gql/src/mutation.rs b/crates/gql/src/mutation.rs index 6d20bbc..8ceb4b7 100644 --- a/crates/gql/src/mutation.rs +++ b/crates/gql/src/mutation.rs @@ -11,3 +11,14 @@ impl Mutation { "1.0" } } + +pub struct DynamicMutation; + +#[graphql_object( + context = Context, +)] +impl DynamicMutation { + fn api_version() -> &'static str { + "1.0" + } +} diff --git a/crates/gql/src/query.rs b/crates/gql/src/query.rs index fbed58c..f016ac1 100644 --- a/crates/gql/src/query.rs +++ b/crates/gql/src/query.rs @@ -1,4 +1,199 @@ -use crate::{types::collection::CollectionData, Context}; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, +}; + +use crate::{ + types::{collection::CollectionData, subgraph::Subgraph}, + Context, +}; +use convert_case::{Case, Casing}; +use juniper::{ + meta::{Field, MetaType, ObjectMeta}, + Arguments, DefaultScalarValue, ExecutionResult, Executor, GraphQLType, GraphQLTypeAsync, + GraphQLValue, GraphQLValueAsync, Registry, +}; +use uuid::Uuid; + +#[derive(Debug)] +pub struct DynamicQuery { + pub subgraphs: HashMap, +} + +#[derive(Clone, Debug)] +pub struct SchemaDatasource { + pub entries: HashMap, +} + +impl SchemaDatasource { + pub fn new() -> Self { + Self { + entries: HashMap::new(), + } + } + + pub fn add_entry(&mut self, entry: SchemaDatasourceEntry) { + self.entries.insert(entry.name.to_case(Case::Camel), entry); + } +} + +#[derive(Clone, Debug)] +pub struct SchemaDatasourceEntry { + pub name: String, + pub description: Option, + pub fields: Vec, +} + +impl SchemaDatasourceEntry { + pub fn new(name: &str) -> Self { + Self { + name: name.to_case(Case::Pascal), + description: None, + fields: vec![], + } + } +} + +impl GraphQLType for SchemaDatasourceEntry { + fn name(spec: &SchemaDatasourceEntry) -> Option<&str> { + Some(spec.name.as_str()) + } + + fn meta<'r>(spec: &SchemaDatasourceEntry, registry: &mut Registry<'r>) -> MetaType<'r> + where + DefaultScalarValue: 'r, + { + let mut fields: Vec> = vec![]; + fields.push(registry.field::<&String>("uuid", &())); + for field in spec.fields.iter() { + fields.push(registry.field::<&String>(field, &())); + } + registry + .build_object_type::(&spec, &fields) + .into_meta() + } +} + +impl GraphQLValue for SchemaDatasourceEntry { + type Context = (); + type TypeInfo = SchemaDatasourceEntry; + + fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { + ::name(info) + } + + fn resolve_field( + &self, + info: &SchemaDatasourceEntry, + field_name: &str, + args: &Arguments, + executor: &Executor<()>, + ) -> ExecutionResult { + // Next, we need to match the queried field name. All arms of this match statement + // return `ExecutionResult`, which makes it hard to statically verify that the type you + // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` + // above. + let database = executor.context(); + match field_name { + // Because scalars are defined with another `Context` associated type, you must use + // `resolve_with_ctx` here to make the `executor` perform automatic type conversion + // of its argument. + // You pass a vector of `User` objects to `executor.resolve`, and it will determine + // which fields of the sub-objects to actually resolve based on the query. + // The `executor` instance keeps track of its current position in the query. + // "friends" => executor.resolve(info, + // &self.friend_ids.iter() + // .filter_map(|id| database.users.get(id)) + // .collect::>() + // ), + // We can only reach this panic in two cases: either a mismatch between the defined + // schema in `meta()` above, or a validation failed because of a this library bug. + // + // In either of those two cases, the only reasonable way out is to panic the thread. + _ => panic!("Field {field_name} not found on type User"), + } + } +} + +impl DynamicQuery { + pub fn new() -> Self { + Self { + subgraphs: HashMap::new(), + } + } +} + +impl GraphQLType for DynamicQuery { + fn name(_spec: &SchemaDatasource) -> Option<&str> { + Some("Query") + } + + fn meta<'r>(spec: &SchemaDatasource, registry: &mut Registry<'r>) -> MetaType<'r> + where + DefaultScalarValue: 'r, + { + // First, we need to define all fields and their types on this type. + // + // If we need arguments, want to implement interfaces, or want to add documentation + // strings, we can do it here. + let mut fields = vec![]; + fields.push(registry.field::<&String>("id", &())); + fields.push(registry.field::<&String>("name", &())); + fields.push(registry.field::<&String>("apiVersion", &())); + + for (name, entry) in spec.entries.iter() { + fields.push(registry.field::<&SchemaDatasourceEntry>(name, &entry)); + } + registry + .build_object_type::(&spec, &fields) + .into_meta() + } +} + +impl GraphQLValue for DynamicQuery { + type Context = Context; + type TypeInfo = SchemaDatasource; + + fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { + ::name(&info) + } + + fn resolve_field( + &self, + info: &SchemaDatasource, + field_name: &str, + args: &Arguments, + executor: &Executor, + ) -> ExecutionResult { + // Next, we need to match the queried field name. All arms of this match statement + // return `ExecutionResult`, which makes it hard to statically verify that the type you + // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` + // above. + match field_name { + // Because scalars are defined with another `Context` associated type, you must use + // `resolve_with_ctx` here to make the `executor` perform automatic type conversion + // of its argument. + "api_version" => executor.resolve_with_ctx(&(), "1.0"), + // subgraph => executor.resolve_with_ctx(&(), &self.name), + // You pass a vector of `User` objects to `executor.resolve`, and it will determine + // which fields of the sub-objects to actually resolve based on the query. + // The `executor` instance keeps track of its current position in the query. + // "friends" => executor.resolve(info, + // &self.friend_ids.iter() + // .filter_map(|id| database.users.get(id)) + // .collect::>() + // ), + // We can only reach this panic in two cases: either a mismatch between the defined + // schema in `meta()` above, or a validation failed because of a this library bug. + // + // In either of those two cases, the only reasonable way out is to panic the thread. + _ => panic!("Field {field_name} not found on type DynamicQuery"), + } + } +} + +impl GraphQLValueAsync for DynamicQuery {} + use juniper_codegen::graphql_object; pub struct Query; @@ -14,6 +209,9 @@ impl Query { async fn collections(context: &Context) -> Vec { let collections_store = context.collections_store.read().unwrap(); collections_store - .values().into_iter().map(|c| c.clone()).collect() + .values() + .into_iter() + .map(|c| c.clone()) + .collect() } } diff --git a/crates/gql/src/subscription.rs b/crates/gql/src/subscription.rs index a824e0e..f733b1a 100644 --- a/crates/gql/src/subscription.rs +++ b/crates/gql/src/subscription.rs @@ -25,3 +25,23 @@ impl Subscription { Box::pin(stream) } } + +pub struct DynamicSubscription; + +#[graphql_subscription( + context = Context, +)] +impl DynamicSubscription { + async fn entries_event(context: &Context) -> GqlEntriesStream { + let entries_tx = context.entries_broadcaster.clone(); + let mut entries_tx = entries_tx.subscribe(); + let stream = async_stream::stream! { + loop { + if let Ok(entry_event) = entries_tx.recv().await { + yield Ok(entry_event) + } + } + }; + Box::pin(stream) + } +} diff --git a/crates/gql/src/types/mod.rs b/crates/gql/src/types/mod.rs index 668b69b..330a50f 100644 --- a/crates/gql/src/types/mod.rs +++ b/crates/gql/src/types/mod.rs @@ -1,2 +1,4 @@ pub mod collection; pub mod entry; + +pub mod subgraph; diff --git a/crates/gql/src/types/subgraph.rs b/crates/gql/src/types/subgraph.rs new file mode 100644 index 0000000..1413ba0 --- /dev/null +++ b/crates/gql/src/types/subgraph.rs @@ -0,0 +1,89 @@ +use juniper::{ + meta::MetaType, Arguments, Context, DefaultScalarValue, ExecutionResult, Executor, GraphQLType, + GraphQLValue, Registry, +}; +use std::collections::HashMap; + +pub struct SubgraphSpecification { + name: String, +} + +#[derive(Debug)] +pub struct Database { + subgraphs: HashMap, +} +impl Context for Database {} + +#[derive(Debug)] +pub struct Subgraph { + id: String, + name: String, + // friend_ids: Vec +} + +impl GraphQLType for Subgraph { + fn name(spec: &SubgraphSpecification) -> Option<&str> { + Some(&spec.name) + } + + fn meta<'r>(spec: &SubgraphSpecification, registry: &mut Registry<'r>) -> MetaType<'r> + where + DefaultScalarValue: 'r, + { + // First, we need to define all fields and their types on this type. + // + // If we need arguments, want to implement interfaces, or want to add documentation + // strings, we can do it here. + let fields = &[ + registry.field::<&String>("id", &()), + registry.field::<&String>("name", &()), + // registry.field::>("friends", &spec), + ]; + registry + .build_object_type::(&spec, fields) + .into_meta() + } +} + +impl GraphQLValue for Subgraph { + type Context = Database; + type TypeInfo = SubgraphSpecification; + + fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { + ::name(&info) + } + + fn resolve_field( + &self, + info: &SubgraphSpecification, + field_name: &str, + args: &Arguments, + executor: &Executor, + ) -> ExecutionResult { + // Next, we need to match the queried field name. All arms of this match statement + // return `ExecutionResult`, which makes it hard to statically verify that the type you + // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` + // above. + let database = executor.context(); + match field_name { + // Because scalars are defined with another `Context` associated type, you must use + // `resolve_with_ctx` here to make the `executor` perform automatic type conversion + // of its argument. + "id" => executor.resolve_with_ctx(&(), &self.id), + "name" => executor.resolve_with_ctx(&(), &self.name), + // You pass a vector of `User` objects to `executor.resolve`, and it will determine + // which fields of the sub-objects to actually resolve based on the query. + // The `executor` instance keeps track of its current position in the query. + // "friends" => executor.resolve(info, + // &self.friend_ids.iter() + // .filter_map(|id| database.users.get(id)) + // .collect::>() + // ), + // We can only reach this panic in two cases: either a mismatch between the defined + // schema in `meta()` above, or a validation failed because of a this library bug. + // + // In either of those two cases, the only reasonable way out is to panic the thread. + _ => panic!("Field {field_name} not found on type User"), + } + } +} diff --git a/crates/subgraph/src/lib.rs b/crates/subgraph/src/lib.rs index 9d0bfa8..b1f41e2 100644 --- a/crates/subgraph/src/lib.rs +++ b/crates/subgraph/src/lib.rs @@ -9,15 +9,16 @@ use { std::sync::Mutex, surfpool_core::{ solana_sdk::{bs58, inner_instruction}, - types::{SubgraphIndexingEvent, SubgraphPluginConfig}, + types::{SchemaDatasourceingEvent, SubgraphPluginConfig}, }, - txtx_addon_network_svm::codec::subgraph::{IndexedSubgraphSourceType, SubgraphRequest}, uuid::Uuid, + txtx_addon_network_svm::codec::subgraph::{IndexedSubgraphSourceType, SubgraphRequest}, + uuid::Uuid, }; #[derive(Default, Debug)] pub struct SurfpoolSubgraph { pub uuid: Uuid, - subgraph_indexing_event_tx: Mutex>>, + subgraph_indexing_event_tx: Mutex>>, subgraph_request: Option, } @@ -30,7 +31,7 @@ impl GeyserPlugin for SurfpoolSubgraph { let config = serde_json::from_str::(&config_file).unwrap(); let oneshot_tx = IpcSender::connect(config.ipc_token).unwrap(); let (tx, rx) = ipc_channel::ipc::channel().unwrap(); - let _ = tx.send(SubgraphIndexingEvent::Rountrip(config.uuid.clone())); + let _ = tx.send(SchemaDatasourceingEvent::Rountrip(config.uuid.clone())); let _ = oneshot_tx.send(rx); self.uuid = config.uuid.clone(); self.subgraph_indexing_event_tx = Mutex::new(Some(tx)); @@ -88,7 +89,7 @@ impl GeyserPlugin for SurfpoolSubgraph { }; match transaction { ReplicaTransactionInfoVersions::V0_0_2(data) => { - let _ = tx.send(SubgraphIndexingEvent::ApplyEntry( + let _ = tx.send(SchemaDatasourceingEvent::ApplyEntry( self.uuid, data.signature.to_string(), // subgraph_request.clone(), @@ -132,7 +133,7 @@ impl GeyserPlugin for SurfpoolSubgraph { "found event with match!!!: {:?}", event_subgraph_source.event.name ); - let _ = tx.send(SubgraphIndexingEvent::ApplyEntry( + let _ = tx.send(SchemaDatasourceingEvent::ApplyEntry( self.uuid, data.signature.to_string(), // subgraph_request.clone(), From 6916f8397e91978a9231fac1862fd0dd1101f1eb Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Sat, 15 Feb 2025 18:51:55 -0500 Subject: [PATCH 18/24] wip --- Cargo.lock | 1 + crates/cli/Cargo.toml | 1 + crates/cli/src/http/mod.rs | 53 ++++----- crates/core/src/simnet/mod.rs | 4 +- crates/core/src/types.rs | 4 +- crates/gql/src/lib.rs | 23 ++-- crates/gql/src/query.rs | 171 ++++++++++++++++++----------- crates/gql/src/subscription.rs | 29 ++++- crates/gql/src/types/collection.rs | 36 ------ crates/gql/src/types/entry.rs | 27 ----- crates/gql/src/types/mod.rs | 4 - crates/gql/src/types/subgraph.rs | 89 --------------- 12 files changed, 179 insertions(+), 263 deletions(-) delete mode 100644 crates/gql/src/types/collection.rs delete mode 100644 crates/gql/src/types/entry.rs delete mode 100644 crates/gql/src/types/mod.rs delete mode 100644 crates/gql/src/types/subgraph.rs diff --git a/Cargo.lock b/Cargo.lock index a9085c5..6e9c3b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9253,6 +9253,7 @@ dependencies = [ "clap 4.5.27", "clap_complete 4.5.44", "clap_generate", + "convert_case 0.7.1", "crossbeam", "crossterm", "ctrlc", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 63df909..8a08fe4 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -49,6 +49,7 @@ juniper_graphql_ws = { version = "0.4.0", features = ["graphql-transport-ws"] } juniper = { version = "0.16.1", features = ["schema-language"] } ipc-channel = { workspace = true } bincode = { workspace = true } +convert_case = "0.7.1" [features] default = ["cli"] diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 3e1c10a..4eaf67e 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -9,15 +9,15 @@ use actix_web::{Error, Responder}; use crossbeam::channel::{Receiver, Select, Sender}; use juniper_actix::{graphiql_handler, graphql_handler, playground_handler, subscriptions}; use juniper_graphql_ws::ConnectionConfig; -use std::collections::BTreeMap; +use surfpool_gql::subscription::EntryData; +use std::collections::{BTreeMap, HashMap}; use std::error::Error as StdError; use std::sync::RwLock; use std::time::Duration; use surfpool_core::types::{ - Collection, Entry, SchemaDatasourceingEvent, SubgraphCommand, SubgraphEvent, SurfpoolConfig, + Entry, SchemaDatasourceingEvent, SubgraphCommand, SubgraphEvent, SurfpoolConfig, }; use surfpool_gql::query::{SchemaDatasource, SchemaDatasourceEntry}; -use surfpool_gql::types::collection::CollectionData; use surfpool_gql::{new_dynamic_schema, Context as GqlContext, GqlDynamicSchema as GqlSchema}; use txtx_core::kit::uuid::Uuid; @@ -64,29 +64,22 @@ pub async fn start_server( Ok(cmd) => match cmd { SubgraphCommand::CreateSubgraph(uuid, config, sender) => { let mut gql_schema = gql_schema_copy.write().unwrap(); + let subgraph_uuid = uuid; let subgraph_name = config.subgraph_name.clone(); - let mut schema = SchemaDatasourceEntry::new(&subgraph_name); + let mut schema = + SchemaDatasourceEntry::new(&subgraph_uuid, &subgraph_name); for fields in config.fields.iter() { schema.fields.push(fields.display_name.clone()); } schema_datasource.add_entry(schema); gql_schema.replace(new_dynamic_schema(schema_datasource.clone())); + use convert_case::{Case, Casing}; let gql_context = gql_context_copy.write().unwrap(); - let mut collections = - gql_context.collections_store.write().unwrap(); - let collection_uuid = uuid; - - collections.insert( - collection_uuid.clone(), - CollectionData { - collection: Collection { - uuid: collection_uuid, - name: config.subgraph_name.clone(), - entries: vec![], - }, - }, - ); + let mut entries_store = gql_context.entries_store.write().unwrap(); + let mut lookup = gql_context.uuid_lookup.write().unwrap(); + lookup.insert(subgraph_uuid.clone(), subgraph_name.to_case(Case::Camel)); + entries_store.insert(subgraph_name.to_case(Case::Camel), (subgraph_uuid.clone(), vec![])); let _ = sender.send("http://127.0.0.1:8900/graphql".into()); } SubgraphCommand::ObserveSubgraph(subgraph_observer_rx) => { @@ -105,13 +98,20 @@ pub async fn start_server( value, /* , request, slot*/ ) => { let gql_context = gql_context_copy.write().unwrap(); - let mut collections = - gql_context.collections_store.write().unwrap(); - let collection_data = collections.get_mut(&uuid).unwrap(); - collection_data.collection.entries.push(Entry { - uuid: Uuid::new_v4(), - value, - }); + let uuid_lookup = gql_context.uuid_lookup.read().unwrap(); + let subgraph_name = uuid_lookup.get(&uuid).unwrap(); + let mut entries_store = gql_context.entries_store.write().unwrap(); + let (_uuid, entries) = entries_store.get_mut(subgraph_name).unwrap(); + let mut values = HashMap::new(); + values.insert("message".to_string(), value); + let entry_uuid = Uuid::new_v4(); + entries.push( + EntryData { + entry: Entry { + uuid: entry_uuid, + values, + }}, + ); } SchemaDatasourceingEvent::Rountrip(uuid) => { println!("Subgraph components initialized {}", uuid); @@ -231,7 +231,8 @@ async fn subscriptions( .read() .map_err(|_| actix_web::error::ErrorInternalServerError("Failed to read context"))?; let ctx = GqlContext { - collections_store: context.collections_store.clone(), + uuid_lookup: context.uuid_lookup.clone(), + entries_store: context.entries_store.clone(), entries_broadcaster: context.entries_broadcaster.clone(), }; let config = ConnectionConfig::new(ctx); diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index 60ee45e..09036fe 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -323,7 +323,7 @@ pub async fn start( match event { PluginManagerCommand::LoadConfig(uuid, config, notifier) => { let _ = subgraph_commands_tx.send(SubgraphCommand::CreateSubgraph(uuid.clone(), config.data.clone(), notifier)); - + let (mut plugin, lib) = unsafe { let lib = match Library::new(&libpath) { Ok(lib) => lib, @@ -338,7 +338,7 @@ pub async fn start( let plugin_raw = constructor(); (Box::from_raw(plugin_raw), lib) }; - + let (server, ipc_token) = IpcOneShotServer::>::new().expect("Failed to create IPC one-shot server."); let subgraph_plugin_config = SubgraphPluginConfig { uuid, diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 9c2bbe7..89f992d 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -11,7 +11,7 @@ use solana_sdk::{ transaction::{SanitizedTransaction, TransactionError, VersionedTransaction}, }; use solana_transaction_status::TransactionConfirmationStatus; -use std::path::PathBuf; +use std::{collections::HashMap, path::PathBuf}; use txtx_addon_network_svm::codec::subgraph::{PluginConfig, SubgraphRequest}; use uuid::Uuid; @@ -32,7 +32,7 @@ pub struct Collection { #[derive(Debug, Clone)] pub struct Entry { pub uuid: Uuid, - pub value: String, + pub values: HashMap, } #[derive(Debug)] diff --git a/crates/gql/src/lib.rs b/crates/gql/src/lib.rs index 9b4a3c0..88628d2 100644 --- a/crates/gql/src/lib.rs +++ b/crates/gql/src/lib.rs @@ -1,20 +1,21 @@ use juniper::RootNode; use mutation::{DynamicMutation, Mutation}; -use query::{DynamicQuery, Query, SchemaDatasource}; +use query::{Query, SchemaDatasource}; use std::sync::RwLock; use std::{collections::BTreeMap, sync::Arc}; -use subscription::{DynamicSubscription, Subscription}; -use types::{collection::CollectionData, entry::EntryData}; +use subscription::{DynamicSubscription, EntryData, Subscription}; +use surfpool_core::types::Entry; use uuid::Uuid; pub mod mutation; pub mod query; pub mod subscription; -pub mod types; +// pub mod types; #[derive(Clone, Debug)] pub struct Context { - pub collections_store: Arc>>, + pub uuid_lookup: Arc>>, + pub entries_store: Arc)>>>, pub entries_broadcaster: tokio::sync::broadcast::Sender, } @@ -22,7 +23,8 @@ impl Context { pub fn new() -> Context { let (entries_broadcaster, _) = tokio::sync::broadcast::channel(128); Context { - collections_store: Arc::new(RwLock::new(BTreeMap::new())), + uuid_lookup: Arc::new(RwLock::new(BTreeMap::new())), + entries_store: Arc::new(RwLock::new(BTreeMap::new())), entries_broadcaster, } } @@ -30,16 +32,11 @@ impl Context { impl juniper::Context for Context {} -pub type GqlStaticSchema = RootNode<'static, Query, Mutation, Subscription>; -pub type GqlDynamicSchema = RootNode<'static, DynamicQuery, Mutation, DynamicSubscription>; - -pub fn new_static_schema() -> GqlStaticSchema { - GqlStaticSchema::new(Query, Mutation, Subscription) -} +pub type GqlDynamicSchema = RootNode<'static, Query, Mutation, DynamicSubscription>; pub fn new_dynamic_schema(subgraph_index: SchemaDatasource) -> GqlDynamicSchema { GqlDynamicSchema::new_with_info( - DynamicQuery::new(), + Query::new(), Mutation, DynamicSubscription, subgraph_index, diff --git a/crates/gql/src/query.rs b/crates/gql/src/query.rs index f016ac1..88418bf 100644 --- a/crates/gql/src/query.rs +++ b/crates/gql/src/query.rs @@ -1,23 +1,14 @@ -use std::{ - borrow::Cow, - collections::{BTreeMap, HashMap}, -}; +use std::collections::HashMap; -use crate::{ - types::{collection::CollectionData, subgraph::Subgraph}, - Context, -}; +use crate::Context; use convert_case::{Case, Casing}; use juniper::{ - meta::{Field, MetaType, ObjectMeta}, - Arguments, DefaultScalarValue, ExecutionResult, Executor, GraphQLType, GraphQLTypeAsync, - GraphQLValue, GraphQLValueAsync, Registry, + meta::{Field, MetaType}, Arguments, BoxFuture, DefaultScalarValue, ExecutionResult, Executor, GraphQLType, GraphQLValue, GraphQLValueAsync, Registry }; use uuid::Uuid; #[derive(Debug)] -pub struct DynamicQuery { - pub subgraphs: HashMap, +pub struct Query { } #[derive(Clone, Debug)] @@ -40,14 +31,16 @@ impl SchemaDatasource { #[derive(Clone, Debug)] pub struct SchemaDatasourceEntry { pub name: String, + pub subgraph_uuid: Uuid, pub description: Option, pub fields: Vec, } impl SchemaDatasourceEntry { - pub fn new(name: &str) -> Self { + pub fn new(uuid: &Uuid, name: &str) -> Self { Self { name: name.to_case(Case::Pascal), + subgraph_uuid: uuid.clone(), description: None, fields: vec![], } @@ -75,7 +68,7 @@ impl GraphQLType for SchemaDatasourceEntry { } impl GraphQLValue for SchemaDatasourceEntry { - type Context = (); + type Context = Context; type TypeInfo = SchemaDatasourceEntry; fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { @@ -87,43 +80,54 @@ impl GraphQLValue for SchemaDatasourceEntry { info: &SchemaDatasourceEntry, field_name: &str, args: &Arguments, - executor: &Executor<()>, + executor: &Executor, ) -> ExecutionResult { // Next, we need to match the queried field name. All arms of this match statement // return `ExecutionResult`, which makes it hard to statically verify that the type you // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` // above. - let database = executor.context(); - match field_name { - // Because scalars are defined with another `Context` associated type, you must use - // `resolve_with_ctx` here to make the `executor` perform automatic type conversion - // of its argument. - // You pass a vector of `User` objects to `executor.resolve`, and it will determine - // which fields of the sub-objects to actually resolve based on the query. - // The `executor` instance keeps track of its current position in the query. - // "friends" => executor.resolve(info, - // &self.friend_ids.iter() - // .filter_map(|id| database.users.get(id)) - // .collect::>() - // ), - // We can only reach this panic in two cases: either a mismatch between the defined - // schema in `meta()` above, or a validation failed because of a this library bug. - // - // In either of those two cases, the only reasonable way out is to panic the thread. - _ => panic!("Field {field_name} not found on type User"), - } + // let database = executor.context(); + // let subgraph_db = database.entries_store.read().unwrap(); + // let entries = subgraph_db.get(&self.subgraph_uuid).unwrap(); + + // match args.get::("uuid")? { + // Some(uuid) => unimplemented!(), + // None => { + // let results = entries + // .iter() + // .filter_map(|(_, e)| e.values.get(field_name).map(|e| e.to_string())) + // .collect::>(); + // println!("{:?}", results); + // unimplemented!() + // // executor.resolve(&info, &results) + // } + // } + unimplemented!() } } -impl DynamicQuery { +// impl GraphQLValueAsync for SchemaDatasourceEntry { +// fn resolve_field_async( +// &self, +// _info: &SchemaDatasourceEntry, +// _field_name: &str, +// _arguments: &Arguments, +// _executor: &Executor, +// ) -> BoxFuture { +// panic!( +// "?", +// ); +// } +// } + +impl Query { pub fn new() -> Self { Self { - subgraphs: HashMap::new(), } } } -impl GraphQLType for DynamicQuery { +impl GraphQLType for Query { fn name(_spec: &SchemaDatasource) -> Option<&str> { Some("Query") } @@ -145,35 +149,62 @@ impl GraphQLType for DynamicQuery { fields.push(registry.field::<&SchemaDatasourceEntry>(name, &entry)); } registry - .build_object_type::(&spec, &fields) + .build_object_type::(&spec, &fields) .into_meta() } } -impl GraphQLValue for DynamicQuery { +impl GraphQLValue for Query { type Context = Context; type TypeInfo = SchemaDatasource; fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { - ::name(&info) + ::name(&info) } fn resolve_field( &self, info: &SchemaDatasource, - field_name: &str, + subgraph_name: &str, args: &Arguments, executor: &Executor, ) -> ExecutionResult { + + // Next, we need to match the queried field name. All arms of this match statement + // return `ExecutionResult`, which makes it hard to statically verify that the type you + // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` + // above. + // let database = executor.context(); + // let subgraph_db = database.entries_store.read().unwrap(); + // let entries = subgraph_db.get(&self.subgraph_uuid).unwrap(); + + // match args.get::("uuid")? { + // Some(uuid) => { + // let results = entries + // .iter() + // .filter_map(|(_, e)| e.values.get(field_name).map(|e| e.to_string())) + // .collect::>(); + // println!("{:?}", results); + // unimplemented!() + // executor.resolve(&info, &results) + // }, + // None => unimplemented!() + // } + + + unimplemented!(); + // Next, we need to match the queried field name. All arms of this match statement // return `ExecutionResult`, which makes it hard to statically verify that the type you // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` // above. - match field_name { + let database = executor.context(); + + // match subgraph_name { // Because scalars are defined with another `Context` associated type, you must use // `resolve_with_ctx` here to make the `executor` perform automatic type conversion // of its argument. - "api_version" => executor.resolve_with_ctx(&(), "1.0"), + // "api_version" => executor.resolve_with_ctx(&(), "1.0"), // subgraph => executor.resolve_with_ctx(&(), &self.name), // You pass a vector of `User` objects to `executor.resolve`, and it will determine // which fields of the sub-objects to actually resolve based on the query. @@ -187,31 +218,45 @@ impl GraphQLValue for DynamicQuery { // schema in `meta()` above, or a validation failed because of a this library bug. // // In either of those two cases, the only reasonable way out is to panic the thread. - _ => panic!("Field {field_name} not found on type DynamicQuery"), - } + // _ => { + // for (name, entry) in info.entries.iter() { + // if name.eq(subgraph_name) { + // return executor.resolve(database, entry) + // } + // } + // panic!("unable to resolve") + + // } + // } } } -impl GraphQLValueAsync for DynamicQuery {} +// impl GraphQLValueAsync for Query {} -use juniper_codegen::graphql_object; +impl GraphQLValueAsync for Query { + fn resolve_field_async( + &self, + _info: &SchemaDatasource, + field_name: &str, + _arguments: &Arguments, + executor: &Executor, + ) -> BoxFuture { -pub struct Query; + println!("{}", field_name); + println!("{:?}", _arguments); -#[graphql_object( - context = Context, -)] -impl Query { - fn api_version() -> &'static str { - "1.0" - } + let res = match field_name { + "apiVersion" => executor.resolve_with_ctx(&(), "1.0"), + subgraph_name => { + let database = executor.context(); + let subgraph_db = database.entries_store.read().unwrap(); + let (_, entries) = subgraph_db.get(subgraph_name).unwrap(); + executor.resolve_with_ctx(&(), entries) + } + }; - async fn collections(context: &Context) -> Vec { - let collections_store = context.collections_store.read().unwrap(); - collections_store - .values() - .into_iter() - .map(|c| c.clone()) - .collect() + Box::pin(async { + res + }) } } diff --git a/crates/gql/src/subscription.rs b/crates/gql/src/subscription.rs index f733b1a..b2372c8 100644 --- a/crates/gql/src/subscription.rs +++ b/crates/gql/src/subscription.rs @@ -1,8 +1,34 @@ use std::pin::Pin; -use crate::{types::entry::EntryData, Context}; +use crate::Context; use futures::Stream; use juniper::{graphql_subscription, FieldError}; +use juniper_codegen::graphql_object; +use surfpool_core::types::Entry; + +#[derive(Debug, Clone)] +pub struct EntryData { + pub entry: Entry, +} + +impl EntryData { + pub fn new(entry: &Entry) -> Self { + Self { + entry: entry.clone(), + } + } +} + +#[graphql_object(context = Context)] +impl EntryData { + pub fn uuid(&self) -> String { + self.entry.uuid.to_string() + } + + pub fn values(&self) -> String { + "values".to_string() + } +} pub struct Subscription; @@ -30,6 +56,7 @@ pub struct DynamicSubscription; #[graphql_subscription( context = Context, + )] impl DynamicSubscription { async fn entries_event(context: &Context) -> GqlEntriesStream { diff --git a/crates/gql/src/types/collection.rs b/crates/gql/src/types/collection.rs deleted file mode 100644 index 2a2b121..0000000 --- a/crates/gql/src/types/collection.rs +++ /dev/null @@ -1,36 +0,0 @@ -use super::entry::EntryData; -use crate::Context; -use juniper_codegen::graphql_object; -use surfpool_core::types::{Collection, Entry}; - -#[derive(Debug, Clone)] -pub struct CollectionData { - pub collection: Collection, -} - -impl CollectionData { - pub fn new(collection: &Collection) -> Self { - Self { - collection: collection.clone(), - } - } -} - -#[graphql_object(context = Context)] -impl CollectionData { - pub fn uuid(&self) -> String { - self.collection.uuid.to_string() - } - - pub fn name(&self) -> String { - self.collection.name.clone() - } - - pub fn entries(&self) -> Vec { - self.collection - .entries - .iter() - .map(|e| EntryData::new(e)) - .collect() - } -} diff --git a/crates/gql/src/types/entry.rs b/crates/gql/src/types/entry.rs deleted file mode 100644 index 61a777e..0000000 --- a/crates/gql/src/types/entry.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::Context; -use juniper_codegen::graphql_object; -use surfpool_core::types::Entry; - -#[derive(Debug, Clone)] -pub struct EntryData { - pub entry: Entry, -} - -impl EntryData { - pub fn new(entry: &Entry) -> Self { - Self { - entry: entry.clone(), - } - } -} - -#[graphql_object(context = Context)] -impl EntryData { - pub fn uuid(&self) -> String { - self.entry.uuid.to_string() - } - - pub fn value(&self) -> String { - self.entry.value.clone() - } -} diff --git a/crates/gql/src/types/mod.rs b/crates/gql/src/types/mod.rs deleted file mode 100644 index 330a50f..0000000 --- a/crates/gql/src/types/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod collection; -pub mod entry; - -pub mod subgraph; diff --git a/crates/gql/src/types/subgraph.rs b/crates/gql/src/types/subgraph.rs deleted file mode 100644 index 1413ba0..0000000 --- a/crates/gql/src/types/subgraph.rs +++ /dev/null @@ -1,89 +0,0 @@ -use juniper::{ - meta::MetaType, Arguments, Context, DefaultScalarValue, ExecutionResult, Executor, GraphQLType, - GraphQLValue, Registry, -}; -use std::collections::HashMap; - -pub struct SubgraphSpecification { - name: String, -} - -#[derive(Debug)] -pub struct Database { - subgraphs: HashMap, -} -impl Context for Database {} - -#[derive(Debug)] -pub struct Subgraph { - id: String, - name: String, - // friend_ids: Vec -} - -impl GraphQLType for Subgraph { - fn name(spec: &SubgraphSpecification) -> Option<&str> { - Some(&spec.name) - } - - fn meta<'r>(spec: &SubgraphSpecification, registry: &mut Registry<'r>) -> MetaType<'r> - where - DefaultScalarValue: 'r, - { - // First, we need to define all fields and their types on this type. - // - // If we need arguments, want to implement interfaces, or want to add documentation - // strings, we can do it here. - let fields = &[ - registry.field::<&String>("id", &()), - registry.field::<&String>("name", &()), - // registry.field::>("friends", &spec), - ]; - registry - .build_object_type::(&spec, fields) - .into_meta() - } -} - -impl GraphQLValue for Subgraph { - type Context = Database; - type TypeInfo = SubgraphSpecification; - - fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { - ::name(&info) - } - - fn resolve_field( - &self, - info: &SubgraphSpecification, - field_name: &str, - args: &Arguments, - executor: &Executor, - ) -> ExecutionResult { - // Next, we need to match the queried field name. All arms of this match statement - // return `ExecutionResult`, which makes it hard to statically verify that the type you - // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` - // above. - let database = executor.context(); - match field_name { - // Because scalars are defined with another `Context` associated type, you must use - // `resolve_with_ctx` here to make the `executor` perform automatic type conversion - // of its argument. - "id" => executor.resolve_with_ctx(&(), &self.id), - "name" => executor.resolve_with_ctx(&(), &self.name), - // You pass a vector of `User` objects to `executor.resolve`, and it will determine - // which fields of the sub-objects to actually resolve based on the query. - // The `executor` instance keeps track of its current position in the query. - // "friends" => executor.resolve(info, - // &self.friend_ids.iter() - // .filter_map(|id| database.users.get(id)) - // .collect::>() - // ), - // We can only reach this panic in two cases: either a mismatch between the defined - // schema in `meta()` above, or a validation failed because of a this library bug. - // - // In either of those two cases, the only reasonable way out is to panic the thread. - _ => panic!("Field {field_name} not found on type User"), - } - } -} From 876e21664bccf3bfb8cfa7e424240f4dd229a0cf Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Sun, 16 Feb 2025 14:35:10 -0600 Subject: [PATCH 19/24] feat: draft resolver --- crates/cli/src/http/mod.rs | 25 +++++++---- crates/gql/src/lib.rs | 7 ++- crates/gql/src/query.rs | 74 +++++++++++++++---------------- crates/gql/src/subscription.rs | 81 ++++++++++++++++++++++++++++++---- 4 files changed, 126 insertions(+), 61 deletions(-) diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 4eaf67e..80a7fe3 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -9,7 +9,6 @@ use actix_web::{Error, Responder}; use crossbeam::channel::{Receiver, Select, Sender}; use juniper_actix::{graphiql_handler, graphql_handler, playground_handler, subscriptions}; use juniper_graphql_ws::ConnectionConfig; -use surfpool_gql::subscription::EntryData; use std::collections::{BTreeMap, HashMap}; use std::error::Error as StdError; use std::sync::RwLock; @@ -18,6 +17,7 @@ use surfpool_core::types::{ Entry, SchemaDatasourceingEvent, SubgraphCommand, SubgraphEvent, SurfpoolConfig, }; use surfpool_gql::query::{SchemaDatasource, SchemaDatasourceEntry}; +use surfpool_gql::subscription::EntryData; use surfpool_gql::{new_dynamic_schema, Context as GqlContext, GqlDynamicSchema as GqlSchema}; use txtx_core::kit::uuid::Uuid; @@ -78,8 +78,14 @@ pub async fn start_server( let gql_context = gql_context_copy.write().unwrap(); let mut entries_store = gql_context.entries_store.write().unwrap(); let mut lookup = gql_context.uuid_lookup.write().unwrap(); - lookup.insert(subgraph_uuid.clone(), subgraph_name.to_case(Case::Camel)); - entries_store.insert(subgraph_name.to_case(Case::Camel), (subgraph_uuid.clone(), vec![])); + lookup.insert( + subgraph_uuid.clone(), + subgraph_name.to_case(Case::Camel), + ); + entries_store.insert( + subgraph_name.to_case(Case::Camel), + (subgraph_uuid.clone(), vec![]), + ); let _ = sender.send("http://127.0.0.1:8900/graphql".into()); } SubgraphCommand::ObserveSubgraph(subgraph_observer_rx) => { @@ -101,17 +107,18 @@ pub async fn start_server( let uuid_lookup = gql_context.uuid_lookup.read().unwrap(); let subgraph_name = uuid_lookup.get(&uuid).unwrap(); let mut entries_store = gql_context.entries_store.write().unwrap(); - let (_uuid, entries) = entries_store.get_mut(subgraph_name).unwrap(); + let (_uuid, entries) = + entries_store.get_mut(subgraph_name).unwrap(); let mut values = HashMap::new(); values.insert("message".to_string(), value); let entry_uuid = Uuid::new_v4(); - entries.push( - EntryData { - entry: Entry { + entries.push(EntryData { + name: subgraph_name.clone(), + entry: Entry { uuid: entry_uuid, values, - }}, - ); + }, + }); } SchemaDatasourceingEvent::Rountrip(uuid) => { println!("Subgraph components initialized {}", uuid); diff --git a/crates/gql/src/lib.rs b/crates/gql/src/lib.rs index 88628d2..33c74c7 100644 --- a/crates/gql/src/lib.rs +++ b/crates/gql/src/lib.rs @@ -1,10 +1,9 @@ use juniper::RootNode; -use mutation::{DynamicMutation, Mutation}; +use mutation::Mutation; use query::{Query, SchemaDatasource}; use std::sync::RwLock; use std::{collections::BTreeMap, sync::Arc}; -use subscription::{DynamicSubscription, EntryData, Subscription}; -use surfpool_core::types::Entry; +use subscription::{DynamicSubscription, EntryData, EntryUpdate}; use uuid::Uuid; pub mod mutation; @@ -16,7 +15,7 @@ pub mod subscription; pub struct Context { pub uuid_lookup: Arc>>, pub entries_store: Arc)>>>, - pub entries_broadcaster: tokio::sync::broadcast::Sender, + pub entries_broadcaster: tokio::sync::broadcast::Sender, } impl Context { diff --git a/crates/gql/src/query.rs b/crates/gql/src/query.rs index 88418bf..4accc9a 100644 --- a/crates/gql/src/query.rs +++ b/crates/gql/src/query.rs @@ -3,13 +3,14 @@ use std::collections::HashMap; use crate::Context; use convert_case::{Case, Casing}; use juniper::{ - meta::{Field, MetaType}, Arguments, BoxFuture, DefaultScalarValue, ExecutionResult, Executor, GraphQLType, GraphQLValue, GraphQLValueAsync, Registry + meta::{Field, MetaType}, + Arguments, BoxFuture, DefaultScalarValue, ExecutionResult, Executor, GraphQLType, GraphQLValue, + GraphQLValueAsync, Registry, }; use uuid::Uuid; #[derive(Debug)] -pub struct Query { -} +pub struct Query {} #[derive(Clone, Debug)] pub struct SchemaDatasource { @@ -122,8 +123,7 @@ impl GraphQLValue for SchemaDatasourceEntry { impl Query { pub fn new() -> Self { - Self { - } + Self {} } } @@ -146,7 +146,7 @@ impl GraphQLType for Query { fields.push(registry.field::<&String>("apiVersion", &())); for (name, entry) in spec.entries.iter() { - fields.push(registry.field::<&SchemaDatasourceEntry>(name, &entry)); + fields.push(registry.field::<&[SchemaDatasourceEntry]>(name, &entry)); } registry .build_object_type::(&spec, &fields) @@ -169,7 +169,6 @@ impl GraphQLValue for Query { args: &Arguments, executor: &Executor, ) -> ExecutionResult { - // Next, we need to match the queried field name. All arms of this match statement // return `ExecutionResult`, which makes it hard to statically verify that the type you // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` @@ -191,7 +190,6 @@ impl GraphQLValue for Query { // None => unimplemented!() // } - unimplemented!(); // Next, we need to match the queried field name. All arms of this match statement @@ -201,32 +199,32 @@ impl GraphQLValue for Query { let database = executor.context(); // match subgraph_name { - // Because scalars are defined with another `Context` associated type, you must use - // `resolve_with_ctx` here to make the `executor` perform automatic type conversion - // of its argument. - // "api_version" => executor.resolve_with_ctx(&(), "1.0"), - // subgraph => executor.resolve_with_ctx(&(), &self.name), - // You pass a vector of `User` objects to `executor.resolve`, and it will determine - // which fields of the sub-objects to actually resolve based on the query. - // The `executor` instance keeps track of its current position in the query. - // "friends" => executor.resolve(info, - // &self.friend_ids.iter() - // .filter_map(|id| database.users.get(id)) - // .collect::>() - // ), - // We can only reach this panic in two cases: either a mismatch between the defined - // schema in `meta()` above, or a validation failed because of a this library bug. - // - // In either of those two cases, the only reasonable way out is to panic the thread. - // _ => { - // for (name, entry) in info.entries.iter() { - // if name.eq(subgraph_name) { - // return executor.resolve(database, entry) - // } - // } - // panic!("unable to resolve") + // Because scalars are defined with another `Context` associated type, you must use + // `resolve_with_ctx` here to make the `executor` perform automatic type conversion + // of its argument. + // "api_version" => executor.resolve_with_ctx(&(), "1.0"), + // subgraph => executor.resolve_with_ctx(&(), &self.name), + // You pass a vector of `User` objects to `executor.resolve`, and it will determine + // which fields of the sub-objects to actually resolve based on the query. + // The `executor` instance keeps track of its current position in the query. + // "friends" => executor.resolve(info, + // &self.friend_ids.iter() + // .filter_map(|id| database.users.get(id)) + // .collect::>() + // ), + // We can only reach this panic in two cases: either a mismatch between the defined + // schema in `meta()` above, or a validation failed because of a this library bug. + // + // In either of those two cases, the only reasonable way out is to panic the thread. + // _ => { + // for (name, entry) in info.entries.iter() { + // if name.eq(subgraph_name) { + // return executor.resolve(database, entry) + // } + // } + // panic!("unable to resolve") - // } + // } // } } } @@ -236,12 +234,11 @@ impl GraphQLValue for Query { impl GraphQLValueAsync for Query { fn resolve_field_async( &self, - _info: &SchemaDatasource, + info: &SchemaDatasource, field_name: &str, _arguments: &Arguments, executor: &Executor, ) -> BoxFuture { - println!("{}", field_name); println!("{:?}", _arguments); @@ -250,13 +247,12 @@ impl GraphQLValueAsync for Query { subgraph_name => { let database = executor.context(); let subgraph_db = database.entries_store.read().unwrap(); + let entry = info.entries.get(subgraph_name).unwrap(); let (_, entries) = subgraph_db.get(subgraph_name).unwrap(); - executor.resolve_with_ctx(&(), entries) + executor.resolve_with_ctx(entry, &entries[..]) } }; - Box::pin(async { - res - }) + Box::pin(async { res }) } } diff --git a/crates/gql/src/subscription.rs b/crates/gql/src/subscription.rs index b2372c8..994022b 100644 --- a/crates/gql/src/subscription.rs +++ b/crates/gql/src/subscription.rs @@ -1,38 +1,101 @@ use std::pin::Pin; -use crate::Context; +use crate::{query::SchemaDatasourceEntry, Context}; use futures::Stream; -use juniper::{graphql_subscription, FieldError}; +use juniper::{ + graphql_subscription, + meta::{Field, MetaType}, + Arguments, DefaultScalarValue, ExecutionResult, Executor, FieldError, GraphQLType, + GraphQLValue, Registry, +}; use juniper_codegen::graphql_object; use surfpool_core::types::Entry; #[derive(Debug, Clone)] pub struct EntryData { pub entry: Entry, + pub name: String, } impl EntryData { - pub fn new(entry: &Entry) -> Self { + pub fn new(name: &String, entry: &Entry) -> Self { Self { entry: entry.clone(), + name: name.clone(), + } + } +} + +impl GraphQLType for EntryData { + fn name(spec: &SchemaDatasourceEntry) -> Option<&str> { + Some(spec.name.as_str()) + } + + fn meta<'r>(spec: &SchemaDatasourceEntry, registry: &mut Registry<'r>) -> MetaType<'r> + where + DefaultScalarValue: 'r, + { + let mut fields: Vec> = vec![]; + fields.push(registry.field::<&String>("uuid", &())); + for field in spec.fields.iter() { + fields.push(registry.field::<&String>(field, &())); + } + registry + .build_object_type::<[EntryData]>(&spec, &fields) + .into_meta() + } +} + +impl GraphQLValue for EntryData { + type Context = Context; + type TypeInfo = SchemaDatasourceEntry; + + fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { + ::name(info) + } + + fn resolve_field( + &self, + info: &SchemaDatasourceEntry, + field_name: &str, + args: &Arguments, + executor: &Executor, + ) -> ExecutionResult { + match field_name { + "uuid" => executor.resolve_with_ctx(&(), &self.entry.uuid.to_string()), + field_name => { + let value = self.entry.values.get(field_name).unwrap(); + executor.resolve_with_ctx(&(), value) + } + } + } +} + +#[derive(Debug, Clone)] +pub struct EntryUpdate { + pub entry: Entry, + pub name: String, +} + +impl EntryUpdate { + pub fn new(name: &String, entry: &Entry) -> Self { + Self { + entry: entry.clone(), + name: name.clone(), } } } #[graphql_object(context = Context)] -impl EntryData { +impl EntryUpdate { pub fn uuid(&self) -> String { self.entry.uuid.to_string() } - - pub fn values(&self) -> String { - "values".to_string() - } } pub struct Subscription; -type GqlEntriesStream = Pin> + Send>>; +type GqlEntriesStream = Pin> + Send>>; #[graphql_subscription( context = Context, From add3bb87c20d2ca411c5d61c35a97801f5148892 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Sun, 16 Feb 2025 16:43:11 -0600 Subject: [PATCH 20/24] chore: clean logs / dead code --- crates/cli/src/http/mod.rs | 5 +- crates/gql/src/query.rs | 122 ------------------------------------- 2 files changed, 1 insertion(+), 126 deletions(-) diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 80a7fe3..57a30cc 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -89,7 +89,6 @@ pub async fn start_server( let _ = sender.send("http://127.0.0.1:8900/graphql".into()); } SubgraphCommand::ObserveSubgraph(subgraph_observer_rx) => { - println!("Observing new graph"); observers.push(subgraph_observer_rx); } SubgraphCommand::Shutdown => { @@ -120,9 +119,7 @@ pub async fn start_server( }, }); } - SchemaDatasourceingEvent::Rountrip(uuid) => { - println!("Subgraph components initialized {}", uuid); - } + SchemaDatasourceingEvent::Rountrip(_uuid) => {} }, Err(_e) => {} }, diff --git a/crates/gql/src/query.rs b/crates/gql/src/query.rs index 4accc9a..7f8f98d 100644 --- a/crates/gql/src/query.rs +++ b/crates/gql/src/query.rs @@ -75,52 +75,8 @@ impl GraphQLValue for SchemaDatasourceEntry { fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { ::name(info) } - - fn resolve_field( - &self, - info: &SchemaDatasourceEntry, - field_name: &str, - args: &Arguments, - executor: &Executor, - ) -> ExecutionResult { - // Next, we need to match the queried field name. All arms of this match statement - // return `ExecutionResult`, which makes it hard to statically verify that the type you - // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` - // above. - // let database = executor.context(); - // let subgraph_db = database.entries_store.read().unwrap(); - // let entries = subgraph_db.get(&self.subgraph_uuid).unwrap(); - - // match args.get::("uuid")? { - // Some(uuid) => unimplemented!(), - // None => { - // let results = entries - // .iter() - // .filter_map(|(_, e)| e.values.get(field_name).map(|e| e.to_string())) - // .collect::>(); - // println!("{:?}", results); - // unimplemented!() - // // executor.resolve(&info, &results) - // } - // } - unimplemented!() - } } -// impl GraphQLValueAsync for SchemaDatasourceEntry { -// fn resolve_field_async( -// &self, -// _info: &SchemaDatasourceEntry, -// _field_name: &str, -// _arguments: &Arguments, -// _executor: &Executor, -// ) -> BoxFuture { -// panic!( -// "?", -// ); -// } -// } - impl Query { pub fn new() -> Self { Self {} @@ -136,13 +92,7 @@ impl GraphQLType for Query { where DefaultScalarValue: 'r, { - // First, we need to define all fields and their types on this type. - // - // If we need arguments, want to implement interfaces, or want to add documentation - // strings, we can do it here. let mut fields = vec![]; - fields.push(registry.field::<&String>("id", &())); - fields.push(registry.field::<&String>("name", &())); fields.push(registry.field::<&String>("apiVersion", &())); for (name, entry) in spec.entries.iter() { @@ -161,76 +111,8 @@ impl GraphQLValue for Query { fn type_name<'i>(&self, info: &'i Self::TypeInfo) -> Option<&'i str> { ::name(&info) } - - fn resolve_field( - &self, - info: &SchemaDatasource, - subgraph_name: &str, - args: &Arguments, - executor: &Executor, - ) -> ExecutionResult { - // Next, we need to match the queried field name. All arms of this match statement - // return `ExecutionResult`, which makes it hard to statically verify that the type you - // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` - // above. - // let database = executor.context(); - // let subgraph_db = database.entries_store.read().unwrap(); - // let entries = subgraph_db.get(&self.subgraph_uuid).unwrap(); - - // match args.get::("uuid")? { - // Some(uuid) => { - // let results = entries - // .iter() - // .filter_map(|(_, e)| e.values.get(field_name).map(|e| e.to_string())) - // .collect::>(); - // println!("{:?}", results); - // unimplemented!() - // executor.resolve(&info, &results) - // }, - // None => unimplemented!() - // } - - unimplemented!(); - - // Next, we need to match the queried field name. All arms of this match statement - // return `ExecutionResult`, which makes it hard to statically verify that the type you - // pass on to `executor.resolve*` actually matches the one that you defined in `meta()` - // above. - let database = executor.context(); - - // match subgraph_name { - // Because scalars are defined with another `Context` associated type, you must use - // `resolve_with_ctx` here to make the `executor` perform automatic type conversion - // of its argument. - // "api_version" => executor.resolve_with_ctx(&(), "1.0"), - // subgraph => executor.resolve_with_ctx(&(), &self.name), - // You pass a vector of `User` objects to `executor.resolve`, and it will determine - // which fields of the sub-objects to actually resolve based on the query. - // The `executor` instance keeps track of its current position in the query. - // "friends" => executor.resolve(info, - // &self.friend_ids.iter() - // .filter_map(|id| database.users.get(id)) - // .collect::>() - // ), - // We can only reach this panic in two cases: either a mismatch between the defined - // schema in `meta()` above, or a validation failed because of a this library bug. - // - // In either of those two cases, the only reasonable way out is to panic the thread. - // _ => { - // for (name, entry) in info.entries.iter() { - // if name.eq(subgraph_name) { - // return executor.resolve(database, entry) - // } - // } - // panic!("unable to resolve") - - // } - // } - } } -// impl GraphQLValueAsync for Query {} - impl GraphQLValueAsync for Query { fn resolve_field_async( &self, @@ -239,9 +121,6 @@ impl GraphQLValueAsync for Query { _arguments: &Arguments, executor: &Executor, ) -> BoxFuture { - println!("{}", field_name); - println!("{:?}", _arguments); - let res = match field_name { "apiVersion" => executor.resolve_with_ctx(&(), "1.0"), subgraph_name => { @@ -252,7 +131,6 @@ impl GraphQLValueAsync for Query { executor.resolve_with_ctx(entry, &entries[..]) } }; - Box::pin(async { res }) } } From 14ddb1e12ca4735034244bcf159f336607d93f2d Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Sun, 16 Feb 2025 20:13:32 -0600 Subject: [PATCH 21/24] chore: update crates --- Cargo.lock | 4 +++- Cargo.toml | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e9c3b5..0c7c81a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10096,6 +10096,8 @@ dependencies = [ [[package]] name = "txtx-addon-kit" version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75191c35eec95ea5087cbf74ebfdbff8ebf1496083afa0392db3061f580d6fd1" dependencies = [ "crossbeam-channel", "dirs 5.0.1", @@ -10126,7 +10128,7 @@ dependencies = [ [[package]] name = "txtx-addon-network-svm" -version = "0.1.3" +version = "0.1.4" dependencies = [ "anchor-lang-idl", "async-recursion", diff --git a/Cargo.toml b/Cargo.toml index f7af5ea..b63beee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,9 +57,9 @@ ipc-channel = "0.19.0" serde = "1.0.217" serde_derive = "1.0.217" # must match the serde version, see https://github.com/serde-rs/serde/issues/2584#issuecomment-1685252251 serde_json = "1.0.135" -txtx-core = { path = "../txtx/crates/txtx-core" } -txtx-addon-network-svm = { package = "txtx-addon-network-svm", path = "../txtx/addons/svm" } +# txtx-core = { path = "../txtx/crates/txtx-core" } +# txtx-addon-network-svm = { package = "txtx-addon-network-svm", path = "../txtx/addons/svm" } +txtx-core = { version = "0.2.3" } +txtx-addon-network-svm = { version = "0.1.4" } bincode = "1.3.3" uuid = "1.7.0" -# txtx-core = { version = "0.2.2" } -# txtx-addon-network-svm = { version = "0.1.3" } From 24be23b8446b834f09b765509851b1e84c1a0b4a Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 17 Feb 2025 07:40:39 -0600 Subject: [PATCH 22/24] fix: reconciliate main --- Cargo.lock | 4 ++++ crates/core/src/rpc/full.rs | 10 ++++++---- crates/core/src/test_helpers.rs | 18 +++++++----------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 829f3eb..54b9c20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10163,6 +10163,8 @@ dependencies = [ [[package]] name = "txtx-addon-network-svm" version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d9ba755577d4b8906d8bac43c82e072ab0aeeb19767aa6fdad8609dec5a62f" dependencies = [ "anchor-lang-idl", "async-recursion", @@ -10184,6 +10186,8 @@ dependencies = [ [[package]] name = "txtx-core" version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e853497b765475d05d5bccefdf3eeda9fe87685632a415ebb842fd4764b7dd9" dependencies = [ "base64 0.22.1", "better-debug", diff --git a/crates/core/src/rpc/full.rs b/crates/core/src/rpc/full.rs index cb2b9c9..46eb57f 100644 --- a/crates/core/src/rpc/full.rs +++ b/crates/core/src/rpc/full.rs @@ -1,6 +1,6 @@ +use super::utils::{decode_and_deserialize, transform_tx_metadata_to_ui_accounts}; use crate::simnet::{EntryStatus, TransactionWithStatusMeta}; use crate::types::TransactionStatusEvent; -use super::utils::{decode_and_deserialize, transform_tx_metadata_to_ui_accounts}; use itertools::Itertools; use jsonrpc_core::futures::future::{self, join_all}; use jsonrpc_core::BoxFuture; @@ -938,12 +938,14 @@ mod tests { .unwrap(); match mempool_rx.recv() { - Ok((_hash, _tx, status_tx)) => { + Ok(SimnetCommand::TransactionReceived(_, _, status_tx, _)) => { status_tx - .send(TransactionConfirmationStatus::Confirmed) + .send(TransactionStatusEvent::Success( + TransactionConfirmationStatus::Confirmed, + )) .unwrap(); } - Err(_) => panic!("failed to receive transaction from mempool"), + _ => panic!("failed to receive transaction from mempool"), } assert_eq!( diff --git a/crates/core/src/test_helpers.rs b/crates/core/src/test_helpers.rs index c6f39f4..75d1e25 100644 --- a/crates/core/src/test_helpers.rs +++ b/crates/core/src/test_helpers.rs @@ -17,6 +17,7 @@ use solana_transaction_status::TransactionConfirmationStatus; use crate::{ rpc::RunloopContext, simnet::{EntryStatus, GlobalState, TransactionWithStatusMeta}, + types::SimnetCommand, }; pub struct TestSetup { @@ -26,7 +27,8 @@ pub struct TestSetup { impl TestSetup { pub fn new(rpc: T) -> Self { - let (mempool_tx, _rx) = crossbeam_channel::unbounded(); + let (simnet_commands_tx, _rx) = crossbeam_channel::unbounded(); + let (plugin_manager_commands_tx, _rx) = crossbeam_channel::unbounded(); let mut svm = LiteSVM::new(); let clock = Clock { @@ -40,7 +42,8 @@ impl TestSetup { TestSetup { context: RunloopContext { - mempool_tx, + simnet_commands_tx, + plugin_manager_commands_tx, id: Hash::new_unique(), state: Arc::new(RwLock::new(GlobalState { svm, @@ -74,16 +77,9 @@ impl TestSetup { setup } - pub fn new_with_mempool( - rpc: T, - mempool_tx: Sender<( - Hash, - VersionedTransaction, - Sender, - )>, - ) -> Self { + pub fn new_with_mempool(rpc: T, simnet_commands_tx: Sender) -> Self { let mut setup = TestSetup::new(rpc); - setup.context.mempool_tx = mempool_tx; + setup.context.simnet_commands_tx = simnet_commands_tx; setup } From c30bc7a7f3e3900e79ca4fb3cf32dfcb8fcafa18 Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 17 Feb 2025 08:09:54 -0600 Subject: [PATCH 23/24] fix: merge conflicts --- crates/cli/src/cli/simnet/mod.rs | 2 +- crates/cli/src/tui/simnet.rs | 3 -- crates/core/src/simnet/mod.rs | 64 +++++++------------------------- crates/core/src/types.rs | 18 ++++++--- 4 files changed, 27 insertions(+), 60 deletions(-) diff --git a/crates/cli/src/cli/simnet/mod.rs b/crates/cli/src/cli/simnet/mod.rs index 9aa75f7..cd20cbe 100644 --- a/crates/cli/src/cli/simnet/mod.rs +++ b/crates/cli/src/cli/simnet/mod.rs @@ -354,7 +354,7 @@ fn log_events( SimnetEvent::WarnLog(_dt, log) => { warn!(ctx.expect_logger(), "{}", log); } - SimnetEvent::TransactionSimulated(_dt, transaction) => { + SimnetEvent::TransactionReceived(_dt, transaction) => { if deployment_completed { info!( ctx.expect_logger(), diff --git a/crates/cli/src/tui/simnet.rs b/crates/cli/src/tui/simnet.rs index 4e1c82c..ca86241 100644 --- a/crates/cli/src/tui/simnet.rs +++ b/crates/cli/src/tui/simnet.rs @@ -278,9 +278,6 @@ fn run_app(terminal: &mut Terminal, mut app: App) -> io::Result<( SimnetEvent::WarnLog(dt, log) => { app.events.push_front((EventType::Warning, dt, log)); } - SimnetEvent::TransactionSimulated(_dt, _transaction) => { - app.successful_transactions += 1; - } SimnetEvent::TransactionReceived(_dt, _transaction) => {} SimnetEvent::TransactionProcessed(_dt, _meta, _err) => { app.successful_transactions += 1; diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index 0077fe2..fd548c2 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -186,46 +186,6 @@ impl EntryStatus { } } -#[derive(Debug)] -pub enum SimnetEvent { - Ready, - Aborted(String), - Shutdown, - ClockUpdate(Clock), - EpochInfoUpdate(EpochInfo), - BlockHashExpired, - InfoLog(DateTime, String), - ErrorLog(DateTime, String), - WarnLog(DateTime, String), - DebugLog(DateTime, String), - TransactionReceived(DateTime, VersionedTransaction), - TransactionProcessed( - DateTime, - TransactionMetadata, - Option, - ), - AccountUpdate(DateTime, Pubkey), -} - -pub enum SimnetCommand { - SlotForward, - SlotBackward, - UpdateClock(ClockCommand), - UpdateRunloopMode(RunloopTriggerMode), -} - -pub enum ClockCommand { - Pause, - Resume, - Toggle, - UpdateSlotInterval(u64), -} - -pub enum ClockEvent { - Tick, - ExpireBlockHash, -} - type PluginConstructor = unsafe fn() -> *mut dyn GeyserPlugin; use libloading::{Library, Symbol}; @@ -252,6 +212,15 @@ pub async fn start( // Todo: should check config first let rpc_client = Arc::new(RpcClient::new(config.simnet.remote_rpc_url.clone())); let epoch_info = rpc_client.get_epoch_info().await?; + // let epoch_info = EpochInfo { + // epoch: 0, + // slot_index: 0, + // slots_in_epoch: 0, + // absolute_slot: 0, + // block_height: 0, + // transaction_count: None, + // }; + // Question: can the value `slots_in_epoch` fluctuate over time? let slots_in_epoch = epoch_info.slots_in_epoch; @@ -546,11 +515,6 @@ pub async fn start( // Handle the transactions accumulated for (key, transaction, status_tx, tx_config) in transactions_to_process.drain(..) { - let _ = simnet_events_tx.try_send(SimnetEvent::TransactionSimulated( - Local::now(), - transaction.clone(), - )); - transaction.verify_with_results(); let transaction = transaction.into_legacy_transaction().unwrap(); let message = &transaction.message; @@ -627,11 +591,11 @@ pub async fn start( let _ = status_tx.try_send(TransactionStatusEvent::Success( TransactionConfirmationStatus::Processed, )); - let _ = simnet_events_tx.try_send(SimnetEvent::TransactionProcessed( - Local::now(), - meta, - err, - )); + let _ = simnet_events_tx.try_send(SimnetEvent::TransactionProcessed( + Local::now(), + meta, + err, + )); } transactions_processed.push((key, transaction, status_tx)); num_transactions += 1; diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index 21c896e..c5176fc 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -15,6 +15,8 @@ use std::{collections::HashMap, path::PathBuf}; use txtx_addon_network_svm::codec::subgraph::{PluginConfig, SubgraphRequest}; use uuid::Uuid; +pub const DEFAULT_RPC_URL: &str = "https://api.mainnet-beta.solana.com"; + #[derive(Clone, Debug, PartialEq, Eq, Default)] pub enum RunloopTriggerMode { #[default] @@ -72,7 +74,12 @@ pub enum SimnetEvent { WarnLog(DateTime, String), DebugLog(DateTime, String), PluginLoaded(String), - TransactionSimulated(DateTime, VersionedTransaction), + TransactionReceived(DateTime, VersionedTransaction), + TransactionProcessed( + DateTime, + TransactionMetadata, + Option, + ), AccountUpdate(DateTime, Pubkey), } @@ -131,7 +138,7 @@ pub struct SimnetConfig { impl Default for SimnetConfig { fn default() -> Self { Self { - remote_rpc_url: "https://api.mainnet-beta.solana.com".to_string(), + remote_rpc_url: DEFAULT_RPC_URL.to_string(), slot_time: 0, runloop_trigger_mode: RunloopTriggerMode::Clock, airdrop_addresses: vec![], @@ -140,7 +147,7 @@ impl Default for SimnetConfig { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct SubgraphConfig {} #[derive(Clone, Debug)] @@ -163,11 +170,10 @@ pub struct SubgraphPluginConfig { pub subgraph_request: SubgraphRequest, } - - impl Default for RpcConfig { +impl Default for RpcConfig { fn default() -> Self { Self { - remote_rpc_url: "https://api.mainnet-beta.solana.com".to_string(), + remote_rpc_url: DEFAULT_RPC_URL.to_string(), bind_host: "127.0.0.1".to_string(), bind_port: 8899, } From 033a4ce9e1cf2321d656816fafb9a5de911a03ae Mon Sep 17 00:00:00 2001 From: Ludo Galabru Date: Mon, 17 Feb 2025 08:29:51 -0600 Subject: [PATCH 24/24] fix: build warnings --- Cargo.lock | 8 +++---- Cargo.toml | 2 +- crates/cli/src/http/mod.rs | 5 ++-- crates/core/src/lib.rs | 4 +++- crates/core/src/rpc/full.rs | 5 ++-- crates/core/src/simnet/mod.rs | 3 ++- crates/core/src/test_helpers.rs | 10 +++----- crates/core/src/types.rs | 5 ++-- crates/core/tests/integration.rs | 39 ++++++++++++++++++++++++-------- crates/gql/src/subscription.rs | 4 ++-- crates/subgraph/src/lib.rs | 6 ++--- 11 files changed, 55 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35bd759..96f8a6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9243,7 +9243,7 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "surfpool-cli" -version = "0.1.7" +version = "0.1.8" dependencies = [ "actix-cors", "actix-web", @@ -9284,7 +9284,7 @@ dependencies = [ [[package]] name = "surfpool-core" -version = "0.1.7" +version = "0.1.8" dependencies = [ "agave-geyser-plugin-interface", "base64 0.22.1", @@ -9353,7 +9353,7 @@ dependencies = [ [[package]] name = "surfpool-gql" -version = "0.1.7" +version = "0.1.8" dependencies = [ "async-stream", "convert_case 0.7.1", @@ -9368,7 +9368,7 @@ dependencies = [ [[package]] name = "surfpool-subgraph" -version = "0.1.7" +version = "0.1.8" dependencies = [ "agave-geyser-plugin-interface", "bincode", diff --git a/Cargo.toml b/Cargo.toml index b63beee..33d6761 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.1.7" +version = "0.1.8" edition = "2021" description = "Surfpool is the best place to train before surfing Solana." license = "Apache-2.0" diff --git a/crates/cli/src/http/mod.rs b/crates/cli/src/http/mod.rs index 57a30cc..2a687d7 100644 --- a/crates/cli/src/http/mod.rs +++ b/crates/cli/src/http/mod.rs @@ -9,7 +9,7 @@ use actix_web::{Error, Responder}; use crossbeam::channel::{Receiver, Select, Sender}; use juniper_actix::{graphiql_handler, graphql_handler, playground_handler, subscriptions}; use juniper_graphql_ws::ConnectionConfig; -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::error::Error as StdError; use std::sync::RwLock; use std::time::Duration; @@ -31,7 +31,7 @@ pub struct Asset; pub async fn start_server( network_binding: String, - config: SurfpoolConfig, + _config: SurfpoolConfig, subgraph_events_tx: Sender, subgraph_commands_rx: Receiver, _ctx: &Context, @@ -125,6 +125,7 @@ pub async fn start_server( }, } } + #[allow(unreachable_code)] Ok::<(), String>(()) }) .map_err(|e| format!("{}", e))?; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index ebfc3e5..ab4ff5a 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -12,7 +12,6 @@ extern crate serde_json; pub mod rpc; pub mod simnet; -mod test_helpers; pub mod types; use crossbeam_channel::{Receiver, Sender}; @@ -39,3 +38,6 @@ pub async fn start_simnet( ) .await } + +#[cfg(test)] +mod test_helpers; diff --git a/crates/core/src/rpc/full.rs b/crates/core/src/rpc/full.rs index 46eb57f..bbd83dd 100644 --- a/crates/core/src/rpc/full.rs +++ b/crates/core/src/rpc/full.rs @@ -766,9 +766,8 @@ mod tests { transaction::{Legacy, TransactionVersion}, }; use solana_transaction_status::{ - option_serializer::OptionSerializer, EncodedTransaction, EncodedTransactionWithStatusMeta, - UiCompiledInstruction, UiMessage, UiRawMessage, UiReturnDataEncoding, UiTransaction, - UiTransactionReturnData, UiTransactionStatusMeta, + EncodedTransaction, EncodedTransactionWithStatusMeta, UiCompiledInstruction, UiMessage, + UiRawMessage, UiTransaction, }; use test_case::test_case; diff --git a/crates/core/src/simnet/mod.rs b/crates/core/src/simnet/mod.rs index fd548c2..fdf962a 100644 --- a/crates/core/src/simnet/mod.rs +++ b/crates/core/src/simnet/mod.rs @@ -30,6 +30,7 @@ use solana_transaction_status::{ UiInstruction, UiMessage, UiRawMessage, UiReturnDataEncoding, UiTransaction, UiTransactionReturnData, UiTransactionStatusMeta, }; +use std::path::PathBuf; use std::{ collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, @@ -37,7 +38,6 @@ use std::{ thread::sleep, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use std::{fs::File, io::Read, path::PathBuf}; use crate::{ rpc::{ @@ -406,6 +406,7 @@ pub async fn start( } } } + #[allow(unreachable_code)] Ok::<(), String>(()) }); } diff --git a/crates/core/src/test_helpers.rs b/crates/core/src/test_helpers.rs index 75d1e25..c4fd850 100644 --- a/crates/core/src/test_helpers.rs +++ b/crates/core/src/test_helpers.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + use std::{ collections::{HashMap, VecDeque}, sync::{Arc, RwLock}, @@ -6,13 +8,7 @@ use std::{ use crossbeam_channel::Sender; use litesvm::LiteSVM; use solana_client::nonblocking::rpc_client::RpcClient; -use solana_sdk::{ - blake3::Hash, - clock::Clock, - epoch_info::EpochInfo, - transaction::{Transaction, VersionedTransaction}, -}; -use solana_transaction_status::TransactionConfirmationStatus; +use solana_sdk::{blake3::Hash, clock::Clock, epoch_info::EpochInfo, transaction::Transaction}; use crate::{ rpc::RunloopContext, diff --git a/crates/core/src/types.rs b/crates/core/src/types.rs index c5176fc..72ab6a2 100644 --- a/crates/core/src/types.rs +++ b/crates/core/src/types.rs @@ -1,14 +1,13 @@ -use agave_geyser_plugin_interface::geyser_plugin_interface::ReplicaTransactionInfoVersions; use chrono::{DateTime, Local}; use crossbeam_channel::{Receiver, Sender}; use litesvm::types::TransactionMetadata; use solana_client::rpc_config::RpcSendTransactionConfig; use solana_sdk::{ blake3::Hash, - clock::{Clock, Slot}, + clock::Clock, epoch_info::EpochInfo, pubkey::Pubkey, - transaction::{SanitizedTransaction, TransactionError, VersionedTransaction}, + transaction::{TransactionError, VersionedTransaction}, }; use solana_transaction_status::TransactionConfirmationStatus; use std::{collections::HashMap, path::PathBuf}; diff --git a/crates/core/tests/integration.rs b/crates/core/tests/integration.rs index 3d7bacc..0a42f44 100644 --- a/crates/core/tests/integration.rs +++ b/crates/core/tests/integration.rs @@ -18,10 +18,10 @@ use solana_sdk::{ use std::{str::FromStr, time::Duration}; use surfpool_core::{ rpc::{full::FullClient, minimal::MinimalClient}, - simnet::{start, SimnetEvent}, - types::{RpcConfig, RunloopTriggerMode, SimnetConfig, SurfpoolConfig}, + simnet::start, + types::{RpcConfig, RunloopTriggerMode, SimnetConfig, SimnetEvent, SurfpoolConfig}, }; -use tokio::{task, time::sleep}; +use tokio::task; mod helpers; @@ -36,10 +36,17 @@ async fn test_simnet_ready() { }; let (simnet_events_tx, simnet_events_rx) = unbounded(); - let (_simnet_commands_tx, simnet_commands_rx) = unbounded(); + let (simnet_commands_tx, simnet_commands_rx) = unbounded(); + let (subgraph_commands_tx, _subgraph_commands_rx) = unbounded(); let _handle = hiro_system_kit::thread_named("test").spawn(move || { - let future = start(config, simnet_events_tx, simnet_commands_rx); + let future = start( + config, + subgraph_commands_tx, + simnet_events_tx, + simnet_commands_tx, + simnet_commands_rx, + ); if let Err(e) = hiro_system_kit::nestable_block_on(future) { panic!("{e:?}"); } @@ -62,11 +69,18 @@ async fn test_simnet_ticks() { }; let (simnet_events_tx, simnet_events_rx) = unbounded(); - let (_simnet_commands_tx, simnet_commands_rx) = unbounded(); + let (simnet_commands_tx, simnet_commands_rx) = unbounded(); + let (subgraph_commands_tx, _subgraph_commands_rx) = unbounded(); let (test_tx, test_rx) = unbounded(); let _handle = hiro_system_kit::thread_named("test").spawn(move || { - let future = start(config, simnet_events_tx, simnet_commands_rx); + let future = start( + config, + subgraph_commands_tx, + simnet_events_tx, + simnet_commands_tx, + simnet_commands_rx, + ); if let Err(e) = hiro_system_kit::nestable_block_on(future) { panic!("{e:?}"); } @@ -116,10 +130,17 @@ async fn test_simnet_some_sol_transfers() { }; let (simnet_events_tx, simnet_events_rx) = unbounded(); - let (_simnet_commands_tx, simnet_commands_rx) = unbounded(); + let (simnet_commands_tx, simnet_commands_rx) = unbounded(); + let (subgraph_commands_tx, _subgraph_commands_rx) = unbounded(); let _handle = hiro_system_kit::thread_named("test").spawn(move || { - let future = start(config, simnet_events_tx, simnet_commands_rx); + let future = start( + config, + subgraph_commands_tx, + simnet_events_tx, + simnet_commands_tx, + simnet_commands_rx, + ); if let Err(e) = hiro_system_kit::nestable_block_on(future) { panic!("{e:?}"); } diff --git a/crates/gql/src/subscription.rs b/crates/gql/src/subscription.rs index 994022b..5f19606 100644 --- a/crates/gql/src/subscription.rs +++ b/crates/gql/src/subscription.rs @@ -56,9 +56,9 @@ impl GraphQLValue for EntryData { fn resolve_field( &self, - info: &SchemaDatasourceEntry, + _info: &SchemaDatasourceEntry, field_name: &str, - args: &Arguments, + _args: &Arguments, executor: &Executor, ) -> ExecutionResult { match field_name { diff --git a/crates/subgraph/src/lib.rs b/crates/subgraph/src/lib.rs index b1f41e2..c9ba442 100644 --- a/crates/subgraph/src/lib.rs +++ b/crates/subgraph/src/lib.rs @@ -8,7 +8,7 @@ use { solana_program::clock::Slot, std::sync::Mutex, surfpool_core::{ - solana_sdk::{bs58, inner_instruction}, + solana_sdk::bs58, types::{SchemaDatasourceingEvent, SubgraphPluginConfig}, }, txtx_addon_network_svm::codec::subgraph::{IndexedSubgraphSourceType, SubgraphRequest}, @@ -78,7 +78,7 @@ impl GeyserPlugin for SurfpoolSubgraph { fn notify_transaction( &self, transaction: ReplicaTransactionInfoVersions, - slot: Slot, + _slot: Slot, ) -> PluginResult<()> { let Ok(tx) = self.subgraph_indexing_event_tx.lock() else { return Ok(()); @@ -119,7 +119,7 @@ impl GeyserPlugin for SurfpoolSubgraph { for field in subgraph_request.fields.iter() { match &field.data_source { IndexedSubgraphSourceType::Instruction( - instruction_subgraph_source, + _instruction_subgraph_source, ) => { continue; }