Skip to content

Commit

Permalink
Add V2 API endpoint for copying tool assets between playgrounds and t…
Browse files Browse the repository at this point in the history
…ool storage. Auto copy from playground on save
  • Loading branch information
acedward committed Feb 26, 2025
1 parent ce033cf commit 875824e
Show file tree
Hide file tree
Showing 4 changed files with 334 additions and 50 deletions.
24 changes: 24 additions & 0 deletions shinkai-bin/shinkai-node/src/network/handle_commands_list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3053,6 +3053,30 @@ impl Node {
let _ = Node::v2_api_set_tool_enabled(db_clone, bearer, tool_router_key, enabled, res).await;
});
}
NodeCommand::V2ApiCopyToolAssets {
bearer,
is_first_playground,
first_path,
is_second_playground,
second_path,
res,
} => {
let db_clone = Arc::clone(&self.db);
let node_env = fetch_node_environment();
tokio::spawn(async move {
let _ = Node::v2_api_copy_tool_assets(
db_clone,
bearer,
node_env,
is_first_playground,
first_path,
is_second_playground,
second_path,
res,
)
.await;
});
}
_ => (),
}
}
Expand Down
249 changes: 207 additions & 42 deletions shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
use crate::{
llm_provider::job_manager::JobManager, managers::{tool_router, IdentityManager}, network::{node_error::NodeError, node_shareable_logic::download_zip_file, Node}, tools::{
tool_definitions::definition_generation::{generate_tool_definitions, get_all_deno_tools}, tool_execution::execution_coordinator::{execute_code, execute_tool_cmd}, tool_generation::v2_create_and_send_job_message, tool_prompts::{generate_code_prompt, tool_metadata_implementation_prompt}
}, utils::environment::NodeEnvironment
llm_provider::job_manager::JobManager,
managers::IdentityManager,
network::{node_error::NodeError, node_shareable_logic::download_zip_file, Node},
tools::{
tool_definitions::definition_generation::{generate_tool_definitions, get_all_deno_tools},
tool_execution::execution_coordinator::{execute_code, execute_tool_cmd},
tool_generation::v2_create_and_send_job_message,
tool_prompts::{generate_code_prompt, tool_metadata_implementation_prompt},
},
utils::environment::NodeEnvironment,
};
use async_channel::Sender;
use chrono::Utc;
Expand All @@ -11,26 +18,44 @@ use serde_json::{json, Map, Value};
use shinkai_http_api::node_api_router::{APIError, SendResponseBodyData};
use shinkai_message_primitives::{
schemas::{
inbox_name::InboxName, indexable_version::IndexableVersion, job::JobLike, job_config::JobConfig, shinkai_name::ShinkaiSubidentityType, tool_router_key::ToolRouterKey
}, shinkai_message::shinkai_message_schemas::{CallbackAction, JobCreationInfo, MessageSchemaType}, shinkai_utils::{
job_scope::MinimalJobScope, shinkai_message_builder::ShinkaiMessageBuilder, signatures::clone_signature_secret_key
}
inbox_name::InboxName, indexable_version::IndexableVersion, job::JobLike, job_config::JobConfig,
shinkai_name::ShinkaiSubidentityType, tool_router_key::ToolRouterKey,
},
shinkai_message::shinkai_message_schemas::{CallbackAction, JobCreationInfo, MessageSchemaType},
shinkai_utils::{
job_scope::MinimalJobScope, shinkai_message_builder::ShinkaiMessageBuilder,
signatures::clone_signature_secret_key,
},
};
use shinkai_message_primitives::{
schemas::{
shinkai_name::ShinkaiName, shinkai_tools::{CodeLanguage, DynamicToolType}
}, shinkai_message::shinkai_message_schemas::JobMessage
shinkai_name::ShinkaiName,
shinkai_tools::{CodeLanguage, DynamicToolType},
},
shinkai_message::shinkai_message_schemas::JobMessage,
};
use shinkai_sqlite::{errors::SqliteManagerError, SqliteManager};
use shinkai_tools_primitives::tools::{
deno_tools::DenoTool, error::ToolError, python_tools::PythonTool, shinkai_tool::{ShinkaiTool, ShinkaiToolWithAssets}, tool_config::{OAuth, ToolConfig}, tool_output_arg::ToolOutputArg, tool_playground::{ToolPlayground, ToolPlaygroundMetadata}
deno_tools::DenoTool,
error::ToolError,
python_tools::PythonTool,
shinkai_tool::{ShinkaiTool, ShinkaiToolWithAssets},
tool_config::{OAuth, ToolConfig},
tool_output_arg::ToolOutputArg,
tool_playground::{ToolPlayground, ToolPlaygroundMetadata},
};
use shinkai_tools_primitives::tools::{
shinkai_tool::ShinkaiToolHeader, tool_types::{OperatingSystem, RunnerType, ToolResult}
shinkai_tool::ShinkaiToolHeader,
tool_types::{OperatingSystem, RunnerType, ToolResult},
};
use std::path::PathBuf;
use std::{
collections::HashMap, env, fs::File, io::{Read, Write}, result, sync::Arc, time::Instant
collections::HashMap,
env,
fs::File,
io::{Read, Write},
sync::Arc,
time::Instant,
};
use tokio::fs;
use tokio::{process::Command, sync::Mutex};
Expand Down Expand Up @@ -585,50 +610,61 @@ impl Node {
origin_path.push(".tools_storage");
origin_path.push("playground");
origin_path.push(app_id);
if let Some(assets) = payload.assets.clone() {
for file_name in assets {
let mut asset_path: PathBuf = origin_path.clone();
asset_path.push(file_name.clone());
if !asset_path.exists() {
return Err(APIError {
code: StatusCode::BAD_REQUEST.as_u16(),
error: "Bad Request".to_string(),
message: format!("Asset file {} does not exist", file_name.clone()),
});
}
}
}

// Copy asset to permanent tool_storage folder
// {storage}/tool_storage/{tool_key}.assets/
// Read all files from origin directory
let origin_files = if origin_path.exists() {
Some(std::fs::read_dir(&origin_path).map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Internal Server Error".to_string(),
message: format!("Failed to read origin directory: {}", e),
})?)
} else {
None
};
// Create destination directory path
let mut perm_file_path = PathBuf::from(storage_path.clone());
perm_file_path.push(".tools_storage");
perm_file_path.push("tools");
perm_file_path.push(shinkai_tool.tool_router_key().convert_to_path());

// Clear destination directory if it exists
if perm_file_path.exists() {
std::fs::remove_dir_all(&perm_file_path).map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Internal Server Error".to_string(),
message: format!("Failed to clear destination directory: {}", e),
})?;
}

// Create destination directory
std::fs::create_dir_all(&perm_file_path).map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Internal Server Error".to_string(),
message: format!("Failed to create permanent storage directory: {}", e),
})?;
if let Some(assets) = payload.assets.clone() {
for file_name in assets {
let mut tool_path = origin_path.clone();
tool_path.push(file_name.clone());
let mut perm_path = perm_file_path.clone();
perm_path.push(file_name.clone());

// Copy all files from origin to destination
if let Some(origin_files) = origin_files {
for entry in origin_files {
let entry = entry.map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Internal Server Error".to_string(),
message: format!("Failed to read directory entry: {}", e),
})?;

let file_name = entry.file_name();
let mut dest_path = perm_file_path.clone();
dest_path.push(&file_name);

println!(
"copying {} to {}",
tool_path.to_string_lossy(),
perm_path.to_string_lossy()
entry.path().to_string_lossy(),
dest_path.to_string_lossy()
);
let _ = std::fs::copy(tool_path, perm_path).map_err(|e| APIError {

std::fs::copy(entry.path(), dest_path).map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Internal Server Error".to_string(),
message: format!(
"Failed to copy asset file {} to permanent storage: {}",
file_name.clone(),
e
),
message: format!("Failed to copy file {}: {}", file_name.to_string_lossy(), e),
})?;
}
}
Expand Down Expand Up @@ -3550,6 +3586,135 @@ Happy coding!"#,

Ok(())
}

pub async fn v2_api_copy_tool_assets(
db: Arc<SqliteManager>,
bearer: String,
node_env: NodeEnvironment,
is_first_playground: bool,
first_path: String,
is_second_playground: bool,
second_path: String,
res: Sender<Result<Value, APIError>>,
) -> Result<(), NodeError> {
if Self::validate_bearer_token(&bearer, db.clone(), &res).await.is_err() {
return Ok(());
}

let response = Self::v2_api_copy_tool_assets_internal(
node_env,
is_first_playground,
first_path,
is_second_playground,
second_path,
)
.await;

let _ = res.send(response).await;

Ok(())
}

async fn v2_api_copy_tool_assets_internal(
node_env: NodeEnvironment,
is_first_playground: bool,
first_path: String,
is_second_playground: bool,
second_path: String,
) -> Result<Value, APIError> {
let storage_path = node_env.node_storage_path.unwrap_or_default();

// Create source path
let mut source_path = PathBuf::from(storage_path.clone());
source_path.push(".tools_storage");
if is_first_playground {
source_path.push("playground");
source_path.push(first_path);
} else {
source_path.push("tools");
source_path.push(ToolRouterKey::from_string(&first_path)?.convert_to_path());
}

// Create destination path
let mut dest_path = PathBuf::from(storage_path);
dest_path.push(".tools_storage");
if is_second_playground {
dest_path.push("playground");
dest_path.push(second_path);
} else {
dest_path.push("tools");
dest_path.push(ToolRouterKey::from_string(&second_path)?.convert_to_path());
}

// Verify source exists
if !source_path.exists() {
return Ok(json!({
"success": false,
"message": "Nothing to copy. Source path does not exist"
}));
}

if dest_path.exists() {
std::fs::remove_dir_all(&dest_path).map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Failed to remove destination directory".to_string(),
message: format!("Error removing destination directory: {}", e),
})?;
}

std::fs::create_dir_all(&dest_path).map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Failed to create destination directory".to_string(),
message: format!("Error creating destination directory: {}", e),
})?;

// Copy all files from source to destination
let entries = std::fs::read_dir(&source_path).map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Failed to read source directory".to_string(),
message: format!("Error reading source directory: {}", e),
})?;

for entry_result in entries {
let entry = match entry_result {
Ok(entry) => entry,
Err(e) => {
return Err(APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Failed to read directory entry".to_string(),
message: format!("Error reading directory entry: {}", e),
});
}
};

let file_name = entry.file_name();
let mut dest_file = dest_path.clone();
dest_file.push(file_name);

match entry.file_type() {
Ok(file_type) if file_type.is_file() => {
std::fs::copy(entry.path(), dest_file).map_err(|e| APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Failed to copy file".to_string(),
message: format!("Error copying file: {}", e),
})?;
}
Ok(_) => continue, // Skip if not a file
Err(e) => {
return Err(APIError {
code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
error: "Failed to get file type".to_string(),
message: format!("Error getting file type: {}", e),
});
}
}
}

Ok(json!({
"success": true,
"message": "Tool assets copied successfully"
}))
}
}

#[cfg(test)]
Expand Down
Loading

0 comments on commit 875824e

Please sign in to comment.